Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

16 files changed, 1498 insertions, 0 deletions

diff --git a/src/pybind/mgr/restful/__init__.py b/src/pybind/mgr/restful/__init__.py
new file mode 100644
index 000000000..8f210ac92
--- /dev/null
+++ b/src/pybind/mgr/restful/__init__.py
@@ -0,0 +1 @@
+from .module import Module
diff --git a/src/pybind/mgr/restful/api/__init__.py b/src/pybind/mgr/restful/api/__init__.py
new file mode 100644
index 000000000..a105dfe87
--- /dev/null
+++ b/src/pybind/mgr/restful/api/__init__.py
@@ -0,0 +1,39 @@
+from pecan import expose
+from pecan.rest import RestController
+
+from .config import Config
+from .crush import Crush
+from .doc import Doc
+from .mon import Mon
+from .osd import Osd
+from .pool import Pool
+from .perf import Perf
+from .request import Request
+from .server import Server
+
+
+class Root(RestController):
+    config = Config()
+    crush = Crush()
+    doc = Doc()
+    mon = Mon()
+    osd = Osd()
+    perf = Perf()
+    pool = Pool()
+    request = Request()
+    server = Server()
+
+    @expose(template='json')
+    def get(self, **kwargs):
+        """
+        Show the basic information for the REST API
+        This includes values like api version or auth method
+        """
+        return {
+            'api_version': 1,
+            'auth':
+                'Use "ceph restful create-key <key>" to create a key pair, '
+                'pass it as HTTP Basic auth to authenticate',
+            'doc': 'See /doc endpoint',
+            'info': "Ceph Manager RESTful API server",
+        }
diff --git a/src/pybind/mgr/restful/api/config.py b/src/pybind/mgr/restful/api/config.py
new file mode 100644
index 000000000..5b0e0af96
--- /dev/null
+++ b/src/pybind/mgr/restful/api/config.py
@@ -0,0 +1,86 @@
+from pecan import expose, request
+from pecan.rest import RestController
+
+from restful import common, context
+from restful.decorators import auth
+
+
+class ConfigOsd(RestController):
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show OSD configuration options
+        """
+        flags = context.instance.get("osd_map")['flags']
+
+        # pause is a valid osd config command that sets pauserd,pausewr
+        flags = flags.replace('pauserd,pausewr', 'pause')
+
+        return flags.split(',')
+
+
+    @expose(template='json')
+    @auth
+    def patch(self, **kwargs):
+        """
+        Modify OSD configuration options
+        """
+        args = request.json
+
+        commands = []
+
+        valid_flags = set(args.keys()) & set(common.OSD_FLAGS)
+        invalid_flags = list(set(args.keys()) - valid_flags)
+        if invalid_flags:
+            context.instance.log.warning("%s not valid to set/unset", invalid_flags)
+
+        for flag in list(valid_flags):
+            if args[flag]:
+                mode = 'set'
+            else:
+                mode = 'unset'
+
+            commands.append({
+                'prefix': 'osd ' + mode,
+                'key': flag,
+            })
+
+        return context.instance.submit_request([commands], **kwargs)
+
+
+
+class ConfigClusterKey(RestController):
+    def __init__(self, key):
+        self.key = key
+
+
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show specific configuration option
+        """
+        return context.instance.get("config").get(self.key, None)
+
+
+
+class ConfigCluster(RestController):
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show all cluster configuration options
+        """
+        return context.instance.get("config")
+
+
+    @expose()
+    def _lookup(self, key, *remainder):
+        return ConfigClusterKey(key), remainder
+
+
+
+class Config(RestController):
+    cluster = ConfigCluster()
+    osd = ConfigOsd()
diff --git a/src/pybind/mgr/restful/api/crush.py b/src/pybind/mgr/restful/api/crush.py
new file mode 100644
index 000000000..79f9007b6
--- /dev/null
+++ b/src/pybind/mgr/restful/api/crush.py
@@ -0,0 +1,25 @@
+from pecan import expose
+from pecan.rest import RestController
+
+from restful import common, context
+
+from restful.decorators import auth
+
+
+class CrushRule(RestController):
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show crush rules
+        """
+        crush = context.instance.get('osd_map_crush')
+        rules = crush['rules']
+
+        for rule in rules:
+            rule['osd_count'] = len(common.crush_rule_osds(crush['buckets'], rule))
+
+        return rules
+
+class Crush(RestController):
+    rule = CrushRule()
diff --git a/src/pybind/mgr/restful/api/doc.py b/src/pybind/mgr/restful/api/doc.py
new file mode 100644
index 000000000..f1038c21b
--- /dev/null
+++ b/src/pybind/mgr/restful/api/doc.py
@@ -0,0 +1,15 @@
+from pecan import expose
+from pecan.rest import RestController
+
+from restful import context
+
+import restful
+
+
+class Doc(RestController):
+    @expose(template='json')
+    def get(self, **kwargs):
+        """
+        Show documentation information
+        """
+        return context.instance.get_doc_api(restful.api.Root)
diff --git a/src/pybind/mgr/restful/api/mon.py b/src/pybind/mgr/restful/api/mon.py
new file mode 100644
index 000000000..20d033605
--- /dev/null
+++ b/src/pybind/mgr/restful/api/mon.py
@@ -0,0 +1,40 @@
+from pecan import expose, response
+from pecan.rest import RestController
+
+from restful import context
+from restful.decorators import auth
+
+
+class MonName(RestController):
+    def __init__(self, name):
+        self.name = name
+
+
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show the information for the monitor name
+        """
+        mon = [x for x in context.instance.get_mons()
+               if x['name'] == self.name]
+        if len(mon) != 1:
+            response.status = 500
+            return {'message': 'Failed to identify the monitor node "{}"'.format(self.name)}
+        return mon[0]
+
+
+
+class Mon(RestController):
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show the information for all the monitors
+        """
+        return context.instance.get_mons()
+
+
+    @expose()
+    def _lookup(self, name, *remainder):
+        return MonName(name), remainder
diff --git a/src/pybind/mgr/restful/api/osd.py b/src/pybind/mgr/restful/api/osd.py
new file mode 100644
index 000000000..8577fae98
--- /dev/null
+++ b/src/pybind/mgr/restful/api/osd.py
@@ -0,0 +1,135 @@
+from pecan import expose, request, response
+from pecan.rest import RestController
+
+from restful import common, context
+from restful.decorators import auth
+
+
+class OsdIdCommand(RestController):
+    def __init__(self, osd_id):
+        self.osd_id = osd_id
+
+
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show implemented commands for the OSD id
+        """
+        osd = context.instance.get_osd_by_id(self.osd_id)
+
+        if not osd:
+            response.status = 500
+            return {'message': 'Failed to identify the OSD id "{}"'.format(self.osd_id)}
+
+        if osd['up']:
+            return common.OSD_IMPLEMENTED_COMMANDS
+        else:
+            return []
+
+
+    @expose(template='json')
+    @auth
+    def post(self, **kwargs):
+        """
+        Run the implemented command for the OSD id
+        """
+        command = request.json.get('command', None)
+
+        osd = context.instance.get_osd_by_id(self.osd_id)
+
+        if not osd:
+            response.status = 500
+            return {'message': 'Failed to identify the OSD id "{}"'.format(self.osd_id)}
+
+        if not osd['up'] or command not in common.OSD_IMPLEMENTED_COMMANDS:
+            response.status = 500
+            return {'message': 'Command "{}" not available'.format(command)}
+
+        return context.instance.submit_request([[{
+            'prefix': 'osd ' + command,
+            'who': str(self.osd_id)
+        }]], **kwargs)
+
+
+
+class OsdId(RestController):
+    def __init__(self, osd_id):
+        self.osd_id = osd_id
+        self.command = OsdIdCommand(osd_id)
+
+
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show the information for the OSD id
+        """
+        osd = context.instance.get_osds(ids=[str(self.osd_id)])
+        if len(osd) != 1:
+            response.status = 500
+            return {'message': 'Failed to identify the OSD id "{}"'.format(self.osd_id)}
+
+        return osd[0]
+
+
+    @expose(template='json')
+    @auth
+    def patch(self, **kwargs):
+        """
+        Modify the state (up, in) of the OSD id or reweight it
+        """
+        args = request.json
+
+        commands = []
+
+        if 'in' in args:
+            if args['in']:
+                commands.append({
+                    'prefix': 'osd in',
+                    'ids': [str(self.osd_id)]
+                })
+            else:
+                commands.append({
+                    'prefix': 'osd out',
+                    'ids': [str(self.osd_id)]
+                })
+
+        if 'up' in args:
+            if args['up']:
+                response.status = 500
+                return {'message': "It is not valid to set a down OSD to be up"}
+            else:
+                commands.append({
+                    'prefix': 'osd down',
+                    'ids': [str(self.osd_id)]
+                })
+
+        if 'reweight' in args:
+            commands.append({
+                'prefix': 'osd reweight',
+                'id': self.osd_id,
+                'weight': args['reweight']
+            })
+
+        return context.instance.submit_request([commands], **kwargs)
+
+
+
+class Osd(RestController):
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show the information for all the OSDs
+        """
+        # Parse request args
+        # TODO Filter by ids
+        pool_id = kwargs.get('pool', None)
+
+        return context.instance.get_osds(pool_id)
+
+
+    @expose()
+    def _lookup(self, osd_id, *remainder):
+        return OsdId(int(osd_id)), remainder
diff --git a/src/pybind/mgr/restful/api/perf.py b/src/pybind/mgr/restful/api/perf.py
new file mode 100644
index 000000000..c484ac55e
--- /dev/null
+++ b/src/pybind/mgr/restful/api/perf.py
@@ -0,0 +1,27 @@
+from pecan import expose, request, response
+from pecan.rest import RestController
+
+from restful import context
+from restful.decorators import auth, lock, paginate
+
+import re
+
+class Perf(RestController):
+    @expose(template='json')
+    @paginate
+    @auth
+    def get(self, **kwargs):
+        """
+        List all the available performance counters
+
+        Options:
+         - 'daemon' -- filter by daemon, accepts Python regexp
+        """
+
+        counters = context.instance.get_unlabeled_perf_counters()
+
+        if 'daemon' in kwargs:
+            _re = re.compile(kwargs['daemon'])
+            counters = {k: v for k, v in counters.items() if _re.match(k)}
+
+        return counters
diff --git a/src/pybind/mgr/restful/api/pool.py b/src/pybind/mgr/restful/api/pool.py
new file mode 100644
index 000000000..40de54eb9
--- /dev/null
+++ b/src/pybind/mgr/restful/api/pool.py
@@ -0,0 +1,140 @@
+from pecan import expose, request, response
+from pecan.rest import RestController
+
+from restful import common, context
+from restful.decorators import auth
+
+
+class PoolId(RestController):
+    def __init__(self, pool_id):
+        self.pool_id = pool_id
+
+
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show the information for the pool id
+        """
+        pool = context.instance.get_pool_by_id(self.pool_id)
+
+        if not pool:
+            response.status = 500
+            return {'message': 'Failed to identify the pool id "{}"'.format(self.pool_id)}
+
+        # pgp_num is called pg_placement_num, deal with that
+        if 'pg_placement_num' in pool:
+            pool['pgp_num'] = pool.pop('pg_placement_num')
+        return pool
+
+
+    @expose(template='json')
+    @auth
+    def patch(self, **kwargs):
+        """
+        Modify the information for the pool id
+        """
+        try:
+            args = request.json
+        except ValueError:
+            response.status = 400
+            return {'message': 'Bad request: malformed JSON or wrong Content-Type'}
+
+        # Get the pool info for its name
+        pool = context.instance.get_pool_by_id(self.pool_id)
+        if not pool:
+            response.status = 500
+            return {'message': 'Failed to identify the pool id "{}"'.format(self.pool_id)}
+
+        # Check for invalid pool args
+        invalid = common.invalid_pool_args(args)
+        if invalid:
+            response.status = 500
+            return {'message': 'Invalid arguments found: "{}"'.format(invalid)}
+
+        # Schedule the update request
+        return context.instance.submit_request(common.pool_update_commands(pool['pool_name'], args), **kwargs)
+
+
+    @expose(template='json')
+    @auth
+    def delete(self, **kwargs):
+        """
+        Remove the pool data for the pool id
+        """
+        pool = context.instance.get_pool_by_id(self.pool_id)
+
+        if not pool:
+            response.status = 500
+            return {'message': 'Failed to identify the pool id "{}"'.format(self.pool_id)}
+
+        return context.instance.submit_request([[{
+            'prefix': 'osd pool delete',
+            'pool': pool['pool_name'],
+            'pool2': pool['pool_name'],
+            'yes_i_really_really_mean_it': True
+        }]], **kwargs)
+
+
+
+class Pool(RestController):
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show the information for all the pools
+        """
+        pools = context.instance.get('osd_map')['pools']
+
+        # pgp_num is called pg_placement_num, deal with that
+        for pool in pools:
+            if 'pg_placement_num' in pool:
+                pool['pgp_num'] = pool.pop('pg_placement_num')
+
+        return pools
+
+
+    @expose(template='json')
+    @auth
+    def post(self, **kwargs):
+        """
+        Create a new pool
+        Requires name and pg_num dict arguments
+        """
+        args = request.json
+
+        # Check for the required arguments
+        pool_name = args.pop('name', None)
+        if pool_name is None:
+            response.status = 500
+            return {'message': 'You need to specify the pool "name" argument'}
+
+        pg_num = args.pop('pg_num', None)
+        if pg_num is None:
+            response.status = 500
+            return {'message': 'You need to specify the "pg_num" argument'}
+
+        # Run the pool create command first
+        create_command = {
+            'prefix': 'osd pool create',
+            'pool': pool_name,
+            'pg_num': pg_num
+        }
+
+        # Check for invalid pool args
+        invalid = common.invalid_pool_args(args)
+        if invalid:
+            response.status = 500
+            return {'message': 'Invalid arguments found: "{}"'.format(invalid)}
+
+        # Schedule the creation and update requests
+        return context.instance.submit_request(
+            [[create_command]] +
+            common.pool_update_commands(pool_name, args),
+            **kwargs
+        )
+
+
+    @expose()
+    def _lookup(self, pool_id, *remainder):
+        return PoolId(int(pool_id)), remainder
diff --git a/src/pybind/mgr/restful/api/request.py b/src/pybind/mgr/restful/api/request.py
new file mode 100644
index 000000000..67143ef50
--- /dev/null
+++ b/src/pybind/mgr/restful/api/request.py
@@ -0,0 +1,93 @@
+from pecan import expose, request, response
+from pecan.rest import RestController
+
+from restful import context
+from restful.decorators import auth, lock, paginate
+
+
+class RequestId(RestController):
+    def __init__(self, request_id):
+        self.request_id = request_id
+
+
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show the information for the request id
+        """
+        request = [x for x in context.instance.requests
+                   if x.id == self.request_id]
+        if len(request) != 1:
+            response.status = 500
+            return {'message': 'Unknown request id "{}"'.format(self.request_id)}
+        return request[0]
+
+
+    @expose(template='json')
+    @auth
+    @lock
+    def delete(self, **kwargs):
+        """
+        Remove the request id from the database
+        """
+        for index in range(len(context.instance.requests)):
+            if context.instance.requests[index].id == self.request_id:
+                return context.instance.requests.pop(index)
+
+        # Failed to find the job to cancel
+        response.status = 500
+        return {'message': 'No such request id'}
+
+
+
+class Request(RestController):
+    @expose(template='json')
+    @paginate
+    @auth
+    def get(self, **kwargs):
+        """
+        List all the available requests
+        """
+        return context.instance.requests
+
+
+    @expose(template='json')
+    @auth
+    @lock
+    def delete(self, **kwargs):
+        """
+        Remove all the finished requests
+        """
+        num_requests = len(context.instance.requests)
+
+        context.instance.requests = [x for x in context.instance.requests
+                                     if not x.is_finished()]
+        remaining = len(context.instance.requests)
+        # Return the job statistics
+        return {
+            'cleaned': num_requests - remaining,
+            'remaining': remaining,
+        }
+
+
+    @expose(template='json')
+    @auth
+    def post(self, **kwargs):
+        """
+        Pass through method to create any request
+        """
+        if isinstance(request.json, list):
+            if all(isinstance(element, list) for element in request.json):
+                return context.instance.submit_request(request.json, **kwargs)
+
+            # The request.json has wrong format
+            response.status = 500
+            return {'message': 'The request format should be [[{c1},{c2}]]'}
+
+        return context.instance.submit_request([[request.json]], **kwargs)
+
+
+    @expose()
+    def _lookup(self, request_id, *remainder):
+        return RequestId(request_id), remainder
diff --git a/src/pybind/mgr/restful/api/server.py b/src/pybind/mgr/restful/api/server.py
new file mode 100644
index 000000000..8ce634937
--- /dev/null
+++ b/src/pybind/mgr/restful/api/server.py
@@ -0,0 +1,35 @@
+from pecan import expose
+from pecan.rest import RestController
+
+from restful import context
+from restful.decorators import auth
+
+
+class ServerFqdn(RestController):
+    def __init__(self, fqdn):
+        self.fqdn = fqdn
+
+
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show the information for the server fqdn
+        """
+        return context.instance.get_server(self.fqdn)
+
+
+
+class Server(RestController):
+    @expose(template='json')
+    @auth
+    def get(self, **kwargs):
+        """
+        Show the information for all the servers
+        """
+        return context.instance.list_servers()
+
+
+    @expose()
+    def _lookup(self, fqdn, *remainder):
+        return ServerFqdn(fqdn), remainder
diff --git a/src/pybind/mgr/restful/common.py b/src/pybind/mgr/restful/common.py
new file mode 100644
index 000000000..1b957d6b5
--- /dev/null
+++ b/src/pybind/mgr/restful/common.py
@@ -0,0 +1,156 @@
+# List of valid osd flags
+OSD_FLAGS = [
+    'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill',
+    'norecover', 'noscrub', 'nodeep-scrub',
+]
+
+# Implemented osd commands
+OSD_IMPLEMENTED_COMMANDS = [
+    'scrub', 'deep-scrub', 'repair'
+]
+
+# Valid values for the 'var' argument to 'ceph osd pool set'
+POOL_PROPERTIES_1 = [
+    'size', 'min_size', 'pg_num',
+    'crush_rule', 'hashpspool',
+]
+
+POOL_PROPERTIES_2 = [
+    'pgp_num'
+]
+
+POOL_PROPERTIES = POOL_PROPERTIES_1 + POOL_PROPERTIES_2
+
+# Valid values for the 'ceph osd pool set-quota' command
+POOL_QUOTA_PROPERTIES = [
+    ('quota_max_bytes', 'max_bytes'),
+    ('quota_max_objects', 'max_objects'),
+]
+
+POOL_ARGS = POOL_PROPERTIES + [x for x,_ in POOL_QUOTA_PROPERTIES]
+
+
+# Transform command to a human readable form
+def humanify_command(command):
+    out = [command['prefix']]
+
+    for arg, val in command.items():
+        if arg != 'prefix':
+            out.append("%s=%s" % (str(arg), str(val)))
+
+    return " ".join(out)
+
+
+def invalid_pool_args(args):
+    invalid = []
+    for arg in args:
+        if arg not in POOL_ARGS:
+            invalid.append(arg)
+
+    return invalid
+
+
+def pool_update_commands(pool_name, args):
+    commands = [[], []]
+
+    # We should increase pgp_num when we are re-setting pg_num
+    if 'pg_num' in args and 'pgp_num' not in args:
+        args['pgp_num'] = args['pg_num']
+
+    # Run the first pool set and quota properties in parallel
+    for var in POOL_PROPERTIES_1:
+        if var in args:
+            commands[0].append({
+                'prefix': 'osd pool set',
+                'pool': pool_name,
+                'var': var,
+                'val': args[var],
+            })
+
+    for (var, field) in POOL_QUOTA_PROPERTIES:
+        if var in args:
+            commands[0].append({
+                'prefix': 'osd pool set-quota',
+                'pool': pool_name,
+                'field': field,
+                'val': str(args[var]),
+            })
+
+    # The second pool set properties need to be run after the first wave
+    for var in POOL_PROPERTIES_2:
+        if var in args:
+            commands[1].append({
+                'prefix': 'osd pool set',
+                'pool': pool_name,
+                'var': var,
+                'val': args[var],
+            })
+
+    return commands
+
+def crush_rule_osds(node_buckets, rule):
+    nodes_by_id = dict((b['id'], b) for b in node_buckets)
+
+    def _gather_leaf_ids(node_id):
+        if node_id >= 0:
+            return set([node_id])
+
+        result = set()
+        for item in nodes_by_id[node_id]['items']:
+            result |= _gather_leaf_ids(item['id'])
+
+        return result
+
+    def _gather_descendent_ids(node, typ):
+        result = set()
+        for item in node['items']:
+            if item['id'] >= 0:
+                if typ == "osd":
+                    result.add(item['id'])
+            else:
+                child_node = nodes_by_id[item['id']]
+                if child_node['type_name'] == typ:
+                    result.add(child_node['id'])
+                elif 'items' in child_node:
+                    result |= _gather_descendent_ids(child_node, typ)
+
+        return result
+
+    def _gather_osds(root, steps):
+        if root['id'] >= 0:
+            return set([root['id']])
+
+        osds = set()
+        step = steps[0]
+        if step['op'] == 'choose_firstn':
+            # Choose all descendents of the current node of type 'type'
+            descendent_ids = _gather_descendent_ids(root, step['type'])
+            for node_id in descendent_ids:
+                if node_id >= 0:
+                    osds.add(node_id)
+                else:
+                    osds |= _gather_osds(nodes_by_id[node_id], steps[1:])
+        elif step['op'] == 'chooseleaf_firstn':
+            # Choose all descendents of the current node of type 'type',
+            # and select all leaves beneath those
+            descendent_ids = _gather_descendent_ids(root, step['type'])
+            for node_id in descendent_ids:
+                if node_id >= 0:
+                    osds.add(node_id)
+                else:
+                    for desc_node in nodes_by_id[node_id]['items']:
+                        # Short circuit another iteration to find the emit
+                        # and assume anything we've done a chooseleaf on
+                        # is going to be part of the selected set of osds
+                        osds |= _gather_leaf_ids(desc_node['id'])
+        elif step['op'] == 'emit':
+            if root['id'] >= 0:
+                osds |= root['id']
+
+        return osds
+
+    osds = set()
+    for i, step in enumerate(rule['steps']):
+        if step['op'] == 'take':
+            osds |= _gather_osds(nodes_by_id[step['item']], rule['steps'][i + 1:])
+    return osds
diff --git a/src/pybind/mgr/restful/context.py b/src/pybind/mgr/restful/context.py
new file mode 100644
index 000000000..a05ea8548
--- /dev/null
+++ b/src/pybind/mgr/restful/context.py
@@ -0,0 +1,2 @@
+# Global instance to share
+instance = None
diff --git a/src/pybind/mgr/restful/decorators.py b/src/pybind/mgr/restful/decorators.py
new file mode 100644
index 000000000..11840a991
--- /dev/null
+++ b/src/pybind/mgr/restful/decorators.py
@@ -0,0 +1,81 @@
+
+from pecan import request, response
+from base64 import b64decode
+from functools import wraps
+
+import traceback
+
+from . import context
+
+
+# Handle authorization
+def auth(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        if not context.instance.enable_auth:
+            return f(*args, **kwargs)
+            
+        if not request.authorization:
+            response.status = 401
+            response.headers['WWW-Authenticate'] = 'Basic realm="Login Required"'
+            return {'message': 'auth: No HTTP username/password'}
+
+        username, password = b64decode(request.authorization[1]).decode('utf-8').split(':')
+
+        # Check that the username exists
+        if username not in context.instance.keys:
+            response.status = 401
+            response.headers['WWW-Authenticate'] = 'Basic realm="Login Required"'
+            return {'message': 'auth: No such user'}
+
+        # Check the password
+        if context.instance.keys[username] != password:
+            response.status = 401
+            response.headers['WWW-Authenticate'] = 'Basic realm="Login Required"'
+            return {'message': 'auth: Incorrect password'}
+
+        return f(*args, **kwargs)
+    return decorated
+
+
+# Helper function to lock the function
+def lock(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        with context.instance.requests_lock:
+            return f(*args, **kwargs)
+    return decorated
+
+
+# Support ?page=N argument
+def paginate(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        _out = f(*args, **kwargs)
+
+        # Do not modify anything without a specific request
+        if not 'page' in kwargs:
+            return _out
+
+        # A pass-through for errors, etc
+        if not isinstance(_out, list):
+            return _out
+
+        # Parse the page argument
+        _page = kwargs['page']
+        try:
+            _page = int(_page)
+        except ValueError:
+            response.status = 500
+            return {'message': 'The requested page is not an integer'}
+
+        # Raise _page so that 0 is the first page and -1 is the last
+        _page += 1
+
+        if _page > 0:
+            _page *= 100
+        else:
+            _page = len(_out) - (_page*100)
+
+        return _out[_page - 100: _page]
+    return decorated
diff --git a/src/pybind/mgr/restful/hooks.py b/src/pybind/mgr/restful/hooks.py
new file mode 100644
index 000000000..c57cbcd40
--- /dev/null
+++ b/src/pybind/mgr/restful/hooks.py
@@ -0,0 +1,10 @@
+
+from pecan.hooks import PecanHook
+
+import traceback
+
+from . import context
+
+class ErrorHook(PecanHook):
+    def on_error(self, stat, exc):
+        context.instance.log.error(str(traceback.format_exc()))
diff --git a/src/pybind/mgr/restful/module.py b/src/pybind/mgr/restful/module.py
new file mode 100644
index 000000000..cb8391ecd
--- /dev/null
+++ b/src/pybind/mgr/restful/module.py
@@ -0,0 +1,613 @@
+"""
+A RESTful API for Ceph
+"""
+
+import os
+import json
+import time
+import errno
+import inspect
+import tempfile
+import threading
+import traceback
+import socket
+import fcntl
+
+from . import common
+from . import context
+
+from uuid import uuid4
+from pecan import jsonify, make_app
+from OpenSSL import crypto
+from pecan.rest import RestController
+from werkzeug.serving import make_server, make_ssl_devcert
+
+from .hooks import ErrorHook
+from mgr_module import MgrModule, CommandResult, NotifyType
+from mgr_util import build_url
+
+
+class CannotServe(Exception):
+    pass
+
+
+class CommandsRequest(object):
+    """
+    This class handles parallel as well as sequential execution of
+    commands. The class accept a list of iterables that should be
+    executed sequentially. Each iterable can contain several commands
+    that can be executed in parallel.
+
+    Example:
+    [[c1,c2],[c3,c4]]
+     - run c1 and c2 in parallel
+     - wait for them to finish
+     - run c3 and c4 in parallel
+     - wait for them to finish
+    """
+
+
+    def __init__(self, commands_arrays):
+        self.id = str(id(self))
+
+        # Filter out empty sub-requests
+        commands_arrays = [x for x in commands_arrays
+                           if len(x) != 0]
+
+        self.running = []
+        self.waiting = commands_arrays[1:]
+        self.finished = []
+        self.failed = []
+
+        self.lock = threading.RLock()
+        if not len(commands_arrays):
+            # Nothing to run
+            return
+
+        # Process first iteration of commands_arrays in parallel
+        results = self.run(commands_arrays[0])
+
+        self.running.extend(results)
+
+
+    def run(self, commands):
+        """
+        A static method that will execute the given list of commands in
+        parallel and will return the list of command results.
+        """
+
+        # Gather the results (in parallel)
+        results = []
+        for index, command in enumerate(commands):
+            tag = '%s:%s:%d' % (__name__, self.id, index)
+
+            # Store the result
+            result = CommandResult(tag)
+            result.command = common.humanify_command(command)
+            results.append(result)
+
+            # Run the command
+            context.instance.send_command(result, 'mon', '', json.dumps(command), tag)
+
+        return results
+
+
+    def next(self):
+        with self.lock:
+            if not self.waiting:
+                # Nothing to run
+                return
+
+            # Run a next iteration of commands
+            commands = self.waiting[0]
+            self.waiting = self.waiting[1:]
+
+            self.running.extend(self.run(commands))
+
+
+    def finish(self, tag):
+        with self.lock:
+            for index in range(len(self.running)):
+                if self.running[index].tag == tag:
+                    if self.running[index].r == 0:
+                        self.finished.append(self.running.pop(index))
+                    else:
+                        self.failed.append(self.running.pop(index))
+                    return True
+
+            # No such tag found
+            return False
+
+
+    def is_running(self, tag):
+        for result in self.running:
+            if result.tag == tag:
+                return True
+        return False
+
+
+    def is_ready(self):
+        with self.lock:
+            return not self.running and self.waiting
+
+
+    def is_waiting(self):
+        return bool(self.waiting)
+
+
+    def is_finished(self):
+        with self.lock:
+            return not self.running and not self.waiting
+
+
+    def has_failed(self):
+        return bool(self.failed)
+
+
+    def get_state(self):
+        with self.lock:
+            if not self.is_finished():
+                return "pending"
+
+            if self.has_failed():
+                return "failed"
+
+            return "success"
+
+
+    def __json__(self):
+        return {
+            'id': self.id,
+            'running': [
+                {
+                    'command': x.command,
+                    'outs': x.outs,
+                    'outb': x.outb,
+                } for x in self.running
+            ],
+            'finished': [
+                {
+                    'command': x.command,
+                    'outs': x.outs,
+                    'outb': x.outb,
+                } for x in self.finished
+            ],
+            'waiting': [
+                [common.humanify_command(y) for y in x]
+                for x in self.waiting
+            ],
+            'failed': [
+                {
+                    'command': x.command,
+                    'outs': x.outs,
+                    'outb': x.outb,
+                } for x in self.failed
+            ],
+            'is_waiting': self.is_waiting(),
+            'is_finished': self.is_finished(),
+            'has_failed': self.has_failed(),
+            'state': self.get_state(),
+        }
+
+
+
+class Module(MgrModule):
+    MODULE_OPTIONS = [
+        {'name': 'server_addr'},
+        {'name': 'server_port'},
+        {'name': 'key_file'},
+        {'name': 'enable_auth', 'type': 'bool', 'default': True},
+    ]
+
+    COMMANDS = [
+        {
+            "cmd": "restful create-key name=key_name,type=CephString",
+            "desc": "Create an API key with this name",
+            "perm": "rw"
+        },
+        {
+            "cmd": "restful delete-key name=key_name,type=CephString",
+            "desc": "Delete an API key with this name",
+            "perm": "rw"
+        },
+        {
+            "cmd": "restful list-keys",
+            "desc": "List all API keys",
+            "perm": "r"
+        },
+        {
+            "cmd": "restful create-self-signed-cert",
+            "desc": "Create localized self signed certificate",
+            "perm": "rw"
+        },
+        {
+            "cmd": "restful restart",
+            "desc": "Restart API server",
+            "perm": "rw"
+        },
+    ]
+
+    NOTIFY_TYPES = [NotifyType.command]
+
+    def __init__(self, *args, **kwargs):
+        super(Module, self).__init__(*args, **kwargs)
+        context.instance = self
+
+        self.requests = []
+        self.requests_lock = threading.RLock()
+
+        self.keys = {}
+        self.enable_auth = True
+
+        self.server = None
+
+        self.stop_server = False
+        self.serve_event = threading.Event()
+
+
+    def serve(self):
+        self.log.debug('serve enter')
+        while not self.stop_server:
+            try:
+                self._serve()
+                self.server.socket.close()
+            except CannotServe as cs:
+                self.log.warning("server not running: %s", cs)
+            except:
+                self.log.error(str(traceback.format_exc()))
+
+            # Wait and clear the threading event
+            self.serve_event.wait()
+            self.serve_event.clear()
+        self.log.debug('serve exit')
+
+    def refresh_keys(self):
+        self.keys = {}
+        rawkeys = self.get_store_prefix('keys/') or {}
+        for k, v in rawkeys.items():
+            self.keys[k[5:]] = v  # strip of keys/ prefix
+
+    def _serve(self):
+        # Load stored authentication keys
+        self.refresh_keys()
+
+        jsonify._instance = jsonify.GenericJSON(
+            sort_keys=True,
+            indent=4,
+            separators=(',', ': '),
+        )
+
+        server_addr = self.get_localized_module_option('server_addr', '::')
+        if server_addr is None:
+            raise CannotServe('no server_addr configured; try "ceph config-key set mgr/restful/server_addr <ip>"')
+
+        server_port = int(self.get_localized_module_option('server_port', '8003'))
+        self.log.info('server_addr: %s server_port: %d',
+                      server_addr, server_port)
+
+        cert = self.get_localized_store("crt")
+        if cert is not None:
+            cert_tmp = tempfile.NamedTemporaryFile()
+            cert_tmp.write(cert.encode('utf-8'))
+            cert_tmp.flush()
+            cert_fname = cert_tmp.name
+        else:
+            cert_fname = self.get_localized_store('crt_file')
+
+        pkey = self.get_localized_store("key")
+        if pkey is not None:
+            pkey_tmp = tempfile.NamedTemporaryFile()
+            pkey_tmp.write(pkey.encode('utf-8'))
+            pkey_tmp.flush()
+            pkey_fname = pkey_tmp.name
+        else:
+            pkey_fname = self.get_localized_module_option('key_file')
+
+        self.enable_auth = self.get_localized_module_option('enable_auth', True)
+        
+        if not cert_fname or not pkey_fname:
+            raise CannotServe('no certificate configured')
+        if not os.path.isfile(cert_fname):
+            raise CannotServe('certificate %s does not exist' % cert_fname)
+        if not os.path.isfile(pkey_fname):
+            raise CannotServe('private key %s does not exist' % pkey_fname)
+
+        # Publish the URI that others may use to access the service we're
+        # about to start serving
+        addr = self.get_mgr_ip() if server_addr == "::" else server_addr
+        self.set_uri(build_url(scheme='https', host=addr, port=server_port, path='/'))
+
+        # Create the HTTPS werkzeug server serving pecan app
+        self.server = make_server(
+            host=server_addr,
+            port=server_port,
+            app=make_app(
+                root='restful.api.Root',
+                hooks = [ErrorHook()],  # use a callable if pecan >= 0.3.2
+            ),
+            ssl_context=(cert_fname, pkey_fname),
+        )
+        sock_fd_flag = fcntl.fcntl(self.server.socket.fileno(), fcntl.F_GETFD)
+        if not (sock_fd_flag & fcntl.FD_CLOEXEC):
+            self.log.debug("set server socket close-on-exec")
+            fcntl.fcntl(self.server.socket.fileno(), fcntl.F_SETFD, sock_fd_flag | fcntl.FD_CLOEXEC)
+        if self.stop_server:
+            self.log.debug('made server, but stop flag set')
+        else:
+            self.log.debug('made server, serving forever')
+            self.server.serve_forever()
+
+
+    def shutdown(self):
+        self.log.debug('shutdown enter')
+        try:
+            self.stop_server = True
+            if self.server:
+                self.log.debug('calling server.shutdown')
+                self.server.shutdown()
+                self.log.debug('called server.shutdown')
+            self.serve_event.set()
+        except:
+            self.log.error(str(traceback.format_exc()))
+            raise
+        self.log.debug('shutdown exit')
+
+
+    def restart(self):
+        try:
+            if self.server:
+                self.server.shutdown()
+            self.serve_event.set()
+        except:
+            self.log.error(str(traceback.format_exc()))
+
+
+    def notify(self, notify_type: NotifyType, tag: str):
+        try:
+            self._notify(notify_type, tag)
+        except:
+            self.log.error(str(traceback.format_exc()))
+
+
+    def _notify(self, notify_type: NotifyType, tag):
+        if notify_type != NotifyType.command:
+            self.log.debug("Unhandled notification type '%s'", notify_type)
+            return
+        # we can safely skip all the sequential commands
+        if tag == 'seq':
+            return
+        try:
+            with self.requests_lock:
+                request = next(x for x in self.requests if x.is_running(tag))
+            request.finish(tag)
+            if request.is_ready():
+                request.next()
+        except StopIteration:
+            # the command was not issued by me
+            pass
+
+    def config_notify(self):
+        self.enable_auth = self.get_localized_module_option('enable_auth', True)
+
+
+    def create_self_signed_cert(self):
+        # create a key pair
+        pkey = crypto.PKey()
+        pkey.generate_key(crypto.TYPE_RSA, 2048)
+
+        # create a self-signed cert
+        cert = crypto.X509()
+        cert.get_subject().O = "IT"
+        cert.get_subject().CN = "ceph-restful"
+        cert.set_serial_number(int(uuid4()))
+        cert.gmtime_adj_notBefore(0)
+        cert.gmtime_adj_notAfter(10*365*24*60*60)
+        cert.set_issuer(cert.get_subject())
+        cert.set_pubkey(pkey)
+        cert.sign(pkey, 'sha512')
+
+        return (
+            crypto.dump_certificate(crypto.FILETYPE_PEM, cert),
+            crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey)
+        )
+
+
+    def handle_command(self, inbuf, command):
+        self.log.warning("Handling command: '%s'" % str(command))
+        if command['prefix'] == "restful create-key":
+            if command['key_name'] in self.keys:
+                return 0, self.keys[command['key_name']], ""
+
+            else:
+                key = str(uuid4())
+                self.keys[command['key_name']] = key
+                self.set_store('keys/' + command['key_name'], key)
+
+            return (
+                0,
+                self.keys[command['key_name']],
+                "",
+            )
+
+        elif command['prefix'] == "restful delete-key":
+            if command['key_name'] in self.keys:
+                del self.keys[command['key_name']]
+                self.set_store('keys/' + command['key_name'], None)
+
+            return (
+                0,
+                "",
+                "",
+            )
+
+        elif command['prefix'] == "restful list-keys":
+            self.refresh_keys()
+            return (
+                0,
+                json.dumps(self.keys, indent=4, sort_keys=True),
+                "",
+            )
+
+        elif command['prefix'] == "restful create-self-signed-cert":
+            cert, pkey = self.create_self_signed_cert()
+            self.set_store(self.get_mgr_id() + '/crt', cert.decode('utf-8'))
+            self.set_store(self.get_mgr_id() + '/key', pkey.decode('utf-8'))
+
+            self.restart()
+            return (
+                0,
+                "Restarting RESTful API server...",
+                ""
+            )
+
+        elif command['prefix'] == 'restful restart':
+            self.restart();
+            return (
+                0,
+                "Restarting RESTful API server...",
+                ""
+            )
+
+        else:
+            return (
+                -errno.EINVAL,
+                "",
+                "Command not found '{0}'".format(command['prefix'])
+            )
+
+
+    def get_doc_api(self, root, prefix=''):
+        doc = {}
+        for _obj in dir(root):
+            obj = getattr(root, _obj)
+
+            if isinstance(obj, RestController):
+                doc.update(self.get_doc_api(obj, prefix + '/' + _obj))
+
+        if getattr(root, '_lookup', None) and isinstance(root._lookup('0')[0], RestController):
+            doc.update(self.get_doc_api(root._lookup('0')[0], prefix + '/<arg>'))
+
+        prefix = prefix or '/'
+
+        doc[prefix] = {}
+        for method in 'get', 'post', 'patch', 'delete':
+            if getattr(root, method, None):
+                doc[prefix][method.upper()] = inspect.getdoc(getattr(root, method)).split('\n')
+
+        if len(doc[prefix]) == 0:
+            del doc[prefix]
+
+        return doc
+
+
+    def get_mons(self):
+        mon_map_mons = self.get('mon_map')['mons']
+        mon_status = json.loads(self.get('mon_status')['json'])
+
+        # Add more information
+        for mon in mon_map_mons:
+            mon['in_quorum'] = mon['rank'] in mon_status['quorum']
+            mon['server'] = self.get_metadata("mon", mon['name'])['hostname']
+            mon['leader'] = mon['rank'] == mon_status['quorum'][0]
+
+        return mon_map_mons
+
+
+    def get_osd_pools(self):
+        osds = dict(map(lambda x: (x['osd'], []), self.get('osd_map')['osds']))
+        pools = dict(map(lambda x: (x['pool'], x), self.get('osd_map')['pools']))
+        crush = self.get('osd_map_crush')
+        crush_rules = crush['rules']
+
+        osds_by_pool = {}
+        for pool_id, pool in pools.items():
+            pool_osds = None
+            for rule in [r for r in crush_rules if r['rule_id'] == pool['crush_rule']]:
+                pool_osds = common.crush_rule_osds(crush['buckets'], rule)
+
+            osds_by_pool[pool_id] = pool_osds
+
+        for pool_id in pools.keys():
+            for in_pool_id in osds_by_pool[pool_id]:
+                osds[in_pool_id].append(pool_id)
+
+        return osds
+
+
+    def get_osds(self, pool_id=None, ids=None):
+        # Get data
+        osd_map = self.get('osd_map')
+        osd_metadata = self.get('osd_metadata')
+
+        # Update the data with the additional info from the osd map
+        osds = osd_map['osds']
+
+        # Filter by osd ids
+        if ids is not None:
+            osds = [x for x in osds if str(x['osd']) in ids]
+
+        # Get list of pools per osd node
+        pools_map = self.get_osd_pools()
+
+        # map osd IDs to reweight
+        reweight_map = dict([
+            (x.get('id'), x.get('reweight', None))
+            for x in self.get('osd_map_tree')['nodes']
+        ])
+
+        # Build OSD data objects
+        for osd in osds:
+            osd['pools'] = pools_map[osd['osd']]
+            osd['server'] = osd_metadata.get(str(osd['osd']), {}).get('hostname', None)
+
+            osd['reweight'] = reweight_map.get(osd['osd'], 0.0)
+
+            if osd['up']:
+                osd['valid_commands'] = common.OSD_IMPLEMENTED_COMMANDS
+            else:
+                osd['valid_commands'] = []
+
+        # Filter by pool
+        if pool_id:
+            pool_id = int(pool_id)
+            osds = [x for x in osds if pool_id in x['pools']]
+
+        return osds
+
+
+    def get_osd_by_id(self, osd_id):
+        osd = [x for x in self.get('osd_map')['osds']
+               if x['osd'] == osd_id]
+
+        if len(osd) != 1:
+            return None
+
+        return osd[0]
+
+
+    def get_pool_by_id(self, pool_id):
+        pool = [x for x in self.get('osd_map')['pools']
+                if x['pool'] == pool_id]
+
+        if len(pool) != 1:
+            return None
+
+        return pool[0]
+
+
+    def submit_request(self, _request, **kwargs):
+        with self.requests_lock:
+            request = CommandsRequest(_request)
+            self.requests.append(request)
+        if kwargs.get('wait', 0):
+            while not request.is_finished():
+                time.sleep(0.001)
+        return request
+
+
+    def run_command(self, command):
+        # tag with 'seq' so that we can ignore these in notify function
+        result = CommandResult('seq')
+
+        self.send_command(result, 'mon', '', json.dumps(command), 'seq')
+        return result.wait()