Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

61 files changed, 60089 insertions, 0 deletions

diff --git a/src/mon/AuthMonitor.cc b/src/mon/AuthMonitor.cc
new file mode 100644
index 000000000..f08608c61
--- /dev/null
+++ b/src/mon/AuthMonitor.cc
@@ -0,0 +1,2033 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <sstream>
+
+#include "mon/AuthMonitor.h"
+#include "mon/Monitor.h"
+#include "mon/MonitorDBStore.h"
+#include "mon/OSDMonitor.h"
+#include "mon/MDSMonitor.h"
+#include "mon/ConfigMonitor.h"
+
+#include "messages/MMonCommand.h"
+#include "messages/MAuth.h"
+#include "messages/MAuthReply.h"
+#include "messages/MMonGlobalID.h"
+#include "msg/Messenger.h"
+
+#include "auth/AuthServiceHandler.h"
+#include "auth/KeyRing.h"
+#include "include/stringify.h"
+#include "include/ceph_assert.h"
+
+#include "mds/MDSAuthCaps.h"
+#include "mgr/MgrCap.h"
+#include "osd/OSDCap.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, get_last_committed())
+using namespace TOPNSPC::common;
+
+using std::cerr;
+using std::cout;
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::setfill;
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+using std::unique_ptr;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::make_message;
+using ceph::mono_clock;
+using ceph::mono_time;
+using ceph::timespan_str;
+static ostream& _prefix(std::ostream *_dout, Monitor &mon, version_t v) {
+  return *_dout << "mon." << mon.name << "@" << mon.rank
+		<< "(" << mon.get_state_name()
+		<< ").auth v" << v << " ";
+}
+
+ostream& operator<<(ostream &out, const AuthMonitor &pm)
+{
+  return out << "auth";
+}
+
+bool AuthMonitor::check_rotate()
+{
+  KeyServerData::Incremental rot_inc;
+  rot_inc.op = KeyServerData::AUTH_INC_SET_ROTATING;
+  if (mon.key_server.prepare_rotating_update(rot_inc.rotating_bl)) {
+    dout(10) << __func__ << " updating rotating" << dendl;
+    push_cephx_inc(rot_inc);
+    return true;
+  }
+  return false;
+}
+
+/*
+ Tick function to update the map based on performance every N seconds
+*/
+
+void AuthMonitor::tick()
+{
+  if (!is_active()) return;
+
+  dout(10) << *this << dendl;
+
+  // increase global_id?
+  bool propose = false;
+  bool increase;
+  {
+    std::lock_guard l(mon.auth_lock);
+    increase = _should_increase_max_global_id();
+  }
+  if (increase) {
+    if (mon.is_leader()) {
+      increase_max_global_id();
+      propose = true;
+    } else {
+      dout(10) << __func__ << "requesting more ids from leader" << dendl;
+      int leader = mon.get_leader();
+      MMonGlobalID *req = new MMonGlobalID();
+      req->old_max_id = max_global_id;
+      mon.send_mon_message(req, leader);
+    }
+  }
+
+  if (!mon.is_leader()) {
+    return;
+  }
+
+  if (check_rotate()) {
+    propose = true;
+  }
+
+  if (propose) {
+    propose_pending();
+  }
+}
+
+void AuthMonitor::on_active()
+{
+  dout(10) << "AuthMonitor::on_active()" << dendl;
+
+  if (!mon.is_leader())
+    return;
+
+  mon.key_server.start_server();
+
+  if (is_writeable()) {
+    bool propose = false;
+    if (check_rotate()) {
+      propose = true;
+    }
+    bool increase;
+    {
+      std::lock_guard l(mon.auth_lock);
+      increase = _should_increase_max_global_id();
+    }
+    if (increase) {
+      increase_max_global_id();
+      propose = true;
+    }
+    if (propose) {
+      propose_pending();
+    }
+  }
+}
+
+bufferlist _encode_cap(const string& cap)
+{
+  bufferlist bl;
+  encode(cap, bl);
+  return bl;
+}
+
+void AuthMonitor::get_initial_keyring(KeyRing *keyring)
+{
+  dout(10) << __func__ << dendl;
+  ceph_assert(keyring != nullptr);
+
+  bufferlist bl;
+  int ret = mon.store->get("mkfs", "keyring", bl);
+  if (ret == -ENOENT) {
+    return;
+  }
+  // fail hard only if there's an error we're not expecting to see
+  ceph_assert(ret == 0);
+
+  auto p = bl.cbegin();
+  decode(*keyring, p);
+}
+
+void _generate_bootstrap_keys(
+    list<pair<EntityName,EntityAuth> >* auth_lst)
+{
+  ceph_assert(auth_lst != nullptr);
+
+  map<string,map<string,bufferlist> > bootstrap = {
+    { "admin", {
+      { "mon", _encode_cap("allow *") },
+      { "osd", _encode_cap("allow *") },
+      { "mds", _encode_cap("allow *") },
+      { "mgr", _encode_cap("allow *") }
+    } },
+    { "bootstrap-osd", {
+      { "mon", _encode_cap("allow profile bootstrap-osd") }
+    } },
+    { "bootstrap-rgw", {
+      { "mon", _encode_cap("allow profile bootstrap-rgw") }
+    } },
+    { "bootstrap-mds", {
+      { "mon", _encode_cap("allow profile bootstrap-mds") }
+    } },
+    { "bootstrap-mgr", {
+      { "mon", _encode_cap("allow profile bootstrap-mgr") }
+    } },
+    { "bootstrap-rbd", {
+      { "mon", _encode_cap("allow profile bootstrap-rbd") }
+    } },
+    { "bootstrap-rbd-mirror", {
+      { "mon", _encode_cap("allow profile bootstrap-rbd-mirror") }
+    } }
+  };
+
+  for (auto &p : bootstrap) {
+    EntityName name;
+    name.from_str("client." + p.first);
+    EntityAuth auth;
+    auth.key.create(g_ceph_context, CEPH_CRYPTO_AES);
+    auth.caps = p.second;
+
+    auth_lst->push_back(make_pair(name, auth));
+  }
+}
+
+void AuthMonitor::create_initial_keys(KeyRing *keyring)
+{
+  dout(10) << __func__ << " with keyring" << dendl;
+  ceph_assert(keyring != nullptr);
+
+  list<pair<EntityName,EntityAuth> > auth_lst;
+  _generate_bootstrap_keys(&auth_lst);
+
+  for (auto &p : auth_lst) {
+    if (keyring->exists(p.first)) {
+      continue;
+    }
+    keyring->add(p.first, p.second);
+  }
+}
+
+void AuthMonitor::create_initial()
+{
+  dout(10) << "create_initial -- creating initial map" << dendl;
+
+  // initialize rotating keys
+  mon.key_server.clear_secrets();
+  check_rotate();
+  ceph_assert(pending_auth.size() == 1);
+
+  if (mon.is_keyring_required()) {
+    KeyRing keyring;
+    // attempt to obtain an existing mkfs-time keyring
+    get_initial_keyring(&keyring);
+    // create missing keys in the keyring
+    create_initial_keys(&keyring);
+    // import the resulting keyring
+    import_keyring(keyring);
+  }
+
+  max_global_id = MIN_GLOBAL_ID;
+
+  Incremental inc;
+  inc.inc_type = GLOBAL_ID;
+  inc.max_global_id = max_global_id;
+  pending_auth.push_back(inc);
+
+  format_version = 3;
+}
+
+void AuthMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  dout(10) << __func__ << dendl;
+  load_health();
+
+  version_t version = get_last_committed();
+  version_t keys_ver = mon.key_server.get_ver();
+  if (version == keys_ver)
+    return;
+  ceph_assert(version > keys_ver);
+
+  version_t latest_full = get_version_latest_full();
+
+  dout(10) << __func__ << " version " << version << " keys ver " << keys_ver
+           << " latest " << latest_full << dendl;
+
+  if ((latest_full > 0) && (latest_full > keys_ver)) {
+    bufferlist latest_bl;
+    int err = get_version_full(latest_full, latest_bl);
+    ceph_assert(err == 0);
+    ceph_assert(latest_bl.length() != 0);
+    dout(7) << __func__ << " loading summary e " << latest_full << dendl;
+    dout(7) << __func__ << " latest length " << latest_bl.length() << dendl;
+    auto p = latest_bl.cbegin();
+    __u8 struct_v;
+    decode(struct_v, p);
+    decode(max_global_id, p);
+    decode(mon.key_server, p);
+    mon.key_server.set_ver(latest_full);
+    keys_ver = latest_full;
+  }
+
+  dout(10) << __func__ << " key server version " << mon.key_server.get_ver() << dendl;
+
+  // walk through incrementals
+  while (version > keys_ver) {
+    bufferlist bl;
+    int ret = get_version(keys_ver+1, bl);
+    ceph_assert(ret == 0);
+    ceph_assert(bl.length());
+
+    // reset if we are moving to initial state.  we will normally have
+    // keys in here temporarily for bootstrapping that we need to
+    // clear out.
+    if (keys_ver == 0)
+      mon.key_server.clear_secrets();
+
+    dout(20) << __func__ << " walking through version " << (keys_ver+1)
+             << " len " << bl.length() << dendl;
+
+    auto p = bl.cbegin();
+    __u8 v;
+    decode(v, p);
+    while (!p.end()) {
+      Incremental inc;
+      decode(inc, p);
+      switch (inc.inc_type) {
+      case GLOBAL_ID:
+	max_global_id = inc.max_global_id;
+	break;
+
+      case AUTH_DATA:
+        {
+          KeyServerData::Incremental auth_inc;
+          auto iter = inc.auth_data.cbegin();
+          decode(auth_inc, iter);
+          mon.key_server.apply_data_incremental(auth_inc);
+          break;
+        }
+      }
+    }
+
+    keys_ver++;
+    mon.key_server.set_ver(keys_ver);
+
+    if (keys_ver == 1 && mon.is_keyring_required()) {
+      auto t(std::make_shared<MonitorDBStore::Transaction>());
+      t->erase("mkfs", "keyring");
+      mon.store->apply_transaction(t);
+    }
+  }
+
+  {
+    std::lock_guard l(mon.auth_lock);
+    if (last_allocated_id == 0) {
+      last_allocated_id = max_global_id;
+      dout(10) << __func__ << " last_allocated_id initialized to "
+	       << max_global_id << dendl;
+    }
+  }
+
+  dout(10) << __func__ << " max_global_id=" << max_global_id
+	   << " format_version " << format_version
+	   << dendl;
+
+  mon.key_server.dump();
+}
+
+bool AuthMonitor::_should_increase_max_global_id()
+{
+  ceph_assert(ceph_mutex_is_locked(mon.auth_lock));
+  auto num_prealloc = g_conf()->mon_globalid_prealloc;
+  if (max_global_id < num_prealloc ||
+      (last_allocated_id + 1) >= max_global_id - num_prealloc / 2) {
+    return true;
+  }
+  return false;
+}
+
+void AuthMonitor::increase_max_global_id()
+{
+  ceph_assert(mon.is_leader());
+
+  Incremental inc;
+  inc.inc_type = GLOBAL_ID;
+  inc.max_global_id = max_global_id + g_conf()->mon_globalid_prealloc;
+  dout(10) << "increasing max_global_id to " << inc.max_global_id << dendl;
+  pending_auth.push_back(inc);
+}
+
+bool AuthMonitor::should_propose(double& delay)
+{
+  return (!pending_auth.empty());
+}
+
+void AuthMonitor::create_pending()
+{
+  pending_auth.clear();
+  dout(10) << "create_pending v " << (get_last_committed() + 1) << dendl;
+}
+
+void AuthMonitor::encode_pending(MonitorDBStore::TransactionRef t)
+{
+  dout(10) << __func__ << " v " << (get_last_committed() + 1) << dendl;
+
+  bufferlist bl;
+
+  __u8 v = 1;
+  encode(v, bl);
+  vector<Incremental>::iterator p;
+  for (p = pending_auth.begin(); p != pending_auth.end(); ++p)
+    p->encode(bl, mon.get_quorum_con_features());
+
+  version_t version = get_last_committed() + 1;
+  put_version(t, version, bl);
+  put_last_committed(t, version);
+
+  // health
+  health_check_map_t next;
+  map<string,list<string>> bad_detail;  // entity -> details
+  for (auto i = mon.key_server.secrets_begin();
+       i != mon.key_server.secrets_end();
+       ++i) {
+    for (auto& p : i->second.caps) {
+      ostringstream ss;
+      if (!valid_caps(p.first, p.second, &ss)) {
+	ostringstream ss2;
+	ss2 << i->first << " " << ss.str();
+	bad_detail[i->first.to_str()].push_back(ss2.str());
+      }
+    }
+  }
+  for (auto& inc : pending_auth) {
+    if (inc.inc_type == AUTH_DATA) {
+      KeyServerData::Incremental auth_inc;
+      auto iter = inc.auth_data.cbegin();
+      decode(auth_inc, iter);
+      if (auth_inc.op == KeyServerData::AUTH_INC_DEL) {
+	bad_detail.erase(auth_inc.name.to_str());
+      } else if (auth_inc.op == KeyServerData::AUTH_INC_ADD) {
+	for (auto& p : auth_inc.auth.caps) {
+	  ostringstream ss;
+	  if (!valid_caps(p.first, p.second, &ss)) {
+	    ostringstream ss2;
+	    ss2 << auth_inc.name << " " << ss.str();
+	    bad_detail[auth_inc.name.to_str()].push_back(ss2.str());
+	  }
+	}
+      }
+    }
+  }
+  if (bad_detail.size()) {
+    ostringstream ss;
+    ss << bad_detail.size() << " auth entities have invalid capabilities";
+    health_check_t *check = &next.add("AUTH_BAD_CAPS", HEALTH_ERR, ss.str(),
+				      bad_detail.size());
+    for (auto& i : bad_detail) {
+      for (auto& j : i.second) {
+	check->detail.push_back(j);
+      }
+    }
+  }
+  encode_health(next, t);
+}
+
+void AuthMonitor::encode_full(MonitorDBStore::TransactionRef t)
+{
+  version_t version = mon.key_server.get_ver();
+  // do not stash full version 0 as it will never be removed nor read
+  if (version == 0)
+    return;
+
+  dout(10) << __func__ << " auth v " << version << dendl;
+  ceph_assert(get_last_committed() == version);
+
+  bufferlist full_bl;
+  std::scoped_lock l{mon.key_server.get_lock()};
+  dout(20) << __func__ << " key server has "
+           << (mon.key_server.has_secrets() ? "" : "no ")
+           << "secrets!" << dendl;
+  __u8 v = 1;
+  encode(v, full_bl);
+  encode(max_global_id, full_bl);
+  encode(mon.key_server, full_bl);
+
+  put_version_full(t, version, full_bl);
+  put_version_latest_full(t, version);
+}
+
+version_t AuthMonitor::get_trim_to() const
+{
+  unsigned max = g_conf()->paxos_max_join_drift * 2;
+  version_t version = get_last_committed();
+  if (mon.is_leader() && (version > max))
+    return version - max;
+  return 0;
+}
+
+bool AuthMonitor::preprocess_query(MonOpRequestRef op)
+{
+  auto m = op->get_req<PaxosServiceMessage>();
+  dout(10) << "preprocess_query " << *m << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    try {
+      return preprocess_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+
+  case CEPH_MSG_AUTH:
+    return prep_auth(op, false);
+
+  case MSG_MON_GLOBAL_ID:
+    return false;
+
+  default:
+    ceph_abort();
+    return true;
+  }
+}
+
+bool AuthMonitor::prepare_update(MonOpRequestRef op)
+{
+  auto m = op->get_req<PaxosServiceMessage>();
+  dout(10) << "prepare_update " << *m << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    try {
+      return prepare_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+  case MSG_MON_GLOBAL_ID:
+    return prepare_global_id(op);
+  case CEPH_MSG_AUTH:
+    return prep_auth(op, true);
+  default:
+    ceph_abort();
+    return false;
+  }
+}
+
+void AuthMonitor::_set_mon_num_rank(int num, int rank)
+{
+  dout(10) << __func__ << " num " << num << " rank " << rank << dendl;
+  ceph_assert(ceph_mutex_is_locked(mon.auth_lock));
+  mon_num = num;
+  mon_rank = rank;
+}
+
+uint64_t AuthMonitor::_assign_global_id()
+{
+  ceph_assert(ceph_mutex_is_locked(mon.auth_lock));
+  if (mon_num < 1 || mon_rank < 0) {
+    dout(10) << __func__ << " inactive (num_mon " << mon_num
+	     << " rank " << mon_rank << ")" << dendl;
+    return 0;
+  }
+  if (!last_allocated_id) {
+    dout(10) << __func__ << " last_allocated_id == 0" << dendl;
+    return 0;
+  }
+
+  uint64_t id = last_allocated_id + 1;
+  int remainder = id % mon_num;
+  if (remainder) {
+    remainder = mon_num - remainder;
+  }
+  id += remainder + mon_rank;
+
+  if (id >= max_global_id) {
+    dout(10) << __func__ << " failed (max " << max_global_id << ")" << dendl;
+    return 0;
+  }
+
+  last_allocated_id = id;
+  dout(10) << __func__ << " " << id << " (max " << max_global_id << ")"
+	   << dendl;
+  return id;
+}
+
+uint64_t AuthMonitor::assign_global_id(bool should_increase_max)
+{
+  uint64_t id;
+  {
+    std::lock_guard l(mon.auth_lock);
+    id =_assign_global_id();
+    if (should_increase_max) {
+      should_increase_max = _should_increase_max_global_id();
+    }
+  }
+  if (mon.is_leader() &&
+      should_increase_max) {
+    increase_max_global_id();
+  }
+  return id;
+}
+
+bool AuthMonitor::prep_auth(MonOpRequestRef op, bool paxos_writable)
+{
+  auto m = op->get_req<MAuth>();
+  dout(10) << "prep_auth() blob_size=" << m->get_auth_payload().length() << dendl;
+
+  MonSession *s = op->get_session();
+  if (!s) {
+    dout(10) << "no session, dropping" << dendl;
+    return true;
+  }
+
+  int ret = 0;
+  MAuthReply *reply;
+  bufferlist response_bl;
+  auto indata = m->auth_payload.cbegin();
+  __u32 proto = m->protocol;
+  bool start = false;
+  bool finished = false;
+  EntityName entity_name;
+  bool is_new_global_id = false;
+
+  // set up handler?
+  if (m->protocol == 0 && !s->auth_handler) {
+    set<__u32> supported;
+
+    try {
+      __u8 struct_v = 1;
+      decode(struct_v, indata);
+      decode(supported, indata);
+      decode(entity_name, indata);
+      decode(s->con->peer_global_id, indata);
+    } catch (const ceph::buffer::error &e) {
+      dout(10) << "failed to decode initial auth message" << dendl;
+      ret = -EINVAL;
+      goto reply;
+    }
+
+    // do we require cephx signatures?
+
+    if (!m->get_connection()->has_feature(CEPH_FEATURE_MSG_AUTH)) {
+      if (entity_name.get_type() == CEPH_ENTITY_TYPE_MON ||
+	  entity_name.get_type() == CEPH_ENTITY_TYPE_OSD ||
+	  entity_name.get_type() == CEPH_ENTITY_TYPE_MDS ||
+	  entity_name.get_type() == CEPH_ENTITY_TYPE_MGR) {
+	if (g_conf()->cephx_cluster_require_signatures ||
+	    g_conf()->cephx_require_signatures) {
+	  dout(1) << m->get_source_inst()
+                  << " supports cephx but not signatures and"
+                  << " 'cephx [cluster] require signatures = true';"
+                  << " disallowing cephx" << dendl;
+	  supported.erase(CEPH_AUTH_CEPHX);
+	}
+      } else {
+	if (g_conf()->cephx_service_require_signatures ||
+	    g_conf()->cephx_require_signatures) {
+	  dout(1) << m->get_source_inst()
+                  << " supports cephx but not signatures and"
+                  << " 'cephx [service] require signatures = true';"
+                  << " disallowing cephx" << dendl;
+	  supported.erase(CEPH_AUTH_CEPHX);
+	}
+      }
+    } else if (!m->get_connection()->has_feature(CEPH_FEATURE_CEPHX_V2)) {
+      if (entity_name.get_type() == CEPH_ENTITY_TYPE_MON ||
+	  entity_name.get_type() == CEPH_ENTITY_TYPE_OSD ||
+	  entity_name.get_type() == CEPH_ENTITY_TYPE_MDS ||
+	  entity_name.get_type() == CEPH_ENTITY_TYPE_MGR) {
+	if (g_conf()->cephx_cluster_require_version >= 2 ||
+	    g_conf()->cephx_require_version >= 2) {
+	  dout(1) << m->get_source_inst()
+                  << " supports cephx but not v2 and"
+                  << " 'cephx [cluster] require version >= 2';"
+                  << " disallowing cephx" << dendl;
+	  supported.erase(CEPH_AUTH_CEPHX);
+	}
+      } else {
+	if (g_conf()->cephx_service_require_version >= 2 ||
+	    g_conf()->cephx_require_version >= 2) {
+	  dout(1) << m->get_source_inst()
+                  << " supports cephx but not v2 and"
+                  << " 'cephx [service] require version >= 2';"
+                  << " disallowing cephx" << dendl;
+	  supported.erase(CEPH_AUTH_CEPHX);
+	}
+      }
+    }
+
+    int type;
+    if (entity_name.get_type() == CEPH_ENTITY_TYPE_MON ||
+	entity_name.get_type() == CEPH_ENTITY_TYPE_OSD ||
+	entity_name.get_type() == CEPH_ENTITY_TYPE_MDS ||
+	entity_name.get_type() == CEPH_ENTITY_TYPE_MGR)
+      type = mon.auth_cluster_required.pick(supported);
+    else
+      type = mon.auth_service_required.pick(supported);
+
+    s->auth_handler = get_auth_service_handler(type, g_ceph_context, &mon.key_server);
+    if (!s->auth_handler) {
+      dout(1) << "client did not provide supported auth type" << dendl;
+      ret = -ENOTSUP;
+      goto reply;
+    }
+    start = true;
+    proto = type;
+  } else if (!s->auth_handler) {
+      dout(10) << "protocol specified but no s->auth_handler" << dendl;
+      ret = -EINVAL;
+      goto reply;
+  }
+
+  /* assign a new global_id? we assume this should only happen on the first
+     request. If a client tries to send it later, it'll screw up its auth
+     session */
+  if (!s->con->peer_global_id) {
+    s->con->peer_global_id = assign_global_id(paxos_writable);
+    if (!s->con->peer_global_id) {
+
+      delete s->auth_handler;
+      s->auth_handler = NULL;
+
+      if (mon.is_leader() && paxos_writable) {
+        dout(10) << "increasing global id, waitlisting message" << dendl;
+        wait_for_active(op, new C_RetryMessage(this, op));
+        goto done;
+      }
+
+      if (!mon.is_leader()) {
+	dout(10) << "not the leader, requesting more ids from leader" << dendl;
+	int leader = mon.get_leader();
+	MMonGlobalID *req = new MMonGlobalID();
+	req->old_max_id = max_global_id;
+	mon.send_mon_message(req, leader);
+	wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+	return true;
+      }
+
+      ceph_assert(!paxos_writable);
+      return false;
+    }
+    is_new_global_id = true;
+  }
+
+  try {
+    if (start) {
+      // new session
+      ret = s->auth_handler->start_session(entity_name,
+					   s->con->peer_global_id,
+					   is_new_global_id,
+					   &response_bl,
+					   &s->con->peer_caps_info);
+    } else {
+      // request
+      ret = s->auth_handler->handle_request(
+	indata,
+	0, // no connection_secret needed
+	&response_bl,
+	&s->con->peer_caps_info,
+	nullptr, nullptr);
+    }
+    if (ret == -EIO) {
+      wait_for_active(op, new C_RetryMessage(this,op));
+      goto done;
+    }
+    if (ret > 0) {
+      if (!s->authenticated &&
+	  mon.ms_handle_authentication(s->con.get()) > 0) {
+	finished = true;
+      }
+      ret = 0;
+    }
+  } catch (const ceph::buffer::error &err) {
+    ret = -EINVAL;
+    dout(0) << "caught error when trying to handle auth request, probably malformed request" << dendl;
+  }
+
+reply:
+  reply = new MAuthReply(proto, &response_bl, ret, s->con->peer_global_id);
+  mon.send_reply(op, reply);
+  if (finished) {
+    // always send the latest monmap.
+    if (m->monmap_epoch < mon.monmap->get_epoch())
+      mon.send_latest_monmap(m->get_connection().get());
+
+    mon.configmon()->check_sub(s);
+  }
+done:
+  return true;
+}
+
+bool AuthMonitor::preprocess_command(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonCommand>();
+  int r = -1;
+  bufferlist rdata;
+  stringstream ss, ds;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    // ss has reason for failure
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+  if (prefix == "auth add" ||
+      prefix == "auth del" ||
+      prefix == "auth rm" ||
+      prefix == "auth get-or-create" ||
+      prefix == "auth get-or-create-key" ||
+      prefix == "fs authorize" ||
+      prefix == "auth import" ||
+      prefix == "auth caps") {
+    return false;
+  }
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    mon.reply_command(op, -EACCES, "access denied", rdata, get_last_committed());
+    return true;
+  }
+
+  // entity might not be supplied, but if it is, it should be valid
+  string entity_name;
+  cmd_getval(cmdmap, "entity", entity_name);
+  EntityName entity;
+  if (!entity_name.empty() && !entity.from_str(entity_name)) {
+    ss << "invalid entity_auth " << entity_name;
+    mon.reply_command(op, -EINVAL, ss.str(), get_last_committed());
+    return true;
+  }
+
+  string format;
+  cmd_getval(cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+  if (prefix == "auth export") {
+    KeyRing keyring;
+    export_keyring(keyring);
+    if (!entity_name.empty()) {
+      EntityAuth eauth;
+      if (keyring.get_auth(entity, eauth)) {
+	KeyRing kr;
+	kr.add(entity, eauth);
+	if (f)
+	  kr.encode_formatted("auth", f.get(), rdata);
+	else
+	  kr.encode_plaintext(rdata);
+	ss << "export " << eauth;
+	r = 0;
+      } else {
+	ss << "no key for " << eauth;
+	r = -ENOENT;
+      }
+    } else {
+      if (f)
+	keyring.encode_formatted("auth", f.get(), rdata);
+      else
+	keyring.encode_plaintext(rdata);
+
+      ss << "exported master keyring";
+      r = 0;
+    }
+  } else if (prefix == "auth get" && !entity_name.empty()) {
+    KeyRing keyring;
+    EntityAuth entity_auth;
+    if(!mon.key_server.get_auth(entity, entity_auth)) {
+      ss << "failed to find " << entity_name << " in keyring";
+      r = -ENOENT;
+    } else {
+      keyring.add(entity, entity_auth);
+      if (f)
+	keyring.encode_formatted("auth", f.get(), rdata);
+      else
+	keyring.encode_plaintext(rdata);
+      ss << "exported keyring for " << entity_name;
+      r = 0;
+    }
+  } else if (prefix == "auth print-key" ||
+	     prefix == "auth print_key" ||
+	     prefix == "auth get-key") {
+    EntityAuth auth;
+    if (!mon.key_server.get_auth(entity, auth)) {
+      ss << "don't have " << entity;
+      r = -ENOENT;
+      goto done;
+    }
+    if (f) {
+      auth.key.encode_formatted("auth", f.get(), rdata);
+    } else {
+      auth.key.encode_plaintext(rdata);
+    }
+    r = 0;
+  } else if (prefix == "auth list" ||
+	     prefix == "auth ls") {
+    if (f) {
+      mon.key_server.encode_formatted("auth", f.get(), rdata);
+    } else {
+      mon.key_server.encode_plaintext(rdata);
+      if (rdata.length() > 0)
+        ss << "installed auth entries:" << std::endl;
+      else
+        ss << "no installed auth entries!" << std::endl;
+    }
+    r = 0;
+    goto done;
+  } else {
+    ss << "invalid command";
+    r = -EINVAL;
+  }
+
+ done:
+  rdata.append(ds);
+  string rs;
+  getline(ss, rs, '\0');
+  mon.reply_command(op, r, rs, rdata, get_last_committed());
+  return true;
+}
+
+void AuthMonitor::export_keyring(KeyRing& keyring)
+{
+  mon.key_server.export_keyring(keyring);
+}
+
+int AuthMonitor::import_keyring(KeyRing& keyring)
+{
+  dout(10) << __func__ << " " << keyring.size() << " keys" << dendl;
+
+  for (map<EntityName, EntityAuth>::iterator p = keyring.get_keys().begin();
+       p != keyring.get_keys().end();
+       ++p) {
+    if (p->second.caps.empty()) {
+      dout(0) << "import: no caps supplied" << dendl;
+      return -EINVAL;
+    }
+    int err = add_entity(p->first, p->second);
+    ceph_assert(err == 0);
+  }
+  return 0;
+}
+
+int AuthMonitor::remove_entity(const EntityName &entity)
+{
+  dout(10) << __func__ << " " << entity << dendl;
+  if (!mon.key_server.contains(entity))
+    return -ENOENT;
+
+  KeyServerData::Incremental auth_inc;
+  auth_inc.name = entity;
+  auth_inc.op = KeyServerData::AUTH_INC_DEL;
+  push_cephx_inc(auth_inc);
+
+  return 0;
+}
+
+bool AuthMonitor::entity_is_pending(EntityName& entity)
+{
+  // are we about to have it?
+  for (auto& p : pending_auth) {
+    if (p.inc_type == AUTH_DATA) {
+      KeyServerData::Incremental inc;
+      auto q = p.auth_data.cbegin();
+      decode(inc, q);
+      if (inc.op == KeyServerData::AUTH_INC_ADD &&
+          inc.name == entity) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+int AuthMonitor::exists_and_matches_entity(
+    const auth_entity_t& entity,
+    bool has_secret,
+    stringstream& ss)
+{
+  return exists_and_matches_entity(entity.name, entity.auth,
+                                   entity.auth.caps, has_secret, ss);
+}
+
+int AuthMonitor::exists_and_matches_entity(
+    const EntityName& name,
+    const EntityAuth& auth,
+    const map<string,bufferlist>& caps,
+    bool has_secret,
+    stringstream& ss)
+{
+
+  dout(20) << __func__ << " entity " << name << " auth " << auth
+           << " caps " << caps << " has_secret " << has_secret << dendl;
+
+  EntityAuth existing_auth;
+  // does entry already exist?
+  if (mon.key_server.get_auth(name, existing_auth)) {
+    // key match?
+    if (has_secret) {
+      if (existing_auth.key.get_secret().cmp(auth.key.get_secret())) {
+        ss << "entity " << name << " exists but key does not match";
+        return -EEXIST;
+      }
+    }
+
+    // caps match?
+    if (caps.size() != existing_auth.caps.size()) {
+      ss << "entity " << name << " exists but caps do not match";
+      return -EINVAL;
+    }
+    for (auto& it : caps) {
+      if (existing_auth.caps.count(it.first) == 0 ||
+          !existing_auth.caps[it.first].contents_equal(it.second)) {
+        ss << "entity " << name << " exists but cap "
+          << it.first << " does not match";
+        return -EINVAL;
+      }
+    }
+
+    // they match, no-op
+    return 0;
+  }
+  return -ENOENT;
+}
+
+int AuthMonitor::add_entity(
+    const EntityName& name,
+    const EntityAuth& auth)
+{
+
+  // okay, add it.
+  KeyServerData::Incremental auth_inc;
+  auth_inc.op = KeyServerData::AUTH_INC_ADD;
+  auth_inc.name = name;
+  auth_inc.auth = auth;
+
+  dout(10) << " add auth entity " << auth_inc.name << dendl;
+  dout(30) << "    " << auth_inc.auth << dendl;
+  push_cephx_inc(auth_inc);
+  return 0;
+}
+
+int AuthMonitor::validate_osd_destroy(
+    int32_t id,
+    const uuid_d& uuid,
+    EntityName& cephx_entity,
+    EntityName& lockbox_entity,
+    stringstream& ss)
+{
+  ceph_assert(paxos.is_plugged());
+
+  dout(10) << __func__ << " id " << id << " uuid " << uuid << dendl;
+
+  string cephx_str = "osd." + stringify(id);
+  string lockbox_str = "client.osd-lockbox." + stringify(uuid);
+
+  if (!cephx_entity.from_str(cephx_str)) {
+    dout(10) << __func__ << " invalid cephx entity '"
+             << cephx_str << "'" << dendl;
+    ss << "invalid cephx key entity '" << cephx_str << "'";
+    return -EINVAL;
+  }
+
+  if (!lockbox_entity.from_str(lockbox_str)) {
+    dout(10) << __func__ << " invalid lockbox entity '"
+             << lockbox_str << "'" << dendl;
+    ss << "invalid lockbox key entity '" << lockbox_str << "'";
+    return -EINVAL;
+  }
+
+  if (!mon.key_server.contains(cephx_entity) &&
+      !mon.key_server.contains(lockbox_entity)) {
+    return -ENOENT;
+  }
+
+  return 0;
+}
+
+int AuthMonitor::do_osd_destroy(
+    const EntityName& cephx_entity,
+    const EntityName& lockbox_entity)
+{
+  ceph_assert(paxos.is_plugged());
+
+  dout(10) << __func__ << " cephx " << cephx_entity
+                       << " lockbox " << lockbox_entity << dendl;
+
+  bool removed = false;
+
+  int err = remove_entity(cephx_entity);
+  if (err == -ENOENT) {
+    dout(10) << __func__ << " " << cephx_entity << " does not exist" << dendl;
+  } else {
+    removed = true;
+  }
+
+  err = remove_entity(lockbox_entity);
+  if (err == -ENOENT) {
+    dout(10) << __func__ << " " << lockbox_entity << " does not exist" << dendl;
+  } else {
+    removed = true;
+  }
+
+  if (!removed) {
+    dout(10) << __func__ << " entities do not exist -- no-op." << dendl;
+    return 0;
+  }
+
+  // given we have paxos plugged, this will not result in a proposal
+  // being triggered, but it will still be needed so that we get our
+  // pending state encoded into the paxos' pending transaction.
+  propose_pending();
+  return 0;
+}
+
+int _create_auth(
+    EntityAuth& auth,
+    const string& key,
+    const map<string,bufferlist>& caps)
+{
+  if (key.empty())
+    return -EINVAL;
+  try {
+    auth.key.decode_base64(key);
+  } catch (ceph::buffer::error& e) {
+    return -EINVAL;
+  }
+  auth.caps = caps;
+  return 0;
+}
+
+int AuthMonitor::validate_osd_new(
+    int32_t id,
+    const uuid_d& uuid,
+    const string& cephx_secret,
+    const string& lockbox_secret,
+    auth_entity_t& cephx_entity,
+    auth_entity_t& lockbox_entity,
+    stringstream& ss)
+{
+
+  dout(10) << __func__ << " osd." << id << " uuid " << uuid << dendl;
+
+  map<string,bufferlist> cephx_caps = {
+    { "osd", _encode_cap("allow *") },
+    { "mon", _encode_cap("allow profile osd") },
+    { "mgr", _encode_cap("allow profile osd") }
+  };
+  map<string,bufferlist> lockbox_caps = {
+    { "mon", _encode_cap("allow command \"config-key get\" "
+        "with key=\"dm-crypt/osd/" +
+        stringify(uuid) +
+        "/luks\"") }
+  };
+
+  bool has_lockbox = !lockbox_secret.empty();
+
+  string cephx_name = "osd." + stringify(id);
+  string lockbox_name = "client.osd-lockbox." + stringify(uuid);
+
+  if (!cephx_entity.name.from_str(cephx_name)) {
+    dout(10) << __func__ << " invalid cephx entity '"
+             << cephx_name << "'" << dendl;
+    ss << "invalid cephx key entity '" << cephx_name << "'";
+    return -EINVAL;
+  }
+
+  if (has_lockbox) {
+    if (!lockbox_entity.name.from_str(lockbox_name)) {
+      dout(10) << __func__ << " invalid cephx lockbox entity '"
+               << lockbox_name << "'" << dendl;
+      ss << "invalid cephx lockbox entity '" << lockbox_name << "'";
+      return -EINVAL;
+    }
+  }
+
+  if (entity_is_pending(cephx_entity.name) ||
+      (has_lockbox && entity_is_pending(lockbox_entity.name))) {
+    // If we have pending entities for either the cephx secret or the
+    // lockbox secret, then our safest bet is to retry the command at
+    // a later time. These entities may be pending because an `osd new`
+    // command has been run (which is unlikely, due to the nature of
+    // the operation, which will force a paxos proposal), or (more likely)
+    // because a competing client created those entities before we handled
+    // the `osd new` command. Regardless, let's wait and see.
+    return -EAGAIN;
+  }
+
+  if (!is_valid_cephx_key(cephx_secret)) {
+    ss << "invalid cephx secret.";
+    return -EINVAL;
+  }
+
+  if (has_lockbox && !is_valid_cephx_key(lockbox_secret)) {
+    ss << "invalid cephx lockbox secret.";
+    return -EINVAL;
+  }
+
+  int err = _create_auth(cephx_entity.auth, cephx_secret, cephx_caps);
+  ceph_assert(0 == err);
+
+  bool cephx_is_idempotent = false, lockbox_is_idempotent = false;
+  err = exists_and_matches_entity(cephx_entity, true, ss);
+
+  if (err != -ENOENT) {
+    if (err < 0) {
+      return err;
+    }
+    ceph_assert(0 == err);
+    cephx_is_idempotent = true;
+  }
+
+  if (has_lockbox) {
+    err = _create_auth(lockbox_entity.auth, lockbox_secret, lockbox_caps);
+    ceph_assert(err == 0);
+    err = exists_and_matches_entity(lockbox_entity, true, ss);
+    if (err != -ENOENT) {
+      if (err < 0) {
+        return err;
+      }
+      ceph_assert(0 == err);
+      lockbox_is_idempotent = true;
+    }
+  }
+
+  if (cephx_is_idempotent && (!has_lockbox || lockbox_is_idempotent)) {
+    return EEXIST;
+  }
+
+  return 0;
+}
+
+int AuthMonitor::do_osd_new(
+    const auth_entity_t& cephx_entity,
+    const auth_entity_t& lockbox_entity,
+    bool has_lockbox)
+{
+  ceph_assert(paxos.is_plugged());
+
+  dout(10) << __func__ << " cephx " << cephx_entity.name
+           << " lockbox ";
+  if (has_lockbox) {
+    *_dout << lockbox_entity.name;
+  } else {
+    *_dout << "n/a";
+  }
+  *_dout << dendl;
+
+  // we must have validated before reaching this point.
+  // if keys exist, then this means they also match; otherwise we would
+  // have failed before calling this function.
+  bool cephx_exists = mon.key_server.contains(cephx_entity.name);
+
+  if (!cephx_exists) {
+    int err = add_entity(cephx_entity.name, cephx_entity.auth);
+    ceph_assert(0 == err);
+  }
+
+  if (has_lockbox &&
+      !mon.key_server.contains(lockbox_entity.name)) {
+    int err = add_entity(lockbox_entity.name, lockbox_entity.auth);
+    ceph_assert(0 == err);
+  }
+
+  // given we have paxos plugged, this will not result in a proposal
+  // being triggered, but it will still be needed so that we get our
+  // pending state encoded into the paxos' pending transaction.
+  propose_pending();
+  return 0;
+}
+
+bool AuthMonitor::valid_caps(
+    const string& type,
+    const string& caps,
+    ostream *out)
+{
+  if (type == "mon") {
+    MonCap moncap;
+    if (!moncap.parse(caps, out)) {
+      return false;
+    }
+    return true;
+  }
+
+  if (!g_conf().get_val<bool>("mon_auth_validate_all_caps")) {
+    return true;
+  }
+
+  if (type == "mgr") {
+    MgrCap mgrcap;
+    if (!mgrcap.parse(caps, out)) {
+      return false;
+    }
+  } else if (type == "osd") {
+    OSDCap ocap;
+    if (!ocap.parse(caps, out)) {
+      return false;
+    }
+  } else if (type == "mds") {
+    MDSAuthCaps mdscap;
+    if (!mdscap.parse(g_ceph_context, caps, out)) {
+      return false;
+    }
+  } else {
+    if (out) {
+      *out << "unknown cap type '" << type << "'";
+    }
+    return false;
+  }
+  return true;
+}
+
+bool AuthMonitor::valid_caps(const vector<string>& caps, ostream *out)
+{
+  for (vector<string>::const_iterator p = caps.begin();
+       p != caps.end(); p += 2) {
+    if ((p+1) == caps.end()) {
+      *out << "cap '" << *p << "' has no value";
+      return false;
+    }
+    if (!valid_caps(*p, *(p+1), out)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+bool AuthMonitor::prepare_command(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonCommand>();
+  stringstream ss, ds;
+  bufferlist rdata;
+  string rs;
+  int err = -EINVAL;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    // ss has reason for failure
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  vector<string>caps_vec;
+  string entity_name;
+  EntityName entity;
+
+  cmd_getval(cmdmap, "prefix", prefix);
+
+  string format;
+  cmd_getval(cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    mon.reply_command(op, -EACCES, "access denied", rdata, get_last_committed());
+    return true;
+  }
+
+  cmd_getval(cmdmap, "caps", caps_vec);
+  // fs authorize command's can have odd number of caps arguments
+  if ((prefix != "fs authorize") && (caps_vec.size() % 2) != 0) {
+    ss << "bad capabilities request; odd number of arguments";
+    err = -EINVAL;
+    goto done;
+  }
+
+  cmd_getval(cmdmap, "entity", entity_name);
+  if (!entity_name.empty() && !entity.from_str(entity_name)) {
+    ss << "bad entity name";
+    err = -EINVAL;
+    goto done;
+  }
+
+  if (prefix == "auth import") {
+    bufferlist bl = m->get_data();
+    if (bl.length() == 0) {
+      ss << "auth import: no data supplied";
+      getline(ss, rs);
+      mon.reply_command(op, -EINVAL, rs, get_last_committed());
+      return true;
+    }
+    auto iter = bl.cbegin();
+    KeyRing keyring;
+    try {
+      decode(keyring, iter);
+    } catch (const ceph::buffer::error &ex) {
+      ss << "error decoding keyring" << " " << ex.what();
+      err = -EINVAL;
+      goto done;
+    }
+    err = import_keyring(keyring);
+    if (err < 0) {
+      ss << "auth import: no caps supplied";
+      getline(ss, rs);
+      mon.reply_command(op, -EINVAL, rs, get_last_committed());
+      return true;
+    }
+    ss << "imported keyring";
+    getline(ss, rs);
+    err = 0;
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "auth add" && !entity_name.empty()) {
+    /* expected behavior:
+     *  - if command reproduces current state, return 0.
+     *  - if command adds brand new entity, handle it.
+     *  - if command adds new state to existing entity, return error.
+     */
+    KeyServerData::Incremental auth_inc;
+    auth_inc.name = entity;
+    bufferlist bl = m->get_data();
+    bool has_keyring = (bl.length() > 0);
+    map<string,bufferlist> new_caps;
+
+    KeyRing new_keyring;
+    if (has_keyring) {
+      auto iter = bl.cbegin();
+      try {
+        decode(new_keyring, iter);
+      } catch (const ceph::buffer::error &ex) {
+        ss << "error decoding keyring";
+        err = -EINVAL;
+        goto done;
+      }
+    }
+
+    if (!valid_caps(caps_vec, &ss)) {
+      err = -EINVAL;
+      goto done;
+    }
+
+    // are we about to have it?
+    if (entity_is_pending(entity)) {
+      wait_for_finished_proposal(op,
+          new Monitor::C_Command(mon, op, 0, rs, get_last_committed() + 1));
+      return true;
+    }
+
+    // build new caps from provided arguments (if available)
+    for (vector<string>::iterator it = caps_vec.begin();
+	 it != caps_vec.end() && (it + 1) != caps_vec.end();
+	 it += 2) {
+      string sys = *it;
+      bufferlist cap;
+      encode(*(it+1), cap);
+      new_caps[sys] = cap;
+    }
+
+    // pull info out of provided keyring
+    EntityAuth new_inc;
+    if (has_keyring) {
+      if (!new_keyring.get_auth(auth_inc.name, new_inc)) {
+	ss << "key for " << auth_inc.name
+	   << " not found in provided keyring";
+	err = -EINVAL;
+	goto done;
+      }
+      if (!new_caps.empty() && !new_inc.caps.empty()) {
+	ss << "caps cannot be specified both in keyring and in command";
+	err = -EINVAL;
+	goto done;
+      }
+      if (new_caps.empty()) {
+	new_caps = new_inc.caps;
+      }
+    }
+
+    err = exists_and_matches_entity(auth_inc.name, new_inc,
+                                    new_caps, has_keyring, ss);
+    // if entity/key/caps do not exist in the keyring, just fall through
+    // and add the entity; otherwise, make sure everything matches (in
+    // which case it's a no-op), because if not we must fail.
+    if (err != -ENOENT) {
+      if (err < 0) {
+        goto done;
+      }
+      // no-op.
+      ceph_assert(err == 0);
+      goto done;
+    }
+    err = 0;
+
+    // okay, add it.
+    if (!has_keyring) {
+      dout(10) << "AuthMonitor::prepare_command generating random key for "
+        << auth_inc.name << dendl;
+      new_inc.key.create(g_ceph_context, CEPH_CRYPTO_AES);
+    }
+    new_inc.caps = new_caps;
+
+    err = add_entity(auth_inc.name, new_inc);
+    ceph_assert(err == 0);
+
+    ss << "added key for " << auth_inc.name;
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						   get_last_committed() + 1));
+    return true;
+  } else if ((prefix == "auth get-or-create-key" ||
+	      prefix == "auth get-or-create") &&
+	     !entity_name.empty()) {
+    // auth get-or-create <name> [mon osdcapa osd osdcapb ...]
+
+    if (!valid_caps(caps_vec, &ss)) {
+      err = -EINVAL;
+      goto done;
+    }
+
+    // Parse the list of caps into a map
+    std::map<std::string, bufferlist> wanted_caps;
+    for (vector<string>::const_iterator it = caps_vec.begin();
+	 it != caps_vec.end() && (it + 1) != caps_vec.end();
+	 it += 2) {
+      const std::string &sys = *it;
+      bufferlist cap;
+      encode(*(it+1), cap);
+      wanted_caps[sys] = cap;
+    }
+
+    // do we have it?
+    EntityAuth entity_auth;
+    if (mon.key_server.get_auth(entity, entity_auth)) {
+      for (const auto &sys_cap : wanted_caps) {
+	if (entity_auth.caps.count(sys_cap.first) == 0 ||
+	    !entity_auth.caps[sys_cap.first].contents_equal(sys_cap.second)) {
+	  ss << "key for " << entity << " exists but cap " << sys_cap.first
+            << " does not match";
+	  err = -EINVAL;
+	  goto done;
+	}
+      }
+
+      if (prefix == "auth get-or-create-key") {
+        if (f) {
+          entity_auth.key.encode_formatted("auth", f.get(), rdata);
+        } else {
+          ds << entity_auth.key;
+        }
+      } else {
+	KeyRing kr;
+	kr.add(entity, entity_auth.key);
+        if (f) {
+          kr.set_caps(entity, entity_auth.caps);
+          kr.encode_formatted("auth", f.get(), rdata);
+        } else {
+          kr.encode_plaintext(rdata);
+        }
+      }
+      err = 0;
+      goto done;
+    }
+
+    // ...or are we about to?
+    for (vector<Incremental>::iterator p = pending_auth.begin();
+	 p != pending_auth.end();
+	 ++p) {
+      if (p->inc_type == AUTH_DATA) {
+	KeyServerData::Incremental auth_inc;
+	auto q = p->auth_data.cbegin();
+	decode(auth_inc, q);
+	if (auth_inc.op == KeyServerData::AUTH_INC_ADD &&
+	    auth_inc.name == entity) {
+	  wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						get_last_committed() + 1));
+	  return true;
+	}
+      }
+    }
+
+    // create it
+    KeyServerData::Incremental auth_inc;
+    auth_inc.op = KeyServerData::AUTH_INC_ADD;
+    auth_inc.name = entity;
+    auth_inc.auth.key.create(g_ceph_context, CEPH_CRYPTO_AES);
+    auth_inc.auth.caps = wanted_caps;
+
+    push_cephx_inc(auth_inc);
+
+    if (prefix == "auth get-or-create-key") {
+      if (f) {
+        auth_inc.auth.key.encode_formatted("auth", f.get(), rdata);
+      } else {
+        ds << auth_inc.auth.key;
+      }
+    } else {
+      KeyRing kr;
+      kr.add(entity, auth_inc.auth.key);
+      if (f) {
+        kr.set_caps(entity, wanted_caps);
+        kr.encode_formatted("auth", f.get(), rdata);
+      } else {
+        kr.encode_plaintext(rdata);
+      }
+    }
+
+    rdata.append(ds);
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs, rdata,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "fs authorize") {
+    string filesystem;
+    cmd_getval(cmdmap, "filesystem", filesystem);
+    string mon_cap_string = "allow r";
+    string mds_cap_string, osd_cap_string;
+    string osd_cap_wanted = "r";
+
+    std::shared_ptr<const Filesystem> fs;
+    if (filesystem != "*" && filesystem != "all") {
+      fs = mon.mdsmon()->get_fsmap().get_filesystem(filesystem);
+      if (fs == nullptr) {
+	ss << "filesystem " << filesystem << " does not exist.";
+	err = -EINVAL;
+	goto done;
+      } else {
+	mon_cap_string += " fsname=" + std::string(fs->mds_map.get_fs_name());
+      }
+    }
+
+    for (auto it = caps_vec.begin();
+	 it != caps_vec.end() && (it + 1) != caps_vec.end();
+	 it += 2) {
+      const string &path = *it;
+      const string &cap = *(it+1);
+      bool root_squash = false;
+      if ((it + 2) != caps_vec.end() && *(it+2) == "root_squash") {
+	root_squash = true;
+	++it;
+      }
+
+      if (cap != "r" && cap.compare(0, 2, "rw")) {
+	ss << "Permission flags must start with 'r' or 'rw'.";
+	err = -EINVAL;
+	goto done;
+      }
+      if (cap.compare(0, 2, "rw") == 0)
+	osd_cap_wanted = "rw";
+
+      char last='\0';
+      for (size_t i = 2; i < cap.size(); ++i) {
+	char c = cap.at(i);
+	if (last >= c) {
+	  ss << "Permission flags (except 'rw') must be specified in alphabetical order.";
+	  err = -EINVAL;
+	  goto done;
+	}
+	switch (c) {
+	case 'p':
+	  break;
+	case 's':
+	  break;
+	default:
+	  ss << "Unknown permission flag '" << c << "'.";
+	  err = -EINVAL;
+	  goto done;
+	}
+      }
+
+      mds_cap_string += mds_cap_string.empty() ? "" : ", ";
+      mds_cap_string += "allow " + cap;
+
+      if (filesystem != "*" && filesystem != "all" && fs != nullptr) {
+	mds_cap_string += " fsname=" + std::string(fs->mds_map.get_fs_name());
+      }
+
+      if (path != "/") {
+	mds_cap_string += " path=" + path;
+      }
+
+      if (root_squash) {
+	mds_cap_string += " root_squash";
+      }
+    }
+
+    osd_cap_string += osd_cap_string.empty() ? "" : ", ";
+    osd_cap_string += "allow " + osd_cap_wanted
+      + " tag " + pg_pool_t::APPLICATION_NAME_CEPHFS
+      + " data=" + filesystem;
+
+    std::map<string, bufferlist> wanted_caps = {
+      { "mon", _encode_cap(mon_cap_string) },
+      { "osd", _encode_cap(osd_cap_string) },
+      { "mds", _encode_cap(mds_cap_string) }
+    };
+
+    if (!valid_caps("mon", mon_cap_string, &ss) ||
+        !valid_caps("osd", osd_cap_string, &ss) ||
+	!valid_caps("mds", mds_cap_string, &ss)) {
+      err = -EINVAL;
+      goto done;
+    }
+
+    EntityAuth entity_auth;
+    if (mon.key_server.get_auth(entity, entity_auth)) {
+      for (const auto &sys_cap : wanted_caps) {
+	if (entity_auth.caps.count(sys_cap.first) == 0 ||
+	    !entity_auth.caps[sys_cap.first].contents_equal(sys_cap.second)) {
+	  ss << entity << " already has fs capabilities that differ from "
+	     << "those supplied. To generate a new auth key for " << entity
+	     << ", first remove " << entity << " from configuration files, "
+	     << "execute 'ceph auth rm " << entity << "', then execute this "
+	     << "command again.";
+	  err = -EINVAL;
+	  goto done;
+	}
+      }
+
+      KeyRing kr;
+      kr.add(entity, entity_auth.key);
+      if (f) {
+	kr.set_caps(entity, entity_auth.caps);
+	kr.encode_formatted("auth", f.get(), rdata);
+      } else {
+	kr.encode_plaintext(rdata);
+      }
+      err = 0;
+      goto done;
+    }
+
+    KeyServerData::Incremental auth_inc;
+    auth_inc.op = KeyServerData::AUTH_INC_ADD;
+    auth_inc.name = entity;
+    auth_inc.auth.key.create(g_ceph_context, CEPH_CRYPTO_AES);
+    auth_inc.auth.caps = wanted_caps;
+
+    push_cephx_inc(auth_inc);
+    KeyRing kr;
+    kr.add(entity, auth_inc.auth.key);
+    if (f) {
+      kr.set_caps(entity, wanted_caps);
+      kr.encode_formatted("auth", f.get(), rdata);
+    } else {
+      kr.encode_plaintext(rdata);
+    }
+
+    rdata.append(ds);
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs, rdata,
+						  get_last_committed() + 1));
+    return true;
+  } else if (prefix == "auth caps" && !entity_name.empty()) {
+    KeyServerData::Incremental auth_inc;
+    auth_inc.name = entity;
+    if (!mon.key_server.get_auth(auth_inc.name, auth_inc.auth)) {
+      ss << "couldn't find entry " << auth_inc.name;
+      err = -ENOENT;
+      goto done;
+    }
+
+    if (!valid_caps(caps_vec, &ss)) {
+      err = -EINVAL;
+      goto done;
+    }
+
+    map<string,bufferlist> newcaps;
+    for (vector<string>::iterator it = caps_vec.begin();
+	 it != caps_vec.end(); it += 2)
+      encode(*(it+1), newcaps[*it]);
+
+    auth_inc.op = KeyServerData::AUTH_INC_ADD;
+    auth_inc.auth.caps = newcaps;
+    push_cephx_inc(auth_inc);
+
+    ss << "updated caps for " << auth_inc.name;
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if ((prefix == "auth del" || prefix == "auth rm") &&
+             !entity_name.empty()) {
+    KeyServerData::Incremental auth_inc;
+    auth_inc.name = entity;
+    if (!mon.key_server.contains(auth_inc.name)) {
+      ss << "entity " << entity << " does not exist";
+      err = 0;
+      goto done;
+    }
+    auth_inc.op = KeyServerData::AUTH_INC_DEL;
+    push_cephx_inc(auth_inc);
+
+    ss << "updated";
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  }
+done:
+  rdata.append(ds);
+  getline(ss, rs, '\0');
+  mon.reply_command(op, err, rs, rdata, get_last_committed());
+  return false;
+}
+
+bool AuthMonitor::prepare_global_id(MonOpRequestRef op)
+{
+  dout(10) << "AuthMonitor::prepare_global_id" << dendl;
+  increase_max_global_id();
+
+  return true;
+}
+
+bool AuthMonitor::_upgrade_format_to_dumpling()
+{
+  dout(1) << __func__ << " upgrading from format 0 to 1" << dendl;
+  ceph_assert(format_version == 0);
+
+  bool changed = false;
+  map<EntityName, EntityAuth>::iterator p;
+  for (p = mon.key_server.secrets_begin();
+       p != mon.key_server.secrets_end();
+       ++p) {
+    // grab mon caps, if any
+    string mon_caps;
+    if (p->second.caps.count("mon") == 0)
+      continue;
+    try {
+      auto it = p->second.caps["mon"].cbegin();
+      decode(mon_caps, it);
+    }
+    catch (const ceph::buffer::error&) {
+      dout(10) << __func__ << " unable to parse mon cap for "
+	       << p->first << dendl;
+      continue;
+    }
+
+    string n = p->first.to_str();
+    string new_caps;
+
+    // set daemon profiles
+    if ((p->first.is_osd() || p->first.is_mds()) &&
+        mon_caps == "allow rwx") {
+      new_caps = string("allow profile ") + std::string(p->first.get_type_name());
+    }
+
+    // update bootstrap keys
+    if (n == "client.bootstrap-osd") {
+      new_caps = "allow profile bootstrap-osd";
+    }
+    if (n == "client.bootstrap-mds") {
+      new_caps = "allow profile bootstrap-mds";
+    }
+
+    if (new_caps.length() > 0) {
+      dout(5) << __func__ << " updating " << p->first << " mon cap from "
+	      << mon_caps << " to " << new_caps << dendl;
+
+      bufferlist bl;
+      encode(new_caps, bl);
+
+      KeyServerData::Incremental auth_inc;
+      auth_inc.name = p->first;
+      auth_inc.auth = p->second;
+      auth_inc.auth.caps["mon"] = bl;
+      auth_inc.op = KeyServerData::AUTH_INC_ADD;
+      push_cephx_inc(auth_inc);
+      changed = true;
+    }
+  }
+  return changed;
+}
+
+bool AuthMonitor::_upgrade_format_to_luminous()
+{
+  dout(1) << __func__ << " upgrading from format 1 to 2" << dendl;
+  ceph_assert(format_version == 1);
+
+  bool changed = false;
+  map<EntityName, EntityAuth>::iterator p;
+  for (p = mon.key_server.secrets_begin();
+       p != mon.key_server.secrets_end();
+       ++p) {
+    string n = p->first.to_str();
+
+    string newcap;
+    if (n == "client.admin") {
+      // admin gets it all
+      newcap = "allow *";
+    } else if (n.find("osd.") == 0 ||
+	       n.find("mds.") == 0 ||
+	       n.find("mon.") == 0) {
+      // daemons follow their profile
+      string type = n.substr(0, 3);
+      newcap = "allow profile " + type;
+    } else if (p->second.caps.count("mon")) {
+      // if there are any mon caps, give them 'r' mgr caps
+      newcap = "allow r";
+    }
+
+    if (newcap.length() > 0) {
+      dout(5) << " giving " << n << " mgr '" << newcap << "'" << dendl;
+      bufferlist bl;
+      encode(newcap, bl);
+
+      EntityAuth auth = p->second;
+      auth.caps["mgr"] = bl;
+
+      add_entity(p->first, auth);
+      changed = true;
+    }
+
+    if (n.find("mgr.") == 0 &&
+	p->second.caps.count("mon")) {
+      // the kraken ceph-mgr@.service set the mon cap to 'allow *'.
+      auto blp = p->second.caps["mon"].cbegin();
+      string oldcaps;
+      decode(oldcaps, blp);
+      if (oldcaps == "allow *") {
+	dout(5) << " fixing " << n << " mon cap to 'allow profile mgr'"
+		<< dendl;
+	bufferlist bl;
+	encode("allow profile mgr", bl);
+
+	EntityAuth auth = p->second;
+	auth.caps["mon"] = bl;
+	add_entity(p->first, p->second);
+	changed = true;
+      }
+    }
+  }
+
+  // add bootstrap key if it does not already exist
+  // (might have already been get-or-create'd by
+  //  ceph-create-keys)
+  EntityName bootstrap_mgr_name;
+  int r = bootstrap_mgr_name.from_str("client.bootstrap-mgr");
+  ceph_assert(r);
+  if (!mon.key_server.contains(bootstrap_mgr_name)) {
+
+    EntityName name = bootstrap_mgr_name;
+    EntityAuth auth;
+    encode("allow profile bootstrap-mgr", auth.caps["mon"]);
+    auth.key.create(g_ceph_context, CEPH_CRYPTO_AES);
+    add_entity(name, auth);
+    changed = true;
+  }
+  return changed;
+}
+
+bool AuthMonitor::_upgrade_format_to_mimic()
+{
+  dout(1) << __func__ << " upgrading from format 2 to 3" << dendl;
+  ceph_assert(format_version == 2);
+
+  list<pair<EntityName,EntityAuth> > auth_lst;
+  _generate_bootstrap_keys(&auth_lst);
+
+  bool changed = false;
+  for (auto &p : auth_lst) {
+    if (mon.key_server.contains(p.first)) {
+      continue;
+    }
+    int err = add_entity(p.first, p.second);
+    ceph_assert(err == 0);
+    changed = true;
+  }
+
+  return changed;
+}
+
+void AuthMonitor::upgrade_format()
+{
+  constexpr unsigned int FORMAT_NONE = 0;
+  constexpr unsigned int FORMAT_DUMPLING = 1;
+  constexpr unsigned int FORMAT_LUMINOUS = 2;
+  constexpr unsigned int FORMAT_MIMIC = 3;
+
+  // when upgrading from the current format to a new format, ensure that
+  // the new format doesn't break the older format. I.e., if a given format N
+  // changes or adds something, ensure that when upgrading from N-1 to N+1, we
+  // still observe the changes for format N if those have not been superseded
+  // by N+1.
+
+  unsigned int current = FORMAT_MIMIC;
+  if (!mon.get_quorum_mon_features().contains_all(
+	ceph::features::mon::FEATURE_LUMINOUS)) {
+    // pre-luminous quorum
+    current = FORMAT_DUMPLING;
+  } else if (!mon.get_quorum_mon_features().contains_all(
+	ceph::features::mon::FEATURE_MIMIC)) {
+    // pre-mimic quorum
+    current = FORMAT_LUMINOUS;
+  }
+  if (format_version >= current) {
+    dout(20) << __func__ << " format " << format_version
+	     << " is current" << dendl;
+    return;
+  }
+
+  // perform a rolling upgrade of the new format, if necessary.
+  // i.e., if we are moving from format NONE to MIMIC, we will first upgrade
+  // to DUMPLING, then to LUMINOUS, and finally to MIMIC, in several different
+  // proposals.
+
+  bool changed = false;
+  if (format_version == FORMAT_NONE) {
+    changed = _upgrade_format_to_dumpling();
+
+  } else if (format_version == FORMAT_DUMPLING) {
+    changed = _upgrade_format_to_luminous();
+  } else if (format_version == FORMAT_LUMINOUS) {
+    changed = _upgrade_format_to_mimic();
+  }
+
+  if (changed) {
+    // note new format
+    dout(10) << __func__ << " proposing update from format " << format_version
+	     << " -> " << current << dendl;
+    format_version = current;
+    propose_pending();
+  }
+}
+
+void AuthMonitor::dump_info(Formatter *f)
+{
+  /*** WARNING: do not include any privileged information here! ***/
+  f->open_object_section("auth");
+  f->dump_unsigned("first_committed", get_first_committed());
+  f->dump_unsigned("last_committed", get_last_committed());
+  f->dump_unsigned("num_secrets", mon.key_server.get_num_secrets());
+  f->close_section();
+}
diff --git a/src/mon/AuthMonitor.h b/src/mon/AuthMonitor.h
new file mode 100644
index 000000000..4312b5607
--- /dev/null
+++ b/src/mon/AuthMonitor.h
@@ -0,0 +1,237 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_AUTHMONITOR_H
+#define CEPH_AUTHMONITOR_H
+
+#include <map>
+#include <set>
+
+#include "global/global_init.h"
+#include "include/ceph_features.h"
+#include "include/types.h"
+#include "mon/PaxosService.h"
+#include "mon/MonitorDBStore.h"
+
+class MAuth;
+class KeyRing;
+class Monitor;
+
+#define MIN_GLOBAL_ID 0x1000
+
+class AuthMonitor : public PaxosService {
+public:
+  enum IncType {
+    GLOBAL_ID,
+    AUTH_DATA,
+  };
+  struct Incremental {
+    IncType inc_type;
+    uint64_t max_global_id;
+    uint32_t auth_type;
+    ceph::buffer::list auth_data;
+
+    Incremental() : inc_type(GLOBAL_ID), max_global_id(0), auth_type(0) {}
+
+    void encode(ceph::buffer::list& bl, uint64_t features=-1) const {
+      using ceph::encode;
+      ENCODE_START(2, 2, bl);
+      __u32 _type = (__u32)inc_type;
+      encode(_type, bl);
+      if (_type == GLOBAL_ID) {
+	encode(max_global_id, bl);
+      } else {
+	encode(auth_type, bl);
+	encode(auth_data, bl);
+      }
+      ENCODE_FINISH(bl);
+    }
+    void decode(ceph::buffer::list::const_iterator& bl) {
+      DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
+      __u32 _type;
+      decode(_type, bl);
+      inc_type = (IncType)_type;
+      ceph_assert(inc_type >= GLOBAL_ID && inc_type <= AUTH_DATA);
+      if (_type == GLOBAL_ID) {
+	decode(max_global_id, bl);
+      } else {
+	decode(auth_type, bl);
+	decode(auth_data, bl);
+      }
+      DECODE_FINISH(bl);
+    }
+    void dump(ceph::Formatter *f) const {
+      f->dump_int("type", inc_type);
+      f->dump_int("max_global_id", max_global_id);
+      f->dump_int("auth_type", auth_type);
+      f->dump_int("auth_data_len", auth_data.length());
+    }
+    static void generate_test_instances(std::list<Incremental*>& ls) {
+      ls.push_back(new Incremental);
+      ls.push_back(new Incremental);
+      ls.back()->inc_type = GLOBAL_ID;
+      ls.back()->max_global_id = 1234;
+      ls.push_back(new Incremental);
+      ls.back()->inc_type = AUTH_DATA;
+      ls.back()->auth_type = 12;
+      ls.back()->auth_data.append("foo");
+    }
+  };
+
+  struct auth_entity_t {
+    EntityName name;
+    EntityAuth auth;
+  };
+
+
+private:
+  std::vector<Incremental> pending_auth;
+  uint64_t max_global_id;
+  uint64_t last_allocated_id;
+
+  // these are protected by mon->auth_lock
+  int mon_num = 0, mon_rank = 0;
+
+  bool _upgrade_format_to_dumpling();
+  bool _upgrade_format_to_luminous();
+  bool _upgrade_format_to_mimic();
+  void upgrade_format() override;
+
+  void export_keyring(KeyRing& keyring);
+  int import_keyring(KeyRing& keyring);
+
+  void push_cephx_inc(KeyServerData::Incremental& auth_inc) {
+    Incremental inc;
+    inc.inc_type = AUTH_DATA;
+    encode(auth_inc, inc.auth_data);
+    inc.auth_type = CEPH_AUTH_CEPHX;
+    pending_auth.push_back(inc);
+  }
+
+  /* validate mon/osd/mds caps; fail on unrecognized service/type */
+  bool valid_caps(const std::string& type, const std::string& caps, std::ostream *out);
+  bool valid_caps(const std::string& type, const ceph::buffer::list& bl, std::ostream *out) {
+    auto p = bl.begin();
+    std::string v;
+    try {
+      using ceph::decode;
+      decode(v, p);
+    } catch (ceph::buffer::error& e) {
+      *out << "corrupt capability encoding";
+      return false;
+    }
+    return valid_caps(type, v, out);
+  }
+  bool valid_caps(const std::vector<std::string>& caps, std::ostream *out);
+
+  void on_active() override;
+  bool should_propose(double& delay) override;
+  void get_initial_keyring(KeyRing *keyring);
+  void create_initial_keys(KeyRing *keyring);
+  void create_initial() override;
+  void update_from_paxos(bool *need_bootstrap) override;
+  void create_pending() override;  // prepare a new pending
+  bool prepare_global_id(MonOpRequestRef op);
+  bool _should_increase_max_global_id(); ///< called under mon->auth_lock
+  void increase_max_global_id();
+  uint64_t assign_global_id(bool should_increase_max);
+public:
+  uint64_t _assign_global_id(); ///< called under mon->auth_lock
+  void _set_mon_num_rank(int num, int rank); ///< called under mon->auth_lock
+
+private:
+  // propose pending update to peers
+  void encode_pending(MonitorDBStore::TransactionRef t) override;
+  void encode_full(MonitorDBStore::TransactionRef t) override;
+  version_t get_trim_to() const override;
+
+  bool preprocess_query(MonOpRequestRef op) override;  // true if processed.
+  bool prepare_update(MonOpRequestRef op) override;
+
+  bool prep_auth(MonOpRequestRef op, bool paxos_writable);
+
+  bool preprocess_command(MonOpRequestRef op);
+  bool prepare_command(MonOpRequestRef op);
+
+  bool check_rotate();
+
+  bool entity_is_pending(EntityName& entity);
+  int exists_and_matches_entity(
+      const auth_entity_t& entity,
+      bool has_secret,
+      std::stringstream& ss);
+  int exists_and_matches_entity(
+      const EntityName& name,
+      const EntityAuth& auth,
+      const std::map<std::string,ceph::buffer::list>& caps,
+      bool has_secret,
+      std::stringstream& ss);
+  int remove_entity(const EntityName &entity);
+  int add_entity(
+      const EntityName& name,
+      const EntityAuth& auth);
+
+ public:
+  AuthMonitor(Monitor &mn, Paxos &p, const std::string& service_name)
+    : PaxosService(mn, p, service_name),
+      max_global_id(0),
+      last_allocated_id(0)
+  {}
+
+  void pre_auth(MAuth *m);
+
+  void tick() override;  // check state, take actions
+
+  int validate_osd_destroy(
+      int32_t id,
+      const uuid_d& uuid,
+      EntityName& cephx_entity,
+      EntityName& lockbox_entity,
+      std::stringstream& ss);
+  int do_osd_destroy(
+      const EntityName& cephx_entity,
+      const EntityName& lockbox_entity);
+
+  int do_osd_new(
+      const auth_entity_t& cephx_entity,
+      const auth_entity_t& lockbox_entity,
+      bool has_lockbox);
+  int validate_osd_new(
+      int32_t id,
+      const uuid_d& uuid,
+      const std::string& cephx_secret,
+      const std::string& lockbox_secret,
+      auth_entity_t& cephx_entity,
+      auth_entity_t& lockbox_entity,
+      std::stringstream& ss);
+
+  void dump_info(ceph::Formatter *f);
+
+  bool is_valid_cephx_key(const std::string& k) {
+    if (k.empty())
+      return false;
+
+    EntityAuth ea;
+    try {
+      ea.key.decode_base64(k);
+      return true;
+    } catch (ceph::buffer::error& e) { /* fallthrough */ }
+    return false;
+  }
+};
+
+
+WRITE_CLASS_ENCODER_FEATURES(AuthMonitor::Incremental)
+
+#endif
diff --git a/src/mon/CMakeLists.txt b/src/mon/CMakeLists.txt
new file mode 100644
index 000000000..b4056fdb1
--- /dev/null
+++ b/src/mon/CMakeLists.txt
@@ -0,0 +1,42 @@
+set(lib_mon_srcs
+  ${CMAKE_SOURCE_DIR}/src/auth/cephx/CephxKeyServer.cc
+  ${CMAKE_SOURCE_DIR}/src/auth/cephx/CephxServiceHandler.cc
+  ${CMAKE_SOURCE_DIR}/src/auth/AuthServiceHandler.cc
+  Paxos.cc
+  PaxosService.cc
+  OSDMonitor.cc
+  MDSMonitor.cc
+  CommandHandler.cc
+  FSCommands.cc
+  MgrMonitor.cc
+  MgrStatMonitor.cc
+  Monitor.cc
+  MonmapMonitor.cc
+  LogMonitor.cc
+  AuthMonitor.cc
+  ConfigMap.cc
+  ConfigMonitor.cc
+  Elector.cc
+  ElectionLogic.cc
+  ConnectionTracker.cc
+  HealthMonitor.cc
+  KVMonitor.cc
+  ../mds/MDSAuthCaps.cc
+  ../mgr/mgr_commands.cc
+  ../osd/OSDCap.cc
+  $<TARGET_OBJECTS:mgr_cap_obj>)
+
+if(HAVE_GSSAPI)
+  list(APPEND lib_mon_srcs
+    ${CMAKE_SOURCE_DIR}/src/auth/krb/KrbServiceHandler.cpp)
+endif()
+
+add_library(mon STATIC
+  ${lib_mon_srcs})
+target_link_libraries(mon
+  kv
+  heap_profiler
+  fmt::fmt)
+if(WITH_JAEGER)
+  target_link_libraries(mon jaeger-base)
+endif()
diff --git a/src/mon/CommandHandler.cc b/src/mon/CommandHandler.cc
new file mode 100644
index 000000000..903d35927
--- /dev/null
+++ b/src/mon/CommandHandler.cc
@@ -0,0 +1,43 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2019 Red Hat Ltd
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "CommandHandler.h"
+
+#include "common/strtol.h"
+#include "include/ceph_assert.h"
+
+#include <ostream>
+#include <string>
+#include <string_view>
+
+int CommandHandler::parse_bool(std::string_view str, bool* result, std::ostream& ss)
+{
+  ceph_assert(result != nullptr);
+
+  std::string interr;
+  int64_t n = strict_strtoll(str.data(), 10, &interr);
+
+  if (str == "false" || str == "no"
+      || (interr.length() == 0 && n == 0)) {
+    *result = false;
+    return 0;
+  } else if (str == "true" || str == "yes"
+      || (interr.length() == 0 && n == 1)) {
+    *result = true;
+    return 0;
+  } else {
+    ss << "value must be false|no|0 or true|yes|1";
+    return -EINVAL;
+  }
+}
diff --git a/src/mon/CommandHandler.h b/src/mon/CommandHandler.h
new file mode 100644
index 000000000..167b4587f
--- /dev/null
+++ b/src/mon/CommandHandler.h
@@ -0,0 +1,35 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2019 Red Hat Ltd
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef COMMAND_HANDLER_H_
+#define COMMAND_HANDLER_H_
+
+#include <ostream>
+#include <string_view>
+
+class CommandHandler
+{
+public:
+  /**
+   * Parse true|yes|1 style boolean string from `bool_str`
+   * `result` must be non-null.
+   * `ss` will be populated with error message on error.
+   *
+   * @return 0 on success, else -EINVAL
+   */
+  int parse_bool(std::string_view str, bool* result, std::ostream& ss);
+};
+
+#endif
diff --git a/src/mon/ConfigMap.cc b/src/mon/ConfigMap.cc
new file mode 100644
index 000000000..763b8ce9b
--- /dev/null
+++ b/src/mon/ConfigMap.cc
@@ -0,0 +1,291 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <boost/algorithm/string/split.hpp>
+
+#include "ConfigMap.h"
+#include "crush/CrushWrapper.h"
+#include "common/entity_name.h"
+
+using namespace std::literals;
+
+using std::cerr;
+using std::cout;
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::setfill;
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+using std::unique_ptr;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::mono_clock;
+using ceph::mono_time;
+using ceph::timespan_str;
+
+int MaskedOption::get_precision(const CrushWrapper *crush)
+{
+  // 0 = most precise
+  if (mask.location_type.size()) {
+    int r = crush->get_type_id(mask.location_type);
+    if (r >= 0) {
+      return r;
+    }
+    // bad type name, ignore it
+  }
+  int num_types = crush->get_num_type_names();
+  if (mask.device_class.size()) {
+    return num_types;
+  }
+  return num_types + 1;
+}
+
+void OptionMask::dump(Formatter *f) const
+{
+  if (location_type.size()) {
+    f->dump_string("location_type", location_type);
+    f->dump_string("location_value", location_value);
+  }
+  if (device_class.size()) {
+    f->dump_string("device_class", device_class);
+  }
+}
+
+void MaskedOption::dump(Formatter *f) const
+{
+  f->dump_string("name", opt->name);
+  f->dump_string("value", raw_value);
+  f->dump_string("level", Option::level_to_str(opt->level));
+  f->dump_bool("can_update_at_runtime", opt->can_update_at_runtime());
+  f->dump_string("mask", mask.to_str());
+  mask.dump(f);
+}
+
+ostream& operator<<(ostream& out, const MaskedOption& o)
+{
+  out << o.opt->name;
+  if (o.mask.location_type.size()) {
+    out << "@" << o.mask.location_type << '=' << o.mask.location_value;
+  }
+  if (o.mask.device_class.size()) {
+    out << "@class=" << o.mask.device_class;
+  }
+  return out;
+}
+
+// ----------
+
+void Section::dump(Formatter *f) const
+{
+  for (auto& i : options) {
+    f->dump_object(i.first.c_str(), i.second);
+  }
+}
+
+std::string Section::get_minimal_conf() const
+{
+  std::string r;
+  for (auto& i : options) {
+    if (i.second.opt->has_flag(Option::FLAG_NO_MON_UPDATE) ||
+	i.second.opt->has_flag(Option::FLAG_MINIMAL_CONF)) {
+      if (i.second.mask.empty()) {
+	r += "\t"s + i.first + " = " + i.second.raw_value + "\n";
+      } else {
+	r += "\t# masked option excluded: " + i.first + " = " +
+	  i.second.raw_value + "\n";
+      }
+    }
+  }
+  return r;
+}
+
+
+// ------------
+
+void ConfigMap::dump(Formatter *f) const
+{
+  f->dump_object("global", global);
+  f->open_object_section("by_type");
+  for (auto& i : by_type) {
+    f->dump_object(i.first.c_str(), i.second);
+  }
+  f->close_section();
+  f->open_object_section("by_id");
+  for (auto& i : by_id) {
+    f->dump_object(i.first.c_str(), i.second);
+  }
+  f->close_section();
+}
+
+std::map<std::string,std::string,std::less<>>
+ConfigMap::generate_entity_map(
+  const EntityName& name,
+  const map<std::string,std::string>& crush_location,
+  const CrushWrapper *crush,
+  const std::string& device_class,
+  std::map<std::string,pair<std::string,const MaskedOption*>> *src)
+{
+  // global, then by type, then by name prefix component(s), then name.
+  // name prefix components are .-separated,
+  // e.g. client.a.b.c -> [global, client, client.a, client.a.b, client.a.b.c]
+  vector<pair<string,Section*>> sections = { make_pair("global", &global) };
+  auto p = by_type.find(name.get_type_name());
+  if (p != by_type.end()) {
+    sections.emplace_back(name.get_type_name(), &p->second);
+  }
+  vector<std::string> name_bits;
+  boost::split(name_bits, name.to_str(), [](char c){ return c == '.'; });
+  std::string tname;
+  for (unsigned p = 0; p < name_bits.size(); ++p) {
+    if (p) {
+      tname += '.';
+    }
+    tname += name_bits[p];
+    auto q = by_id.find(tname);
+    if (q != by_id.end()) {
+      sections.push_back(make_pair(tname, &q->second));
+    }
+  }
+  std::map<std::string,std::string,std::less<>> out;
+  MaskedOption *prev = nullptr;
+  for (auto s : sections) {
+    for (auto& i : s.second->options) {
+      auto& o = i.second;
+      // match against crush location, class
+      if (o.mask.device_class.size() &&
+	  o.mask.device_class != device_class) {
+	continue;
+      }
+      if (o.mask.location_type.size()) {
+	auto p = crush_location.find(o.mask.location_type);
+	if (p == crush_location.end() ||
+	    p->second != o.mask.location_value) {
+	  continue;
+	}
+      }
+      if (prev && prev->opt->name != i.first) {
+	prev = nullptr;
+      }
+      if (prev &&
+	  prev->get_precision(crush) < o.get_precision(crush)) {
+	continue;
+      }
+      out[i.first] = o.raw_value;
+      if (src) {
+	(*src)[i.first] = make_pair(s.first, &o);
+      }
+      prev = &o;
+    }
+  }
+  return out;
+}
+
+bool ConfigMap::parse_mask(
+  const std::string& who,
+  std::string *section,
+  OptionMask *mask)
+{
+  vector<std::string> split;
+  boost::split(split, who, [](char c){ return c == '/'; });
+  for (unsigned j = 0; j < split.size(); ++j) {
+    auto& i = split[j];
+    if (i == "global") {
+      *section = "global";
+      continue;
+    }
+    size_t delim = i.find(':');
+    if (delim != std::string::npos) {
+      string k = i.substr(0, delim);
+      if (k == "class") {
+	mask->device_class = i.substr(delim + 1);
+      } else {
+	mask->location_type = k;
+	mask->location_value = i.substr(delim + 1);
+      }
+      continue;
+    }
+    string type, id;
+    auto dotpos = i.find('.');
+    if (dotpos != std::string::npos) {
+      type = i.substr(0, dotpos);
+      id = i.substr(dotpos + 1);
+    } else {
+      type = i;
+    }
+    if (EntityName::str_to_ceph_entity_type(type) == CEPH_ENTITY_TYPE_ANY) {
+      return false;
+    }
+    *section = i;
+  }
+  return true;
+}
+
+void ConfigMap::parse_key(
+  const std::string& key,
+  std::string *name,
+  std::string *who)
+{
+  auto last_slash = key.rfind('/');
+  if (last_slash == std::string::npos) {
+    *name = key;
+  } else if (auto mgrpos = key.find("/mgr/"); mgrpos != std::string::npos) {
+    *name = key.substr(mgrpos + 1);
+    *who = key.substr(0, mgrpos);
+  } else {
+    *name = key.substr(last_slash + 1);
+    *who = key.substr(0, last_slash);
+  }
+}
+
+
+// --------------
+
+void ConfigChangeSet::dump(Formatter *f) const
+{
+  f->dump_int("version", version);
+  f->dump_stream("timestamp") << stamp;
+  f->dump_string("name", name);
+  f->open_array_section("changes");
+  for (auto& i : diff) {
+    f->open_object_section("change");
+    f->dump_string("name", i.first);
+    if (i.second.first) {
+      f->dump_string("previous_value", *i.second.first);
+    }
+    if (i.second.second) {
+      f->dump_string("new_value", *i.second.second);
+    }
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void ConfigChangeSet::print(ostream& out) const
+{
+  out << "--- " << version << " --- " << stamp;
+  if (name.size()) {
+    out << " --- " << name;
+  }
+  out << " ---\n";
+  for (auto& i : diff) {
+    if (i.second.first) {
+      out << "- " << i.first << " = " << *i.second.first << "\n";
+    }
+    if (i.second.second) {
+      out << "+ " << i.first << " = " << *i.second.second << "\n";
+    }
+  }
+}
diff --git a/src/mon/ConfigMap.h b/src/mon/ConfigMap.h
new file mode 100644
index 000000000..2ecdcc071
--- /dev/null
+++ b/src/mon/ConfigMap.h
@@ -0,0 +1,153 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <map>
+#include <ostream>
+#include <string>
+
+#include "include/utime.h"
+#include "common/options.h"
+#include "common/entity_name.h"
+
+class CrushWrapper;
+
+// the precedence is thus:
+//
+//  global
+//   crush location (coarse to fine, ordered by type id)
+//  daemon type (e.g., osd)
+//   device class (osd only)
+//   crush location (coarse to fine, ordered by type id)
+//  daemon name (e.g., mds.foo)
+//
+// Note that this means that if we have
+//
+//  config/host:foo/a = 1
+//  config/osd/rack:foo/a = 2
+//
+// then we get a = 2.  The osd-level config wins, even though rack
+// is less precise than host, because the crush limiters are only
+// resolved within a section (global, per-daemon, per-instance).
+
+struct OptionMask {
+  std::string location_type, location_value; ///< matches crush_location
+  std::string device_class;                  ///< matches device class
+
+  bool empty() const {
+    return location_type.size() == 0
+      && location_value.size() == 0
+      && device_class.size() == 0;
+  }
+
+  std::string to_str() const {
+    std::string r;
+    if (location_type.size()) {
+      r += location_type + ":" + location_value;
+    }
+    if (device_class.size()) {
+      if (r.size()) {
+	r += "/";
+      }
+      r += "class:" + device_class;
+    }
+    return r;
+  }
+  void dump(ceph::Formatter *f) const;
+};
+
+struct MaskedOption {
+  std::string raw_value;               ///< raw, unparsed, unvalidated value
+  const Option *opt;              ///< the option
+  OptionMask mask;
+  std::unique_ptr<const Option> unknown_opt; ///< if fabricated for an unknown option
+
+  MaskedOption(const Option *o, bool fab=false) : opt(o) {
+    if (fab) {
+      unknown_opt.reset(o);
+    }
+  }
+  MaskedOption(MaskedOption&& o) {
+    raw_value = std::move(o.raw_value);
+    opt = o.opt;
+    mask = std::move(o.mask);
+    unknown_opt = std::move(o.unknown_opt);
+  }
+  const MaskedOption& operator=(const MaskedOption& o) = delete;
+  const MaskedOption& operator=(MaskedOption&& o) = delete;
+
+  /// return a precision metric (smaller is more precise)
+  int get_precision(const CrushWrapper *crush);
+
+  friend std::ostream& operator<<(std::ostream& out, const MaskedOption& o);
+
+  void dump(ceph::Formatter *f) const;
+};
+
+struct Section {
+  std::multimap<std::string,MaskedOption> options;
+
+  void clear() {
+    options.clear();
+  }
+  void dump(ceph::Formatter *f) const;
+  std::string get_minimal_conf() const;
+};
+
+struct ConfigMap {
+  Section global;
+  std::map<std::string,Section, std::less<>> by_type;
+  std::map<std::string,Section, std::less<>> by_id;
+  std::list<std::unique_ptr<Option>> stray_options;
+
+  Section *find_section(const std::string& name) {
+    if (name == "global") {
+      return &global;
+    }
+    auto i = by_type.find(name);
+    if (i != by_type.end()) {
+      return &i->second;
+    }
+    i = by_id.find(name);
+    if (i != by_id.end()) {
+      return &i->second;
+    }
+    return nullptr;
+  }
+  void clear() {
+    global.clear();
+    by_type.clear();
+    by_id.clear();
+    stray_options.clear();
+  }
+  void dump(ceph::Formatter *f) const;
+  std::map<std::string,std::string,std::less<>> generate_entity_map(
+    const EntityName& name,
+    const std::map<std::string,std::string>& crush_location,
+    const CrushWrapper *crush,
+    const std::string& device_class,
+    std::map<std::string,std::pair<std::string,const MaskedOption*>> *src=0);
+
+  void parse_key(
+    const std::string& key,
+    std::string *name,
+    std::string *who);
+  static bool parse_mask(
+    const std::string& in,
+    std::string *section,
+    OptionMask *mask);
+};
+
+
+struct ConfigChangeSet {
+  version_t version;
+  utime_t stamp;
+  std::string name;
+
+  // key -> (old value, new value)
+  std::map<std::string,std::pair<boost::optional<std::string>,boost::optional<std::string>>> diff;
+
+  void dump(ceph::Formatter *f) const;
+  void print(std::ostream& out) const;
+};
diff --git a/src/mon/ConfigMonitor.cc b/src/mon/ConfigMonitor.cc
new file mode 100644
index 000000000..c82a8417a
--- /dev/null
+++ b/src/mon/ConfigMonitor.cc
@@ -0,0 +1,1028 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <boost/algorithm/string/predicate.hpp>
+
+#include "mon/Monitor.h"
+#include "mon/ConfigMonitor.h"
+#include "mon/KVMonitor.h"
+#include "mon/MgrMonitor.h"
+#include "mon/OSDMonitor.h"
+#include "messages/MConfig.h"
+#include "messages/MGetConfig.h"
+#include "messages/MMonCommand.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include "common/cmdparse.h"
+#include "include/stringify.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, this)
+using namespace TOPNSPC::common;
+
+using namespace std::literals;
+
+using std::cerr;
+using std::cout;
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::setfill;
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+using std::unique_ptr;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::mono_clock;
+using ceph::mono_time;
+using ceph::timespan_str;
+static ostream& _prefix(std::ostream *_dout, const Monitor &mon,
+                        const ConfigMonitor *hmon) {
+  return *_dout << "mon." << mon.name << "@" << mon.rank
+		<< "(" << mon.get_state_name() << ").config ";
+}
+
+const string KEY_PREFIX("config/");
+const string HISTORY_PREFIX("config-history/");
+
+ConfigMonitor::ConfigMonitor(Monitor &m, Paxos &p, const string& service_name)
+  : PaxosService(m, p, service_name) {
+}
+
+void ConfigMonitor::init()
+{
+  dout(10) << __func__ << dendl;
+}
+
+void ConfigMonitor::create_initial()
+{
+  dout(10) << __func__ << dendl;
+  version = 0;
+  pending.clear();
+}
+
+void ConfigMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  if (version == get_last_committed()) {
+    return;
+  }
+  version = get_last_committed();
+  dout(10) << __func__ << " " << version << dendl;
+  load_config();
+  check_all_subs();
+}
+
+void ConfigMonitor::create_pending()
+{
+  dout(10) << " " << version << dendl;
+  pending.clear();
+  pending_description.clear();
+}
+
+void ConfigMonitor::encode_pending(MonitorDBStore::TransactionRef t)
+{
+  dout(10) << " " << (version+1) << dendl;
+  put_last_committed(t, version+1);
+  // NOTE: caller should have done encode_pending_to_kvmon() and
+  // kvmon->propose_pending() to commit the actual config changes.
+}
+
+void ConfigMonitor::encode_pending_to_kvmon()
+{
+  // we need to pass our data through KVMonitor so that it is properly
+  // versioned and shared with subscribers.
+  for (auto& [key, value] : pending_cleanup) {
+    if (pending.count(key) == 0) {
+      derr << __func__ << " repair: adjusting config key '" << key << "'"
+	   << dendl;
+      pending[key] = value;
+    }
+  }
+  pending_cleanup.clear();
+
+  // TODO: record changed sections (osd, mds.foo, rack:bar, ...)
+
+  string history = HISTORY_PREFIX + stringify(version+1) + "/";
+  {
+    bufferlist metabl;
+    ::encode(ceph_clock_now(), metabl);
+    ::encode(pending_description, metabl);
+    mon.kvmon()->enqueue_set(history, metabl);
+  }
+  for (auto& p : pending) {
+    string key = KEY_PREFIX + p.first;
+    auto q = current.find(p.first);
+    if (q != current.end()) {
+      if (p.second && *p.second == q->second) {
+	continue;
+      }
+      mon.kvmon()->enqueue_set(history + "-" + p.first, q->second);
+    } else if (!p.second) {
+      continue;
+    }
+    if (p.second) {
+      dout(20) << __func__ << " set " << key << dendl;
+      mon.kvmon()->enqueue_set(key, *p.second);
+      mon.kvmon()->enqueue_set(history + "+" + p.first, *p.second);
+   } else {
+      dout(20) << __func__ << " rm " << key << dendl;
+      mon.kvmon()->enqueue_rm(key);
+    }
+  }
+}
+
+version_t ConfigMonitor::get_trim_to() const
+{
+  // we don't actually need *any* old states, but keep a few.
+  if (version > 5) {
+    return version - 5;
+  }
+  return 0;
+}
+
+bool ConfigMonitor::preprocess_query(MonOpRequestRef op)
+{
+  switch (op->get_req()->get_type()) {
+  case MSG_MON_COMMAND:
+    try {
+      return preprocess_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+  }
+  return false;
+}
+
+static string indent_who(const string& who)
+{
+  if (who == "global") {
+    return who;
+  }
+  if (who.find('.') == string::npos) {
+    return "  " + who;
+  }
+  return "    " + who;
+}
+
+bool ConfigMonitor::preprocess_command(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonCommand>();
+  std::stringstream ss;
+  int err = 0;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+  string format;
+  cmd_getval(cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+
+  bufferlist odata;
+  if (prefix == "config help") {
+    stringstream ss;
+    string name;
+    cmd_getval(cmdmap, "key", name);
+    name = ConfFile::normalize_key_name(name);
+    const Option *opt = g_conf().find_option(name);
+    if (!opt) {
+      opt = mon.mgrmon()->find_module_option(name);
+    }
+    if (opt) {
+      if (f) {
+	f->dump_object("option", *opt);
+      } else {
+	opt->print(&ss);
+      }
+    } else {
+      ss << "configuration option '" << name << "' not recognized";
+      err = -ENOENT;
+      goto reply;
+    }
+    if (f) {
+      f->flush(odata);
+    } else {
+      odata.append(ss.str());
+    }
+  } else if (prefix == "config ls") {
+    ostringstream ss;
+    if (f) {
+      f->open_array_section("options");
+    }
+    for (auto& i : ceph_options) {
+      if (f) {
+	f->dump_string("option", i.name);
+      } else {
+	ss << i.name << "\n";
+      }
+    }
+    for (auto& i : mon.mgrmon()->get_mgr_module_options()) {
+      if (f) {
+	f->dump_string("option", i.first);
+      } else {
+	ss << i.first << "\n";
+      }
+    }
+    if (f) {
+      f->close_section();
+      f->flush(odata);
+    } else {
+      odata.append(ss.str());
+    }
+  } else if (prefix == "config dump") {
+    list<pair<string,Section*>> sections = {
+      make_pair("global", &config_map.global)
+    };
+    for (string type : { "mon", "mgr", "osd", "mds", "client" }) {
+      auto i = config_map.by_type.find(type);
+      if (i != config_map.by_type.end()) {
+	sections.push_back(make_pair(i->first, &i->second));
+      }
+      auto j = config_map.by_id.lower_bound(type);
+      while (j != config_map.by_id.end() &&
+	     j->first.find(type) == 0) {
+	sections.push_back(make_pair(j->first, &j->second));
+	++j;
+      }
+    }
+    TextTable tbl;
+    if (!f) {
+      tbl.define_column("WHO", TextTable::LEFT, TextTable::LEFT);
+      tbl.define_column("MASK", TextTable::LEFT, TextTable::LEFT);
+      tbl.define_column("LEVEL", TextTable::LEFT, TextTable::LEFT);
+      tbl.define_column("OPTION", TextTable::LEFT, TextTable::LEFT);
+      tbl.define_column("VALUE", TextTable::LEFT, TextTable::LEFT);
+      tbl.define_column("RO", TextTable::LEFT, TextTable::LEFT);
+    } else {
+      f->open_array_section("config");
+    }
+    for (auto s : sections) {
+      for (auto& i : s.second->options) {
+	if (!f) {
+	  tbl << indent_who(s.first);
+	  tbl << i.second.mask.to_str();
+	  tbl << Option::level_to_str(i.second.opt->level);
+          tbl << i.first;
+	  tbl << i.second.raw_value;
+	  tbl << (i.second.opt->can_update_at_runtime() ? "" : "*");
+	  tbl << TextTable::endrow;
+	} else {
+	  f->open_object_section("option");
+	  f->dump_string("section", s.first);
+	  i.second.dump(f.get());
+	  f->close_section();
+	}
+      }
+    }
+    if (!f) {
+      odata.append(stringify(tbl));
+    } else {
+      f->close_section();
+      f->flush(odata);
+    }
+  } else if (prefix == "config get") {
+    string who, name;
+    cmd_getval(cmdmap, "who", who);
+
+    EntityName entity;
+    if (!entity.from_str(who) &&
+	!entity.from_str(who + ".")) {
+      ss << "unrecognized entity '" << who << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    map<string,string> crush_location;
+    string device_class;
+    if (entity.is_osd()) {
+      mon.osdmon()->osdmap.crush->get_full_location(who, &crush_location);
+      int id = atoi(entity.get_id().c_str());
+      const char *c = mon.osdmon()->osdmap.crush->get_item_class(id);
+      if (c) {
+	device_class = c;
+      }
+      dout(10) << __func__ << " crush_location " << crush_location
+	       << " class " << device_class << dendl;
+    }
+
+    std::map<std::string,pair<std::string,const MaskedOption*>> src;
+    auto config = config_map.generate_entity_map(
+      entity,
+      crush_location,
+      mon.osdmon()->osdmap.crush.get(),
+      device_class,
+      &src);
+
+    if (cmd_getval(cmdmap, "key", name)) {
+      name = ConfFile::normalize_key_name(name);
+      const Option *opt = g_conf().find_option(name);
+      if (!opt) {
+	opt = mon.mgrmon()->find_module_option(name);
+      }
+      if (!opt) {
+        ss << "unrecognized key '" << name << "'";
+	err = -ENOENT;
+	goto reply;
+      }
+      if (opt->has_flag(Option::FLAG_NO_MON_UPDATE)) {
+	// handle special options
+	if (name == "fsid") {
+	  odata.append(stringify(mon.monmap->get_fsid()));
+	  odata.append("\n");
+	  goto reply;
+	}
+	err = -EINVAL;
+	ss << name << " is special and cannot be stored by the mon";
+	goto reply;
+      }
+      // get a single value
+      auto p = config.find(name);
+      if (p != config.end()) {
+	odata.append(p->second);
+	odata.append("\n");
+	goto reply;
+      }
+      if (!entity.is_client() &&
+	  !boost::get<boost::blank>(&opt->daemon_value)) {
+	odata.append(Option::to_str(opt->daemon_value));
+      } else {
+	odata.append(Option::to_str(opt->value));
+      }
+      odata.append("\n");
+    } else {
+      // dump all (non-default) values for this entity
+      TextTable tbl;
+      if (!f) {
+	tbl.define_column("WHO", TextTable::LEFT, TextTable::LEFT);
+	tbl.define_column("MASK", TextTable::LEFT, TextTable::LEFT);
+	tbl.define_column("LEVEL", TextTable::LEFT, TextTable::LEFT);
+	tbl.define_column("OPTION", TextTable::LEFT, TextTable::LEFT);
+	tbl.define_column("VALUE", TextTable::LEFT, TextTable::LEFT);
+	tbl.define_column("RO", TextTable::LEFT, TextTable::LEFT);
+      } else {
+	f->open_object_section("config");
+      }
+      auto p = config.begin();
+      auto q = src.begin();
+      for (; p != config.end(); ++p, ++q) {
+	if (name.size() && p->first != name) {
+	  continue;
+	}
+	if (!f) {
+	  tbl << q->second.first;
+	  tbl << q->second.second->mask.to_str();
+	  tbl << Option::level_to_str(q->second.second->opt->level);
+	  tbl << p->first;
+	  tbl << p->second;
+	  tbl << (q->second.second->opt->can_update_at_runtime() ? "" : "*");
+	  tbl << TextTable::endrow;
+	} else {
+	  f->open_object_section(p->first.c_str());
+	  f->dump_string("value", p->second);
+	  f->dump_string("section", q->second.first);
+	  f->dump_object("mask", q->second.second->mask);
+	  f->dump_bool("can_update_at_runtime",
+		       q->second.second->opt->can_update_at_runtime());
+	  f->close_section();
+	}
+      }
+      if (!f) {
+	odata.append(stringify(tbl));
+      } else {
+	f->close_section();
+	f->flush(odata);
+      }
+    }
+  } else if (prefix == "config log") {
+    int64_t num = 10;
+    cmd_getval(cmdmap, "num", num);
+    ostringstream ds;
+    if (f) {
+      f->open_array_section("changesets");
+    }
+    for (version_t v = version; v > version - std::min(version, (version_t)num); --v) {
+      ConfigChangeSet ch;
+      load_changeset(v, &ch);
+      if (f) {
+	f->dump_object("changeset", ch);
+      } else {
+	ch.print(ds);
+      }
+    }
+    if (f) {
+      f->close_section();
+      f->flush(odata);
+    } else {
+      odata.append(ds.str());
+    }
+  } else if (prefix == "config generate-minimal-conf") {
+    ostringstream conf;
+    conf << "# minimal ceph.conf for " << mon.monmap->get_fsid() << "\n";
+
+    // the basics
+    conf << "[global]\n";
+    conf << "\tfsid = " << mon.monmap->get_fsid() << "\n";
+    conf << "\tmon_host = ";
+    for (auto i = mon.monmap->mon_info.begin();
+	 i != mon.monmap->mon_info.end();
+	 ++i) {
+      if (i != mon.monmap->mon_info.begin()) {
+	conf << " ";
+      }
+      if (i->second.public_addrs.size() == 1 &&
+	  i->second.public_addrs.front().is_legacy() &&
+	  i->second.public_addrs.front().get_port() == CEPH_MON_PORT_LEGACY) {
+	// if this is a legacy addr on the legacy default port, then
+	// use the legacy-compatible formatting so that old clients
+	// can use this config.  new code will see the :6789 and correctly
+	// interpret this as a v1 address.
+	conf << i->second.public_addrs.get_legacy_str();
+      } else {
+	conf << i->second.public_addrs;
+      }
+    }
+    conf << "\n";
+    conf << config_map.global.get_minimal_conf();
+    for (auto m : { &config_map.by_type, &config_map.by_id }) {
+      for (auto& i : *m) {
+	auto s = i.second.get_minimal_conf();
+	if (s.size()) {
+	  conf << "\n[" << i.first << "]\n" << s;
+	}
+      }
+    }
+    odata.append(conf.str());
+    err = 0;
+  } else {
+    return false;
+  }
+
+  reply:
+  mon.reply_command(op, err, ss.str(), odata, get_last_committed());
+  return true;
+}
+
+void ConfigMonitor::handle_get_config(MonOpRequestRef op)
+{
+  auto m = op->get_req<MGetConfig>();
+  dout(10) << __func__ << " " << m->name << " host " << m->host << dendl;
+
+  const OSDMap& osdmap = mon.osdmon()->osdmap;
+  map<string,string> crush_location;
+  osdmap.crush->get_full_location(m->host, &crush_location);
+  auto out = config_map.generate_entity_map(
+    m->name,
+    crush_location,
+    osdmap.crush.get(),
+    m->device_class);
+  dout(20) << " config is " << out << dendl;
+  m->get_connection()->send_message(new MConfig{std::move(out)});
+}
+
+bool ConfigMonitor::prepare_update(MonOpRequestRef op)
+{
+  Message *m = op->get_req();
+  dout(7) << "prepare_update " << *m
+	  << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    try {
+      return prepare_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+  }
+  return false;
+}
+
+bool ConfigMonitor::prepare_command(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonCommand>();
+  std::stringstream ss;
+  int err = -EINVAL;
+
+  // make sure kv is writeable.
+  if (!mon.kvmon()->is_writeable()) {
+    dout(10) << __func__ << " waiting for kv mon to be writeable" << dendl;
+    mon.kvmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+    return false;
+  }
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+  bufferlist odata;
+
+  if (prefix == "config set" ||
+      prefix == "config rm") {
+    string who;
+    string name, value;
+    bool force = false;
+    cmd_getval(cmdmap, "who", who);
+    cmd_getval(cmdmap, "name", name);
+    cmd_getval(cmdmap, "value", value);
+    cmd_getval(cmdmap, "force", force);
+    name = ConfFile::normalize_key_name(name);
+    
+    if (prefix == "config set" && !force) {
+      const Option *opt = g_conf().find_option(name);
+      if (!opt) {
+	opt = mon.mgrmon()->find_module_option(name);
+      }
+      if (!opt) {
+	ss << "unrecognized config option '" << name << "'";
+	err = -EINVAL;
+	goto reply;
+      }
+
+      Option::value_t real_value;
+      string errstr;
+      err = opt->parse_value(value, &real_value, &errstr, &value);
+      if (err < 0) {
+	ss << "error parsing value: " << errstr;
+	goto reply;
+      }
+
+      if (opt->has_flag(Option::FLAG_NO_MON_UPDATE)) {
+	err = -EINVAL;
+	ss << name << " is special and cannot be stored by the mon";
+	goto reply;
+      }
+    }
+
+    string section;
+    OptionMask mask;
+    if (!ConfigMap::parse_mask(who, &section, &mask)) {
+      ss << "unrecognized config target '" << who << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    string key;
+    if (section.size()) {
+      key += section + "/";
+    } else {
+      key += "global/";
+    }
+    string mask_str = mask.to_str();
+    if (mask_str.size()) {
+      key += mask_str + "/";
+    }
+    key += name;
+
+    if (prefix == "config set") {
+      bufferlist bl;
+      bl.append(value);
+      pending[key] = bl;
+    } else {
+      pending[key] = boost::none;
+    }
+    goto update;
+  } else if (prefix == "config reset") {
+    int64_t revert_to = -1;
+    cmd_getval(cmdmap, "num", revert_to);
+    if (revert_to < 0 ||
+        revert_to > (int64_t)version) {
+      err = -EINVAL;
+      ss << "must specify a valid historical version to revert to; "
+         << "see 'ceph config log' for a list of avialable configuration "
+         << "historical versions";
+      goto reply;
+    }
+    if (revert_to == (int64_t)version) {
+      err = 0;
+      goto reply;
+    }
+    for (int64_t v = version; v > revert_to; --v) {
+      ConfigChangeSet ch;
+      load_changeset(v, &ch);
+      for (auto& i : ch.diff) {
+	if (i.second.first) {
+	  bufferlist bl;
+	  bl.append(*i.second.first);
+	  pending[i.first] = bl;
+	} else if (i.second.second) {
+	  pending[i.first] = boost::none;
+	}
+      }
+    }
+    pending_description = string("reset to ") + stringify(revert_to);
+    goto update;
+  } else if (prefix == "config assimilate-conf") {
+    ConfFile cf;
+    bufferlist bl = m->get_data();
+    err = cf.parse_bufferlist(&bl, &ss);
+    if (err < 0) {
+      goto reply;
+    }
+    bool updated = false;
+    ostringstream newconf;
+    for (auto& [section, s] : cf) {
+      dout(20) << __func__ << " [" << section << "]" << dendl;
+      bool did_section = false;
+      for (auto& [key, val] : s) {
+	Option::value_t real_value;
+	string value;
+	string errstr;
+	if (key.empty()) {
+	  continue;
+	}
+	// a known and worthy option?
+	const Option *o = g_conf().find_option(key);
+	if (!o) {
+	  o = mon.mgrmon()->find_module_option(key);
+	}
+	if (!o ||
+	    (o->flags & Option::FLAG_NO_MON_UPDATE) ||
+	    (o->flags & Option::FLAG_CLUSTER_CREATE)) {
+	  goto skip;
+	}
+	// normalize
+	err = o->parse_value(val, &real_value, &errstr, &value);
+	if (err < 0) {
+	  dout(20) << __func__ << " failed to parse " << key << " = '"
+		   << val << "'" << dendl;
+	  goto skip;
+	}
+	// does it conflict with an existing value?
+	{
+	  const Section *s = config_map.find_section(section);
+	  if (s) {
+	    auto k = s->options.find(key);
+	    if (k != s->options.end()) {
+	      if (value != k->second.raw_value) {
+		dout(20) << __func__ << " have " << key
+			 << " = " << k->second.raw_value
+			 << " (not " << value << ")" << dendl;
+		goto skip;
+	      }
+	      dout(20) << __func__ << " already have " << key
+		       << " = " << k->second.raw_value << dendl;
+	      continue;
+	    }
+	  }
+	}
+	dout(20) << __func__ << "  add " << key << " = " << value
+		 << " (" << val << ")" << dendl;
+	{
+	  bufferlist bl;
+	  bl.append(value);
+	  pending[section + "/" + key] = bl;
+	  updated = true;
+	}
+	continue;
+
+       skip:
+	dout(20) << __func__ << " skip " << key << " = " << value
+		 << " (" << val << ")" << dendl;
+	if (!did_section) {
+	  newconf << "\n[" << section << "]\n";
+	  did_section = true;
+	}
+	newconf << "\t" << key << " = " << val << "\n";
+      }
+    }
+    odata.append(newconf.str());
+    if (updated) {
+      goto update;
+    }
+  } else {
+    ss << "unknown command " << prefix;
+    err = -EINVAL;
+  }
+
+reply:
+  mon.reply_command(op, err, ss.str(), odata, get_last_committed());
+  return false;
+
+update:
+  // see if there is an actual change
+  auto p = pending.begin();
+  while (p != pending.end()) {
+    auto q = current.find(p->first);
+    if (p->second && q != current.end() && *p->second == q->second) {
+      // set to same value
+      p = pending.erase(p);
+    } else if (!p->second && q == current.end()) {
+      // erasing non-existent value
+      p = pending.erase(p);
+    } else {
+      ++p;
+    }
+  }
+  if (pending.empty()) {
+    err = 0;
+    goto reply;
+  }
+  // immediately propose *with* KV mon
+  encode_pending_to_kvmon();
+  paxos.plug();
+  mon.kvmon()->propose_pending();
+  paxos.unplug();
+  force_immediate_propose();
+  wait_for_finished_proposal(
+    op,
+    new Monitor::C_Command(
+      mon, op, 0, ss.str(), odata,
+      get_last_committed() + 1));
+  return true;
+}
+
+void ConfigMonitor::tick()
+{
+  if (!is_active() || !mon.is_leader()) {
+    return;
+  }
+  dout(10) << __func__ << dendl;
+  bool changed = false;
+  if (!pending_cleanup.empty()) {
+    changed = true;
+  }
+  if (changed && mon.kvmon()->is_writeable()) {
+    paxos.plug();
+    encode_pending_to_kvmon();
+    mon.kvmon()->propose_pending();
+    paxos.unplug();
+    propose_pending();
+  }
+}
+
+void ConfigMonitor::on_active()
+{
+}
+
+void ConfigMonitor::load_config()
+{
+  std::map<std::string,std::string> renamed_pacific = {
+    { "mon_osd_blacklist_default_expire", "mon_osd_blocklist_default_expire" },
+    { "mon_mds_blacklist_interval", "mon_mds_blocklist_interval" },
+    { "mon_mgr_blacklist_interval", "mon_mgr_blocklist_interval" },
+    { "rbd_blacklist_on_break_lock", "rbd_blocklist_on_break_lock" },
+    { "rbd_blacklist_expire_seconds", "rbd_blocklist_expire_seconds" },
+    { "mds_session_blacklist_on_timeout", "mds_session_blocklist_on_timeout" },
+    { "mds_session_blacklist_on_evict", "mds_session_blocklist_on_evict" },
+  };
+
+  unsigned num = 0;
+  KeyValueDB::Iterator it = mon.store->get_iterator(KV_PREFIX);
+  it->lower_bound(KEY_PREFIX);
+  config_map.clear();
+  current.clear();
+  pending_cleanup.clear();
+  while (it->valid() &&
+	 it->key().compare(0, KEY_PREFIX.size(), KEY_PREFIX) == 0) {
+    string key = it->key().substr(KEY_PREFIX.size());
+    string value = it->value().to_str();
+
+    current[key] = it->value();
+
+    string name;
+    string who;
+    config_map.parse_key(key, &name, &who);
+
+    // has this option been renamed?
+    {
+      auto p = renamed_pacific.find(name);
+      if (p != renamed_pacific.end()) {
+	if (mon.monmap->min_mon_release >= ceph_release_t::pacific) {
+	  // schedule a cleanup
+	  pending_cleanup[key] = boost::none;
+	  pending_cleanup[who + "/" + p->second] = it->value();
+	}
+	// continue loading under the new name
+	name = p->second;
+      }
+    }
+
+    const Option *opt = g_conf().find_option(name);
+    if (!opt) {
+      opt = mon.mgrmon()->find_module_option(name);
+    }
+    if (!opt) {
+      dout(10) << __func__ << " unrecognized option '" << name << "'" << dendl;
+      config_map.stray_options.push_back(
+	std::unique_ptr<Option>(
+	  new Option(name, Option::TYPE_STR, Option::LEVEL_UNKNOWN)));
+      opt = config_map.stray_options.back().get();
+    }
+
+    string err;
+    int r = opt->pre_validate(&value, &err);
+    if (r < 0) {
+      dout(10) << __func__ << " pre-validate failed on '" << name << "' = '"
+	       << value << "' for " << name << dendl;
+    }
+    
+    MaskedOption mopt(opt);
+    mopt.raw_value = value;
+    string section_name;
+    if (who.size() &&
+	!ConfigMap::parse_mask(who, &section_name, &mopt.mask)) {
+      derr << __func__ << " invalid mask for key " << key << dendl;
+      pending_cleanup[key] = boost::none;
+    } else if (opt->has_flag(Option::FLAG_NO_MON_UPDATE)) {
+      dout(10) << __func__ << " NO_MON_UPDATE option '"
+	       << name << "' = '" << value << "' for " << name
+	       << dendl;
+      pending_cleanup[key] = boost::none;
+    } else {
+      if (section_name.empty()) {
+	// we prefer global/$option instead of just $option
+	derr << __func__ << " adding global/ prefix to key '" << key << "'"
+	     << dendl;
+	pending_cleanup[key] = boost::none;
+	pending_cleanup["global/"s + key] = it->value();
+      }
+      Section *section = &config_map.global;;
+      if (section_name.size() && section_name != "global") {
+	if (section_name.find('.') != std::string::npos) {
+	  section = &config_map.by_id[section_name];
+	} else {
+	  section = &config_map.by_type[section_name];
+	}
+      }
+      section->options.insert(make_pair(name, std::move(mopt)));
+      ++num;
+    }
+    it->next();
+  }
+  dout(10) << __func__ << " got " << num << " keys" << dendl;
+
+  // refresh our own config
+  {
+    const OSDMap& osdmap = mon.osdmon()->osdmap;
+    map<string,string> crush_location;
+    osdmap.crush->get_full_location(g_conf()->host, &crush_location);
+    auto out = config_map.generate_entity_map(
+      g_conf()->name,
+      crush_location,
+      osdmap.crush.get(),
+      string{}); // no device class
+    g_conf().set_mon_vals(g_ceph_context, out, nullptr);
+  }
+}
+
+void ConfigMonitor::load_changeset(version_t v, ConfigChangeSet *ch)
+{
+  ch->version = v;
+  string prefix = HISTORY_PREFIX + stringify(v) + "/";
+  KeyValueDB::Iterator it = mon.store->get_iterator(KV_PREFIX);
+  it->lower_bound(prefix);
+  while (it->valid() && it->key().find(prefix) == 0) {
+    if (it->key() == prefix) {
+      bufferlist bl = it->value();
+      auto p = bl.cbegin();
+      try {
+	decode(ch->stamp, p);
+	decode(ch->name, p);
+      }
+      catch (ceph::buffer::error& e) {
+	derr << __func__ << " failure decoding changeset " << v << dendl;
+      }
+    } else {
+      char op = it->key()[prefix.length()];
+      string key = it->key().substr(prefix.length() + 1);
+      if (op == '-') {
+	ch->diff[key].first = it->value().to_str();
+      } else if (op == '+') {
+	ch->diff[key].second = it->value().to_str();
+      }
+    }
+    it->next();
+  }
+}
+
+bool ConfigMonitor::refresh_config(MonSession *s)
+{
+  const OSDMap& osdmap = mon.osdmon()->osdmap;
+  map<string,string> crush_location;
+  if (s->remote_host.size()) {
+    osdmap.crush->get_full_location(s->remote_host, &crush_location);
+    dout(10) << __func__ << " crush_location for remote_host " << s->remote_host
+	     << " is " << crush_location << dendl;
+  }
+
+  string device_class;
+  if (s->name.is_osd()) {
+    const char *c = osdmap.crush->get_item_class(s->name.num());
+    if (c) {
+      device_class = c;
+      dout(10) << __func__ << " device_class " << device_class << dendl;
+    }
+  }
+
+  dout(20) << __func__ << " " << s->entity_name << " crush " << crush_location
+	   << " device_class " << device_class << dendl;
+  auto out = config_map.generate_entity_map(
+    s->entity_name,
+    crush_location,
+    osdmap.crush.get(),
+    device_class);
+
+  if (out == s->last_config && s->any_config) {
+    dout(20) << __func__ << " no change, " << out << dendl;
+    return false;
+  }
+  // removing this to hide sensitive data going into logs
+  // leaving this for debugging purposes
+ //  dout(20) << __func__ << " " << out << dendl;
+  s->last_config = std::move(out);
+  s->any_config = true;
+  return true;
+}
+
+bool ConfigMonitor::maybe_send_config(MonSession *s)
+{
+  bool changed = refresh_config(s);
+  dout(10) << __func__ << " to " << s->name << " "
+	   << (changed ? "(changed)" : "(unchanged)")
+	   << dendl;
+  if (changed) {
+    send_config(s);
+  }
+  return changed;
+}
+
+void ConfigMonitor::send_config(MonSession *s)
+{
+  dout(10) << __func__ << " to " << s->name << dendl;
+  auto m = new MConfig(s->last_config);
+  s->con->send_message(m);
+}
+
+void ConfigMonitor::check_sub(MonSession *s)
+{
+  if (!s->authenticated) {
+    dout(20) << __func__ << " not authenticated " << s->entity_name << dendl;
+    return;
+  }
+  auto p = s->sub_map.find("config");
+  if (p != s->sub_map.end()) {
+    check_sub(p->second);
+  }
+}
+
+void ConfigMonitor::check_sub(Subscription *sub)
+{
+  dout(10) << __func__
+	   << " next " << sub->next
+	   << " have " << version << dendl;
+  if (sub->next <= version) {
+    maybe_send_config(sub->session);
+    if (sub->onetime) {
+      mon.with_session_map([sub](MonSessionMap& session_map) {
+	  session_map.remove_sub(sub);
+	});
+    } else {
+      sub->next = version + 1;
+    }
+  }
+}
+
+void ConfigMonitor::check_all_subs()
+{
+  dout(10) << __func__ << dendl;
+  auto subs = mon.session_map.subs.find("config");
+  if (subs == mon.session_map.subs.end()) {
+    return;
+  }
+  int updated = 0, total = 0;
+  auto p = subs->second->begin();
+  while (!p.end()) {
+    auto sub = *p;
+    ++p;
+    ++total;
+    if (maybe_send_config(sub->session)) {
+      ++updated;
+    }
+  }
+  dout(10) << __func__ << " updated " << updated << " / " << total << dendl;
+}
diff --git a/src/mon/ConfigMonitor.h b/src/mon/ConfigMonitor.h
new file mode 100644
index 000000000..e6c12a3d7
--- /dev/null
+++ b/src/mon/ConfigMonitor.h
@@ -0,0 +1,58 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <boost/optional.hpp>
+
+#include "ConfigMap.h"
+#include "mon/PaxosService.h"
+
+class MonSession;
+
+class ConfigMonitor : public PaxosService
+{
+  version_t version = 0;
+  ConfigMap config_map;
+  std::map<std::string,boost::optional<ceph::buffer::list>> pending;
+  std::string pending_description;
+  std::map<std::string,boost::optional<ceph::buffer::list>> pending_cleanup;
+
+  std::map<std::string,ceph::buffer::list> current;
+
+  void encode_pending_to_kvmon();
+
+public:
+  ConfigMonitor(Monitor &m, Paxos &p, const std::string& service_name);
+
+  void init() override;
+
+  void load_config();
+  void load_changeset(version_t v, ConfigChangeSet *ch);
+
+  bool preprocess_query(MonOpRequestRef op) override;
+  bool prepare_update(MonOpRequestRef op) override;
+
+  bool preprocess_command(MonOpRequestRef op);
+  bool prepare_command(MonOpRequestRef op);
+
+  void handle_get_config(MonOpRequestRef op);
+
+  void create_initial() override;
+  void update_from_paxos(bool *need_bootstrap) override;
+  void create_pending() override;
+  void encode_pending(MonitorDBStore::TransactionRef t) override;
+  version_t get_trim_to() const override;
+
+  void encode_full(MonitorDBStore::TransactionRef t) override { }
+
+  void on_active() override;
+  void tick() override;
+
+  bool refresh_config(MonSession *s);
+  bool maybe_send_config(MonSession *s);
+  void send_config(MonSession *s);
+  void check_sub(MonSession *s);
+  void check_sub(Subscription *sub);
+  void check_all_subs();
+};
diff --git a/src/mon/ConnectionTracker.cc b/src/mon/ConnectionTracker.cc
new file mode 100644
index 000000000..272ad40c2
--- /dev/null
+++ b/src/mon/ConnectionTracker.cc
@@ -0,0 +1,361 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2019 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "ConnectionTracker.h"
+#include "common/Formatter.h"
+#include "common/dout.h"
+#include "include/ceph_assert.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, rank, epoch, version)
+
+static std::ostream& _prefix(std::ostream *_dout, int rank, epoch_t epoch, uint64_t version) {
+  return *_dout << "rank: " << rank << " version: "<< version << " ConnectionTracker(" << epoch << ") ";
+}
+
+std::ostream& operator<<(std::ostream&o, const ConnectionReport& c) {
+  o << "rank=" << c.rank << ",epoch=" << c.epoch << ",version=" << c.epoch_version
+    << ", current links: " << c.current << ", history: " << c.history;
+  return o;
+}
+
+std::ostream& operator<<(std::ostream& o, const ConnectionTracker& c) {
+  o << "rank=" << c.rank << ", epoch=" << c.epoch << ", version=" << c.version
+    << ", half_life=" << c.half_life << ", reports: " << c.peer_reports;
+  return o;
+}
+
+ConnectionReport *ConnectionTracker::reports(int p)
+{
+  auto i = peer_reports.find(p);
+  if (i == peer_reports.end()) {
+    ceph_assert(p != rank);
+    auto[j,k] = peer_reports.insert(std::pair<int,ConnectionReport>(p,ConnectionReport()));
+    i = j;
+  }
+  return &i->second;
+}
+
+const ConnectionReport *ConnectionTracker::reports(int p) const
+{
+  auto i = peer_reports.find(p);
+  if (i == peer_reports.end()) {
+    return NULL;
+  }
+  return &i->second;
+}
+
+void ConnectionTracker::receive_peer_report(const ConnectionTracker& o)
+{
+  ldout(cct, 30) << __func__ << dendl;
+  for (auto& i : o.peer_reports) {
+    const ConnectionReport& report = i.second;
+    if (i.first == rank) continue;
+    ConnectionReport& existing = *reports(i.first);
+    if (report.epoch > existing.epoch ||
+	(report.epoch == existing.epoch &&
+	 report.epoch_version > existing.epoch_version)) {
+      ldout(cct, 30) << " new peer_report is more updated" << dendl;
+      ldout(cct, 30) << "existing: " << existing << dendl;
+      ldout(cct, 30) << "new: " << report << dendl;
+      existing = report;
+    }
+  }
+  encoding.clear();
+}
+
+bool ConnectionTracker::increase_epoch(epoch_t e)
+{
+  ldout(cct, 30) << __func__ << " to " << e << dendl;
+  if (e > epoch) {
+    my_reports.epoch_version = version = 0;
+    my_reports.epoch = epoch = e;
+    peer_reports[rank] = my_reports;
+    encoding.clear();
+    return true;
+  }
+  return false;
+}
+
+void ConnectionTracker::increase_version()
+{
+  ldout(cct, 30) << __func__ << " to " << version+1 << dendl;
+  encoding.clear();
+  ++version;
+  my_reports.epoch_version = version;
+  peer_reports[rank] = my_reports;
+  if ((version % persist_interval) == 0 ) {
+    ldout(cct, 30) << version << " % " << persist_interval << " == 0" << dendl;
+    owner->persist_connectivity_scores();
+  }
+}
+
+void ConnectionTracker::report_live_connection(int peer_rank, double units_alive)
+{
+  ldout(cct, 30) << __func__ << " peer_rank: " << peer_rank << " units_alive: " << units_alive << dendl;
+  ldout(cct, 30) << "my_reports before: " << my_reports << dendl;
+  if (peer_rank == rank) {
+    lderr(cct) << "Got a report from my own rank, hopefully this is startup weirdness, dropping" << dendl;
+    return;
+  }
+  // we need to "auto-initialize" to 1, do shenanigans
+  auto i = my_reports.history.find(peer_rank);
+  if (i == my_reports.history.end()) {
+    ldout(cct, 30) << "couldn't find: " << peer_rank
+      << " in my_reports.history" << "... inserting: "
+      << "(" << peer_rank << ", 1" << dendl;
+    auto[j,k] = my_reports.history.insert(std::pair<int,double>(peer_rank,1.0));
+    i = j;
+  }
+  double& pscore = i->second;
+  ldout(cct, 30) << "adding new pscore to my_reports" << dendl;
+  pscore = pscore * (1 - units_alive / (2 * half_life)) +
+    (units_alive / (2 * half_life));
+  pscore = std::min(pscore, 1.0);
+  my_reports.current[peer_rank] = true;
+
+  increase_version();
+  ldout(cct, 30) << "my_reports after: " << my_reports << dendl;
+}
+
+void ConnectionTracker::report_dead_connection(int peer_rank, double units_dead)
+{
+  ldout(cct, 30) << __func__ << " peer_rank: " << peer_rank << " units_dead: " << units_dead << dendl;
+  ldout(cct, 30) << "my_reports before: " << my_reports << dendl;
+  if (peer_rank == rank) {
+    lderr(cct) << "Got a report from my own rank, hopefully this is startup weirdness, dropping" << dendl;
+    return;
+  }
+  // we need to "auto-initialize" to 1, do shenanigans
+  auto i = my_reports.history.find(peer_rank);
+  if (i == my_reports.history.end()) {
+    ldout(cct, 30) << "couldn't find: " << peer_rank
+    << " in my_reports.history" << "... inserting: "
+    << "(" << peer_rank << ", 1" << dendl;
+    auto[j,k] = my_reports.history.insert(std::pair<int,double>(peer_rank,1.0));
+    i = j;
+  }
+  double& pscore = i->second;
+  ldout(cct, 30) << "adding new pscore to my_reports" << dendl;
+  pscore = pscore * (1 - units_dead / (2 * half_life)) -
+    (units_dead / (2*half_life));
+  pscore = std::max(pscore, 0.0);
+  my_reports.current[peer_rank] = false;
+  
+  increase_version();
+  ldout(cct, 30) << "my_reports after: " << my_reports << dendl;
+}
+
+void ConnectionTracker::get_total_connection_score(int peer_rank, double *rating,
+						    int *live_count) const
+{
+  ldout(cct, 30) << __func__ << dendl;
+  *rating = 0;
+  *live_count = 0;
+  double rate = 0;
+  int live = 0;
+
+  for (const auto& i : peer_reports) { // loop through all the scores
+    if (i.first == peer_rank) { // ... except the ones it has for itself, of course!
+      continue;
+    }
+    const auto& report = i.second;
+    auto score_i = report.history.find(peer_rank);
+    auto live_i = report.current.find(peer_rank);
+    if (score_i != report.history.end()) {
+      if (live_i->second) {
+	rate += score_i->second;
+	++live;
+      }
+    }
+  }
+  *rating = rate;
+  *live_count = live;
+}
+
+void ConnectionTracker::notify_rank_changed(int new_rank)
+{
+  ldout(cct, 20) << __func__ << " to " << new_rank << dendl;
+  if (new_rank == rank) return;
+  ldout(cct, 20) << "peer_reports before: " << peer_reports << dendl;
+  peer_reports.erase(rank);
+  peer_reports.erase(new_rank);
+  my_reports.rank = new_rank;
+  rank = new_rank;
+  encoding.clear();
+  ldout(cct, 20) << "peer_reports after: " << peer_reports << dendl;
+
+  increase_version();
+}
+
+void ConnectionTracker::notify_rank_removed(int rank_removed, int new_rank)
+{
+  ldout(cct, 20) << __func__ << " " << rank_removed
+    << " new_rank: " << new_rank << dendl;
+  ldout(cct, 20) << "my_reports before: " << my_reports << dendl;
+  ldout(cct, 20) << "peer_reports before: " << peer_reports << dendl;
+  ldout(cct, 20) << "my rank before: " << rank << dendl;
+
+  encoding.clear();
+  size_t starting_size_current = my_reports.current.size();
+  // Lets adjust everything in my report.
+  my_reports.current.erase(rank_removed);
+  my_reports.history.erase(rank_removed);
+  auto ci = my_reports.current.upper_bound(rank_removed);
+  auto hi = my_reports.history.upper_bound(rank_removed);
+  while (ci != my_reports.current.end()) {
+    ceph_assert(ci->first == hi->first);
+    my_reports.current[ci->first - 1] = ci->second;
+    my_reports.history[hi->first - 1] = hi->second;
+    my_reports.current.erase(ci++);
+    my_reports.history.erase(hi++);
+  }
+  ceph_assert((my_reports.current.size() == starting_size_current) ||
+    (my_reports.current.size() + 1 == starting_size_current));
+
+  size_t starting_size = peer_reports.size();
+  auto pi = peer_reports.upper_bound(rank_removed);
+  // Remove the target rank and adjust everything that comes after.
+  // Note that we don't adjust current and history for our peer_reports
+  // because it is better to rely on our peers on that information.
+  peer_reports.erase(rank_removed);
+  while (pi != peer_reports.end()) {
+    peer_reports[pi->first - 1] = pi->second; // copy content of next rank to ourself.
+    peer_reports.erase(pi++); // destroy our next rank and move on.
+  }
+
+  ceph_assert((peer_reports.size() == starting_size) ||
+	  (peer_reports.size() + 1 == starting_size));
+
+  if (rank_removed < rank) { // if the rank removed is lower than us, we need to adjust.
+    --rank;
+    my_reports.rank = rank; // also adjust my_reports.rank.
+  }
+
+  ldout(cct, 20) << "my rank after: " << rank << dendl;
+  ldout(cct, 20) << "peer_reports after: " << peer_reports << dendl;
+  ldout(cct, 20) << "my_reports after: " << my_reports << dendl;
+
+  //check if the new_rank from monmap is equal to our adjusted rank.
+  ceph_assert(rank == new_rank);
+
+  increase_version();
+}
+
+bool ConnectionTracker::is_clean(int mon_rank, int monmap_size)
+{
+  ldout(cct, 30) << __func__ << dendl;
+  // check consistency between our rank according
+  // to monmap and our rank according to our report.
+  if (rank != mon_rank ||
+    my_reports.rank != mon_rank) {
+    return false;
+  } else if (!peer_reports.empty()){
+    // if peer_report max rank is greater than monmap max rank
+    // then there is a problem.
+    if (peer_reports.rbegin()->first > monmap_size - 1) return false;
+  }
+  return true;
+}
+
+void ConnectionTracker::encode(bufferlist &bl) const
+{
+  ENCODE_START(1, 1, bl);
+  encode(rank, bl);
+  encode(epoch, bl);
+  encode(version, bl);
+  encode(half_life, bl);
+  encode(peer_reports, bl);
+  ENCODE_FINISH(bl);
+}
+
+void ConnectionTracker::decode(bufferlist::const_iterator& bl) {
+  clear_peer_reports();
+  encoding.clear();
+
+  DECODE_START(1, bl);
+  decode(rank, bl);
+  decode(epoch, bl);
+  decode(version, bl);
+  decode(half_life, bl);
+  decode(peer_reports, bl);
+  DECODE_FINISH(bl);
+  if (rank >=0)
+    my_reports = peer_reports[rank];
+}
+
+const bufferlist& ConnectionTracker::get_encoded_bl()
+{
+  if (!encoding.length()) {
+    encode(encoding);
+  }
+  return encoding;
+}
+
+void ConnectionReport::dump(ceph::Formatter *f) const
+{
+  f->dump_int("rank", rank);
+  f->dump_int("epoch", epoch);
+  f->dump_int("version", epoch_version);
+  f->open_object_section("peer_scores");
+  for (auto i : history) {
+    f->open_object_section("peer");
+    f->dump_int("peer_rank", i.first);
+    f->dump_float("peer_score", i.second);
+    f->dump_bool("peer_alive", current.find(i.first)->second);
+    f->close_section();
+  }
+  f->close_section(); // peer scores
+}
+
+void ConnectionReport::generate_test_instances(std::list<ConnectionReport*>& o)
+{
+  o.push_back(new ConnectionReport);
+  o.push_back(new ConnectionReport);
+  o.back()->rank = 1;
+  o.back()->epoch = 2;
+  o.back()->epoch_version = 3;
+  o.back()->current[0] = true;
+  o.back()->history[0] = .4;
+}
+
+void ConnectionTracker::dump(ceph::Formatter *f) const
+{
+  f->dump_int("rank", rank);
+  f->dump_int("epoch", epoch);
+  f->dump_int("version", version);
+  f->dump_float("half_life", half_life);
+  f->dump_int("persist_interval", persist_interval);
+  f->open_object_section("reports");
+  for (const auto& i : peer_reports) {
+    f->open_object_section("report");
+    i.second.dump(f);
+    f->close_section();
+  }
+  f->close_section(); // reports
+}
+
+void ConnectionTracker::generate_test_instances(std::list<ConnectionTracker*>& o)
+{
+  o.push_back(new ConnectionTracker);
+  o.push_back(new ConnectionTracker);
+  ConnectionTracker *e = o.back();
+  e->rank = 2;
+  e->epoch = 3;
+  e->version = 4;
+  e->peer_reports[0];
+  e->peer_reports[1];
+  e->my_reports = e->peer_reports[2];
+}
diff --git a/src/mon/ConnectionTracker.h b/src/mon/ConnectionTracker.h
new file mode 100644
index 000000000..09506636d
--- /dev/null
+++ b/src/mon/ConnectionTracker.h
@@ -0,0 +1,205 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2019 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#pragma once
+#include "include/types.h"
+
+struct ConnectionReport {
+  int rank = -1; // mon rank this state belongs to
+  std::map<int, bool> current; // true if connected to the other mon
+  std::map<int, double> history; // [0-1]; the connection reliability
+  epoch_t epoch = 0; // the (local) election epoch the ConnectionReport came from
+  uint64_t epoch_version = 0; // version of the ConnectionReport within the epoch
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    encode(rank, bl);
+    encode(current, bl);
+    encode(history, bl);
+    encode(epoch, bl);
+    encode(epoch_version, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::const_iterator& bl) {
+    DECODE_START(1, bl);
+    decode(rank, bl);
+    decode(current, bl);
+    decode(history, bl);
+    decode(epoch, bl);
+    decode(epoch_version, bl);
+    DECODE_FINISH(bl);
+  }
+  bool operator==(const ConnectionReport& o) const {
+    return o.rank == rank && o.current == current &&
+      o.history == history && o.epoch == epoch &&
+      o.epoch_version == epoch_version;
+  }
+  friend std::ostream& operator<<(std::ostream&o, const ConnectionReport& c);
+
+  void dump(ceph::Formatter *f) const;
+  static void generate_test_instances(std::list<ConnectionReport*>& o);
+};
+WRITE_CLASS_ENCODER(ConnectionReport);
+
+class RankProvider {
+ public:
+  /**
+   * Get the rank of the running daemon.
+   * It can be -1, meaning unknown/invalid, or it
+   * can be >1.
+   * You should not invoke the function get_total_connection_score()
+   * with an unknown rank.
+   */
+  virtual int get_my_rank() const = 0;
+  /**
+   * Asks our owner to encode us and persist it to disk.
+   * Presently we do this every tenth update.
+   */
+  virtual void persist_connectivity_scores() = 0;
+  virtual ~RankProvider() {}
+};
+
+class ConnectionTracker {
+ public:
+  /**
+   * Receive a report from a peer and update our internal state
+   * if the peer has newer data.
+   */
+  void receive_peer_report(const ConnectionTracker& o);
+  /**
+   * Bump up the epoch to the specified number.
+   * Validates that it is > current epoch and resets
+   * version to 0; returns false if not.
+   */
+  bool increase_epoch(epoch_t e);
+  /**
+   * Bump up the version within our epoch.
+   * If the new version is a multiple of ten, we also persist it.
+   */
+  void increase_version();
+  
+  /**
+   * Report a connection to a peer rank has been considered alive for
+   * the given time duration. We assume the units_alive is <= the time
+   * since the previous reporting call.
+   * (Or, more precisely, we assume that the total amount of time
+   * passed in is less than or equal to the time which has actually
+   * passed -- you can report a 10-second death immediately followed
+   * by reporting 5 seconds of liveness if your metrics are delayed.)
+   */
+  void report_live_connection(int peer_rank, double units_alive);
+  /**
+   * Report a connection to a peer rank has been considered dead for
+   * the given time duration, analogous to that above.
+   */
+  void report_dead_connection(int peer_rank, double units_dead);
+  /**
+   * Set the half-life for dropping connection state
+   * out of the ongoing score.
+   * Whenever you add a new data point:
+   * new_score = old_score * ( 1 - units / (2d)) + (units/(2d))
+   * where units is the units reported alive (for dead, you subtract them).
+   */
+  void set_half_life(double d) {
+    half_life = d;
+  }
+  /**
+   * Get the total connection score of a rank across
+   * all peers, and the count of how many electors think it's alive.
+   * For this summation, if a rank reports a peer as down its score is zero.
+   */
+  void get_total_connection_score(int peer_rank, double *rating,
+				  int *live_count) const;
+  /**
+  * Check if our ranks are clean and make
+  * sure there are no extra peer_report lingering.
+  * In the future we also want to check the reports
+  * current and history of each peer_report.
+  */
+  bool is_clean(int mon_rank, int monmap_size);
+  /**
+   * Encode this ConnectionTracker. Useful both for storing on disk
+   * and for sending off to peers for decoding and import
+   * with receive_peer_report() above.
+   */
+  void encode(bufferlist &bl) const;
+  void decode(bufferlist::const_iterator& bl);
+  /**
+   * Get a bufferlist containing the ConnectionTracker.
+   * This is like encode() but holds a copy so it
+   * doesn't re-encode on every invocation.
+   */
+  const bufferlist& get_encoded_bl();
+ private:
+  epoch_t epoch;
+  uint64_t version;
+  map<int,ConnectionReport> peer_reports;
+  ConnectionReport my_reports;
+  double half_life;
+  RankProvider *owner;
+  int rank;
+  int persist_interval;
+  bufferlist encoding;
+  CephContext *cct;
+  int get_my_rank() const { return rank; }
+  ConnectionReport *reports(int p);
+  const ConnectionReport *reports(int p) const;
+
+  void clear_peer_reports() {
+    encoding.clear();
+    peer_reports.clear();
+    my_reports = ConnectionReport();
+    my_reports.rank = rank;
+  }
+
+ public:
+  ConnectionTracker() : epoch(0), version(0), half_life(12*60*60),
+			owner(NULL), rank(-1), persist_interval(10) {
+  }
+  ConnectionTracker(RankProvider *o, int rank, double hl,
+		    int persist_i, CephContext *c) :
+    epoch(0), version(0),
+    half_life(hl), owner(o), rank(rank), persist_interval(persist_i), cct(c) {
+    my_reports.rank = rank;
+  }
+  ConnectionTracker(const bufferlist& bl, CephContext *c) :
+    epoch(0), version(0),
+    half_life(0), owner(NULL), rank(-1), persist_interval(10), cct(c)
+  {
+    auto bi = bl.cbegin();
+    decode(bi);
+  }
+  ConnectionTracker(const ConnectionTracker& o) :
+    epoch(o.epoch), version(o.version),
+    half_life(o.half_life), owner(o.owner), rank(o.rank),
+    persist_interval(o.persist_interval), cct(o.cct)
+  {
+    peer_reports = o.peer_reports;
+    my_reports = o.my_reports;
+  }
+  void notify_reset() { clear_peer_reports(); }
+  void set_rank(int new_rank) {
+    rank = new_rank;
+    my_reports.rank = rank;
+  }
+
+  void notify_rank_changed(int new_rank);
+  void notify_rank_removed(int rank_removed, int new_rank);
+  friend std::ostream& operator<<(std::ostream& o, const ConnectionTracker& c);
+  friend ConnectionReport *get_connection_reports(ConnectionTracker& ct);
+  friend map<int,ConnectionReport> *get_peer_reports(ConnectionTracker& ct);
+  void dump(ceph::Formatter *f) const;
+  static void generate_test_instances(std::list<ConnectionTracker*>& o);
+};
+
+WRITE_CLASS_ENCODER(ConnectionTracker);
diff --git a/src/mon/CreatingPGs.h b/src/mon/CreatingPGs.h
new file mode 100644
index 000000000..0075f81e7
--- /dev/null
+++ b/src/mon/CreatingPGs.h
@@ -0,0 +1,234 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <map>
+#include <set>
+#include <vector>
+
+#include "include/encoding.h"
+#include "include/utime.h"
+
+#include "osd/osd_types.h"
+
+struct creating_pgs_t {
+  epoch_t last_scan_epoch = 0;
+
+  struct pg_create_info {
+    epoch_t create_epoch;
+    utime_t create_stamp;
+
+    // NOTE: pre-octopus instances of this class will have a
+    // zeroed-out history
+    std::vector<int> up;
+    int up_primary = -1;
+    std::vector<int> acting;
+    int acting_primary = -1;
+    pg_history_t history;
+    PastIntervals past_intervals;
+
+    void encode(ceph::buffer::list& bl, uint64_t features) const {
+      using ceph::encode;
+      if (!HAVE_FEATURE(features, SERVER_OCTOPUS)) {
+	// was pair<epoch_t,utime_t> prior to octopus
+	encode(create_epoch, bl);
+	encode(create_stamp, bl);
+	return;
+      }
+      ENCODE_START(1, 1, bl);
+      encode(create_epoch, bl);
+      encode(create_stamp, bl);
+      encode(up, bl);
+      encode(up_primary, bl);
+      encode(acting, bl);
+      encode(acting_primary, bl);
+      encode(history, bl);
+      encode(past_intervals, bl);
+      ENCODE_FINISH(bl);
+    }
+    void decode_legacy(ceph::buffer::list::const_iterator& p) {
+      using ceph::decode;
+      decode(create_epoch, p);
+      decode(create_stamp, p);
+    }
+    void decode(ceph::buffer::list::const_iterator& p) {
+      using ceph::decode;
+      DECODE_START(1, p);
+      decode(create_epoch, p);
+      decode(create_stamp, p);
+      decode(up, p);
+      decode(up_primary, p);
+      decode(acting, p);
+      decode(acting_primary, p);
+      decode(history, p);
+      decode(past_intervals, p);
+      DECODE_FINISH(p);
+    }
+    void dump(ceph::Formatter *f) const {
+      f->dump_unsigned("create_epoch", create_epoch);
+      f->dump_stream("create_stamp") << create_stamp;
+      f->open_array_section("up");
+      for (auto& i : up) {
+	f->dump_unsigned("osd", i);
+      }
+      f->close_section();
+      f->dump_int("up_primary", up_primary);
+      f->open_array_section("acting");
+      for (auto& i : acting) {
+	f->dump_unsigned("osd", i);
+      }
+      f->close_section();
+      f->dump_int("acting_primary", up_primary);
+      f->dump_object("pg_history", history);
+      f->dump_object("past_intervals", past_intervals);
+    }
+
+    pg_create_info() {}
+    pg_create_info(epoch_t e, utime_t t)
+      : create_epoch(e),
+	create_stamp(t) {
+      // NOTE: we don't initialize the other fields here; see
+      // OSDMonitor::update_pending_pgs()
+    }
+  };
+
+  /// pgs we are currently creating
+  std::map<pg_t, pg_create_info> pgs;
+
+  struct pool_create_info {
+    epoch_t created;
+    utime_t modified;
+    uint64_t start = 0;
+    uint64_t end = 0;
+    bool done() const {
+      return start >= end;
+    }
+    void encode(ceph::buffer::list& bl) const {
+      using ceph::encode;
+      encode(created, bl);
+      encode(modified, bl);
+      encode(start, bl);
+      encode(end, bl);
+    }
+    void decode(ceph::buffer::list::const_iterator& p) {
+      using ceph::decode;
+      decode(created, p);
+      decode(modified, p);
+      decode(start, p);
+      decode(end, p);
+    }
+  };
+
+  /// queue of pgs we still need to create (poolid -> <created, set of ps>)
+  std::map<int64_t,pool_create_info> queue;
+
+  /// pools that exist in the osdmap for which at least one pg has been created
+  std::set<int64_t> created_pools;
+
+  bool still_creating_pool(int64_t poolid) {
+    for (auto& i : pgs) {
+      if (i.first.pool() == poolid) {
+	return true;
+      }
+    }
+    if (queue.count(poolid)) {
+      return true;
+    }
+    return false;
+  }
+  void create_pool(int64_t poolid, uint32_t pg_num,
+		   epoch_t created, utime_t modified) {
+    ceph_assert(created_pools.count(poolid) == 0);
+    auto& c = queue[poolid];
+    c.created = created;
+    c.modified = modified;
+    c.end = pg_num;
+    created_pools.insert(poolid);
+  }
+  unsigned remove_pool(int64_t removed_pool) {
+    const unsigned total = pgs.size();
+    auto first = pgs.lower_bound(pg_t{0, (uint64_t)removed_pool});
+    auto last = pgs.lower_bound(pg_t{0, (uint64_t)removed_pool + 1});
+    pgs.erase(first, last);
+    created_pools.erase(removed_pool);
+    queue.erase(removed_pool);
+    return total - pgs.size();
+  }
+  void encode(ceph::buffer::list& bl, uint64_t features) const {
+    unsigned v = 3;
+    if (!HAVE_FEATURE(features, SERVER_OCTOPUS)) {
+      v = 2;
+    }
+    ENCODE_START(v, 1, bl);
+    encode(last_scan_epoch, bl);
+    encode(pgs, bl, features);
+    encode(created_pools, bl);
+    encode(queue, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(ceph::buffer::list::const_iterator& bl) {
+    DECODE_START(3, bl);
+    decode(last_scan_epoch, bl);
+    if (struct_v >= 3) {
+      decode(pgs, bl);
+    } else {
+      // legacy pg encoding
+      pgs.clear();
+      uint32_t num;
+      decode(num, bl);
+      while (num--) {
+	pg_t pgid;
+	decode(pgid, bl);
+	pgs[pgid].decode_legacy(bl);
+      }
+    }
+    decode(created_pools, bl);
+    if (struct_v >= 2)
+      decode(queue, bl);
+    DECODE_FINISH(bl);
+  }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("last_scan_epoch", last_scan_epoch);
+    f->open_array_section("creating_pgs");
+    for (auto& pg : pgs) {
+      f->open_object_section("pg");
+      f->dump_stream("pgid") << pg.first;
+      f->dump_object("pg_create_info", pg.second);
+      f->close_section();
+    }
+    f->close_section();
+    f->open_array_section("queue");
+    for (auto& p : queue) {
+      f->open_object_section("pool");
+      f->dump_unsigned("pool", p.first);
+      f->dump_unsigned("created", p.second.created);
+      f->dump_stream("modified") << p.second.modified;
+      f->dump_unsigned("ps_start", p.second.start);
+      f->dump_unsigned("ps_end", p.second.end);
+      f->close_section();
+    }
+    f->close_section();
+    f->open_array_section("created_pools");
+    for (auto pool : created_pools) {
+      f->dump_unsigned("pool", pool);
+    }
+    f->close_section();
+  }
+  static void generate_test_instances(std::list<creating_pgs_t*>& o) {
+    auto c = new creating_pgs_t;
+    c->last_scan_epoch = 17;
+    c->pgs.emplace(pg_t{42, 2}, pg_create_info(31, utime_t{891, 113}));
+    c->pgs.emplace(pg_t{44, 2}, pg_create_info(31, utime_t{891, 113}));
+    c->created_pools = {0, 1};
+    o.push_back(c);
+    c = new creating_pgs_t;
+    c->last_scan_epoch = 18;
+    c->pgs.emplace(pg_t{42, 3}, pg_create_info(31, utime_t{891, 113}));
+    c->created_pools = {};
+    o.push_back(c);
+  }
+};
+WRITE_CLASS_ENCODER_FEATURES(creating_pgs_t::pg_create_info)
+WRITE_CLASS_ENCODER(creating_pgs_t::pool_create_info)
+WRITE_CLASS_ENCODER_FEATURES(creating_pgs_t)
diff --git a/src/mon/ElectionLogic.cc b/src/mon/ElectionLogic.cc
new file mode 100644
index 000000000..e22a85bed
--- /dev/null
+++ b/src/mon/ElectionLogic.cc
@@ -0,0 +1,556 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "ElectionLogic.h"
+
+#include "include/ceph_assert.h"
+#include "common/dout.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, epoch, elector)
+using std::cerr;
+using std::cout;
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::setfill;
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+using std::unique_ptr;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::mono_clock;
+using ceph::mono_time;
+using ceph::timespan_str;
+static ostream& _prefix(std::ostream *_dout, epoch_t epoch, ElectionOwner* elector) {
+  return *_dout << "paxos." << elector->get_my_rank()
+		<< ").electionLogic(" <<  epoch << ") ";
+}
+void ElectionLogic::init()
+{
+  epoch = elector->read_persisted_epoch();
+  if (!epoch) {
+    ldout(cct, 1) << "init, first boot, initializing epoch at 1 " << dendl;
+    epoch = 1;
+  } else if (epoch % 2) {
+    ldout(cct, 1) << "init, last seen epoch " << epoch
+	    << ", mid-election, bumping" << dendl;
+    ++epoch;
+    elector->persist_epoch(epoch);
+  } else {
+    ldout(cct, 1) << "init, last seen epoch " << epoch << dendl;
+  }
+}
+
+void ElectionLogic::bump_epoch(epoch_t e)
+{
+  ldout(cct, 10) << __func__ << " to "  << e << dendl;
+  ceph_assert(epoch <= e);
+  epoch = e;
+  peer_tracker->increase_epoch(e);
+  elector->persist_epoch(epoch);
+  // clear up some state
+  electing_me = false;
+  acked_me.clear();
+  elector->notify_bump_epoch();
+}
+
+void ElectionLogic::declare_standalone_victory()
+{
+  assert(elector->paxos_size() == 1 && elector->get_my_rank() == 0);
+  init();
+  bump_epoch(epoch+1);
+}
+
+void ElectionLogic::clear_live_election_state()
+{
+  leader_acked = -1;
+  electing_me = false;
+  reset_stable_tracker();
+  leader_peer_tracker.reset();
+}
+
+void ElectionLogic::reset_stable_tracker()
+{
+  stable_peer_tracker.reset(new ConnectionTracker(*peer_tracker));
+}
+
+void ElectionLogic::connectivity_bump_epoch_in_election(epoch_t mepoch)
+{
+  ldout(cct, 30) << __func__ << " to " << mepoch << dendl;
+  ceph_assert(mepoch > epoch);
+  bump_epoch(mepoch);
+  reset_stable_tracker();
+  double lscore, my_score;
+  my_score = connectivity_election_score(elector->get_my_rank());
+  lscore = connectivity_election_score(leader_acked);
+  if (my_score > lscore) {
+    leader_acked = -1;
+    leader_peer_tracker.reset();
+  }
+}
+
+void ElectionLogic::start()
+{
+  if (!participating) {
+    ldout(cct, 0) << "not starting new election -- not participating" << dendl;
+    return;
+  }
+  ldout(cct, 5) << "start -- can i be leader?" << dendl;
+
+  acked_me.clear();
+  init();
+  
+  // start by trying to elect me
+  if (epoch % 2 == 0) {
+    bump_epoch(epoch+1);  // odd == election cycle
+  } else {
+    elector->validate_store();
+  }
+  acked_me.insert(elector->get_my_rank());
+  clear_live_election_state();
+  reset_stable_tracker();
+  electing_me = true;
+
+  bufferlist bl;
+  if (strategy == CONNECTIVITY) {
+    stable_peer_tracker->encode(bl);
+  }
+  elector->propose_to_peers(epoch, bl);
+  elector->_start();
+}
+
+void ElectionLogic::defer(int who)
+{
+  if (strategy == CLASSIC) {
+      ldout(cct, 5) << "defer to " << who << dendl;
+      ceph_assert(who < elector->get_my_rank());
+  } else {
+    ldout(cct, 5) << "defer to " << who << ", disallowed_leaders=" << elector->get_disallowed_leaders() << dendl;
+    ceph_assert(!elector->get_disallowed_leaders().count(who));
+  }
+
+  if (electing_me) {
+    // drop out
+    acked_me.clear();
+    electing_me = false;
+  }
+
+  // ack them
+  leader_acked = who;
+  elector->_defer_to(who);
+}
+
+void ElectionLogic::end_election_period()
+{
+  ldout(cct, 5) << "election period ended" << dendl;
+  
+  // did i win?
+  if (electing_me &&
+      acked_me.size() > (elector->paxos_size() / 2)) {
+    // i win
+    declare_victory();
+  } else {
+    // whoever i deferred to didn't declare victory quickly enough.
+    if (elector->ever_participated())
+      start();
+    else
+      elector->reset_election();
+  }
+}
+
+
+void ElectionLogic::declare_victory()
+{
+  ldout(cct, 5) << "I win! acked_me=" << acked_me << dendl;
+  last_election_winner = elector->get_my_rank();
+  last_voted_for = last_election_winner;
+  clear_live_election_state();
+
+  set<int> new_quorum;
+  new_quorum.swap(acked_me);
+  
+  ceph_assert(epoch % 2 == 1);  // election
+  bump_epoch(epoch+1);     // is over!
+
+  elector->message_victory(new_quorum);
+}
+
+bool ElectionLogic::propose_classic_prefix(int from, epoch_t mepoch)
+{
+  if (mepoch > epoch) {
+    bump_epoch(mepoch);
+  } else if (mepoch < epoch) {
+    // got an "old" propose,
+    if (epoch % 2 == 0 &&    // in a non-election cycle
+	!elector->is_current_member(from)) {  // from someone outside the quorum
+      // a mon just started up, call a new election so they can rejoin!
+      ldout(cct, 5) << " got propose from old epoch, "
+	      << from << " must have just started" << dendl;
+      // we may be active; make sure we reset things in the monitor appropriately.
+      elector->trigger_new_election();
+    } else {
+      ldout(cct, 5) << " ignoring old propose" << dendl;
+    }
+    return true;
+  }
+  return false;
+}
+
+void ElectionLogic::receive_propose(int from, epoch_t mepoch,
+				    const ConnectionTracker *ct)
+{
+  ldout(cct, 20) << __func__ << " from " << from << dendl;
+  if (from == elector->get_my_rank()) {
+    lderr(cct) << "I got a propose from my own rank, hopefully this is startup weirdness,dropping" << dendl;
+    return;
+  }
+  switch (strategy) {
+  case CLASSIC:
+    propose_classic_handler(from, mepoch);
+    break;
+  case DISALLOW:
+    propose_disallow_handler(from, mepoch);
+    break;
+  case CONNECTIVITY:
+    propose_connectivity_handler(from, mepoch, ct);
+    break;
+  default:
+    ceph_assert(0 == "how did election strategy become an invalid value?");
+  }
+}
+
+void ElectionLogic::propose_disallow_handler(int from, epoch_t mepoch)
+{
+  if (propose_classic_prefix(from, mepoch)) {
+    return;
+  }
+  const set<int>& disallowed_leaders = elector->get_disallowed_leaders();
+  int my_rank = elector->get_my_rank();
+  bool me_disallowed = disallowed_leaders.count(my_rank);
+  bool from_disallowed = disallowed_leaders.count(from);
+  bool my_win = !me_disallowed && // we are allowed to lead
+    (my_rank < from || from_disallowed); // we are a better choice than them
+  bool their_win = !from_disallowed && // they are allowed to lead
+    (my_rank > from || me_disallowed) && // they are a better choice than us
+    (leader_acked < 0 || leader_acked >= from); // they are a better choice than our previously-acked choice
+    
+  
+  if (my_win) {
+    // i would win over them.
+    if (leader_acked >= 0) {        // we already acked someone
+      ceph_assert(leader_acked < from || from_disallowed);  // and they still win, of course
+      ldout(cct, 5) << "no, we already acked " << leader_acked << dendl;
+    } else {
+      // wait, i should win!
+      if (!electing_me) {
+	elector->trigger_new_election();
+      }
+    }
+  } else {
+    // they would win over me
+    if (their_win) {
+      defer(from);
+    } else {
+      // ignore them!
+      ldout(cct, 5) << "no, we already acked " << leader_acked << dendl;
+    }
+  }
+}
+
+void ElectionLogic::propose_classic_handler(int from, epoch_t mepoch)
+{
+  if (propose_classic_prefix(from, mepoch)) {
+    return;
+  }
+  if (elector->get_my_rank() < from) {
+    // i would win over them.
+    if (leader_acked >= 0) {        // we already acked someone
+      ceph_assert(leader_acked < from);  // and they still win, of course
+      ldout(cct, 5) << "no, we already acked " << leader_acked << dendl;
+    } else {
+      // wait, i should win!
+      if (!electing_me) {
+	elector->trigger_new_election();
+      }
+    }
+  } else {
+    // they would win over me
+    if (leader_acked < 0 || // haven't acked anyone yet, or
+	leader_acked > from ||   // they would win over who you did ack, or
+	leader_acked == from) {  // this is the guy we're already deferring to
+      defer(from);
+    } else {
+      // ignore them!
+      ldout(cct, 5) << "no, we already acked " << leader_acked << dendl;
+    }
+  }
+}
+
+double ElectionLogic::connectivity_election_score(int rank)
+{
+  ldout(cct, 30) << __func__ << " of " << rank << dendl;
+  if (elector->get_disallowed_leaders().count(rank)) {
+    return -1;
+  }
+  double score;
+  int liveness;
+  if (stable_peer_tracker) {
+    ldout(cct, 30) << "stable_peer_tracker exists so using that ..." << dendl;
+    stable_peer_tracker->get_total_connection_score(rank, &score, &liveness);
+  } else {
+    ldout(cct, 30) << "stable_peer_tracker does not exists, using peer_tracker ..." << dendl;
+    peer_tracker->get_total_connection_score(rank, &score, &liveness);
+  }
+  return score;
+}
+
+void ElectionLogic::propose_connectivity_handler(int from, epoch_t mepoch,
+						 const ConnectionTracker *ct)
+{
+  ldout(cct, 10) << __func__ << " from " << from << " mepoch: "
+    << mepoch << " epoch: " << epoch << dendl;
+  ldout(cct, 30) << "last_election_winner: " << last_election_winner << dendl;
+  if ((epoch % 2 == 0) &&
+      last_election_winner != elector->get_my_rank() &&
+      !elector->is_current_member(from)) {
+    // To prevent election flapping, peons ignore proposals from out-of-quorum
+    // peers unless their vote would materially change from the last election
+    ldout(cct, 30) << "Lets see if this out-of-quorum peer is worth it " << dendl;
+    int best_scorer = 0;
+    double best_score = 0;
+    double last_voted_for_score = 0;
+    ldout(cct, 30) << "elector->paxos_size(): " << elector->paxos_size() << dendl;
+    for (unsigned i = 0; i < elector->paxos_size(); ++i) {
+      double score = connectivity_election_score(i);
+      if (score > best_score) {
+	best_scorer = i;
+	best_score = score;
+      }
+      if (last_voted_for >= 0 && i == static_cast<unsigned>(last_voted_for)) {
+	last_voted_for_score = score;
+      }
+    }
+    ldout(cct, 30) << "best_scorer: " << best_scorer << " best_score: " << best_score
+      << " last_voted_for: " << last_voted_for << " last_voted_for_score: " 
+      << last_voted_for_score << dendl;
+    if (best_scorer == last_voted_for ||
+	(best_score - last_voted_for_score < ignore_propose_margin)) {
+      // drop this message; it won't change our vote so we defer to leader
+      ldout(cct, 30) << "drop this message; it won't change our vote so we defer to leader " << dendl;
+      return;
+    }
+  }
+  if (mepoch > epoch) {
+    ldout(cct, 20) << "mepoch > epoch" << dendl;
+    connectivity_bump_epoch_in_election(mepoch);
+  } else if (mepoch < epoch) {
+    // got an "old" propose,
+    if (epoch % 2 == 0 &&    // in a non-election cycle
+	!elector->is_current_member(from)) {  // from someone outside the quorum
+      // a mon just started up, call a new election so they can rejoin!
+      ldout(cct, 5) << " got propose from old epoch, "
+	      << from << " must have just started" << dendl;
+      ldout(cct, 10) << "triggering new election" << dendl;
+      // we may be active; make sure we reset things in the monitor appropriately.
+      elector->trigger_new_election();
+    } else {
+      ldout(cct, 5) << " ignoring old propose" << dendl;
+    }
+    return;
+  }
+
+  int my_rank = elector->get_my_rank();
+  double my_score = connectivity_election_score(my_rank);
+  double from_score = connectivity_election_score(from);
+  double leader_score = -1;
+  if (leader_acked >= 0) {
+    leader_score = connectivity_election_score(leader_acked);
+  }
+
+  ldout(cct, 20) << "propose from rank=" << from << ", tracker: "
+		 << (stable_peer_tracker ? *stable_peer_tracker : *peer_tracker) << dendl;
+
+  ldout(cct, 10) << "propose from rank=" << from << ",from_score=" << from_score
+		 << "; my score=" << my_score
+		 << "; currently acked " << leader_acked
+		 << ",leader_score=" << leader_score << dendl;
+
+  bool my_win = (my_score >= 0) && // My score is non-zero; I am allowed to lead
+    ((my_rank < from && my_score >= from_score) || // We have same scores and I have lower rank, or
+     (my_score > from_score)); // my score is higher
+  
+  bool their_win = (from_score >= 0) && // Their score is non-zero; they're allowed to lead, AND
+    ((from < my_rank && from_score >= my_score) || // Either they have lower rank and same score, or
+     (from_score > my_score)) && // their score is higher, AND
+    ((from <= leader_acked && from_score >= leader_score) || // same conditions compared to leader, or IS leader
+     (from_score > leader_score));
+
+  if (my_win) {
+    ldout(cct, 10) << " conditionally I win" << dendl;
+    // i would win over them.
+    if (leader_acked >= 0) {        // we already acked someone
+      ceph_assert(leader_score >= from_score);  // and they still win, of course
+      ldout(cct, 5) << "no, we already acked " << leader_acked << dendl;
+    } else {
+      // wait, i should win!
+      if (!electing_me) {
+      ldout(cct, 10) << " wait, i should win! triggering new election ..." << dendl;
+	elector->trigger_new_election();
+      }
+    }
+  } else {
+    ldout(cct, 10) << " conditionally they win" << dendl;
+    // they would win over me
+    if (their_win || from == leader_acked) {
+      if (leader_acked >= 0 && from != leader_acked) {
+	// we have to make sure our acked leader will ALSO defer to them, or else
+	// we can't, to maintain guarantees!
+  ldout(cct, 10) << " make sure acked leader defer to: " << from << dendl;
+	double leader_from_score;
+	int leader_from_liveness;
+	leader_peer_tracker->
+	  get_total_connection_score(from, &leader_from_score,
+				     &leader_from_liveness);
+	double leader_leader_score;
+	int leader_leader_liveness;
+	leader_peer_tracker->
+	  get_total_connection_score(leader_acked, &leader_leader_score,
+				     &leader_leader_liveness);
+	if ((from < leader_acked && leader_from_score >= leader_leader_score) ||
+	    (leader_from_score > leader_leader_score)) {
+    ldout(cct, 10) << "defering to " << from << dendl;
+	  defer(from);
+	  leader_peer_tracker.reset(new ConnectionTracker(*ct));
+	} else { // we can't defer to them *this* round even though they should win...
+	  double cur_leader_score, cur_from_score;
+	  int cur_leader_live, cur_from_live;
+	  peer_tracker->get_total_connection_score(leader_acked, &cur_leader_score, &cur_leader_live);
+	  peer_tracker->get_total_connection_score(from, &cur_from_score, &cur_from_live);
+	  if ((from < leader_acked && cur_from_score >= cur_leader_score) ||
+	      (cur_from_score > cur_leader_score)) {
+	    ldout(cct, 5) << "Bumping epoch and starting new election; acked "
+			  << leader_acked << " should defer to " << from
+			  << " but there is score disagreement!" << dendl;
+	    bump_epoch(epoch+1);
+	    start();
+	  } else {
+	    ldout(cct, 5) << "no, we already acked " << leader_acked
+			  << " and it won't defer to " << from
+			  << " despite better round scores" << dendl;
+	  }
+	}
+      } else {
+  ldout(cct, 10) << "defering to " << from << dendl;
+	defer(from);
+	leader_peer_tracker.reset(new ConnectionTracker(*ct));
+      }
+    } else {
+      // ignore them!
+      ldout(cct, 5) << "no, we already acked " << leader_acked << " with score >=" << from_score << dendl;
+    }
+  }
+}
+
+void ElectionLogic::receive_ack(int from, epoch_t from_epoch)
+{
+  ceph_assert(from_epoch % 2 == 1); // sender in an election epoch
+  if (from_epoch > epoch) {
+    ldout(cct, 5) << "woah, that's a newer epoch, i must have rebooted.  bumping and re-starting!" << dendl;
+    bump_epoch(from_epoch);
+    start();
+    return;
+  }
+  // is that _everyone_?
+  if (electing_me) {
+    acked_me.insert(from);
+    if (acked_me.size() == elector->paxos_size()) {
+      // if yes, shortcut to election finish
+      declare_victory();
+    }
+  } else {
+    // ignore, i'm deferring already.
+    ceph_assert(leader_acked >= 0);
+  }
+}
+
+bool ElectionLogic::victory_makes_sense(int from)
+{
+  bool makes_sense = false;
+  switch (strategy) {
+  case CLASSIC:
+    makes_sense = (from < elector->get_my_rank());
+    break;
+  case DISALLOW:
+    makes_sense = (from < elector->get_my_rank()) ||
+      elector->get_disallowed_leaders().count(elector->get_my_rank());
+    break;
+  case CONNECTIVITY:
+    double my_score, leader_score;
+    my_score = connectivity_election_score(elector->get_my_rank());
+    leader_score = connectivity_election_score(from);
+    ldout(cct, 5) << "victory from " << from << " makes sense? lscore:"
+		  << leader_score
+		  << "; my score:" << my_score << dendl;
+
+    makes_sense = (leader_score >= my_score);
+    break;
+  default:
+    ceph_assert(0 == "how did you get a nonsense election strategy assigned?");
+  }
+  return makes_sense;
+}
+
+bool ElectionLogic::receive_victory_claim(int from, epoch_t from_epoch)
+{
+  bool election_okay = victory_makes_sense(from);
+
+  last_election_winner = from;
+  last_voted_for = leader_acked;
+  clear_live_election_state();
+
+  if (!election_okay) {
+    ceph_assert(strategy == CONNECTIVITY);
+    ldout(cct, 1) << "I should have been elected over this leader; bumping and restarting!" << dendl;
+    bump_epoch(from_epoch);
+    start();
+    return false;
+  }
+
+  // i should have seen this election if i'm getting the victory.
+  if (from_epoch != epoch + 1) { 
+    ldout(cct, 5) << "woah, that's a funny epoch, i must have rebooted.  bumping and re-starting!" << dendl;
+    bump_epoch(from_epoch);
+    start();
+    return false;
+  }
+
+  bump_epoch(from_epoch);
+
+  // they win
+  return true;
+}
diff --git a/src/mon/ElectionLogic.h b/src/mon/ElectionLogic.h
new file mode 100644
index 000000000..65c727ca1
--- /dev/null
+++ b/src/mon/ElectionLogic.h
@@ -0,0 +1,459 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_ELECTIONLOGIC_H
+#define CEPH_ELECTIONLOGIC_H
+
+#include <map>
+#include "include/types.h"
+#include "ConnectionTracker.h"
+
+class ElectionOwner {
+public:
+  /**
+   * Write down the given epoch in persistent storage, such that it
+   * can later be retrieved by read_persisted_epoch even across process
+   * or machine restarts.
+   *
+   * @param e The epoch to write
+   */
+  virtual void persist_epoch(epoch_t e) = 0;
+  /**
+   * Retrieve the most-previously-persisted epoch.
+   *
+   * @returns The latest epoch passed to persist_epoch()
+   */
+  virtual epoch_t read_persisted_epoch() const = 0;
+  /**
+   * Validate that the persistent store is working by committing
+   * to it. (There is no interface for retrieving the value; this
+   * tests local functionality before doing things like triggering
+   * elections to try and join a quorum.)
+   */
+  virtual void validate_store() = 0;
+  /**
+   * Notify the ElectionOwner that ElectionLogic has increased its
+   * election epoch. This resets an election (either on local loss or victory,
+   * or when trying a new election round) and the ElectionOwner
+   * should reset any tracking of its own to match. (The ElectionLogic
+   * will further trigger sending election messages if that is
+   * appropriate.)
+   */
+  virtual void notify_bump_epoch() = 0;
+  /**
+   * Notify the ElectionOwner we must start a new election.
+   */
+  virtual void trigger_new_election() = 0;
+  /**
+   * Retrieve this Paxos instance's rank.
+   */
+  virtual int get_my_rank() const = 0;
+  /**
+   * Send a PROPOSE message to all our peers. This happens when
+   * we have started a new election (which may mean attempting to
+   * override a current one).
+   *
+   * @param e The election epoch of our proposal.
+   * @param bl A bufferlist containing data the logic wishes to share
+   */
+  virtual void propose_to_peers(epoch_t e, bufferlist& bl) = 0;
+  /**
+   * The election has failed and we aren't sure what the state of the
+   * quorum is, so reset the entire system as if from scratch.
+   */
+  virtual void reset_election() = 0;
+  /**
+   * Ask the ElectionOwner if we-the-Monitor have ever participated in the
+   * quorum (including across process restarts!).
+   *
+   * @returns true if we have participated, false otherwise
+   */
+  virtual bool ever_participated() const = 0;
+  /**
+   * Ask the ElectionOwner for the size of the Paxos set. This includes
+   * those monitors which may not be in the current quorum!
+   * The value returned by this function can change between elections,
+   * but not during them. (In practical terms, it can be updated
+   * by making a paxos commit, but not by injecting values while
+   * an election is ongoing.)
+   */
+  virtual unsigned paxos_size() const = 0;
+  /**
+   * Retrieve a set of ranks which are not allowed to become the leader.
+   * Like paxos_size(), This set can change between elections, but not
+   * during them.
+   */
+  virtual const set<int>& get_disallowed_leaders() const = 0;
+  /**
+   * Tell the ElectionOwner we have started a new election.
+   *
+   * The ElectionOwner is responsible for timing out the election (by invoking
+   * end_election_period()) if it takes too long (as defined by the ElectionOwner).
+   * This function is the opportunity to do that and to clean up any other external 
+   * election state it may be maintaining.
+   */
+  virtual void _start() = 0;
+  /**
+   * Tell the ElectionOwner to defer to the identified peer. Tell that peer
+   * we have deferred to it.
+   *
+   * @post  we sent an ack message to @p who
+   */
+  virtual void _defer_to(int who) = 0;
+  /**
+   * We have won an election, so have the ElectionOwner message that to
+   * our new quorum!
+   *
+   * @param quorum The ranks of our peers which deferred to us and
+   *        must be told of our victory
+   */
+  virtual void message_victory(const std::set<int>& quorum) = 0;
+  /**
+   * Query the ElectionOwner about if a given rank is in the
+   * currently active quorum.
+   * @param rank the Paxos rank whose status we are checking
+   * @returns true if the rank is in our current quorum, false otherwise.
+   */
+  virtual bool is_current_member(int rank) const = 0;
+  virtual ~ElectionOwner() {}
+};
+
+/**
+ * This class maintains local state for running an election
+ * between Paxos instances. It receives input requests
+ * and calls back out to its ElectionOwner to do persistence
+ * and message other entities.
+ */
+
+class ElectionLogic {
+  ElectionOwner *elector;
+  ConnectionTracker *peer_tracker;
+  
+  CephContext *cct;
+  /**
+   * Latest epoch we've seen.
+   *
+   * @remarks if its value is odd, we're electing; if it's even, then we're
+   *	      stable.
+   */
+  epoch_t epoch = 0;
+  /**
+   * The last rank which won an election we participated in
+   */
+  int last_election_winner = -1;
+  /**
+   * Only used in the connectivity handler.
+   * The rank we voted for in the last election we voted in.
+   */
+  int last_voted_for = -1;
+  double ignore_propose_margin = 0.0001;
+  /**
+   * Only used in the connectivity handler.
+   * Points at a stable copy of the peer_tracker we use to keep scores
+   * throughout an election period.
+   */
+  std::unique_ptr<ConnectionTracker> stable_peer_tracker;
+  std::unique_ptr<ConnectionTracker> leader_peer_tracker;
+  /**
+   * Indicates who we have acked
+   */
+  int leader_acked;
+  
+public:
+  enum election_strategy {
+			  // Keep in sync with MonMap.h!
+    CLASSIC = 1, // the original rank-based one
+    DISALLOW = 2, // disallow a set from being leader
+    CONNECTIVITY = 3 // includes DISALLOW, extends to prefer stronger connections
+  };
+  election_strategy strategy;
+    
+  /**
+   * Indicates if we are participating in the quorum.
+   *
+   * @remarks By default, we are created as participating. We may stop
+   *	      participating if something explicitly sets our value
+   *	      false, though. If that happens, it will
+   *	      have to set participating=true and invoke start() for us to resume
+   *	      participating in the quorum.
+   */
+  bool participating;
+  /**
+   * Indicates if we are the ones being elected.
+   *
+   * We always attempt to be the one being elected if we are the ones starting
+   * the election. If we are not the ones that started it, we will only attempt
+   * to be elected if we think we might have a chance (i.e., the other guy's
+   * rank is lower than ours).
+   */
+  bool electing_me;
+  /**
+   * Set containing all those that acked our proposal to become the Leader.
+   *
+   * If we are acked by ElectionOwner::paxos_size() peers, we will declare
+   * victory.
+   */
+  std::set<int> acked_me;
+
+  ElectionLogic(ElectionOwner *e, election_strategy es, ConnectionTracker *t,
+		double ipm,
+		CephContext *c) : elector(e), peer_tracker(t), cct(c),
+				  last_election_winner(-1), last_voted_for(-1),
+				  ignore_propose_margin(ipm),
+				  stable_peer_tracker(),
+				  leader_peer_tracker(),
+				  leader_acked(-1),
+				  strategy(es),
+				  participating(true),
+				  electing_me(false) {}
+  /**
+   * Set the election strategy to use. If this is not consistent across the
+   * electing cluster, you're going to have a bad time.
+   * Defaults to CLASSIC.
+   */
+  void set_election_strategy(election_strategy es) {
+    strategy = es;
+  }
+  /**
+   * If there are no other peers in this Paxos group, ElectionOwner
+   * can simply declare victory and we will make it so.
+   *
+   * @pre paxos_size() is 1
+   * @pre get_my_rank is 0
+   */
+  void declare_standalone_victory();
+  /**
+   * Start a new election by proposing ourselves as the new Leader.
+   *
+   * Basically, send propose messages to all the peers.
+   *
+   * @pre   participating is true
+   * @post  epoch is an odd value
+   * @post  electing_me is true
+   * @post  We have invoked propose_to_peers() on our ElectionOwner
+   * @post  We have invoked _start() on our ElectionOwner
+   */
+  void start();
+  /**
+   * ElectionOwner has decided the election has taken too long and expired.
+   *
+   * This will happen when no one declared victory or started a new election
+   * during the allowed time span.
+   *
+   * When the election expires, we will check if we were the ones who won, and
+   * if so we will declare victory. If that is not the case, then we assume
+   * that the one we deferred to didn't declare victory quickly enough (in fact,
+   * as far as we know, it may even be dead); so, just propose ourselves as the
+   * Leader.
+   */
+  void end_election_period();
+  /**
+   * Handle a proposal from some other node proposing asking to become
+   * the Leader.
+   *
+   * If the message appears to be old (i.e., its epoch is lower than our epoch),
+   * then we may take one of two actions:
+   *
+   *  @li Ignore it because it's nothing more than an old proposal
+   *  @li Start new elections if we verify that it was sent by a monitor from
+   *	  outside the quorum; given its old state, it's fair to assume it just
+   *	  started, so we should start new elections so it may rejoin. (Some
+   *      handlers may choose to ignore even these, if they think it's flapping.)
+   *
+   * We pass the propose off to a propose_*_handler function based
+   * on the election strategy we're using.
+   * Only the Connectivity strategy cares about the ConnectionTracker; it should
+   * be NULL if other strategies are in use. Otherwise, it will take ownership
+   * of the underlying data and delete it as needed.
+   *
+   * @pre   Message epoch is from the current or a newer epoch
+   * @param mepoch The epoch of the proposal
+   * @param from The rank proposing itself as leader
+   * @param ct Any incoming ConnectionTracker data sent with the message.
+   *  Callers are responsible for deleting this -- we will copy it if we want
+   *  to keep the data.
+   */
+  void receive_propose(int from, epoch_t mepoch, const ConnectionTracker *ct);
+  /**
+   * Handle a message from some other participant Acking us as the Leader.
+   *
+   * When we receive such a message, one of three thing may be happening:
+   *  @li We received a message with a newer epoch, which means we must have
+   *	  somehow lost track of what was going on (maybe we rebooted), thus we
+   *	  will start a new election
+   *  @li We consider ourselves in the run for the Leader (i.e., @p electing_me 
+   *	  is true), and we are actually being Acked by someone; thus simply add
+   *	  the one acking us to the @p acked_me set. If we do now have acks from
+   *	  all the participants, then we can declare victory
+   *  @li We already deferred the election to somebody else, so we will just
+   *	  ignore this message
+   *
+   * @pre   Message epoch is from the current or a newer epoch
+   * @post  Election is on-going if we deferred to somebody else
+   * @post  Election is on-going if we are still waiting for further Acks
+   * @post  Election is not on-going if we are victorious
+   * @post  Election is not on-going if we must start a new one
+   *
+   * @param from The rank which acked us
+   * @param from_epoch The election epoch the ack belongs to
+   */
+  void receive_ack(int from, epoch_t from_epoch);
+  /**
+   * Handle a message from some other participant declaring Victory.
+   *
+   * We just got a message from someone declaring themselves Victorious, thus
+   * the new Leader.
+   *
+   * However, if the message's epoch happens to be different from our epoch+1,
+   * then it means we lost track of something and we must start a new election.
+   *
+   * If that is not the case, then we will simply update our epoch to the one
+   * in the message and invoke start() to reset the quorum.
+   *
+   * @pre   from_epoch is the current or a newer epoch
+   * @post  Election is not on-going
+   * @post  Updated @p epoch
+   * @post  We are a peon in a new quorum if we lost the election
+   *
+   * @param from The victory-claiming rank
+   * @param from_epoch The election epoch in which they claim victory
+   */
+  bool receive_victory_claim(int from, epoch_t from_epoch);
+  /**
+   * Obtain our epoch
+   *
+   * @returns Our current epoch number
+   */
+  epoch_t get_epoch() const { return epoch; }
+  int get_election_winner() { return last_election_winner; }
+
+private:
+  /**
+   * Initiate the ElectionLogic class.
+   *
+   * Basically, we will simply read whatever epoch value we have in our stable
+   * storage, or consider it to be 1 if none is read.
+   *
+   * @post @p epoch is set to 1 or higher.
+   */
+  void init();
+  /**
+   * Update our epoch.
+   *
+   * If we come across a higher epoch, we simply update ours, also making
+   * sure we are no longer being elected (even though we could have been,
+   * we no longer are since we no longer are on that old epoch).
+   *
+   * @pre Our epoch is not larger than @p e
+   * @post Our epoch equals @p e
+   *
+   * @param e Epoch to which we will update our epoch
+   */
+  void bump_epoch(epoch_t e);
+  /**
+   * If the incoming proposal is newer, bump our own epoch; if
+   * it comes from an out-of-quorum peer, trigger a new eleciton.
+   * @returns true if you should drop this proposal, false otherwise.
+   */
+  bool propose_classic_prefix(int from, epoch_t mepoch);
+  /**
+   * Handle a proposal from another rank using the classic strategy.
+   * We will take one of the following actions:
+   *
+   *  @li Ignore it because we already acked another node with higher rank
+   *  @li Ignore it and start a new election because we outrank it
+   *  @li Defer to it because it outranks us and the node we previously
+   *	  acked, if any
+   */
+  void propose_classic_handler(int from, epoch_t mepoch);
+  /**
+   * Handle a proposal from another rank using our disallow strategy.
+   * This is the same as the classic strategy except we also disallow
+   * certain ranks from becoming the leader.
+   */
+  void propose_disallow_handler(int from, epoch_t mepoch);
+  /**
+   * Handle a proposal from another rank using the connectivity strategy.
+   * We will choose to defer or not based on the ordered criteria:
+   *
+   * @li Whether the other monitor (or ourself) is on the disallow list
+   * @li Whether the other monitor or ourself has the most connectivity to peers
+   * @li Whether the other monitor or ourself has the lower rank
+   */
+  void propose_connectivity_handler(int from, epoch_t mepoch, const ConnectionTracker *ct);
+  /**
+   * Helper function for connectivity handler. Combines the disallowed list
+   * with ConnectionTracker scores.
+   */
+  double connectivity_election_score(int rank);
+  /**
+   * Defer the current election to some other monitor.
+   *
+   * This means that we will ack some other monitor and drop out from the run
+   * to become the Leader. We will only defer an election if the monitor we
+   * are deferring to outranks us.
+   *
+   * @pre   @p who outranks us (i.e., who < our rank)
+   * @pre   @p who outranks any other monitor we have deferred to in the past
+   * @post  electing_me is false
+   * @post  leader_acked equals @p who
+   * @post  we triggered ElectionOwner's _defer_to() on @p who
+   *
+   * @param who Some other monitor's numeric identifier. 
+   */
+  void defer(int who);
+  /**
+   * Declare Victory.
+   * 
+   * We won. Or at least we believe we won, but for all intents and purposes
+   * that does not matter. What matters is that we Won.
+   *
+   * That said, we must now bump our epoch to reflect that the election is over
+   * and then we must let everybody in the quorum know we are their brand new
+   * Leader.
+   *
+   * Actually, the quorum will be now defined as the group of monitors that
+   * acked us during the election process.
+   *
+   * @pre   Election is on-going
+   * @pre   electing_me is true
+   * @post  electing_me is false
+   * @post  epoch is bumped up into an even value
+   * @post  Election is not on-going
+   * @post  We have a quorum, composed of the monitors that acked us
+   * @post  We invoked message_victory() on the ElectionOwner
+   */
+  void declare_victory();
+  /**
+   * This is just a helper function to validate that the victory claim we
+   * get from another rank makes any sense.
+   */
+  bool victory_makes_sense(int from);
+  /**
+   * Reset some data members which we only care about while we are in an election
+   * or need to be set consistently during stable states.
+   */
+  void clear_live_election_state();
+  void reset_stable_tracker();
+  /**
+   * Only for the connectivity handler, Bump the epoch
+   * when we get a message from a newer one and clear
+   * out leader and stable tracker
+   * data so that we can switch our allegiance.
+   */
+  void connectivity_bump_epoch_in_election(epoch_t mepoch);
+};
+
+#endif
diff --git a/src/mon/Elector.cc b/src/mon/Elector.cc
new file mode 100644
index 000000000..671c08d85
--- /dev/null
+++ b/src/mon/Elector.cc
@@ -0,0 +1,807 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "Elector.h"
+#include "Monitor.h"
+
+#include "common/Timer.h"
+#include "MonitorDBStore.h"
+#include "messages/MMonElection.h"
+#include "messages/MMonPing.h"
+
+#include "common/config.h"
+#include "include/ceph_assert.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, get_epoch())
+using std::cerr;
+using std::cout;
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::setfill;
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+using std::unique_ptr;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::mono_clock;
+using ceph::mono_time;
+using ceph::timespan_str;
+static ostream& _prefix(std::ostream *_dout, Monitor *mon, epoch_t epoch) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name()
+		<< ").elector(" << epoch << ") ";
+}
+
+Elector::Elector(Monitor *m, int strategy) : logic(this, static_cast<ElectionLogic::election_strategy>(strategy),
+						   &peer_tracker,
+						   m->cct->_conf.get_val<double>("mon_elector_ignore_propose_margin"),
+						   m->cct),
+					     peer_tracker(this, m->rank,
+					    m->cct->_conf.get_val<uint64_t>("mon_con_tracker_score_halflife"),
+					    m->cct->_conf.get_val<uint64_t>("mon_con_tracker_persist_interval"), m->cct),
+			       ping_timeout(m->cct->_conf.get_val<double>("mon_elector_ping_timeout")),
+			       PING_DIVISOR(m->cct->_conf.get_val<uint64_t>("mon_elector_ping_divisor")),
+			       mon(m), elector(this) {
+  bufferlist bl;
+  mon->store->get(Monitor::MONITOR_NAME, "connectivity_scores", bl);
+  if (bl.length()) {
+    bufferlist::const_iterator bi = bl.begin();
+    peer_tracker.decode(bi);
+  }
+}
+
+
+void Elector::persist_epoch(epoch_t e)
+{
+  auto t(std::make_shared<MonitorDBStore::Transaction>());
+  t->put(Monitor::MONITOR_NAME, "election_epoch", e);
+  t->put(Monitor::MONITOR_NAME, "connectivity_scores", peer_tracker.get_encoded_bl());
+  mon->store->apply_transaction(t);
+}
+
+void Elector::persist_connectivity_scores()
+{
+  dout(20) << __func__ << dendl;
+  auto t(std::make_shared<MonitorDBStore::Transaction>());
+  t->put(Monitor::MONITOR_NAME, "connectivity_scores", peer_tracker.get_encoded_bl());
+  mon->store->apply_transaction(t);
+}
+
+epoch_t Elector::read_persisted_epoch() const
+{
+  return mon->store->get(Monitor::MONITOR_NAME, "election_epoch");
+}
+
+void Elector::validate_store()
+{
+  auto t(std::make_shared<MonitorDBStore::Transaction>());
+  t->put(Monitor::MONITOR_NAME, "election_writeable_test", rand());
+  int r = mon->store->apply_transaction(t);
+  ceph_assert(r >= 0);
+}
+
+bool Elector::is_current_member(int rank) const
+{
+  return mon->quorum.count(rank);
+}
+
+void Elector::trigger_new_election()
+{
+  mon->start_election();
+}
+
+int Elector::get_my_rank() const
+{
+  return mon->rank;
+}
+
+void Elector::reset_election()
+{
+  mon->bootstrap();
+}
+
+bool Elector::ever_participated() const
+{
+  return mon->has_ever_joined;
+}
+
+unsigned Elector::paxos_size() const
+{
+  return (unsigned)mon->monmap->size();
+}
+
+void Elector::shutdown()
+{
+  cancel_timer();
+}
+
+void Elector::notify_bump_epoch()
+{
+  mon->join_election();
+}
+
+void Elector::propose_to_peers(epoch_t e, bufferlist& logic_bl)
+{
+  // bcast to everyone else
+  for (unsigned i=0; i<mon->monmap->size(); ++i) {
+    if ((int)i == mon->rank) continue;
+    MMonElection *m =
+      new MMonElection(MMonElection::OP_PROPOSE, e,
+		       peer_tracker.get_encoded_bl(),
+		       logic.strategy, mon->monmap);
+    m->sharing_bl = logic_bl;
+    m->mon_features = ceph::features::mon::get_supported();
+    m->mon_release = ceph_release();
+    mon->send_mon_message(m, i);
+  }  
+}
+
+void Elector::_start()
+{
+  peer_info.clear();
+  peer_info[mon->rank].cluster_features = CEPH_FEATURES_ALL;
+  peer_info[mon->rank].mon_release = ceph_release();
+  peer_info[mon->rank].mon_features = ceph::features::mon::get_supported();
+  mon->collect_metadata(&peer_info[mon->rank].metadata);
+  reset_timer();
+}
+
+void Elector::_defer_to(int who)
+{
+  MMonElection *m = new MMonElection(MMonElection::OP_ACK, get_epoch(),
+				     peer_tracker.get_encoded_bl(),
+				     logic.strategy, mon->monmap);
+  m->mon_features = ceph::features::mon::get_supported();
+  m->mon_release = ceph_release();
+  mon->collect_metadata(&m->metadata);
+
+  mon->send_mon_message(m, who);
+  
+  // set a timer
+  reset_timer(1.0);  // give the leader some extra time to declare victory
+}
+
+
+void Elector::reset_timer(double plus)
+{
+  // set the timer
+  cancel_timer();
+  /**
+   * This class is used as the callback when the expire_event timer fires up.
+   *
+   * If the expire_event is fired, then it means that we had an election going,
+   * either started by us or by some other participant, but it took too long,
+   * thus expiring.
+   *
+   * When the election expires, we will check if we were the ones who won, and
+   * if so we will declare victory. If that is not the case, then we assume
+   * that the one we defered to didn't declare victory quickly enough (in fact,
+   * as far as we know, we may even be dead); so, just propose ourselves as the
+   * Leader.
+   */
+  expire_event = mon->timer.add_event_after(
+    g_conf()->mon_election_timeout + plus,
+    new C_MonContext{mon, [this](int) {
+	logic.end_election_period();
+      }});
+}
+
+
+void Elector::cancel_timer()
+{
+  if (expire_event) {
+    mon->timer.cancel_event(expire_event);
+    expire_event = 0;
+  }
+}
+
+void Elector::assimilate_connection_reports(const bufferlist& tbl)
+{
+  dout(10) << __func__ << dendl;
+  ConnectionTracker pct(tbl, mon->cct);
+  peer_tracker.receive_peer_report(pct);
+}
+
+void Elector::message_victory(const std::set<int>& quorum)
+{
+  uint64_t cluster_features = CEPH_FEATURES_ALL;
+  mon_feature_t mon_features = ceph::features::mon::get_supported();
+  map<int,Metadata> metadata;
+  ceph_release_t min_mon_release{ceph_release_t::unknown};
+  for (auto id : quorum) {
+    auto i = peer_info.find(id);
+    ceph_assert(i != peer_info.end());
+    auto& info = i->second;
+    cluster_features &= info.cluster_features;
+    mon_features &= info.mon_features;
+    metadata[id] = info.metadata;
+    if (min_mon_release == ceph_release_t::unknown ||
+	info.mon_release < min_mon_release) {
+      min_mon_release = info.mon_release;
+    }
+  }
+
+  cancel_timer();
+  
+
+  // tell everyone!
+  for (set<int>::iterator p = quorum.begin();
+       p != quorum.end();
+       ++p) {
+    if (*p == mon->rank) continue;
+    MMonElection *m = new MMonElection(MMonElection::OP_VICTORY, get_epoch(),
+				       peer_tracker.get_encoded_bl(),
+				       logic.strategy, mon->monmap);
+    m->quorum = quorum;
+    m->quorum_features = cluster_features;
+    m->mon_features = mon_features;
+    m->sharing_bl = mon->get_local_commands_bl(mon_features);
+    m->mon_release = min_mon_release;
+    mon->send_mon_message(m, *p);
+  }
+
+  // tell monitor
+  mon->win_election(get_epoch(), quorum,
+                    cluster_features, mon_features, min_mon_release,
+		    metadata);
+}
+
+
+void Elector::handle_propose(MonOpRequestRef op)
+{
+  op->mark_event("elector:handle_propose");
+  auto m = op->get_req<MMonElection>();
+  dout(5) << "handle_propose from " << m->get_source() << dendl;
+  int from = m->get_source().num();
+
+  ceph_assert(m->epoch % 2 == 1); // election
+  uint64_t required_features = mon->get_required_features();
+  mon_feature_t required_mon_features = mon->get_required_mon_features();
+
+  dout(10) << __func__ << " required features " << required_features
+           << " " << required_mon_features
+           << ", peer features " << m->get_connection()->get_features()
+           << " " << m->mon_features
+           << dendl;
+
+  if ((required_features ^ m->get_connection()->get_features()) &
+      required_features) {
+    dout(5) << " ignoring propose from mon" << from
+	    << " without required features" << dendl;
+    nak_old_peer(op);
+    return;
+  } else if (mon->monmap->min_mon_release > m->mon_release) {
+    dout(5) << " ignoring propose from mon" << from
+	    << " release " << (int)m->mon_release
+	    << " < min_mon_release " << (int)mon->monmap->min_mon_release
+	    << dendl;
+    nak_old_peer(op);
+    return;
+  } else if (!m->mon_features.contains_all(required_mon_features)) {
+    // all the features in 'required_mon_features' not in 'm->mon_features'
+    mon_feature_t missing = required_mon_features.diff(m->mon_features);
+    dout(5) << " ignoring propose from mon." << from
+            << " without required mon_features " << missing
+            << dendl;
+    nak_old_peer(op);
+  }
+  ConnectionTracker *oct = NULL;
+  if (m->sharing_bl.length()) {
+    oct = new ConnectionTracker(m->sharing_bl, mon->cct);
+  }
+  logic.receive_propose(from, m->epoch, oct);
+  delete oct;
+}
+
+void Elector::handle_ack(MonOpRequestRef op)
+{
+  op->mark_event("elector:handle_ack");
+  auto m = op->get_req<MMonElection>();
+  dout(5) << "handle_ack from " << m->get_source() << dendl;
+  int from = m->get_source().num();
+
+  ceph_assert(m->epoch == get_epoch());
+  uint64_t required_features = mon->get_required_features();
+  if ((required_features ^ m->get_connection()->get_features()) &
+      required_features) {
+    dout(5) << " ignoring ack from mon" << from
+	    << " without required features" << dendl;
+    return;
+  }
+
+  mon_feature_t required_mon_features = mon->get_required_mon_features();
+  if (!m->mon_features.contains_all(required_mon_features)) {
+    mon_feature_t missing = required_mon_features.diff(m->mon_features);
+    dout(5) << " ignoring ack from mon." << from
+            << " without required mon_features " << missing
+            << dendl;
+    return;
+  }
+
+  if (logic.electing_me) {
+    // thanks
+    peer_info[from].cluster_features = m->get_connection()->get_features();
+    peer_info[from].mon_features = m->mon_features;
+    peer_info[from].mon_release = m->mon_release;
+    peer_info[from].metadata = m->metadata;
+    dout(5) << " so far i have {";
+    for (auto q = logic.acked_me.begin();
+         q != logic.acked_me.end();
+         ++q) {
+      auto p = peer_info.find(*q);
+      ceph_assert(p != peer_info.end());
+      if (q != logic.acked_me.begin())
+        *_dout << ",";
+      *_dout << " mon." << p->first << ":"
+             << " features " << p->second.cluster_features
+             << " " << p->second.mon_features;
+    }
+    *_dout << " }" << dendl;
+  }
+
+  logic.receive_ack(from, m->epoch);
+}
+
+void Elector::handle_victory(MonOpRequestRef op)
+{
+  op->mark_event("elector:handle_victory");
+  auto m = op->get_req<MMonElection>();
+  dout(5) << "handle_victory from " << m->get_source()
+          << " quorum_features " << m->quorum_features
+          << " " << m->mon_features
+          << dendl;
+  int from = m->get_source().num();
+
+  bool accept_victory = logic.receive_victory_claim(from, m->epoch);
+
+  if (!accept_victory) {
+    return;
+  }
+
+  mon->lose_election(get_epoch(), m->quorum, from,
+                     m->quorum_features, m->mon_features, m->mon_release);
+
+  // cancel my timer
+  cancel_timer();
+
+  // stash leader's commands
+  ceph_assert(m->sharing_bl.length());
+  vector<MonCommand> new_cmds;
+  auto bi = m->sharing_bl.cbegin();
+  MonCommand::decode_vector(new_cmds, bi);
+  mon->set_leader_commands(new_cmds);
+}
+
+void Elector::nak_old_peer(MonOpRequestRef op)
+{
+  op->mark_event("elector:nak_old_peer");
+  auto m = op->get_req<MMonElection>();
+  uint64_t supported_features = m->get_connection()->get_features();
+  uint64_t required_features = mon->get_required_features();
+  mon_feature_t required_mon_features = mon->get_required_mon_features();
+  dout(10) << "sending nak to peer " << m->get_source()
+	   << " supports " << supported_features << " " << m->mon_features
+	   << ", required " << required_features << " " << required_mon_features
+	   << ", release " << (int)m->mon_release
+	   << " vs required " << (int)mon->monmap->min_mon_release
+	   << dendl;
+  MMonElection *reply = new MMonElection(MMonElection::OP_NAK, m->epoch,
+                                         peer_tracker.get_encoded_bl(),
+					 logic.strategy, mon->monmap);
+  reply->quorum_features = required_features;
+  reply->mon_features = required_mon_features;
+  reply->mon_release = mon->monmap->min_mon_release;
+  mon->features.encode(reply->sharing_bl);
+  m->get_connection()->send_message(reply);
+}
+
+void Elector::handle_nak(MonOpRequestRef op)
+{
+  op->mark_event("elector:handle_nak");
+  auto m = op->get_req<MMonElection>();
+  dout(1) << "handle_nak from " << m->get_source()
+	  << " quorum_features " << m->quorum_features
+          << " " << m->mon_features
+	  << " min_mon_release " << (int)m->mon_release
+          << dendl;
+
+  if (m->mon_release > ceph_release()) {
+    derr << "Shutting down because I am release " << (int)ceph_release()
+	 << " < min_mon_release " << (int)m->mon_release << dendl;
+  } else {
+    CompatSet other;
+    auto bi = m->sharing_bl.cbegin();
+    other.decode(bi);
+    CompatSet diff = Monitor::get_supported_features().unsupported(other);
+
+    mon_feature_t mon_supported = ceph::features::mon::get_supported();
+    // all features in 'm->mon_features' not in 'mon_supported'
+    mon_feature_t mon_diff = m->mon_features.diff(mon_supported);
+
+    derr << "Shutting down because I lack required monitor features: { "
+	 << diff << " } " << mon_diff << dendl;
+  }
+  exit(0);
+  // the end!
+}
+
+void Elector::begin_peer_ping(int peer)
+{
+  dout(20) << __func__ << " against " << peer << dendl;
+  if (live_pinging.count(peer)) {
+    dout(20) << peer << " already in live_pinging ... return " << dendl;
+    return;
+  }
+
+  if (!mon->get_quorum_mon_features().contains_all(
+				      ceph::features::mon::FEATURE_PINGING)) {
+    return;
+  }
+
+  peer_tracker.report_live_connection(peer, 0); // init this peer as existing
+  live_pinging.insert(peer);
+  dead_pinging.erase(peer);
+  peer_acked_ping[peer] = ceph_clock_now();
+  if (!send_peer_ping(peer)) return;
+  mon->timer.add_event_after(ping_timeout / PING_DIVISOR,
+			     new C_MonContext{mon, [this, peer](int) {
+				 ping_check(peer);
+			       }});
+}
+
+bool Elector::send_peer_ping(int peer, const utime_t *n)
+{
+  dout(10) << __func__ << " to peer " << peer << dendl;
+  if (peer >= mon->monmap->ranks.size()) {
+    // Monitor no longer exists in the monmap,
+    // therefore, we shouldn't ping this monitor
+    // since we cannot lookup the address!
+    dout(5) << "peer: " << peer << " >= ranks_size: "
+      << mon->monmap->ranks.size() << " ... dropping to prevent "
+      << "https://tracker.ceph.com/issues/50089" << dendl;
+    live_pinging.erase(peer);
+    return false;
+  }
+  utime_t now;
+  if (n != NULL) {
+    now = *n;
+  } else {
+    now = ceph_clock_now();
+  }
+  MMonPing *ping = new MMonPing(MMonPing::PING, now, peer_tracker.get_encoded_bl());
+  mon->messenger->send_to_mon(ping, mon->monmap->get_addrs(peer));
+  peer_sent_ping[peer] = now;
+  return true;
+}
+
+void Elector::ping_check(int peer)
+{
+  dout(20) << __func__ << " to peer " << peer << dendl;
+
+  if (!live_pinging.count(peer) &&
+      !dead_pinging.count(peer)) {
+    dout(20) << __func__ << peer << " is no longer marked for pinging" << dendl;
+    return;
+  }
+  utime_t now = ceph_clock_now();
+  utime_t& acked_ping = peer_acked_ping[peer];
+  utime_t& newest_ping = peer_sent_ping[peer];
+  if (!acked_ping.is_zero() && acked_ping < now - ping_timeout) {
+    peer_tracker.report_dead_connection(peer, now - acked_ping);
+    acked_ping = now;
+    begin_dead_ping(peer);
+    return;
+  }
+
+  if (acked_ping == newest_ping) {
+    if (!send_peer_ping(peer, &now)) return;
+  }
+
+  mon->timer.add_event_after(ping_timeout / PING_DIVISOR,
+			     new C_MonContext{mon, [this, peer](int) {
+				 ping_check(peer);
+			       }});
+}
+
+void Elector::begin_dead_ping(int peer)
+{
+  dout(20) << __func__ << " to peer " << peer << dendl;  
+  if (dead_pinging.count(peer)) {
+    return;
+  }
+  
+  live_pinging.erase(peer);
+  dead_pinging.insert(peer);
+  mon->timer.add_event_after(ping_timeout,
+			     new C_MonContext{mon, [this, peer](int) {
+				 dead_ping(peer);
+			       }});
+}
+
+void Elector::dead_ping(int peer)
+{
+  dout(20) << __func__ << " to peer " << peer << dendl;
+  if (!dead_pinging.count(peer)) {
+    dout(20) << __func__ << peer << " is no longer marked for dead pinging" << dendl;
+    return;
+  }
+  ceph_assert(!live_pinging.count(peer));
+
+  utime_t now = ceph_clock_now();
+  utime_t& acked_ping = peer_acked_ping[peer];
+
+  peer_tracker.report_dead_connection(peer, now - acked_ping);
+  acked_ping = now;
+  mon->timer.add_event_after(ping_timeout,
+			       new C_MonContext{mon, [this, peer](int) {
+				   dead_ping(peer);
+				 }});
+}
+
+void Elector::handle_ping(MonOpRequestRef op)
+{
+  MMonPing *m = static_cast<MMonPing*>(op->get_req());
+  int prank = mon->monmap->get_rank(m->get_source_addr());
+  dout(20) << __func__ << " from: " << prank << dendl;
+  begin_peer_ping(prank);
+  assimilate_connection_reports(m->tracker_bl);
+  switch(m->op) {
+  case MMonPing::PING:
+    {
+      MMonPing *reply = new MMonPing(MMonPing::PING_REPLY, m->stamp, peer_tracker.get_encoded_bl());
+      m->get_connection()->send_message(reply);
+    }
+    break;
+
+  case MMonPing::PING_REPLY:
+
+    const utime_t& previous_acked = peer_acked_ping[prank];
+    const utime_t& newest = peer_sent_ping[prank];
+
+    if (m->stamp > newest && !newest.is_zero()) {
+      derr << "dropping PING_REPLY stamp " << m->stamp
+	   << " as it is newer than newest sent " << newest << dendl;
+      return;
+    }
+
+    if (m->stamp > previous_acked) {
+      dout(20) << "m->stamp > previous_acked" << dendl;
+      peer_tracker.report_live_connection(prank, m->stamp - previous_acked);
+      peer_acked_ping[prank] = m->stamp;
+    } else{
+      dout(20) << "m->stamp <= previous_acked .. we don't report_live_connection" << dendl;
+    }
+    utime_t now = ceph_clock_now();
+    dout(30) << "now: " << now << " m->stamp: " << m->stamp << " ping_timeout: "
+      << ping_timeout << " PING_DIVISOR: " << PING_DIVISOR << dendl;
+    if (now - m->stamp > ping_timeout / PING_DIVISOR) {
+      if (!send_peer_ping(prank, &now)) return;
+    }
+    break;
+  }
+}
+
+void Elector::dispatch(MonOpRequestRef op)
+{
+  op->mark_event("elector:dispatch");
+  ceph_assert(op->is_type_election_or_ping());
+
+  switch (op->get_req()->get_type()) {
+    
+  case MSG_MON_ELECTION:
+    {
+      if (!logic.participating) {
+        return;
+      }
+      if (op->get_req()->get_source().num() >= mon->monmap->size()) {
+	dout(5) << " ignoring bogus election message with bad mon rank " 
+		<< op->get_req()->get_source() << dendl;
+	return;
+      }
+
+      auto em = op->get_req<MMonElection>();
+      dout(20) << __func__ << " from: " << mon->monmap->get_rank(em->get_source_addr()) << dendl;
+      // assume an old message encoding would have matched
+      if (em->fsid != mon->monmap->fsid) {
+	dout(0) << " ignoring election msg fsid " 
+		<< em->fsid << " != " << mon->monmap->fsid << dendl;
+	return;
+      }
+
+      if (!mon->monmap->contains(em->get_source_addr())) {
+	dout(1) << "discarding election message: " << em->get_source_addr()
+		<< " not in my monmap " << *mon->monmap << dendl;
+	return;
+      }
+
+      MonMap peermap;
+      peermap.decode(em->monmap_bl);
+      if (peermap.epoch > mon->monmap->epoch) {
+	dout(0) << em->get_source_inst() << " has newer monmap epoch " << peermap.epoch
+		<< " > my epoch " << mon->monmap->epoch 
+		<< ", taking it"
+		<< dendl;
+	mon->monmap->decode(em->monmap_bl);
+        auto t(std::make_shared<MonitorDBStore::Transaction>());
+        t->put("monmap", mon->monmap->epoch, em->monmap_bl);
+        t->put("monmap", "last_committed", mon->monmap->epoch);
+        mon->store->apply_transaction(t);
+	//mon->monmon()->paxos->stash_latest(mon->monmap->epoch, em->monmap_bl);
+	cancel_timer();
+	mon->notify_new_monmap(false);
+	mon->bootstrap();
+	return;
+      }
+      if (peermap.epoch < mon->monmap->epoch) {
+	dout(0) << em->get_source_inst() << " has older monmap epoch " << peermap.epoch
+		<< " < my epoch " << mon->monmap->epoch 
+		<< dendl;
+      }
+
+      if (em->strategy != logic.strategy) {
+	dout(5) << __func__ << " somehow got an Election message with different strategy "
+		<< em->strategy << " from local " << logic.strategy
+		<< "; dropping for now to let race resolve" << dendl;
+	return;
+      }
+
+      if (em->scoring_bl.length()) {
+	assimilate_connection_reports(em->scoring_bl);
+      }
+
+      begin_peer_ping(mon->monmap->get_rank(em->get_source_addr()));
+      switch (em->op) {
+      case MMonElection::OP_PROPOSE:
+	handle_propose(op);
+	return;
+      }
+
+      if (em->epoch < get_epoch()) {
+	dout(5) << "old epoch, dropping" << dendl;
+	break;
+      }
+
+      switch (em->op) {
+      case MMonElection::OP_ACK:
+	handle_ack(op);
+	return;
+      case MMonElection::OP_VICTORY:
+	handle_victory(op);
+	return;
+      case MMonElection::OP_NAK:
+	handle_nak(op);
+	return;
+      default:
+	ceph_abort();
+      }
+    }
+    break;
+
+  case MSG_MON_PING:
+    handle_ping(op);
+    break;
+    
+  default: 
+    ceph_abort();
+  }
+}
+
+void Elector::start_participating()
+{
+  logic.participating = true;
+}
+
+bool Elector::peer_tracker_is_clean()
+{
+  return peer_tracker.is_clean(mon->rank, paxos_size());
+}
+
+void Elector::notify_clear_peer_state()
+{
+  dout(10) << __func__ << dendl;
+  dout(20) << " peer_tracker before: " << peer_tracker << dendl;
+  peer_tracker.notify_reset();
+  peer_tracker.set_rank(mon->rank);
+  dout(20) << " peer_tracker after: " << peer_tracker << dendl;
+}
+
+void Elector::notify_rank_changed(int new_rank)
+{
+  dout(10) << __func__ << " to " << new_rank << dendl; 
+  peer_tracker.notify_rank_changed(new_rank);
+  live_pinging.erase(new_rank);
+  dead_pinging.erase(new_rank);
+}
+
+void Elector::notify_rank_removed(int rank_removed, int new_rank)
+{
+  dout(10) << __func__ << ": " << rank_removed << dendl; 
+  peer_tracker.notify_rank_removed(rank_removed, new_rank);
+  /* we have to clean up the pinging state, which is annoying
+     because it's not indexed anywhere (and adding indexing
+     would also be annoying).
+     In the case where we are removing any rank that is not the
+     higest, we start with the removed rank and examine the state
+     of the surrounding ranks.
+     Everybody who remains with larger rank gets a new rank one lower
+     than before, and we have to figure out the remaining scheduled
+     ping contexts. So, starting one past with the removed rank, we:
+     * check if the current rank is alive or dead
+     * examine our new rank (one less than before, initially the removed
+     rank)
+     * * erase it if it's in the wrong set
+     * * start pinging it if we're not already
+     * check if the next rank is in the same pinging set, and delete
+     * ourselves if not.
+     In the case where we are removing the highest rank,
+     we erase the removed rank from all sets.
+   */
+  if (rank_removed < paxos_size()) {
+    for (unsigned i = rank_removed + 1; i <= paxos_size() ; ++i) {
+      if (live_pinging.count(i)) {
+        dead_pinging.erase(i-1);
+        if (!live_pinging.count(i-1)) {
+	  begin_peer_ping(i-1);
+        }
+        if (!live_pinging.count(i+1)) {
+	  live_pinging.erase(i);
+        }
+      }
+      else if (dead_pinging.count(i)) {
+        live_pinging.erase(i-1);
+        if (!dead_pinging.count(i-1)) {
+	  begin_dead_ping(i-1);
+        }
+        if (!dead_pinging.count(i+1)) {
+	  dead_pinging.erase(i);
+        }
+      } else {
+        // we aren't pinging rank i at all
+        if (i-1 == (unsigned)rank_removed) {
+	  // so we special case to make sure we
+	  // actually nuke the removed rank
+	  dead_pinging.erase(rank_removed);
+	  live_pinging.erase(rank_removed);
+        }
+      }
+     }
+   } else {
+     if (live_pinging.count(rank_removed)) {
+       live_pinging.erase(rank_removed);
+     }
+     if (dead_pinging.count(rank_removed)) {
+       dead_pinging.erase(rank_removed);
+     }
+   }
+}
+
+void Elector::notify_strategy_maybe_changed(int strategy)
+{
+  logic.set_election_strategy(static_cast<ElectionLogic::election_strategy>(strategy));
+}
diff --git a/src/mon/Elector.h b/src/mon/Elector.h
new file mode 100644
index 000000000..2a53c1fc4
--- /dev/null
+++ b/src/mon/Elector.h
@@ -0,0 +1,406 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef CEPH_MON_ELECTOR_H
+#define CEPH_MON_ELECTOR_H
+
+#include <map>
+
+#include "include/types.h"
+#include "include/Context.h"
+#include "mon/MonOpRequest.h"
+#include "mon/mon_types.h"
+#include "mon/ElectionLogic.h"
+#include "mon/ConnectionTracker.h"
+
+class Monitor;
+
+
+/**
+ * This class is responsible for handling messages and maintaining
+ * an ElectionLogic which holds the local state when electing
+ * a new Leader. We may win or we may lose. If we win, it means we became the
+ * Leader; if we lose, it means we are a Peon.
+ */
+class Elector : public ElectionOwner, RankProvider {
+  /**
+   * @defgroup Elector_h_class Elector
+   * @{
+   */
+  ElectionLogic logic;
+  // connectivity validation and scoring
+  ConnectionTracker peer_tracker;
+  map<int, utime_t> peer_acked_ping; // rank -> last ping stamp they acked
+  map<int, utime_t> peer_sent_ping; // rank -> last ping stamp we sent
+  set<int> live_pinging; // ranks which we are currently pinging
+  set<int> dead_pinging; // ranks which didn't answer (degrading scores)
+  double ping_timeout; // the timeout after which we consider a ping to be dead
+  int PING_DIVISOR = 2;  // we time out pings
+
+   /**
+   * @defgroup Elector_h_internal_types Internal Types
+   * @{
+   */
+  /**
+   * This struct will hold the features from a given peer.
+   * Features may both be the cluster's (in the form of a uint64_t), or
+   * mon-specific features. Instead of keeping maps to hold them both, or
+   * a pair, which would be weird, a struct to keep them seems appropriate.
+   */
+  struct elector_info_t {
+    uint64_t cluster_features = 0;
+    mon_feature_t mon_features;
+    ceph_release_t mon_release{0};
+    std::map<std::string,std::string> metadata;
+  };
+
+  /**
+   * @}
+   */
+
+  /**
+   * The Monitor instance associated with this class.
+   */
+  Monitor *mon;
+
+  /**
+   * Event callback responsible for dealing with an expired election once a
+   * timer runs out and fires up.
+   */
+  Context *expire_event = nullptr;
+
+  /**
+   * Resets the expire_event timer, by cancelling any existing one and
+   * scheduling a new one.
+   *
+   * @remarks This function assumes as a default firing value the duration of
+   *	      the monitor's lease interval, and adds to it the value specified
+   *	      in @e plus
+   *
+   * @post expire_event is set
+   *
+   * @param plus The amount of time to be added to the default firing value.
+   */
+  void reset_timer(double plus=0.0);
+  /**
+   * Cancel the expire_event timer, if it is defined.
+   *
+   * @post expire_event is not set
+   */
+  void cancel_timer();
+
+  // electing me
+  /**
+   * @defgroup Elector_h_electing_me_vars We are being elected
+   * @{
+   */
+  /**
+   * Map containing info of all those that acked our proposal to become the Leader.
+   * Note each peer's info.
+   */
+  std::map<int, elector_info_t> peer_info;
+  /**
+   * @}
+   */
+ 
+  /**
+   * Handle a message from some other node proposing itself to become it
+   * the Leader.
+   *
+   * We validate that the sending Monitor is allowed to participate based on
+   * its supported features, then pass the request to our ElectionLogic.
+   *
+   * @invariant The received message is an operation of type OP_PROPOSE
+   *
+   * @pre   Message epoch is from the current or a newer epoch
+   * 
+   * @param m A message sent by another participant in the quorum.
+   */
+  void handle_propose(MonOpRequestRef op);
+  /**
+   * Handle a message from some other participant Acking us as the Leader.
+   *
+   * We validate that the sending Monitor is allowed to participate based on
+   * its supported features, add it to peer_info, and pass the ack to our
+   * ElectionLogic.
+   *
+   * @pre   Message epoch is from the current or a newer epoch
+   *
+   * @param m A message with an operation type of OP_ACK
+   */
+  void handle_ack(MonOpRequestRef op);
+  /**
+   * Handle a message from some other participant declaring Victory.
+   *
+   * We just got a message from someone declaring themselves Victorious, thus
+   * the new Leader.
+   *
+   * We pass the Victory to our ElectionLogic, and if it confirms the
+   * victory we lose the election and start following this Leader. Otherwise,
+   * drop the message.
+   *
+   * @pre   Message epoch is from the current or a newer epoch
+   * @post  Election is not on-going
+   * @post  Updated @p epoch
+   * @post  We have a new quorum if we lost the election
+   *
+   * @param m A message with an operation type of OP_VICTORY
+   */
+  void handle_victory(MonOpRequestRef op);
+  /**
+   * Send a nak to a peer who's out of date, containing information about why.
+   *
+   * If we get a message from a peer who can't support the required quorum
+   * features, we have to ignore them. This function will at least send
+   * them a message about *why* they're being ignored -- if they're new
+   * enough to support such a message.
+   *
+   * @param m A message from a monitor not supporting required features. We
+   * take ownership of the reference.
+   */
+  void nak_old_peer(MonOpRequestRef op);
+  /**
+   * Handle a message from some other participant declaring
+   * we cannot join the quorum.
+   *
+   * Apparently the quorum requires some feature that we do not implement. Shut
+   * down gracefully.
+   *
+   * @pre Election is on-going.
+   * @post We've shut down.
+   *
+   * @param m A message with an operation type of OP_NAK
+   */
+  void handle_nak(MonOpRequestRef op);
+  /**
+   * Send a ping to the specified peer.
+   * @n optional time that we will use instead of calling ceph_clock_now()
+   */
+  bool send_peer_ping(int peer, const utime_t *n=NULL);
+  /**
+   * Check the state of pinging the specified peer. This is our
+   * "tick" for heartbeating; scheduled by itself and begin_peer_ping().
+   */
+  void ping_check(int peer);
+  /**
+   * Move the peer out of live_pinging into dead_pinging set
+   * and schedule dead_ping()ing on it.
+   */
+  void begin_dead_ping(int peer);
+  /**
+   * Checks that the peer is still marked for dead pinging,
+   * and then marks it as dead for the appropriate interval.
+   */
+  void dead_ping(int peer);
+  /**
+   * Handle a ping from another monitor and assimilate the data it contains.
+   */
+  void handle_ping(MonOpRequestRef op);
+  /**
+   * Update our view of everybody else's connectivity based on the provided
+   * tracker bufferlist
+   */
+  void assimilate_connection_reports(const bufferlist& bl);
+  
+ public:
+  /**
+   * @defgroup Elector_h_ElectionOwner Functions from the ElectionOwner interface
+   * @{
+   */
+  /* Commit the given epoch to our MonStore.
+   * We also take the opportunity to persist our peer_tracker.
+   */
+  void persist_epoch(epoch_t e);
+  /* Read the epoch out of our MonStore */
+  epoch_t read_persisted_epoch() const;
+  /* Write a nonsense key "election_writeable_test" to our MonStore */
+  void validate_store();
+  /* Reset my tracking. Currently, just call Monitor::join_election() */
+  void notify_bump_epoch();
+  /* Call a new election: Invoke Monitor::start_election() */
+  void trigger_new_election();
+  /* Retrieve rank from the Monitor */
+  int get_my_rank() const;
+  /* Send MMonElection OP_PROPOSE to every monitor in the map. */
+  void propose_to_peers(epoch_t e, bufferlist &bl);
+  /* bootstrap() the Monitor */
+  void reset_election();
+  /* Retrieve the Monitor::has_ever_joined member */
+  bool ever_participated() const;
+  /* Retrieve monmap->size() */
+  unsigned paxos_size() const;
+  /* Right now we don't disallow anybody */
+  set<int> disallowed_leaders;
+  const set<int>& get_disallowed_leaders() const { return disallowed_leaders; }
+  /**
+   * Reset the expire_event timer so we can limit the amount of time we 
+   * will be electing. Clean up our peer_info.
+   *
+   * @post  we reset the expire_event timer
+   */
+  void _start();
+  /**
+   * Send an MMonElection message deferring to the identified monitor. We
+   * also increase the election timeout so the monitor we defer to
+   * has some time to gather deferrals and actually win. (FIXME: necessary to protocol?)
+   *
+   * @post  we sent an ack message to @p who
+   * @post  we reset the expire_event timer
+   *
+   * @param who Some other monitor's numeric identifier. 
+   */
+  void _defer_to(int who);
+  /**
+   * Our ElectionLogic told us we won an election! Identify the quorum
+   * features, tell our new peons we've won, and invoke Monitor::win_election().
+   */
+  void message_victory(const std::set<int>& quorum);
+  /* Check if rank is in mon->quorum */
+  bool is_current_member(int rank) const;
+  /*
+   * @}
+   */
+  /**
+   * Persist our peer_tracker to disk.
+   */
+  void persist_connectivity_scores();
+
+  Elector *elector;
+  
+  /**
+   * Create an Elector class
+   *
+   * @param m A Monitor instance
+   * @param strategy The election strategy to use, defined in MonMap/ElectionLogic
+   */
+  explicit Elector(Monitor *m, int strategy);
+  virtual ~Elector() {}
+
+  /**
+   * Inform this class it is supposed to shutdown.
+   *
+   * We will simply cancel the @p expire_event if any exists.
+   *
+   * @post @p expire_event is cancelled 
+   */
+  void shutdown();
+
+  /**
+   * Obtain our epoch from ElectionLogic.
+   *
+   * @returns Our current epoch number
+   */
+  epoch_t get_epoch() { return logic.get_epoch(); }
+
+  /**
+   * If the Monitor knows there are no Paxos peers (so
+   * we are rank 0 and there are no others) we can declare victory.
+   */
+  void declare_standalone_victory() {
+    logic.declare_standalone_victory();
+  }
+  /**
+   * Tell the Elector to start pinging a given peer.
+   * Do this when you discover a peer and it has a rank assigned.
+   * We do it ourselves on receipt of pings and when receiving other messages.
+   */
+  void begin_peer_ping(int peer);
+  /**
+   * Handle received messages.
+   *
+   * We will ignore all messages that are not of type @p MSG_MON_ELECTION
+   * (i.e., messages whose interface is not of type @p MMonElection). All of
+   * those that are will then be dispatched to their operation-specific
+   * functions.
+   *
+   * @param m A received message
+   */
+  void dispatch(MonOpRequestRef op);
+
+  /**
+   * Call an election.
+   *
+   * This function simply calls ElectionLogic::start.
+   */
+  void call_election() {
+    logic.start();
+  }
+
+  /**
+   * Stop participating in subsequent Elections.
+   *
+   * @post @p participating is false
+   */
+  void stop_participating() { logic.participating = false; }
+  /**
+   * Start participating in Elections.
+   *
+   * If we are already participating (i.e., @p participating is true), then
+   * calling this function is moot.
+   *
+   * However, if we are not participating (i.e., @p participating is false),
+   * then we will start participating by setting @p participating to true and
+   * we will call for an Election.
+   *
+   * @post  @p participating is true
+   */
+  void start_participating();
+  /**
+  * Check if our peer_tracker is self-consistent, not suffering from
+  * https://tracker.ceph.com/issues/58049
+  */
+  bool peer_tracker_is_clean();
+  /**
+   * Forget everything about our peers. :(
+   */
+  void notify_clear_peer_state();
+  /**
+   * Notify that our local rank has changed
+   * and we may need to update internal data structures.
+   */
+  void notify_rank_changed(int new_rank);
+  /**
+   * A peer has been removed so we should clean up state related to it.
+   * This is safe to call even if we haven't joined or are currently
+   * in a quorum.
+   */
+  void notify_rank_removed(int rank_removed, int new_rank);
+  void notify_strategy_maybe_changed(int strategy);
+  /**
+   * Set the disallowed leaders.
+   *
+   * If you call this and the new disallowed set
+   * contains your current leader, you are
+   * responsible for calling an election!
+   *
+   * @returns false if the set is unchanged,
+   *   true if the set changed
+   */
+  bool set_disallowed_leaders(const set<int>& dl) {
+    if (dl == disallowed_leaders) return false;
+    disallowed_leaders = dl;
+    return true;
+  }
+  void dump_connection_scores(Formatter *f) {
+    f->open_object_section("connection scores");
+    peer_tracker.dump(f);
+    f->close_section();
+  }
+  /**
+   * @}
+   */
+};
+
+#endif
diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc
new file mode 100644
index 000000000..0b1bb2a03
--- /dev/null
+++ b/src/mon/FSCommands.cc
@@ -0,0 +1,1516 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2017 Red Hat Ltd
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#include "OSDMonitor.h"
+
+#include "FSCommands.h"
+#include "MDSMonitor.h"
+#include "MgrStatMonitor.h"
+#include "mds/cephfs_features.h"
+
+using TOPNSPC::common::cmd_getval;
+
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::pair;
+using std::set;
+using std::string;
+using std::to_string;
+using std::vector;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::ErasureCodeInterfaceRef;
+using ceph::ErasureCodeProfile;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::make_message;
+using ceph::mono_clock;
+using ceph::mono_time;
+
+class FlagSetHandler : public FileSystemCommandHandler
+{
+  public:
+  FlagSetHandler()
+    : FileSystemCommandHandler("fs flag set")
+  {
+  }
+
+  int handle(
+      Monitor *mon,
+      FSMap& fsmap,
+      MonOpRequestRef op,
+      const cmdmap_t& cmdmap,
+      std::ostream &ss) override
+  {
+    string flag_name;
+    cmd_getval(cmdmap, "flag_name", flag_name);
+
+    string flag_val;
+    cmd_getval(cmdmap, "val", flag_val);
+
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+
+    if (flag_name == "enable_multiple") {
+      bool flag_bool = false;
+      int r = parse_bool(flag_val, &flag_bool, ss);
+      if (r != 0) {
+        ss << "Invalid boolean value '" << flag_val << "'";
+        return r;
+      }
+
+      fsmap.set_enable_multiple(flag_bool);
+      return 0;
+    } else {
+      ss << "Unknown flag '" << flag_name << "'";
+      return -EINVAL;
+    }
+  }
+};
+
+class FailHandler : public FileSystemCommandHandler
+{
+  public:
+  FailHandler()
+    : FileSystemCommandHandler("fs fail")
+  {
+  }
+
+  int handle(
+      Monitor* mon,
+      FSMap& fsmap,
+      MonOpRequestRef op,
+      const cmdmap_t& cmdmap,
+      std::ostream& ss) override
+  {
+    if (!mon->osdmon()->is_writeable()) {
+      // not allowed to write yet, so retry when we can
+      mon->osdmon()->wait_for_writeable(op, new PaxosService::C_RetryMessage(mon->mdsmon(), op));
+      return -EAGAIN;
+    }
+
+    std::string fs_name;
+    if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+      ss << "Missing filesystem name";
+      return -EINVAL;
+    }
+
+    auto fs = fsmap.get_filesystem(fs_name);
+
+    auto f = [](auto fs) {
+      fs->mds_map.set_flag(CEPH_MDSMAP_NOT_JOINABLE);
+    };
+    fsmap.modify_filesystem(fs->fscid, std::move(f));
+
+    std::vector<mds_gid_t> to_fail;
+    for (const auto& p : fs->mds_map.get_mds_info()) {
+      to_fail.push_back(p.first);
+    }
+
+    for (const auto& gid : to_fail) {
+      mon->mdsmon()->fail_mds_gid(fsmap, gid);
+    }
+    if (!to_fail.empty()) {
+      mon->osdmon()->propose_pending();
+    }
+
+    ss << fs_name;
+    ss << " marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed.";
+
+    return 0;
+  }
+};
+
+class FsNewHandler : public FileSystemCommandHandler
+{
+  public:
+  explicit FsNewHandler(Paxos *paxos)
+    : FileSystemCommandHandler("fs new"), m_paxos(paxos)
+  {
+  }
+
+  bool batched_propose() override {
+    return true;
+  }
+
+  int handle(
+      Monitor *mon,
+      FSMap& fsmap,
+      MonOpRequestRef op,
+      const cmdmap_t& cmdmap,
+      std::ostream &ss) override
+  {
+    ceph_assert(m_paxos->is_plugged());
+
+    string metadata_name;
+    cmd_getval(cmdmap, "metadata", metadata_name);
+    int64_t metadata = mon->osdmon()->osdmap.lookup_pg_pool_name(metadata_name);
+    if (metadata < 0) {
+      ss << "pool '" << metadata_name << "' does not exist";
+      return -ENOENT;
+    }
+
+    string data_name;
+    cmd_getval(cmdmap, "data", data_name);
+    int64_t data = mon->osdmon()->osdmap.lookup_pg_pool_name(data_name);
+    if (data < 0) {
+      ss << "pool '" << data_name << "' does not exist";
+      return -ENOENT;
+    }
+    if (data == 0) {
+      ss << "pool '" << data_name << "' has id 0, which CephFS does not allow. Use another pool or recreate it to get a non-zero pool id.";
+      return -EINVAL;
+    }
+
+    string fs_name;
+    cmd_getval(cmdmap, "fs_name", fs_name);
+    if (fs_name.empty()) {
+        // Ensure fs name is not empty so that we can implement
+        // commmands that refer to FS by name in future.
+        ss << "Filesystem name may not be empty";
+        return -EINVAL;
+    }
+
+    if (fsmap.get_filesystem(fs_name)) {
+      auto fs = fsmap.get_filesystem(fs_name);
+      if (*(fs->mds_map.get_data_pools().begin()) == data
+          && fs->mds_map.get_metadata_pool() == metadata) {
+        // Identical FS created already, this is a no-op
+        ss << "filesystem '" << fs_name << "' already exists";
+        return 0;
+      } else {
+        ss << "filesystem already exists with name '" << fs_name << "'";
+        return -EINVAL;
+      }
+    }
+
+    bool force = false;
+    cmd_getval(cmdmap, "force", force);
+
+    const pool_stat_t *stat = mon->mgrstatmon()->get_pool_stat(metadata);
+    if (stat) {
+      int64_t metadata_num_objects = stat->stats.sum.num_objects;
+      if (!force && metadata_num_objects > 0) {
+	ss << "pool '" << metadata_name
+	   << "' already contains some objects. Use an empty pool instead.";
+	return -EINVAL;
+      }
+    }
+
+    if (fsmap.filesystem_count() > 0
+        && !fsmap.get_enable_multiple()) {
+      ss << "Creation of multiple filesystems is disabled.  To enable "
+            "this experimental feature, use 'ceph fs flag set enable_multiple "
+            "true'";
+      return -EINVAL;
+    }
+
+    for (auto& fs : fsmap.get_filesystems()) {
+      const std::vector<int64_t> &data_pools = fs->mds_map.get_data_pools();
+
+      bool sure = false;
+      cmd_getval(cmdmap,
+                 "allow_dangerous_metadata_overlay", sure);
+
+      if ((std::find(data_pools.begin(), data_pools.end(), data) != data_pools.end()
+	   || fs->mds_map.get_metadata_pool() == metadata)
+	  && !sure) {
+	ss << "Filesystem '" << fs_name
+	   << "' is already using one of the specified RADOS pools. This should ONLY be done in emergencies and after careful reading of the documentation. Pass --allow-dangerous-metadata-overlay to permit this.";
+	return -EEXIST;
+      }
+    }
+
+    int64_t fscid = FS_CLUSTER_ID_NONE;
+    if (cmd_getval(cmdmap, "fscid", fscid)) {
+      if (!force) {
+        ss << "Pass --force to create a file system with a specific ID";
+        return -EINVAL;
+      }
+      if (fsmap.filesystem_exists(fscid)) {
+        ss << "filesystem already exists with id '" << fscid << "'";
+        return -EINVAL;
+      }
+    }
+
+    pg_pool_t const *data_pool = mon->osdmon()->osdmap.get_pg_pool(data);
+    ceph_assert(data_pool != NULL);  // Checked it existed above
+    pg_pool_t const *metadata_pool = mon->osdmon()->osdmap.get_pg_pool(metadata);
+    ceph_assert(metadata_pool != NULL);  // Checked it existed above
+
+    int r = _check_pool(mon->osdmon()->osdmap, data, POOL_DATA_DEFAULT, force, &ss);
+    if (r < 0) {
+      return r;
+    }
+
+    r = _check_pool(mon->osdmon()->osdmap, metadata, POOL_METADATA, force, &ss);
+    if (r < 0) {
+      return r;
+    }
+    
+    if (!mon->osdmon()->is_writeable()) {
+      // not allowed to write yet, so retry when we can
+      mon->osdmon()->wait_for_writeable(op, new PaxosService::C_RetryMessage(mon->mdsmon(), op));
+      return -EAGAIN;
+    }
+    mon->osdmon()->do_application_enable(data,
+					 pg_pool_t::APPLICATION_NAME_CEPHFS,
+					 "data", fs_name, true);
+    mon->osdmon()->do_application_enable(metadata,
+					 pg_pool_t::APPLICATION_NAME_CEPHFS,
+					 "metadata", fs_name, true);
+    mon->osdmon()->do_set_pool_opt(metadata,
+				   pool_opts_t::RECOVERY_PRIORITY,
+				   static_cast<int64_t>(5));
+    mon->osdmon()->do_set_pool_opt(metadata,
+				   pool_opts_t::PG_NUM_MIN,
+				   static_cast<int64_t>(16));
+    mon->osdmon()->do_set_pool_opt(metadata,
+				   pool_opts_t::PG_AUTOSCALE_BIAS,
+				   static_cast<double>(4.0));
+    mon->osdmon()->propose_pending();
+
+    bool recover = false;
+    cmd_getval(cmdmap, "recover", recover);
+
+    // All checks passed, go ahead and create.
+    auto&& fs = fsmap.create_filesystem(fs_name, metadata, data,
+        mon->get_quorum_con_features(), fscid, recover);
+
+    ss << "new fs with metadata pool " << metadata << " and data pool " << data;
+
+    if (recover) {
+      return 0;
+    }
+
+    // assign a standby to rank 0 to avoid health warnings
+    auto info = fsmap.find_replacement_for({fs->fscid, 0});
+
+    if (info) {
+      mon->clog->info() << info->human_name() << " assigned to filesystem "
+          << fs_name << " as rank 0";
+      fsmap.promote(info->global_id, *fs, 0);
+    }
+
+    return 0;
+  }
+
+private:
+  Paxos *m_paxos;
+};
+
+class SetHandler : public FileSystemCommandHandler
+{
+public:
+  SetHandler()
+    : FileSystemCommandHandler("fs set")
+  {}
+
+  int handle(
+      Monitor *mon,
+      FSMap& fsmap,
+      MonOpRequestRef op,
+      const cmdmap_t& cmdmap,
+      std::ostream &ss) override
+  {
+    std::string fs_name;
+    if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+      ss << "Missing filesystem name";
+      return -EINVAL;
+    }
+
+    auto fs = fsmap.get_filesystem(fs_name);
+    string var;
+    if (!cmd_getval(cmdmap, "var", var) || var.empty()) {
+      ss << "Invalid variable";
+      return -EINVAL;
+    }
+    string val;
+    string interr;
+    int64_t n = 0;
+    if (!cmd_getval(cmdmap, "val", val)) {
+      return -EINVAL;
+    }
+    // we got a string.  see if it contains an int.
+    n = strict_strtoll(val.c_str(), 10, &interr);
+    if (var == "max_mds") {
+      // NOTE: see also "mds set_max_mds", which can modify the same field.
+      if (interr.length()) {
+        ss << interr;
+	return -EINVAL;
+      }
+
+      if (n <= 0) {
+        ss << "You must specify at least one MDS";
+        return -EINVAL;
+      }
+
+      if (n > 1 && n > fs->mds_map.get_max_mds()) {
+	if (fs->mds_map.was_snaps_ever_allowed() &&
+	    !fs->mds_map.allows_multimds_snaps()) {
+	  ss << "multi-active MDS is not allowed while there are snapshots possibly created by pre-mimic MDS";
+	  return -EINVAL;
+	}
+      }
+      if (n > MAX_MDS) {
+        ss << "may not have more than " << MAX_MDS << " MDS ranks";
+        return -EINVAL;
+      }
+
+      fsmap.modify_filesystem(
+          fs->fscid,
+          [n](std::shared_ptr<Filesystem> fs)
+      {
+	fs->mds_map.clear_flag(CEPH_MDSMAP_NOT_JOINABLE);
+        fs->mds_map.set_max_mds(n);
+      });
+    } else if (var == "inline_data") {
+      bool enable_inline = false;
+      int r = parse_bool(val, &enable_inline, ss);
+      if (r != 0) {
+        return r;
+      }
+
+      if (enable_inline) {
+        bool confirm = false;
+        cmd_getval(cmdmap, "yes_i_really_really_mean_it", confirm);
+	if (!confirm) {
+	  ss << "Inline data support is deprecated and will be removed in a future release. "
+	     << "Add --yes-i-really-really-mean-it if you are certain you want this enabled.";
+	  return -EPERM;
+	}
+	ss << "inline data enabled";
+
+        fsmap.modify_filesystem(
+            fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+        {
+          fs->mds_map.set_inline_data_enabled(true);
+        });
+      } else {
+	ss << "inline data disabled";
+        fsmap.modify_filesystem(
+            fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+        {
+          fs->mds_map.set_inline_data_enabled(false);
+        });
+      }
+    } else if (var == "balancer") {
+      if (val.empty()) {
+        ss << "unsetting the metadata load balancer";
+      } else {
+        ss << "setting the metadata load balancer to " << val;
+      }
+      fsmap.modify_filesystem(
+	fs->fscid,
+	[val](std::shared_ptr<Filesystem> fs)
+        {
+          fs->mds_map.set_balancer(val);
+        });
+      return true;
+    } else if (var == "max_file_size") {
+      if (interr.length()) {
+	ss << var << " requires an integer value";
+	return -EINVAL;
+      }
+      if (n < CEPH_MIN_STRIPE_UNIT) {
+	ss << var << " must at least " << CEPH_MIN_STRIPE_UNIT;
+	return -ERANGE;
+      }
+      fsmap.modify_filesystem(
+          fs->fscid,
+          [n](std::shared_ptr<Filesystem> fs)
+      {
+        fs->mds_map.set_max_filesize(n);
+      });
+    } else if (var == "allow_new_snaps") {
+      bool enable_snaps = false;
+      int r = parse_bool(val, &enable_snaps, ss);
+      if (r != 0) {
+        return r;
+      }
+
+      if (!enable_snaps) {
+        fsmap.modify_filesystem(
+            fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+        {
+          fs->mds_map.clear_snaps_allowed();
+        });
+	ss << "disabled new snapshots";
+      } else {
+        fsmap.modify_filesystem(
+            fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+        {
+          fs->mds_map.set_snaps_allowed();
+        });
+	ss << "enabled new snapshots";
+      }
+    } else if (var == "allow_multimds") {
+        ss << "Multiple MDS is always enabled. Use the max_mds"
+           << " parameter to control the number of active MDSs"
+           << " allowed. This command is DEPRECATED and will be"
+           << " REMOVED from future releases.";
+    } else if (var == "allow_multimds_snaps") {
+      bool enable = false;
+      int r = parse_bool(val, &enable, ss);
+      if (r != 0) {
+        return r;
+      }
+
+      string confirm;
+      if (!cmd_getval(cmdmap, "confirm", confirm) ||
+	  confirm != "--yes-i-am-really-a-mds") {
+	ss << "Warning! This command is for MDS only. Do not run it manually";
+	return -EPERM;
+      }
+
+      if (enable) {
+	ss << "enabled multimds with snapshot";
+        fsmap.modify_filesystem(
+            fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+        {
+	  fs->mds_map.set_multimds_snaps_allowed();
+        });
+      } else {
+	ss << "disabled multimds with snapshot";
+        fsmap.modify_filesystem(
+            fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+        {
+	  fs->mds_map.clear_multimds_snaps_allowed();
+        });
+      }
+    } else if (var == "allow_dirfrags") {
+        ss << "Directory fragmentation is now permanently enabled."
+           << " This command is DEPRECATED and will be REMOVED from future releases.";
+    } else if (var == "down") {
+      bool is_down = false;
+      int r = parse_bool(val, &is_down, ss);
+      if (r != 0) {
+        return r;
+      }
+
+      ss << fs->mds_map.get_fs_name();
+
+      fsmap.modify_filesystem(
+          fs->fscid,
+          [is_down](std::shared_ptr<Filesystem> fs)
+      {
+	if (is_down) {
+          if (fs->mds_map.get_max_mds() > 0) {
+	    fs->mds_map.set_old_max_mds();
+	    fs->mds_map.set_max_mds(0);
+          } /* else already down! */
+	} else {
+	  mds_rank_t oldmax = fs->mds_map.get_old_max_mds();
+	  fs->mds_map.set_max_mds(oldmax ? oldmax : 1);
+	}
+      });
+
+      if (is_down) {
+	ss << " marked down. ";
+      } else {
+	ss << " marked up, max_mds = " << fs->mds_map.get_max_mds();
+      }
+    } else if (var == "cluster_down" || var == "joinable") {
+      bool joinable = true;
+      int r = parse_bool(val, &joinable, ss);
+      if (r != 0) {
+        return r;
+      }
+      if (var == "cluster_down") {
+        joinable = !joinable;
+      }
+
+      ss << fs->mds_map.get_fs_name();
+
+      fsmap.modify_filesystem(
+          fs->fscid,
+          [joinable](std::shared_ptr<Filesystem> fs)
+      {
+	if (joinable) {
+	  fs->mds_map.clear_flag(CEPH_MDSMAP_NOT_JOINABLE);
+	} else {
+	  fs->mds_map.set_flag(CEPH_MDSMAP_NOT_JOINABLE);
+	}
+      });
+
+      if (joinable) {
+	ss << " marked joinable; MDS may join as newly active.";
+      } else {
+	ss << " marked not joinable; MDS cannot join as newly active.";
+      }
+
+      if (var == "cluster_down") {
+        ss << " WARNING: cluster_down flag is deprecated and will be"
+           << " removed in a future version. Please use \"joinable\".";
+      }
+    } else if (var == "standby_count_wanted") {
+      if (interr.length()) {
+       ss << var << " requires an integer value";
+       return -EINVAL;
+      }
+      if (n < 0) {
+       ss << var << " must be non-negative";
+       return -ERANGE;
+      }
+      fsmap.modify_filesystem(
+          fs->fscid,
+          [n](std::shared_ptr<Filesystem> fs)
+      {
+        fs->mds_map.set_standby_count_wanted(n);
+      });
+    } else if (var == "session_timeout") {
+      if (interr.length()) {
+       ss << var << " requires an integer value";
+       return -EINVAL;
+      }
+      if (n < 30) {
+       ss << var << " must be at least 30s";
+       return -ERANGE;
+      }
+      fsmap.modify_filesystem(
+          fs->fscid,
+          [n](std::shared_ptr<Filesystem> fs)
+      {
+        fs->mds_map.set_session_timeout((uint32_t)n);
+      });
+    } else if (var == "session_autoclose") {
+      if (interr.length()) {
+       ss << var << " requires an integer value";
+       return -EINVAL;
+      }
+      if (n < 30) {
+       ss << var << " must be at least 30s";
+       return -ERANGE;
+      }
+      fsmap.modify_filesystem(
+          fs->fscid,
+          [n](std::shared_ptr<Filesystem> fs)
+      {
+        fs->mds_map.set_session_autoclose((uint32_t)n);
+      });
+    } else if (var == "allow_standby_replay") {
+      bool allow = false;
+      int r = parse_bool(val, &allow, ss);
+      if (r != 0) {
+        return r;
+      }
+
+      if (!allow) {
+        if (!mon->osdmon()->is_writeable()) {
+          // not allowed to write yet, so retry when we can
+          mon->osdmon()->wait_for_writeable(op, new PaxosService::C_RetryMessage(mon->mdsmon(), op));
+          return -EAGAIN;
+        }
+        std::vector<mds_gid_t> to_fail;
+        for (const auto& [gid, info]: fs->mds_map.get_mds_info()) {
+          if (info.state == MDSMap::STATE_STANDBY_REPLAY) {
+            to_fail.push_back(gid);
+          }
+        }
+
+        for (const auto& gid : to_fail) {
+          mon->mdsmon()->fail_mds_gid(fsmap, gid);
+        }
+        if (!to_fail.empty()) {
+          mon->osdmon()->propose_pending();
+        }
+      }
+
+      auto f = [allow](auto& fs) {
+        if (allow) {
+          fs->mds_map.set_standby_replay_allowed();
+        } else {
+          fs->mds_map.clear_standby_replay_allowed();
+        }
+      };
+      fsmap.modify_filesystem(fs->fscid, std::move(f));
+    } else if (var == "min_compat_client") {
+      auto vno = ceph_release_from_name(val.c_str());
+      if (!vno) {
+	ss << "version " << val << " is not recognized";
+	return -EINVAL;
+      }
+      ss << "WARNING: setting min_compat_client is deprecated"
+            " and may not do what you want.\n"
+            "The oldest release to set is octopus.\n"
+            "Please migrate to `ceph fs required_client_features ...`.";
+      auto f = [vno](auto&& fs) {
+        fs->mds_map.set_min_compat_client(vno);
+      };
+      fsmap.modify_filesystem(fs->fscid, std::move(f));
+    } else {
+      ss << "unknown variable " << var;
+      return -EINVAL;
+    }
+
+    return 0;
+  }
+};
+
+class CompatSetHandler : public FileSystemCommandHandler
+{
+  public:
+    CompatSetHandler()
+      : FileSystemCommandHandler("fs compat")
+    {
+    }
+
+    int handle(
+	Monitor *mon,
+	FSMap &fsmap,
+	MonOpRequestRef op,
+	const cmdmap_t& cmdmap,
+	std::ostream &ss) override
+    {
+      static const std::set<std::string> subops = {"rm_incompat", "rm_compat", "add_incompat", "add_compat"};
+
+      std::string fs_name;
+      if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+	ss << "Missing filesystem name";
+	return -EINVAL;
+      }
+      auto fs = fsmap.get_filesystem(fs_name);
+      if (fs == nullptr) {
+	ss << "Not found: '" << fs_name << "'";
+	return -ENOENT;
+      }
+
+      string subop;
+      if (!cmd_getval(cmdmap, "subop", subop) || subops.count(subop) == 0) {
+	ss << "subop `" << subop << "' not recognized. Must be one of: " << subops;
+	return -EINVAL;
+      }
+
+      int64_t feature;
+      if (!cmd_getval(cmdmap, "feature", feature) || feature <= 0) {
+        ss << "Invalid feature";
+        return -EINVAL;
+      }
+
+      if (fs->mds_map.get_num_up_mds() > 0) {
+        ss << "file system must be failed or down; use `ceph fs fail` to bring down";
+        return -EBUSY;
+      }
+
+      CompatSet cs = fs->mds_map.compat;
+      if (subop == "rm_compat") {
+        if (cs.compat.contains(feature)) {
+          ss << "removed compat feature " << feature;
+          cs.compat.remove(feature);
+        } else {
+          ss << "already removed compat feature " << feature;
+        }
+      } else if (subop == "rm_incompat") {
+        if (cs.incompat.contains(feature)) {
+          ss << "removed incompat feature " << feature;
+          cs.incompat.remove(feature);
+        } else {
+          ss << "already removed incompat feature " << feature;
+        }
+      } else if (subop == "add_compat" || subop == "add_incompat") {
+        string feature_str;
+        if (!cmd_getval(cmdmap, "feature_str", feature_str) || feature_str.empty()) {
+          ss << "adding a feature requires a feature string";
+          return -EINVAL;
+        }
+        auto f = CompatSet::Feature(feature, feature_str);
+        if (subop == "add_compat") {
+          if (cs.compat.contains(feature)) {
+            auto name = cs.compat.get_name(feature);
+            if (name == feature_str) {
+              ss << "feature already exists";
+            } else {
+              ss << "feature with differing name `" << name << "' exists";
+              return -EEXIST;
+            }
+          } else {
+            cs.compat.insert(f);
+            ss << "added compat feature " << f;
+          }
+        } else if (subop == "add_incompat") {
+          if (cs.incompat.contains(feature)) {
+            auto name = cs.incompat.get_name(feature);
+            if (name == feature_str) {
+              ss << "feature already exists";
+            } else {
+              ss << "feature with differing name `" << name << "' exists";
+              return -EEXIST;
+            }
+          } else {
+            cs.incompat.insert(f);
+            ss << "added incompat feature " << f;
+          }
+        } else ceph_assert(0);
+      } else ceph_assert(0);
+
+      auto modifyf = [cs = std::move(cs)](auto&& fs) {
+        fs->mds_map.compat = cs;
+      };
+
+      fsmap.modify_filesystem(fs->fscid, std::move(modifyf));
+      return 0;
+    }
+};
+
+class RequiredClientFeaturesHandler : public FileSystemCommandHandler
+{
+  public:
+    RequiredClientFeaturesHandler()
+      : FileSystemCommandHandler("fs required_client_features")
+    {
+    }
+
+    int handle(
+	Monitor *mon,
+	FSMap &fsmap,
+	MonOpRequestRef op,
+	const cmdmap_t& cmdmap,
+	std::ostream &ss) override
+    {
+      std::string fs_name;
+      if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+	ss << "Missing filesystem name";
+	return -EINVAL;
+      }
+      auto fs = fsmap.get_filesystem(fs_name);
+      if (fs == nullptr) {
+	ss << "Not found: '" << fs_name << "'";
+	return -ENOENT;
+      }
+      string subop;
+      if (!cmd_getval(cmdmap, "subop", subop) ||
+	  (subop != "add" && subop != "rm")) {
+	ss << "Must either add or rm a feature; " << subop << " is not recognized";
+	return -EINVAL;
+      }
+      string val;
+      if (!cmd_getval(cmdmap, "val", val) || val.empty()) {
+	ss << "Missing feature id/name";
+	return -EINVAL;
+      }
+
+      int feature = cephfs_feature_from_name(val);
+      if (feature < 0) {
+	string err;
+	feature = strict_strtol(val.c_str(), 10, &err);
+	if (err.length()) {
+	  ss << "Invalid feature name: " << val;
+	  return -EINVAL;
+	}
+	if (feature < 0 || feature > CEPHFS_FEATURE_MAX) {
+	  ss << "Invalid feature id: " << feature;
+	  return -EINVAL;
+	}
+      }
+
+      if (subop == "add") {
+	bool ret = false;
+	fsmap.modify_filesystem(
+	    fs->fscid,
+	    [feature, &ret](auto&& fs)
+	{
+	  if (fs->mds_map.get_required_client_features().test(feature))
+	    return;
+	  fs->mds_map.add_required_client_feature(feature);
+	  ret = true;
+	});
+	if (ret) {
+	  ss << "added feature '" << cephfs_feature_name(feature) << "' to required_client_features";
+	} else {
+	  ss << "feature '" << cephfs_feature_name(feature) << "' is already set";
+	}
+      } else {
+	bool ret = false;
+	fsmap.modify_filesystem(
+	    fs->fscid,
+	    [feature, &ret](auto&& fs)
+	{
+          if (!fs->mds_map.get_required_client_features().test(feature))
+            return;
+          fs->mds_map.remove_required_client_feature(feature);
+          ret = true;
+	});
+	if (ret) {
+	  ss << "removed feature '" << cephfs_feature_name(feature) << "' from required_client_features";
+	} else {
+	  ss << "feature '" << cephfs_feature_name(feature) << "' is already unset";
+	}
+      }
+      return 0;
+   }
+};
+
+
+class AddDataPoolHandler : public FileSystemCommandHandler
+{
+  public:
+  explicit AddDataPoolHandler(Paxos *paxos)
+    : FileSystemCommandHandler("fs add_data_pool"), m_paxos(paxos)
+  {}
+
+  bool batched_propose() override {
+    return true;
+  }
+
+  int handle(
+      Monitor *mon,
+      FSMap& fsmap,
+      MonOpRequestRef op,
+      const cmdmap_t& cmdmap,
+      std::ostream &ss) override
+  {
+    ceph_assert(m_paxos->is_plugged());
+
+    string poolname;
+    cmd_getval(cmdmap, "pool", poolname);
+
+    std::string fs_name;
+    if (!cmd_getval(cmdmap, "fs_name", fs_name)
+        || fs_name.empty()) {
+      ss << "Missing filesystem name";
+      return -EINVAL;
+    }
+
+    int64_t poolid = mon->osdmon()->osdmap.lookup_pg_pool_name(poolname);
+    if (poolid < 0) {
+      string err;
+      poolid = strict_strtol(poolname.c_str(), 10, &err);
+      if (err.length()) {
+	ss << "pool '" << poolname << "' does not exist";
+	return -ENOENT;
+      }
+    }
+
+    int r = _check_pool(mon->osdmon()->osdmap, poolid, POOL_DATA_EXTRA, false, &ss);
+    if (r != 0) {
+      return r;
+    }
+
+    auto fs = fsmap.get_filesystem(fs_name);
+    // no-op when the data_pool already on fs
+    if (fs->mds_map.is_data_pool(poolid)) {
+      ss << "data pool " << poolid << " is already on fs " << fs_name;
+      return 0;
+    }
+
+    if (!mon->osdmon()->is_writeable()) {
+      // not allowed to write yet, so retry when we can
+      mon->osdmon()->wait_for_writeable(op, new PaxosService::C_RetryMessage(mon->mdsmon(), op));
+      return -EAGAIN;
+    }
+    mon->osdmon()->do_application_enable(poolid,
+					 pg_pool_t::APPLICATION_NAME_CEPHFS,
+					 "data", fs_name, true);
+    mon->osdmon()->propose_pending();
+
+    fsmap.modify_filesystem(
+        fs->fscid,
+        [poolid](std::shared_ptr<Filesystem> fs)
+    {
+      fs->mds_map.add_data_pool(poolid);
+    });
+
+    ss << "added data pool " << poolid << " to fsmap";
+
+    return 0;
+  }
+
+private:
+  Paxos *m_paxos;
+};
+
+class SetDefaultHandler : public FileSystemCommandHandler
+{
+  public:
+  SetDefaultHandler()
+    : FileSystemCommandHandler("fs set-default")
+  {}
+
+  int handle(
+      Monitor *mon,
+      FSMap& fsmap,
+      MonOpRequestRef op,
+      const cmdmap_t& cmdmap,
+      std::ostream &ss) override
+  {
+    std::string fs_name;
+    cmd_getval(cmdmap, "fs_name", fs_name);
+    auto fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+        ss << "filesystem '" << fs_name << "' does not exist";
+        return -ENOENT;
+    }
+
+    fsmap.set_legacy_client_fscid(fs->fscid);
+    return 0;
+  }
+};
+
+class RemoveFilesystemHandler : public FileSystemCommandHandler
+{
+  public:
+  RemoveFilesystemHandler()
+    : FileSystemCommandHandler("fs rm")
+  {}
+
+  int handle(
+      Monitor *mon,
+      FSMap& fsmap,
+      MonOpRequestRef op,
+      const cmdmap_t& cmdmap,
+      std::ostream &ss) override
+  {
+    /* We may need to blocklist ranks. */
+    if (!mon->osdmon()->is_writeable()) {
+      // not allowed to write yet, so retry when we can
+      mon->osdmon()->wait_for_writeable(op, new PaxosService::C_RetryMessage(mon->mdsmon(), op));
+      return -EAGAIN;
+    }
+
+    // Check caller has correctly named the FS to delete
+    // (redundant while there is only one FS, but command
+    //  syntax should apply to multi-FS future)
+    string fs_name;
+    cmd_getval(cmdmap, "fs_name", fs_name);
+    auto fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+        // Consider absence success to make deletes idempotent
+        ss << "filesystem '" << fs_name << "' does not exist";
+        return 0;
+    }
+
+    // Check that no MDS daemons are active
+    if (fs->mds_map.get_num_up_mds() > 0) {
+      ss << "all MDS daemons must be inactive/failed before removing filesystem. See `ceph fs fail`.";
+      return -EINVAL;
+    }
+
+    // Check for confirmation flag
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+    if (!sure) {
+      ss << "this is a DESTRUCTIVE operation and will make data in your filesystem permanently" \
+            " inaccessible.  Add --yes-i-really-mean-it if you are sure you wish to continue.";
+      return -EPERM;
+    }
+
+    if (fsmap.get_legacy_client_fscid() == fs->fscid) {
+      fsmap.set_legacy_client_fscid(FS_CLUSTER_ID_NONE);
+    }
+
+    std::vector<mds_gid_t> to_fail;
+    // There may be standby_replay daemons left here
+    for (const auto &i : fs->mds_map.get_mds_info()) {
+      ceph_assert(i.second.state == MDSMap::STATE_STANDBY_REPLAY);
+      to_fail.push_back(i.first);
+    }
+
+    for (const auto &gid : to_fail) {
+      // Standby replays don't write, so it isn't important to
+      // wait for an osdmap propose here: ignore return value.
+      mon->mdsmon()->fail_mds_gid(fsmap, gid);
+    }
+    if (!to_fail.empty()) {
+      mon->osdmon()->propose_pending(); /* maybe new blocklists */
+    }
+
+    fsmap.erase_filesystem(fs->fscid);
+
+    return 0;
+  }
+};
+
+class ResetFilesystemHandler : public FileSystemCommandHandler
+{
+  public:
+  ResetFilesystemHandler()
+    : FileSystemCommandHandler("fs reset")
+  {}
+
+  int handle(
+      Monitor *mon,
+      FSMap& fsmap,
+      MonOpRequestRef op,
+      const cmdmap_t& cmdmap,
+      std::ostream &ss) override
+  {
+    string fs_name;
+    cmd_getval(cmdmap, "fs_name", fs_name);
+    auto fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+        ss << "filesystem '" << fs_name << "' does not exist";
+        // Unlike fs rm, we consider this case an error
+        return -ENOENT;
+    }
+
+    // Check that no MDS daemons are active
+    if (fs->mds_map.get_num_up_mds() > 0) {
+      ss << "all MDS daemons must be inactive before resetting filesystem: set the cluster_down flag"
+            " and use `ceph mds fail` to make this so";
+      return -EINVAL;
+    }
+
+    // Check for confirmation flag
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+    if (!sure) {
+      ss << "this is a potentially destructive operation, only for use by experts in disaster recovery.  "
+        "Add --yes-i-really-mean-it if you are sure you wish to continue.";
+      return -EPERM;
+    }
+
+    fsmap.reset_filesystem(fs->fscid);
+
+    return 0;
+  }
+};
+
+class RemoveDataPoolHandler : public FileSystemCommandHandler
+{
+  public:
+  RemoveDataPoolHandler()
+    : FileSystemCommandHandler("fs rm_data_pool")
+  {}
+
+  int handle(
+      Monitor *mon,
+      FSMap& fsmap,
+      MonOpRequestRef op,
+      const cmdmap_t& cmdmap,
+      std::ostream &ss) override
+  {
+    string poolname;
+    cmd_getval(cmdmap, "pool", poolname);
+
+    std::string fs_name;
+    if (!cmd_getval(cmdmap, "fs_name", fs_name)
+        || fs_name.empty()) {
+      ss << "Missing filesystem name";
+      return -EINVAL;
+    }
+
+    int64_t poolid = mon->osdmon()->osdmap.lookup_pg_pool_name(poolname);
+    if (poolid < 0) {
+      string err;
+      poolid = strict_strtol(poolname.c_str(), 10, &err);
+      if (err.length()) {
+	ss << "pool '" << poolname << "' does not exist";
+        return -ENOENT;
+      } else if (poolid < 0) {
+        ss << "invalid pool id '" << poolid << "'";
+        return -EINVAL;
+      }
+    }
+
+    ceph_assert(poolid >= 0);  // Checked by parsing code above
+
+    auto fs = fsmap.get_filesystem(fs_name);
+    if (fs->mds_map.get_first_data_pool() == poolid) {
+      ss << "cannot remove default data pool";
+      return -EINVAL;
+    }
+
+    int r = 0;
+    fsmap.modify_filesystem(fs->fscid,
+        [&r, poolid](std::shared_ptr<Filesystem> fs)
+    {
+      r = fs->mds_map.remove_data_pool(poolid);
+    });
+    if (r == -ENOENT) {
+      // It was already removed, succeed in silence
+      return 0;
+    } else if (r == 0) {
+      // We removed it, succeed
+      ss << "removed data pool " << poolid << " from fsmap";
+      return 0;
+    } else {
+      // Unexpected error, bubble up
+      return r;
+    }
+  }
+};
+
+/**
+ * For commands with an alternative prefix
+ */
+template<typename T>
+class AliasHandler : public T
+{
+  std::string alias_prefix;
+
+  public:
+  explicit AliasHandler(const std::string &new_prefix)
+    : T()
+  {
+    alias_prefix = new_prefix;
+  }
+
+  std::string const &get_prefix() const override {return alias_prefix;}
+
+  int handle(
+      Monitor *mon,
+      FSMap& fsmap,
+      MonOpRequestRef op,
+      const cmdmap_t& cmdmap,
+      std::ostream &ss) override
+  {
+    return T::handle(mon, fsmap, op, cmdmap, ss);
+  }
+};
+
+class MirrorHandlerEnable : public FileSystemCommandHandler
+{
+public:
+  MirrorHandlerEnable()
+    : FileSystemCommandHandler("fs mirror enable")
+  {}
+
+  int handle(Monitor *mon,
+             FSMap &fsmap, MonOpRequestRef op,
+             const cmdmap_t& cmdmap, std::ostream &ss) override {
+    std::string fs_name;
+    if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+      ss << "Missing filesystem name";
+      return -EINVAL;
+    }
+
+    auto fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+      ss << "Filesystem '" << fs_name << "' not found";
+      return -ENOENT;
+    }
+
+    if (fs->mirror_info.is_mirrored()) {
+      return 0;
+    }
+
+    auto f = [](auto &&fs) {
+               fs->mirror_info.enable_mirroring();
+    };
+    fsmap.modify_filesystem(fs->fscid, std::move(f));
+
+    return 0;
+  }
+};
+
+class MirrorHandlerDisable : public FileSystemCommandHandler
+{
+public:
+  MirrorHandlerDisable()
+    : FileSystemCommandHandler("fs mirror disable")
+  {}
+
+  int handle(Monitor *mon,
+             FSMap &fsmap, MonOpRequestRef op,
+             const cmdmap_t& cmdmap, std::ostream &ss) override {
+    std::string fs_name;
+    if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+      ss << "Missing filesystem name";
+      return -EINVAL;
+    }
+
+    auto fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+      ss << "Filesystem '" << fs_name << "' not found";
+      return -ENOENT;
+    }
+
+    if (!fs->mirror_info.is_mirrored()) {
+      return 0;
+    }
+
+    auto f = [](auto &&fs) {
+      fs->mirror_info.disable_mirroring();
+    };
+    fsmap.modify_filesystem(fs->fscid, std::move(f));
+
+    return 0;
+  }
+};
+
+class MirrorHandlerAddPeer : public FileSystemCommandHandler
+{
+public:
+  MirrorHandlerAddPeer()
+    : FileSystemCommandHandler("fs mirror peer_add")
+  {}
+
+  boost::optional<std::pair<string, string>>
+  extract_remote_cluster_conf(const std::string &spec) {
+    auto pos = spec.find("@");
+    if (pos == std::string_view::npos) {
+      return boost::optional<std::pair<string, string>>();
+    }
+
+    auto client = spec.substr(0, pos);
+    auto cluster = spec.substr(pos+1);
+
+    return std::make_pair(client, cluster);
+  }
+
+  bool peer_add(FSMap &fsmap, Filesystem::const_ref &&fs,
+                const cmdmap_t &cmdmap, std::ostream &ss) {
+    string peer_uuid;
+    string remote_spec;
+    string remote_fs_name;
+    cmd_getval(cmdmap, "uuid", peer_uuid);
+    cmd_getval(cmdmap, "remote_cluster_spec", remote_spec);
+    cmd_getval(cmdmap, "remote_fs_name", remote_fs_name);
+
+    // verify (and extract) remote cluster specification
+    auto remote_conf = extract_remote_cluster_conf(remote_spec);
+    if (!remote_conf) {
+      ss << "invalid remote cluster spec -- should be <client>@<cluster>";
+      return false;
+    }
+
+    if (fs->mirror_info.has_peer(peer_uuid)) {
+      ss << "peer already exists";
+      return true;
+    }
+    if (fs->mirror_info.has_peer((*remote_conf).first, (*remote_conf).second,
+                                 remote_fs_name)) {
+      ss << "peer already exists";
+      return true;
+    }
+
+    auto f = [peer_uuid, remote_conf, remote_fs_name](auto &&fs) {
+               fs->mirror_info.peer_add(peer_uuid, (*remote_conf).first,
+                                        (*remote_conf).second, remote_fs_name);
+             };
+    fsmap.modify_filesystem(fs->fscid, std::move(f));
+    return true;
+  }
+
+  int handle(Monitor *mon,
+             FSMap &fsmap, MonOpRequestRef op,
+             const cmdmap_t& cmdmap, std::ostream &ss) override {
+    std::string fs_name;
+    if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+      ss << "Missing filesystem name";
+      return -EINVAL;
+    }
+
+    auto fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+      ss << "Filesystem '" << fs_name << "' not found";
+      return -ENOENT;
+    }
+
+    if (!fs->mirror_info.is_mirrored()) {
+      ss << "Mirroring not enabled for filesystem '" << fs_name << "'";
+      return -EINVAL;
+    }
+
+    auto res = peer_add(fsmap, std::move(fs), cmdmap, ss);
+    if (!res) {
+      return -EINVAL;
+    }
+
+    return 0;
+  }
+};
+
+class MirrorHandlerRemovePeer : public FileSystemCommandHandler
+{
+public:
+  MirrorHandlerRemovePeer()
+    : FileSystemCommandHandler("fs mirror peer_remove")
+  {}
+
+  bool peer_remove(FSMap &fsmap, Filesystem::const_ref &&fs,
+                   const cmdmap_t &cmdmap, std::ostream &ss) {
+    string peer_uuid;
+    cmd_getval(cmdmap, "uuid", peer_uuid);
+
+    if (!fs->mirror_info.has_peer(peer_uuid)) {
+      ss << "cannot find peer with uuid: " << peer_uuid;
+      return true;
+    }
+
+    auto f = [peer_uuid](auto &&fs) {
+               fs->mirror_info.peer_remove(peer_uuid);
+             };
+    fsmap.modify_filesystem(fs->fscid, std::move(f));
+    return true;
+  }
+
+  int handle(Monitor *mon,
+             FSMap &fsmap, MonOpRequestRef op,
+             const cmdmap_t& cmdmap, std::ostream &ss) override {
+    std::string fs_name;
+    if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+      ss << "Missing filesystem name";
+      return -EINVAL;
+    }
+
+    auto fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+      ss << "Filesystem '" << fs_name << "' not found";
+      return -ENOENT;
+    }
+
+    if (!fs->mirror_info.is_mirrored()) {
+      ss << "Mirroring not enabled for filesystem '" << fs_name << "'";
+      return -EINVAL;
+    }
+
+    auto res = peer_remove(fsmap, std::move(fs), cmdmap, ss);
+    if (!res) {
+      return -EINVAL;
+    }
+
+    return 0;
+  }
+};
+
+std::list<std::shared_ptr<FileSystemCommandHandler> >
+FileSystemCommandHandler::load(Paxos *paxos)
+{
+  std::list<std::shared_ptr<FileSystemCommandHandler> > handlers;
+
+  handlers.push_back(std::make_shared<SetHandler>());
+  handlers.push_back(std::make_shared<FailHandler>());
+  handlers.push_back(std::make_shared<FlagSetHandler>());
+  handlers.push_back(std::make_shared<CompatSetHandler>());
+  handlers.push_back(std::make_shared<RequiredClientFeaturesHandler>());
+  handlers.push_back(std::make_shared<AddDataPoolHandler>(paxos));
+  handlers.push_back(std::make_shared<RemoveDataPoolHandler>());
+  handlers.push_back(std::make_shared<FsNewHandler>(paxos));
+  handlers.push_back(std::make_shared<RemoveFilesystemHandler>());
+  handlers.push_back(std::make_shared<ResetFilesystemHandler>());
+
+  handlers.push_back(std::make_shared<SetDefaultHandler>());
+  handlers.push_back(std::make_shared<AliasHandler<SetDefaultHandler> >(
+        "fs set_default"));
+  handlers.push_back(std::make_shared<MirrorHandlerEnable>());
+  handlers.push_back(std::make_shared<MirrorHandlerDisable>());
+  handlers.push_back(std::make_shared<MirrorHandlerAddPeer>());
+  handlers.push_back(std::make_shared<MirrorHandlerRemovePeer>());
+
+  return handlers;
+}
+
+int FileSystemCommandHandler::_check_pool(
+    OSDMap &osd_map,
+    const int64_t pool_id,
+    int type,
+    bool force,
+    std::ostream *ss) const
+{
+  ceph_assert(ss != NULL);
+
+  const pg_pool_t *pool = osd_map.get_pg_pool(pool_id);
+  if (!pool) {
+    *ss << "pool id '" << pool_id << "' does not exist";
+    return -ENOENT;
+  }
+
+  const string& pool_name = osd_map.get_pool_name(pool_id);
+
+  if (pool->is_erasure()) {
+    if (type == POOL_METADATA) {
+      *ss << "pool '" << pool_name << "' (id '" << pool_id << "')"
+         << " is an erasure-coded pool.  Use of erasure-coded pools"
+         << " for CephFS metadata is not permitted";
+      return -EINVAL;
+    } else if (type == POOL_DATA_DEFAULT && !force) {
+      *ss << "pool '" << pool_name << "' (id '" << pool_id << "')"
+             " is an erasure-coded pool."
+             " Use of an EC pool for the default data pool is discouraged;"
+             " see the online CephFS documentation for more information."
+             " Use --force to override.";
+      return -EINVAL;
+    } else if (!pool->allows_ecoverwrites()) {
+      // non-overwriteable EC pools are only acceptable with a cache tier overlay
+      if (!pool->has_tiers() || !pool->has_read_tier() || !pool->has_write_tier()) {
+        *ss << "pool '" << pool_name << "' (id '" << pool_id << "')"
+            << " is an erasure-coded pool, with no overwrite support";
+        return -EINVAL;
+      }
+
+      // That cache tier overlay must be writeback, not readonly (it's the
+      // write operations like modify+truncate we care about support for)
+      const pg_pool_t *write_tier = osd_map.get_pg_pool(
+          pool->write_tier);
+      ceph_assert(write_tier != NULL);  // OSDMonitor shouldn't allow DNE tier
+      if (write_tier->cache_mode == pg_pool_t::CACHEMODE_FORWARD
+          || write_tier->cache_mode == pg_pool_t::CACHEMODE_READONLY) {
+        *ss << "EC pool '" << pool_name << "' has a write tier ("
+            << osd_map.get_pool_name(pool->write_tier)
+            << ") that is configured "
+               "to forward writes.  Use a cache mode such as 'writeback' for "
+               "CephFS";
+        return -EINVAL;
+      }
+    }
+  }
+
+  if (pool->is_tier()) {
+    *ss << " pool '" << pool_name << "' (id '" << pool_id
+      << "') is already in use as a cache tier.";
+    return -EINVAL;
+  }
+
+  if (!force && !pool->application_metadata.empty() &&
+      pool->application_metadata.count(
+        pg_pool_t::APPLICATION_NAME_CEPHFS) == 0) {
+    *ss << " pool '" << pool_name << "' (id '" << pool_id
+        << "') has a non-CephFS application enabled.";
+    return -EINVAL;
+  }
+
+  // Nothing special about this pool, so it is permissible
+  return 0;
+}
+
+int FileSystemCommandHandler::is_op_allowed(
+    const MonOpRequestRef& op, const FSMap& fsmap, const cmdmap_t& cmdmap,
+    std::ostream &ss) const
+{
+    string fs_name;
+    cmd_getval(cmdmap, "fs_name", fs_name);
+
+    // so that fsmap can filtered and the original copy is untouched.
+    FSMap fsmap_copy = fsmap;
+    fsmap_copy.filter(op->get_session()->get_allowed_fs_names());
+
+    auto fs = fsmap_copy.get_filesystem(fs_name);
+    if (fs == nullptr) {
+      /* let "fs rm" handle idempotent case where file system does not exist */
+      if (!(get_prefix() == "fs rm" && fsmap.get_filesystem(fs_name) == nullptr)) {
+        ss << "Filesystem not found: '" << fs_name << "'";
+        return -ENOENT;
+      }
+    }
+
+    if (!op->get_session()->fs_name_capable(fs_name, MON_CAP_W)) {
+      ss << "Permission denied: '" << fs_name << "'";
+      return -EPERM;
+    }
+
+  return 1;
+}
diff --git a/src/mon/FSCommands.h b/src/mon/FSCommands.h
new file mode 100644
index 000000000..4b59225f9
--- /dev/null
+++ b/src/mon/FSCommands.h
@@ -0,0 +1,93 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2017 Red Hat Ltd
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef FS_COMMANDS_H_
+#define FS_COMMANDS_H_
+
+#include "Monitor.h"
+#include "CommandHandler.h"
+
+#include "osd/OSDMap.h"
+#include "mds/FSMap.h"
+
+#include <string>
+#include <ostream>
+
+class FileSystemCommandHandler : protected CommandHandler
+{
+protected:
+  std::string prefix;
+
+  enum {
+    POOL_METADATA,
+    POOL_DATA_DEFAULT,
+    POOL_DATA_EXTRA,
+  };
+  /**
+   * Return 0 if the pool is suitable for use with CephFS, or
+   * in case of errors return a negative error code, and populate
+   * the passed ostream with an explanation.
+   *
+   * @param metadata whether the pool will be for metadata (stricter checks)
+   */
+  int _check_pool(
+      OSDMap &osd_map,
+      const int64_t pool_id,
+      int type,
+      bool force,
+      std::ostream *ss) const;
+
+  virtual std::string const &get_prefix() const {return prefix;}
+
+public:
+  FileSystemCommandHandler(const std::string &prefix_)
+    : prefix(prefix_)
+  {}
+
+  virtual ~FileSystemCommandHandler()
+  {}
+
+  int is_op_allowed(const MonOpRequestRef& op, const FSMap& fsmap,
+		    const cmdmap_t& cmdmap, std::ostream &ss) const;
+
+  int can_handle(std::string const &prefix_, MonOpRequestRef& op, FSMap& fsmap,
+	         const cmdmap_t& cmdmap, std::ostream &ss) const
+  {
+    if (get_prefix() != prefix_) {
+      return 0;
+    }
+
+    if (get_prefix() == "fs new" || get_prefix() == "fs flag set") {
+      return 1;
+    }
+
+    return is_op_allowed(op, fsmap, cmdmap, ss);
+  }
+
+  static std::list<std::shared_ptr<FileSystemCommandHandler> > load(Paxos *paxos);
+
+  virtual bool batched_propose() {
+    return false;
+  }
+
+  virtual int handle(
+    Monitor *mon,
+    FSMap &fsmap,
+    MonOpRequestRef op,
+    const cmdmap_t& cmdmap,
+    std::ostream &ss) = 0;
+};
+
+#endif
diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc
new file mode 100644
index 000000000..a45159e7c
--- /dev/null
+++ b/src/mon/HealthMonitor.cc
@@ -0,0 +1,877 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <stdlib.h>
+#include <limits.h>
+#include <sstream>
+#include <regex>
+#include <time.h>
+#include <iterator>
+
+#include "include/ceph_assert.h"
+#include "include/common_fwd.h"
+#include "include/stringify.h"
+
+#include "mon/Monitor.h"
+#include "mon/HealthMonitor.h"
+
+#include "messages/MMonHealthChecks.h"
+
+#include "common/Formatter.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, this)
+using namespace TOPNSPC::common;
+
+using namespace std::literals;
+using std::cerr;
+using std::cout;
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::setfill;
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+using std::unique_ptr;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::mono_clock;
+using ceph::mono_time;
+using ceph::parse_timespan;
+using ceph::timespan_str;
+static ostream& _prefix(std::ostream *_dout, const Monitor &mon,
+                        const HealthMonitor *hmon) {
+  return *_dout << "mon." << mon.name << "@" << mon.rank
+		<< "(" << mon.get_state_name() << ").health ";
+}
+
+HealthMonitor::HealthMonitor(Monitor &m, Paxos &p, const string& service_name)
+  : PaxosService(m, p, service_name) {
+}
+
+void HealthMonitor::init()
+{
+  dout(10) << __func__ << dendl;
+}
+
+void HealthMonitor::create_initial()
+{
+  dout(10) << __func__ << dendl;
+}
+
+void HealthMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  version = get_last_committed();
+  dout(10) << __func__ << dendl;
+  load_health();
+
+  bufferlist qbl;
+  mon.store->get(service_name, "quorum", qbl);
+  if (qbl.length()) {
+    auto p = qbl.cbegin();
+    decode(quorum_checks, p);
+  } else {
+    quorum_checks.clear();
+  }
+
+  bufferlist lbl;
+  mon.store->get(service_name, "leader", lbl);
+  if (lbl.length()) {
+    auto p = lbl.cbegin();
+    decode(leader_checks, p);
+  } else {
+    leader_checks.clear();
+  }
+
+  {
+    bufferlist bl;
+    mon.store->get(service_name, "mutes", bl);
+    if (bl.length()) {
+      auto p = bl.cbegin();
+      decode(mutes, p);
+    } else {
+      mutes.clear();
+    }
+  }
+
+  dout(20) << "dump:";
+  JSONFormatter jf(true);
+  jf.open_object_section("health");
+  jf.open_object_section("quorum_health");
+  for (auto& p : quorum_checks) {
+    string s = string("mon.") + stringify(p.first);
+    jf.dump_object(s.c_str(), p.second);
+  }
+  jf.close_section();
+  jf.dump_object("leader_health", leader_checks);
+  jf.close_section();
+  jf.flush(*_dout);
+  *_dout << dendl;
+}
+
+void HealthMonitor::create_pending()
+{
+  dout(10) << " " << version << dendl;
+  pending_mutes = mutes;
+}
+
+void HealthMonitor::encode_pending(MonitorDBStore::TransactionRef t)
+{
+  ++version;
+  dout(10) << " " << version << dendl;
+  put_last_committed(t, version);
+
+  bufferlist qbl;
+  encode(quorum_checks, qbl);
+  t->put(service_name, "quorum", qbl);
+  bufferlist lbl;
+  encode(leader_checks, lbl);
+  t->put(service_name, "leader", lbl);
+  {
+    bufferlist bl;
+    encode(pending_mutes, bl);
+    t->put(service_name, "mutes", bl);
+  }
+
+  health_check_map_t pending_health;
+
+  // combine per-mon details carefully...
+  map<string,set<string>> names; // code -> <mon names>
+  for (auto p : quorum_checks) {
+    for (auto q : p.second.checks) {
+      names[q.first].insert(mon.monmap->get_name(p.first));
+    }
+    pending_health.merge(p.second);
+  }
+  for (auto &p : pending_health.checks) {
+    p.second.summary = std::regex_replace(
+      p.second.summary,
+      std::regex("%hasorhave%"),
+      names[p.first].size() > 1 ? "have" : "has");
+    p.second.summary = std::regex_replace(
+      p.second.summary,
+      std::regex("%names%"), stringify(names[p.first]));
+    p.second.summary = std::regex_replace(
+      p.second.summary,
+      std::regex("%plurals%"),
+      names[p.first].size() > 1 ? "s" : "");
+    p.second.summary = std::regex_replace(
+      p.second.summary,
+      std::regex("%isorare%"),
+      names[p.first].size() > 1 ? "are" : "is");
+  }
+
+  pending_health.merge(leader_checks);
+  encode_health(pending_health, t);
+}
+
+version_t HealthMonitor::get_trim_to() const
+{
+  // we don't actually need *any* old states, but keep a few.
+  if (version > 5) {
+    return version - 5;
+  }
+  return 0;
+}
+
+bool HealthMonitor::preprocess_query(MonOpRequestRef op)
+{
+  auto m = op->get_req<PaxosServiceMessage>();
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    return preprocess_command(op);
+  case MSG_MON_HEALTH_CHECKS:
+    return false;
+  default:
+    mon.no_reply(op);
+    derr << "Unhandled message type " << m->get_type() << dendl;
+    return true;
+  }
+}
+
+bool HealthMonitor::prepare_update(MonOpRequestRef op)
+{
+  Message *m = op->get_req();
+  dout(7) << "prepare_update " << *m
+	  << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case MSG_MON_HEALTH_CHECKS:
+    return prepare_health_checks(op);
+  case MSG_MON_COMMAND:
+    return prepare_command(op);
+  default:
+    return false;
+  }
+}
+
+bool HealthMonitor::preprocess_command(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonCommand>();
+  std::stringstream ss;
+  bufferlist rdata;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    mon.reply_command(op, -EACCES, "access denied", rdata,
+		       get_last_committed());
+    return true;
+  }
+  // more sanity checks
+  try {
+    string format;
+    cmd_getval(cmdmap, "format", format);
+    string prefix;
+    cmd_getval(cmdmap, "prefix", prefix);
+  } catch (const bad_cmd_get& e) {
+    mon.reply_command(op, -EINVAL, e.what(), rdata, get_last_committed());
+    return true;
+  }
+  return false;
+}
+
+bool HealthMonitor::prepare_command(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonCommand>();
+
+  std::stringstream ss;
+  bufferlist rdata;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    mon.reply_command(op, -EACCES, "access denied", rdata, get_last_committed());
+    return true;
+  }
+
+  string format;
+  cmd_getval(cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+
+  int r = 0;
+
+  if (prefix == "health mute") {
+    string code;
+    bool sticky = false;
+    if (!cmd_getval(cmdmap, "code", code) ||
+	code == "") {
+      r = -EINVAL;
+      ss << "must specify an alert code to mute";
+      goto out;
+    }
+    cmd_getval(cmdmap, "sticky", sticky);
+    string ttl_str;
+    utime_t ttl;
+    if (cmd_getval(cmdmap, "ttl", ttl_str)) {
+      auto secs = parse_timespan(ttl_str);
+      if (secs == 0s) {
+	r = -EINVAL;
+	ss << "not a valid duration: " << ttl_str;
+	goto out;
+      }
+      ttl = ceph_clock_now();
+      ttl += std::chrono::duration<double>(secs).count();
+    }
+    health_check_map_t all;
+    gather_all_health_checks(&all);
+    string summary;
+    int64_t count = 0;
+    if (!sticky) {
+      auto p = all.checks.find(code);
+      if (p == all.checks.end()) {
+	r = -ENOENT;
+	ss << "health alert " << code << " is not currently raised";
+	goto out;
+      }
+      count = p->second.count;
+      summary = p->second.summary;
+    }
+    auto& m = pending_mutes[code];
+    m.code = code;
+    m.ttl = ttl;
+    m.sticky = sticky;
+    m.summary = summary;
+    m.count = count;
+  } else if (prefix == "health unmute") {
+    string code;
+    if (cmd_getval(cmdmap, "code", code)) {
+      pending_mutes.erase(code);
+    } else {
+      pending_mutes.clear();
+    }
+  } else {
+    ss << "Command '" << prefix << "' not implemented!";
+    r = -ENOSYS;
+  }
+
+out:
+  dout(4) << __func__ << " done, r=" << r << dendl;
+  /* Compose response */
+  string rs;
+  getline(ss, rs);
+
+  if (r >= 0) {
+    // success.. delay reply
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, r, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else {
+    // reply immediately
+    mon.reply_command(op, r, rs, rdata, get_last_committed());
+    return false;
+  }
+}
+
+bool HealthMonitor::prepare_health_checks(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonHealthChecks>();
+  // no need to check if it's changed, the peon has done so
+  quorum_checks[m->get_source().num()] = std::move(m->health_checks);
+  return true;
+}
+
+void HealthMonitor::tick()
+{
+  if (!is_active()) {
+    return;
+  }
+  dout(10) << __func__ << dendl;
+  bool changed = false;
+  if (check_member_health()) {
+    changed = true;
+  }
+  if (!mon.is_leader()) {
+    return;
+  }
+  if (check_leader_health()) {
+    changed = true;
+  }
+  if (check_mutes()) {
+    changed = true;
+  }
+  if (changed) {
+    propose_pending();
+  }
+}
+
+bool HealthMonitor::check_mutes()
+{
+  bool changed = true;
+  auto now = ceph_clock_now();
+  health_check_map_t all;
+  gather_all_health_checks(&all);
+  auto p = pending_mutes.begin();
+  while (p != pending_mutes.end()) {
+    if (p->second.ttl != utime_t() &&
+	p->second.ttl <= now) {
+      mon.clog->info() << "Health alert mute " << p->first
+			<< " cleared (passed TTL " << p->second.ttl << ")";
+      p = pending_mutes.erase(p);
+      changed = true;
+      continue;
+    }
+    if (!p->second.sticky) {
+      auto q = all.checks.find(p->first);
+      if (q == all.checks.end()) {
+	mon.clog->info() << "Health alert mute " << p->first
+			  << " cleared (health alert cleared)";
+	p = pending_mutes.erase(p);
+	changed = true;
+	continue;
+      }
+      if (p->second.count) {
+	// count-based mute
+	if (q->second.count > p->second.count) {
+	  mon.clog->info() << "Health alert mute " << p->first
+			    << " cleared (count increased from " << p->second.count
+			    << " to " << q->second.count << ")";
+	  p = pending_mutes.erase(p);
+	  changed = true;
+	  continue;
+	}
+	if (q->second.count < p->second.count) {
+	  // rachet down the mute
+	  dout(10) << __func__ << " mute " << p->first << " count "
+		   << p->second.count << " -> " << q->second.count
+		   << dendl;
+	  p->second.count = q->second.count;
+	  changed = true;
+	}
+      } else {
+	// summary-based mute
+	if (p->second.summary != q->second.summary) {
+	  mon.clog->info() << "Health alert mute " << p->first
+			    << " cleared (summary changed)";
+	  p = pending_mutes.erase(p);
+	  changed = true;
+	  continue;
+	}
+      }
+    }
+    ++p;
+  }
+  return changed;
+}
+
+void HealthMonitor::gather_all_health_checks(health_check_map_t *all)
+{
+  for (auto& svc : mon.paxos_service) {
+    all->merge(svc->get_health_checks());
+  }
+}
+
+health_status_t HealthMonitor::get_health_status(
+  bool want_detail,
+  Formatter *f,
+  std::string *plain,
+  const char *sep1,
+  const char *sep2)
+{
+  health_check_map_t all;
+  gather_all_health_checks(&all);
+  health_status_t r = HEALTH_OK;
+  for (auto& p : all.checks) {
+    if (!mutes.count(p.first)) {
+      if (r > p.second.severity) {
+	r = p.second.severity;
+      }
+    }
+  }
+  if (f) {
+    f->open_object_section("health");
+    f->dump_stream("status") << r;
+    f->open_object_section("checks");
+    for (auto& p : all.checks) {
+      f->open_object_section(p.first.c_str());
+      p.second.dump(f, want_detail);
+      f->dump_bool("muted", mutes.count(p.first));
+      f->close_section();
+    }
+    f->close_section();
+    f->open_array_section("mutes");
+    for (auto& p : mutes) {
+      f->dump_object("mute", p.second);
+    }
+    f->close_section();
+    f->close_section();
+  } else {
+    auto now = ceph_clock_now();
+    // one-liner: HEALTH_FOO[ thing1[; thing2 ...]]
+    string summary;
+    for (auto& p : all.checks) {
+      if (!mutes.count(p.first)) {
+	if (!summary.empty()) {
+	  summary += sep2;
+	}
+	summary += p.second.summary;
+      }
+    }
+    *plain = stringify(r);
+    if (summary.size()) {
+      *plain += sep1;
+      *plain += summary;
+    }
+    if (!mutes.empty()) {
+      if (summary.size()) {
+	*plain += sep2;
+      } else {
+	*plain += sep1;
+      }
+      *plain += "(muted:";
+      for (auto& p : mutes) {
+	*plain += " ";
+	*plain += p.first;
+	if (p.second.ttl) {
+	  if (p.second.ttl > now) {
+	    auto left = p.second.ttl;
+	    left -= now;
+	    *plain += "("s + utimespan_str(left) + ")";
+	  } else {
+	    *plain += "(0s)";
+	  }
+	}
+      }
+      *plain += ")";
+    }
+    *plain += "\n";
+    // detail
+    if (want_detail) {
+      for (auto& p : all.checks) {
+	auto q = mutes.find(p.first);
+	if (q != mutes.end()) {
+	  *plain += "(MUTED";
+	  if (q->second.ttl != utime_t()) {
+	    if (q->second.ttl > now) {
+	      auto left = q->second.ttl;
+	      left -= now;
+	      *plain += " ttl ";
+	      *plain += utimespan_str(left);
+	    } else {
+	      *plain += "0s";
+	    }
+	  }
+	  if (q->second.sticky) {
+	    *plain += ", STICKY";
+	  }
+	  *plain += ") ";
+	}
+	*plain += "["s + short_health_string(p.second.severity) + "] " +
+	  p.first + ": " + p.second.summary + "\n";
+	for (auto& d : p.second.detail) {
+	  *plain += "    ";
+	  *plain += d;
+	  *plain += "\n";
+	}
+      }
+    }
+  }
+  return r;
+}
+
+bool HealthMonitor::check_member_health()
+{
+  dout(20) << __func__ << dendl;
+  bool changed = false;
+  const auto max = g_conf().get_val<uint64_t>("mon_health_max_detail");
+
+  // snapshot of usage
+  DataStats stats;
+  get_fs_stats(stats.fs_stats, g_conf()->mon_data.c_str());
+  map<string,uint64_t> extra;
+  uint64_t store_size = mon.store->get_estimated_size(extra);
+  ceph_assert(store_size > 0);
+  stats.store_stats.bytes_total = store_size;
+  stats.store_stats.bytes_sst = extra["sst"];
+  stats.store_stats.bytes_log = extra["log"];
+  stats.store_stats.bytes_misc = extra["misc"];
+  stats.last_update = ceph_clock_now();
+  dout(10) << __func__ << " avail " << stats.fs_stats.avail_percent << "%"
+	   << " total " << byte_u_t(stats.fs_stats.byte_total)
+	   << ", used " << byte_u_t(stats.fs_stats.byte_used)
+	   << ", avail " << byte_u_t(stats.fs_stats.byte_avail) << dendl;
+
+  // MON_DISK_{LOW,CRIT,BIG}
+  health_check_map_t next;
+  if (stats.fs_stats.avail_percent <= g_conf()->mon_data_avail_crit) {
+    stringstream ss, ss2;
+    ss << "mon%plurals% %names% %isorare% very low on available space";
+    auto& d = next.add("MON_DISK_CRIT", HEALTH_ERR, ss.str(), 1);
+    ss2 << "mon." << mon.name << " has " << stats.fs_stats.avail_percent
+	<< "% avail";
+    d.detail.push_back(ss2.str());
+  } else if (stats.fs_stats.avail_percent <= g_conf()->mon_data_avail_warn) {
+    stringstream ss, ss2;
+    ss << "mon%plurals% %names% %isorare% low on available space";
+    auto& d = next.add("MON_DISK_LOW", HEALTH_WARN, ss.str(), 1);
+    ss2 << "mon." << mon.name << " has " << stats.fs_stats.avail_percent
+	<< "% avail";
+    d.detail.push_back(ss2.str());
+  }
+  if (stats.store_stats.bytes_total >= g_conf()->mon_data_size_warn) {
+    stringstream ss, ss2;
+    ss << "mon%plurals% %names% %isorare% using a lot of disk space";
+    auto& d = next.add("MON_DISK_BIG", HEALTH_WARN, ss.str(), 1);
+    ss2 << "mon." << mon.name << " is "
+	<< byte_u_t(stats.store_stats.bytes_total)
+	<< " >= mon_data_size_warn ("
+	<< byte_u_t(g_conf()->mon_data_size_warn) << ")";
+    d.detail.push_back(ss2.str());
+  }
+
+  // OSD_NO_DOWN_OUT_INTERVAL
+  {
+    // Warn if 'mon_osd_down_out_interval' is set to zero.
+    // Having this option set to zero on the leader acts much like the
+    // 'noout' flag.  It's hard to figure out what's going wrong with clusters
+    // without the 'noout' flag set but acting like that just the same, so
+    // we report a HEALTH_WARN in case this option is set to zero.
+    // This is an ugly hack to get the warning out, but until we find a way
+    // to spread global options throughout the mon cluster and have all mons
+    // using a base set of the same options, we need to work around this sort
+    // of things.
+    // There's also the obvious drawback that if this is set on a single
+    // monitor on a 3-monitor cluster, this warning will only be shown every
+    // third monitor connection.
+    if (g_conf()->mon_warn_on_osd_down_out_interval_zero &&
+        g_conf()->mon_osd_down_out_interval == 0) {
+      ostringstream ss, ds;
+      ss << "mon%plurals% %names% %hasorhave% mon_osd_down_out_interval set to 0";
+      auto& d = next.add("OSD_NO_DOWN_OUT_INTERVAL", HEALTH_WARN, ss.str(), 1);
+      ds << "mon." << mon.name << " has mon_osd_down_out_interval set to 0";
+      d.detail.push_back(ds.str());
+    }
+  }
+
+  // AUTH_INSECURE_GLOBAL_ID_RECLAIM
+  if (g_conf().get_val<bool>("mon_warn_on_insecure_global_id_reclaim") &&
+      g_conf().get_val<bool>("auth_allow_insecure_global_id_reclaim")) {
+    // Warn if there are any clients that are insecurely renewing their global_id
+    std::lock_guard l(mon.session_map_lock);
+    list<std::string> detail;
+    for (auto p = mon.session_map.sessions.begin();
+	 p != mon.session_map.sessions.end();
+	 ++p) {
+      if ((*p)->global_id_status == global_id_status_t::RECLAIM_INSECURE) {
+	ostringstream ds;
+	ds << (*p)->entity_name << " at " << (*p)->addrs
+	   << " is using insecure global_id reclaim";
+	detail.push_back(ds.str());
+	if (detail.size() >= max) {
+	  detail.push_back("...");
+	  break;
+	}
+      }
+    }
+    if (!detail.empty()) {
+      ostringstream ss;
+      ss << "client%plurals% %isorare% using insecure global_id reclaim";
+      auto& d = next.add("AUTH_INSECURE_GLOBAL_ID_RECLAIM", HEALTH_WARN, ss.str(),
+			 detail.size());
+      d.detail.swap(detail);
+    }
+  }
+  // AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED
+  if (g_conf().get_val<bool>("mon_warn_on_insecure_global_id_reclaim_allowed") &&
+      g_conf().get_val<bool>("auth_allow_insecure_global_id_reclaim")) {
+    ostringstream ss, ds;
+    ss << "mon%plurals% %isorare% allowing insecure global_id reclaim";
+    auto& d = next.add("AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED", HEALTH_WARN, ss.str(), 1);
+    ds << "mon." << mon.name << " has auth_allow_insecure_global_id_reclaim set to true";
+    d.detail.push_back(ds.str());
+  }
+
+  auto p = quorum_checks.find(mon.rank);
+  if (p == quorum_checks.end()) {
+    if (next.empty()) {
+      return false;
+    }
+  } else {
+    if (p->second == next) {
+      return false;
+    }
+  }
+
+  if (mon.is_leader()) {
+    // prepare to propose
+    quorum_checks[mon.rank] = next;
+    changed = true;
+  } else {
+    // tell the leader
+    mon.send_mon_message(new MMonHealthChecks(next), mon.get_leader());
+  }
+
+  return changed;
+}
+
+bool HealthMonitor::check_leader_health()
+{
+  dout(20) << __func__ << dendl;
+  bool changed = false;
+
+  // prune quorum_health
+  {
+    auto& qset = mon.get_quorum();
+    auto p = quorum_checks.begin();
+    while (p != quorum_checks.end()) {
+      if (qset.count(p->first) == 0) {
+	p = quorum_checks.erase(p);
+	changed = true;
+      } else {
+	++p;
+      }
+    }
+  }
+
+  health_check_map_t next;
+
+ // DAEMON_OLD_VERSION
+  if (g_conf().get_val<bool>("mon_warn_on_older_version")) {
+    check_for_older_version(&next);
+  }
+  // MON_DOWN
+  check_for_mon_down(&next);
+  // MON_CLOCK_SKEW
+  check_for_clock_skew(&next);
+  // MON_MSGR2_NOT_ENABLED
+  if (g_conf().get_val<bool>("mon_warn_on_msgr2_not_enabled")) {
+    check_if_msgr2_enabled(&next);
+  }
+
+  if (next != leader_checks) {
+    changed = true;
+    leader_checks = next;
+  }
+  return changed;
+}
+
+void HealthMonitor::check_for_older_version(health_check_map_t *checks)
+{
+  static ceph::coarse_mono_time old_version_first_time =
+    ceph::coarse_mono_clock::zero();
+
+  auto now = ceph::coarse_mono_clock::now();
+  if (ceph::coarse_mono_clock::is_zero(old_version_first_time)) {
+    old_version_first_time = now;
+  }
+  const auto warn_delay = g_conf().get_val<std::chrono::seconds>("mon_warn_older_version_delay");
+  if (now - old_version_first_time > warn_delay) {
+    std::map<string, std::list<string> > all_versions;
+    mon.get_all_versions(all_versions);
+    if (all_versions.size() > 1) {
+      dout(20) << __func__ << " all_versions=" << all_versions << dendl;
+      // The last entry has the largest version
+      dout(20) << __func__ << " highest version daemon count "
+	       << all_versions.rbegin()->second.size() << dendl;
+      // Erase last element (the highest version running)
+      all_versions.erase(all_versions.rbegin()->first);
+      ceph_assert(all_versions.size() > 0);
+      ostringstream ss;
+      unsigned daemon_count = 0;
+      for (auto& g : all_versions) {
+	daemon_count += g.second.size();
+      }
+      int ver_count = all_versions.size();
+      ceph_assert(!(daemon_count == 1 && ver_count != 1));
+      ss << "There " << (daemon_count == 1 ? "is a daemon" : "are daemons")
+	 << " running " << (ver_count > 1 ? "multiple old versions" : "an older version")  << " of ceph";
+      health_status_t status;
+      if (ver_count > 1)
+	status = HEALTH_ERR;
+      else
+	status = HEALTH_WARN;
+      auto& d = checks->add("DAEMON_OLD_VERSION", status, ss.str(), all_versions.size());
+      for (auto& g : all_versions) {
+	ostringstream ds;
+	for (auto& i : g.second) { // Daemon list
+	  ds << i << " ";
+	}
+	ds << (g.second.size() == 1 ? "is" : "are")
+	   << " running an older version of ceph: " << g.first;
+	d.detail.push_back(ds.str());
+      }
+    } else {
+      old_version_first_time = ceph::coarse_mono_clock::zero();
+    }
+  }
+}
+
+void HealthMonitor::check_for_mon_down(health_check_map_t *checks)
+{
+  int max = mon.monmap->size();
+  int actual = mon.get_quorum().size();
+  const auto now = ceph::real_clock::now();
+  if (actual < max &&
+      now > mon.monmap->created.to_real_time() + g_conf().get_val<std::chrono::seconds>("mon_down_mkfs_grace")) {
+    ostringstream ss;
+    ss << (max-actual) << "/" << max << " mons down, quorum "
+       << mon.get_quorum_names();
+    auto& d = checks->add("MON_DOWN", HEALTH_WARN, ss.str(), max - actual);
+    set<int> q = mon.get_quorum();
+    for (int i=0; i<max; i++) {
+      if (q.count(i) == 0) {
+	ostringstream ss;
+	ss << "mon." << mon.monmap->get_name(i) << " (rank " << i
+	   << ") addr " << mon.monmap->get_addrs(i)
+	   << " is down (out of quorum)";
+	d.detail.push_back(ss.str());
+      }
+    }
+  }
+}
+
+void HealthMonitor::check_for_clock_skew(health_check_map_t *checks)
+{
+  if (!mon.timecheck_skews.empty()) {
+    list<string> warns;
+    list<string> details;
+    for (auto& i : mon.timecheck_skews) {
+      double skew = i.second;
+      double latency = mon.timecheck_latencies[i.first];
+      string name = mon.monmap->get_name(i.first);
+      ostringstream tcss;
+      health_status_t tcstatus = mon.timecheck_status(tcss, skew, latency);
+      if (tcstatus != HEALTH_OK) {
+	warns.push_back(name);
+	ostringstream tmp_ss;
+	tmp_ss << "mon." << name << " " << tcss.str()
+	       << " (latency " << latency << "s)";
+	details.push_back(tmp_ss.str());
+      }
+    }
+    if (!warns.empty()) {
+      ostringstream ss;
+      ss << "clock skew detected on";
+      while (!warns.empty()) {
+	ss << " mon." << warns.front();
+	warns.pop_front();
+	if (!warns.empty())
+	  ss << ",";
+      }
+      auto& d = checks->add("MON_CLOCK_SKEW", HEALTH_WARN, ss.str(), details.size());
+      d.detail.swap(details);
+    }
+  }
+}
+
+void HealthMonitor::check_if_msgr2_enabled(health_check_map_t *checks)
+{
+  if (g_conf().get_val<bool>("ms_bind_msgr2") &&
+      mon.monmap->get_required_features().contains_all(
+	ceph::features::mon::FEATURE_NAUTILUS)) {
+    list<string> details;
+    for (auto& i : mon.monmap->mon_info) {
+      if (!i.second.public_addrs.has_msgr2()) {
+	ostringstream ds;
+	ds << "mon." << i.first << " is not bound to a msgr2 port, only "
+	   << i.second.public_addrs;
+	details.push_back(ds.str());
+      }
+    }
+    if (!details.empty()) {
+      ostringstream ss;
+      ss << details.size() << " monitors have not enabled msgr2";
+      auto &d = checks->add("MON_MSGR2_NOT_ENABLED", HEALTH_WARN, ss.str(),
+			    details.size());
+      d.detail.swap(details);
+    }
+  }
+}
diff --git a/src/mon/HealthMonitor.h b/src/mon/HealthMonitor.h
new file mode 100644
index 000000000..c0e79d033
--- /dev/null
+++ b/src/mon/HealthMonitor.h
@@ -0,0 +1,75 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#ifndef CEPH_HEALTH_MONITOR_H
+#define CEPH_HEALTH_MONITOR_H
+
+#include "mon/PaxosService.h"
+
+class HealthMonitor : public PaxosService
+{
+  version_t version = 0;
+  std::map<int,health_check_map_t> quorum_checks;  // for each quorum member
+  health_check_map_t leader_checks;           // leader only
+  std::map<std::string,health_mute_t> mutes;
+
+  std::map<std::string,health_mute_t> pending_mutes;
+
+public:
+  HealthMonitor(Monitor &m, Paxos &p, const std::string& service_name);
+
+  /**
+   * @defgroup HealthMonitor_Inherited_h Inherited abstract methods
+   * @{
+   */
+  void init() override;
+
+  bool preprocess_query(MonOpRequestRef op) override;
+  bool prepare_update(MonOpRequestRef op) override;
+
+  void create_initial() override;
+  void update_from_paxos(bool *need_bootstrap) override;
+  void create_pending() override;
+  void encode_pending(MonitorDBStore::TransactionRef t) override;
+  version_t get_trim_to() const override;
+
+  void encode_full(MonitorDBStore::TransactionRef t) override { }
+
+  void tick() override;
+
+  void gather_all_health_checks(health_check_map_t *all);
+  health_status_t get_health_status(
+    bool want_detail,
+    ceph::Formatter *f,
+    std::string *plain,
+    const char *sep1 = " ",
+    const char *sep2 = "; ");
+
+  /**
+   * @} // HealthMonitor_Inherited_h
+   */
+private:
+  bool preprocess_command(MonOpRequestRef op);
+
+  bool prepare_command(MonOpRequestRef op);
+  bool prepare_health_checks(MonOpRequestRef op);
+  void check_for_older_version(health_check_map_t *checks);
+  void check_for_mon_down(health_check_map_t *checks);
+  void check_for_clock_skew(health_check_map_t *checks);
+  void check_if_msgr2_enabled(health_check_map_t *checks);
+  bool check_leader_health();
+  bool check_member_health();
+  bool check_mutes();
+};
+
+#endif // CEPH_HEALTH_MONITOR_H
diff --git a/src/mon/KVMonitor.cc b/src/mon/KVMonitor.cc
new file mode 100644
index 000000000..699cbe417
--- /dev/null
+++ b/src/mon/KVMonitor.cc
@@ -0,0 +1,525 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "mon/Monitor.h"
+#include "mon/KVMonitor.h"
+#include "include/stringify.h"
+#include "messages/MKVData.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, this)
+
+static ostream& _prefix(std::ostream *_dout, const Monitor &mon,
+                        const KVMonitor *hmon) {
+  return *_dout << "mon." << mon.name << "@" << mon.rank
+		<< "(" << mon.get_state_name() << ").kv ";
+}
+
+const string KV_PREFIX = "mon_config_key";
+
+const int MAX_HISTORY = 50;
+
+
+static bool is_binary_string(const string& s)
+{
+  for (auto c : s) {
+    // \n and \t are escaped in JSON; other control characters are not.
+    if ((c < 0x20 && c != '\n' && c != '\t') || c >= 0x7f) {
+      return true;
+    }
+  }
+  return false;
+}
+
+
+KVMonitor::KVMonitor(Monitor &m, Paxos &p, const string& service_name)
+  : PaxosService(m, p, service_name) {
+}
+
+void KVMonitor::init()
+{
+  dout(10) << __func__ << dendl;
+}
+
+void KVMonitor::create_initial()
+{
+  dout(10) << __func__ << dendl;
+  version = 0;
+  pending.clear();
+}
+
+void KVMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  if (version == get_last_committed()) {
+    return;
+  }
+  version = get_last_committed();
+  dout(10) << __func__ << " " << version << dendl;
+  check_all_subs();
+}
+
+void KVMonitor::create_pending()
+{
+  dout(10) << " " << version << dendl;
+  pending.clear();
+}
+
+void KVMonitor::encode_pending(MonitorDBStore::TransactionRef t)
+{
+  dout(10) << " " << (version+1) << dendl;
+  put_last_committed(t, version+1);
+
+  // record the delta for this commit point
+  bufferlist bl;
+  encode(pending, bl);
+  put_version(t, version+1, bl);
+  
+  // make actual changes
+  for (auto& p : pending) {
+    string key = p.first;
+    if (p.second) {
+      dout(20) << __func__ << " set " << key << dendl;
+      t->put(KV_PREFIX, key, *p.second);
+    } else {
+      dout(20) << __func__ << " rm " << key << dendl;
+      t->erase(KV_PREFIX, key);
+    }
+  }
+}
+
+version_t KVMonitor::get_trim_to() const
+{
+  // we don't need that many old states, but keep a few
+  if (version > MAX_HISTORY) {
+    return version - MAX_HISTORY;
+  }
+  return 0;
+}
+
+void KVMonitor::get_store_prefixes(set<string>& s) const
+{
+  s.insert(service_name);
+  s.insert(KV_PREFIX);
+}
+
+void KVMonitor::tick()
+{
+  if (!is_active() || !mon.is_leader()) {
+    return;
+  }
+  dout(10) << __func__ << dendl;
+}
+
+void KVMonitor::on_active()
+{
+}
+
+
+bool KVMonitor::preprocess_query(MonOpRequestRef op)
+{
+  switch (op->get_req()->get_type()) {
+  case MSG_MON_COMMAND:
+    try {
+      return preprocess_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+  }
+  return false;
+}
+
+bool KVMonitor::preprocess_command(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonCommand>();
+  std::stringstream ss;
+  int err = 0;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+  string format;
+  cmd_getval(cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+  string key;
+  cmd_getval(cmdmap, "key", key);
+
+  bufferlist odata;
+
+  if (prefix == "config-key get") {
+    err = mon.store->get(KV_PREFIX, key, odata);
+  }
+  else if (prefix == "config-key exists") {
+    bool exists = mon.store->exists(KV_PREFIX, key);
+    ss << "key '" << key << "'";
+    if (exists) {
+      ss << " exists";
+      err = 0;
+    } else {
+      ss << " doesn't exist";
+      err = -ENOENT;
+    }
+  }
+  else if (prefix == "config-key list" ||
+	   prefix == "config-key ls") {
+    if (!f) {
+      f.reset(Formatter::create("json-pretty"));
+    }
+    KeyValueDB::Iterator iter = mon.store->get_iterator(KV_PREFIX);
+    f->open_array_section("keys");
+    while (iter->valid()) {
+      string key(iter->key());
+      f->dump_string("key", key);
+      iter->next();
+    }
+    f->close_section();
+
+    stringstream tmp_ss;
+    f->flush(tmp_ss);
+    odata.append(tmp_ss);
+    err = 0;
+  }
+  else if (prefix == "config-key dump") {
+    if (!f) {
+      f.reset(Formatter::create("json-pretty"));
+    }
+
+    KeyValueDB::Iterator iter = mon.store->get_iterator(KV_PREFIX);
+    if (key.size()) {
+      iter->lower_bound(key);
+    }
+    f->open_object_section("config-key store");
+    while (iter->valid()) {
+      if (key.size() &&
+	  iter->key().find(key) != 0) {
+	break;
+      }
+      string s = iter->value().to_str();
+      if (is_binary_string(s)) {
+	ostringstream ss;
+	ss << "<<< binary blob of length " << s.size() << " >>>";
+	f->dump_string(iter->key().c_str(), ss.str());
+      } else {
+	f->dump_string(iter->key().c_str(), s);
+      }
+      iter->next();
+    }
+    f->close_section();
+    
+    stringstream tmp_ss;
+    f->flush(tmp_ss);
+    odata.append(tmp_ss);
+    err = 0;
+  }
+  else {
+    return false;
+  }
+
+  mon.reply_command(op, err, ss.str(), odata, get_last_committed());
+  return true;
+}
+
+bool KVMonitor::prepare_update(MonOpRequestRef op)
+{
+  Message *m = op->get_req();
+  dout(7) << "prepare_update " << *m
+	  << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    try {
+      return prepare_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+  }
+  return false;
+}
+
+
+bool KVMonitor::prepare_command(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonCommand>();
+  std::stringstream ss;
+  int err = 0;
+  bufferlist odata;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+  string key;
+  if (!cmd_getval(cmdmap, "key", key)) {
+    err = -EINVAL;
+    ss << "must specify a key";
+    goto reply;
+  }
+
+
+  if (prefix == "config-key set" ||
+      prefix == "config-key put") {
+    bufferlist data;
+    string val;
+    if (cmd_getval(cmdmap, "val", val)) {
+      // they specified a value in the command instead of a file
+      data.append(val);
+    } else if (m->get_data_len() > 0) {
+      // they specified '-i <file>'
+      data = m->get_data();
+    }
+    if (data.length() > (size_t) g_conf()->mon_config_key_max_entry_size) {
+      err = -EFBIG; // File too large
+      ss << "error: entry size limited to "
+         << g_conf()->mon_config_key_max_entry_size << " bytes. "
+         << "Use 'mon config key max entry size' to manually adjust";
+      goto reply;
+    }
+
+    ss << "set " << key;
+    pending[key] = data;
+    goto update;
+  }
+  else if (prefix == "config-key del" ||
+	   prefix == "config-key rm") {
+    ss << "key deleted";
+    pending[key] = boost::none;
+    goto update;
+  }
+  else {
+    ss << "unknown command " << prefix;
+    err = -EINVAL;
+  }
+
+reply:
+  mon.reply_command(op, err, ss.str(), odata, get_last_committed());
+  return false;
+
+update:
+  // see if there is an actual change
+  if (pending.empty()) {
+    err = 0;
+    goto reply;
+  }
+  force_immediate_propose();  // faster response
+  wait_for_finished_proposal(
+    op,
+    new Monitor::C_Command(
+      mon, op, 0, ss.str(), odata,
+      get_last_committed() + 1));
+  return true;
+}
+
+
+
+
+static string _get_dmcrypt_prefix(const uuid_d& uuid, const string k)
+{
+  return "dm-crypt/osd/" + stringify(uuid) + "/" + k;
+}
+
+bool KVMonitor::_have_prefix(const string &prefix)
+{
+  KeyValueDB::Iterator iter = mon.store->get_iterator(KV_PREFIX);
+
+  while (iter->valid()) {
+    string key(iter->key());
+    size_t p = key.find(prefix);
+    if (p != string::npos && p == 0) {
+      return true;
+    }
+    iter->next();
+  }
+  return false;
+}
+
+int KVMonitor::validate_osd_destroy(
+  const int32_t id,
+  const uuid_d& uuid)
+{
+  string dmcrypt_prefix = _get_dmcrypt_prefix(uuid, "");
+  string daemon_prefix =
+    "daemon-private/osd." + stringify(id) + "/";
+
+  if (!_have_prefix(dmcrypt_prefix) &&
+      !_have_prefix(daemon_prefix)) {
+    return -ENOENT;
+  }
+  return 0;
+}
+
+void KVMonitor::do_osd_destroy(int32_t id, uuid_d& uuid)
+{
+  string dmcrypt_prefix = _get_dmcrypt_prefix(uuid, "");
+  string daemon_prefix =
+    "daemon-private/osd." + stringify(id) + "/";
+
+  for (auto& prefix : { dmcrypt_prefix, daemon_prefix }) {
+    KeyValueDB::Iterator iter = mon.store->get_iterator(KV_PREFIX);
+    iter->lower_bound(prefix);
+    if (iter->key().find(prefix) != 0) {
+      break;
+    }
+    pending[iter->key()] = boost::none;
+  }
+
+  propose_pending();
+}
+
+int KVMonitor::validate_osd_new(
+  const uuid_d& uuid,
+  const string& dmcrypt_key,
+  stringstream& ss)
+{
+  string dmcrypt_prefix = _get_dmcrypt_prefix(uuid, "luks");
+  bufferlist value;
+  value.append(dmcrypt_key);
+  
+  if (mon.store->exists(KV_PREFIX, dmcrypt_prefix)) {
+    bufferlist existing_value;
+    int err = mon.store->get(KV_PREFIX, dmcrypt_prefix, existing_value);
+    if (err < 0) {
+      dout(10) << __func__ << " unable to get dm-crypt key from store (r = "
+               << err << ")" << dendl;
+      return err;
+    }
+    if (existing_value.contents_equal(value)) {
+      // both values match; this will be an idempotent op.
+      return EEXIST;
+    }
+    ss << "dm-crypt key already exists and does not match";
+    return -EEXIST;
+  }
+  return 0;
+}
+
+void KVMonitor::do_osd_new(
+  const uuid_d& uuid,
+  const string& dmcrypt_key)
+{
+  ceph_assert(paxos.is_plugged());
+
+  string dmcrypt_key_prefix = _get_dmcrypt_prefix(uuid, "luks");
+  bufferlist dmcrypt_key_value;
+  dmcrypt_key_value.append(dmcrypt_key);
+
+  pending[dmcrypt_key_prefix] = dmcrypt_key_value;
+
+  propose_pending();
+}
+
+
+void KVMonitor::check_sub(MonSession *s)
+{
+  if (!s->authenticated) {
+    dout(20) << __func__ << " not authenticated " << s->entity_name << dendl;
+    return;
+  }
+  for (auto& p : s->sub_map) {
+    if (p.first.find("kv:") == 0) {
+      check_sub(p.second);
+    }
+  }
+}
+
+void KVMonitor::check_sub(Subscription *sub)
+{
+  dout(10) << __func__
+	   << " next " << sub->next
+	   << " have " << version << dendl;
+  if (sub->next <= version) {
+    maybe_send_update(sub);
+    if (sub->onetime) {
+      mon.with_session_map([sub](MonSessionMap& session_map) {
+	  session_map.remove_sub(sub);
+	});
+    }
+  }
+}
+
+void KVMonitor::check_all_subs()
+{
+  dout(10) << __func__ << dendl;
+  int updated = 0, total = 0;
+  for (auto& i : mon.session_map.subs) {
+    if (i.first.find("kv:") == 0) {
+      auto p = i.second->begin();
+      while (!p.end()) {
+	auto sub = *p;
+	++p;
+	++total;
+	if (maybe_send_update(sub)) {
+	  ++updated;
+	}
+      }
+    }
+  }
+  dout(10) << __func__ << " updated " << updated << " / " << total << dendl;
+}
+
+bool KVMonitor::maybe_send_update(Subscription *sub)
+{
+  if (sub->next > version) {
+    return false;
+  }
+
+  auto m = new MKVData;
+  m->prefix = sub->type.substr(3);
+  m->version = version;
+
+  if (sub->next && sub->next > get_first_committed()) {
+    // incremental
+    m->incremental = true;
+
+    for (version_t cur = sub->next; cur <= version; ++cur) {
+      bufferlist bl;
+      int err = get_version(cur, bl);
+      ceph_assert(err == 0);
+
+      std::map<std::string,boost::optional<ceph::buffer::list>> pending;
+      auto p = bl.cbegin();
+      ceph::decode(pending, p);
+
+      for (auto& i : pending) {
+	if (i.first.find(m->prefix) == 0) {
+	  m->data[i.first] = i.second;
+	}
+      }
+    }
+
+    dout(10) << __func__ << " incremental keys for " << m->prefix
+	     << ", v " << sub->next << ".." << version
+	     << ", " << m->data.size() << " keys"
+	     << dendl;
+  } else {
+    m->incremental = false;
+
+    KeyValueDB::Iterator iter = mon.store->get_iterator(KV_PREFIX);
+    iter->lower_bound(m->prefix);
+    while (iter->valid() &&
+	   iter->key().find(m->prefix) == 0) {
+      m->data[iter->key()] = iter->value();
+      iter->next();
+    }
+
+    dout(10) << __func__ << " sending full dump of " << m->prefix
+	     << ", " << m->data.size() << " keys"
+	     << dendl;
+  }
+  sub->session->con->send_message(m);
+  sub->next = version + 1;
+  return true;
+}
diff --git a/src/mon/KVMonitor.h b/src/mon/KVMonitor.h
new file mode 100644
index 000000000..c14c16380
--- /dev/null
+++ b/src/mon/KVMonitor.h
@@ -0,0 +1,69 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <boost/optional.hpp>
+
+#include "mon/PaxosService.h"
+
+class MonSession;
+
+extern const std::string KV_PREFIX;
+
+class KVMonitor : public PaxosService
+{
+  version_t version = 0;
+  std::map<std::string,boost::optional<ceph::buffer::list>> pending;
+
+  bool _have_prefix(const string &prefix);
+
+public:
+  KVMonitor(Monitor &m, Paxos &p, const std::string& service_name);
+
+  void init() override;
+
+  void get_store_prefixes(set<string>& s) const override;
+
+  bool preprocess_command(MonOpRequestRef op);
+  bool prepare_command(MonOpRequestRef op);
+  
+  bool preprocess_query(MonOpRequestRef op) override;
+  bool prepare_update(MonOpRequestRef op) override;
+
+  void create_initial() override;
+  void update_from_paxos(bool *need_bootstrap) override;
+  void create_pending() override;
+  void encode_pending(MonitorDBStore::TransactionRef t) override;
+  version_t get_trim_to() const override;
+
+  void encode_full(MonitorDBStore::TransactionRef t) override { }
+
+  void on_active() override;
+  void tick() override;
+
+  int validate_osd_destroy(const int32_t id, const uuid_d& uuid);
+  void do_osd_destroy(int32_t id, uuid_d& uuid);
+  int validate_osd_new(
+      const uuid_d& uuid,
+      const std::string& dmcrypt_key,
+      std::stringstream& ss);
+  void do_osd_new(const uuid_d& uuid, const std::string& dmcrypt_key);
+
+  void check_sub(MonSession *s);
+  void check_sub(Subscription *sub);
+  void check_all_subs();
+
+  bool maybe_send_update(Subscription *sub);
+
+
+  // used by other services to adjust kv content; note that callers MUST ensure that
+  // propose_pending() is called and a commit is forced to provide atomicity and
+  // proper subscriber notifications.
+  void enqueue_set(const std::string& key, bufferlist &v) {
+    pending[key] = v;
+  }
+  void enqueue_rm(const std::string& key) {
+    pending[key] = boost::none;
+  }
+};
diff --git a/src/mon/LogMonitor.cc b/src/mon/LogMonitor.cc
new file mode 100644
index 000000000..88327663a
--- /dev/null
+++ b/src/mon/LogMonitor.cc
@@ -0,0 +1,947 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include <boost/algorithm/string/predicate.hpp>
+
+#include <sstream>
+#include <syslog.h>
+
+#include "LogMonitor.h"
+#include "Monitor.h"
+#include "MonitorDBStore.h"
+
+#include "messages/MMonCommand.h"
+#include "messages/MLog.h"
+#include "messages/MLogAck.h"
+#include "common/Graylog.h"
+#include "common/errno.h"
+#include "common/strtol.h"
+#include "include/ceph_assert.h"
+#include "include/str_list.h"
+#include "include/str_map.h"
+#include "include/compat.h"
+
+#define dout_subsys ceph_subsys_mon
+
+using namespace TOPNSPC::common;
+
+using std::cerr;
+using std::cout;
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::setfill;
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+using std::unique_ptr;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::make_message;
+using ceph::mono_clock;
+using ceph::mono_time;
+using ceph::timespan_str;
+
+string LogMonitor::log_channel_info::get_log_file(const string &channel)
+{
+  dout(25) << __func__ << " for channel '"
+	   << channel << "'" << dendl;
+
+  if (expanded_log_file.count(channel) == 0) {
+    string fname = expand_channel_meta(
+      get_str_map_key(log_file, channel, &CLOG_CONFIG_DEFAULT_KEY),
+      channel);
+    expanded_log_file[channel] = fname;
+
+    dout(20) << __func__ << " for channel '"
+	     << channel << "' expanded to '"
+	     << fname << "'" << dendl;
+  }
+  return expanded_log_file[channel];
+}
+
+
+void LogMonitor::log_channel_info::expand_channel_meta(map<string,string> &m)
+{
+  dout(20) << __func__ << " expand map: " << m << dendl;
+  for (map<string,string>::iterator p = m.begin(); p != m.end(); ++p) {
+    m[p->first] = expand_channel_meta(p->second, p->first);
+  }
+  dout(20) << __func__ << " expanded map: " << m << dendl;
+}
+
+string LogMonitor::log_channel_info::expand_channel_meta(
+    const string &input,
+    const string &change_to)
+{
+  size_t pos = string::npos;
+  string s(input);
+  while ((pos = s.find(LOG_META_CHANNEL)) != string::npos) {
+    string tmp = s.substr(0, pos) + change_to;
+    if (pos+LOG_META_CHANNEL.length() < s.length())
+      tmp += s.substr(pos+LOG_META_CHANNEL.length());
+    s = tmp;
+  }
+  dout(20) << __func__ << " from '" << input
+	   << "' to '" << s << "'" << dendl;
+
+  return s;
+}
+
+bool LogMonitor::log_channel_info::do_log_to_syslog(const string &channel) {
+  string v = get_str_map_key(log_to_syslog, channel,
+                             &CLOG_CONFIG_DEFAULT_KEY);
+  // We expect booleans, but they are in k/v pairs, kept
+  // as strings, in 'log_to_syslog'. We must ensure
+  // compatibility with existing boolean handling, and so
+  // we are here using a modified version of how
+  // md_config_t::set_val_raw() handles booleans. We will
+  // accept both 'true' and 'false', but will also check for
+  // '1' and '0'. The main distiction between this and the
+  // original code is that we will assume everything not '1',
+  // '0', 'true' or 'false' to be 'false'.
+  bool ret = false;
+
+  if (boost::iequals(v, "false")) {
+    ret = false;
+  } else if (boost::iequals(v, "true")) {
+    ret = true;
+  } else {
+    std::string err;
+    int b = strict_strtol(v.c_str(), 10, &err);
+    ret = (err.empty() && b == 1);
+  }
+
+  return ret;
+}
+
+ceph::logging::Graylog::Ref LogMonitor::log_channel_info::get_graylog(
+    const string &channel)
+{
+  dout(25) << __func__ << " for channel '"
+	   << channel << "'" << dendl;
+
+  if (graylogs.count(channel) == 0) {
+    auto graylog(std::make_shared<ceph::logging::Graylog>("mon"));
+
+    graylog->set_fsid(g_conf().get_val<uuid_d>("fsid"));
+    graylog->set_hostname(g_conf()->host);
+    graylog->set_destination(get_str_map_key(log_to_graylog_host, channel,
+					     &CLOG_CONFIG_DEFAULT_KEY),
+			     atoi(get_str_map_key(log_to_graylog_port, channel,
+						  &CLOG_CONFIG_DEFAULT_KEY).c_str()));
+
+    graylogs[channel] = graylog;
+    dout(20) << __func__ << " for channel '"
+	     << channel << "' to graylog host '"
+	     << log_to_graylog_host[channel] << ":"
+	     << log_to_graylog_port[channel]
+	     << "'" << dendl;
+  }
+  return graylogs[channel];
+}
+
+
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, get_last_committed())
+static ostream& _prefix(std::ostream *_dout, Monitor &mon, version_t v) {
+  return *_dout << "mon." << mon.name << "@" << mon.rank
+		<< "(" << mon.get_state_name()
+		<< ").log v" << v << " ";
+}
+
+ostream& operator<<(ostream &out, const LogMonitor &pm)
+{
+  return out << "log";
+}
+
+/*
+ Tick function to update the map based on performance every N seconds
+*/
+
+void LogMonitor::tick() 
+{
+  if (!is_active()) return;
+
+  dout(10) << *this << dendl;
+
+}
+
+void LogMonitor::create_initial()
+{
+  dout(10) << "create_initial -- creating initial map" << dendl;
+  LogEntry e;
+  e.name = g_conf()->name;
+  e.rank = entity_name_t::MON(mon.rank);
+  e.addrs = mon.messenger->get_myaddrs();
+  e.stamp = ceph_clock_now();
+  e.prio = CLOG_INFO;
+  std::stringstream ss;
+  ss << "mkfs " << mon.monmap->get_fsid();
+  e.msg = ss.str();
+  e.seq = 0;
+  pending_log.insert(pair<utime_t,LogEntry>(e.stamp, e));
+}
+
+void LogMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  dout(10) << __func__ << dendl;
+  version_t version = get_last_committed();
+  dout(10) << __func__ << " version " << version
+           << " summary v " << summary.version << dendl;
+  if (version == summary.version)
+    return;
+  ceph_assert(version >= summary.version);
+
+  map<string,bufferlist> channel_blog;
+
+  version_t latest_full = get_version_latest_full();
+  dout(10) << __func__ << " latest full " << latest_full << dendl;
+  if ((latest_full > 0) && (latest_full > summary.version)) {
+    bufferlist latest_bl;
+    get_version_full(latest_full, latest_bl);
+    ceph_assert(latest_bl.length() != 0);
+    dout(7) << __func__ << " loading summary e" << latest_full << dendl;
+    auto p = latest_bl.cbegin();
+    decode(summary, p);
+    dout(7) << __func__ << " loaded summary e" << summary.version << dendl;
+  }
+
+  // walk through incrementals
+  while (version > summary.version) {
+    bufferlist bl;
+    int err = get_version(summary.version+1, bl);
+    ceph_assert(err == 0);
+    ceph_assert(bl.length());
+
+    auto p = bl.cbegin();
+    __u8 v;
+    decode(v, p);
+    while (!p.end()) {
+      LogEntry le;
+      le.decode(p);
+      dout(7) << "update_from_paxos applying incremental log " << summary.version+1 <<  " " << le << dendl;
+
+      string channel = le.channel;
+      if (channel.empty()) // keep retrocompatibility
+        channel = CLOG_CHANNEL_CLUSTER;
+
+      if (g_conf().get_val<bool>("mon_cluster_log_to_stderr")) {
+	cerr << channel << " " << le << std::endl;
+      }
+
+      if (channels.do_log_to_syslog(channel)) {
+        string level = channels.get_level(channel);
+        string facility = channels.get_facility(channel);
+        if (level.empty() || facility.empty()) {
+          derr << __func__ << " unable to log to syslog -- level or facility"
+               << " not defined (level: " << level << ", facility: "
+               << facility << ")" << dendl;
+          continue;
+        }
+        le.log_to_syslog(channels.get_level(channel),
+                         channels.get_facility(channel));
+      }
+
+      if (channels.do_log_to_graylog(channel)) {
+	ceph::logging::Graylog::Ref graylog = channels.get_graylog(channel);
+	if (graylog) {
+	  graylog->log_log_entry(&le);
+	}
+	dout(7) << "graylog: " << channel << " " << graylog
+		<< " host:" << channels.log_to_graylog_host << dendl;
+      }
+
+      if (g_conf()->mon_cluster_log_to_file) {
+	string log_file = channels.get_log_file(channel);
+	dout(20) << __func__ << " logging for channel '" << channel
+		 << "' to file '" << log_file << "'" << dendl;
+
+	if (!log_file.empty()) {
+	  string log_file_level = channels.get_log_file_level(channel);
+	  if (log_file_level.empty()) {
+	    dout(1) << __func__ << " warning: log file level not defined for"
+		    << " channel '" << channel << "' yet a log file is --"
+		    << " will assume lowest level possible" << dendl;
+	  }
+
+	  int min = string_to_syslog_level(log_file_level);
+	  int l = clog_type_to_syslog_level(le.prio);
+	  if (l <= min) {
+	    stringstream ss;
+	    ss << le << "\n";
+	    // init entry if DNE
+	    bufferlist &blog = channel_blog[channel];
+	    blog.append(ss.str());
+	  }
+	}
+      }
+
+      summary.add(le);
+    }
+
+    summary.version++;
+    summary.prune(g_conf()->mon_log_max_summary);
+  }
+
+  dout(15) << __func__ << " logging for "
+           << channel_blog.size() << " channels" << dendl;
+  for(map<string,bufferlist>::iterator p = channel_blog.begin();
+      p != channel_blog.end(); ++p) {
+    if (!p->second.length()) {
+      dout(15) << __func__ << " channel '" << p->first
+               << "': nothing to log" << dendl;
+      continue;
+    }
+
+    dout(15) << __func__ << " channel '" << p->first
+             << "' logging " << p->second.length() << " bytes" << dendl;
+    string log_file = channels.get_log_file(p->first);
+
+    int fd = ::open(log_file.c_str(), O_WRONLY|O_APPEND|O_CREAT|O_CLOEXEC, 0600);
+    if (fd < 0) {
+      int err = -errno;
+      dout(1) << "unable to write to '" << log_file << "' for channel '"
+              << p->first << "': " << cpp_strerror(err) << dendl;
+    } else {
+      int err = p->second.write_fd(fd);
+      if (err < 0) {
+	dout(1) << "error writing to '" << log_file << "' for channel '"
+                << p->first << ": " << cpp_strerror(err) << dendl;
+      }
+      VOID_TEMP_FAILURE_RETRY(::close(fd));
+    }
+  }
+
+  check_subs();
+}
+
+void LogMonitor::create_pending()
+{
+  pending_log.clear();
+  pending_summary = summary;
+  dout(10) << "create_pending v " << (get_last_committed() + 1) << dendl;
+}
+
+void LogMonitor::encode_pending(MonitorDBStore::TransactionRef t)
+{
+  version_t version = get_last_committed() + 1;
+  bufferlist bl;
+  dout(10) << __func__ << " v" << version << dendl;
+  __u8 v = 1;
+  encode(v, bl);
+  for (auto p = pending_log.begin(); p != pending_log.end(); ++p)
+    p->second.encode(bl, mon.get_quorum_con_features());
+
+  put_version(t, version, bl);
+  put_last_committed(t, version);
+}
+
+void LogMonitor::encode_full(MonitorDBStore::TransactionRef t)
+{
+  dout(10) << __func__ << " log v " << summary.version << dendl;
+  ceph_assert(get_last_committed() == summary.version);
+
+  bufferlist summary_bl;
+  encode(summary, summary_bl, mon.get_quorum_con_features());
+
+  put_version_full(t, summary.version, summary_bl);
+  put_version_latest_full(t, summary.version);
+}
+
+version_t LogMonitor::get_trim_to() const
+{
+  if (!mon.is_leader())
+    return 0;
+
+  unsigned max = g_conf()->mon_max_log_epochs;
+  version_t version = get_last_committed();
+  if (version > max)
+    return version - max;
+  return 0;
+}
+
+bool LogMonitor::preprocess_query(MonOpRequestRef op)
+{
+  op->mark_logmon_event("preprocess_query");
+  auto m = op->get_req<PaxosServiceMessage>();
+  dout(10) << "preprocess_query " << *m << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    try {
+      return preprocess_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+
+  case MSG_LOG:
+    return preprocess_log(op);
+
+  default:
+    ceph_abort();
+    return true;
+  }
+}
+
+bool LogMonitor::prepare_update(MonOpRequestRef op)
+{
+  op->mark_logmon_event("prepare_update");
+  auto m = op->get_req<PaxosServiceMessage>();
+  dout(10) << "prepare_update " << *m << " from " << m->get_orig_source_inst() << dendl;
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    try {
+      return prepare_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+  case MSG_LOG:
+    return prepare_log(op);
+  default:
+    ceph_abort();
+    return false;
+  }
+}
+
+bool LogMonitor::preprocess_log(MonOpRequestRef op)
+{
+  op->mark_logmon_event("preprocess_log");
+  auto m = op->get_req<MLog>();
+  dout(10) << "preprocess_log " << *m << " from " << m->get_orig_source() << dendl;
+  int num_new = 0;
+
+  MonSession *session = op->get_session();
+  if (!session)
+    goto done;
+  if (!session->is_capable("log", MON_CAP_W)) {
+    dout(0) << "preprocess_log got MLog from entity with insufficient privileges "
+	    << session->caps << dendl;
+    goto done;
+  }
+  
+  for (auto p = m->entries.begin();
+       p != m->entries.end();
+       ++p) {
+    if (!pending_summary.contains(p->key()))
+      num_new++;
+  }
+  if (!num_new) {
+    dout(10) << "  nothing new" << dendl;
+    goto done;
+  }
+
+  return false;
+
+ done:
+  mon.no_reply(op);
+  return true;
+}
+
+struct LogMonitor::C_Log : public C_MonOp {
+  LogMonitor *logmon;
+  C_Log(LogMonitor *p, MonOpRequestRef o) :
+    C_MonOp(o), logmon(p) {}
+  void _finish(int r) override {
+    if (r == -ECANCELED) {
+      return;
+    }
+    logmon->_updated_log(op);
+  }
+};
+
+bool LogMonitor::prepare_log(MonOpRequestRef op) 
+{
+  op->mark_logmon_event("prepare_log");
+  auto m = op->get_req<MLog>();
+  dout(10) << "prepare_log " << *m << " from " << m->get_orig_source() << dendl;
+
+  if (m->fsid != mon.monmap->fsid) {
+    dout(0) << "handle_log on fsid " << m->fsid << " != " << mon.monmap->fsid 
+	    << dendl;
+    return false;
+  }
+
+  for (auto p = m->entries.begin();
+       p != m->entries.end();
+       ++p) {
+    dout(10) << " logging " << *p << dendl;
+    if (!pending_summary.contains(p->key())) {
+      pending_summary.add(*p);
+      pending_log.insert(pair<utime_t,LogEntry>(p->stamp, *p));
+    }
+  }
+  pending_summary.prune(g_conf()->mon_log_max_summary);
+  wait_for_finished_proposal(op, new C_Log(this, op));
+  return true;
+}
+
+void LogMonitor::_updated_log(MonOpRequestRef op)
+{
+  auto m = op->get_req<MLog>();
+  dout(7) << "_updated_log for " << m->get_orig_source_inst() << dendl;
+  mon.send_reply(op, new MLogAck(m->fsid, m->entries.rbegin()->seq));
+}
+
+bool LogMonitor::should_propose(double& delay)
+{
+  // commit now if we have a lot of pending events
+  if (g_conf()->mon_max_log_entries_per_event > 0 &&
+      pending_log.size() >= (unsigned)g_conf()->mon_max_log_entries_per_event)
+    return true;
+
+  // otherwise fall back to generic policy
+  return PaxosService::should_propose(delay);
+}
+
+
+bool LogMonitor::preprocess_command(MonOpRequestRef op)
+{
+  op->mark_logmon_event("preprocess_command");
+  auto m = op->get_req<MMonCommand>();
+  int r = -EINVAL;
+  bufferlist rdata;
+  stringstream ss;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+  MonSession *session = op->get_session();
+  if (!session) {
+    mon.reply_command(op, -EACCES, "access denied", get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+
+  string format;
+  cmd_getval(cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+  if (prefix == "log last") {
+    int64_t num = 20;
+    cmd_getval(cmdmap, "num", num);
+    if (f) {
+      f->open_array_section("tail");
+    }
+
+    std::string level_str;
+    clog_type level;
+    if (cmd_getval(cmdmap, "level", level_str)) {
+      level = LogEntry::str_to_level(level_str);
+      if (level == CLOG_UNKNOWN) {
+        ss << "Invalid severity '" << level_str << "'";
+        mon.reply_command(op, -EINVAL, ss.str(), get_last_committed());
+        return true;
+      }
+    } else {
+      level = CLOG_INFO;
+    }
+
+    std::string channel;
+    if (!cmd_getval(cmdmap, "channel", channel)) {
+      channel = CLOG_CHANNEL_DEFAULT;
+    }
+
+    // We'll apply this twice, once while counting out lines
+    // and once while outputting them.
+    auto match = [level](const LogEntry &entry) {
+      return entry.prio >= level;
+    };
+
+    // Decrement operation that sets to container end when hitting rbegin
+    ostringstream ss;
+    if (channel == "*") {
+      list<LogEntry> full_tail;
+      summary.build_ordered_tail(&full_tail);
+      derr << "full " << full_tail << dendl;
+      auto rp = full_tail.rbegin();
+      for (; num > 0 && rp != full_tail.rend(); ++rp) {
+	if (match(*rp)) {
+	  num--;
+	}
+      }
+      if (rp == full_tail.rend()) {
+	--rp;
+      }
+
+      // Decrement a reverse iterator such that going past rbegin()
+      // sets it to rend().  This is for writing a for() loop that
+      // goes up to (and including) rbegin()
+      auto dec = [&rp, &full_tail] () {
+        if (rp == full_tail.rbegin()) {
+          rp = full_tail.rend();
+        } else {
+          --rp;
+        }
+      };
+
+      // Move forward to the end of the container (decrement the reverse
+      // iterator).
+      for (; rp != full_tail.rend(); dec()) {
+	if (!match(*rp)) {
+	  continue;
+	}
+	if (f) {
+	  f->dump_object("entry", *rp);
+	} else {
+	  ss << *rp << "\n";
+	}
+      }
+    } else {
+      auto p = summary.tail_by_channel.find(channel);
+      if (p != summary.tail_by_channel.end()) {
+	auto rp = p->second.rbegin();
+	for (; num > 0 && rp != p->second.rend(); ++rp) {
+	  if (match(rp->second)) {
+	    num--;
+	  }
+	}
+	if (rp == p->second.rend()) {
+	  --rp;
+	}
+
+        // Decrement a reverse iterator such that going past rbegin()
+        // sets it to rend().  This is for writing a for() loop that
+        // goes up to (and including) rbegin()
+        auto dec = [&rp, &p] () {
+          if (rp == p->second.rbegin()) {
+            rp = p->second.rend();
+          } else {
+            --rp;
+          }
+        };
+
+        // Move forward to the end of the container (decrement the reverse
+        // iterator).
+	for (; rp != p->second.rend(); dec()) {
+	  if (!match(rp->second)) {
+	    continue;
+	  }
+	  if (f) {
+	    f->dump_object("entry", rp->second);
+	  } else {
+	    ss << rp->second << "\n";
+	  }
+	}
+      }
+    }
+    if (f) {
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      rdata.append(ss.str());
+    }
+    r = 0;
+  } else {
+    return false;
+  }
+
+  string rs;
+  getline(ss, rs);
+  mon.reply_command(op, r, rs, rdata, get_last_committed());
+  return true;
+}
+
+
+bool LogMonitor::prepare_command(MonOpRequestRef op)
+{
+  op->mark_logmon_event("prepare_command");
+  auto m = op->get_req<MMonCommand>();
+  stringstream ss;
+  string rs;
+  int err = -EINVAL;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    // ss has reason for failure
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    mon.reply_command(op, -EACCES, "access denied", get_last_committed());
+    return true;
+  }
+
+  if (prefix == "log") {
+    vector<string> logtext;
+    string level_str;
+    cmd_getval(cmdmap, "logtext", logtext);
+    LogEntry le;
+    le.rank = m->get_orig_source();
+    le.addrs.v.push_back(m->get_orig_source_addr());
+    le.name = session->entity_name;
+    le.stamp = m->get_recv_stamp();
+    le.seq = 0;
+    cmd_getval(cmdmap, "level", level_str, string("info"));
+    le.prio = LogEntry::str_to_level(level_str);
+    le.channel = CLOG_CHANNEL_DEFAULT;
+    le.msg = str_join(logtext, " ");
+    pending_summary.add(le);
+    pending_summary.prune(g_conf()->mon_log_max_summary);
+    pending_log.insert(pair<utime_t,LogEntry>(le.stamp, le));
+    wait_for_finished_proposal(op, new Monitor::C_Command(
+          mon, op, 0, string(), get_last_committed() + 1));
+    return true;
+  }
+
+  getline(ss, rs);
+  mon.reply_command(op, err, rs, get_last_committed());
+  return false;
+}
+
+
+int LogMonitor::sub_name_to_id(const string& n)
+{
+  if (n.substr(0, 4) == "log-" && n.size() > 4) {
+    return LogEntry::str_to_level(n.substr(4));
+  } else {
+    return CLOG_UNKNOWN;
+  }
+}
+
+void LogMonitor::check_subs()
+{
+  dout(10) << __func__ << dendl;
+  for (map<string, xlist<Subscription*>*>::iterator i = mon.session_map.subs.begin();
+       i != mon.session_map.subs.end();
+       ++i) {
+    for (xlist<Subscription*>::iterator j = i->second->begin(); !j.end(); ++j) {
+      if (sub_name_to_id((*j)->type) >= 0)
+	check_sub(*j);
+    }
+  }
+}
+
+void LogMonitor::check_sub(Subscription *s)
+{
+  dout(10) << __func__ << " client wants " << s->type << " ver " << s->next << dendl;
+
+  int sub_level = sub_name_to_id(s->type);
+  ceph_assert(sub_level >= 0);
+
+  version_t summary_version = summary.version;
+  if (s->next > summary_version) {
+    dout(10) << __func__ << " client " << s->session->name
+	    << " requested version (" << s->next << ") is greater than ours (" 
+	    << summary_version << "), which means we already sent him" 
+	    << " everything we have." << dendl;
+    return;
+  } 
+ 
+  MLog *mlog = new MLog(mon.monmap->fsid);
+
+  if (s->next == 0) { 
+    /* First timer, heh? */
+    _create_sub_incremental(mlog, sub_level, get_last_committed());
+  } else {
+    /* let us send you an incremental log... */
+    _create_sub_incremental(mlog, sub_level, s->next);
+  }
+
+  dout(10) << __func__ << " sending message to " << s->session->name
+	  << " with " << mlog->entries.size() << " entries"
+	  << " (version " << mlog->version << ")" << dendl;
+  
+  if (!mlog->entries.empty()) {
+    s->session->con->send_message(mlog);
+  } else {
+    mlog->put();
+  }
+  if (s->onetime)
+    mon.session_map.remove_sub(s);
+  else
+    s->next = summary_version+1;
+}
+
+/**
+ * Create an incremental log message from version \p sv to \p summary.version
+ *
+ * @param mlog	Log message we'll send to the client with the messages received
+ *		since version \p sv, inclusive.
+ * @param level	The max log level of the messages the client is interested in.
+ * @param sv	The version the client is looking for.
+ */
+void LogMonitor::_create_sub_incremental(MLog *mlog, int level, version_t sv)
+{
+  dout(10) << __func__ << " level " << level << " ver " << sv 
+	  << " cur summary ver " << summary.version << dendl; 
+
+  if (sv < get_first_committed()) {
+    dout(10) << __func__ << " skipped from " << sv
+	     << " to first_committed " << get_first_committed() << dendl;
+    LogEntry le;
+    le.stamp = ceph_clock_now();
+    le.prio = CLOG_WARN;
+    ostringstream ss;
+    ss << "skipped log messages from " << sv << " to " << get_first_committed();
+    le.msg = ss.str();
+    mlog->entries.push_back(le);
+    sv = get_first_committed();
+  }
+
+  version_t summary_ver = summary.version;
+  while (sv && sv <= summary_ver) {
+    bufferlist bl;
+    int err = get_version(sv, bl);
+    ceph_assert(err == 0);
+    ceph_assert(bl.length());
+    auto p = bl.cbegin();
+    __u8 v;
+    decode(v,p);
+    while (!p.end()) {
+      LogEntry le;
+      le.decode(p);
+
+      if (le.prio < level) {
+	dout(20) << __func__ << " requested " << level 
+		 << " entry " << le.prio << dendl;
+	continue;
+      }
+
+      mlog->entries.push_back(le);
+    }
+    mlog->version = sv++;
+  }
+
+  dout(10) << __func__ << " incremental message ready (" 
+	   << mlog->entries.size() << " entries)" << dendl;
+}
+
+void LogMonitor::update_log_channels()
+{
+  ostringstream oss;
+
+  channels.clear();
+
+  int r = get_conf_str_map_helper(
+    g_conf().get_val<string>("mon_cluster_log_to_syslog"),
+    oss, &channels.log_to_syslog,
+    CLOG_CONFIG_DEFAULT_KEY);
+  if (r < 0) {
+    derr << __func__ << " error parsing 'mon_cluster_log_to_syslog'" << dendl;
+    return;
+  }
+
+  r = get_conf_str_map_helper(
+    g_conf().get_val<string>("mon_cluster_log_to_syslog_level"),
+    oss, &channels.syslog_level,
+    CLOG_CONFIG_DEFAULT_KEY);
+  if (r < 0) {
+    derr << __func__ << " error parsing 'mon_cluster_log_to_syslog_level'"
+         << dendl;
+    return;
+  }
+
+  r = get_conf_str_map_helper(
+    g_conf().get_val<string>("mon_cluster_log_to_syslog_facility"),
+    oss, &channels.syslog_facility,
+    CLOG_CONFIG_DEFAULT_KEY);
+  if (r < 0) {
+    derr << __func__ << " error parsing 'mon_cluster_log_to_syslog_facility'"
+         << dendl;
+    return;
+  }
+
+  r = get_conf_str_map_helper(
+    g_conf().get_val<string>("mon_cluster_log_file"), oss,
+    &channels.log_file,
+    CLOG_CONFIG_DEFAULT_KEY);
+  if (r < 0) {
+    derr << __func__ << " error parsing 'mon_cluster_log_file'" << dendl;
+    return;
+  }
+
+  r = get_conf_str_map_helper(
+    g_conf().get_val<string>("mon_cluster_log_file_level"), oss,
+    &channels.log_file_level,
+    CLOG_CONFIG_DEFAULT_KEY);
+  if (r < 0) {
+    derr << __func__ << " error parsing 'mon_cluster_log_file_level'"
+         << dendl;
+    return;
+  }
+
+  r = get_conf_str_map_helper(
+    g_conf().get_val<string>("mon_cluster_log_to_graylog"), oss,
+    &channels.log_to_graylog,
+    CLOG_CONFIG_DEFAULT_KEY);
+  if (r < 0) {
+    derr << __func__ << " error parsing 'mon_cluster_log_to_graylog'"
+         << dendl;
+    return;
+  }
+
+  r = get_conf_str_map_helper(
+    g_conf().get_val<string>("mon_cluster_log_to_graylog_host"), oss,
+    &channels.log_to_graylog_host,
+    CLOG_CONFIG_DEFAULT_KEY);
+  if (r < 0) {
+    derr << __func__ << " error parsing 'mon_cluster_log_to_graylog_host'"
+         << dendl;
+    return;
+  }
+
+  r = get_conf_str_map_helper(
+    g_conf().get_val<string>("mon_cluster_log_to_graylog_port"), oss,
+    &channels.log_to_graylog_port,
+    CLOG_CONFIG_DEFAULT_KEY);
+  if (r < 0) {
+    derr << __func__ << " error parsing 'mon_cluster_log_to_graylog_port'"
+         << dendl;
+    return;
+  }
+
+  channels.expand_channel_meta();
+}
+
+
+void LogMonitor::handle_conf_change(const ConfigProxy& conf,
+                                    const std::set<std::string> &changed)
+{
+  if (changed.count("mon_cluster_log_to_syslog") ||
+      changed.count("mon_cluster_log_to_syslog_level") ||
+      changed.count("mon_cluster_log_to_syslog_facility") ||
+      changed.count("mon_cluster_log_file") ||
+      changed.count("mon_cluster_log_file_level") ||
+      changed.count("mon_cluster_log_to_graylog") ||
+      changed.count("mon_cluster_log_to_graylog_host") ||
+      changed.count("mon_cluster_log_to_graylog_port")) {
+    update_log_channels();
+  }
+}
diff --git a/src/mon/LogMonitor.h b/src/mon/LogMonitor.h
new file mode 100644
index 000000000..6d6a0b71c
--- /dev/null
+++ b/src/mon/LogMonitor.h
@@ -0,0 +1,189 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_LOGMONITOR_H
+#define CEPH_LOGMONITOR_H
+
+#include <map>
+#include <set>
+
+#include "include/types.h"
+#include "PaxosService.h"
+
+#include "common/config_fwd.h"
+#include "common/LogEntry.h"
+#include "include/str_map.h"
+
+class MLog;
+
+static const std::string LOG_META_CHANNEL = "$channel";
+
+namespace ceph {
+namespace logging {
+  class Graylog;
+}
+}
+
+class LogMonitor : public PaxosService,
+                   public md_config_obs_t {
+private:
+  std::multimap<utime_t,LogEntry> pending_log;
+  LogSummary pending_summary, summary;
+
+  struct log_channel_info {
+
+    std::map<std::string,std::string> log_to_syslog;
+    std::map<std::string,std::string> syslog_level;
+    std::map<std::string,std::string> syslog_facility;
+    std::map<std::string,std::string> log_file;
+    std::map<std::string,std::string> expanded_log_file;
+    std::map<std::string,std::string> log_file_level;
+    std::map<std::string,std::string> log_to_graylog;
+    std::map<std::string,std::string> log_to_graylog_host;
+    std::map<std::string,std::string> log_to_graylog_port;
+
+    std::map<std::string, std::shared_ptr<ceph::logging::Graylog>> graylogs;
+    uuid_d fsid;
+    std::string host;
+
+    void clear() {
+      log_to_syslog.clear();
+      syslog_level.clear();
+      syslog_facility.clear();
+      log_file.clear();
+      expanded_log_file.clear();
+      log_file_level.clear();
+      log_to_graylog.clear();
+      log_to_graylog_host.clear();
+      log_to_graylog_port.clear();
+      graylogs.clear();
+    }
+
+    /** expands $channel meta variable on all maps *EXCEPT* log_file
+     *
+     * We won't expand the log_file map meta variables here because we
+     * intend to do that selectively during get_log_file()
+     */
+    void expand_channel_meta() {
+      expand_channel_meta(log_to_syslog);
+      expand_channel_meta(syslog_level);
+      expand_channel_meta(syslog_facility);
+      expand_channel_meta(log_file_level);
+    }
+    void expand_channel_meta(std::map<std::string,std::string> &m);
+    std::string expand_channel_meta(const std::string &input,
+				    const std::string &change_to);
+
+    bool do_log_to_syslog(const std::string &channel);
+
+    std::string get_facility(const std::string &channel) {
+      return get_str_map_key(syslog_facility, channel,
+                             &CLOG_CONFIG_DEFAULT_KEY);
+    }
+
+    std::string get_level(const std::string &channel) {
+      return get_str_map_key(syslog_level, channel,
+                             &CLOG_CONFIG_DEFAULT_KEY);
+    }
+
+    std::string get_log_file(const std::string &channel);
+
+    std::string get_log_file_level(const std::string &channel) {
+      return get_str_map_key(log_file_level, channel,
+                             &CLOG_CONFIG_DEFAULT_KEY);
+    }
+
+    bool do_log_to_graylog(const std::string &channel) {
+      return (get_str_map_key(log_to_graylog, channel,
+			      &CLOG_CONFIG_DEFAULT_KEY) == "true");
+    }
+
+    std::shared_ptr<ceph::logging::Graylog> get_graylog(const std::string &channel);
+  } channels;
+
+  void update_log_channels();
+
+  void create_initial() override;
+  void update_from_paxos(bool *need_bootstrap) override;
+  void create_pending() override;  // prepare a new pending
+  // propose pending update to peers
+  void encode_pending(MonitorDBStore::TransactionRef t) override;
+  void encode_full(MonitorDBStore::TransactionRef t) override;
+  version_t get_trim_to() const override;
+  bool preprocess_query(MonOpRequestRef op) override;  // true if processed.
+  bool prepare_update(MonOpRequestRef op) override;
+
+  bool preprocess_log(MonOpRequestRef op);
+  bool prepare_log(MonOpRequestRef op);
+  void _updated_log(MonOpRequestRef op);
+
+  bool should_propose(double& delay) override;
+
+  bool should_stash_full() override {
+    // commit a LogSummary on every commit
+    return true;
+  }
+
+  struct C_Log;
+
+  bool preprocess_command(MonOpRequestRef op);
+  bool prepare_command(MonOpRequestRef op);
+
+  void _create_sub_incremental(MLog *mlog, int level, version_t sv);
+
+ public:
+  LogMonitor(Monitor &mn, Paxos &p, const std::string& service_name)
+    : PaxosService(mn, p, service_name) { }
+
+  void init() override {
+    generic_dout(10) << "LogMonitor::init" << dendl;
+    g_conf().add_observer(this);
+    update_log_channels();
+  }
+  
+  void tick() override;  // check state, take actions
+
+  void check_subs();
+  void check_sub(Subscription *s);
+
+  /**
+   * translate log sub name ('log-info') to integer id
+   *
+   * @param n name
+   * @return id, or -1 if unrecognized
+   */
+  int sub_name_to_id(const std::string& n);
+
+  void on_shutdown() override {
+    g_conf().remove_observer(this);
+  }
+
+  const char **get_tracked_conf_keys() const override {
+    static const char* KEYS[] = {
+      "mon_cluster_log_to_syslog",
+      "mon_cluster_log_to_syslog_level",
+      "mon_cluster_log_to_syslog_facility",
+      "mon_cluster_log_file",
+      "mon_cluster_log_file_level",
+      "mon_cluster_log_to_graylog",
+      "mon_cluster_log_to_graylog_host",
+      "mon_cluster_log_to_graylog_port",
+      NULL
+    };
+    return KEYS;
+  }
+  void handle_conf_change(const ConfigProxy& conf,
+                          const std::set<std::string> &changed) override;
+};
+#endif
diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
new file mode 100644
index 000000000..2ec7a2018
--- /dev/null
+++ b/src/mon/MDSMonitor.cc
@@ -0,0 +1,2370 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include <regex>
+#include <sstream>
+#include <boost/utility.hpp>
+
+#include "MDSMonitor.h"
+#include "FSCommands.h"
+#include "Monitor.h"
+#include "MonitorDBStore.h"
+#include "OSDMonitor.h"
+
+#include "common/strtol.h"
+#include "common/perf_counters.h"
+#include "common/config.h"
+#include "common/cmdparse.h"
+#include "messages/MMDSMap.h"
+#include "messages/MFSMap.h"
+#include "messages/MFSMapUser.h"
+#include "messages/MMDSLoadTargets.h"
+#include "messages/MMonCommand.h"
+#include "messages/MGenericMessage.h"
+
+#include "include/ceph_assert.h"
+#include "include/str_list.h"
+#include "include/stringify.h"
+#include "mds/mdstypes.h"
+#include "Session.h"
+
+using namespace TOPNSPC::common;
+
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::string;
+using std::string_view;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::ErasureCodeInterfaceRef;
+using ceph::ErasureCodeProfile;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::make_message;
+using ceph::mono_clock;
+using ceph::mono_time;
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, get_fsmap())
+static ostream& _prefix(std::ostream *_dout, Monitor &mon, const FSMap& fsmap) {
+  return *_dout << "mon." << mon.name << "@" << mon.rank
+		<< "(" << mon.get_state_name()
+		<< ").mds e" << fsmap.get_epoch() << " ";
+}
+
+static const string MDS_METADATA_PREFIX("mds_metadata");
+static const string MDS_HEALTH_PREFIX("mds_health");
+
+
+/*
+ * Specialized implementation of cmd_getval to allow us to parse
+ * out strongly-typedef'd types
+ */
+namespace TOPNSPC::common {
+template<> bool cmd_getval(const cmdmap_t& cmdmap,
+			   const std::string& k, mds_gid_t &val)
+{
+  return cmd_getval(cmdmap, k, (int64_t&)val);
+}
+
+template<> bool cmd_getval(const cmdmap_t& cmdmap,
+			   const std::string& k, mds_rank_t &val)
+{
+  return cmd_getval(cmdmap, k, (int64_t&)val);
+}
+
+template<> bool cmd_getval(const cmdmap_t& cmdmap,
+			   const std::string& k, MDSMap::DaemonState &val)
+{
+  return cmd_getval(cmdmap, k, (int64_t&)val);
+}
+}
+// my methods
+
+template <int dblV>
+void MDSMonitor::print_map(const FSMap& m)
+{
+  dout(dblV) << "print_map\n";
+  m.print(*_dout);
+  *_dout << dendl;
+}
+
+// service methods
+void MDSMonitor::create_initial()
+{
+  dout(10) << "create_initial" << dendl;
+}
+
+void MDSMonitor::get_store_prefixes(std::set<string>& s) const
+{
+  s.insert(service_name);
+  s.insert(MDS_METADATA_PREFIX);
+  s.insert(MDS_HEALTH_PREFIX);
+}
+
+void MDSMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  version_t version = get_last_committed();
+  if (version == get_fsmap().epoch)
+    return;
+
+  dout(10) << __func__ << " version " << version
+	   << ", my e " << get_fsmap().epoch << dendl;
+  ceph_assert(version > get_fsmap().epoch);
+
+  load_health();
+
+  // read and decode
+  bufferlist fsmap_bl;
+  fsmap_bl.clear();
+  int err = get_version(version, fsmap_bl);
+  ceph_assert(err == 0);
+
+  ceph_assert(fsmap_bl.length() > 0);
+  dout(10) << __func__ << " got " << version << dendl;
+  try {
+    PaxosFSMap::decode(fsmap_bl);
+  } catch (const ceph::buffer::malformed_input& e) {
+    derr << "unable to decode FSMap: " << e.what() << dendl;
+    throw;
+  }
+
+  // new map
+  dout(0) << "new map" << dendl;
+  print_map<0>(get_fsmap());
+  if (!g_conf()->mon_mds_skip_sanity) {
+    get_fsmap().sanity();
+  }
+
+  check_subs();
+}
+
+void MDSMonitor::init()
+{
+  (void)load_metadata(pending_metadata);
+}
+
+void MDSMonitor::create_pending()
+{
+  auto &fsmap = PaxosFSMap::create_pending();
+
+  if (mon.osdmon()->is_readable()) {
+    const auto &osdmap = mon.osdmon()->osdmap;
+    fsmap.sanitize([&osdmap](int64_t pool){return osdmap.have_pg_pool(pool);});
+  }
+
+  dout(10) << "create_pending e" << fsmap.epoch << dendl;
+}
+
+void MDSMonitor::encode_pending(MonitorDBStore::TransactionRef t)
+{
+  auto &pending = get_pending_fsmap_writeable();
+  auto &epoch = pending.epoch;
+
+  dout(10) << "encode_pending e" << epoch << dendl;
+
+  // print map iff 'debug mon = 30' or higher
+  print_map<30>(pending);
+  if (!g_conf()->mon_mds_skip_sanity) {
+    pending.sanity(true);
+  }
+
+  // Set 'modified' on maps modified this epoch
+  for (auto &p : pending.filesystems) {
+    if (p.second->mds_map.epoch == epoch) {
+      p.second->mds_map.modified = ceph_clock_now();
+    }
+  }
+
+  // apply to paxos
+  ceph_assert(get_last_committed() + 1 == pending.epoch);
+  bufferlist pending_bl;
+  pending.encode(pending_bl, mon.get_quorum_con_features());
+
+  /* put everything in the transaction */
+  put_version(t, pending.epoch, pending_bl);
+  put_last_committed(t, pending.epoch);
+
+  // Encode MDSHealth data
+  for (std::map<uint64_t, MDSHealth>::iterator i = pending_daemon_health.begin();
+      i != pending_daemon_health.end(); ++i) {
+    bufferlist bl;
+    i->second.encode(bl);
+    t->put(MDS_HEALTH_PREFIX, stringify(i->first), bl);
+  }
+
+  for (std::set<uint64_t>::iterator i = pending_daemon_health_rm.begin();
+      i != pending_daemon_health_rm.end(); ++i) {
+    t->erase(MDS_HEALTH_PREFIX, stringify(*i));
+  }
+  pending_daemon_health_rm.clear();
+  remove_from_metadata(pending, t);
+
+  // health
+  health_check_map_t new_checks;
+  const auto &info_map = pending.get_mds_info();
+  for (const auto &i : info_map) {
+    const auto &gid = i.first;
+    const auto &info = i.second;
+    if (pending_daemon_health_rm.count(gid)) {
+      continue;
+    }
+    MDSHealth health;
+    auto p = pending_daemon_health.find(gid);
+    if (p != pending_daemon_health.end()) {
+      health = p->second;
+    } else {
+      bufferlist bl;
+      mon.store->get(MDS_HEALTH_PREFIX, stringify(gid), bl);
+      if (!bl.length()) {
+	derr << "Missing health data for MDS " << gid << dendl;
+	continue;
+      }
+      auto bl_i = bl.cbegin();
+      health.decode(bl_i);
+    }
+    for (const auto &metric : health.metrics) {
+      if (metric.type == MDS_HEALTH_DUMMY) {
+        continue;
+      }
+      const auto rank = info.rank;
+      health_check_t *check = &new_checks.get_or_add(
+	mds_metric_name(metric.type),
+	metric.sev,
+	mds_metric_summary(metric.type),
+	1);
+      ostringstream ss;
+      ss << "mds." << info.name << "(mds." << rank << "): " << metric.message;
+      bool first = true;
+      for (auto &p : metric.metadata) {
+	if (first) {
+	  ss << " ";
+	} else {
+	  ss << ", ";
+        }
+	ss << p.first << ": " << p.second;
+        first = false;
+      }
+      check->detail.push_back(ss.str());
+    }
+  }
+  pending.get_health_checks(&new_checks);
+  for (auto& p : new_checks.checks) {
+    p.second.summary = std::regex_replace(
+      p.second.summary,
+      std::regex("%num%"),
+      stringify(p.second.detail.size()));
+    p.second.summary = std::regex_replace(
+      p.second.summary,
+      std::regex("%plurals%"),
+      p.second.detail.size() > 1 ? "s" : "");
+    p.second.summary = std::regex_replace(
+      p.second.summary,
+      std::regex("%isorare%"),
+      p.second.detail.size() > 1 ? "are" : "is");
+    p.second.summary = std::regex_replace(
+      p.second.summary,
+      std::regex("%hasorhave%"),
+      p.second.detail.size() > 1 ? "have" : "has");
+  }
+  encode_health(new_checks, t);
+}
+
+version_t MDSMonitor::get_trim_to() const
+{
+  version_t floor = 0;
+  if (g_conf()->mon_mds_force_trim_to > 0 &&
+      g_conf()->mon_mds_force_trim_to <= (int)get_last_committed()) {
+    floor = g_conf()->mon_mds_force_trim_to;
+    dout(10) << __func__ << " explicit mon_mds_force_trim_to = "
+             << floor << dendl;
+  }
+
+  unsigned max = g_conf()->mon_max_mdsmap_epochs;
+  version_t last = get_last_committed();
+
+  if (last - get_first_committed() > max && floor < last - max) {
+    floor = last-max;
+  }
+
+  dout(20) << __func__ << " = " << floor << dendl;
+  return floor;
+}
+
+bool MDSMonitor::preprocess_query(MonOpRequestRef op)
+{
+  op->mark_mdsmon_event(__func__);
+  auto m = op->get_req<PaxosServiceMessage>();
+  dout(10) << "preprocess_query " << *m << " from " << m->get_orig_source()
+	   << " " << m->get_orig_source_addrs() << dendl;
+
+  switch (m->get_type()) {
+    
+  case MSG_MDS_BEACON:
+    return preprocess_beacon(op);
+    
+  case MSG_MON_COMMAND:
+    try {
+      return preprocess_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+
+  case MSG_MDS_OFFLOAD_TARGETS:
+    return preprocess_offload_targets(op);
+
+  default:
+    ceph_abort();
+    return true;
+  }
+}
+
+void MDSMonitor::_note_beacon(MMDSBeacon *m)
+{
+  mds_gid_t gid = mds_gid_t(m->get_global_id());
+  version_t seq = m->get_seq();
+
+  dout(5) << "_note_beacon " << *m << " noting time" << dendl;
+  auto &beacon = last_beacon[gid];
+  beacon.stamp = mono_clock::now();
+  beacon.seq = seq;
+}
+
+bool MDSMonitor::preprocess_beacon(MonOpRequestRef op)
+{
+  op->mark_mdsmon_event(__func__);
+  auto m = op->get_req<MMDSBeacon>();
+  MDSMap::DaemonState state = m->get_state();
+  mds_gid_t gid = m->get_global_id();
+  version_t seq = m->get_seq();
+  MDSMap::mds_info_t info;
+  epoch_t effective_epoch = 0;
+
+  const auto &fsmap = get_fsmap();
+
+  // check privileges, ignore if fails
+  MonSession *session = op->get_session();
+  if (!session)
+    goto ignore;
+  if (!session->is_capable("mds", MON_CAP_X)) {
+    dout(0) << "preprocess_beacon got MMDSBeacon from entity with insufficient privileges "
+	    << session->caps << dendl;
+    goto ignore;
+  }
+
+  if (m->get_fsid() != mon.monmap->fsid) {
+    dout(0) << "preprocess_beacon on fsid " << m->get_fsid() << " != " << mon.monmap->fsid << dendl;
+    goto ignore;
+  }
+
+  dout(5)  << "preprocess_beacon " << *m
+	   << " from " << m->get_orig_source()
+	   << " " << m->get_orig_source_addrs()
+	   << " " << m->get_compat()
+	   << dendl;
+
+  // make sure the address has a port
+  if (m->get_orig_source_addr().get_port() == 0) {
+    dout(1) << " ignoring boot message without a port" << dendl;
+    goto ignore;
+  }
+
+  // fw to leader?
+  if (!is_leader())
+    return false;
+
+  // booted, but not in map?
+  if (!fsmap.gid_exists(gid)) {
+    if (state != MDSMap::STATE_BOOT) {
+      dout(7) << "mds_beacon " << *m << " is not in fsmap (state "
+              << ceph_mds_state_name(state) << ")" << dendl;
+
+      /* We can't send an MDSMap this MDS was a part of because we no longer
+       * know which FS it was part of. Nor does this matter. Sending an empty
+       * MDSMap is sufficient for getting the MDS to respawn.
+       */
+      auto m = make_message<MMDSMap>(mon.monmap->fsid, MDSMap::create_null_mdsmap());
+      mon.send_reply(op, m.detach());
+      return true;
+    } else {
+      return false;  // not booted yet.
+    }
+  }
+  dout(10) << __func__ << ": GID exists in map: " << gid << dendl;
+  info = fsmap.get_info_gid(gid);
+
+  if (state == MDSMap::STATE_DNE) {
+    return false;
+  }
+
+  // old seq?
+  if (info.state_seq > seq) {
+    dout(7) << "mds_beacon " << *m << " has old seq, ignoring" << dendl;
+    goto ignore;
+  }
+
+  // Work out the latest epoch that this daemon should have seen
+  {
+    fs_cluster_id_t fscid = fsmap.mds_roles.at(gid);
+    if (fscid == FS_CLUSTER_ID_NONE) {
+      effective_epoch = fsmap.standby_epochs.at(gid);
+    } else {
+      effective_epoch = fsmap.get_filesystem(fscid)->mds_map.epoch;
+    }
+    if (effective_epoch != m->get_last_epoch_seen()) {
+      dout(10) << "mds_beacon " << *m
+               << " ignoring requested state, because mds hasn't seen latest map" << dendl;
+      goto reply;
+    }
+  }
+
+  if (info.laggy()) {
+    _note_beacon(m);
+    return false;  // no longer laggy, need to update map.
+  }
+  if (state == MDSMap::STATE_BOOT) {
+    // ignore, already booted.
+    goto ignore;
+  }
+
+  // did the join_fscid change
+  if (m->get_fs().size()) {
+    fs_cluster_id_t fscid = FS_CLUSTER_ID_NONE;
+    auto f = fsmap.get_filesystem(m->get_fs());
+    if (f) {
+      fscid = f->fscid;
+    }
+    if (info.join_fscid != fscid) {
+      dout(10) << __func__ << " standby mds_join_fs changed to " << fscid
+               << " (" << m->get_fs() << ")" << dendl;
+      _note_beacon(m);
+      return false;
+    }
+  } else {
+    if (info.join_fscid != FS_CLUSTER_ID_NONE) {
+      dout(10) << __func__ << " standby mds_join_fs was cleared" << dendl;
+      _note_beacon(m);
+      return false;
+    }
+  }
+
+  // is there a state change here?
+  if (info.state != state) {
+    _note_beacon(m);
+    return false;
+  }
+
+  // Comparing known daemon health with m->get_health()
+  // and return false (i.e. require proposal) if they
+  // do not match, to update our stored
+  if (!(pending_daemon_health[gid] == m->get_health())) {
+    dout(10) << __func__ << " health metrics for gid " << gid << " were updated" << dendl;
+    _note_beacon(m);
+    return false;
+  }
+
+ reply:
+  // note time and reply
+  ceph_assert(effective_epoch > 0);
+  _note_beacon(m);
+  {
+    auto beacon = make_message<MMDSBeacon>(mon.monmap->fsid,
+        m->get_global_id(), m->get_name(), effective_epoch,
+        state, seq, CEPH_FEATURES_SUPPORTED_DEFAULT);
+    mon.send_reply(op, beacon.detach());
+  }
+  return true;
+
+ ignore:
+  // I won't reply this beacon, drop it.
+  mon.no_reply(op);
+  return true;
+}
+
+bool MDSMonitor::preprocess_offload_targets(MonOpRequestRef op)
+{
+  op->mark_mdsmon_event(__func__);
+  auto m = op->get_req<MMDSLoadTargets>();
+  dout(10) << "preprocess_offload_targets " << *m << " from " << m->get_orig_source() << dendl;
+
+  const auto &fsmap = get_fsmap();
+  
+  // check privileges, ignore message if fails
+  MonSession *session = op->get_session();
+  if (!session)
+    goto ignore;
+  if (!session->is_capable("mds", MON_CAP_X)) {
+    dout(0) << "preprocess_offload_targets got MMDSLoadTargets from entity with insufficient caps "
+	    << session->caps << dendl;
+    goto ignore;
+  }
+
+  if (fsmap.gid_exists(m->global_id) &&
+      m->targets == fsmap.get_info_gid(m->global_id).export_targets)
+    goto ignore;
+
+  return false;
+
+ ignore:
+  mon.no_reply(op);
+  return true;
+}
+
+
+bool MDSMonitor::prepare_update(MonOpRequestRef op)
+{
+  op->mark_mdsmon_event(__func__);
+  auto m = op->get_req<PaxosServiceMessage>();
+  dout(7) << "prepare_update " << *m << dendl;
+
+  switch (m->get_type()) {
+    
+  case MSG_MDS_BEACON:
+    return prepare_beacon(op);
+
+  case MSG_MON_COMMAND:
+    try {
+      return prepare_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+
+  case MSG_MDS_OFFLOAD_TARGETS:
+    return prepare_offload_targets(op);
+  
+  default:
+    ceph_abort();
+  }
+
+  return true;
+}
+
+bool MDSMonitor::prepare_beacon(MonOpRequestRef op)
+{
+  op->mark_mdsmon_event(__func__);
+  auto m = op->get_req<MMDSBeacon>();
+  // -- this is an update --
+  dout(12) << "prepare_beacon " << *m << " from " << m->get_orig_source()
+	   << " " << m->get_orig_source_addrs() << dendl;
+  entity_addrvec_t addrs = m->get_orig_source_addrs();
+  mds_gid_t gid = m->get_global_id();
+  MDSMap::DaemonState state = m->get_state();
+  version_t seq = m->get_seq();
+
+  auto &pending = get_pending_fsmap_writeable();
+
+  dout(15) << __func__ << " got health from gid " << gid << " with " << m->get_health().metrics.size() << " metrics." << dendl;
+
+  // Calculate deltas of health metrics created and removed
+  // Do this by type rather than MDSHealthMetric equality, because messages can
+  // change a lot when they include e.g. a number of items.
+  const auto &old_health = pending_daemon_health[gid].metrics;
+  const auto &new_health = m->get_health().metrics;
+
+  std::set<mds_metric_t> old_types;
+  for (const auto &i : old_health) {
+    old_types.insert(i.type);
+  }
+
+  std::set<mds_metric_t> new_types;
+  for (const auto &i : new_health) {
+    if (i.type == MDS_HEALTH_DUMMY) {
+      continue;
+    }
+    new_types.insert(i.type);
+  }
+
+  for (const auto &new_metric: new_health) {
+    if (new_metric.type == MDS_HEALTH_DUMMY) {
+      continue;
+    }
+    if (old_types.count(new_metric.type) == 0) {
+      dout(10) << "MDS health message (" << m->get_orig_source()
+	       << "): " << new_metric.sev << " " << new_metric.message << dendl;
+    }
+  }
+
+  // Log the disappearance of health messages at INFO
+  for (const auto &old_metric : old_health) {
+    if (new_types.count(old_metric.type) == 0) {
+      mon.clog->info() << "MDS health message cleared ("
+        << m->get_orig_source() << "): " << old_metric.message;
+    }
+  }
+
+  // Store health
+  pending_daemon_health[gid] = m->get_health();
+
+  const auto& cs = m->get_compat();
+  if (state == MDSMap::STATE_BOOT) {
+    // zap previous instance of this name?
+    if (g_conf()->mds_enforce_unique_name) {
+      bool failed_mds = false;
+      while (mds_gid_t existing = pending.find_mds_gid_by_name(m->get_name())) {
+        if (!mon.osdmon()->is_writeable()) {
+          mon.osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+          return false;
+        }
+        const auto& existing_info = pending.get_info_gid(existing);
+        mon.clog->info() << existing_info.human_name() << " restarted";
+	fail_mds_gid(pending, existing);
+        failed_mds = true;
+      }
+      if (failed_mds) {
+        ceph_assert(mon.osdmon()->is_writeable());
+        request_proposal(mon.osdmon());
+      }
+    }
+
+    // Add this daemon to the map
+    if (pending.mds_roles.count(gid) == 0) {
+      MDSMap::mds_info_t new_info;
+      new_info.global_id = gid;
+      new_info.name = m->get_name();
+      new_info.addrs = addrs;
+      new_info.mds_features = m->get_mds_features();
+      new_info.state = MDSMap::STATE_STANDBY;
+      new_info.state_seq = seq;
+      new_info.compat = cs;
+      if (m->get_fs().size()) {
+	fs_cluster_id_t fscid = FS_CLUSTER_ID_NONE;
+	auto f = pending.get_filesystem(m->get_fs());
+	if (f) {
+	  fscid = f->fscid;
+	}
+        new_info.join_fscid = fscid;
+      }
+      pending.insert(new_info);
+    }
+
+    // initialize the beacon timer
+    auto &beacon = last_beacon[gid];
+    beacon.stamp = mono_clock::now();
+    beacon.seq = seq;
+
+    update_metadata(m->get_global_id(), m->get_sys_info());
+  } else {
+    // state update
+
+    if (!pending.gid_exists(gid)) {
+      /* gid has been removed from pending, send null map */
+      dout(5) << "mds_beacon " << *m << " is not in fsmap (state "
+              << ceph_mds_state_name(state) << ")" << dendl;
+
+      /* We can't send an MDSMap this MDS was a part of because we no longer
+       * know which FS it was part of. Nor does this matter. Sending an empty
+       * MDSMap is sufficient for getting the MDS to respawn.
+       */
+      goto null;
+    }
+
+    const auto& info = pending.get_info_gid(gid);
+
+    // did the reported compat change? That's illegal!
+    if (cs.compare(info.compat) != 0) {
+      if (!mon.osdmon()->is_writeable()) {
+        mon.osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+        return false;
+      }
+      mon.clog->warn() << info.human_name() << " compat changed unexpectedly";
+      fail_mds_gid(pending, gid);
+      request_proposal(mon.osdmon());
+      return true;
+    }
+
+    if (state == MDSMap::STATE_DNE) {
+      dout(1) << __func__ << ": DNE from " << info << dendl;
+      goto evict;
+    }
+
+    // legal state change?
+    if ((info.state == MDSMap::STATE_STANDBY && state != info.state) ||
+        (info.state == MDSMap::STATE_STANDBY_REPLAY && state != info.state && state != MDSMap::STATE_DAMAGED)) {
+      // Standby daemons should never modify their own state.
+      // Except that standby-replay can indicate the rank is damaged due to failure to replay.
+      // Reject any attempts to do so.
+      derr << "standby " << gid << " attempted to change state to "
+           << ceph_mds_state_name(state) << ", rejecting" << dendl;
+      goto evict;
+    } else if (info.state != MDSMap::STATE_STANDBY && state != info.state &&
+               !MDSMap::state_transition_valid(info.state, state)) {
+      // Validate state transitions for daemons that hold a rank
+      derr << "daemon " << gid << " (rank " << info.rank << ") "
+           << "reported invalid state transition "
+           << ceph_mds_state_name(info.state) << " -> "
+           << ceph_mds_state_name(state) << dendl;
+      goto evict;
+    }
+
+    if (info.laggy()) {
+      dout(1) << "prepare_beacon clearing laggy flag on " << addrs << dendl;
+      pending.modify_daemon(info.global_id, [](auto& info)
+        {
+          info.clear_laggy();
+        }
+      );
+    }
+
+    dout(5)  << "prepare_beacon mds." << info.rank
+	     << " " << ceph_mds_state_name(info.state)
+	     << " -> " << ceph_mds_state_name(state)
+	     << dendl;
+
+    fs_cluster_id_t fscid = FS_CLUSTER_ID_NONE;
+    if (m->get_fs().size()) {
+      auto f = pending.get_filesystem(m->get_fs());
+      if (f) {
+        fscid = f->fscid;
+      }
+    }
+    pending.modify_daemon(gid, [fscid](auto& info) {
+      info.join_fscid = fscid;
+    });
+
+    if (state == MDSMap::STATE_STOPPED) {
+      const auto fscid = pending.mds_roles.at(gid);
+      const auto &fs = pending.get_filesystem(fscid);
+
+      mon.clog->info() << info.human_name() << " finished "
+                        << "stopping rank " << info.rank << " in filesystem "
+                        << fs->mds_map.fs_name << " (now has "
+                        << fs->mds_map.get_num_in_mds() - 1 << " ranks)";
+
+      auto erased = pending.stop(gid);
+      erased.push_back(gid);
+
+      for (const auto& erased_gid : erased) {
+        last_beacon.erase(erased_gid);
+        if (pending_daemon_health.count(erased_gid)) {
+          pending_daemon_health.erase(erased_gid);
+          pending_daemon_health_rm.insert(erased_gid);
+        }
+      }
+    } else if (state == MDSMap::STATE_DAMAGED) {
+      if (!mon.osdmon()->is_writeable()) {
+        dout(1) << __func__ << ": DAMAGED from rank " << info.rank
+                << " waiting for osdmon writeable to blocklist it" << dendl;
+        mon.osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+        return false;
+      }
+
+      auto rank = info.rank;
+
+      // Record this MDS rank as damaged, so that other daemons
+      // won't try to run it.
+      dout(0) << __func__ << ": marking rank " << rank << " damaged" << dendl;
+
+      auto fs = pending.get_filesystem(gid);
+      auto rankgid = fs->mds_map.get_gid(rank);
+      auto rankinfo = pending.get_info_gid(rankgid);
+      auto followergid = fs->mds_map.get_standby_replay(rank);
+
+      ceph_assert(gid == rankgid || gid == followergid);
+
+      utime_t until = ceph_clock_now();
+      until += g_conf().get_val<double>("mon_mds_blocklist_interval");
+      const auto blocklist_epoch = mon.osdmon()->blocklist(rankinfo.addrs, until);
+      if (followergid != MDS_GID_NONE) {
+        fail_mds_gid(pending, followergid);
+        last_beacon.erase(followergid);
+      }
+      request_proposal(mon.osdmon());
+      pending.damaged(rankgid, blocklist_epoch);
+      last_beacon.erase(rankgid);
+
+      /* MDS expects beacon reply back */
+    } else {
+      if (info.state != MDSMap::STATE_ACTIVE && state == MDSMap::STATE_ACTIVE) {
+        const auto &fscid = pending.mds_roles.at(gid);
+        const auto &fs = pending.get_filesystem(fscid);
+        mon.clog->info() << info.human_name() << " is now active in "
+                          << "filesystem " << fs->mds_map.fs_name << " as rank "
+                          << info.rank;
+      }
+
+      // Made it through special cases and validations, record the
+      // daemon's reported state to the FSMap.
+      pending.modify_daemon(gid, [state, seq](auto& info) {
+        info.state = state;
+        info.state_seq = seq;
+      });
+    }
+  }
+
+  dout(5) << "prepare_beacon pending map now:" << dendl;
+  print_map(pending);
+  
+  wait_for_finished_proposal(op, new LambdaContext([op, this](int r){
+    if (r >= 0)
+      _updated(op);   // success
+    else if (r == -ECANCELED) {
+      mon.no_reply(op);
+    } else {
+      dispatch(op);        // try again
+    }
+  }));
+
+  return true;
+
+evict:
+  if (!mon.osdmon()->is_writeable()) {
+    dout(1) << __func__ << ": waiting for writeable OSDMap to evict" << dendl;
+    mon.osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+    return false;
+  }
+
+  fail_mds_gid(pending, gid);
+  request_proposal(mon.osdmon());
+  dout(5) << __func__ << ": pending map now:" << dendl;
+  print_map(pending);
+
+  goto null;
+
+null:
+  wait_for_finished_proposal(op, new LambdaContext([op, this](int r){
+    if (r >= 0) {
+      auto m = make_message<MMDSMap>(mon.monmap->fsid, MDSMap::create_null_mdsmap());
+      mon.send_reply(op, m.detach());
+    } else {
+      dispatch(op);        // try again
+    }
+  }));
+
+  return true;
+}
+
+bool MDSMonitor::prepare_offload_targets(MonOpRequestRef op)
+{
+  auto &pending = get_pending_fsmap_writeable();
+
+  op->mark_mdsmon_event(__func__);
+  auto m = op->get_req<MMDSLoadTargets>();
+  mds_gid_t gid = m->global_id;
+  if (pending.gid_has_rank(gid)) {
+    dout(10) << "prepare_offload_targets " << gid << " " << m->targets << dendl;
+    pending.update_export_targets(gid, m->targets);
+  } else {
+    dout(10) << "prepare_offload_targets " << gid << " not in map" << dendl;
+  }
+  mon.no_reply(op);
+  return true;
+}
+
+bool MDSMonitor::should_propose(double& delay)
+{
+  // delegate to PaxosService to assess whether we should propose
+  return PaxosService::should_propose(delay);
+}
+
+void MDSMonitor::_updated(MonOpRequestRef op)
+{
+  const auto &fsmap = get_fsmap();
+  op->mark_mdsmon_event(__func__);
+  auto m = op->get_req<MMDSBeacon>();
+  dout(10) << "_updated " << m->get_orig_source() << " " << *m << dendl;
+  mon.clog->debug() << m->get_orig_source() << " "
+		     << m->get_orig_source_addrs() << " "
+		     << ceph_mds_state_name(m->get_state());
+
+  if (m->get_state() == MDSMap::STATE_STOPPED) {
+    // send the map manually (they're out of the map, so they won't get it automatic)
+    auto m = make_message<MMDSMap>(mon.monmap->fsid, MDSMap::create_null_mdsmap());
+    mon.send_reply(op, m.detach());
+  } else {
+    auto beacon = make_message<MMDSBeacon>(mon.monmap->fsid,
+        m->get_global_id(), m->get_name(), fsmap.get_epoch(),
+        m->get_state(), m->get_seq(), CEPH_FEATURES_SUPPORTED_DEFAULT);
+    mon.send_reply(op, beacon.detach());
+  }
+}
+
+void MDSMonitor::on_active()
+{
+  tick();
+
+  if (is_leader()) {
+    mon.clog->debug() << "fsmap " << get_fsmap();
+  }
+}
+
+void MDSMonitor::dump_info(Formatter *f)
+{
+  f->open_object_section("fsmap");
+  get_fsmap().dump(f);
+  f->close_section();
+
+  f->dump_unsigned("mdsmap_first_committed", get_first_committed());
+  f->dump_unsigned("mdsmap_last_committed", get_last_committed());
+}
+
+bool MDSMonitor::preprocess_command(MonOpRequestRef op)
+{
+  op->mark_mdsmon_event(__func__);
+  auto m = op->get_req<MMonCommand>();
+  int r = -1;
+  bufferlist rdata;
+  stringstream ss, ds;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    // ss has reason for failure
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+  string format;
+  cmd_getval(cmdmap, "format", format, string("plain"));
+  std::unique_ptr<Formatter> f(Formatter::create(format));
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    mon.reply_command(op, -EACCES, "access denied", rdata, get_last_committed());
+    return true;
+  }
+
+  // to use const qualifier filter fsmap beforehand
+  FSMap _fsmap_copy = get_fsmap();
+  _fsmap_copy.filter(session->get_allowed_fs_names());
+  const auto& fsmap = _fsmap_copy;
+
+  if (prefix == "mds stat") {
+    if (f) {
+      f->open_object_section("mds_stat");
+      dump_info(f.get());
+      f->close_section();
+      f->flush(ds);
+    } else {
+      ds << fsmap;
+    }
+    r = 0;
+  } else if (prefix == "mds ok-to-stop") {
+    vector<string> ids;
+    if (!cmd_getval(cmdmap, "ids", ids)) {
+      r = -EINVAL;
+      ss << "must specify mds id";
+      goto out;
+    }
+    if (fsmap.is_any_degraded()) {
+      ss << "one or more filesystems is currently degraded";
+      r = -EBUSY;
+      goto out;
+    }
+    set<mds_gid_t> stopping;
+    for (auto& id : ids) {
+      ostringstream ess;
+      mds_gid_t gid = gid_from_arg(fsmap, id, ess);
+      if (gid == MDS_GID_NONE) {
+	// the mds doesn't exist, but no file systems are unhappy, so losing it
+	// can't have any effect.
+	continue;
+      }
+      stopping.insert(gid);
+    }
+    set<mds_gid_t> active;
+    set<mds_gid_t> standby;
+    for (auto gid : stopping) {
+      if (fsmap.gid_has_rank(gid)) {
+	// ignore standby-replay daemons (at this level)
+	if (!fsmap.is_standby_replay(gid)) {
+	  auto standby = fsmap.get_standby_replay(gid);
+	  if (standby == MDS_GID_NONE ||
+	      stopping.count(standby)) {
+	    // no standby-replay, or we're also stopping the standby-replay
+	    // for this mds
+	    active.insert(gid);
+	  }
+	}
+      } else {
+	// net loss of a standby
+	standby.insert(gid);
+      }
+    }
+    if (fsmap.get_num_standby() - standby.size() < active.size()) {
+      r = -EBUSY;
+      ss << "insufficent standby MDS daemons to stop active gids "
+	 << stringify(active)
+	 << " and/or standby gids " << stringify(standby);;
+      goto out;
+    }
+    r = 0;
+    ss << "should be safe to stop " << ids;
+  } else if (prefix == "fs dump") {
+    int64_t epocharg;
+    epoch_t epoch;
+
+    const FSMap *fsmapp = &fsmap;
+    FSMap dummy;
+    if (cmd_getval(cmdmap, "epoch", epocharg)) {
+      epoch = epocharg;
+      bufferlist b;
+      int err = get_version(epoch, b);
+      if (err == -ENOENT) {
+	r = -ENOENT;
+        goto out;
+      } else {
+	ceph_assert(err == 0);
+	ceph_assert(b.length());
+	dummy.decode(b);
+        fsmapp = &dummy;
+      }
+    }
+
+    stringstream ds;
+    if (f != NULL) {
+      f->open_object_section("fsmap");
+      fsmapp->dump(f.get());
+      f->close_section();
+      f->flush(ds);
+      r = 0;
+    } else {
+      fsmapp->print(ds);
+      r = 0;
+    }
+
+    rdata.append(ds);
+    ss << "dumped fsmap epoch " << fsmapp->get_epoch();
+  } else if (prefix == "mds metadata") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+
+    string who;
+    bool all = !cmd_getval(cmdmap, "who", who);
+    dout(1) << "all = " << all << dendl;
+    if (all) {
+      r = 0;
+      // Dump all MDSs' metadata
+      const auto all_info = fsmap.get_mds_info();
+
+      f->open_array_section("mds_metadata");
+      for(const auto &i : all_info) {
+        const auto &info = i.second;
+
+        f->open_object_section("mds");
+        f->dump_string("name", info.name);
+        std::ostringstream get_err;
+        r = dump_metadata(fsmap, info.name, f.get(), get_err);
+        if (r == -EINVAL || r == -ENOENT) {
+          // Drop error, list what metadata we do have
+          dout(1) << get_err.str() << dendl;
+          r = 0;
+        } else if (r != 0) {
+          derr << "Unexpected error reading metadata: " << cpp_strerror(r)
+               << dendl;
+          ss << get_err.str();
+          f->close_section();
+          break;
+        }
+        f->close_section();
+      }
+      f->close_section();
+    } else {
+      // Dump a single daemon's metadata
+      f->open_object_section("mds_metadata");
+      r = dump_metadata(fsmap, who, f.get(), ss);
+      f->close_section();
+    }
+    f->flush(ds);
+  } else if (prefix == "mds versions") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    count_metadata("ceph_version", f.get());
+    f->flush(ds);
+    r = 0;
+  } else if (prefix == "mds count-metadata") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    string field;
+    cmd_getval(cmdmap, "property", field);
+    count_metadata(field, f.get());
+    f->flush(ds);
+    r = 0;
+  } else if (prefix == "fs compat show") {
+    string fs_name;
+    cmd_getval(cmdmap, "fs_name", fs_name);
+    const auto &fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+      ss << "filesystem '" << fs_name << "' not found";
+      r = -ENOENT;
+      goto out;
+    }
+
+    if (f) {
+      f->open_object_section("mds_compat");
+      fs->mds_map.compat.dump(f.get());
+      f->close_section();
+      f->flush(ds);
+    } else {
+      ds << fs->mds_map.compat;
+    }
+    r = 0;
+  } else if (prefix == "mds compat show") {
+      if (f) {
+	f->open_object_section("mds_compat");
+	fsmap.default_compat.dump(f.get());
+	f->close_section();
+	f->flush(ds);
+      } else {
+	ds << fsmap.default_compat;
+      }
+      r = 0;
+  } else if (prefix == "fs get") {
+    string fs_name;
+    cmd_getval(cmdmap, "fs_name", fs_name);
+    const auto &fs = fsmap.get_filesystem(fs_name);
+    if (fs == nullptr) {
+      ss << "filesystem '" << fs_name << "' not found";
+      r = -ENOENT;
+    } else {
+      if (f != nullptr) {
+        f->open_object_section("filesystem");
+        fs->dump(f.get());
+        f->close_section();
+        f->flush(ds);
+        r = 0;
+      } else {
+        fs->print(ds);
+        r = 0;
+      }
+    }
+  } else if (prefix == "fs ls") {
+    if (f) {
+      f->open_array_section("filesystems");
+      for (const auto &p : fsmap.filesystems) {
+        const auto &fs = p.second;
+        f->open_object_section("filesystem");
+        {
+          const MDSMap &mds_map = fs->mds_map;
+          f->dump_string("name", mds_map.fs_name);
+          /* Output both the names and IDs of pools, for use by
+           * humans and machines respectively */
+          f->dump_string("metadata_pool", mon.osdmon()->osdmap.get_pool_name(
+                mds_map.metadata_pool));
+          f->dump_int("metadata_pool_id", mds_map.metadata_pool);
+          f->open_array_section("data_pool_ids");
+          for (const auto &id : mds_map.data_pools) {
+            f->dump_int("data_pool_id", id);
+          }
+          f->close_section();
+
+          f->open_array_section("data_pools");
+          for (const auto &id : mds_map.data_pools) {
+            const auto &name = mon.osdmon()->osdmap.get_pool_name(id);
+            f->dump_string("data_pool", name);
+          }
+          f->close_section();
+        }
+        f->close_section();
+      }
+      f->close_section();
+      f->flush(ds);
+    } else {
+      for (const auto &p : fsmap.filesystems) {
+        const auto &fs = p.second;
+        const MDSMap &mds_map = fs->mds_map;
+        const string &md_pool_name = mon.osdmon()->osdmap.get_pool_name(
+            mds_map.metadata_pool);
+        
+        ds << "name: " << mds_map.fs_name << ", metadata pool: "
+           << md_pool_name << ", data pools: [";
+        for (const auto &id : mds_map.data_pools) {
+          const string &pool_name = mon.osdmon()->osdmap.get_pool_name(id);
+          ds << pool_name << " ";
+        }
+        ds << "]" << std::endl;
+      }
+
+      if (fsmap.filesystems.empty()) {
+        ds << "No filesystems enabled" << std::endl;
+      }
+    }
+    r = 0;
+  } else if (prefix == "fs feature ls") {
+    if (f) {
+      f->open_array_section("cephfs_features");
+      for (size_t i = 0; i <= CEPHFS_FEATURE_MAX; ++i) {
+	f->open_object_section("feature");
+	f->dump_int("index", i);
+	f->dump_string("name", cephfs_feature_name(i));
+	f->close_section();
+      }
+      f->close_section();
+      f->flush(ds);
+    } else {
+      for (size_t i = 0; i <= CEPHFS_FEATURE_MAX; ++i) {
+        ds << i << " " << cephfs_feature_name(i) << std::endl;
+      }
+    }
+    r = 0;
+  }
+
+out:
+  if (r != -1) {
+    rdata.append(ds);
+    string rs;
+    getline(ss, rs);
+    mon.reply_command(op, r, rs, rdata, get_last_committed());
+    return true;
+  } else
+    return false;
+}
+
+bool MDSMonitor::fail_mds_gid(FSMap &fsmap, mds_gid_t gid)
+{
+  const auto& info = fsmap.get_info_gid(gid);
+  dout(1) << "fail_mds_gid " << gid << " mds." << info.name << " role " << info.rank << dendl;
+
+  ceph_assert(mon.osdmon()->is_writeable());
+
+  epoch_t blocklist_epoch = 0;
+  if (info.rank >= 0 && info.state != MDSMap::STATE_STANDBY_REPLAY) {
+    utime_t until = ceph_clock_now();
+    until += g_conf().get_val<double>("mon_mds_blocklist_interval");
+    blocklist_epoch = mon.osdmon()->blocklist(info.addrs, until);
+  }
+
+  fsmap.erase(gid, blocklist_epoch);
+  last_beacon.erase(gid);
+  if (pending_daemon_health.count(gid)) {
+    pending_daemon_health.erase(gid);
+    pending_daemon_health_rm.insert(gid);
+  }
+
+  return blocklist_epoch != 0;
+}
+
+mds_gid_t MDSMonitor::gid_from_arg(const FSMap &fsmap, const std::string &arg, std::ostream &ss)
+{
+  // Try parsing as a role
+  mds_role_t role;
+  std::ostringstream ignore_err;  // Don't spam 'ss' with parse_role errors
+  int r = fsmap.parse_role(arg, &role, ignore_err);
+  if (r == 0) {
+    // See if a GID is assigned to this role
+    const auto &fs = fsmap.get_filesystem(role.fscid);
+    ceph_assert(fs != nullptr);  // parse_role ensures it exists
+    if (fs->mds_map.is_up(role.rank)) {
+      dout(10) << __func__ << ": validated rank/GID " << role
+               << " as a rank" << dendl;
+      return fs->mds_map.get_mds_info(role.rank).global_id;
+    }
+  }
+
+  // Try parsing as a gid
+  std::string err;
+  unsigned long long maybe_gid = strict_strtoll(arg.c_str(), 10, &err);
+  if (!err.empty()) {
+    // Not a role or a GID, try as a daemon name
+    const MDSMap::mds_info_t *mds_info = fsmap.find_by_name(arg);
+    if (!mds_info) {
+      ss << "MDS named '" << arg
+	 << "' does not exist, or is not up";
+      return MDS_GID_NONE;
+    }
+    dout(10) << __func__ << ": resolved MDS name '" << arg
+             << "' to GID " << mds_info->global_id << dendl;
+    return mds_info->global_id;
+  } else {
+    // Not a role, but parses as a an integer, might be a GID
+    dout(10) << __func__ << ": treating MDS reference '" << arg
+	     << "' as an integer " << maybe_gid << dendl;
+
+    if (fsmap.gid_exists(mds_gid_t(maybe_gid))) {
+      return mds_gid_t(maybe_gid);
+    }
+  }
+
+  dout(1) << __func__ << ": rank/GID " << arg
+	  << " not a existent rank or GID" << dendl;
+  return MDS_GID_NONE;
+}
+
+int MDSMonitor::fail_mds(FSMap &fsmap, std::ostream &ss,
+    const std::string &arg, MDSMap::mds_info_t *failed_info)
+{
+  ceph_assert(failed_info != nullptr);
+
+  mds_gid_t gid = gid_from_arg(fsmap, arg, ss);
+  if (gid == MDS_GID_NONE) {
+    return 0;
+  }
+  if (!mon.osdmon()->is_writeable()) {
+    return -EAGAIN;
+  }
+
+  // Take a copy of the info before removing the MDS from the map,
+  // so that the caller knows which mds (if any) they ended up removing.
+  *failed_info = fsmap.get_info_gid(gid);
+
+  fail_mds_gid(fsmap, gid);
+  ss << "failed mds gid " << gid;
+  ceph_assert(mon.osdmon()->is_writeable());
+  request_proposal(mon.osdmon());
+  return 0;
+}
+
+bool MDSMonitor::prepare_command(MonOpRequestRef op)
+{
+  op->mark_mdsmon_event(__func__);
+  auto m = op->get_req<MMonCommand>();
+  int r = -EINVAL;
+  stringstream ss;
+  bufferlist rdata;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+
+  /* Refuse access if message not associated with a valid session */
+  MonSession *session = op->get_session();
+  if (!session) {
+    mon.reply_command(op, -EACCES, "access denied", rdata, get_last_committed());
+    return true;
+  }
+
+  auto &pending = get_pending_fsmap_writeable();
+
+  bool batched_propose = false;
+  for (const auto &h : handlers) {
+    r = h->can_handle(prefix, op, pending, cmdmap, ss);
+    if (r == 1) {
+      ; // pass, since we got the right handler.
+    } else if (r == 0) {
+      continue;
+    } else {
+      goto out;
+    }
+
+    batched_propose = h->batched_propose();
+    if (batched_propose) {
+      paxos.plug();
+    }
+    r = h->handle(&mon, pending, op, cmdmap, ss);
+    if (batched_propose) {
+      paxos.unplug();
+    }
+
+    if (r == -EAGAIN) {
+      // message has been enqueued for retry; return.
+      dout(4) << __func__ << " enqueue for retry by prepare_command" << dendl;
+      return false;
+    } else {
+      if (r == 0) {
+	// On successful updates, print the updated map
+	print_map(pending);
+      }
+      // Successful or not, we're done: respond.
+      goto out;
+    }
+  }
+
+  r = filesystem_command(pending, op, prefix, cmdmap, ss);
+  if (r >= 0) {
+    goto out;
+  } else if (r == -EAGAIN) {
+    // Do not reply, the message has been enqueued for retry
+    dout(4) << __func__ << " enqueue for retry by filesystem_command" << dendl;
+    return false;
+  } else if (r != -ENOSYS) {
+    goto out;
+  }
+
+  if (r == -ENOSYS && ss.str().empty()) {
+    ss << "unrecognized command";
+  }
+
+out:
+  dout(4) << __func__ << " done, r=" << r << dendl;
+  /* Compose response */
+  string rs;
+  getline(ss, rs);
+
+  if (r >= 0) {
+    // success.. delay reply
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, r, rs,
+					      get_last_committed() + 1));
+    if (batched_propose) {
+      force_immediate_propose();
+    }
+    return true;
+  } else {
+    // reply immediately
+    mon.reply_command(op, r, rs, rdata, get_last_committed());
+    return false;
+  }
+}
+
+int MDSMonitor::filesystem_command(
+    FSMap &fsmap,
+    MonOpRequestRef op,
+    std::string const &prefix,
+    const cmdmap_t& cmdmap,
+    std::stringstream &ss)
+{
+  dout(4) << __func__ << " prefix='" << prefix << "'" << dendl;
+  op->mark_mdsmon_event(__func__);
+  int r = 0;
+  string whostr;
+  cmd_getval(cmdmap, "role", whostr);
+
+  if (prefix == "mds set_state") {
+    mds_gid_t gid;
+    if (!cmd_getval(cmdmap, "gid", gid)) {
+      ss << "error parsing 'gid' value '"
+         << cmd_vartype_stringify(cmdmap.at("gid")) << "'";
+      return -EINVAL;
+    }
+    MDSMap::DaemonState state;
+    if (!cmd_getval(cmdmap, "state", state)) {
+      ss << "error parsing 'state' string value '"
+         << cmd_vartype_stringify(cmdmap.at("state")) << "'";
+      return -EINVAL;
+    }
+    if (fsmap.gid_exists(gid, op->get_session()->get_allowed_fs_names())) {
+      fsmap.modify_daemon(gid, [state](auto& info) {
+        info.state = state;
+      });
+      ss << "set mds gid " << gid << " to state " << state << " "
+         << ceph_mds_state_name(state);
+      return 0;
+    }
+  } else if (prefix == "mds fail") {
+    string who;
+    cmd_getval(cmdmap, "role_or_gid", who);
+
+    MDSMap::mds_info_t failed_info;
+    mds_gid_t gid = gid_from_arg(fsmap, who, ss);
+    if (gid == MDS_GID_NONE) {
+      ss << "MDS named '" << who << "' does not exist, is not up or you "
+	 << "lack the permission to see.";
+      return 0;
+    }
+    if(!fsmap.gid_exists(gid, op->get_session()->get_allowed_fs_names())) {
+      ss << "MDS named '" << who << "' does not exist, is not up or you "
+	 << "lack the permission to see.";
+      return -EINVAL;
+    }
+    string_view fs_name = fsmap.fs_name_from_gid(gid);
+    if (!op->get_session()->fs_name_capable(fs_name, MON_CAP_W)) {
+      ss << "Permission denied.";
+      return -EPERM;
+    }
+
+    r = fail_mds(fsmap, ss, who, &failed_info);
+    if (r < 0 && r == -EAGAIN) {
+      mon.osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+      return -EAGAIN; // don't propose yet; wait for message to be retried
+    } else if (r == 0) {
+      // Only log if we really did something (not when was already gone)
+      if (failed_info.global_id != MDS_GID_NONE) {
+        mon.clog->info() << failed_info.human_name() << " marked failed by "
+                          << op->get_session()->entity_name;
+      }
+    }
+  } else if (prefix == "mds rm") {
+    mds_gid_t gid;
+    if (!cmd_getval(cmdmap, "gid", gid)) {
+      ss << "error parsing 'gid' value '"
+         << cmd_vartype_stringify(cmdmap.at("gid")) << "'";
+      return -EINVAL;
+    }
+    if (!fsmap.gid_exists(gid, op->get_session()->get_allowed_fs_names())) {
+      ss << "mds gid " << gid << " does not exist";
+      return 0;
+    }
+    string_view fs_name = fsmap.fs_name_from_gid(gid);
+    if (!op->get_session()->fs_name_capable(fs_name, MON_CAP_W)) {
+      ss << "Permission denied.";
+      return -EPERM;
+    }
+    const auto &info = fsmap.get_info_gid(gid);
+    MDSMap::DaemonState state = info.state;
+    if (state > 0) {
+    ss << "cannot remove active mds." << info.name
+	<< " rank " << info.rank;
+    return -EBUSY;
+    } else {
+    fsmap.erase(gid, {});
+    ss << "removed mds gid " << gid;
+    return 0;
+    }
+  } else if (prefix == "mds rmfailed") {
+    bool confirm = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", confirm);
+    if (!confirm) {
+         ss << "WARNING: this can make your filesystem inaccessible! "
+               "Add --yes-i-really-mean-it if you are sure you wish to continue.";
+         return -EPERM;
+    }
+    
+    std::string role_str;
+    cmd_getval(cmdmap, "role", role_str);
+    mds_role_t role;
+    const auto fs_names = op->get_session()->get_allowed_fs_names();
+    int r = fsmap.parse_role(role_str, &role, ss, fs_names);
+    if (r < 0) {
+      ss << "invalid role '" << role_str << "'";
+      return -EINVAL;
+    }
+    string_view fs_name = fsmap.get_filesystem(role.fscid)->mds_map.get_fs_name();
+    if (!op->get_session()->fs_name_capable(fs_name, MON_CAP_W)) {
+      ss << "Permission denied.";
+      return -EPERM;
+    }
+
+    fsmap.modify_filesystem(
+        role.fscid,
+        [role](std::shared_ptr<Filesystem> fs)
+    {
+      fs->mds_map.failed.erase(role.rank);
+    });
+
+    ss << "removed failed mds." << role;
+    return 0;
+    /* TODO: convert to fs commands to update defaults */
+  } else if (prefix == "mds compat rm_compat") {
+    int64_t f;
+    if (!cmd_getval(cmdmap, "feature", f)) {
+      ss << "error parsing feature value '"
+         << cmd_vartype_stringify(cmdmap.at("feature")) << "'";
+      return -EINVAL;
+    }
+    if (fsmap.default_compat.compat.contains(f)) {
+      ss << "removing compat feature " << f;
+      fsmap.default_compat.compat.remove(f);
+    } else {
+      ss << "compat feature " << f << " not present in " << fsmap.default_compat;
+    }
+    r = 0;
+  } else if (prefix == "mds compat rm_incompat") {
+    int64_t f;
+    if (!cmd_getval(cmdmap, "feature", f)) {
+      ss << "error parsing feature value '"
+         << cmd_vartype_stringify(cmdmap.at("feature")) << "'";
+      return -EINVAL;
+    }
+    if (fsmap.default_compat.incompat.contains(f)) {
+      ss << "removing incompat feature " << f;
+      fsmap.default_compat.incompat.remove(f);
+    } else {
+      ss << "incompat feature " << f << " not present in " << fsmap.default_compat;
+    }
+    r = 0;
+  } else if (prefix == "mds repaired") {
+    std::string role_str;
+    cmd_getval(cmdmap, "role", role_str);
+    mds_role_t role;
+    const auto fs_names = op->get_session()->get_allowed_fs_names();
+    r = fsmap.parse_role(role_str, &role, ss, fs_names);
+    if (r < 0) {
+      return r;
+    }
+    string_view fs_name = fsmap.get_filesystem(role.fscid)->mds_map.get_fs_name();
+    if (!op->get_session()->fs_name_capable(fs_name, MON_CAP_W)) {
+      ss << "Permission denied.";
+      return -EPERM;
+    }
+
+    bool modified = fsmap.undamaged(role.fscid, role.rank);
+    if (modified) {
+      ss << "repaired: restoring rank " << role;
+    } else {
+      ss << "nothing to do: rank is not damaged";
+    }
+
+    r = 0;
+  } else if (prefix == "mds freeze") {
+    std::string who;
+    cmd_getval(cmdmap, "role_or_gid", who);
+    mds_gid_t gid = gid_from_arg(fsmap, who, ss);
+    if (gid == MDS_GID_NONE) {
+      return -EINVAL;
+    }
+
+    string_view fs_name = fsmap.fs_name_from_gid(gid);
+    if (!op->get_session()->fs_name_capable(fs_name, MON_CAP_W)) {
+      ss << "Permission denied.";
+      return -EPERM;
+    }
+
+    bool freeze = false;
+    {
+      std::string str;
+      cmd_getval(cmdmap, "val", str);
+      if ((r = parse_bool(str, &freeze, ss)) != 0) {
+        return r;
+      }
+    }
+
+    auto f = [freeze,gid,&ss](auto& info) {
+      if (freeze) {
+        ss << "freezing mds." << gid;
+        info.freeze();
+      } else {
+        ss << "unfreezing mds." << gid;
+        info.unfreeze();
+      }
+    };
+    fsmap.modify_daemon(gid, f);
+    r = 0;
+  } else {
+    return -ENOSYS;
+  }
+
+  return r;
+}
+
+void MDSMonitor::check_subs()
+{
+  // Subscriptions may be to "mdsmap" (MDS and legacy clients),
+  // "mdsmap.<namespace>", or to "fsmap" for the full state of all
+  // filesystems.  Build a list of all the types we service
+  // subscriptions for.
+
+  std::vector<std::string> types = {
+    "fsmap",
+    "fsmap.user",
+    "mdsmap",
+  };
+
+  for (const auto &p : get_fsmap().filesystems) {
+    const auto &fscid = p.first;
+    CachedStackStringStream cos;
+    *cos << "mdsmap." << fscid;
+    types.push_back(std::string(cos->strv()));
+  }
+
+  for (const auto &type : types) {
+    auto& subs = mon.session_map.subs;
+    auto subs_it = subs.find(type);
+    if (subs_it == subs.end())
+      continue;
+    auto sub_it = subs_it->second->begin();
+    while (!sub_it.end()) {
+      auto sub = *sub_it;
+      ++sub_it; // N.B. check_sub may remove sub!
+      check_sub(sub);
+    }
+  }
+}
+
+
+void MDSMonitor::check_sub(Subscription *sub)
+{
+  dout(20) << __func__ << ": " << sub->type << dendl;
+
+  // to use const qualifier filter fsmap beforehand
+  FSMap _fsmap_copy = get_fsmap();
+  _fsmap_copy.filter(sub->session->get_allowed_fs_names());
+  const auto& fsmap = _fsmap_copy;
+  if (sub->next > fsmap.get_epoch()) {
+    return;
+  }
+
+  if (sub->type == "fsmap") {
+    sub->session->con->send_message(new MFSMap(mon.monmap->fsid, fsmap));
+    if (sub->onetime) {
+      mon.session_map.remove_sub(sub);
+    } else {
+      sub->next = fsmap.get_epoch() + 1;
+    }
+  } else if (sub->type == "fsmap.user") {
+    FSMapUser fsmap_u;
+    fsmap_u.epoch = fsmap.get_epoch();
+    fsmap_u.legacy_client_fscid = fsmap.legacy_client_fscid;
+    for (const auto &p : fsmap.filesystems) {
+      FSMapUser::fs_info_t& fs_info = fsmap_u.filesystems[p.second->fscid];
+      fs_info.cid = p.second->fscid;
+      fs_info.name = p.second->mds_map.fs_name;
+    }
+    sub->session->con->send_message(new MFSMapUser(mon.monmap->fsid, fsmap_u));
+    if (sub->onetime) {
+      mon.session_map.remove_sub(sub);
+    } else {
+      sub->next = fsmap.get_epoch() + 1;
+    }
+  } else if (sub->type.compare(0, 6, "mdsmap") == 0) {
+    const bool is_mds = sub->session->name.is_mds();
+    mds_gid_t mds_gid = MDS_GID_NONE;
+    fs_cluster_id_t fscid = FS_CLUSTER_ID_NONE;
+    if (is_mds) {
+      // What (if any) namespace are you assigned to?
+      auto mds_info = fsmap.get_mds_info();
+      for (const auto &p : mds_info) {
+        if (p.second.addrs == sub->session->addrs) {
+          mds_gid = p.first;
+          fscid = fsmap.mds_roles.at(mds_gid);
+        }
+      }
+    } else {
+      // You're a client.  Did you request a particular
+      // namespace?
+      if (sub->type.compare(0, 7, "mdsmap.") == 0) {
+        auto namespace_id_str = sub->type.substr(std::string("mdsmap.").size());
+        dout(10) << __func__ << ": namespace_id " << namespace_id_str << dendl;
+        std::string err;
+        fscid = strict_strtoll(namespace_id_str.c_str(), 10, &err);
+        if (!err.empty()) {
+          // Client asked for a non-existent namespace, send them nothing
+          dout(1) << "Invalid client subscription '" << sub->type
+                  << "'" << dendl;
+          return;
+        }
+      } else {
+        // Unqualified request for "mdsmap": give it the one marked
+        // for use by legacy clients.
+        if (fsmap.legacy_client_fscid != FS_CLUSTER_ID_NONE) {
+          fscid = fsmap.legacy_client_fscid;
+        } else {
+          dout(1) << "Client subscribed for legacy filesystem but "
+                     "none is configured" << dendl;
+          return;
+        }
+      }
+      if (!fsmap.filesystem_exists(fscid)) {
+        // Client asked for a non-existent namespace, send them nothing
+        // TODO: something more graceful for when a client has a filesystem
+        // mounted, and the fileysstem is deleted.  Add a "shut down you fool"
+        // flag to MMDSMap?
+        dout(1) << "Client subscribed to non-existent namespace '" <<
+                fscid << "'" << dendl;
+        return;
+      }
+    }
+    dout(10) << __func__ << ": is_mds=" << is_mds << ", fscid= " << fscid << dendl;
+
+    // Work out the effective latest epoch
+    const MDSMap *mds_map = nullptr;
+    MDSMap null_map = MDSMap::create_null_mdsmap();
+    if (fscid == FS_CLUSTER_ID_NONE) {
+      // For a client, we should have already dropped out
+      ceph_assert(is_mds);
+
+      auto it = fsmap.standby_daemons.find(mds_gid);
+      if (it != fsmap.standby_daemons.end()) {
+        // For an MDS, we need to feed it an MDSMap with its own state in
+        null_map.mds_info[mds_gid] = it->second;
+        null_map.epoch = fsmap.standby_epochs.at(mds_gid);
+      } else {
+        null_map.epoch = fsmap.epoch;
+      }
+      mds_map = &null_map;
+    } else {
+      // Check the effective epoch 
+      mds_map = &fsmap.get_filesystem(fscid)->mds_map;
+    }
+
+    ceph_assert(mds_map != nullptr);
+    dout(10) << __func__ << " selected MDS map epoch " <<
+      mds_map->epoch << " for namespace " << fscid << " for subscriber "
+      << sub->session->name << " who wants epoch " << sub->next << dendl;
+
+    if (sub->next > mds_map->epoch) {
+      return;
+    }
+    auto msg = make_message<MMDSMap>(mon.monmap->fsid, *mds_map,
+			             mds_map->fs_name);
+
+    sub->session->con->send_message(msg.detach());
+    if (sub->onetime) {
+      mon.session_map.remove_sub(sub);
+    } else {
+      sub->next = mds_map->get_epoch() + 1;
+    }
+  }
+}
+
+
+void MDSMonitor::update_metadata(mds_gid_t gid,
+				 const map<string, string>& metadata)
+{
+  if (metadata.empty()) {
+    return;
+  }
+  pending_metadata[gid] = metadata;
+
+  MonitorDBStore::TransactionRef t = paxos.get_pending_transaction();
+  bufferlist bl;
+  encode(pending_metadata, bl);
+  t->put(MDS_METADATA_PREFIX, "last_metadata", bl);
+  paxos.trigger_propose();
+}
+
+void MDSMonitor::remove_from_metadata(const FSMap &fsmap, MonitorDBStore::TransactionRef t)
+{
+  bool update = false;
+  for (auto it = pending_metadata.begin(); it != pending_metadata.end(); ) {
+    if (!fsmap.gid_exists(it->first)) {
+      it = pending_metadata.erase(it);
+      update = true;
+    } else {
+      ++it;
+    }
+  }
+  if (!update)
+    return;
+  bufferlist bl;
+  encode(pending_metadata, bl);
+  t->put(MDS_METADATA_PREFIX, "last_metadata", bl);
+}
+
+int MDSMonitor::load_metadata(map<mds_gid_t, Metadata>& m)
+{
+  bufferlist bl;
+  int r = mon.store->get(MDS_METADATA_PREFIX, "last_metadata", bl);
+  if (r) {
+    dout(5) << "Unable to load 'last_metadata'" << dendl;
+    return r;
+  }
+
+  auto it = bl.cbegin();
+  ceph::decode(m, it);
+  return 0;
+}
+
+void MDSMonitor::count_metadata(const std::string &field, map<string,int> *out)
+{
+  map<mds_gid_t,Metadata> meta;
+  load_metadata(meta);
+  for (auto& p : meta) {
+    auto q = p.second.find(field);
+    if (q == p.second.end()) {
+      (*out)["unknown"]++;
+    } else {
+      (*out)[q->second]++;
+    }
+  }
+}
+
+void MDSMonitor::count_metadata(const std::string &field, Formatter *f)
+{
+  map<string,int> by_val;
+  count_metadata(field, &by_val);
+  f->open_object_section(field.c_str());
+  for (auto& p : by_val) {
+    f->dump_int(p.first.c_str(), p.second);
+  }
+  f->close_section();
+}
+
+void MDSMonitor::get_versions(std::map<string, list<string> > &versions)
+{
+  map<mds_gid_t,Metadata> meta;
+  load_metadata(meta);
+  const auto &fsmap = get_fsmap();
+  std::map<mds_gid_t, mds_info_t> map = fsmap.get_mds_info();
+  dout(10) << __func__ << " mds meta=" << meta << dendl;
+  for (auto& p : meta) {
+    auto q = p.second.find("ceph_version_short");
+    if (q == p.second.end()) continue;
+    versions[q->second].push_back(string("mds.") + map[p.first].name);
+  }
+}
+
+int MDSMonitor::dump_metadata(const FSMap& fsmap, const std::string &who,
+    Formatter *f, ostream& err)
+{
+  ceph_assert(f);
+
+  mds_gid_t gid = gid_from_arg(fsmap, who, err);
+  if (gid == MDS_GID_NONE) {
+    return -EINVAL;
+  }
+
+  map<mds_gid_t, Metadata> metadata;
+  if (int r = load_metadata(metadata)) {
+    err << "Unable to load 'last_metadata'";
+    return r;
+  }
+
+  if (!metadata.count(gid)) {
+    return -ENOENT;
+  }
+  const Metadata& m = metadata[gid];
+  for (Metadata::const_iterator p = m.begin(); p != m.end(); ++p) {
+    f->dump_string(p->first.c_str(), p->second);
+  }
+  return 0;
+}
+
+int MDSMonitor::print_nodes(Formatter *f)
+{
+  ceph_assert(f);
+
+  const auto &fsmap = get_fsmap();
+
+  map<mds_gid_t, Metadata> metadata;
+  if (int r = load_metadata(metadata)) {
+    return r;
+  }
+
+  map<string, list<string> > mdses; // hostname => mds
+  for (const auto &p : metadata) {
+    const mds_gid_t& gid = p.first;
+    const Metadata& m = p.second;
+    Metadata::const_iterator hostname = m.find("hostname");
+    if (hostname == m.end()) {
+      // not likely though
+      continue;
+    }
+    if (!fsmap.gid_exists(gid)) {
+      dout(5) << __func__ << ": GID " << gid << " not existent" << dendl;
+      continue;
+    }
+    const MDSMap::mds_info_t& mds_info = fsmap.get_info_gid(gid);
+    mdses[hostname->second].push_back(mds_info.name);
+  }
+
+  dump_services(f, mdses, "mds");
+  return 0;
+}
+
+/**
+ * If a cluster is undersized (with respect to max_mds), then
+ * attempt to find daemons to grow it. If the cluster is oversized
+ * (with respect to max_mds) then shrink it by stopping its highest rank.
+ */
+bool MDSMonitor::maybe_resize_cluster(FSMap &fsmap, fs_cluster_id_t fscid)
+{
+  auto &current_mds_map = get_fsmap().get_filesystem(fscid)->mds_map;
+  auto&& fs = fsmap.get_filesystem(fscid);
+  auto &mds_map = fs->mds_map;
+
+  int in = mds_map.get_num_in_mds();
+  int max = mds_map.get_max_mds();
+
+  dout(20) << __func__ << " in " << in << " max " << max << dendl;
+
+  /* Check that both the current epoch mds_map is resizeable as well as the
+   * current batch of changes in pending. This is important if an MDS is
+   * becoming active in the next epoch.
+   */
+  if (!current_mds_map.is_resizeable() ||
+      !mds_map.is_resizeable()) {
+    dout(5) << __func__ << " mds_map is not currently resizeable" << dendl;
+    return false;
+  }
+
+  if (in < max && !mds_map.test_flag(CEPH_MDSMAP_NOT_JOINABLE)) {
+    mds_rank_t mds = mds_rank_t(0);
+    while (mds_map.is_in(mds)) {
+      mds++;
+    }
+    auto info = fsmap.find_replacement_for({fscid, mds});
+    if (!info) {
+      return false;
+    }
+
+    dout(1) << "assigned standby " << info->addrs
+            << " as mds." << mds << dendl;
+    mon.clog->info() << info->human_name() << " assigned to "
+                         "filesystem " << mds_map.fs_name << " as rank "
+                      << mds << " (now has " << mds_map.get_num_in_mds() + 1
+                      << " ranks)";
+    fsmap.promote(info->global_id, *fs, mds);
+    return true;
+  } else if (in > max) {
+    mds_rank_t target = in - 1;
+    const auto &info = mds_map.get_info(target);
+    if (mds_map.is_active(target)) {
+      dout(1) << "stopping " << target << dendl;
+      mon.clog->info() << "stopping " << info.human_name();
+      auto f = [](auto& info) {
+        info.state = MDSMap::STATE_STOPPING;
+      };
+      fsmap.modify_daemon(info.global_id, f);
+      return true;
+    } else {
+      dout(20) << "skipping stop of " << target << dendl;
+      return false;
+    }
+  }
+
+  return false;
+}
+
+
+/**
+ * Fail a daemon and replace it with a suitable standby.
+ */
+bool MDSMonitor::drop_mds(FSMap &fsmap, mds_gid_t gid, const mds_info_t* rep_info, bool *osd_propose)
+{
+  ceph_assert(osd_propose != nullptr);
+
+  const auto fscid = fsmap.mds_roles.at(gid);
+  const auto& info = fsmap.get_info_gid(gid);
+  const auto rank = info.rank;
+  const auto state = info.state;
+
+  if (info.is_frozen()) {
+    return false;
+  } else if (state == MDSMap::STATE_STANDBY_REPLAY ||
+             state == MDSMap::STATE_STANDBY) {
+    dout(1)  << " failing and removing standby " << gid << " " << info.addrs
+	     << " mds." << rank
+	     << "." << info.inc << " " << ceph_mds_state_name(state)
+	     << dendl;
+    *osd_propose |= fail_mds_gid(fsmap, gid);
+    return true;
+  } else if (rank >= 0 && rep_info) {
+    auto fs = fsmap.filesystems.at(fscid);
+    if (fs->mds_map.test_flag(CEPH_MDSMAP_NOT_JOINABLE)) {
+      return false;
+    }
+    // are we in?
+    // and is there a non-laggy standby that can take over for us?
+    dout(1)  << " replacing " << gid << " " << info.addrs
+	     << " mds." << rank << "." << info.inc
+	     << " " << ceph_mds_state_name(state)
+	     << " with " << rep_info->global_id << "/" << rep_info->name << " " << rep_info->addrs
+	     << dendl;
+
+    mon.clog->warn() << "Replacing " << info.human_name()
+                      << " as rank " << rank
+                      << " with standby " << rep_info->human_name();
+
+    // Remove the old one
+    *osd_propose |= fail_mds_gid(fsmap, gid);
+
+    // Promote the replacement
+    fsmap.promote(rep_info->global_id, *fs, rank);
+
+    return true;
+  }
+  return false;
+}
+
+bool MDSMonitor::check_health(FSMap& fsmap, bool* propose_osdmap)
+{
+  bool do_propose = false;
+  const auto now = mono_clock::now();
+  const bool osdmap_writeable = mon.osdmon()->is_writeable();
+  const auto mds_beacon_grace = g_conf().get_val<double>("mds_beacon_grace");
+  const auto mds_beacon_interval = g_conf().get_val<double>("mds_beacon_interval");
+
+  if (mono_clock::is_zero(last_tick)) {
+    last_tick = now;
+  }
+
+  {
+    auto since_last = std::chrono::duration<double>(now-last_tick);
+
+    if (since_last.count() > (mds_beacon_grace-mds_beacon_interval)) {
+      // This case handles either local slowness (calls being delayed
+      // for whatever reason) or cluster election slowness (a long gap
+      // between calls while an election happened)
+      dout(1) << __func__ << ": resetting beacon timeouts due to mon delay "
+              "(slow election?) of " << since_last.count() << " seconds" << dendl;
+      for (auto& p : last_beacon) {
+        p.second.stamp = now;
+      }
+    }
+  }
+
+  // make sure last_beacon is fully populated
+  for (auto& p : fsmap.mds_roles) {
+    auto& gid = p.first;
+    last_beacon.emplace(std::piecewise_construct,
+        std::forward_as_tuple(gid),
+        std::forward_as_tuple(now, 0));
+  }
+
+  // We will only take decisive action (replacing/removing a daemon)
+  // if we have some indication that some other daemon(s) are successfully
+  // getting beacons through recently.
+  mono_time latest_beacon = mono_clock::zero();
+  for (const auto& p : last_beacon) {
+    latest_beacon = std::max(p.second.stamp, latest_beacon);
+  }
+  auto since = std::chrono::duration<double>(now-latest_beacon);
+  const bool may_replace = since.count() <
+      std::max(g_conf()->mds_beacon_interval, g_conf()->mds_beacon_grace * 0.5);
+
+  // check beacon timestamps
+  std::vector<mds_gid_t> to_remove;
+  const bool mon_down = mon.is_mon_down();
+  const auto mds_beacon_mon_down_grace =
+      g_conf().get_val<std::chrono::seconds>("mds_beacon_mon_down_grace");
+  const auto quorum_age = std::chrono::seconds(mon.quorum_age());
+  const bool new_quorum = quorum_age < mds_beacon_mon_down_grace;
+  for (auto it = last_beacon.begin(); it != last_beacon.end(); ) {
+    auto& [gid, beacon_info] = *it;
+    auto since_last = std::chrono::duration<double>(now-beacon_info.stamp);
+
+    if (!fsmap.gid_exists(gid)) {
+      // gid no longer exists, remove from tracked beacons
+      it = last_beacon.erase(it);
+      continue;
+    }
+
+    if (since_last.count() >= g_conf()->mds_beacon_grace) {
+      auto& info = fsmap.get_info_gid(gid);
+      dout(1) << "no beacon from mds." << info.rank << "." << info.inc
+              << " (gid: " << gid << " addr: " << info.addrs
+              << " state: " << ceph_mds_state_name(info.state) << ")"
+              << " since " << since_last.count() << dendl;
+      if ((mon_down || new_quorum) && since_last < mds_beacon_mon_down_grace) {
+        /* The MDS may be sending beacons to a monitor not yet in quorum or
+         * temporarily partitioned. Hold off on removal for a little longer...
+         */
+        dout(10) << "deferring removal for mds_beacon_mon_down_grace during MON_DOWN" << dendl;
+        ++it;
+        continue;
+      }
+      // If the OSDMap is writeable, we can blocklist things, so we can
+      // try failing any laggy MDS daemons.  Consider each one for failure.
+      if (!info.laggy()) {
+        dout(1)  << " marking " << gid << " " << info.addrs
+	         << " mds." << info.rank << "." << info.inc
+	         << " " << ceph_mds_state_name(info.state)
+	         << " laggy" << dendl;
+        fsmap.modify_daemon(info.global_id, [](auto& info) {
+            info.laggy_since = ceph_clock_now();
+        });
+        do_propose = true;
+      }
+      if (osdmap_writeable && may_replace) {
+        to_remove.push_back(gid); // drop_mds may invalidate iterator
+      }
+    }
+
+    ++it;
+  }
+
+  for (const auto& gid : to_remove) {
+    auto info = fsmap.get_info_gid(gid);
+    const mds_info_t* rep_info = nullptr;
+    if (info.rank >= 0) {
+      auto fscid = fsmap.fscid_from_gid(gid);
+      rep_info = fsmap.find_replacement_for({fscid, info.rank});
+    }
+    bool dropped = drop_mds(fsmap, gid, rep_info, propose_osdmap);
+    if (dropped) {
+      mon.clog->info() << "MDS " << info.human_name()
+                        << " is removed because it is dead or otherwise unavailable.";
+      do_propose = true;
+    }
+  }
+
+  if (osdmap_writeable) {
+    for (auto& [fscid, fs] : fsmap.filesystems) {
+      if (!fs->mds_map.test_flag(CEPH_MDSMAP_NOT_JOINABLE) &&
+          fs->mds_map.is_resizeable()) {
+        // Check if a rank or standby-replay should be replaced with a stronger
+        // affinity standby. This looks at ranks and standby-replay:
+        for (const auto& [gid, info] : fs->mds_map.get_mds_info()) {
+          const auto join_fscid = info.join_fscid;
+          if (join_fscid == fscid)
+            continue;
+          const auto rank = info.rank;
+          const auto state = info.state;
+          const mds_info_t* rep_info = nullptr;
+          if (state == MDSMap::STATE_STANDBY_REPLAY) {
+            rep_info = fsmap.get_available_standby(*fs);
+          } else if (state == MDSMap::STATE_ACTIVE) {
+            rep_info = fsmap.find_replacement_for({fscid, rank});
+          } else {
+            /* N.B. !is_degraded() */
+            ceph_abort_msg("invalid state in MDSMap");
+          }
+          if (!rep_info) {
+            break;
+          }
+          bool better_affinity = false;
+          if (join_fscid == FS_CLUSTER_ID_NONE) {
+            better_affinity = (rep_info->join_fscid == fscid);
+          } else {
+            better_affinity = (rep_info->join_fscid == fscid) ||
+                              (rep_info->join_fscid == FS_CLUSTER_ID_NONE);
+          }
+          if (better_affinity) {
+            if (state == MDSMap::STATE_STANDBY_REPLAY) {
+              mon.clog->info() << "Dropping low affinity standby-replay "
+                                << info.human_name()
+                                << " in favor of higher affinity standby.";
+              *propose_osdmap |= fail_mds_gid(fsmap, gid);
+              /* Now let maybe_promote_standby do the promotion. */
+            } else {
+              mon.clog->info() << "Dropping low affinity active "
+                                << info.human_name()
+                                << " in favor of higher affinity standby.";
+              do_propose |= drop_mds(fsmap, gid, rep_info, propose_osdmap);
+            }
+            break; /* don't replace more than one per tick per fs */
+          }
+        }
+      }
+    }
+  }
+  return do_propose;
+}
+
+bool MDSMonitor::maybe_promote_standby(FSMap &fsmap, Filesystem& fs)
+{
+  if (fs.mds_map.test_flag(CEPH_MDSMAP_NOT_JOINABLE)) {
+    return false;
+  }
+
+  bool do_propose = false;
+
+  // have a standby take over?
+  set<mds_rank_t> failed;
+  fs.mds_map.get_failed_mds_set(failed);
+  for (const auto& rank : failed) {
+    auto info = fsmap.find_replacement_for({fs.fscid, rank});
+    if (info) {
+      dout(1) << " taking over failed mds." << rank << " with " << info->global_id
+              << "/" << info->name << " " << info->addrs << dendl;
+      mon.clog->info() << "Standby " << info->human_name()
+                        << " assigned to filesystem " << fs.mds_map.fs_name
+                        << " as rank " << rank;
+
+      fsmap.promote(info->global_id, fs, rank);
+      do_propose = true;
+    }
+  }
+
+  if (fs.mds_map.is_resizeable() && fs.mds_map.allows_standby_replay()) {
+    // There were no failures to replace, so try using any available standbys
+    // as standby-replay daemons. Don't do this when the cluster is degraded
+    // as a standby-replay daemon may try to read a journal being migrated.
+    for (;;) {
+      auto info = fsmap.get_available_standby(fs);
+      if (!info) break;
+      dout(20) << "standby available mds." << info->global_id << dendl;
+      bool changed = false;
+      for (const auto& rank : fs.mds_map.in) {
+        dout(20) << "examining " << rank << dendl;
+        if (fs.mds_map.is_followable(rank)) {
+          dout(1) << "  setting mds." << info->global_id
+                  << " to follow mds rank " << rank << dendl;
+          fsmap.assign_standby_replay(info->global_id, fs.fscid, rank);
+          do_propose = true;
+          changed = true;
+          break;
+        }
+      }
+      if (!changed) break;
+    }
+  }
+
+  return do_propose;
+}
+
+void MDSMonitor::tick()
+{
+  if (!is_active() || !is_leader()) return;
+
+  auto &pending = get_pending_fsmap_writeable();
+
+  bool do_propose = false;
+  bool propose_osdmap = false;
+
+  if (check_fsmap_struct_version) {
+    /* Allow time for trimming otherwise PaxosService::is_writeable will always
+     * be false.
+     */
+
+    auto now = clock::now();
+    auto elapsed = now - last_fsmap_struct_flush;
+    if (elapsed > std::chrono::seconds(30)) {
+      FSMap fsmap;
+      bufferlist bl;
+      auto v = get_first_committed();
+      int err = get_version(v, bl);
+      if (err) {
+        derr << "could not get version " << v << dendl;
+        ceph_abort();
+      }
+      try {
+        fsmap.decode(bl);
+      } catch (const ceph::buffer::malformed_input& e) {
+        dout(5) << "flushing old fsmap struct because unable to decode FSMap: " << e.what() << dendl;
+      }
+      /* N.B. FSMap::is_struct_old is also true for undecoded (failed to decode) FSMap */
+      if (fsmap.is_struct_old()) {
+        dout(5) << "fsmap struct is too old; proposing to flush out old versions" << dendl;
+        do_propose = true;
+        last_fsmap_struct_flush = now;
+      } else {
+        dout(20) << "struct is recent" << dendl;
+        check_fsmap_struct_version = false;
+      }
+    }
+  }
+
+  do_propose |= pending.check_health();
+
+  /* Check health and affinity of ranks */
+  do_propose |= check_health(pending, &propose_osdmap);
+
+  /* Resize the cluster according to max_mds. */
+  for (auto& p : pending.filesystems) {
+    do_propose |= maybe_resize_cluster(pending, p.second->fscid);
+  }
+
+  /* Replace any failed ranks. */
+  for (auto& p : pending.filesystems) {
+    do_propose |= maybe_promote_standby(pending, *p.second);
+  }
+
+  if (propose_osdmap) {
+    request_proposal(mon.osdmon());
+  }
+
+  if (do_propose) {
+    propose_pending();
+  }
+
+  last_tick = mono_clock::now();
+}
+
+MDSMonitor::MDSMonitor(Monitor &mn, Paxos &p, string service_name)
+  : PaxosService(mn, p, service_name)
+{
+  handlers = FileSystemCommandHandler::load(&p);
+}
+
+void MDSMonitor::on_restart()
+{
+  // Clear out the leader-specific state.
+  last_tick = mono_clock::now();
+  last_beacon.clear();
+}
+
diff --git a/src/mon/MDSMonitor.h b/src/mon/MDSMonitor.h
new file mode 100644
index 000000000..c70814996
--- /dev/null
+++ b/src/mon/MDSMonitor.h
@@ -0,0 +1,158 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+ 
+/* Metadata Server Monitor
+ */
+
+#ifndef CEPH_MDSMONITOR_H
+#define CEPH_MDSMONITOR_H
+
+#include <map>
+#include <set>
+
+#include "include/types.h"
+#include "PaxosFSMap.h"
+#include "PaxosService.h"
+#include "msg/Messenger.h"
+#include "messages/MMDSBeacon.h"
+#include "CommandHandler.h"
+
+class FileSystemCommandHandler;
+
+class MDSMonitor : public PaxosService, public PaxosFSMap, protected CommandHandler {
+ public:
+  using clock = ceph::coarse_mono_clock;
+  using time = ceph::coarse_mono_time;
+
+  MDSMonitor(Monitor &mn, Paxos &p, std::string service_name);
+
+  // service methods
+  void create_initial() override;
+  void get_store_prefixes(std::set<std::string>& s) const override;
+  void update_from_paxos(bool *need_bootstrap) override;
+  void init() override;
+  void create_pending() override;
+  void encode_pending(MonitorDBStore::TransactionRef t) override;
+  // we don't require full versions; don't encode any.
+  void encode_full(MonitorDBStore::TransactionRef t) override { }
+  version_t get_trim_to() const override;
+
+  bool preprocess_query(MonOpRequestRef op) override;  // true if processed.
+  bool prepare_update(MonOpRequestRef op) override;
+  bool should_propose(double& delay) override;
+
+  bool should_print_status() const {
+    auto& fs = get_fsmap();
+    auto fs_count = fs.filesystem_count();
+    auto standby_count = fs.get_num_standby();
+    return fs_count > 0 || standby_count > 0;
+  }
+
+  void on_active() override;
+  void on_restart() override;
+
+  void check_subs();
+  void check_sub(Subscription *sub);
+
+  void dump_info(ceph::Formatter *f);
+  int print_nodes(ceph::Formatter *f);
+
+  /**
+   * Return true if a blocklist was done (i.e. OSD propose needed)
+   */
+  bool fail_mds_gid(FSMap &fsmap, mds_gid_t gid);
+
+  bool is_leader() const override { return mon.is_leader(); }
+
+ protected:
+  using mds_info_t = MDSMap::mds_info_t;
+
+  // my helpers
+  template<int dblV = 7>
+  void print_map(const FSMap &m);
+
+  void _updated(MonOpRequestRef op);
+
+  void _note_beacon(class MMDSBeacon *m);
+  bool preprocess_beacon(MonOpRequestRef op);
+  bool prepare_beacon(MonOpRequestRef op);
+
+  bool preprocess_offload_targets(MonOpRequestRef op);
+  bool prepare_offload_targets(MonOpRequestRef op);
+
+  int fail_mds(FSMap &fsmap, std::ostream &ss,
+      const std::string &arg, mds_info_t *failed_info);
+
+  bool preprocess_command(MonOpRequestRef op);
+  bool prepare_command(MonOpRequestRef op);
+
+  int filesystem_command(
+      FSMap &fsmap,
+      MonOpRequestRef op,
+      std::string const &prefix,
+      const cmdmap_t& cmdmap,
+      std::stringstream &ss);
+
+  // beacons
+  struct beacon_info_t {
+    ceph::mono_time stamp = ceph::mono_clock::zero();
+    uint64_t seq = 0;
+    beacon_info_t() {}
+    beacon_info_t(ceph::mono_time stamp, uint64_t seq) : stamp(stamp), seq(seq) {}
+  };
+  std::map<mds_gid_t, beacon_info_t> last_beacon;
+
+  std::list<std::shared_ptr<FileSystemCommandHandler> > handlers;
+
+  bool maybe_promote_standby(FSMap& fsmap, Filesystem& fs);
+  bool maybe_resize_cluster(FSMap &fsmap, fs_cluster_id_t fscid);
+  bool drop_mds(FSMap &fsmap, mds_gid_t gid, const mds_info_t* rep_info, bool* osd_propose);
+  bool check_health(FSMap &fsmap, bool* osd_propose);
+  void tick() override;     // check state, take actions
+
+  int dump_metadata(const FSMap &fsmap, const std::string &who, ceph::Formatter *f,
+		    std::ostream& err);
+
+  void update_metadata(mds_gid_t gid, const Metadata& metadata);
+  void remove_from_metadata(const FSMap &fsmap, MonitorDBStore::TransactionRef t);
+  int load_metadata(std::map<mds_gid_t, Metadata>& m);
+  void count_metadata(const std::string& field, ceph::Formatter *f);
+
+public:
+  void print_fs_summary(ostream& out) {
+    get_fsmap().print_fs_summary(out);
+  }
+  void count_metadata(const std::string& field, std::map<std::string,int> *out);
+  void get_versions(std::map<std::string, std::list<std::string>> &versions);
+
+protected:
+  // MDS daemon GID to latest health state from that GID
+  std::map<uint64_t, MDSHealth> pending_daemon_health;
+  std::set<uint64_t> pending_daemon_health_rm;
+
+  std::map<mds_gid_t, Metadata> pending_metadata;
+
+  mds_gid_t gid_from_arg(const FSMap &fsmap, const std::string &arg, std::ostream& err);
+
+  // When did the mon last call into our tick() method?  Used for detecting
+  // when the mon was not updating us for some period (e.g. during slow
+  // election) to reset last_beacon timeouts
+  ceph::mono_time last_tick = ceph::mono_clock::zero();
+
+private:
+  time last_fsmap_struct_flush = clock::zero();
+  bool check_fsmap_struct_version = true;
+};
+
+#endif
diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h
new file mode 100644
index 000000000..5342fc51f
--- /dev/null
+++ b/src/mon/MgrMap.h
@@ -0,0 +1,601 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2016 John Spray <john.spray@redhat.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#ifndef MGR_MAP_H_
+#define MGR_MAP_H_
+
+#include <sstream>
+#include <set>
+
+#include "msg/msg_types.h"
+#include "include/encoding.h"
+#include "include/utime.h"
+#include "common/Formatter.h"
+#include "common/ceph_releases.h"
+#include "common/version.h"
+#include "common/options.h"
+#include "common/Clock.h"
+
+
+class MgrMap
+{
+public:
+  struct ModuleOption {
+    std::string name;
+    uint8_t type = Option::TYPE_STR;         // Option::type_t TYPE_*
+    uint8_t level = Option::LEVEL_ADVANCED;  // Option::level_t LEVEL_*
+    uint32_t flags = 0; // Option::flag_t FLAG_*
+    std::string default_value;
+    std::string min, max;
+    std::set<std::string> enum_allowed;
+    std::string desc, long_desc;
+    std::set<std::string> tags;
+    std::set<std::string> see_also;
+
+    void encode(ceph::buffer::list& bl) const {
+      ENCODE_START(1, 1, bl);
+      encode(name, bl);
+      encode(type, bl);
+      encode(level, bl);
+      encode(flags, bl);
+      encode(default_value, bl);
+      encode(min, bl);
+      encode(max, bl);
+      encode(enum_allowed, bl);
+      encode(desc, bl);
+      encode(long_desc, bl);
+      encode(tags, bl);
+      encode(see_also, bl);
+      ENCODE_FINISH(bl);
+    }
+    void decode(ceph::buffer::list::const_iterator& p) {
+      DECODE_START(1, p);
+      decode(name, p);
+      decode(type, p);
+      decode(level, p);
+      decode(flags, p);
+      decode(default_value, p);
+      decode(min, p);
+      decode(max, p);
+      decode(enum_allowed, p);
+      decode(desc, p);
+      decode(long_desc, p);
+      decode(tags, p);
+      decode(see_also, p);
+      DECODE_FINISH(p);
+    }
+    void dump(ceph::Formatter *f) const {
+      f->dump_string("name", name);
+      f->dump_string("type", Option::type_to_str(
+		       static_cast<Option::type_t>(type)));
+      f->dump_string("level", Option::level_to_str(
+		       static_cast<Option::level_t>(level)));
+      f->dump_unsigned("flags", flags);
+      f->dump_string("default_value", default_value);
+      f->dump_string("min", min);
+      f->dump_string("max", max);
+      f->open_array_section("enum_allowed");
+      for (auto& i : enum_allowed) {
+	f->dump_string("value", i);
+      }
+      f->close_section();
+      f->dump_string("desc", desc);
+      f->dump_string("long_desc", long_desc);
+      f->open_array_section("tags");
+      for (auto& i : tags) {
+	f->dump_string("tag", i);
+      }
+      f->close_section();
+      f->open_array_section("see_also");
+      for (auto& i : see_also) {
+	f->dump_string("option", i);
+      }
+      f->close_section();
+    }
+  };
+
+  class ModuleInfo
+  {
+    public:
+    std::string name;
+    bool can_run = true;
+    std::string error_string;
+    std::map<std::string,ModuleOption> module_options;
+
+    // We do not include the module's `failed` field in the beacon,
+    // because it is exposed via health checks.
+    void encode(ceph::buffer::list &bl) const {
+      ENCODE_START(2, 1, bl);
+      encode(name, bl);
+      encode(can_run, bl);
+      encode(error_string, bl);
+      encode(module_options, bl);
+      ENCODE_FINISH(bl);
+    }
+
+    void decode(ceph::buffer::list::const_iterator &bl) {
+      DECODE_START(1, bl);
+      decode(name, bl);
+      decode(can_run, bl);
+      decode(error_string, bl);
+      if (struct_v >= 2) {
+	decode(module_options, bl);
+      }
+      DECODE_FINISH(bl);
+    }
+
+    bool operator==(const ModuleInfo &rhs) const
+    {
+      return (name == rhs.name) && (can_run == rhs.can_run);
+    }
+
+    void dump(ceph::Formatter *f) const {
+      f->open_object_section("module");
+      f->dump_string("name", name);
+      f->dump_bool("can_run", can_run);
+      f->dump_string("error_string", error_string);
+      f->open_object_section("module_options");
+      for (auto& i : module_options) {
+	f->dump_object(i.first.c_str(), i.second);
+      }
+      f->close_section();
+      f->close_section();
+    }
+  };
+
+  class StandbyInfo
+  {
+  public:
+    uint64_t gid = 0;
+    std::string name;
+    std::vector<ModuleInfo> available_modules;
+    uint64_t mgr_features = 0;
+
+    StandbyInfo(uint64_t gid_, const std::string &name_,
+                const std::vector<ModuleInfo>& am,
+		uint64_t feat)
+      : gid(gid_), name(name_), available_modules(am),
+	mgr_features(feat)
+    {}
+
+    StandbyInfo() {}
+
+    void encode(ceph::buffer::list& bl) const
+    {
+      ENCODE_START(4, 1, bl);
+      encode(gid, bl);
+      encode(name, bl);
+      std::set<std::string> old_available_modules;
+      for (const auto &i : available_modules) {
+        old_available_modules.insert(i.name);
+      }
+      encode(old_available_modules, bl);  // version 2
+      encode(available_modules, bl);  // version 3
+      encode(mgr_features, bl); // v4
+      ENCODE_FINISH(bl);
+    }
+
+    void decode(ceph::buffer::list::const_iterator& p)
+    {
+      DECODE_START(4, p);
+      decode(gid, p);
+      decode(name, p);
+      if (struct_v >= 2) {
+        std::set<std::string> old_available_modules;
+        decode(old_available_modules, p);
+        if (struct_v < 3) {
+          for (const auto &name : old_available_modules) {
+            MgrMap::ModuleInfo info;
+            info.name = name;
+            available_modules.push_back(std::move(info));
+          }
+        }
+      }
+      if (struct_v >= 3) {
+        decode(available_modules, p);
+      }
+      if (struct_v >= 4) {
+	decode(mgr_features, p);
+      }
+      DECODE_FINISH(p);
+    }
+
+    bool have_module(const std::string &module_name) const
+    {
+      auto it = std::find_if(available_modules.begin(),
+          available_modules.end(),
+          [module_name](const ModuleInfo &m) -> bool {
+            return m.name == module_name;
+          });
+
+      return it != available_modules.end();
+    }
+  };
+
+  epoch_t epoch = 0;
+  epoch_t last_failure_osd_epoch = 0;
+
+  /// global_id of the ceph-mgr instance selected as a leader
+  uint64_t active_gid = 0;
+  /// server address reported by the leader once it is active
+  entity_addrvec_t active_addrs;
+  /// whether the nominated leader is active (i.e. has initialized its server)
+  bool available = false;
+  /// the name (foo in mgr.<foo>) of the active daemon
+  std::string active_name;
+  /// when the active mgr became active, or we lost the active mgr
+  utime_t active_change;
+  /// features
+  uint64_t active_mgr_features = 0;
+
+  std::vector<entity_addrvec_t> clients; // for blocklist
+
+  std::map<uint64_t, StandbyInfo> standbys;
+
+  // Modules which are enabled
+  std::set<std::string> modules;
+
+  // Modules which should always be enabled. A manager daemon will enable
+  // modules from the union of this set and the `modules` set above, latest
+  // active version.
+  std::map<uint32_t, std::set<std::string>> always_on_modules;
+
+  // Modules which are reported to exist
+  std::vector<ModuleInfo> available_modules;
+
+  // Map of module name to URI, indicating services exposed by
+  // running modules on the active mgr daemon.
+  std::map<std::string, std::string> services;
+
+  epoch_t get_epoch() const { return epoch; }
+  epoch_t get_last_failure_osd_epoch() const { return last_failure_osd_epoch; }
+  const entity_addrvec_t& get_active_addrs() const { return active_addrs; }
+  uint64_t get_active_gid() const { return active_gid; }
+  bool get_available() const { return available; }
+  const std::string &get_active_name() const { return active_name; }
+  const utime_t& get_active_change() const { return active_change; }
+  int get_num_standby() const { return standbys.size(); }
+
+  bool all_support_module(const std::string& module) {
+    if (!have_module(module)) {
+      return false;
+    }
+    for (auto& p : standbys) {
+      if (!p.second.have_module(module)) {
+	return false;
+      }
+    }
+    return true;
+  }
+
+  bool have_module(const std::string &module_name) const
+  {
+    for (const auto &i : available_modules) {
+      if (i.name == module_name) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  const ModuleInfo *get_module_info(const std::string &module_name) const {
+    for (const auto &i : available_modules) {
+      if (i.name == module_name) {
+        return &i;
+      }
+    }
+    return nullptr;
+  }
+
+  bool can_run_module(const std::string &module_name, std::string *error) const
+  {
+    for (const auto &i : available_modules) {
+      if (i.name == module_name) {
+        *error = i.error_string;
+        return i.can_run;
+      }
+    }
+
+    std::ostringstream oss;
+    oss << "Module '" << module_name << "' does not exist";
+    throw std::logic_error(oss.str());
+  }
+
+  bool module_enabled(const std::string& module_name) const
+  {
+    return modules.find(module_name) != modules.end();
+  }
+
+  bool any_supports_module(const std::string& module) const {
+    if (have_module(module)) {
+      return true;
+    }
+    for (auto& p : standbys) {
+      if (p.second.have_module(module)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  bool have_name(const std::string& name) const {
+    if (active_name == name) {
+      return true;
+    }
+    for (auto& p : standbys) {
+      if (p.second.name == name) {
+	return true;
+      }
+    }
+    return false;
+  }
+
+  std::set<std::string> get_all_names() const {
+    std::set<std::string> ls;
+    if (active_name.size()) {
+      ls.insert(active_name);
+    }
+    for (auto& p : standbys) {
+      ls.insert(p.second.name);
+    }
+    return ls;
+  }
+
+  std::set<std::string> get_always_on_modules() const {
+    unsigned rnum = to_integer<uint32_t>(ceph_release());
+    auto it = always_on_modules.find(rnum);
+    if (it == always_on_modules.end()) {
+      // ok, try the most recent release
+      if (always_on_modules.empty()) {
+	return {}; // ugh
+      }
+      --it;
+      if (it->first < rnum) {
+	return it->second;
+      }
+      return {};      // wth
+    }
+    return it->second;
+  }
+
+  void encode(ceph::buffer::list& bl, uint64_t features) const
+  {
+    if (!HAVE_FEATURE(features, SERVER_NAUTILUS)) {
+      ENCODE_START(5, 1, bl);
+      encode(epoch, bl);
+      encode(active_addrs.legacy_addr(), bl, features);
+      encode(active_gid, bl);
+      encode(available, bl);
+      encode(active_name, bl);
+      encode(standbys, bl);
+      encode(modules, bl);
+
+      // Pre-version 4 std::string std::list of available modules
+      // (replaced by direct encode of ModuleInfo below)
+      std::set<std::string> old_available_modules;
+      for (const auto &i : available_modules) {
+	old_available_modules.insert(i.name);
+      }
+      encode(old_available_modules, bl);
+
+      encode(services, bl);
+      encode(available_modules, bl);
+      ENCODE_FINISH(bl);
+      return;
+    }
+    ENCODE_START(11, 6, bl);
+    encode(epoch, bl);
+    encode(active_addrs, bl, features);
+    encode(active_gid, bl);
+    encode(available, bl);
+    encode(active_name, bl);
+    encode(standbys, bl);
+    encode(modules, bl);
+    encode(services, bl);
+    encode(available_modules, bl);
+    encode(active_change, bl);
+    encode(always_on_modules, bl);
+    encode(active_mgr_features, bl);
+    encode(last_failure_osd_epoch, bl);
+    encode(clients, bl, features);
+    ENCODE_FINISH(bl);
+    return;
+  }
+
+  void decode(ceph::buffer::list::const_iterator& p)
+  {
+    DECODE_START(11, p);
+    decode(epoch, p);
+    decode(active_addrs, p);
+    decode(active_gid, p);
+    decode(available, p);
+    decode(active_name, p);
+    decode(standbys, p);
+    if (struct_v >= 2) {
+      decode(modules, p);
+
+      if (struct_v < 6) {
+	// Reconstitute ModuleInfos from names
+	std::set<std::string> module_name_list;
+	decode(module_name_list, p);
+	// Only need to unpack this field if we won't have the full
+	// MgrMap::ModuleInfo structures added in v4
+	if (struct_v < 4) {
+	  for (const auto &i : module_name_list) {
+	    MgrMap::ModuleInfo info;
+	    info.name = i;
+	    available_modules.push_back(std::move(info));
+	  }
+	}
+      }
+    }
+    if (struct_v >= 3) {
+      decode(services, p);
+    }
+    if (struct_v >= 4) {
+      decode(available_modules, p);
+    }
+    if (struct_v >= 7) {
+      decode(active_change, p);
+    } else {
+      active_change = {};
+    }
+    if (struct_v >= 8) {
+      decode(always_on_modules, p);
+    }
+    if (struct_v >= 9) {
+      decode(active_mgr_features, p);
+    }
+    if (struct_v >= 10) {
+      decode(last_failure_osd_epoch, p);
+    }
+    if (struct_v >= 11) {
+      decode(clients, p);
+    }
+    DECODE_FINISH(p);
+  }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_int("epoch", epoch);
+    f->dump_int("active_gid", get_active_gid());
+    f->dump_string("active_name", get_active_name());
+    f->dump_object("active_addrs", active_addrs);
+    f->dump_stream("active_addr") << active_addrs.get_legacy_str();
+    f->dump_stream("active_change") << active_change;
+    f->dump_unsigned("active_mgr_features", active_mgr_features);
+    f->dump_bool("available", available);
+    f->open_array_section("standbys");
+    for (const auto &i : standbys) {
+      f->open_object_section("standby");
+      f->dump_int("gid", i.second.gid);
+      f->dump_string("name", i.second.name);
+      f->dump_unsigned("mgr_features", i.second.mgr_features);
+      f->open_array_section("available_modules");
+      for (const auto& j : i.second.available_modules) {
+        j.dump(f);
+      }
+      f->close_section();
+      f->close_section();
+    }
+    f->close_section();
+    f->open_array_section("modules");
+    for (auto& i : modules) {
+      f->dump_string("module", i);
+    }
+    f->close_section();
+    f->open_array_section("available_modules");
+    for (const auto& j : available_modules) {
+      j.dump(f);
+    }
+    f->close_section();
+
+    f->open_object_section("services");
+    for (const auto &i : services) {
+      f->dump_string(i.first.c_str(), i.second);
+    }
+    f->close_section();
+
+    f->open_object_section("always_on_modules");
+    for (auto& v : always_on_modules) {
+      f->open_array_section(ceph_release_name(v.first));
+      for (auto& m : v.second) {
+        f->dump_string("module", m);
+      }
+      f->close_section();
+    }
+    f->dump_int("last_failure_osd_epoch", last_failure_osd_epoch);
+    f->open_array_section("active_clients");
+    for (const auto &c : clients) {
+      f->dump_object("client", c);
+    }
+    f->close_section();
+    f->close_section();
+  }
+
+  static void generate_test_instances(std::list<MgrMap*> &l) {
+    l.push_back(new MgrMap);
+  }
+
+  void print_summary(ceph::Formatter *f, std::ostream *ss) const
+  {
+    // One or the other, not both
+    ceph_assert((ss != nullptr) != (f != nullptr));
+    if (f) {
+      f->dump_bool("available", available);
+      f->dump_int("num_standbys", standbys.size());
+      f->open_array_section("modules");
+      for (auto& i : modules) {
+	f->dump_string("module", i);
+      }
+      f->close_section();
+      f->open_object_section("services");
+      for (const auto &i : services) {
+	f->dump_string(i.first.c_str(), i.second);
+      }
+      f->close_section();
+    } else {
+      utime_t now = ceph_clock_now();
+      if (get_active_gid() != 0) {
+	*ss << get_active_name();
+        if (!available) {
+          // If the daemon hasn't gone active yet, indicate that.
+          *ss << "(active, starting";
+        } else {
+          *ss << "(active";
+        }
+	if (active_change) {
+	  *ss << ", since " << utimespan_str(now - active_change);
+	}
+	*ss << ")";
+      } else {
+	*ss << "no daemons active";
+	if (active_change) {
+	  *ss << " (since " << utimespan_str(now - active_change) << ")";
+	}
+      }
+      if (standbys.size()) {
+	*ss << ", standbys: ";
+	bool first = true;
+	for (const auto &i : standbys) {
+	  if (!first) {
+	    *ss << ", ";
+	  }
+	  *ss << i.second.name;
+	  first = false;
+	}
+      }
+    }
+  }
+
+  friend std::ostream& operator<<(std::ostream& out, const MgrMap& m) {
+    std::ostringstream ss;
+    m.print_summary(nullptr, &ss);
+    return out << ss.str();
+  }
+
+  friend std::ostream& operator<<(std::ostream& out, const std::vector<ModuleInfo>& mi) {
+    for (const auto &i : mi) {
+      out << i.name << " ";
+    }
+    return out;
+  }
+};
+
+WRITE_CLASS_ENCODER_FEATURES(MgrMap)
+WRITE_CLASS_ENCODER(MgrMap::StandbyInfo)
+WRITE_CLASS_ENCODER(MgrMap::ModuleInfo);
+WRITE_CLASS_ENCODER(MgrMap::ModuleOption);
+
+#endif
+
diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc
new file mode 100644
index 000000000..bf5e2ed31
--- /dev/null
+++ b/src/mon/MgrMonitor.cc
@@ -0,0 +1,1356 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2016 John Spray <john.spray@redhat.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#include <boost/tokenizer.hpp>
+
+#include "messages/MMgrBeacon.h"
+#include "messages/MMgrMap.h"
+#include "messages/MMgrDigest.h"
+
+#include "include/stringify.h"
+#include "mgr/MgrContext.h"
+#include "mgr/mgr_commands.h"
+#include "OSDMonitor.h"
+#include "ConfigMonitor.h"
+#include "HealthMonitor.h"
+
+#include "MgrMonitor.h"
+
+#define MGR_METADATA_PREFIX "mgr_metadata"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, map)
+using namespace TOPNSPC::common;
+
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::ErasureCodeInterfaceRef;
+using ceph::ErasureCodeProfile;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::make_message;
+using ceph::mono_clock;
+using ceph::mono_time;
+
+static ostream& _prefix(std::ostream *_dout, Monitor &mon,
+			const MgrMap& mgrmap) {
+  return *_dout << "mon." << mon.name << "@" << mon.rank
+		<< "(" << mon.get_state_name()
+		<< ").mgr e" << mgrmap.get_epoch() << " ";
+}
+
+// the system treats always_on_modules as if they provide built-in functionality
+// by ensuring that they are always enabled.
+const static std::map<uint32_t, std::set<std::string>> always_on_modules = {
+  {
+    CEPH_RELEASE_NAUTILUS, {
+      "crash",
+      "status",
+      "progress",
+      "balancer",
+      "devicehealth",
+      "orchestrator_cli",
+      "rbd_support",
+      "volumes",
+    }
+  },
+  {
+    CEPH_RELEASE_OCTOPUS, {
+      "crash",
+      "status",
+      "progress",
+      "balancer",
+      "devicehealth",
+      "orchestrator",
+      "rbd_support",
+      "volumes",
+      "pg_autoscaler",
+      "telemetry",
+    }
+  },
+  {
+    CEPH_RELEASE_PACIFIC, {
+      "crash",
+      "status",
+      "progress",
+      "balancer",
+      "devicehealth",
+      "orchestrator",
+      "rbd_support",
+      "volumes",
+      "pg_autoscaler",
+      "telemetry",
+    }
+  }
+};
+
+// Prefix for mon store of active mgr's command descriptions
+const static std::string command_descs_prefix = "mgr_command_descs";
+
+const Option *MgrMonitor::find_module_option(const string& name)
+{
+  // we have two forms of names: "mgr/$module/$option" and
+  // localized "mgr/$module/$instance/$option".  normalize to the
+  // former by stripping out $instance.
+  string real_name;
+  if (name.substr(0, 4) != "mgr/") {
+    return nullptr;
+  }
+  auto second_slash = name.find('/', 5);
+  if (second_slash == std::string::npos) {
+    return nullptr;
+  }
+  auto third_slash = name.find('/', second_slash + 1);
+  if (third_slash != std::string::npos) {
+    // drop the $instance part between the second and third slash
+    real_name = name.substr(0, second_slash) + name.substr(third_slash);
+  } else {
+    real_name = name;
+  }
+  auto p = mgr_module_options.find(real_name);
+  if (p != mgr_module_options.end()) {
+    return &p->second;
+  }
+  return nullptr;
+}
+
+version_t MgrMonitor::get_trim_to() const
+{
+  int64_t max = g_conf().get_val<int64_t>("mon_max_mgrmap_epochs");
+  if (map.epoch > max) {
+    return map.epoch - max;
+  }
+  return 0;
+}
+
+void MgrMonitor::create_initial()
+{
+  // Take a local copy of initial_modules for tokenizer to iterate over.
+  auto initial_modules = g_conf().get_val<std::string>("mgr_initial_modules");
+  boost::tokenizer<> tok(initial_modules);
+  for (auto& m : tok) {
+    pending_map.modules.insert(m);
+  }
+  pending_map.always_on_modules = always_on_modules;
+  pending_command_descs = mgr_commands;
+  dout(10) << __func__ << " initial modules " << pending_map.modules
+	   << ", always on modules " << pending_map.get_always_on_modules()
+           << ", " << pending_command_descs.size() << " commands"
+	   << dendl;
+}
+
+void MgrMonitor::get_store_prefixes(std::set<string>& s) const
+{
+  s.insert(service_name);
+  s.insert(command_descs_prefix);
+  s.insert(MGR_METADATA_PREFIX);
+}
+
+void MgrMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  version_t version = get_last_committed();
+  if (version != map.epoch) {
+    dout(4) << "loading version " << version << dendl;
+
+    bufferlist bl;
+    int err = get_version(version, bl);
+    ceph_assert(err == 0);
+
+    bool old_available = map.get_available();
+    uint64_t old_gid = map.get_active_gid();
+
+    auto p = bl.cbegin();
+    map.decode(p);
+
+    dout(4) << "active server: " << map.active_addrs
+	    << "(" << map.active_gid << ")" << dendl;
+
+    ever_had_active_mgr = get_value("ever_had_active_mgr");
+
+    load_health();
+
+    if (map.available) {
+      first_seen_inactive = utime_t();
+    } else {
+      first_seen_inactive = ceph_clock_now();
+    }
+
+    check_subs();
+
+    if (version == 1
+        || command_descs.empty()
+        || (map.get_available()
+            && (!old_available || old_gid != map.get_active_gid()))) {
+      dout(4) << "mkfs or daemon transitioned to available, loading commands"
+	      << dendl;
+      bufferlist loaded_commands;
+      int r = mon.store->get(command_descs_prefix, "", loaded_commands);
+      if (r < 0) {
+        derr << "Failed to load mgr commands: " << cpp_strerror(r) << dendl;
+      } else {
+        auto p = loaded_commands.cbegin();
+        decode(command_descs, p);
+      }
+    }
+  }
+
+  // populate module options
+  mgr_module_options.clear();
+  misc_option_strings.clear();
+  for (auto& i : map.available_modules) {
+    for (auto& j : i.module_options) {
+      string name = string("mgr/") + i.name + "/" + j.second.name;
+      auto p = mgr_module_options.emplace(
+	name,
+	Option(name, static_cast<Option::type_t>(j.second.type),
+	       static_cast<Option::level_t>(j.second.level)));
+      Option& opt = p.first->second;
+      opt.set_flags(static_cast<Option::flag_t>(j.second.flags));
+      opt.set_flag(Option::FLAG_MGR);
+      opt.set_description(j.second.desc.c_str());
+      opt.set_long_description(j.second.long_desc.c_str());
+      for (auto& k : j.second.tags) {
+	opt.add_tag(k.c_str());
+      }
+      for (auto& k : j.second.see_also) {
+	if (i.module_options.count(k)) {
+	  // it's another module option
+	  misc_option_strings.push_back(string("mgr/") + i.name + "/" + k);
+	  opt.add_see_also(misc_option_strings.back().c_str());
+	} else {
+	  // it's a native option
+	  opt.add_see_also(k.c_str());
+	}
+      }
+      Option::value_t v, v2;
+      std::string err;
+      if (j.second.default_value.size() &&
+	  !opt.parse_value(j.second.default_value, &v, &err)) {
+	opt.set_default(v);
+      }
+      if (j.second.min.size() &&
+	  j.second.max.size() &&
+	  !opt.parse_value(j.second.min, &v, &err) &&
+	  !opt.parse_value(j.second.max, &v2, &err)) {
+	opt.set_min_max(v, v2);
+      }
+      std::vector<const char *> enum_allowed;
+      for (auto& k : j.second.enum_allowed) {
+	enum_allowed.push_back(k.c_str());
+      }
+      opt.set_enum_allowed(enum_allowed);
+    }
+  }
+  // force ConfigMonitor to refresh, since it uses const Option *
+  // pointers into our mgr_module_options (which we just rebuilt).
+  mon.configmon()->load_config();
+
+  if (!mon.is_init()) {
+    // feed our pet MgrClient, unless we are in Monitor::[pre]init()
+    prime_mgr_client();
+  }
+}
+
+void MgrMonitor::prime_mgr_client()
+{
+  dout(10) << __func__ << dendl;
+  mon.mgr_client.ms_dispatch2(make_message<MMgrMap>(map));
+}
+
+void MgrMonitor::create_pending()
+{
+  pending_map = map;
+  pending_map.epoch++;
+}
+
+health_status_t MgrMonitor::should_warn_about_mgr_down()
+{
+  utime_t now = ceph_clock_now();
+  // we warn if we have osds AND we've exceeded the grace period
+  // which means a new mon cluster and be HEALTH_OK indefinitely as long as
+  // no OSDs are ever created.
+  if (mon.osdmon()->osdmap.get_num_osds() > 0 &&
+       now > mon.monmap->created + g_conf().get_val<int64_t>("mon_mgr_mkfs_grace")) {
+    health_status_t level = HEALTH_WARN;
+    if (first_seen_inactive != utime_t() &&
+	now - first_seen_inactive > g_conf().get_val<int64_t>("mon_mgr_inactive_grace")) {
+      level = HEALTH_ERR;
+    }
+    return level;
+  }
+  return HEALTH_OK;
+}
+
+void MgrMonitor::post_paxos_update()
+{
+  // are we handling digest subscribers?
+  if (digest_event) {
+    bool send = false;
+    if (prev_health_checks.empty()) {
+      prev_health_checks.resize(mon.paxos_service.size());
+      send = true;
+    }
+    ceph_assert(prev_health_checks.size() == mon.paxos_service.size());
+    for (auto i = 0u; i < prev_health_checks.size(); i++) {
+      const auto& curr = mon.paxos_service[i]->get_health_checks();
+      if (!send && curr != prev_health_checks[i]) {
+        send = true;
+      }
+      prev_health_checks[i] = curr;
+    }
+    if (send) {
+      if (is_active()) {
+        send_digests();
+      } else {
+        cancel_timer();
+        wait_for_active_ctx(new C_MonContext{&mon, [this](int) {
+          send_digests();
+        }});
+      }
+    }
+  }
+}
+
+void MgrMonitor::encode_pending(MonitorDBStore::TransactionRef t)
+{
+  dout(10) << __func__ << " " << pending_map << dendl;
+  bufferlist bl;
+  pending_map.encode(bl, mon.get_quorum_con_features());
+  put_version(t, pending_map.epoch, bl);
+  put_last_committed(t, pending_map.epoch);
+
+  for (auto& p : pending_metadata) {
+    dout(10) << __func__ << " set metadata for " << p.first << dendl;
+    t->put(MGR_METADATA_PREFIX, p.first, p.second);
+  }
+  for (auto& name : pending_metadata_rm) {
+    dout(10) << __func__ << " rm metadata for " << name << dendl;
+    t->erase(MGR_METADATA_PREFIX, name);
+  }
+  pending_metadata.clear();
+  pending_metadata_rm.clear();
+
+  health_check_map_t next;
+  if (pending_map.active_gid == 0) {
+    auto level = should_warn_about_mgr_down();
+    if (level != HEALTH_OK) {
+      next.add("MGR_DOWN", level, "no active mgr", 0);
+    } else {
+      dout(10) << __func__ << " no health warning (never active and new cluster)"
+	       << dendl;
+    }
+  } else {
+    put_value(t, "ever_had_active_mgr", 1);
+  }
+  encode_health(next, t);
+
+  if (pending_command_descs.size()) {
+    dout(4) << __func__ << " encoding " << pending_command_descs.size()
+            << " command_descs" << dendl;
+    for (auto& p : pending_command_descs) {
+      p.set_flag(MonCommand::FLAG_MGR);
+    }
+    bufferlist bl;
+    encode(pending_command_descs, bl);
+    t->put(command_descs_prefix, "", bl);
+    pending_command_descs.clear();
+  }
+}
+
+bool MgrMonitor::check_caps(MonOpRequestRef op, const uuid_d& fsid)
+{
+  // check permissions
+  MonSession *session = op->get_session();
+  if (!session)
+    return false;
+  if (!session->is_capable("mgr", MON_CAP_X)) {
+    dout(1) << __func__ << " insufficient caps " << session->caps << dendl;
+    return false;
+  }
+  if (fsid != mon.monmap->fsid) {
+    dout(1) << __func__ << " op fsid " << fsid
+	    << " != " << mon.monmap->fsid << dendl;
+    return false;
+  }
+  return true;
+}
+
+bool MgrMonitor::preprocess_query(MonOpRequestRef op)
+{
+  auto m = op->get_req<PaxosServiceMessage>();
+  switch (m->get_type()) {
+    case MSG_MGR_BEACON:
+      return preprocess_beacon(op);
+    case MSG_MON_COMMAND:
+      try {
+	return preprocess_command(op);
+      } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+
+    default:
+      mon.no_reply(op);
+      derr << "Unhandled message type " << m->get_type() << dendl;
+      return true;
+  }
+}
+
+bool MgrMonitor::prepare_update(MonOpRequestRef op)
+{
+  auto m = op->get_req<PaxosServiceMessage>();
+  switch (m->get_type()) {
+    case MSG_MGR_BEACON:
+      return prepare_beacon(op);
+
+    case MSG_MON_COMMAND:
+      try {
+	return prepare_command(op);
+      } catch (const bad_cmd_get& e) {
+	bufferlist bl;
+	mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+	return true;
+      }
+
+    default:
+      mon.no_reply(op);
+      derr << "Unhandled message type " << m->get_type() << dendl;
+      return true;
+  }
+}
+
+
+
+class C_Updated : public Context {
+  MgrMonitor *mm;
+  MonOpRequestRef op;
+public:
+  C_Updated(MgrMonitor *a, MonOpRequestRef c) :
+    mm(a), op(c) {}
+  void finish(int r) override {
+    if (r >= 0) {
+      // Success 
+    } else if (r == -ECANCELED) {
+      mm->mon.no_reply(op);
+    } else {
+      mm->dispatch(op);        // try again
+    }
+  }
+};
+
+bool MgrMonitor::preprocess_beacon(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMgrBeacon>();
+  mon.no_reply(op); // we never reply to beacons
+  dout(4) << "beacon from " << m->get_gid() << dendl;
+
+  if (!check_caps(op, m->get_fsid())) {
+    // drop it on the floor
+    return true;
+  }
+
+  // always send this to the leader's prepare_beacon()
+  return false;
+}
+
+bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMgrBeacon>();
+  dout(4) << "beacon from " << m->get_gid() << dendl;
+
+  // See if we are seeing same name, new GID for the active daemon
+  if (m->get_name() == pending_map.active_name
+      && m->get_gid() != pending_map.active_gid)
+  {
+    dout(4) << "Active daemon restart (mgr." << m->get_name() << ")" << dendl;
+    mon.clog->info() << "Active manager daemon " << m->get_name()
+                      << " restarted";
+    if (!mon.osdmon()->is_writeable()) {
+      dout(1) << __func__ << ":  waiting for osdmon writeable to"
+                 " blocklist old instance." << dendl;
+      mon.osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+      return false;
+    }
+    drop_active();
+  }
+
+  // See if we are seeing same name, new GID for any standbys
+  for (const auto &i : pending_map.standbys) {
+    const MgrMap::StandbyInfo &s = i.second;
+    if (s.name == m->get_name() && s.gid != m->get_gid()) {
+      dout(4) << "Standby daemon restart (mgr." << m->get_name() << ")" << dendl;
+      mon.clog->debug() << "Standby manager daemon " << m->get_name()
+                         << " restarted";
+      drop_standby(i.first);
+      break;
+    }
+  }
+
+  last_beacon[m->get_gid()] = ceph::coarse_mono_clock::now();
+
+  // Track whether we modified pending_map
+  bool updated = false;
+
+  if (pending_map.active_gid == m->get_gid()) {
+    if (pending_map.services != m->get_services()) {
+      dout(4) << "updated services from mgr." << m->get_name()
+              << ": " << m->get_services() << dendl;
+      pending_map.services = m->get_services();
+      updated = true;
+    }
+
+    // A beacon from the currently active daemon
+    if (pending_map.active_addrs != m->get_server_addrs()) {
+      dout(4) << "learned address " << m->get_server_addrs()
+	      << " (was " << pending_map.active_addrs << ")" << dendl;
+      pending_map.active_addrs = m->get_server_addrs();
+      updated = true;
+    }
+
+    if (pending_map.get_available() != m->get_available()) {
+      dout(4) << "available " << m->get_gid() << dendl;
+      mon.clog->info() << "Manager daemon " << pending_map.active_name
+                        << " is now available";
+
+      // This beacon should include command descriptions
+      pending_command_descs = m->get_command_descs();
+      if (pending_command_descs.empty()) {
+        // This should not happen, but it also isn't fatal: we just
+        // won't successfully update our list of commands.
+        dout(4) << "First available beacon from " << pending_map.active_name
+                << "(" << m->get_gid() << ") does not include command descs"
+                << dendl;
+      } else {
+        dout(4) << "First available beacon from " << pending_map.active_name
+                << "(" << m->get_gid() << ") includes "
+                << pending_command_descs.size() << " command descs" << dendl;
+      }
+
+      pending_map.available = m->get_available();
+      updated = true;
+    }
+    if (pending_map.available_modules != m->get_available_modules()) {
+      dout(4) << "available_modules " << m->get_available_modules()
+	      << " (was " << pending_map.available_modules << ")" << dendl;
+      pending_map.available_modules = m->get_available_modules();
+      updated = true;
+    }
+    const auto& clients = m->get_clients();
+    if (pending_map.clients != clients) {
+      dout(4) << "active's RADOS clients " << clients
+	      << " (was " << pending_map.clients << ")" << dendl;
+      pending_map.clients = clients;
+      updated = true;
+    }
+  } else if (pending_map.active_gid == 0) {
+    // There is no currently active daemon, select this one.
+    if (pending_map.standbys.count(m->get_gid())) {
+      drop_standby(m->get_gid(), false);
+    }
+    dout(4) << "selecting new active " << m->get_gid()
+	    << " " << m->get_name()
+	    << " (was " << pending_map.active_gid << " "
+	    << pending_map.active_name << ")" << dendl;
+    pending_map.active_gid = m->get_gid();
+    pending_map.active_name = m->get_name();
+    pending_map.active_change = ceph_clock_now();
+    pending_map.active_mgr_features = m->get_mgr_features();
+    pending_map.available_modules = m->get_available_modules();
+    encode(m->get_metadata(), pending_metadata[m->get_name()]);
+    pending_metadata_rm.erase(m->get_name());
+
+    mon.clog->info() << "Activating manager daemon "
+                      << pending_map.active_name;
+
+    updated = true;
+  } else {
+    if (pending_map.standbys.count(m->get_gid()) > 0) {
+      dout(10) << "from existing standby " << m->get_gid() << dendl;
+      if (pending_map.standbys[m->get_gid()].available_modules !=
+	  m->get_available_modules()) {
+	dout(10) << "existing standby " << m->get_gid() << " available_modules "
+		 << m->get_available_modules() << " (was "
+		 << pending_map.standbys[m->get_gid()].available_modules << ")"
+		 << dendl;
+	pending_map.standbys[m->get_gid()].available_modules =
+	  m->get_available_modules();
+	updated = true;
+      }
+    } else {
+      dout(10) << "new standby " << m->get_gid() << dendl;
+      mon.clog->debug() << "Standby manager daemon " << m->get_name()
+                         << " started";
+      pending_map.standbys[m->get_gid()] = {m->get_gid(), m->get_name(),
+					    m->get_available_modules(),
+					    m->get_mgr_features()};
+      encode(m->get_metadata(), pending_metadata[m->get_name()]);
+      pending_metadata_rm.erase(m->get_name());
+      updated = true;
+    }
+  }
+
+  if (updated) {
+    dout(4) << "updating map" << dendl;
+    wait_for_finished_proposal(op, new C_Updated(this, op));
+  } else {
+    dout(10) << "no change" << dendl;
+  }
+
+  return updated;
+}
+
+void MgrMonitor::check_subs()
+{
+  const std::string type = "mgrmap";
+  if (mon.session_map.subs.count(type) == 0)
+    return;
+  for (auto sub : *(mon.session_map.subs[type])) {
+    check_sub(sub);
+  }
+}
+
+void MgrMonitor::check_sub(Subscription *sub)
+{
+  if (sub->type == "mgrmap") {
+    if (sub->next <= map.get_epoch()) {
+      dout(20) << "Sending map to subscriber " << sub->session->con
+	       << " " << sub->session->con->get_peer_addr() << dendl;
+      sub->session->con->send_message2(make_message<MMgrMap>(map));
+      if (sub->onetime) {
+        mon.session_map.remove_sub(sub);
+      } else {
+        sub->next = map.get_epoch() + 1;
+      }
+    }
+  } else {
+    ceph_assert(sub->type == "mgrdigest");
+    if (sub->next == 0) {
+      // new registration; cancel previous timer
+      cancel_timer();
+    }
+    if (digest_event == nullptr) {
+      send_digests();
+    }
+  }
+}
+
+/**
+ * Handle digest subscriptions separately (outside of check_sub) because
+ * they are going to be periodic rather than version-driven.
+ */
+void MgrMonitor::send_digests()
+{
+  cancel_timer();
+
+  const std::string type = "mgrdigest";
+  if (mon.session_map.subs.count(type) == 0) {
+    prev_health_checks.clear();
+    return;
+  }
+
+  if (!is_active()) {
+    // if paxos is currently not active, don't send a digest but reenable timer
+    goto timer;
+  }
+  dout(10) << __func__ << dendl;
+
+  for (auto sub : *(mon.session_map.subs[type])) {
+    dout(10) << __func__ << " sending digest to subscriber " << sub->session->con
+	     << " " << sub->session->con->get_peer_addr() << dendl;
+    auto mdigest = make_message<MMgrDigest>();
+
+    JSONFormatter f;
+    mon.healthmon()->get_health_status(true, &f, nullptr, nullptr, nullptr);
+    f.flush(mdigest->health_json);
+    f.reset();
+
+    mon.get_mon_status(&f);
+    f.flush(mdigest->mon_status_json);
+    f.reset();
+
+    sub->session->con->send_message2(mdigest);
+  }
+
+timer:
+  digest_event = mon.timer.add_event_after(
+    g_conf().get_val<int64_t>("mon_mgr_digest_period"),
+    new C_MonContext{&mon, [this](int) {
+      send_digests();
+  }});
+}
+
+void MgrMonitor::cancel_timer()
+{
+  if (digest_event) {
+    mon.timer.cancel_event(digest_event);
+    digest_event = nullptr;
+  }
+}
+
+void MgrMonitor::on_active()
+{
+  if (!mon.is_leader()) {
+    return;
+  }
+  mon.clog->debug() << "mgrmap e" << map.epoch << ": " << map;
+  if (!HAVE_FEATURE(mon.get_quorum_con_features(), SERVER_NAUTILUS)) {
+    return;
+  }
+  if (pending_map.always_on_modules == always_on_modules) {
+    return;
+  }
+  dout(4) << "always on modules changed, pending "
+          << pending_map.always_on_modules << " != wanted "
+          << always_on_modules << dendl;
+  pending_map.always_on_modules = always_on_modules;
+  propose_pending();
+}
+
+void MgrMonitor::tick()
+{
+  if (!is_active() || !mon.is_leader())
+    return;
+
+  const auto now = ceph::coarse_mono_clock::now();
+
+  const auto mgr_beacon_grace =
+      g_conf().get_val<std::chrono::seconds>("mon_mgr_beacon_grace");
+
+  // Note that this is the mgr daemon's tick period, not ours (the
+  // beacon is sent with this period).
+  const auto mgr_tick_period =
+      g_conf().get_val<std::chrono::seconds>("mgr_tick_period");
+
+  if (last_tick != ceph::coarse_mono_clock::time_point::min()
+      && (now - last_tick > (mgr_beacon_grace - mgr_tick_period))) {
+    // This case handles either local slowness (calls being delayed
+    // for whatever reason) or cluster election slowness (a long gap
+    // between calls while an election happened)
+    dout(4) << __func__ << ": resetting beacon timeouts due to mon delay "
+            "(slow election?) of " << now - last_tick << " seconds" << dendl;
+    for (auto &i : last_beacon) {
+      i.second = now;
+    }
+  }
+
+  last_tick = now;
+
+  // Populate any missing beacons (i.e. no beacon since MgrMonitor
+  // instantiation) with the current time, so that they will
+  // eventually look laggy if they fail to give us a beacon.
+  if (pending_map.active_gid != 0
+      && last_beacon.count(pending_map.active_gid) == 0) {
+    last_beacon[pending_map.active_gid] = now;
+  }
+  for (auto s : pending_map.standbys) {
+    if (last_beacon.count(s.first) == 0) {
+      last_beacon[s.first] = now;
+    }
+  }
+
+  // Cull standbys first so that any remaining standbys
+  // will be eligible to take over from the active if we cull him.
+  std::list<uint64_t> dead_standbys;
+  const auto cutoff = now - mgr_beacon_grace;
+  for (const auto &i : pending_map.standbys) {
+    auto last_beacon_time = last_beacon.at(i.first);
+    if (last_beacon_time < cutoff) {
+      dead_standbys.push_back(i.first);
+    }
+  }
+
+  bool propose = false;
+
+  for (auto i : dead_standbys) {
+    dout(4) << "Dropping laggy standby " << i << dendl;
+    drop_standby(i);
+    propose = true;
+  }
+
+  if (pending_map.active_gid != 0
+      && last_beacon.at(pending_map.active_gid) < cutoff
+      && mon.osdmon()->is_writeable()) {
+    const std::string old_active_name = pending_map.active_name;
+    drop_active();
+    propose = true;
+    dout(4) << "Dropping active" << pending_map.active_gid << dendl;
+    if (promote_standby()) {
+      dout(4) << "Promoted standby " << pending_map.active_gid << dendl;
+      mon.clog->info() << "Manager daemon " << old_active_name
+                        << " is unresponsive, replacing it with standby"
+                        << " daemon " << pending_map.active_name;
+    } else {
+      dout(4) << "Active is laggy but have no standbys to replace it" << dendl;
+      mon.clog->info() << "Manager daemon " << old_active_name
+                        << " is unresponsive.  No standby daemons available.";
+    }
+  } else if (pending_map.active_gid == 0) {
+    if (promote_standby()) {
+      dout(4) << "Promoted standby " << pending_map.active_gid << dendl;
+      mon.clog->info() << "Activating manager daemon "
+                      << pending_map.active_name;
+      propose = true;
+    }
+  }
+
+  if (!pending_map.available &&
+      !ever_had_active_mgr &&
+      should_warn_about_mgr_down() != HEALTH_OK) {
+    dout(10) << " exceeded mon_mgr_mkfs_grace "
+             << g_conf().get_val<int64_t>("mon_mgr_mkfs_grace")
+             << " seconds" << dendl;
+    propose = true;
+  }
+
+  // obsolete modules?
+  if (mon.monmap->min_mon_release >= ceph_release_t::octopus &&
+      pending_map.module_enabled("orchestrator_cli")) {
+    dout(10) << " disabling obsolete/renamed 'orchestrator_cli'" << dendl;
+    // we don't need to enable 'orchestrator' because it's now always-on
+    pending_map.modules.erase("orchestrator_cli");
+    propose = true;
+  }
+
+  if (propose) {
+    propose_pending();
+  }
+}
+
+void MgrMonitor::on_restart()
+{
+  // Clear out the leader-specific state.
+  last_beacon.clear();
+  last_tick = ceph::coarse_mono_clock::now();
+}
+
+
+bool MgrMonitor::promote_standby()
+{
+  ceph_assert(pending_map.active_gid == 0);
+  if (pending_map.standbys.size()) {
+    // Promote a replacement (arbitrary choice of standby)
+    auto replacement_gid = pending_map.standbys.begin()->first;
+    pending_map.active_gid = replacement_gid;
+    pending_map.active_name = pending_map.standbys.at(replacement_gid).name;
+    pending_map.available_modules =
+      pending_map.standbys.at(replacement_gid).available_modules;
+    pending_map.active_mgr_features =
+      pending_map.standbys.at(replacement_gid).mgr_features;
+    pending_map.available = false;
+    pending_map.active_addrs = entity_addrvec_t();
+    pending_map.active_change = ceph_clock_now();
+
+    drop_standby(replacement_gid, false);
+
+    return true;
+  } else {
+    return false;
+  }
+}
+
+void MgrMonitor::drop_active()
+{
+  ceph_assert(mon.osdmon()->is_writeable());
+
+  if (last_beacon.count(pending_map.active_gid) > 0) {
+    last_beacon.erase(pending_map.active_gid);
+  }
+
+  ceph_assert(pending_map.active_gid > 0);
+  auto until = ceph_clock_now();
+  until += g_conf().get_val<double>("mon_mgr_blocklist_interval");
+  dout(5) << "blocklisting previous mgr." << pending_map.active_name << "."
+          << pending_map.active_gid << " ("
+          << pending_map.active_addrs << ")" << dendl;
+  auto blocklist_epoch = mon.osdmon()->blocklist(pending_map.active_addrs, until);
+
+  /* blocklist RADOS clients in use by the mgr */
+  for (const auto& a : pending_map.clients) {
+    mon.osdmon()->blocklist(a, until);
+  }
+  request_proposal(mon.osdmon());
+
+  pending_metadata_rm.insert(pending_map.active_name);
+  pending_metadata.erase(pending_map.active_name);
+  pending_map.active_name = "";
+  pending_map.active_gid = 0;
+  pending_map.active_change = ceph_clock_now();
+  pending_map.active_mgr_features = 0;
+  pending_map.available = false;
+  pending_map.active_addrs = entity_addrvec_t();
+  pending_map.services.clear();
+  pending_map.clients.clear();
+  pending_map.last_failure_osd_epoch = blocklist_epoch;
+
+  // So that when new active mgr subscribes to mgrdigest, it will
+  // get an immediate response instead of waiting for next timer
+  cancel_timer();
+}
+
+void MgrMonitor::drop_standby(uint64_t gid, bool drop_meta)
+{
+  if (drop_meta) {
+    pending_metadata_rm.insert(pending_map.standbys[gid].name);
+    pending_metadata.erase(pending_map.standbys[gid].name);
+  }
+  pending_map.standbys.erase(gid);
+  if (last_beacon.count(gid) > 0) {
+    last_beacon.erase(gid);
+  }
+}
+
+bool MgrMonitor::preprocess_command(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonCommand>();
+  std::stringstream ss;
+  bufferlist rdata;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    mon.reply_command(op, -EACCES, "access denied", rdata,
+		       get_last_committed());
+    return true;
+  }
+
+  string format;
+  cmd_getval(cmdmap, "format", format);
+  boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty",
+						   "json-pretty"));
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+  int r = 0;
+
+  if (prefix == "mgr stat") {
+    f->open_object_section("stat");
+    f->dump_unsigned("epoch", map.get_epoch());
+    f->dump_bool("available", map.get_available());
+    f->dump_string("active_name", map.get_active_name());
+    f->dump_unsigned("num_standby", map.get_num_standby());
+    f->close_section();
+    f->flush(rdata);
+  } else if (prefix == "mgr dump") {
+    int64_t epoch = 0;
+    cmd_getval(cmdmap, "epoch", epoch, (int64_t)map.get_epoch());
+    if (epoch == (int64_t)map.get_epoch()) {
+      f->dump_object("mgrmap", map);
+    } else {
+      bufferlist bl;
+      int err = get_version(epoch, bl);
+      if (err == -ENOENT) {
+	r = -ENOENT;
+	ss << "there is no map for epoch " << epoch;
+	goto reply;
+      }
+      MgrMap m;
+      auto p = bl.cbegin();
+      m.decode(p);
+      f->dump_object("mgrmap", m);
+    }
+    f->flush(rdata);
+  } else if (prefix == "mgr module ls") {
+    f->open_object_section("modules");
+    {
+      f->open_array_section("always_on_modules");
+      for (auto& p : map.get_always_on_modules()) {
+        f->dump_string("module", p);
+      }
+      f->close_section();
+      f->open_array_section("enabled_modules");
+      for (auto& p : map.modules) {
+        if (map.get_always_on_modules().count(p) > 0)
+          continue;
+        // We only show the name for enabled modules.  The any errors
+        // etc will show up as a health checks.
+        f->dump_string("module", p);
+      }
+      f->close_section();
+      f->open_array_section("disabled_modules");
+      for (auto& p : map.available_modules) {
+        if (map.modules.count(p.name) == 0 &&
+            map.get_always_on_modules().count(p.name) == 0) {
+          // For disabled modules, we show the full info, to
+          // give a hint about whether enabling it will work
+          p.dump(f.get());
+        }
+      }
+      f->close_section();
+    }
+    f->close_section();
+    f->flush(rdata);
+  } else if (prefix == "mgr services") {
+    f->open_object_section("services");
+    for (const auto &i : map.services) {
+      f->dump_string(i.first.c_str(), i.second);
+    }
+    f->close_section();
+    f->flush(rdata);
+  } else if (prefix == "mgr metadata") {
+    string name;
+    cmd_getval(cmdmap, "who", name);
+    if (name.size() > 0 && !map.have_name(name)) {
+      ss << "mgr." << name << " does not exist";
+      r = -ENOENT;
+      goto reply;
+    }
+    string format;
+    cmd_getval(cmdmap, "format", format);
+    boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
+    if (name.size()) {
+      f->open_object_section("mgr_metadata");
+      f->dump_string("name", name);
+      r = dump_metadata(name, f.get(), &ss);
+      if (r < 0)
+        goto reply;
+      f->close_section();
+    } else {
+      r = 0;
+      f->open_array_section("mgr_metadata");
+      for (auto& i : map.get_all_names()) {
+	f->open_object_section("mgr");
+	f->dump_string("name", i);
+	r = dump_metadata(i, f.get(), NULL);
+	if (r == -EINVAL || r == -ENOENT) {
+	  // Drop error, continue to get other daemons' metadata
+	  dout(4) << "No metadata for mgr." << i << dendl;
+	  r = 0;
+	} else if (r < 0) {
+	  // Unexpected error
+	  goto reply;
+	}
+	f->close_section();
+      }
+      f->close_section();
+    }
+    f->flush(rdata);
+  } else if (prefix == "mgr versions") {
+    count_metadata("ceph_version", f.get());
+    f->flush(rdata);
+    r = 0;
+  } else if (prefix == "mgr count-metadata") {
+    string field;
+    cmd_getval(cmdmap, "property", field);
+    count_metadata(field, f.get());
+    f->flush(rdata);
+    r = 0;
+  } else {
+    return false;
+  }
+
+reply:
+  string rs;
+  getline(ss, rs);
+  mon.reply_command(op, r, rs, rdata, get_last_committed());
+  return true;
+}
+
+bool MgrMonitor::prepare_command(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonCommand>();
+
+  std::stringstream ss;
+  bufferlist rdata;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    mon.reply_command(op, -EACCES, "access denied", rdata, get_last_committed());
+    return true;
+  }
+
+  string format;
+  cmd_getval(cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+
+  int r = 0;
+
+  if (prefix == "mgr fail") {
+    string who;
+    if (!cmd_getval(cmdmap, "who", who)) {
+      if (!map.active_gid) {
+	ss << "Currently no active mgr";
+	goto out;
+      }
+      who = map.active_name;
+    }
+
+    std::string err;
+    uint64_t gid = strict_strtol(who.c_str(), 10, &err);
+    bool changed = false;
+    if (!err.empty()) {
+      // Does not parse as a gid, treat it as a name
+      if (pending_map.active_name == who) {
+        if (!mon.osdmon()->is_writeable()) {
+          mon.osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+          return false;
+        }
+        drop_active();
+        changed = true;
+      } else {
+        gid = 0;
+        for (const auto &i : pending_map.standbys) {
+          if (i.second.name == who) {
+            gid = i.first;
+            break;
+          }
+        }
+        if (gid != 0) {
+          drop_standby(gid);
+          changed = true;
+        } else {
+          ss << "Daemon not found '" << who << "', already failed?";
+        }
+      }
+    } else {
+      if (pending_map.active_gid == gid) {
+        if (!mon.osdmon()->is_writeable()) {
+          mon.osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+          return false;
+        }
+        drop_active();
+        changed = true;
+      } else if (pending_map.standbys.count(gid) > 0) {
+        drop_standby(gid);
+        changed = true;
+      } else {
+        ss << "Daemon not found '" << gid << "', already failed?";
+      }
+    }
+
+    if (changed && pending_map.active_gid == 0) {
+      promote_standby();
+    }
+  } else if (prefix == "mgr module enable") {
+    string module;
+    cmd_getval(cmdmap, "module", module);
+    if (module.empty()) {
+      r = -EINVAL;
+      goto out;
+    }
+    if (pending_map.get_always_on_modules().count(module) > 0) {
+      ss << "module '" << module << "' is already enabled (always-on)";
+      goto out;
+    }
+    string force;
+    cmd_getval(cmdmap, "force", force);
+    if (!pending_map.all_support_module(module) &&
+	force != "--force") {
+      ss << "all mgr daemons do not support module '" << module << "', pass "
+	 << "--force to force enablement";
+      r = -ENOENT;
+      goto out;
+    }
+
+    std::string can_run_error;
+    if (force != "--force" && !pending_map.can_run_module(module, &can_run_error)) {
+      ss << "module '" << module << "' reports that it cannot run on the active "
+            "manager daemon: " << can_run_error << " (pass --force to force "
+            "enablement)";
+      r = -ENOENT;
+      goto out;
+    }
+
+    if (pending_map.module_enabled(module)) {
+      ss << "module '" << module << "' is already enabled";
+      r = 0;
+      goto out;
+    }
+    pending_map.modules.insert(module);
+  } else if (prefix == "mgr module disable") {
+    string module;
+    cmd_getval(cmdmap, "module", module);
+    if (module.empty()) {
+      r = -EINVAL;
+      goto out;
+    }
+    if (pending_map.get_always_on_modules().count(module) > 0) {
+      ss << "module '" << module << "' cannot be disabled (always-on)";
+      r = -EINVAL;
+      goto out;
+    }
+    if (!pending_map.module_enabled(module)) {
+      ss << "module '" << module << "' is already disabled";
+      r = 0;
+      goto out;
+    }
+    if (!pending_map.modules.count(module)) {
+      ss << "module '" << module << "' is not enabled";
+    }
+    pending_map.modules.erase(module);
+  } else {
+    ss << "Command '" << prefix << "' not implemented!";
+    r = -ENOSYS;
+  }
+
+out:
+  dout(4) << __func__ << " done, r=" << r << dendl;
+  /* Compose response */
+  string rs;
+  getline(ss, rs);
+
+  if (r >= 0) {
+    // success.. delay reply
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, r, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else {
+    // reply immediately
+    mon.reply_command(op, r, rs, rdata, get_last_committed());
+    return false;
+  }
+}
+
+void MgrMonitor::init()
+{
+  if (digest_event == nullptr) {
+    send_digests();  // To get it to schedule its own event
+  }
+}
+
+void MgrMonitor::on_shutdown()
+{
+  cancel_timer();
+}
+
+int MgrMonitor::load_metadata(const string& name, std::map<string, string>& m,
+			      ostream *err) const
+{
+  bufferlist bl;
+  int r = mon.store->get(MGR_METADATA_PREFIX, name, bl);
+  if (r < 0)
+    return r;
+  try {
+    auto p = bl.cbegin();
+    decode(m, p);
+  }
+  catch (ceph::buffer::error& e) {
+    if (err)
+      *err << "mgr." << name << " metadata is corrupt";
+    return -EIO;
+  }
+  return 0;
+}
+
+void MgrMonitor::count_metadata(const string& field, std::map<string,int> *out)
+{
+  std::set<string> ls = map.get_all_names();
+  for (auto& name : ls) {
+    std::map<string,string> meta;
+    load_metadata(name, meta, nullptr);
+    auto p = meta.find(field);
+    if (p == meta.end()) {
+      (*out)["unknown"]++;
+    } else {
+      (*out)[p->second]++;
+    }
+  }
+}
+
+void MgrMonitor::count_metadata(const string& field, Formatter *f)
+{
+  std::map<string,int> by_val;
+  count_metadata(field, &by_val);
+  f->open_object_section(field.c_str());
+  for (auto& p : by_val) {
+    f->dump_int(p.first.c_str(), p.second);
+  }
+  f->close_section();
+}
+
+void MgrMonitor::get_versions(std::map<string, list<string> > &versions)
+{
+  std::set<string> ls = map.get_all_names();
+  for (auto& name : ls) {
+    std::map<string,string> meta;
+    load_metadata(name, meta, nullptr);
+    auto p = meta.find("ceph_version_short");
+    if (p == meta.end()) continue;
+    versions[p->second].push_back(string("mgr.") + name);
+  }
+}
+
+int MgrMonitor::dump_metadata(const string& name, Formatter *f, ostream *err)
+{
+  std::map<string,string> m;
+  if (int r = load_metadata(name, m, err))
+    return r;
+  for (auto& p : m) {
+    f->dump_string(p.first.c_str(), p.second);
+  }
+  return 0;
+}
+
+void MgrMonitor::print_nodes(Formatter *f) const
+{
+  ceph_assert(f);
+
+  std::map<string, list<string> > mgrs; // hostname => mgr
+  auto ls = map.get_all_names();
+  for (auto& name : ls) {
+    std::map<string,string> meta;
+    if (load_metadata(name, meta, nullptr)) {
+      continue;
+    }
+    auto hostname = meta.find("hostname");
+    if (hostname == meta.end()) {
+      // not likely though
+      continue;
+    }
+    mgrs[hostname->second].push_back(name);
+  }
+
+  dump_services(f, mgrs, "mgr");
+}
+
+const std::vector<MonCommand> &MgrMonitor::get_command_descs() const
+{
+  if (command_descs.empty()) {
+    // must have just upgraded; fallback to static commands
+    return mgr_commands;
+  } else {
+    return command_descs;
+  }
+}
diff --git a/src/mon/MgrMonitor.h b/src/mon/MgrMonitor.h
new file mode 100644
index 000000000..be75602ab
--- /dev/null
+++ b/src/mon/MgrMonitor.h
@@ -0,0 +1,140 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2016 John Spray <john.spray@redhat.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#ifndef CEPH_MGRMONITOR_H
+#define CEPH_MGRMONITOR_H
+
+#include <map>
+#include <set>
+
+#include "include/Context.h"
+#include "MgrMap.h"
+#include "PaxosService.h"
+#include "MonCommand.h"
+
+class MgrMonitor: public PaxosService
+{
+  MgrMap map;
+  MgrMap pending_map;
+  bool ever_had_active_mgr = false;
+
+  std::map<std::string, ceph::buffer::list> pending_metadata;
+  std::set<std::string> pending_metadata_rm;
+
+  std::map<std::string,Option> mgr_module_options;
+  std::list<std::string> misc_option_strings;
+
+  utime_t first_seen_inactive;
+
+  std::map<uint64_t, ceph::coarse_mono_clock::time_point> last_beacon;
+
+  /**
+   * If a standby is available, make it active, given that
+   * there is currently no active daemon.
+   *
+   * @return true if a standby was promoted
+   */
+  bool promote_standby();
+  void drop_active();
+
+  /**
+   * Remove this gid from the list of standbys.  By default,
+   * also remove metadata (i.e. forget the daemon entirely).
+   *
+   * Set `drop_meta` to false if you would like to keep
+   * the daemon's metadata, for example if you're dropping
+   * it as a standby before reinstating it as the active daemon.
+   */
+  void drop_standby(uint64_t gid, bool drop_meta=true);
+
+  Context *digest_event = nullptr;
+  void cancel_timer();
+
+  std::vector<health_check_map_t> prev_health_checks;
+
+  bool check_caps(MonOpRequestRef op, const uuid_d& fsid);
+
+  health_status_t should_warn_about_mgr_down();
+
+  // Command descriptions we've learned from the active mgr
+  std::vector<MonCommand> command_descs;
+  std::vector<MonCommand> pending_command_descs;
+
+public:
+  MgrMonitor(Monitor &mn, Paxos &p, const std::string& service_name)
+    : PaxosService(mn, p, service_name)
+  {}
+  ~MgrMonitor() override {}
+
+  void init() override;
+  void on_shutdown() override;
+
+  const MgrMap &get_map() const { return map; }
+
+  const std::map<std::string,Option>& get_mgr_module_options() {
+    return mgr_module_options;
+  }
+  const Option *find_module_option(const std::string& name);
+
+  bool in_use() const { return map.epoch > 0; }
+
+  version_t get_trim_to() const override;
+
+  void prime_mgr_client();
+
+  void create_initial() override;
+  void get_store_prefixes(std::set<std::string>& s) const override;
+  void update_from_paxos(bool *need_bootstrap) override;
+  void post_paxos_update() override;
+  void create_pending() override;
+  void encode_pending(MonitorDBStore::TransactionRef t) override;
+
+  bool preprocess_query(MonOpRequestRef op) override;
+  bool prepare_update(MonOpRequestRef op) override;
+
+  bool preprocess_command(MonOpRequestRef op);
+  bool prepare_command(MonOpRequestRef op);
+
+  void encode_full(MonitorDBStore::TransactionRef t) override { }
+
+  bool preprocess_beacon(MonOpRequestRef op);
+  bool prepare_beacon(MonOpRequestRef op);
+
+  void check_sub(Subscription *sub);
+  void check_subs();
+  void send_digests();
+
+  void on_active() override;
+  void on_restart() override;
+
+  void tick() override;
+
+  void print_summary(ceph::Formatter *f, std::ostream *ss) const;
+
+  const std::vector<MonCommand> &get_command_descs() const;
+
+  int load_metadata(const std::string& name, std::map<std::string, std::string>& m,
+		    std::ostream *err) const;
+  int dump_metadata(const std::string& name, ceph::Formatter *f, std::ostream *err);
+  void print_nodes(ceph::Formatter *f) const;
+  void count_metadata(const std::string& field, ceph::Formatter *f);
+  void count_metadata(const std::string& field, std::map<std::string,int> *out);
+  void get_versions(std::map<std::string, std::list<std::string>> &versions);
+
+  // When did the mon last call into our tick() method?  Used for detecting
+  // when the mon was not updating us for some period (e.g. during slow
+  // election) to reset last_beacon timeouts
+  ceph::coarse_mono_clock::time_point last_tick;
+};
+
+#endif
diff --git a/src/mon/MgrStatMonitor.cc b/src/mon/MgrStatMonitor.cc
new file mode 100644
index 000000000..9da4c50da
--- /dev/null
+++ b/src/mon/MgrStatMonitor.cc
@@ -0,0 +1,367 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "MgrStatMonitor.h"
+#include "mon/OSDMonitor.h"
+#include "mon/MgrMonitor.h"
+#include "mon/PGMap.h"
+#include "messages/MGetPoolStats.h"
+#include "messages/MGetPoolStatsReply.h"
+#include "messages/MMonMgrReport.h"
+#include "messages/MStatfs.h"
+#include "messages/MStatfsReply.h"
+#include "messages/MServiceMap.h"
+
+#include "include/ceph_assert.h"	// re-clobber assert
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon)
+
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::ErasureCodeInterfaceRef;
+using ceph::ErasureCodeProfile;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::make_message;
+using ceph::mono_clock;
+using ceph::mono_time;
+
+static ostream& _prefix(std::ostream *_dout, Monitor &mon) {
+  return *_dout << "mon." << mon.name << "@" << mon.rank
+		<< "(" << mon.get_state_name()
+		<< ").mgrstat ";
+}
+
+MgrStatMonitor::MgrStatMonitor(Monitor &mn, Paxos &p, const string& service_name)
+  : PaxosService(mn, p, service_name)
+{
+}
+
+MgrStatMonitor::~MgrStatMonitor() = default;
+
+void MgrStatMonitor::create_initial()
+{
+  dout(10) << __func__ << dendl;
+  version = 0;
+  service_map.epoch = 1;
+  service_map.modified = ceph_clock_now();
+  pending_service_map_bl.clear();
+  encode(service_map, pending_service_map_bl, CEPH_FEATURES_ALL);
+}
+
+void MgrStatMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  version = get_last_committed();
+  dout(10) << " " << version << dendl;
+  load_health();
+  bufferlist bl;
+  get_version(version, bl);
+  if (version) {
+    ceph_assert(bl.length());
+    try {
+      auto p = bl.cbegin();
+      decode(digest, p);
+      decode(service_map, p);
+      if (!p.end()) {
+	decode(progress_events, p);
+      }
+      dout(10) << __func__ << " v" << version
+	       << " service_map e" << service_map.epoch
+	       << " " << progress_events.size() << " progress events"
+	       << dendl;
+    }
+    catch (ceph::buffer::error& e) {
+      derr << "failed to decode mgrstat state; luminous dev version? "
+	   << e.what() << dendl;
+    }
+  }
+  check_subs();
+  update_logger();
+  mon.osdmon()->notify_new_pg_digest();
+}
+
+void MgrStatMonitor::update_logger()
+{
+  dout(20) << __func__ << dendl;
+
+  mon.cluster_logger->set(l_cluster_osd_bytes, digest.osd_sum.statfs.total);
+  mon.cluster_logger->set(l_cluster_osd_bytes_used,
+                           digest.osd_sum.statfs.get_used_raw());
+  mon.cluster_logger->set(l_cluster_osd_bytes_avail,
+                           digest.osd_sum.statfs.available);
+
+  mon.cluster_logger->set(l_cluster_num_pool, digest.pg_pool_sum.size());
+  uint64_t num_pg = 0;
+  for (auto i : digest.num_pg_by_pool) {
+    num_pg += i.second;
+  }
+  mon.cluster_logger->set(l_cluster_num_pg, num_pg);
+
+  unsigned active = 0, active_clean = 0, peering = 0;
+  for (auto p = digest.num_pg_by_state.begin();
+       p != digest.num_pg_by_state.end();
+       ++p) {
+    if (p->first & PG_STATE_ACTIVE) {
+      active += p->second;
+      if (p->first & PG_STATE_CLEAN)
+	active_clean += p->second;
+    }
+    if (p->first & PG_STATE_PEERING)
+      peering += p->second;
+  }
+  mon.cluster_logger->set(l_cluster_num_pg_active_clean, active_clean);
+  mon.cluster_logger->set(l_cluster_num_pg_active, active);
+  mon.cluster_logger->set(l_cluster_num_pg_peering, peering);
+
+  mon.cluster_logger->set(l_cluster_num_object, digest.pg_sum.stats.sum.num_objects);
+  mon.cluster_logger->set(l_cluster_num_object_degraded, digest.pg_sum.stats.sum.num_objects_degraded);
+  mon.cluster_logger->set(l_cluster_num_object_misplaced, digest.pg_sum.stats.sum.num_objects_misplaced);
+  mon.cluster_logger->set(l_cluster_num_object_unfound, digest.pg_sum.stats.sum.num_objects_unfound);
+  mon.cluster_logger->set(l_cluster_num_bytes, digest.pg_sum.stats.sum.num_bytes);
+
+}
+
+void MgrStatMonitor::create_pending()
+{
+  dout(10) << " " << version << dendl;
+  pending_digest = digest;
+  pending_health_checks = get_health_checks();
+  pending_service_map_bl.clear();
+  encode(service_map, pending_service_map_bl, mon.get_quorum_con_features());
+}
+
+void MgrStatMonitor::encode_pending(MonitorDBStore::TransactionRef t)
+{
+  ++version;
+  dout(10) << " " << version << dendl;
+  bufferlist bl;
+  encode(pending_digest, bl, mon.get_quorum_con_features());
+  ceph_assert(pending_service_map_bl.length());
+  bl.append(pending_service_map_bl);
+  encode(pending_progress_events, bl);
+  put_version(t, version, bl);
+  put_last_committed(t, version);
+
+  encode_health(pending_health_checks, t);
+}
+
+version_t MgrStatMonitor::get_trim_to() const
+{
+  // we don't actually need *any* old states, but keep a few.
+  if (version > 5) {
+    return version - 5;
+  }
+  return 0;
+}
+
+void MgrStatMonitor::on_active()
+{
+  update_logger();
+}
+
+void MgrStatMonitor::tick()
+{
+}
+
+bool MgrStatMonitor::preprocess_query(MonOpRequestRef op)
+{
+  auto m = op->get_req<PaxosServiceMessage>();
+  switch (m->get_type()) {
+  case CEPH_MSG_STATFS:
+    return preprocess_statfs(op);
+  case MSG_MON_MGR_REPORT:
+    return preprocess_report(op);
+  case MSG_GETPOOLSTATS:
+    return preprocess_getpoolstats(op);
+  default:
+    mon.no_reply(op);
+    derr << "Unhandled message type " << m->get_type() << dendl;
+    return true;
+  }
+}
+
+bool MgrStatMonitor::prepare_update(MonOpRequestRef op)
+{
+  auto m = op->get_req<PaxosServiceMessage>();
+  switch (m->get_type()) {
+  case MSG_MON_MGR_REPORT:
+    return prepare_report(op);
+  default:
+    mon.no_reply(op);
+    derr << "Unhandled message type " << m->get_type() << dendl;
+    return true;
+  }
+}
+
+bool MgrStatMonitor::preprocess_report(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonMgrReport>();
+  mon.no_reply(op);
+  if (m->gid &&
+      m->gid != mon.mgrmon()->get_map().get_active_gid()) {
+    dout(10) << "ignoring report from non-active mgr " << m->gid
+	     << dendl;
+    return true;
+  }
+  return false;
+}
+
+bool MgrStatMonitor::prepare_report(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonMgrReport>();
+  bufferlist bl = m->get_data();
+  auto p = bl.cbegin();
+  decode(pending_digest, p);
+  pending_health_checks.swap(m->health_checks);
+  if (m->service_map_bl.length()) {
+    pending_service_map_bl.swap(m->service_map_bl);
+  }
+  pending_progress_events.swap(m->progress_events);
+  dout(10) << __func__ << " " << pending_digest << ", "
+	   << pending_health_checks.checks.size() << " health checks, "
+	   << progress_events.size() << " progress events" << dendl;
+  dout(20) << "pending_digest:\n";
+  JSONFormatter jf(true);
+  jf.open_object_section("pending_digest");
+  pending_digest.dump(&jf);
+  jf.close_section();
+  jf.flush(*_dout);
+  *_dout << dendl;
+  dout(20) << "health checks:\n";
+  JSONFormatter jf(true);
+  jf.open_object_section("health_checks");
+  pending_health_checks.dump(&jf);
+  jf.close_section();
+  jf.flush(*_dout);
+  *_dout << dendl;
+  dout(20) << "progress events:\n";
+  JSONFormatter jf(true);
+  jf.open_object_section("progress_events");
+  for (auto& i : pending_progress_events) {
+    jf.dump_object(i.first.c_str(), i.second);
+  }
+  jf.close_section();
+  jf.flush(*_dout);
+  *_dout << dendl;
+  return true;
+}
+
+bool MgrStatMonitor::preprocess_getpoolstats(MonOpRequestRef op)
+{
+  op->mark_pgmon_event(__func__);
+  auto m = op->get_req<MGetPoolStats>();
+  auto session = op->get_session();
+  if (!session)
+    return true;
+  if (!session->is_capable("pg", MON_CAP_R)) {
+    dout(0) << "MGetPoolStats received from entity with insufficient caps "
+            << session->caps << dendl;
+    return true;
+  }
+  if (m->fsid != mon.monmap->fsid) {
+    dout(0) << __func__ << " on fsid "
+	    << m->fsid << " != " << mon.monmap->fsid << dendl;
+    return true;
+  }
+  epoch_t ver = get_last_committed();
+  auto reply = new MGetPoolStatsReply(m->fsid, m->get_tid(), ver);
+  reply->per_pool = digest.use_per_pool_stats();
+  for (const auto& pool_name : m->pools) {
+    const auto pool_id = mon.osdmon()->osdmap.lookup_pg_pool_name(pool_name);
+    if (pool_id == -ENOENT)
+      continue;
+    auto pool_stat = get_pool_stat(pool_id);
+    if (!pool_stat)
+      continue;
+    reply->pool_stats[pool_name] = *pool_stat;
+  }
+  mon.send_reply(op, reply);
+  return true;
+}
+
+bool MgrStatMonitor::preprocess_statfs(MonOpRequestRef op)
+{
+  op->mark_pgmon_event(__func__);
+  auto statfs = op->get_req<MStatfs>();
+  auto session = op->get_session();
+
+  if (!session)
+    return true;
+  if (!session->is_capable("pg", MON_CAP_R)) {
+    dout(0) << "MStatfs received from entity with insufficient privileges "
+            << session->caps << dendl;
+    return true;
+  }
+  if (statfs->fsid != mon.monmap->fsid) {
+    dout(0) << __func__ << " on fsid " << statfs->fsid
+            << " != " << mon.monmap->fsid << dendl;
+    return true;
+  }
+  const auto& pool = statfs->data_pool;
+  if (pool && !mon.osdmon()->osdmap.have_pg_pool(*pool)) {
+    // There's no error field for MStatfsReply so just ignore the request.
+    // This is known to happen when a client is still accessing a removed fs.
+    dout(1) << __func__ << " on removed pool " << *pool << dendl;
+    return true;
+  }
+  dout(10) << __func__ << " " << *statfs
+           << " from " << statfs->get_orig_source() << dendl;
+  epoch_t ver = get_last_committed();
+  auto reply = new MStatfsReply(statfs->fsid, statfs->get_tid(), ver);
+  reply->h.st = get_statfs(mon.osdmon()->osdmap, pool);
+  mon.send_reply(op, reply);
+  return true;
+}
+
+void MgrStatMonitor::check_sub(Subscription *sub)
+{
+  dout(10) << __func__
+	   << " next " << sub->next
+	   << " vs service_map.epoch " << service_map.epoch << dendl;
+  if (sub->next <= service_map.epoch) {
+    auto m = new MServiceMap(service_map);
+    sub->session->con->send_message(m);
+    if (sub->onetime) {
+      mon.with_session_map([sub](MonSessionMap& session_map) {
+	  session_map.remove_sub(sub);
+	});
+    } else {
+      sub->next = service_map.epoch + 1;
+    }
+  }
+}
+
+void MgrStatMonitor::check_subs()
+{
+  dout(10) << __func__ << dendl;
+  if (!service_map.epoch) {
+    return;
+  }
+  auto subs = mon.session_map.subs.find("servicemap");
+  if (subs == mon.session_map.subs.end()) {
+    return;
+  }
+  auto p = subs->second->begin();
+  while (!p.end()) {
+    auto sub = *p;
+    ++p;
+    check_sub(sub);
+  }
+}
diff --git a/src/mon/MgrStatMonitor.h b/src/mon/MgrStatMonitor.h
new file mode 100644
index 000000000..7c31f2c13
--- /dev/null
+++ b/src/mon/MgrStatMonitor.h
@@ -0,0 +1,107 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include "include/Context.h"
+#include "PaxosService.h"
+#include "mon/PGMap.h"
+#include "mgr/ServiceMap.h"
+
+class MgrStatMonitor : public PaxosService {
+  // live version
+  version_t version = 0;
+  PGMapDigest digest;
+  ServiceMap service_map;
+  std::map<std::string,ProgressEvent> progress_events;
+
+  // pending commit
+  PGMapDigest pending_digest;
+  health_check_map_t pending_health_checks;
+  std::map<std::string,ProgressEvent> pending_progress_events;
+  ceph::buffer::list pending_service_map_bl;
+
+public:
+  MgrStatMonitor(Monitor &mn, Paxos &p, const std::string& service_name);
+  ~MgrStatMonitor() override;
+
+  void init() override {}
+  void on_shutdown() override {}
+
+  void create_initial() override;
+  void update_from_paxos(bool *need_bootstrap) override;
+  void create_pending() override;
+  void encode_pending(MonitorDBStore::TransactionRef t) override;
+  version_t get_trim_to() const override;
+
+  bool definitely_converted_snapsets() const {
+    return digest.definitely_converted_snapsets();
+  }
+
+  bool preprocess_query(MonOpRequestRef op) override;
+  bool prepare_update(MonOpRequestRef op) override;
+
+  void encode_full(MonitorDBStore::TransactionRef t) override { }
+
+  bool preprocess_report(MonOpRequestRef op);
+  bool prepare_report(MonOpRequestRef op);
+
+  bool preprocess_getpoolstats(MonOpRequestRef op);
+  bool preprocess_statfs(MonOpRequestRef op);
+
+  void check_sub(Subscription *sub);
+  void check_subs();
+  void send_digests();
+
+  void on_active() override;
+  void tick() override;
+
+  uint64_t get_last_osd_stat_seq(int osd) {
+    return digest.get_last_osd_stat_seq(osd);
+  }
+
+  void update_logger();
+
+  const ServiceMap& get_service_map() const {
+    return service_map;
+  }
+
+  const std::map<std::string,ProgressEvent>& get_progress_events() {
+    return progress_events;
+  }
+
+  // pg stat access
+  const pool_stat_t* get_pool_stat(int64_t poolid) const {
+    auto i = digest.pg_pool_sum.find(poolid);
+    if (i != digest.pg_pool_sum.end()) {
+      return &i->second;
+    }
+    return nullptr;
+  }
+
+  const PGMapDigest& get_digest() {
+    return digest;
+  }
+
+  ceph_statfs get_statfs(OSDMap& osdmap,
+			 boost::optional<int64_t> data_pool) const {
+    return digest.get_statfs(osdmap, data_pool);
+  }
+
+  void print_summary(ceph::Formatter *f, std::ostream *out) const {
+    digest.print_summary(f, out);
+  }
+  void dump_info(ceph::Formatter *f) const {
+    digest.dump(f);
+    f->dump_object("servicemap", get_service_map());
+  }
+  void dump_cluster_stats(std::stringstream *ss,
+			  ceph::Formatter *f,
+			  bool verbose) const {
+    digest.dump_cluster_stats(ss, f, verbose);
+  }
+  void dump_pool_stats(const OSDMap& osdm, std::stringstream *ss, ceph::Formatter *f,
+		       bool verbose) const {
+    digest.dump_pool_stats_full(osdm, ss, f, verbose);
+  }
+};
diff --git a/src/mon/MonCap.cc b/src/mon/MonCap.cc
new file mode 100644
index 000000000..e1dc37239
--- /dev/null
+++ b/src/mon/MonCap.cc
@@ -0,0 +1,679 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/spirit/include/qi_uint.hpp>
+#include <boost/spirit/include/qi.hpp>
+#include <boost/fusion/include/std_pair.hpp>
+#include <boost/spirit/include/phoenix.hpp>
+#include <boost/fusion/adapted/struct/adapt_struct.hpp>
+#include <boost/fusion/include/adapt_struct.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+
+#include "MonCap.h"
+#include "include/stringify.h"
+#include "include/ipaddr.h"
+#include "common/debug.h"
+#include "common/Formatter.h"
+
+#include <algorithm>
+#include <regex>
+
+#include "include/ceph_assert.h"
+
+using std::list;
+using std::map;
+using std::ostream;
+using std::pair;
+using std::string;
+using std::vector;
+
+using ceph::bufferlist;
+using ceph::Formatter;
+
+static inline bool is_not_alnum_space(char c)
+{
+  return !(isalpha(c) || isdigit(c) || (c == '-') || (c == '_'));
+}
+
+static std::string maybe_quote_string(const std::string& str)
+{
+  if (find_if(str.begin(), str.end(), is_not_alnum_space) == str.end())
+    return str;
+  return string("\"") + str + string("\"");
+}
+
+#define dout_subsys ceph_subsys_mon
+
+ostream& operator<<(ostream& out, const mon_rwxa_t& p)
+{ 
+  if (p == MON_CAP_ANY)
+    return out << "*";
+
+  if (p & MON_CAP_R)
+    out << "r";
+  if (p & MON_CAP_W)
+    out << "w";
+  if (p & MON_CAP_X)
+    out << "x";
+  return out;
+}
+
+ostream& operator<<(ostream& out, const StringConstraint& c)
+{
+  switch (c.match_type) {
+  case StringConstraint::MATCH_TYPE_EQUAL:
+    return out << "value " << c.value;
+  case StringConstraint::MATCH_TYPE_PREFIX:
+    return out << "prefix " << c.value;
+  case StringConstraint::MATCH_TYPE_REGEX:
+    return out << "regex " << c.value;
+  default:
+    break;
+  }
+  return out;
+}
+
+ostream& operator<<(ostream& out, const MonCapGrant& m)
+{
+  out << "allow";
+  if (m.service.length()) {
+    out << " service " << maybe_quote_string(m.service);
+  }
+  if (m.command.length()) {
+    out << " command " << maybe_quote_string(m.command);
+    if (!m.command_args.empty()) {
+      out << " with";
+      for (auto p = m.command_args.begin();
+	   p != m.command_args.end();
+	   ++p) {
+        switch (p->second.match_type) {
+        case StringConstraint::MATCH_TYPE_EQUAL:
+	  out << " " << maybe_quote_string(p->first) << "="
+              << maybe_quote_string(p->second.value);
+          break;
+        case StringConstraint::MATCH_TYPE_PREFIX:
+	  out << " " << maybe_quote_string(p->first) << " prefix "
+              << maybe_quote_string(p->second.value);
+          break;
+        case StringConstraint::MATCH_TYPE_REGEX:
+	  out << " " << maybe_quote_string(p->first) << " regex "
+              << maybe_quote_string(p->second.value);
+          break;
+        default:
+          break;
+        }
+      }
+    }
+  }
+  if (m.profile.length()) {
+    out << " profile " << maybe_quote_string(m.profile);
+  }
+  if (m.allow != 0)
+    out << " " << m.allow;
+  if (m.network.size())
+    out << " network " << m.network;
+  return out;
+}
+
+
+// <magic>
+//  fusion lets us easily populate structs via the qi parser.
+
+typedef map<string,StringConstraint> kvmap;
+
+BOOST_FUSION_ADAPT_STRUCT(MonCapGrant,
+			  (std::string, service)
+			  (std::string, profile)
+			  (std::string, command)
+			  (kvmap, command_args)
+			  (mon_rwxa_t, allow)
+			  (std::string, network)
+                          (std::string, fs_name))
+
+BOOST_FUSION_ADAPT_STRUCT(StringConstraint,
+                          (StringConstraint::MatchType, match_type)
+			  (std::string, value))
+
+// </magic>
+
+void MonCapGrant::parse_network()
+{
+  network_valid = ::parse_network(network.c_str(), &network_parsed,
+				  &network_prefix);
+}
+
+void MonCapGrant::expand_profile(const EntityName& name) const
+{
+  // only generate this list once
+  if (!profile_grants.empty())
+    return;
+
+  if (profile == "read-only") {
+    // grants READ-ONLY caps monitor-wide
+    // 'auth' requires MON_CAP_X even for RO, which we do not grant here.
+    profile_grants.push_back(mon_rwxa_t(MON_CAP_R));
+    return;
+  }
+
+  if (profile == "read-write") {
+    // grants READ-WRITE caps monitor-wide
+    // 'auth' requires MON_CAP_X for all operations, which we do not grant.
+    profile_grants.push_back(mon_rwxa_t(MON_CAP_R | MON_CAP_W));
+    return;
+  }
+
+  if (profile == "mon") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_ALL));
+    profile_grants.push_back(MonCapGrant("log", MON_CAP_ALL));
+  }
+  if (profile == "osd") {
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_ALL));
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("pg", MON_CAP_R | MON_CAP_W));
+    profile_grants.push_back(MonCapGrant("log", MON_CAP_W));
+    StringConstraint constraint(StringConstraint::MATCH_TYPE_REGEX,
+                                string("osd_mclock_max_capacity_iops_(hdd|ssd)"));
+    profile_grants.push_back(MonCapGrant("config set", "name", constraint));
+  }
+  if (profile == "mds") {
+    profile_grants.push_back(MonCapGrant("mds", MON_CAP_ALL));
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));
+    // This command grant is checked explicitly in MRemoveSnaps handling
+    profile_grants.push_back(MonCapGrant("osd pool rmsnap"));
+    profile_grants.push_back(MonCapGrant("osd blocklist"));
+    profile_grants.push_back(MonCapGrant("osd blacklist")); // for compat
+    profile_grants.push_back(MonCapGrant("log", MON_CAP_W));
+  }
+  if (profile == "mgr") {
+    profile_grants.push_back(MonCapGrant("mgr", MON_CAP_ALL));
+    profile_grants.push_back(MonCapGrant("log", MON_CAP_R | MON_CAP_W));
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R | MON_CAP_W));
+    profile_grants.push_back(MonCapGrant("mds", MON_CAP_R | MON_CAP_W));
+    profile_grants.push_back(MonCapGrant("fs", MON_CAP_R | MON_CAP_W));
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R | MON_CAP_W));
+    profile_grants.push_back(MonCapGrant("auth", MON_CAP_R | MON_CAP_W | MON_CAP_X));
+    profile_grants.push_back(MonCapGrant("config-key", MON_CAP_R | MON_CAP_W));
+    profile_grants.push_back(MonCapGrant("config", MON_CAP_R | MON_CAP_W));
+    // cephadm orchestrator provisions new daemon keys and updates caps
+    profile_grants.push_back(MonCapGrant("auth get-or-create"));
+    profile_grants.push_back(MonCapGrant("auth caps"));
+    profile_grants.push_back(MonCapGrant("auth rm"));
+    // tell commands (this is a bit of a kludge)
+    profile_grants.push_back(MonCapGrant("smart"));
+  }
+  if (profile == "osd" || profile == "mds" || profile == "mon" ||
+      profile == "mgr") {
+    StringConstraint constraint(StringConstraint::MATCH_TYPE_PREFIX,
+                                string("daemon-private/") + stringify(name) +
+                                string("/"));
+    std::string prefix = string("daemon-private/") + stringify(name) + string("/");
+    profile_grants.push_back(MonCapGrant("config-key get", "key", constraint));
+    profile_grants.push_back(MonCapGrant("config-key put", "key", constraint));
+    profile_grants.push_back(MonCapGrant("config-key set", "key", constraint));
+    profile_grants.push_back(MonCapGrant("config-key exists", "key", constraint));
+    profile_grants.push_back(MonCapGrant("config-key delete", "key", constraint));
+  }
+  if (profile == "bootstrap-osd") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));  // read monmap
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));  // read osdmap
+    profile_grants.push_back(MonCapGrant("mon getmap"));
+    profile_grants.push_back(MonCapGrant("osd new"));
+    profile_grants.push_back(MonCapGrant("osd purge-new"));
+  }
+  if (profile == "bootstrap-mds") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));  // read monmap
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));  // read osdmap
+    profile_grants.push_back(MonCapGrant("mon getmap"));
+    profile_grants.push_back(MonCapGrant("auth get-or-create"));  // FIXME: this can expose other mds keys
+    profile_grants.back().command_args["entity"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_PREFIX, "mds.");
+    profile_grants.back().command_args["caps_mon"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_EQUAL, "allow profile mds");
+    profile_grants.back().command_args["caps_osd"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_EQUAL, "allow rwx");
+    profile_grants.back().command_args["caps_mds"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_EQUAL, "allow");
+  }
+  if (profile == "bootstrap-mgr") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));  // read monmap
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));  // read osdmap
+    profile_grants.push_back(MonCapGrant("mon getmap"));
+    profile_grants.push_back(MonCapGrant("auth get-or-create"));  // FIXME: this can expose other mgr keys
+    profile_grants.back().command_args["entity"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_PREFIX, "mgr.");
+    profile_grants.back().command_args["caps_mon"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_EQUAL, "allow profile mgr");
+  }
+  if (profile == "bootstrap-rgw") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));  // read monmap
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));  // read osdmap
+    profile_grants.push_back(MonCapGrant("mon getmap"));
+    profile_grants.push_back(MonCapGrant("auth get-or-create"));  // FIXME: this can expose other mds keys
+    profile_grants.back().command_args["entity"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_PREFIX, "client.rgw.");
+    profile_grants.back().command_args["caps_mon"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_EQUAL, "allow rw");
+    profile_grants.back().command_args["caps_osd"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_EQUAL, "allow rwx");
+  }
+  if (profile == "bootstrap-rbd" || profile == "bootstrap-rbd-mirror") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));  // read monmap
+    profile_grants.push_back(MonCapGrant("auth get-or-create"));  // FIXME: this can expose other rbd keys
+    profile_grants.back().command_args["entity"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_PREFIX, "client.");
+    profile_grants.back().command_args["caps_mon"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_EQUAL,
+      (profile == "bootstrap-rbd-mirror" ? "profile rbd-mirror" :
+                                           "profile rbd"));
+    profile_grants.back().command_args["caps_osd"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_REGEX,
+      "^([ ,]*profile(=|[ ]+)['\"]?rbd[^ ,'\"]*['\"]?([ ]+pool(=|[ ]+)['\"]?[^,'\"]+['\"]?)?)+$");
+  }
+  if (profile == "fs-client") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("mds", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("pg", MON_CAP_R));
+  }
+  if (profile == "simple-rados-client") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("pg", MON_CAP_R));
+  }
+  if (profile == "simple-rados-client-with-blocklist") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("pg", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("osd blocklist"));
+    profile_grants.back().command_args["blocklistop"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_EQUAL, "add");
+    profile_grants.back().command_args["addr"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_REGEX, "^[^/]+/[0-9]+$");
+
+  }
+  if (boost::starts_with(profile, "rbd")) {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("pg", MON_CAP_R));
+
+    // exclusive lock dead-client blocklisting (IP+nonce required)
+    profile_grants.push_back(MonCapGrant("osd blocklist"));
+    profile_grants.back().command_args["blocklistop"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_EQUAL, "add");
+    profile_grants.back().command_args["addr"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_REGEX, "^[^/]+/[0-9]+$");
+
+    // for compat,
+    profile_grants.push_back(MonCapGrant("osd blacklist"));
+    profile_grants.back().command_args["blacklistop"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_EQUAL, "add");
+    profile_grants.back().command_args["addr"] = StringConstraint(
+      StringConstraint::MATCH_TYPE_REGEX, "^[^/]+/[0-9]+$");
+
+  }
+  if (profile == "rbd-mirror") {
+    StringConstraint constraint(StringConstraint::MATCH_TYPE_PREFIX,
+                                "rbd/mirror/");
+    profile_grants.push_back(MonCapGrant("config-key get", "key", constraint));
+  } else if (profile == "rbd-mirror-peer") {
+    StringConstraint constraint(StringConstraint::MATCH_TYPE_REGEX,
+                                "rbd/mirror/[^/]+");
+    profile_grants.push_back(MonCapGrant("config-key get", "key", constraint));
+
+    constraint = StringConstraint(StringConstraint::MATCH_TYPE_PREFIX,
+                                  "rbd/mirror/peer/");
+    profile_grants.push_back(MonCapGrant("config-key set", "key", constraint));
+  }
+  else if (profile == "crash") {
+    // TODO: we could limit this to getting the monmap and mgrmap...
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+  }
+  if (profile == "cephfs-mirror") {
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("mds", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("pg", MON_CAP_R));
+    StringConstraint constraint(StringConstraint::MATCH_TYPE_PREFIX,
+                                "cephfs/mirror/peer/");
+    profile_grants.push_back(MonCapGrant("config-key get", "key", constraint));
+
+  }
+  if (profile == "role-definer") {
+    // grants ALL caps to the auth subsystem, read-only on the
+    // monitor subsystem and nothing else.
+    profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
+    profile_grants.push_back(MonCapGrant("auth", MON_CAP_ALL));
+  }
+}
+
+mon_rwxa_t MonCapGrant::get_allowed(CephContext *cct,
+				    EntityName name,
+				    const std::string& s, const std::string& c,
+				    const map<string,string>& c_args) const
+{
+  if (profile.length()) {
+    expand_profile(name);
+    mon_rwxa_t a;
+    for (auto p = profile_grants.begin();
+	 p != profile_grants.end(); ++p)
+      a = a | p->get_allowed(cct, name, s, c, c_args);
+    return a;
+  }
+  if (service.length()) {
+    if (service != s)
+      return 0;
+    return allow;
+  }
+  if (command.length()) {
+    if (command != c)
+      return 0;
+    for (map<string,StringConstraint>::const_iterator p = command_args.begin(); p != command_args.end(); ++p) {
+      map<string,string>::const_iterator q = c_args.find(p->first);
+      // argument must be present if a constraint exists
+      if (q == c_args.end())
+	return 0;
+      switch (p->second.match_type) {
+      case StringConstraint::MATCH_TYPE_EQUAL:
+	if (p->second.value != q->second)
+	  return 0;
+        break;
+      case StringConstraint::MATCH_TYPE_PREFIX:
+	if (q->second.find(p->second.value) != 0)
+	  return 0;
+        break;
+      case StringConstraint::MATCH_TYPE_REGEX:
+        try {
+	  std::regex pattern(
+            p->second.value, std::regex::extended);
+          if (!std::regex_match(q->second, pattern))
+	    return 0;
+        } catch(const std::regex_error&) {
+	  return 0;
+	}
+        break;
+      default:
+        break;
+      }
+    }
+    return MON_CAP_ALL;
+  }
+  // we don't allow config-key service to be accessed with blanket caps other
+  // than '*' (i.e., 'any'), and that should have been checked by the caller
+  // via 'is_allow_all()'.
+  if (s == "config-key") {
+    return 0;
+  }
+  return allow;
+}
+
+ostream& operator<<(ostream&out, const MonCap& m)
+{
+  for (vector<MonCapGrant>::const_iterator p = m.grants.begin(); p != m.grants.end(); ++p) {
+    if (p != m.grants.begin())
+      out << ", ";
+    out << *p;
+  }
+  return out;
+}
+
+bool MonCap::is_allow_all() const
+{
+  for (vector<MonCapGrant>::const_iterator p = grants.begin(); p != grants.end(); ++p)
+    if (p->is_allow_all())
+      return true;
+  return false;
+}
+
+void MonCap::set_allow_all()
+{
+  grants.clear();
+  grants.push_back(MonCapGrant(MON_CAP_ANY));
+  text = "allow *";
+}
+
+bool MonCap::is_capable(
+  CephContext *cct,
+  EntityName name,
+  const string& service,
+  const string& command, const map<string,string>& command_args,
+  bool op_may_read, bool op_may_write, bool op_may_exec,
+  const entity_addr_t& addr) const
+{
+  if (cct)
+    ldout(cct, 20) << "is_capable service=" << service << " command=" << command
+		   << (op_may_read ? " read":"")
+		   << (op_may_write ? " write":"")
+		   << (op_may_exec ? " exec":"")
+		   << " addr " << addr
+		   << " on cap " << *this
+		   << dendl;
+
+  mon_rwxa_t allow = 0;
+  for (vector<MonCapGrant>::const_iterator p = grants.begin();
+       p != grants.end(); ++p) {
+    if (cct)
+      ldout(cct, 20) << " allow so far " << allow << ", doing grant " << *p
+		     << dendl;
+
+    if (p->network.size() &&
+	(!p->network_valid ||
+	 !network_contains(p->network_parsed,
+			   p->network_prefix,
+			   addr))) {
+      continue;
+    }
+
+    if (p->is_allow_all()) {
+      if (cct)
+	ldout(cct, 20) << " allow all" << dendl;
+      return true;
+    }
+
+    // check enumerated caps
+    allow = allow | p->get_allowed(cct, name, service, command, command_args);
+    if ((!op_may_read || (allow & MON_CAP_R)) &&
+	(!op_may_write || (allow & MON_CAP_W)) &&
+	(!op_may_exec || (allow & MON_CAP_X))) {
+      if (cct)
+	ldout(cct, 20) << " match" << dendl;
+      return true;
+    }
+  }
+  return false;
+}
+
+void MonCap::encode(bufferlist& bl) const
+{
+  ENCODE_START(4, 4, bl);   // legacy MonCaps was 3, 3
+  encode(text, bl);
+  ENCODE_FINISH(bl);
+}
+
+void MonCap::decode(bufferlist::const_iterator& bl)
+{
+  std::string s;
+  DECODE_START(4, bl);
+  decode(s, bl);
+  DECODE_FINISH(bl);
+  parse(s, NULL);
+}
+
+void MonCap::dump(Formatter *f) const
+{
+  f->dump_string("text", text);
+}
+
+void MonCap::generate_test_instances(list<MonCap*>& ls)
+{
+  ls.push_back(new MonCap);
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow *");
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow rwx");
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow service foo x");
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow command bar x");
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow service foo r, allow command bar x");
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow command bar with k1=v1 x");
+  ls.push_back(new MonCap);
+  ls.back()->parse("allow command bar with k1=v1 k2=v2 x");
+}
+
+// grammar
+namespace qi = boost::spirit::qi;
+namespace ascii = boost::spirit::ascii;
+namespace phoenix = boost::phoenix;
+
+
+template <typename Iterator>
+struct MonCapParser : qi::grammar<Iterator, MonCap()>
+{
+  MonCapParser() : MonCapParser::base_type(moncap)
+  {
+    using qi::char_;
+    using qi::int_;
+    using qi::ulong_long;
+    using qi::lexeme;
+    using qi::alnum;
+    using qi::_val;
+    using qi::_1;
+    using qi::_2;
+    using qi::_3;
+    using qi::eps;
+    using qi::lit;
+
+    quoted_string %=
+      lexeme['"' >> +(char_ - '"') >> '"'] | 
+      lexeme['\'' >> +(char_ - '\'') >> '\''];
+    unquoted_word %= +char_("a-zA-Z0-9_./-");
+    str %= quoted_string | unquoted_word;
+    network_str %= +char_("/.:a-fA-F0-9][");
+    fs_name_str %= +char_("a-zA-Z0-9_.-");
+
+    spaces = +(lit(' ') | lit('\n') | lit('\t'));
+
+    // command := command[=]cmd [k1=v1 k2=v2 ...]
+    str_match = '=' >> qi::attr(StringConstraint::MATCH_TYPE_EQUAL) >> str;
+    str_prefix = spaces >> lit("prefix") >> spaces >>
+                 qi::attr(StringConstraint::MATCH_TYPE_PREFIX) >> str;
+    str_regex = spaces >> lit("regex") >> spaces >>
+                 qi::attr(StringConstraint::MATCH_TYPE_REGEX) >> str;
+    kv_pair = str >> (str_match | str_prefix | str_regex);
+    kv_map %= kv_pair >> *(spaces >> kv_pair);
+    command_match = -spaces >> lit("allow") >> spaces >> lit("command") >> (lit('=') | spaces)
+			    >> qi::attr(string()) >> qi::attr(string())
+			    >> str
+			    >> -(spaces >> lit("with") >> spaces >> kv_map)
+			    >> qi::attr(0)
+			    >> -(spaces >> lit("network") >> spaces >> network_str);
+
+    // service foo rwxa
+    service_match %= -spaces >> lit("allow") >> spaces >> lit("service") >> (lit('=') | spaces)
+			     >> str >> qi::attr(string()) >> qi::attr(string())
+			     >> qi::attr(map<string,StringConstraint>())
+                             >> spaces >> rwxa
+			     >> -(spaces >> lit("network") >> spaces >> network_str);
+
+    // profile foo
+    profile_match %= -spaces >> -(lit("allow") >> spaces)
+                             >> lit("profile") >> (lit('=') | spaces)
+			     >> qi::attr(string())
+			     >> str
+			     >> qi::attr(string())
+			     >> qi::attr(map<string,StringConstraint>())
+			     >> qi::attr(0)
+			     >> -(spaces >> lit("network") >> spaces >> network_str);
+
+    // rwxa
+    rwxa_match %= -spaces >> lit("allow") >> spaces
+			  >> qi::attr(string()) >> qi::attr(string()) >> qi::attr(string())
+			  >> qi::attr(map<string,StringConstraint>())
+			  >> rwxa
+			  >> -(spaces >> lit("network") >> spaces >> network_str)
+			  >> -(spaces >> lit("fsname") >> (lit('=') | spaces) >> fs_name_str);
+
+    // rwxa := * | [r][w][x]
+    rwxa =
+      (lit("*")[_val = MON_CAP_ANY]) |
+      (lit("all")[_val = MON_CAP_ANY]) |
+      ( eps[_val = 0] >>
+	( lit('r')[_val |= MON_CAP_R] ||
+	  lit('w')[_val |= MON_CAP_W] ||
+	  lit('x')[_val |= MON_CAP_X]
+	  )
+	);
+
+    // grant := allow ...
+    grant = -spaces >> (rwxa_match | profile_match | service_match | command_match) >> -spaces;
+
+    // moncap := grant [grant ...]
+    grants %= (grant % (*lit(' ') >> (lit(';') | lit(',')) >> *lit(' ')));
+    moncap = grants  [_val = phoenix::construct<MonCap>(_1)]; 
+
+  }
+  qi::rule<Iterator> spaces;
+  qi::rule<Iterator, unsigned()> rwxa;
+  qi::rule<Iterator, string()> quoted_string;
+  qi::rule<Iterator, string()> unquoted_word;
+  qi::rule<Iterator, string()> str, network_str;
+  qi::rule<Iterator, string()> fs_name_str;
+
+  qi::rule<Iterator, StringConstraint()> str_match, str_prefix, str_regex;
+  qi::rule<Iterator, pair<string, StringConstraint>()> kv_pair;
+  qi::rule<Iterator, map<string, StringConstraint>()> kv_map;
+
+  qi::rule<Iterator, MonCapGrant()> rwxa_match;
+  qi::rule<Iterator, MonCapGrant()> command_match;
+  qi::rule<Iterator, MonCapGrant()> service_match;
+  qi::rule<Iterator, MonCapGrant()> profile_match;
+  qi::rule<Iterator, MonCapGrant()> grant;
+  qi::rule<Iterator, std::vector<MonCapGrant>()> grants;
+  qi::rule<Iterator, MonCap()> moncap;
+};
+
+bool MonCap::parse(const string& str, ostream *err)
+{
+  auto iter = str.begin();
+  auto end = str.end();
+
+  MonCapParser<string::const_iterator> exp;
+  bool r = qi::parse(iter, end, exp, *this);
+  if (r && iter == end) {
+    text = str;
+    for (auto& g : grants) {
+      g.parse_network();
+    }
+    return true;
+  }
+
+  // Make sure no grants are kept after parsing failed!
+  grants.clear();
+
+  if (err) {
+    if (iter != end)
+      *err << "mon capability parse failed, stopped at '"
+	   << std::string(iter, end)
+	   << "' of '" << str << "'";
+    else
+      *err << "mon capability parse failed, stopped at end of '" << str << "'";
+  }
+
+  return false; 
+}
+
diff --git a/src/mon/MonCap.h b/src/mon/MonCap.h
new file mode 100644
index 000000000..ab4e35bc9
--- /dev/null
+++ b/src/mon/MonCap.h
@@ -0,0 +1,227 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_MONCAP_H
+#define CEPH_MONCAP_H
+
+#include <ostream>
+
+#include "include/common_fwd.h"
+#include "include/types.h"
+#include "common/entity_name.h"
+#include "mds/mdstypes.h"
+
+static const __u8 MON_CAP_R     = (1 << 1);      // read
+static const __u8 MON_CAP_W     = (1 << 2);      // write
+static const __u8 MON_CAP_X     = (1 << 3);      // execute
+static const __u8 MON_CAP_ALL   = MON_CAP_R | MON_CAP_W | MON_CAP_X;
+static const __u8 MON_CAP_ANY   = 0xff;          // *
+
+struct mon_rwxa_t {
+  __u8 val;
+
+  // cppcheck-suppress noExplicitConstructor
+  mon_rwxa_t(__u8 v = 0) : val(v) {}
+  mon_rwxa_t& operator=(__u8 v) {
+    val = v;
+    return *this;
+  }
+  operator __u8() const {
+    return val;
+  }
+};
+
+std::ostream& operator<<(std::ostream& out, const mon_rwxa_t& p);
+
+struct StringConstraint {
+  enum MatchType {
+    MATCH_TYPE_NONE,
+    MATCH_TYPE_EQUAL,
+    MATCH_TYPE_PREFIX,
+    MATCH_TYPE_REGEX
+  };
+
+  MatchType match_type = MATCH_TYPE_NONE;
+  std::string value;
+
+  StringConstraint() {}
+  StringConstraint(MatchType match_type, std::string value)
+    : match_type(match_type), value(value) {
+  }
+};
+
+std::ostream& operator<<(std::ostream& out, const StringConstraint& c);
+
+struct MonCapGrant {
+  /*
+   * A grant can come in one of five forms:
+   *
+   *  - a blanket allow ('allow rw', 'allow *')
+   *    - this will match against any service and the read/write/exec flags
+   *      in the mon code.  semantics of what X means are somewhat ad hoc.
+   *
+   *  - a service allow ('allow service mds rw')
+   *    - this will match against a specific service and the r/w/x flags.
+   *
+   *  - a profile ('allow profile osd')
+   *    - this will match against specific monitor-enforced semantics of what
+   *      this type of user should need to do.  examples include 'osd', 'mds',
+   *      'bootstrap-osd'.
+   *
+   *  - a command ('allow command foo', 'allow command bar with arg1=val1 arg2 prefix val2')
+   *      this includes the command name (the prefix string), and a set
+   *      of key/value pairs that constrain use of that command.  if no pairs
+   *      are specified, any arguments are allowed; if a pair is specified, that
+   *      argument must be present and equal or match a prefix.
+   *
+   *  - an fs name ('allow fsname foo')
+   *    - this will restrict access to MDSMaps in the FSMap to the provided
+   *      fs name.
+   */
+  std::string service;
+  std::string profile;
+  std::string command;
+  std::map<std::string, StringConstraint> command_args;
+  std::string fs_name;
+
+  // restrict by network
+  std::string network;
+
+  // these are filled in by parse_network(), called by MonCap::parse()
+  entity_addr_t network_parsed;
+  unsigned network_prefix = 0;
+  bool network_valid = true;
+
+  void parse_network();
+
+  mon_rwxa_t allow;
+
+  // explicit grants that a profile grant expands to; populated as
+  // needed by expand_profile() (via is_match()) and cached here.
+  mutable std::list<MonCapGrant> profile_grants;
+
+  void expand_profile(const EntityName& name) const;
+
+  MonCapGrant() : allow(0) {}
+  // cppcheck-suppress noExplicitConstructor
+  MonCapGrant(mon_rwxa_t a) : allow(a) {}
+  MonCapGrant(std::string s, mon_rwxa_t a) : service(std::move(s)), allow(a) {}
+  // cppcheck-suppress noExplicitConstructor
+  MonCapGrant(std::string c) : command(std::move(c)) {}
+  MonCapGrant(std::string c, std::string a, StringConstraint co) : command(std::move(c)) {
+    command_args[a] = co;
+  }
+  MonCapGrant(mon_rwxa_t a, std::string fsname) : fs_name(fsname), allow(a) {}
+
+  /**
+   * check if given request parameters match our constraints
+   *
+   * @param cct context
+   * @param name entity name
+   * @param service service (if any)
+   * @param command command (if any)
+   * @param command_args command args (if any)
+   * @return bits we allow
+   */
+  mon_rwxa_t get_allowed(CephContext *cct,
+			 EntityName name,
+			 const std::string& service,
+			 const std::string& command,
+			 const std::map<std::string, std::string>& command_args) const;
+
+  bool is_allow_all() const {
+    return
+      allow == MON_CAP_ANY &&
+      service.length() == 0 &&
+      profile.length() == 0 &&
+      command.length() == 0 &&
+      fs_name.empty();
+  }
+};
+
+std::ostream& operator<<(std::ostream& out, const MonCapGrant& g);
+
+struct MonCap {
+  std::string text;
+  std::vector<MonCapGrant> grants;
+
+  MonCap() {}
+  explicit MonCap(const std::vector<MonCapGrant> &g) : grants(g) {}
+
+  std::string get_str() const {
+    return text;
+  }
+
+  bool is_allow_all() const;
+  void set_allow_all();
+  bool parse(const std::string& str, std::ostream *err=NULL);
+
+  /**
+   * check if we are capable of something
+   *
+   * This method actually checks a description of a particular operation against
+   * what the capability has specified.
+   *
+   * @param service service name
+   * @param command command id
+   * @param command_args
+   * @param op_may_read whether the operation may need to read
+   * @param op_may_write whether the operation may need to write
+   * @param op_may_exec whether the operation may exec
+   * @return true if the operation is allowed, false otherwise
+   */
+  bool is_capable(CephContext *cct,
+		  EntityName name,
+		  const std::string& service,
+		  const std::string& command,
+		  const std::map<std::string, std::string>& command_args,
+		  bool op_may_read, bool op_may_write, bool op_may_exec,
+		  const entity_addr_t& addr) const;
+
+  void encode(ceph::buffer::list& bl) const;
+  void decode(ceph::buffer::list::const_iterator& bl);
+  void dump(ceph::Formatter *f) const;
+  static void generate_test_instances(std::list<MonCap*>& ls);
+
+  std::vector<string> allowed_fs_names() const {
+    std::vector<string> ret;
+    for (auto& g : grants) {
+      if (not g.fs_name.empty()) {
+	ret.push_back(g.fs_name);
+      } else {
+	return {};
+      }
+    }
+    return ret;
+  }
+
+  bool fs_name_capable(const EntityName& ename, string_view fs_name,
+		       __u8 mask) {
+    for (auto& g : grants) {
+      if (g.is_allow_all()) {
+	return true;
+      }
+
+      if ((g.fs_name.empty() || g.fs_name == fs_name) && (mask & g.allow)) {
+	  return true;
+      }
+
+      g.expand_profile(ename);
+      for (auto& pg : g.profile_grants) {
+	if ((pg.service == "fs" || pg.service == "mds") &&
+	    (pg.fs_name.empty() || pg.fs_name == fs_name) &&
+	    (pg.allow & mask)) {
+	  return true;
+	}
+      }
+    }
+
+    return false;
+  }
+
+};
+WRITE_CLASS_ENCODER(MonCap)
+
+std::ostream& operator<<(std::ostream& out, const MonCap& cap);
+
+#endif
diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
new file mode 100644
index 000000000..9c637bf8a
--- /dev/null
+++ b/src/mon/MonClient.cc
@@ -0,0 +1,2025 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include <algorithm>
+#include <iterator>
+#include <random>
+#include <boost/range/adaptor/map.hpp>
+#include <boost/range/adaptor/filtered.hpp>
+#include <boost/range/algorithm/copy.hpp>
+#include <boost/range/algorithm_ext/copy_n.hpp>
+#include "common/weighted_shuffle.h"
+
+#include "include/random.h"
+#include "include/scope_guard.h"
+#include "include/stringify.h"
+
+#include "messages/MMonGetMap.h"
+#include "messages/MMonGetVersion.h"
+#include "messages/MMonGetMap.h"
+#include "messages/MMonGetVersionReply.h"
+#include "messages/MMonMap.h"
+#include "messages/MConfig.h"
+#include "messages/MGetConfig.h"
+#include "messages/MAuth.h"
+#include "messages/MLogAck.h"
+#include "messages/MAuthReply.h"
+#include "messages/MMonCommand.h"
+#include "messages/MMonCommandAck.h"
+#include "messages/MCommand.h"
+#include "messages/MCommandReply.h"
+#include "messages/MPing.h"
+
+#include "messages/MMonSubscribe.h"
+#include "messages/MMonSubscribeAck.h"
+#include "common/errno.h"
+#include "common/hostname.h"
+#include "common/LogClient.h"
+
+#include "MonClient.h"
+#include "error_code.h"
+#include "MonMap.h"
+
+#include "auth/Auth.h"
+#include "auth/KeyRing.h"
+#include "auth/AuthClientHandler.h"
+#include "auth/AuthRegistry.h"
+#include "auth/RotatingKeyRing.h"
+
+#define dout_subsys ceph_subsys_monc
+#undef dout_prefix
+#define dout_prefix *_dout << "monclient" << (_hunting() ? "(hunting)":"") << ": "
+
+namespace bs = boost::system;
+using std::string;
+using namespace std::literals;
+
+MonClient::MonClient(CephContext *cct_, boost::asio::io_context& service) :
+  Dispatcher(cct_),
+  AuthServer(cct_),
+  messenger(NULL),
+  timer(cct_, monc_lock),
+  service(service),
+  initialized(false),
+  log_client(NULL),
+  more_log_pending(false),
+  want_monmap(true),
+  had_a_connection(false),
+  reopen_interval_multiplier(
+    cct_->_conf.get_val<double>("mon_client_hunt_interval_min_multiple")),
+  last_mon_command_tid(0),
+  version_req_id(0)
+{}
+
+MonClient::~MonClient()
+{
+}
+
+int MonClient::build_initial_monmap()
+{
+  ldout(cct, 10) << __func__ << dendl;
+  int r = monmap.build_initial(cct, false, std::cerr);
+  ldout(cct,10) << "monmap:\n";
+  monmap.print(*_dout);
+  *_dout << dendl;
+  return r;
+}
+
+int MonClient::get_monmap()
+{
+  ldout(cct, 10) << __func__ << dendl;
+  std::unique_lock l(monc_lock);
+
+  sub.want("monmap", 0, 0);
+  if (!_opened())
+    _reopen_session();
+  map_cond.wait(l, [this] { return !want_monmap; });
+  ldout(cct, 10) << __func__ << " done" << dendl;
+  return 0;
+}
+
+int MonClient::get_monmap_and_config()
+{
+  ldout(cct, 10) << __func__ << dendl;
+  ceph_assert(!messenger);
+
+  int tries = 10;
+
+  cct->init_crypto();
+  auto shutdown_crypto = make_scope_guard([this] {
+    cct->shutdown_crypto();
+  });
+
+  int r = build_initial_monmap();
+  if (r < 0) {
+    lderr(cct) << __func__ << " cannot identify monitors to contact" << dendl;
+    return r;
+  }
+
+  messenger = Messenger::create_client_messenger(
+    cct, "temp_mon_client");
+  ceph_assert(messenger);
+  messenger->add_dispatcher_head(this);
+  messenger->start();
+  auto shutdown_msgr = make_scope_guard([this] {
+    messenger->shutdown();
+    messenger->wait();
+    delete messenger;
+    messenger = nullptr;
+    if (!monmap.fsid.is_zero()) {
+      cct->_conf.set_val("fsid", stringify(monmap.fsid));
+    }
+  });
+
+  want_bootstrap_config = true;
+  auto shutdown_config = make_scope_guard([this] {
+    std::unique_lock l(monc_lock);
+    want_bootstrap_config = false;
+    bootstrap_config.reset();
+  });
+
+  ceph::ref_t<MConfig> config;
+  while (tries-- > 0) {
+    r = init();
+    if (r < 0) {
+      return r;
+    }
+    r = authenticate(std::chrono::duration<double>(cct->_conf.get_val<std::chrono::seconds>("client_mount_timeout")).count());
+    if (r == -ETIMEDOUT) {
+      shutdown();
+      continue;
+    }
+    if (r < 0) {
+      break;
+    }
+    {
+      std::unique_lock l(monc_lock);
+      if (monmap.get_epoch() &&
+	  !monmap.persistent_features.contains_all(
+	    ceph::features::mon::FEATURE_MIMIC)) {
+	ldout(cct,10) << __func__ << " pre-mimic monitor, no config to fetch"
+		      << dendl;
+	r = 0;
+	break;
+      }
+      while ((!bootstrap_config || monmap.get_epoch() == 0) && r == 0) {
+	ldout(cct,20) << __func__ << " waiting for monmap|config" << dendl;
+	auto status = map_cond.wait_for(l, ceph::make_timespan(
+	    cct->_conf->mon_client_hunt_interval));
+	if (status == std::cv_status::timeout) {
+	  r = -ETIMEDOUT;
+	}
+      }
+
+      if (bootstrap_config) {
+	ldout(cct,10) << __func__ << " success" << dendl;
+	config = std::move(bootstrap_config);
+	r = 0;
+	break;
+      }
+    }
+    lderr(cct) << __func__ << " failed to get config" << dendl;
+    shutdown();
+    continue;
+  }
+
+  if (config) {
+    // apply the bootstrap config to ensure its applied prior to completing
+    // the bootstrap
+    cct->_conf.set_mon_vals(cct, config->config, config_cb);
+  }
+
+  shutdown();
+  return r;
+}
+
+
+/**
+ * Ping the monitor with id @p mon_id and set the resulting reply in
+ * the provided @p result_reply, if this last parameter is not NULL.
+ *
+ * So that we don't rely on the MonClient's default messenger, set up
+ * during connect(), we create our own messenger to comunicate with the
+ * specified monitor.  This is advantageous in the following ways:
+ *
+ * - Isolate the ping procedure from the rest of the MonClient's operations,
+ *   allowing us to not acquire or manage the big monc_lock, thus not
+ *   having to block waiting for some other operation to finish before we
+ *   can proceed.
+ *   * for instance, we can ping mon.FOO even if we are currently hunting
+ *     or blocked waiting for auth to complete with mon.BAR.
+ *
+ * - Ping a monitor prior to establishing a connection (using connect())
+ *   and properly establish the MonClient's messenger.  This frees us
+ *   from dealing with the complex foo that happens in connect().
+ *
+ * We also don't rely on MonClient as a dispatcher for this messenger,
+ * unlike what happens with the MonClient's default messenger.  This allows
+ * us to sandbox the whole ping, having it much as a separate entity in
+ * the MonClient class, considerably simplifying the handling and dispatching
+ * of messages without needing to consider monc_lock.
+ *
+ * Current drawback is that we will establish a messenger for each ping
+ * we want to issue, instead of keeping a single messenger instance that
+ * would be used for all pings.
+ */
+int MonClient::ping_monitor(const string &mon_id, string *result_reply)
+{
+  ldout(cct, 10) << __func__ << dendl;
+
+  string new_mon_id;
+  if (monmap.contains("noname-"+mon_id)) {
+    new_mon_id = "noname-"+mon_id;
+  } else {
+    new_mon_id = mon_id;
+  }
+
+  if (new_mon_id.empty()) {
+    ldout(cct, 10) << __func__ << " specified mon id is empty!" << dendl;
+    return -EINVAL;
+  } else if (!monmap.contains(new_mon_id)) {
+    ldout(cct, 10) << __func__ << " no such monitor 'mon." << new_mon_id << "'"
+                   << dendl;
+    return -ENOENT;
+  }
+
+  // N.B. monc isn't initialized
+
+  auth_registry.refresh_config();
+
+  KeyRing keyring;
+  keyring.from_ceph_context(cct);
+  RotatingKeyRing rkeyring(cct, cct->get_module_type(), &keyring);
+
+  MonClientPinger *pinger = new MonClientPinger(cct,
+						&rkeyring,
+						result_reply);
+
+  Messenger *smsgr = Messenger::create_client_messenger(cct, "temp_ping_client");
+  smsgr->add_dispatcher_head(pinger);
+  smsgr->set_auth_client(pinger);
+  smsgr->start();
+
+  ConnectionRef con = smsgr->connect_to_mon(monmap.get_addrs(new_mon_id));
+  ldout(cct, 10) << __func__ << " ping mon." << new_mon_id
+                 << " " << con->get_peer_addr() << dendl;
+
+  pinger->mc.reset(new MonConnection(cct, con, 0, &auth_registry));
+  pinger->mc->start(monmap.get_epoch(), entity_name);
+  con->send_message(new MPing);
+
+  int ret = pinger->wait_for_reply(cct->_conf->mon_client_ping_timeout);
+  if (ret == 0) {
+    ldout(cct,10) << __func__ << " got ping reply" << dendl;
+  } else {
+    ret = -ret;
+  }
+
+  con->mark_down();
+  pinger->mc.reset();
+  smsgr->shutdown();
+  smsgr->wait();
+  delete smsgr;
+  delete pinger;
+  return ret;
+}
+
+bool MonClient::ms_dispatch(Message *m)
+{
+  // we only care about these message types
+  switch (m->get_type()) {
+  case CEPH_MSG_MON_MAP:
+  case CEPH_MSG_AUTH_REPLY:
+  case CEPH_MSG_MON_SUBSCRIBE_ACK:
+  case CEPH_MSG_MON_GET_VERSION_REPLY:
+  case MSG_MON_COMMAND_ACK:
+  case MSG_COMMAND_REPLY:
+  case MSG_LOGACK:
+  case MSG_CONFIG:
+    break;
+  case CEPH_MSG_PING:
+    m->put();
+    return true;
+  default:
+    return false;
+  }
+
+  std::lock_guard lock(monc_lock);
+
+  if (!m->get_connection()->is_anon() &&
+      m->get_source().type() == CEPH_ENTITY_TYPE_MON) {
+    if (_hunting()) {
+      auto p = _find_pending_con(m->get_connection());
+      if (p == pending_cons.end()) {
+	// ignore any messages outside hunting sessions
+	ldout(cct, 10) << "discarding stray monitor message " << *m << dendl;
+	m->put();
+	return true;
+      }
+    } else if (!active_con || active_con->get_con() != m->get_connection()) {
+      // ignore any messages outside our session(s)
+      ldout(cct, 10) << "discarding stray monitor message " << *m << dendl;
+      m->put();
+      return true;
+    }
+  }
+
+  switch (m->get_type()) {
+  case CEPH_MSG_MON_MAP:
+    handle_monmap(static_cast<MMonMap*>(m));
+    if (passthrough_monmap) {
+      return false;
+    } else {
+      m->put();
+    }
+    break;
+  case CEPH_MSG_AUTH_REPLY:
+    handle_auth(static_cast<MAuthReply*>(m));
+    break;
+  case CEPH_MSG_MON_SUBSCRIBE_ACK:
+    handle_subscribe_ack(static_cast<MMonSubscribeAck*>(m));
+    break;
+  case CEPH_MSG_MON_GET_VERSION_REPLY:
+    handle_get_version_reply(static_cast<MMonGetVersionReply*>(m));
+    break;
+  case MSG_MON_COMMAND_ACK:
+    handle_mon_command_ack(static_cast<MMonCommandAck*>(m));
+    break;
+  case MSG_COMMAND_REPLY:
+    if (m->get_connection()->is_anon() &&
+        m->get_source().type() == CEPH_ENTITY_TYPE_MON) {
+      // this connection is from 'tell'... ignore everything except our command
+      // reply.  (we'll get misc other message because we authenticated, but we
+      // don't need them.)
+      handle_command_reply(static_cast<MCommandReply*>(m));
+      return true;
+    }
+    // leave the message for another dispatch handler (e.g., Objecter)
+    return false;
+  case MSG_LOGACK:
+    if (log_client) {
+      log_client->handle_log_ack(static_cast<MLogAck*>(m));
+      m->put();
+      if (more_log_pending) {
+	send_log();
+      }
+    } else {
+      m->put();
+    }
+    break;
+  case MSG_CONFIG:
+    handle_config(static_cast<MConfig*>(m));
+    break;
+  }
+  return true;
+}
+
+void MonClient::send_log(bool flush)
+{
+  if (log_client) {
+    auto lm = log_client->get_mon_log_message(flush);
+    if (lm)
+      _send_mon_message(std::move(lm));
+    more_log_pending = log_client->are_pending();
+  }
+}
+
+void MonClient::flush_log()
+{
+  std::lock_guard l(monc_lock);
+  send_log();
+}
+
+/* Unlike all the other message-handling functions, we don't put away a reference
+* because we want to support MMonMap passthrough to other Dispatchers. */
+void MonClient::handle_monmap(MMonMap *m)
+{
+  ldout(cct, 10) << __func__ << " " << *m << dendl;
+  auto con_addrs = m->get_source_addrs();
+  string old_name = monmap.get_name(con_addrs);
+  const auto old_epoch = monmap.get_epoch();
+
+  auto p = m->monmapbl.cbegin();
+  decode(monmap, p);
+
+  ldout(cct, 10) << " got monmap " << monmap.epoch
+		 << " from mon." << old_name
+		 << " (according to old e" << monmap.get_epoch() << ")"
+ 		 << dendl;
+  ldout(cct, 10) << "dump:\n";
+  monmap.print(*_dout);
+  *_dout << dendl;
+
+  if (old_epoch != monmap.get_epoch()) {
+    tried.clear();
+  }
+  if (old_name.size() == 0) {
+    ldout(cct,10) << " can't identify which mon we were connected to" << dendl;
+    _reopen_session();
+  } else {
+    auto new_name = monmap.get_name(con_addrs);
+    if (new_name.empty()) {
+      ldout(cct, 10) << "mon." << old_name << " at " << con_addrs
+		     << " went away" << dendl;
+      // can't find the mon we were talking to (above)
+      _reopen_session();
+    } else if (messenger->should_use_msgr2() &&
+	       monmap.get_addrs(new_name).has_msgr2() &&
+	       !con_addrs.has_msgr2()) {
+      ldout(cct,1) << " mon." << new_name << " has (v2) addrs "
+		   << monmap.get_addrs(new_name) << " but i'm connected to "
+		   << con_addrs << ", reconnecting" << dendl;
+      _reopen_session();
+    }
+  }
+
+  cct->set_mon_addrs(monmap);
+
+  sub.got("monmap", monmap.get_epoch());
+  map_cond.notify_all();
+  want_monmap = false;
+
+  if (authenticate_err == 1) {
+    _finish_auth(0);
+  }
+}
+
+void MonClient::handle_config(MConfig *m)
+{
+  ldout(cct,10) << __func__ << " " << *m << dendl;
+
+  if (want_bootstrap_config) {
+    // get_monmap_and_config is waiting for config which it will apply
+    // synchronously
+    bootstrap_config = ceph::ref_t<MConfig>(m, false);
+    map_cond.notify_all();
+    return;
+  }
+
+  // Take the sledgehammer approach to ensuring we don't depend on
+  // anything in MonClient.
+  boost::asio::post(finish_strand,
+		    [m, cct = boost::intrusive_ptr<CephContext>(cct),
+		     config_notify_cb = config_notify_cb,
+		     config_cb = config_cb]() {
+		      cct->_conf.set_mon_vals(cct.get(), m->config, config_cb);
+		      if (config_notify_cb) {
+			config_notify_cb();
+		      }
+		      m->put();
+		    });
+}
+
+// ----------------------
+
+int MonClient::init()
+{
+  ldout(cct, 10) << __func__ << dendl;
+
+  entity_name = cct->_conf->name;
+
+  auth_registry.refresh_config();
+
+  std::lock_guard l(monc_lock);
+  keyring.reset(new KeyRing);
+  if (auth_registry.is_supported_method(messenger->get_mytype(),
+					CEPH_AUTH_CEPHX)) {
+    // this should succeed, because auth_registry just checked!
+    int r = keyring->from_ceph_context(cct);
+    if (r != 0) {
+      // but be somewhat graceful in case there was a race condition
+      lderr(cct) << "keyring not found" << dendl;
+      return r;
+    }
+  }
+  if (!auth_registry.any_supported_methods(messenger->get_mytype())) {
+    return -ENOENT;
+  }
+
+  rotating_secrets.reset(
+    new RotatingKeyRing(cct, cct->get_module_type(), keyring.get()));
+
+  initialized = true;
+
+  messenger->set_auth_client(this);
+  messenger->add_dispatcher_head(this);
+
+  timer.init();
+  schedule_tick();
+
+  return 0;
+}
+
+void MonClient::shutdown()
+{
+  ldout(cct, 10) << __func__ << dendl;
+  monc_lock.lock();
+  stopping = true;
+  while (!version_requests.empty()) {
+    ceph::async::post(std::move(version_requests.begin()->second),
+		      monc_errc::shutting_down, 0, 0);
+    ldout(cct, 20) << __func__ << " canceling and discarding version request "
+		   << version_requests.begin()->first << dendl;
+    version_requests.erase(version_requests.begin());
+  }
+  while (!mon_commands.empty()) {
+    auto tid = mon_commands.begin()->first;
+    _cancel_mon_command(tid);
+  }
+  ldout(cct, 20) << __func__ << " discarding " << waiting_for_session.size()
+		 << " pending message(s)" << dendl;
+  waiting_for_session.clear();
+
+  active_con.reset();
+  pending_cons.clear();
+
+  auth.reset();
+  global_id = 0;
+  authenticate_err = 0;
+  authenticated = false;
+
+  monc_lock.unlock();
+
+  if (initialized) {
+    initialized = false;
+  }
+  monc_lock.lock();
+  timer.shutdown();
+  stopping = false;
+  monc_lock.unlock();
+}
+
+int MonClient::authenticate(double timeout)
+{
+  std::unique_lock lock{monc_lock};
+
+  if (active_con) {
+    ldout(cct, 5) << "already authenticated" << dendl;
+    return 0;
+  }
+  sub.want("monmap", monmap.get_epoch() ? monmap.get_epoch() + 1 : 0, 0);
+  sub.want("config", 0, 0);
+  if (!_opened())
+    _reopen_session();
+
+  auto until = ceph::real_clock::now();
+  until += ceph::make_timespan(timeout);
+  if (timeout > 0.0)
+    ldout(cct, 10) << "authenticate will time out at " << until << dendl;
+  while (!active_con && authenticate_err >= 0) {
+    if (timeout > 0.0) {
+      auto r = auth_cond.wait_until(lock, until);
+      if (r == std::cv_status::timeout && !active_con) {
+	ldout(cct, 0) << "authenticate timed out after " << timeout << dendl;
+	authenticate_err = -ETIMEDOUT;
+      }
+    } else {
+      auth_cond.wait(lock);
+    }
+  }
+
+  if (active_con) {
+    ldout(cct, 5) << __func__ << " success, global_id "
+		  << active_con->get_global_id() << dendl;
+    // active_con should not have been set if there was an error
+    ceph_assert(authenticate_err >= 0);
+    authenticated = true;
+  }
+
+  if (authenticate_err < 0 && auth_registry.no_keyring_disabled_cephx()) {
+    lderr(cct) << __func__ << " NOTE: no keyring found; disabled cephx authentication" << dendl;
+  }
+
+  return authenticate_err;
+}
+
+void MonClient::handle_auth(MAuthReply *m)
+{
+  ceph_assert(ceph_mutex_is_locked(monc_lock));
+
+  if (m->get_connection()->is_anon()) {
+    // anon connection, used for mon tell commands
+    for (auto& p : mon_commands) {
+      if (p.second->target_con == m->get_connection()) {
+	auto& mc = p.second->target_session;
+	int ret = mc->handle_auth(m, entity_name,
+				  CEPH_ENTITY_TYPE_MON,
+				  rotating_secrets.get());
+	(void)ret; // we don't care
+	break;
+      }
+    }
+    m->put();
+    return;
+  }
+
+  if (!_hunting()) {
+    std::swap(active_con->get_auth(), auth);
+    int ret = active_con->authenticate(m);
+    m->put();
+    std::swap(auth, active_con->get_auth());
+    if (global_id != active_con->get_global_id()) {
+      lderr(cct) << __func__ << " peer assigned me a different global_id: "
+		 << active_con->get_global_id() << dendl;
+    }
+    if (ret != -EAGAIN) {
+      _finish_auth(ret);
+    }
+    return;
+  }
+
+  // hunting
+  auto found = _find_pending_con(m->get_connection());
+  ceph_assert(found != pending_cons.end());
+  int auth_err = found->second.handle_auth(m, entity_name, want_keys,
+					   rotating_secrets.get());
+  m->put();
+  if (auth_err == -EAGAIN) {
+    return;
+  }
+  if (auth_err) {
+    pending_cons.erase(found);
+    if (!pending_cons.empty()) {
+      // keep trying with pending connections
+      return;
+    }
+    // the last try just failed, give up.
+  } else {
+    auto& mc = found->second;
+    ceph_assert(mc.have_session());
+    active_con.reset(new MonConnection(std::move(mc)));
+    pending_cons.clear();
+  }
+
+  _finish_hunting(auth_err);
+  _finish_auth(auth_err);
+}
+
+void MonClient::_finish_auth(int auth_err)
+{
+  ldout(cct,10) << __func__ << " " << auth_err << dendl;
+  authenticate_err = auth_err;
+  // _resend_mon_commands() could _reopen_session() if the connected mon is not
+  // the one the MonCommand is targeting.
+  if (!auth_err && active_con) {
+    ceph_assert(auth);
+    _check_auth_tickets();
+  }
+  auth_cond.notify_all();
+}
+
+// ---------
+
+void MonClient::send_mon_message(MessageRef m)
+{
+  std::lock_guard l{monc_lock};
+  _send_mon_message(std::move(m));
+}
+
+void MonClient::_send_mon_message(MessageRef m)
+{
+  ceph_assert(ceph_mutex_is_locked(monc_lock));
+  if (active_con) {
+    auto cur_con = active_con->get_con();
+    ldout(cct, 10) << "_send_mon_message to mon."
+		   << monmap.get_name(cur_con->get_peer_addr())
+		   << " at " << cur_con->get_peer_addr() << dendl;
+    cur_con->send_message2(std::move(m));
+  } else {
+    waiting_for_session.push_back(std::move(m));
+  }
+}
+
+void MonClient::_reopen_session(int rank)
+{
+  ceph_assert(ceph_mutex_is_locked(monc_lock));
+  ldout(cct, 10) << __func__ << " rank " << rank << dendl;
+
+  active_con.reset();
+  pending_cons.clear();
+
+  authenticate_err = 1;  // == in progress
+
+  _start_hunting();
+
+  if (rank >= 0) {
+    _add_conn(rank);
+  } else {
+    _add_conns();
+  }
+
+  // throw out old queued messages
+  waiting_for_session.clear();
+
+  // throw out version check requests
+  while (!version_requests.empty()) {
+    ceph::async::post(std::move(version_requests.begin()->second),
+		      monc_errc::session_reset, 0, 0);
+    version_requests.erase(version_requests.begin());
+  }
+
+  for (auto& c : pending_cons) {
+    c.second.start(monmap.get_epoch(), entity_name);
+  }
+
+  if (sub.reload()) {
+    _renew_subs();
+  }
+}
+
+void MonClient::_add_conn(unsigned rank)
+{
+  auto peer = monmap.get_addrs(rank);
+  auto conn = messenger->connect_to_mon(peer);
+  MonConnection mc(cct, conn, global_id, &auth_registry);
+  if (auth) {
+    mc.get_auth().reset(auth->clone());
+  }
+  pending_cons.insert(std::make_pair(peer, std::move(mc)));
+  ldout(cct, 10) << "picked mon." << monmap.get_name(rank)
+                 << " con " << conn
+                 << " addr " << peer
+                 << dendl;
+}
+
+void MonClient::_add_conns()
+{
+  // collect the next batch of candidates who are listed right next to the ones
+  // already tried
+  auto get_next_batch = [this]() -> std::vector<unsigned> {
+    std::multimap<uint16_t, unsigned> ranks_by_priority;
+    boost::copy(
+      monmap.mon_info | boost::adaptors::filtered(
+        [this](auto& info) {
+          auto rank = monmap.get_rank(info.first);
+          return tried.count(rank) == 0;
+        }) | boost::adaptors::transformed(
+          [this](auto& info) {
+            auto rank = monmap.get_rank(info.first);
+            return std::make_pair(info.second.priority, rank);
+          }), std::inserter(ranks_by_priority, end(ranks_by_priority)));
+    if (ranks_by_priority.empty()) {
+      return {};
+    }
+    // only choose the monitors with lowest priority
+    auto cands = boost::make_iterator_range(
+      ranks_by_priority.equal_range(ranks_by_priority.begin()->first));
+    std::vector<unsigned> ranks;
+    boost::range::copy(cands | boost::adaptors::map_values,
+		       std::back_inserter(ranks));
+    return ranks;
+  };
+  auto ranks = get_next_batch();
+  if (ranks.empty()) {
+    tried.clear();  // start over
+    ranks = get_next_batch();
+  }
+  ceph_assert(!ranks.empty());
+  if (ranks.size() > 1) {
+    std::vector<uint16_t> weights;
+    for (auto i : ranks) {
+      auto rank_name = monmap.get_name(i);
+      weights.push_back(monmap.get_weight(rank_name));
+    }
+    random_device_t rd;
+    if (std::accumulate(begin(weights), end(weights), 0u) == 0) {
+      std::shuffle(begin(ranks), end(ranks), std::mt19937{rd()});
+    } else {
+      weighted_shuffle(begin(ranks), end(ranks), begin(weights), end(weights),
+		       std::mt19937{rd()});
+    }
+  }
+  ldout(cct, 10) << __func__ << " ranks=" << ranks << dendl;
+  unsigned n = cct->_conf->mon_client_hunt_parallel;
+  if (n == 0 || n > ranks.size()) {
+    n = ranks.size();
+  }
+  for (unsigned i = 0; i < n; i++) {
+    _add_conn(ranks[i]);
+    tried.insert(ranks[i]);
+  }
+}
+
+bool MonClient::ms_handle_reset(Connection *con)
+{
+  std::lock_guard lock(monc_lock);
+
+  if (con->get_peer_type() != CEPH_ENTITY_TYPE_MON)
+    return false;
+
+  if (con->is_anon()) {
+    auto p = mon_commands.begin();
+    while (p != mon_commands.end()) {
+      auto cmd = p->second;
+      ++p;
+      if (cmd->target_con == con) {
+	_send_command(cmd); // may retry or fail
+	break;
+      }
+    }
+    return true;
+  }
+
+  if (_hunting()) {
+    if (pending_cons.count(con->get_peer_addrs())) {
+      ldout(cct, 10) << __func__ << " hunted mon " << con->get_peer_addrs()
+		     << dendl;
+    } else {
+      ldout(cct, 10) << __func__ << " stray mon " << con->get_peer_addrs()
+		     << dendl;
+    }
+    return true;
+  } else {
+    if (active_con && con == active_con->get_con()) {
+      ldout(cct, 10) << __func__ << " current mon " << con->get_peer_addrs()
+		     << dendl;
+      _reopen_session();
+      return false;
+    } else {
+      ldout(cct, 10) << "ms_handle_reset stray mon " << con->get_peer_addrs()
+		     << dendl;
+      return true;
+    }
+  }
+}
+
+bool MonClient::_opened() const
+{
+  ceph_assert(ceph_mutex_is_locked(monc_lock));
+  return active_con || _hunting();
+}
+
+bool MonClient::_hunting() const
+{
+  return !pending_cons.empty();
+}
+
+void MonClient::_start_hunting()
+{
+  ceph_assert(!_hunting());
+  // adjust timeouts if necessary
+  if (!had_a_connection)
+    return;
+  reopen_interval_multiplier *= cct->_conf->mon_client_hunt_interval_backoff;
+  if (reopen_interval_multiplier >
+      cct->_conf->mon_client_hunt_interval_max_multiple) {
+    reopen_interval_multiplier =
+      cct->_conf->mon_client_hunt_interval_max_multiple;
+  }
+}
+
+void MonClient::_finish_hunting(int auth_err)
+{
+  ldout(cct,10) << __func__ << " " << auth_err << dendl;
+  ceph_assert(ceph_mutex_is_locked(monc_lock));
+  // the pending conns have been cleaned.
+  ceph_assert(!_hunting());
+  if (active_con) {
+    auto con = active_con->get_con();
+    ldout(cct, 1) << "found mon."
+		  << monmap.get_name(con->get_peer_addr())
+		  << dendl;
+  } else {
+    ldout(cct, 1) << "no mon sessions established" << dendl;
+  }
+
+  had_a_connection = true;
+  _un_backoff();
+
+  if (!auth_err) {
+    last_rotating_renew_sent = utime_t();
+    while (!waiting_for_session.empty()) {
+      _send_mon_message(std::move(waiting_for_session.front()));
+      waiting_for_session.pop_front();
+    }
+    _resend_mon_commands();
+    send_log(true);
+    if (active_con) {
+      auth = std::move(active_con->get_auth());
+      if (global_id && global_id != active_con->get_global_id()) {
+	lderr(cct) << __func__ << " global_id changed from " << global_id
+		   << " to " << active_con->get_global_id() << dendl;
+      }
+      global_id = active_con->get_global_id();
+    }
+  }
+}
+
+void MonClient::tick()
+{
+  ldout(cct, 10) << __func__ << dendl;
+
+  utime_t now = ceph_clock_now();
+
+  auto reschedule_tick = make_scope_guard([this] {
+      schedule_tick();
+    });
+
+  _check_auth_tickets();
+  _check_tell_commands();
+  
+  if (_hunting()) {
+    ldout(cct, 1) << "continuing hunt" << dendl;
+    return _reopen_session();
+  } else if (active_con) {
+    // just renew as needed
+    auto cur_con = active_con->get_con();
+    if (!cur_con->has_feature(CEPH_FEATURE_MON_STATEFUL_SUB)) {
+      const bool maybe_renew = sub.need_renew();
+      ldout(cct, 10) << "renew subs? -- " << (maybe_renew ? "yes" : "no")
+		     << dendl;
+      if (maybe_renew) {
+	_renew_subs();
+      }
+    }
+
+    if (now > last_keepalive + cct->_conf->mon_client_ping_interval) {
+      cur_con->send_keepalive();
+      last_keepalive = now;
+
+      if (cct->_conf->mon_client_ping_timeout > 0 &&
+	  cur_con->has_feature(CEPH_FEATURE_MSGR_KEEPALIVE2)) {
+	utime_t lk = cur_con->get_last_keepalive_ack();
+	utime_t interval = now - lk;
+	if (interval > cct->_conf->mon_client_ping_timeout) {
+	  ldout(cct, 1) << "no keepalive since " << lk << " (" << interval
+			<< " seconds), reconnecting" << dendl;
+	  return _reopen_session();
+	}
+      }
+
+      _un_backoff();
+    }
+
+    if (now > last_send_log + cct->_conf->mon_client_log_interval) {
+      send_log();
+      last_send_log = now;
+    }
+  }
+}
+
+void MonClient::_un_backoff()
+{
+  // un-backoff our reconnect interval
+  reopen_interval_multiplier = std::max(
+    cct->_conf.get_val<double>("mon_client_hunt_interval_min_multiple"),
+    reopen_interval_multiplier /
+    cct->_conf.get_val<double>("mon_client_hunt_interval_backoff"));
+  ldout(cct, 20) << __func__ << " reopen_interval_multipler now "
+		 << reopen_interval_multiplier << dendl;
+}
+
+void MonClient::schedule_tick()
+{
+  auto do_tick = make_lambda_context([this](int) { tick(); });
+  if (!is_connected()) {
+    // start another round of hunting
+    const auto hunt_interval = (cct->_conf->mon_client_hunt_interval *
+				reopen_interval_multiplier);
+    timer.add_event_after(hunt_interval, do_tick);
+  } else {
+    // keep in touch
+    timer.add_event_after(std::min(cct->_conf->mon_client_ping_interval,
+				   cct->_conf->mon_client_log_interval),
+			  do_tick);
+  }
+}
+
+// ---------
+
+void MonClient::_renew_subs()
+{
+  ceph_assert(ceph_mutex_is_locked(monc_lock));
+  if (!sub.have_new()) {
+    ldout(cct, 10) << __func__ << " - empty" << dendl;
+    return;
+  }
+
+  ldout(cct, 10) << __func__ << dendl;
+  if (!_opened())
+    _reopen_session();
+  else {
+    auto m = ceph::make_message<MMonSubscribe>();
+    m->what = sub.get_subs();
+    m->hostname = ceph_get_short_hostname();
+    _send_mon_message(std::move(m));
+    sub.renewed();
+  }
+}
+
+void MonClient::handle_subscribe_ack(MMonSubscribeAck *m)
+{
+  sub.acked(m->interval);
+  m->put();
+}
+
+int MonClient::_check_auth_tickets()
+{
+  ceph_assert(ceph_mutex_is_locked(monc_lock));
+  if (active_con && auth) {
+    if (auth->need_tickets()) {
+      ldout(cct, 10) << __func__ << " getting new tickets!" << dendl;
+      auto m = ceph::make_message<MAuth>();
+      m->protocol = auth->get_protocol();
+      auth->prepare_build_request();
+      auth->build_request(m->auth_payload);
+      _send_mon_message(m);
+    }
+
+    _check_auth_rotating();
+  }
+  return 0;
+}
+
+int MonClient::_check_auth_rotating()
+{
+  ceph_assert(ceph_mutex_is_locked(monc_lock));
+  if (!rotating_secrets ||
+      !auth_principal_needs_rotating_keys(entity_name)) {
+    ldout(cct, 20) << "_check_auth_rotating not needed by " << entity_name << dendl;
+    return 0;
+  }
+
+  if (!active_con || !auth) {
+    ldout(cct, 10) << "_check_auth_rotating waiting for auth session" << dendl;
+    return 0;
+  }
+
+  utime_t now = ceph_clock_now();
+  utime_t cutoff = now;
+  cutoff -= std::min(30.0, cct->_conf->auth_service_ticket_ttl / 4.0);
+  utime_t issued_at_lower_bound = now;
+  issued_at_lower_bound -= cct->_conf->auth_service_ticket_ttl;
+  if (!rotating_secrets->need_new_secrets(cutoff)) {
+    ldout(cct, 10) << "_check_auth_rotating have uptodate secrets (they expire after " << cutoff << ")" << dendl;
+    rotating_secrets->dump_rotating();
+    return 0;
+  }
+
+  ldout(cct, 10) << "_check_auth_rotating renewing rotating keys (they expired before " << cutoff << ")" << dendl;
+  if (!rotating_secrets->need_new_secrets() &&
+      rotating_secrets->need_new_secrets(issued_at_lower_bound)) {
+    // the key has expired before it has been issued?
+    lderr(cct) << __func__ << " possible clock skew, rotating keys expired way too early"
+               << " (before " << issued_at_lower_bound << ")" << dendl;
+  }
+  if ((now > last_rotating_renew_sent) &&
+      double(now - last_rotating_renew_sent) < 1) {
+    ldout(cct, 10) << __func__ << " called too often (last: "
+                   << last_rotating_renew_sent << "), skipping refresh" << dendl;
+    return 0;
+  }
+  auto m = ceph::make_message<MAuth>();
+  m->protocol = auth->get_protocol();
+  if (auth->build_rotating_request(m->auth_payload)) {
+    last_rotating_renew_sent = now;
+    _send_mon_message(std::move(m));
+  }
+  return 0;
+}
+
+int MonClient::wait_auth_rotating(double timeout)
+{
+  std::unique_lock l(monc_lock);
+
+  // Must be initialized
+  ceph_assert(auth != nullptr);
+
+  if (auth->get_protocol() == CEPH_AUTH_NONE)
+    return 0;
+  
+  if (!rotating_secrets)
+    return 0;
+
+  ldout(cct, 10) << __func__ << " waiting for " << timeout << dendl;
+  utime_t cutoff = ceph_clock_now();
+  cutoff -= std::min(30.0, cct->_conf->auth_service_ticket_ttl / 4.0);
+  if (auth_cond.wait_for(l, ceph::make_timespan(timeout), [this, cutoff] {
+    return (!auth_principal_needs_rotating_keys(entity_name) ||
+	    !rotating_secrets->need_new_secrets(cutoff));
+  })) {
+    ldout(cct, 10) << __func__ << " done" << dendl;
+    return 0;
+  } else {
+    ldout(cct, 0) << __func__ << " timed out after " << timeout << dendl;
+    return -ETIMEDOUT;
+  }
+}
+
+// ---------
+
+void MonClient::_send_command(MonCommand *r)
+{
+  if (r->is_tell()) {
+    ++r->send_attempts;
+    if (r->send_attempts > cct->_conf->mon_client_directed_command_retry) {
+      _finish_command(r, monc_errc::mon_unavailable, "mon unavailable", {});
+      return;
+    }
+    // tell-style command
+    if (monmap.min_mon_release >= ceph_release_t::octopus) {
+      if (r->target_con) {
+	r->target_con->mark_down();
+      }
+      if (r->target_rank >= 0) {
+	if (r->target_rank >= (int)monmap.size()) {
+	  ldout(cct, 10) << " target " << r->target_rank
+			 << " >= max mon " << monmap.size() << dendl;
+	  _finish_command(r, monc_errc::rank_dne, "mon rank dne"sv, {});
+	  return;
+	}
+	r->target_con = messenger->connect_to_mon(
+	  monmap.get_addrs(r->target_rank), true /* anon */);
+      } else {
+	if (!monmap.contains(r->target_name)) {
+	  ldout(cct, 10) << " target " << r->target_name
+			 << " not present in monmap" << dendl;
+	  _finish_command(r, monc_errc::mon_dne, "mon dne"sv, {});
+	  return;
+	}
+	r->target_con = messenger->connect_to_mon(
+	  monmap.get_addrs(r->target_name), true /* anon */);
+      }
+
+      r->target_session.reset(new MonConnection(cct, r->target_con, 0,
+						&auth_registry));
+      r->target_session->start(monmap.get_epoch(), entity_name);
+      r->last_send_attempt = ceph_clock_now();
+
+      MCommand *m = new MCommand(monmap.fsid);
+      m->set_tid(r->tid);
+      m->cmd = r->cmd;
+      m->set_data(r->inbl);
+      r->target_session->queue_command(m);
+      return;
+    }
+
+    // ugly legacy handling of pre-octopus mons
+    entity_addr_t peer;
+    if (active_con) {
+      peer = active_con->get_con()->get_peer_addr();
+    }
+
+    if (r->target_rank >= 0 &&
+	r->target_rank != monmap.get_rank(peer)) {
+      ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd
+		     << " wants rank " << r->target_rank
+		     << ", reopening session"
+		     << dendl;
+      if (r->target_rank >= (int)monmap.size()) {
+	ldout(cct, 10) << " target " << r->target_rank
+		       << " >= max mon " << monmap.size() << dendl;
+	_finish_command(r, monc_errc::rank_dne, "mon rank dne"sv, {});
+	return;
+      }
+      _reopen_session(r->target_rank);
+      return;
+    }
+    if (r->target_name.length() &&
+	r->target_name != monmap.get_name(peer)) {
+      ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd
+		     << " wants mon " << r->target_name
+		     << ", reopening session"
+		     << dendl;
+      if (!monmap.contains(r->target_name)) {
+	ldout(cct, 10) << " target " << r->target_name
+		       << " not present in monmap" << dendl;
+	_finish_command(r, monc_errc::mon_dne, "mon dne"sv, {});
+	return;
+      }
+      _reopen_session(monmap.get_rank(r->target_name));
+      return;
+    }
+    // fall-thru to send 'normal' CLI command
+  }
+
+  // normal CLI command
+  ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd << dendl;
+  auto m = ceph::make_message<MMonCommand>(monmap.fsid);
+  m->set_tid(r->tid);
+  m->cmd = r->cmd;
+  m->set_data(r->inbl);
+  _send_mon_message(std::move(m));
+  return;
+}
+
+void MonClient::_check_tell_commands()
+{
+  // resend any requests
+  auto now = ceph_clock_now();
+  auto p = mon_commands.begin();
+  while (p != mon_commands.end()) {
+    auto cmd = p->second;
+    ++p;
+    if (cmd->is_tell() &&
+	cmd->last_send_attempt != utime_t() &&
+	now - cmd->last_send_attempt > cct->_conf->mon_client_hunt_interval) {
+      ldout(cct,5) << __func__ << " timeout tell command " << cmd->tid << dendl;
+      _send_command(cmd); // might remove cmd from mon_commands
+    }
+  }
+}
+
+void MonClient::_resend_mon_commands()
+{
+  // resend any requests
+  auto p = mon_commands.begin();
+  while (p != mon_commands.end()) {
+    auto cmd = p->second;
+    ++p;
+    if (cmd->is_tell() && monmap.min_mon_release >= ceph_release_t::octopus) {
+      // starting with octopus, tell commands use their own connetion and need no
+      // special resend when we finish hunting.
+    } else {
+      _send_command(cmd); // might remove cmd from mon_commands
+    }
+  }
+}
+
+void MonClient::handle_mon_command_ack(MMonCommandAck *ack)
+{
+  MonCommand *r = NULL;
+  uint64_t tid = ack->get_tid();
+
+  if (tid == 0 && !mon_commands.empty()) {
+    r = mon_commands.begin()->second;
+    ldout(cct, 10) << __func__ << " has tid 0, assuming it is " << r->tid << dendl;
+  } else {
+    auto p = mon_commands.find(tid);
+    if (p == mon_commands.end()) {
+      ldout(cct, 10) << __func__ << " " << ack->get_tid() << " not found" << dendl;
+      ack->put();
+      return;
+    }
+    r = p->second;
+  }
+
+  ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd << dendl;
+  auto ec = ack->r < 0 ? bs::error_code(-ack->r, mon_category())
+    : bs::error_code();
+  _finish_command(r, ec, ack->rs,
+		  std::move(ack->get_data()));
+  ack->put();
+}
+
+void MonClient::handle_command_reply(MCommandReply *reply)
+{
+  MonCommand *r = NULL;
+  uint64_t tid = reply->get_tid();
+
+  if (tid == 0 && !mon_commands.empty()) {
+    r = mon_commands.begin()->second;
+    ldout(cct, 10) << __func__ << " has tid 0, assuming it is " << r->tid
+		   << dendl;
+  } else {
+    auto p = mon_commands.find(tid);
+    if (p == mon_commands.end()) {
+      ldout(cct, 10) << __func__ << " " << reply->get_tid() << " not found"
+		     << dendl;
+      reply->put();
+      return;
+    }
+    r = p->second;
+  }
+
+  ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd << dendl;
+  auto ec = reply->r < 0 ? bs::error_code(-reply->r, mon_category())
+    : bs::error_code();
+  _finish_command(r, ec, reply->rs, std::move(reply->get_data()));
+  reply->put();
+}
+
+int MonClient::_cancel_mon_command(uint64_t tid)
+{
+  ceph_assert(ceph_mutex_is_locked(monc_lock));
+
+  auto it = mon_commands.find(tid);
+  if (it == mon_commands.end()) {
+    ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
+    return -ENOENT;
+  }
+
+  ldout(cct, 10) << __func__ << " tid " << tid << dendl;
+
+  MonCommand *cmd = it->second;
+  _finish_command(cmd, monc_errc::timed_out, "timed out"sv, {});
+  return 0;
+}
+
+void MonClient::_finish_command(MonCommand *r, bs::error_code ret,
+				std::string_view rs, ceph::buffer::list&& bl)
+{
+  ldout(cct, 10) << __func__ << " " << r->tid << " = " << ret << " " << rs
+		 << dendl;
+  ceph::async::post(std::move(r->onfinish), ret, std::string(rs),
+		    std::move(bl));
+  if (r->target_con) {
+    r->target_con->mark_down();
+  }
+  mon_commands.erase(r->tid);
+  delete r;
+}
+
+// ---------
+
+void MonClient::handle_get_version_reply(MMonGetVersionReply* m)
+{
+  ceph_assert(ceph_mutex_is_locked(monc_lock));
+  auto iter = version_requests.find(m->handle);
+  if (iter == version_requests.end()) {
+    ldout(cct, 0) << __func__ << " version request with handle " << m->handle
+		  << " not found" << dendl;
+  } else {
+    auto req = std::move(iter->second);
+    ldout(cct, 10) << __func__ << " finishing " << iter->first << " version "
+		   << m->version << dendl;
+    version_requests.erase(iter);
+    ceph::async::post(std::move(req), bs::error_code(),
+		      m->version, m->oldest_version);
+  }
+  m->put();
+}
+
+int MonClient::get_auth_request(
+  Connection *con,
+  AuthConnectionMeta *auth_meta,
+  uint32_t *auth_method,
+  std::vector<uint32_t> *preferred_modes,
+  ceph::buffer::list *bl)
+{
+  std::lock_guard l(monc_lock);
+  ldout(cct,10) << __func__ << " con " << con << " auth_method " << *auth_method
+		<< dendl;
+
+  // connection to mon?
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) {
+    ceph_assert(!auth_meta->authorizer);
+    if (con->is_anon()) {
+      for (auto& i : mon_commands) {
+	if (i.second->target_con == con) {
+	  return i.second->target_session->get_auth_request(
+	    auth_method, preferred_modes, bl,
+	    entity_name, want_keys, rotating_secrets.get());
+	}
+      }
+    }
+    for (auto& i : pending_cons) {
+      if (i.second.is_con(con)) {
+	return i.second.get_auth_request(
+	  auth_method, preferred_modes, bl,
+	  entity_name, want_keys, rotating_secrets.get());
+      }
+    }
+    return -ENOENT;
+  }
+
+  // generate authorizer
+  if (!auth) {
+    lderr(cct) << __func__ << " but no auth handler is set up" << dendl;
+    return -EACCES;
+  }
+  auth_meta->authorizer.reset(auth->build_authorizer(con->get_peer_type()));
+  if (!auth_meta->authorizer) {
+    lderr(cct) << __func__ << " failed to build_authorizer for type "
+	       << ceph_entity_type_name(con->get_peer_type()) << dendl;
+    return -EACCES;
+  }
+  auth_meta->auth_method = auth_meta->authorizer->protocol;
+  auth_registry.get_supported_modes(con->get_peer_type(),
+				    auth_meta->auth_method,
+				    preferred_modes);
+  *bl = auth_meta->authorizer->bl;
+  return 0;
+}
+
+int MonClient::handle_auth_reply_more(
+  Connection *con,
+  AuthConnectionMeta *auth_meta,
+  const ceph::buffer::list& bl,
+  ceph::buffer::list *reply)
+{
+  std::lock_guard l(monc_lock);
+
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) {
+    if (con->is_anon()) {
+      for (auto& i : mon_commands) {
+	if (i.second->target_con == con) {
+	  return i.second->target_session->handle_auth_reply_more(
+	    auth_meta, bl, reply);
+	}
+      }
+    }
+    for (auto& i : pending_cons) {
+      if (i.second.is_con(con)) {
+	return i.second.handle_auth_reply_more(auth_meta, bl, reply);
+      }
+    }
+    return -ENOENT;
+  }
+
+  // authorizer challenges
+  if (!auth || !auth_meta->authorizer) {
+    lderr(cct) << __func__ << " no authorizer?" << dendl;
+    return -1;
+  }
+  auth_meta->authorizer->add_challenge(cct, bl);
+  *reply = auth_meta->authorizer->bl;
+  return 0;
+}
+
+int MonClient::handle_auth_done(
+  Connection *con,
+  AuthConnectionMeta *auth_meta,
+  uint64_t global_id,
+  uint32_t con_mode,
+  const ceph::buffer::list& bl,
+  CryptoKey *session_key,
+  std::string *connection_secret)
+{
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) {
+    std::lock_guard l(monc_lock);
+    if (con->is_anon()) {
+      for (auto& i : mon_commands) {
+	if (i.second->target_con == con) {
+	  return i.second->target_session->handle_auth_done(
+	    auth_meta, global_id, bl,
+	    session_key, connection_secret);
+	}
+      }
+    }
+    for (auto& i : pending_cons) {
+      if (i.second.is_con(con)) {
+	int r = i.second.handle_auth_done(
+	  auth_meta, global_id, bl,
+	  session_key, connection_secret);
+	if (r) {
+	  pending_cons.erase(i.first);
+	  if (!pending_cons.empty()) {
+	    return r;
+	  }
+	} else {
+	  active_con.reset(new MonConnection(std::move(i.second)));
+	  pending_cons.clear();
+	  ceph_assert(active_con->have_session());
+	}
+
+	_finish_hunting(r);
+	if (r || monmap.get_epoch() > 0) {
+	  _finish_auth(r);
+	}
+	return r;
+      }
+    }
+    return -ENOENT;
+  } else {
+    // verify authorizer reply
+    auto p = bl.begin();
+    if (!auth_meta->authorizer->verify_reply(p, &auth_meta->connection_secret)) {
+      ldout(cct, 0) << __func__ << " failed verifying authorizer reply"
+		    << dendl;
+      return -EACCES;
+    }
+    auth_meta->session_key = auth_meta->authorizer->session_key;
+    return 0;
+  }
+}
+
+int MonClient::handle_auth_bad_method(
+  Connection *con,
+  AuthConnectionMeta *auth_meta,
+  uint32_t old_auth_method,
+  int result,
+  const std::vector<uint32_t>& allowed_methods,
+  const std::vector<uint32_t>& allowed_modes)
+{
+  auth_meta->allowed_methods = allowed_methods;
+
+  std::lock_guard l(monc_lock);
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) {
+    if (con->is_anon()) {
+      for (auto& i : mon_commands) {
+	if (i.second->target_con == con) {
+	  int r = i.second->target_session->handle_auth_bad_method(
+	    old_auth_method,
+	    result,
+	    allowed_methods,
+	    allowed_modes);
+	  if (r < 0) {
+	    auto ec = bs::error_code(-r, mon_category());
+	    _finish_command(i.second, ec, "auth failed"sv, {});
+	  }
+	  return r;
+	}
+      }
+    }
+    for (auto& i : pending_cons) {
+      if (i.second.is_con(con)) {
+	int r = i.second.handle_auth_bad_method(old_auth_method,
+						result,
+						allowed_methods,
+						allowed_modes);
+	if (r == 0) {
+	  return r; // try another method on this con
+	}
+	pending_cons.erase(i.first);
+	if (!pending_cons.empty()) {
+	  return r;  // fail this con, maybe another con will succeed
+	}
+	// fail hunt
+	_finish_hunting(r);
+	_finish_auth(r);
+	return r;
+      }
+    }
+    return -ENOENT;
+  } else {
+    // huh...
+    ldout(cct,10) << __func__ << " hmm, they didn't like " << old_auth_method
+		  << " result " << cpp_strerror(result)
+		  << " and auth is " << (auth ? auth->get_protocol() : 0)
+		  << dendl;
+    return -EACCES;
+  }
+}
+
+int MonClient::handle_auth_request(
+  Connection *con,
+  AuthConnectionMeta *auth_meta,
+  bool more,
+  uint32_t auth_method,
+  const ceph::buffer::list& payload,
+  ceph::buffer::list *reply)
+{
+  if (payload.length() == 0) {
+    // for some channels prior to nautilus (osd heartbeat), we
+    // tolerate the lack of an authorizer.
+    if (!con->get_messenger()->require_authorizer) {
+      handle_authentication_dispatcher->ms_handle_authentication(con);
+      return 1;
+    }
+    return -EACCES;
+  }
+  auth_meta->auth_mode = payload[0];
+  if (auth_meta->auth_mode < AUTH_MODE_AUTHORIZER ||
+      auth_meta->auth_mode > AUTH_MODE_AUTHORIZER_MAX) {
+    return -EACCES;
+  }
+  AuthAuthorizeHandler *ah = get_auth_authorize_handler(con->get_peer_type(),
+							auth_method);
+  if (!ah) {
+    lderr(cct) << __func__ << " no AuthAuthorizeHandler found for auth method "
+	       << auth_method << dendl;
+    return -EOPNOTSUPP;
+  }
+
+  auto ac = &auth_meta->authorizer_challenge;
+  if (auth_meta->skip_authorizer_challenge) {
+    ldout(cct, 10) << __func__ << " skipping challenge on " << con << dendl;
+    ac = nullptr;
+  }
+
+  bool was_challenge = (bool)auth_meta->authorizer_challenge;
+  bool isvalid = ah->verify_authorizer(
+    cct,
+    *rotating_secrets,
+    payload,
+    auth_meta->get_connection_secret_length(),
+    reply,
+    &con->peer_name,
+    &con->peer_global_id,
+    &con->peer_caps_info,
+    &auth_meta->session_key,
+    &auth_meta->connection_secret,
+    ac);
+  if (isvalid) {
+    handle_authentication_dispatcher->ms_handle_authentication(con);
+    return 1;
+  }
+  if (!more && !was_challenge && auth_meta->authorizer_challenge) {
+    ldout(cct,10) << __func__ << " added challenge on " << con << dendl;
+    return 0;
+  }
+  ldout(cct,10) << __func__ << " bad authorizer on " << con << dendl;
+  // discard old challenge
+  auth_meta->authorizer_challenge.reset();
+  return -EACCES;
+}
+
+AuthAuthorizer* MonClient::build_authorizer(int service_id) const {
+  std::lock_guard l(monc_lock);
+  if (auth) {
+    return auth->build_authorizer(service_id);
+  } else {
+    ldout(cct, 0) << __func__ << " for " << ceph_entity_type_name(service_id)
+		  << ", but no auth is available now" << dendl;
+    return nullptr;
+  }
+}
+
+#define dout_subsys ceph_subsys_monc
+#undef dout_prefix
+#define dout_prefix *_dout << "monclient" << (have_session() ? ": " : "(hunting): ")
+
+MonConnection::MonConnection(
+  CephContext *cct, ConnectionRef con, uint64_t global_id,
+  AuthRegistry *ar)
+  : cct(cct), con(con), global_id(global_id), auth_registry(ar)
+{}
+
+MonConnection::~MonConnection()
+{
+  if (con) {
+    con->mark_down();
+    con.reset();
+  }
+}
+
+bool MonConnection::have_session() const
+{
+  return state == State::HAVE_SESSION;
+}
+
+void MonConnection::start(epoch_t epoch,
+			  const EntityName& entity_name)
+{
+  using ceph::encode;
+  auth_start = ceph_clock_now();
+
+  if (con->get_peer_addr().is_msgr2()) {
+    ldout(cct, 10) << __func__ << " opening mon connection" << dendl;
+    state = State::AUTHENTICATING;
+    con->send_message(new MMonGetMap());
+    return;
+  }
+
+  // restart authentication handshake
+  state = State::NEGOTIATING;
+
+  // send an initial keepalive to ensure our timestamp is valid by the
+  // time we are in an OPENED state (by sequencing this before
+  // authentication).
+  con->send_keepalive();
+
+  auto m = new MAuth;
+  m->protocol = CEPH_AUTH_UNKNOWN;
+  m->monmap_epoch = epoch;
+  __u8 struct_v = 1;
+  encode(struct_v, m->auth_payload);
+  std::vector<uint32_t> auth_supported;
+  auth_registry->get_supported_methods(con->get_peer_type(), &auth_supported);
+  encode(auth_supported, m->auth_payload);
+  encode(entity_name, m->auth_payload);
+  encode(global_id, m->auth_payload);
+  con->send_message(m);
+}
+
+int MonConnection::get_auth_request(
+  uint32_t *method,
+  std::vector<uint32_t> *preferred_modes,
+  ceph::buffer::list *bl,
+  const EntityName& entity_name,
+  uint32_t want_keys,
+  RotatingKeyRing* keyring)
+{
+  using ceph::encode;
+  // choose method
+  if (auth_method < 0) {
+    std::vector<uint32_t> as;
+    auth_registry->get_supported_methods(con->get_peer_type(), &as);
+    if (as.empty()) {
+      return -EACCES;
+    }
+    auth_method = as.front();
+  }
+  *method = auth_method;
+  auth_registry->get_supported_modes(con->get_peer_type(), auth_method,
+				     preferred_modes);
+  ldout(cct,10) << __func__ << " method " << *method
+		<< " preferred_modes " << *preferred_modes << dendl;
+  if (preferred_modes->empty()) {
+    return -EACCES;
+  }
+
+  int r = _init_auth(*method, entity_name, want_keys, keyring, true);
+  ceph_assert(r == 0);
+
+  // initial requset includes some boilerplate...
+  encode((char)AUTH_MODE_MON, *bl);
+  encode(entity_name, *bl);
+  encode(global_id, *bl);
+
+  // and (maybe) some method-specific initial payload
+  auth->build_initial_request(bl);
+
+  return 0;
+}
+
+int MonConnection::handle_auth_reply_more(
+  AuthConnectionMeta *auth_meta,
+  const ceph::buffer::list& bl,
+  ceph::buffer::list *reply)
+{
+  ldout(cct, 10) << __func__ << " payload " << bl.length() << dendl;
+  ldout(cct, 30) << __func__ << " got\n";
+  bl.hexdump(*_dout);
+  *_dout << dendl;
+
+  auto p = bl.cbegin();
+  ldout(cct, 10) << __func__ << " payload_len " << bl.length() << dendl;
+  int r = auth->handle_response(0, p, &auth_meta->session_key,
+				&auth_meta->connection_secret);
+  if (r == -EAGAIN) {
+    auth->prepare_build_request();
+    auth->build_request(*reply);
+    ldout(cct, 10) << __func__ << " responding with " << reply->length()
+		   << " bytes" << dendl;
+    r = 0;
+  } else if (r < 0) {
+    lderr(cct) << __func__ << " handle_response returned " << r << dendl;
+  } else {
+    ldout(cct, 10) << __func__ << " authenticated!" << dendl;
+    // FIXME
+    ceph_abort(cct, "write me");
+  }
+  return r;
+}
+
+int MonConnection::handle_auth_done(
+  AuthConnectionMeta *auth_meta,
+  uint64_t new_global_id,
+  const ceph::buffer::list& bl,
+  CryptoKey *session_key,
+  std::string *connection_secret)
+{
+  ldout(cct,10) << __func__ << " global_id " << new_global_id
+		<< " payload " << bl.length()
+		<< dendl;
+  global_id = new_global_id;
+  auth->set_global_id(global_id);
+  auto p = bl.begin();
+  int auth_err = auth->handle_response(0, p, &auth_meta->session_key,
+				       &auth_meta->connection_secret);
+  if (auth_err >= 0) {
+    state = State::HAVE_SESSION;
+  }
+  con->set_last_keepalive_ack(auth_start);
+
+  if (pending_tell_command) {
+    con->send_message2(std::move(pending_tell_command));
+  }
+  return auth_err;
+}
+
+int MonConnection::handle_auth_bad_method(
+  uint32_t old_auth_method,
+  int result,
+  const std::vector<uint32_t>& allowed_methods,
+  const std::vector<uint32_t>& allowed_modes)
+{
+  ldout(cct,10) << __func__ << " old_auth_method " << old_auth_method
+		<< " result " << cpp_strerror(result)
+		<< " allowed_methods " << allowed_methods << dendl;
+  std::vector<uint32_t> auth_supported;
+  auth_registry->get_supported_methods(con->get_peer_type(), &auth_supported);
+  auto p = std::find(auth_supported.begin(), auth_supported.end(),
+		     old_auth_method);
+  assert(p != auth_supported.end());
+  p = std::find_first_of(std::next(p), auth_supported.end(),
+			 allowed_methods.begin(), allowed_methods.end());
+  if (p == auth_supported.end()) {
+    lderr(cct) << __func__ << " server allowed_methods " << allowed_methods
+	       << " but i only support " << auth_supported << dendl;
+    return -EACCES;
+  }
+  auth_method = *p;
+  ldout(cct,10) << __func__ << " will try " << auth_method << " next" << dendl;
+  return 0;
+}
+
+int MonConnection::handle_auth(MAuthReply* m,
+			       const EntityName& entity_name,
+			       uint32_t want_keys,
+			       RotatingKeyRing* keyring)
+{
+  if (state == State::NEGOTIATING) {
+    int r = _negotiate(m, entity_name, want_keys, keyring);
+    if (r) {
+      return r;
+    }
+    state = State::AUTHENTICATING;
+  }
+  int r = authenticate(m);
+  if (!r) {
+    state = State::HAVE_SESSION;
+  }
+  return r;
+}
+
+int MonConnection::_negotiate(MAuthReply *m,
+			      const EntityName& entity_name,
+			      uint32_t want_keys,
+			      RotatingKeyRing* keyring)
+{
+  int r = _init_auth(m->protocol, entity_name, want_keys, keyring, false);
+  if (r == -ENOTSUP) {
+    if (m->result == -ENOTSUP) {
+      ldout(cct, 10) << "none of our auth protocols are supported by the server"
+		     << dendl;
+    }
+    return m->result;
+  }
+  return r;
+}
+
+int MonConnection::_init_auth(
+  uint32_t method,
+  const EntityName& entity_name,
+  uint32_t want_keys,
+  RotatingKeyRing* keyring,
+  bool msgr2)
+{
+  ldout(cct, 10) << __func__ << " method " << method << dendl;
+  if (auth && auth->get_protocol() == (int)method) {
+    ldout(cct, 10) << __func__ << " already have auth, reseting" << dendl;
+    auth->reset();
+    return 0;
+  }
+
+  ldout(cct, 10) << __func__ << " creating new auth" << dendl;
+  auth.reset(AuthClientHandler::create(cct, method, keyring));
+  if (!auth) {
+    ldout(cct, 10) << " no handler for protocol " << method << dendl;
+    return -ENOTSUP;
+  }
+
+  // do not request MGR key unless the mon has the SERVER_KRAKEN
+  // feature.  otherwise it will give us an auth error.  note that
+  // we have to use the FEATUREMASK because pre-jewel the kraken
+  // feature bit was used for something else.
+  if (!msgr2 &&
+      (want_keys & CEPH_ENTITY_TYPE_MGR) &&
+      !(con->has_features(CEPH_FEATUREMASK_SERVER_KRAKEN))) {
+    ldout(cct, 1) << __func__
+		  << " not requesting MGR keys from pre-kraken monitor"
+		  << dendl;
+    want_keys &= ~CEPH_ENTITY_TYPE_MGR;
+  }
+  auth->set_want_keys(want_keys);
+  auth->init(entity_name);
+  auth->set_global_id(global_id);
+  return 0;
+}
+
+int MonConnection::authenticate(MAuthReply *m)
+{
+  ceph_assert(auth);
+  if (!m->global_id) {
+    ldout(cct, 1) << "peer sent an invalid global_id" << dendl;
+  }
+  if (m->global_id != global_id) {
+    // it's a new session
+    auth->reset();
+    global_id = m->global_id;
+    auth->set_global_id(global_id);
+    ldout(cct, 10) << "my global_id is " << m->global_id << dendl;
+  }
+  auto p = m->result_bl.cbegin();
+  int ret = auth->handle_response(m->result, p, nullptr, nullptr);
+  if (ret == -EAGAIN) {
+    auto ma = new MAuth;
+    ma->protocol = auth->get_protocol();
+    auth->prepare_build_request();
+    auth->build_request(ma->auth_payload);
+    con->send_message(ma);
+  }
+  if (ret == 0 && pending_tell_command) {
+    con->send_message2(std::move(pending_tell_command));
+  }
+
+  return ret;
+}
+
+void MonClient::register_config_callback(md_config_t::config_callback fn) {
+  ceph_assert(!config_cb);
+  config_cb = fn;
+}
+
+md_config_t::config_callback MonClient::get_config_callback() {
+  return config_cb;
+}
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wnon-virtual-dtor"
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wnon-virtual-dtor"
+class monc_error_category : public ceph::converting_category {
+public:
+  monc_error_category(){}
+  const char* name() const noexcept override;
+  const char* message(int ev, char*, std::size_t) const noexcept override;
+  std::string message(int ev) const override;
+  bs::error_condition default_error_condition(int ev) const noexcept
+    override;
+  bool equivalent(int ev, const bs::error_condition& c) const
+    noexcept override;
+  using ceph::converting_category::equivalent;
+  int from_code(int ev) const noexcept override;
+};
+#pragma GCC diagnostic pop
+#pragma clang diagnostic pop
+
+const char* monc_error_category::name() const noexcept {
+  return "monc";
+}
+
+const char* monc_error_category::message(int ev, char*, std::size_t) const noexcept {
+  if (ev == 0)
+    return "No error";
+
+  switch (static_cast<monc_errc>(ev)) {
+  case monc_errc::shutting_down: // Command failed due to MonClient shutting down
+    return "Command failed due to MonClient shutting down";
+  case monc_errc::session_reset:
+    return "Monitor session was reset";
+  case monc_errc::rank_dne:
+    return "Requested monitor rank does not exist";
+  case monc_errc::mon_dne:
+    return "Requested monitor does not exist";
+  case monc_errc::timed_out:
+    return "Monitor operation timed out";
+  case monc_errc::mon_unavailable:
+    return "Monitor unavailable";
+  }
+
+  return "Unknown error";
+}
+
+std::string monc_error_category::message(int ev) const {
+  return message(ev, nullptr, 0);
+}
+
+bs::error_condition monc_error_category::default_error_condition(int ev) const noexcept {
+  switch (static_cast<monc_errc>(ev)) {
+  case monc_errc::shutting_down:
+    return bs::errc::operation_canceled;
+  case monc_errc::session_reset:
+    return bs::errc::resource_unavailable_try_again;
+  case monc_errc::rank_dne:
+    [[fallthrough]];
+  case monc_errc::mon_dne:
+    return ceph::errc::not_in_map;
+  case monc_errc::timed_out:
+    return bs::errc::timed_out;
+  case monc_errc::mon_unavailable:
+    return bs::errc::no_such_device;
+  }
+  return { ev, *this };
+}
+
+bool monc_error_category::equivalent(int ev, const bs::error_condition& c) const noexcept {
+  switch (static_cast<monc_errc>(ev)) {
+  case monc_errc::rank_dne:
+    [[fallthrough]];
+  case monc_errc::mon_dne:
+      return c == bs::errc::no_such_file_or_directory;
+  default:
+    return default_error_condition(ev) == c;
+  }
+}
+
+int monc_error_category::from_code(int ev) const noexcept {
+  if (ev == 0)
+    return 0;
+
+  switch (static_cast<monc_errc>(ev)) {
+  case monc_errc::shutting_down:
+    return -ECANCELED;
+  case monc_errc::session_reset:
+    return -EAGAIN;
+  case monc_errc::rank_dne:
+    [[fallthrough]];
+  case monc_errc::mon_dne:
+    return -ENOENT;
+  case monc_errc::timed_out:
+    return -ETIMEDOUT;
+  case monc_errc::mon_unavailable:
+    return -ENXIO;
+  }
+  return -EDOM;
+}
+
+const bs::error_category& monc_category() noexcept {
+  static const monc_error_category c;
+  return c;
+}
diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h
new file mode 100644
index 000000000..6a7daa814
--- /dev/null
+++ b/src/mon/MonClient.h
@@ -0,0 +1,774 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#ifndef CEPH_MONCLIENT_H
+#define CEPH_MONCLIENT_H
+
+#include <functional>
+#include <list>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "msg/Messenger.h"
+
+#include "MonMap.h"
+#include "MonSub.h"
+
+#include "common/async/completion.h"
+#include "common/Timer.h"
+#include "common/config.h"
+#include "messages/MMonGetVersion.h"
+
+#include "auth/AuthClient.h"
+#include "auth/AuthServer.h"
+
+class MMonMap;
+class MConfig;
+class MMonGetVersionReply;
+class MMonCommandAck;
+class LogClient;
+class AuthClientHandler;
+class AuthRegistry;
+class KeyRing;
+class RotatingKeyRing;
+
+class MonConnection {
+public:
+  MonConnection(CephContext *cct,
+		ConnectionRef conn,
+		uint64_t global_id,
+		AuthRegistry *auth_registry);
+  ~MonConnection();
+  MonConnection(MonConnection&& rhs) = default;
+  MonConnection& operator=(MonConnection&&) = default;
+  MonConnection(const MonConnection& rhs) = delete;
+  MonConnection& operator=(const MonConnection&) = delete;
+  int handle_auth(MAuthReply *m,
+		  const EntityName& entity_name,
+		  uint32_t want_keys,
+		  RotatingKeyRing* keyring);
+  int authenticate(MAuthReply *m);
+  void start(epoch_t epoch,
+             const EntityName& entity_name);
+  bool have_session() const;
+  uint64_t get_global_id() const {
+    return global_id;
+  }
+  ConnectionRef get_con() {
+    return con;
+  }
+  std::unique_ptr<AuthClientHandler>& get_auth() {
+    return auth;
+  }
+
+  int get_auth_request(
+    uint32_t *method,
+    std::vector<uint32_t> *preferred_modes,
+    ceph::buffer::list *out,
+    const EntityName& entity_name,
+    uint32_t want_keys,
+    RotatingKeyRing* keyring);
+  int handle_auth_reply_more(
+    AuthConnectionMeta *auth_meta,
+    const ceph::buffer::list& bl,
+    ceph::buffer::list *reply);
+  int handle_auth_done(
+    AuthConnectionMeta *auth_meta,
+    uint64_t global_id,
+    const ceph::buffer::list& bl,
+    CryptoKey *session_key,
+    std::string *connection_secret);
+  int handle_auth_bad_method(
+    uint32_t old_auth_method,
+    int result,
+    const std::vector<uint32_t>& allowed_methods,
+    const std::vector<uint32_t>& allowed_modes);
+
+  bool is_con(Connection *c) const {
+    return con.get() == c;
+  }
+  void queue_command(Message *m) {
+    pending_tell_command = m;
+  }
+
+private:
+  int _negotiate(MAuthReply *m,
+		 const EntityName& entity_name,
+		 uint32_t want_keys,
+		 RotatingKeyRing* keyring);
+  int _init_auth(uint32_t method,
+		 const EntityName& entity_name,
+		 uint32_t want_keys,
+		 RotatingKeyRing* keyring,
+		 bool msgr2);
+
+private:
+  CephContext *cct;
+  enum class State {
+    NONE,
+    NEGOTIATING,       // v1 only
+    AUTHENTICATING,    // v1 and v2
+    HAVE_SESSION,
+  };
+  State state = State::NONE;
+  ConnectionRef con;
+  int auth_method = -1;
+  utime_t auth_start;
+
+  std::unique_ptr<AuthClientHandler> auth;
+  uint64_t global_id;
+
+  MessageRef pending_tell_command;
+
+  AuthRegistry *auth_registry;
+};
+
+
+struct MonClientPinger : public Dispatcher,
+			 public AuthClient {
+  ceph::mutex lock = ceph::make_mutex("MonClientPinger::lock");
+  ceph::condition_variable ping_recvd_cond;
+  std::string *result;
+  bool done;
+  RotatingKeyRing *keyring;
+  std::unique_ptr<MonConnection> mc;
+
+  MonClientPinger(CephContext *cct_,
+		  RotatingKeyRing *keyring,
+		  std::string *res_) :
+    Dispatcher(cct_),
+    result(res_),
+    done(false),
+    keyring(keyring)
+  { }
+
+  int wait_for_reply(double timeout = 0.0) {
+    std::unique_lock locker{lock};
+    if (timeout <= 0) {
+      timeout = std::chrono::duration<double>(cct->_conf.get_val<std::chrono::seconds>("client_mount_timeout")).count();
+    }
+    done = false;
+    if (ping_recvd_cond.wait_for(locker,
+				 ceph::make_timespan(timeout),
+				 [this] { return done; })) {
+      return 0;
+    } else {
+      return ETIMEDOUT;
+    }
+  }
+
+  bool ms_dispatch(Message *m) override {
+    using ceph::decode;
+    std::lock_guard l(lock);
+    if (m->get_type() != CEPH_MSG_PING)
+      return false;
+
+    ceph::buffer::list &payload = m->get_payload();
+    if (result && payload.length() > 0) {
+      auto p = std::cbegin(payload);
+      decode(*result, p);
+    }
+    done = true;
+    ping_recvd_cond.notify_all();
+    m->put();
+    return true;
+  }
+  bool ms_handle_reset(Connection *con) override {
+    std::lock_guard l(lock);
+    done = true;
+    ping_recvd_cond.notify_all();
+    return true;
+  }
+  void ms_handle_remote_reset(Connection *con) override {}
+  bool ms_handle_refused(Connection *con) override {
+    return false;
+  }
+
+  // AuthClient
+  int get_auth_request(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    uint32_t *auth_method,
+    std::vector<uint32_t> *preferred_modes,
+    ceph::buffer::list *bl) override {
+    return mc->get_auth_request(auth_method, preferred_modes, bl,
+				cct->_conf->name, 0, keyring);
+  }
+  int handle_auth_reply_more(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    const ceph::buffer::list& bl,
+    ceph::buffer::list *reply) override {
+    return mc->handle_auth_reply_more(auth_meta, bl, reply);
+  }
+  int handle_auth_done(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    uint64_t global_id,
+    uint32_t con_mode,
+    const ceph::buffer::list& bl,
+    CryptoKey *session_key,
+    std::string *connection_secret) override {
+    return mc->handle_auth_done(auth_meta, global_id, bl,
+				session_key, connection_secret);
+  }
+  int handle_auth_bad_method(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    uint32_t old_auth_method,
+    int result,
+    const std::vector<uint32_t>& allowed_methods,
+    const std::vector<uint32_t>& allowed_modes) override {
+    return mc->handle_auth_bad_method(old_auth_method, result,
+				      allowed_methods, allowed_modes);
+  }
+};
+
+const boost::system::error_category& monc_category() noexcept;
+
+enum class monc_errc {
+  shutting_down = 1, // Command failed due to MonClient shutting down
+  session_reset, // Monitor session was reset
+  rank_dne, // Requested monitor rank does not exist
+  mon_dne, // Requested monitor does not exist
+  timed_out, // Monitor operation timed out
+  mon_unavailable // Monitor unavailable
+};
+
+namespace boost::system {
+template<>
+struct is_error_code_enum<::monc_errc> {
+  static const bool value = true;
+};
+}
+
+//  implicit conversion:
+inline boost::system::error_code make_error_code(monc_errc e) noexcept {
+  return { static_cast<int>(e), monc_category() };
+}
+
+// explicit conversion:
+inline boost::system::error_condition make_error_condition(monc_errc e) noexcept {
+  return { static_cast<int>(e), monc_category() };
+}
+
+const boost::system::error_category& monc_category() noexcept;
+
+class MonClient : public Dispatcher,
+		  public AuthClient,
+		  public AuthServer /* for mgr, osd, mds */ {
+  static constexpr auto dout_subsys = ceph_subsys_monc;
+public:
+  // Error, Newest, Oldest
+  using VersionSig = void(boost::system::error_code, version_t, version_t);
+  using VersionCompletion = ceph::async::Completion<VersionSig>;
+
+  using CommandSig = void(boost::system::error_code, std::string,
+			  ceph::buffer::list);
+  using CommandCompletion = ceph::async::Completion<CommandSig>;
+
+  MonMap monmap;
+  std::map<std::string,std::string> config_mgr;
+
+private:
+  Messenger *messenger;
+
+  std::unique_ptr<MonConnection> active_con;
+  std::map<entity_addrvec_t, MonConnection> pending_cons;
+  std::set<unsigned> tried;
+
+  EntityName entity_name;
+
+  mutable ceph::mutex monc_lock = ceph::make_mutex("MonClient::monc_lock");
+  SafeTimer timer;
+  boost::asio::io_context& service;
+  boost::asio::io_context::strand finish_strand{service};
+
+  bool initialized;
+  bool stopping = false;
+
+  LogClient *log_client;
+  bool more_log_pending;
+
+  void send_log(bool flush = false);
+
+  bool ms_dispatch(Message *m) override;
+  bool ms_handle_reset(Connection *con) override;
+  void ms_handle_remote_reset(Connection *con) override {}
+  bool ms_handle_refused(Connection *con) override { return false; }
+
+  void handle_monmap(MMonMap *m);
+  void handle_config(MConfig *m);
+
+  void handle_auth(MAuthReply *m);
+
+  // monitor session
+  utime_t last_keepalive;
+  utime_t last_send_log;
+
+  void tick();
+  void schedule_tick();
+
+  // monclient
+  bool want_monmap;
+  ceph::condition_variable map_cond;
+  bool passthrough_monmap = false;
+
+  bool want_bootstrap_config = false;
+  ceph::ref_t<MConfig> bootstrap_config;
+
+  // authenticate
+  std::unique_ptr<AuthClientHandler> auth;
+  uint32_t want_keys = 0;
+  uint64_t global_id = 0;
+  ceph::condition_variable auth_cond;
+  int authenticate_err = 0;
+  bool authenticated = false;
+
+  std::list<MessageRef> waiting_for_session;
+  utime_t last_rotating_renew_sent;
+  bool had_a_connection;
+  double reopen_interval_multiplier;
+
+  Dispatcher *handle_authentication_dispatcher = nullptr;
+  bool _opened() const;
+  bool _hunting() const;
+  void _start_hunting();
+  void _finish_hunting(int auth_err);
+  void _finish_auth(int auth_err);
+  void _reopen_session(int rank = -1);
+  void _add_conn(unsigned rank);
+  void _add_conns();
+  void _un_backoff();
+  void _send_mon_message(MessageRef m);
+
+  std::map<entity_addrvec_t, MonConnection>::iterator _find_pending_con(
+    const ConnectionRef& con) {
+    for (auto i = pending_cons.begin(); i != pending_cons.end(); ++i) {
+      if (i->second.get_con() == con) {
+	return i;
+      }
+    }
+    return pending_cons.end();
+  }
+
+public:
+  // AuthClient
+  int get_auth_request(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    uint32_t *method,
+    std::vector<uint32_t> *preferred_modes,
+    ceph::buffer::list *bl) override;
+  int handle_auth_reply_more(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    const ceph::buffer::list& bl,
+    ceph::buffer::list *reply) override;
+  int handle_auth_done(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    uint64_t global_id,
+    uint32_t con_mode,
+    const ceph::buffer::list& bl,
+    CryptoKey *session_key,
+    std::string *connection_secret) override;
+  int handle_auth_bad_method(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    uint32_t old_auth_method,
+    int result,
+    const std::vector<uint32_t>& allowed_methods,
+    const std::vector<uint32_t>& allowed_modes) override;
+  // AuthServer
+  int handle_auth_request(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    bool more,
+    uint32_t auth_method,
+    const ceph::buffer::list& bl,
+    ceph::buffer::list *reply) override;
+
+  void set_entity_name(EntityName name) { entity_name = name; }
+  void set_handle_authentication_dispatcher(Dispatcher *d) {
+    handle_authentication_dispatcher = d;
+  }
+  int _check_auth_tickets();
+  int _check_auth_rotating();
+  int wait_auth_rotating(double timeout);
+
+  int authenticate(double timeout=0.0);
+  bool is_authenticated() const {return authenticated;}
+
+  bool is_connected() const { return active_con != nullptr; }
+
+  /**
+   * Try to flush as many log messages as we can in a single
+   * message.  Use this before shutting down to transmit your
+   * last message.
+   */
+  void flush_log();
+
+private:
+  // mon subscriptions
+  MonSub sub;
+  void _renew_subs();
+  void handle_subscribe_ack(MMonSubscribeAck* m);
+
+public:
+  void renew_subs() {
+    std::lock_guard l(monc_lock);
+    _renew_subs();
+  }
+  bool sub_want(std::string what, version_t start, unsigned flags) {
+    std::lock_guard l(monc_lock);
+    return sub.want(what, start, flags);
+  }
+  void sub_got(std::string what, version_t have) {
+    std::lock_guard l(monc_lock);
+    sub.got(what, have);
+  }
+  void sub_unwant(std::string what) {
+    std::lock_guard l(monc_lock);
+    sub.unwant(what);
+  }
+  bool sub_want_increment(std::string what, version_t start, unsigned flags) {
+    std::lock_guard l(monc_lock);
+    return sub.inc_want(what, start, flags);
+  }
+
+  std::unique_ptr<KeyRing> keyring;
+  std::unique_ptr<RotatingKeyRing> rotating_secrets;
+
+ public:
+  MonClient(CephContext *cct_, boost::asio::io_context& service);
+  MonClient(const MonClient &) = delete;
+  MonClient& operator=(const MonClient &) = delete;
+  ~MonClient() override;
+
+  int init();
+  void shutdown();
+
+  void set_log_client(LogClient *clog) {
+    log_client = clog;
+  }
+  LogClient *get_log_client() {
+    return log_client;
+  }
+
+  int build_initial_monmap();
+  int get_monmap();
+  int get_monmap_and_config();
+  /**
+   * If you want to see MonMap messages, set this and
+   * the MonClient will tell the Messenger it hasn't
+   * dealt with it.
+   * Note that if you do this, *you* are of course responsible for
+   * putting the message reference!
+   */
+  void set_passthrough_monmap() {
+    std::lock_guard l(monc_lock);
+    passthrough_monmap = true;
+  }
+  void unset_passthrough_monmap() {
+    std::lock_guard l(monc_lock);
+    passthrough_monmap = false;
+  }
+  /**
+   * Ping monitor with ID @p mon_id and record the resulting
+   * reply in @p result_reply.
+   *
+   * @param[in]  mon_id Target monitor's ID
+   * @param[out] result_reply reply from mon.ID, if param != NULL
+   * @returns    0 in case of success; < 0 in case of error,
+   *             -ETIMEDOUT if monitor didn't reply before timeout
+   *             expired (default: conf->client_mount_timeout).
+   */
+  int ping_monitor(const std::string &mon_id, std::string *result_reply);
+
+  void send_mon_message(Message *m) {
+    send_mon_message(MessageRef{m, false});
+  }
+  void send_mon_message(MessageRef m);
+
+  void reopen_session() {
+    std::lock_guard l(monc_lock);
+    _reopen_session();
+  }
+
+  const uuid_d& get_fsid() const {
+    return monmap.fsid;
+  }
+
+  entity_addrvec_t get_mon_addrs(unsigned i) const {
+    std::lock_guard l(monc_lock);
+    if (i < monmap.size())
+      return monmap.get_addrs(i);
+    return entity_addrvec_t();
+  }
+  int get_num_mon() const {
+    std::lock_guard l(monc_lock);
+    return monmap.size();
+  }
+
+  uint64_t get_global_id() const {
+    std::lock_guard l(monc_lock);
+    return global_id;
+  }
+
+  void set_messenger(Messenger *m) { messenger = m; }
+  entity_addrvec_t get_myaddrs() const { return messenger->get_myaddrs(); }
+  AuthAuthorizer* build_authorizer(int service_id) const;
+
+  void set_want_keys(uint32_t want) {
+    want_keys = want;
+  }
+
+  // admin commands
+private:
+  uint64_t last_mon_command_tid;
+
+  struct MonCommand {
+    // for tell only
+    std::string target_name;
+    int target_rank = -1;
+    ConnectionRef target_con;
+    std::unique_ptr<MonConnection> target_session;
+    unsigned send_attempts = 0;  ///< attempt count for legacy mons
+    utime_t last_send_attempt;
+    uint64_t tid;
+    std::vector<std::string> cmd;
+    ceph::buffer::list inbl;
+    std::unique_ptr<CommandCompletion> onfinish;
+    std::optional<boost::asio::steady_timer> cancel_timer;
+
+    MonCommand(MonClient& monc, uint64_t t, std::unique_ptr<CommandCompletion> onfinish)
+      : tid(t), onfinish(std::move(onfinish)) {
+      auto timeout =
+          monc.cct->_conf.get_val<std::chrono::seconds>("rados_mon_op_timeout");
+      if (timeout.count() > 0) {
+	cancel_timer.emplace(monc.service, timeout);
+	cancel_timer->async_wait(
+          [this, &monc](boost::system::error_code ec) {
+	    if (ec)
+	      return;
+	    std::scoped_lock l(monc.monc_lock);
+	    monc._cancel_mon_command(tid);
+	  });
+      }
+    }
+
+    bool is_tell() const {
+      return target_name.size() || target_rank >= 0;
+    }
+  };
+  friend MonCommand;
+  std::map<uint64_t,MonCommand*> mon_commands;
+
+  void _send_command(MonCommand *r);
+  void _check_tell_commands();
+  void _resend_mon_commands();
+  int _cancel_mon_command(uint64_t tid);
+  void _finish_command(MonCommand *r, boost::system::error_code ret, std::string_view rs,
+		       bufferlist&& bl);
+  void _finish_auth();
+  void handle_mon_command_ack(MMonCommandAck *ack);
+  void handle_command_reply(MCommandReply *reply);
+
+public:
+  template<typename CompletionToken>
+  auto start_mon_command(const std::vector<std::string>& cmd,
+                         const ceph::buffer::list& inbl,
+			 CompletionToken&& token) {
+    ldout(cct,10) << __func__ << " cmd=" << cmd << dendl;
+    boost::asio::async_completion<CompletionToken, CommandSig> init(token);
+    {
+      std::scoped_lock l(monc_lock);
+      auto h = CommandCompletion::create(service.get_executor(),
+					 std::move(init.completion_handler));
+      if (!initialized || stopping) {
+	ceph::async::post(std::move(h), monc_errc::shutting_down, std::string{},
+			  bufferlist{});
+      } else {
+	auto r = new MonCommand(*this, ++last_mon_command_tid, std::move(h));
+	r->cmd = cmd;
+	r->inbl = inbl;
+	mon_commands.emplace(r->tid, r);
+	_send_command(r);
+      }
+    }
+    return init.result.get();
+  }
+
+  template<typename CompletionToken>
+  auto start_mon_command(int mon_rank, const std::vector<std::string>& cmd,
+			 const ceph::buffer::list& inbl, CompletionToken&& token) {
+    ldout(cct,10) << __func__ << " cmd=" << cmd << dendl;
+    boost::asio::async_completion<CompletionToken, CommandSig> init(token);
+    {
+      std::scoped_lock l(monc_lock);
+      auto h = CommandCompletion::create(service.get_executor(),
+					 std::move(init.completion_handler));
+      if (!initialized || stopping) {
+	ceph::async::post(std::move(h), monc_errc::shutting_down, std::string{},
+			  bufferlist{});
+      } else {
+	auto r = new MonCommand(*this, ++last_mon_command_tid, std::move(h));
+	r->target_rank = mon_rank;
+	r->cmd = cmd;
+	r->inbl = inbl;
+	mon_commands.emplace(r->tid, r);
+	_send_command(r);
+      }
+    }
+    return init.result.get();
+  }
+
+  template<typename CompletionToken>
+  auto start_mon_command(const std::string& mon_name,
+                         const std::vector<std::string>& cmd,
+			 const ceph::buffer::list& inbl,
+			 CompletionToken&& token) {
+    ldout(cct,10) << __func__ << " cmd=" << cmd << dendl;
+    boost::asio::async_completion<CompletionToken, CommandSig> init(token);
+    {
+      std::scoped_lock l(monc_lock);
+      auto h = CommandCompletion::create(service.get_executor(),
+					 std::move(init.completion_handler));
+      if (!initialized || stopping) {
+	ceph::async::post(std::move(h), monc_errc::shutting_down, std::string{},
+			  bufferlist{});
+      } else {
+	auto r = new MonCommand(*this, ++last_mon_command_tid, std::move(h));
+	// detect/tolerate mon *rank* passed as a string
+	std::string err;
+	int rank = strict_strtoll(mon_name.c_str(), 10, &err);
+	if (err.size() == 0 && rank >= 0) {
+	  ldout(cct,10) << __func__ << " interpreting name '" << mon_name
+			<< "' as rank " << rank << dendl;
+	  r->target_rank = rank;
+	} else {
+	  r->target_name = mon_name;
+	}
+	r->cmd = cmd;
+	r->inbl = inbl;
+	mon_commands.emplace(r->tid, r);
+	_send_command(r);
+      }
+    }
+    return init.result.get();
+  }
+
+  class ContextVerter {
+    std::string* outs;
+    ceph::bufferlist* outbl;
+    Context* onfinish;
+
+  public:
+    ContextVerter(std::string* outs, ceph::bufferlist* outbl, Context* onfinish)
+      : outs(outs), outbl(outbl), onfinish(onfinish) {}
+    ~ContextVerter() = default;
+    ContextVerter(const ContextVerter&) = default;
+    ContextVerter& operator =(const ContextVerter&) = default;
+    ContextVerter(ContextVerter&&) = default;
+    ContextVerter& operator =(ContextVerter&&) = default;
+
+    void operator()(boost::system::error_code e,
+		    std::string s,
+		    ceph::bufferlist bl) {
+      if (outs)
+	*outs = std::move(s);
+      if (outbl)
+	*outbl = std::move(bl);
+      if (onfinish)
+	onfinish->complete(ceph::from_error_code(e));
+    }
+  };
+
+  void start_mon_command(const vector<string>& cmd, const bufferlist& inbl,
+			 bufferlist *outbl, string *outs,
+			 Context *onfinish) {
+    start_mon_command(cmd, inbl, ContextVerter(outs, outbl, onfinish));
+  }
+  void start_mon_command(int mon_rank,
+			 const vector<string>& cmd, const bufferlist& inbl,
+			 bufferlist *outbl, string *outs,
+			 Context *onfinish) {
+    start_mon_command(mon_rank, cmd, inbl, ContextVerter(outs, outbl, onfinish));
+  }
+  void start_mon_command(const string &mon_name,  ///< mon name, with mon. prefix
+			 const vector<string>& cmd, const bufferlist& inbl,
+			 bufferlist *outbl, string *outs,
+			 Context *onfinish) {
+    start_mon_command(mon_name, cmd, inbl, ContextVerter(outs, outbl, onfinish));
+  }
+
+
+  // version requests
+public:
+  /**
+   * get latest known version(s) of cluster map
+   *
+   * @param map string name of map (e.g., 'osdmap')
+   * @param token context that will be triggered on completion
+   * @return (via Completion) {} on success,
+   *         boost::system::errc::resource_unavailable_try_again if we need to
+   *         resubmit our request
+   */
+  template<typename CompletionToken>
+  auto get_version(std::string&& map, CompletionToken&& token) {
+    boost::asio::async_completion<CompletionToken, VersionSig> init(token);
+    {
+      std::scoped_lock l(monc_lock);
+      auto m = ceph::make_message<MMonGetVersion>();
+      m->what = std::move(map);
+      m->handle = ++version_req_id;
+      version_requests.emplace(m->handle,
+			       VersionCompletion::create(
+				 service.get_executor(),
+				 std::move(init.completion_handler)));
+      _send_mon_message(m);
+    }
+    return init.result.get();
+  }
+
+  /**
+   * Run a callback within our lock, with a reference
+   * to the MonMap
+   */
+  template<typename Callback, typename...Args>
+  auto with_monmap(Callback&& cb, Args&&...args) const ->
+    decltype(cb(monmap, std::forward<Args>(args)...)) {
+    std::lock_guard l(monc_lock);
+    return std::forward<Callback>(cb)(monmap, std::forward<Args>(args)...);
+  }
+
+  void register_config_callback(md_config_t::config_callback fn);
+  void register_config_notify_callback(std::function<void(void)> f) {
+    config_notify_cb = f;
+  }
+  md_config_t::config_callback get_config_callback();
+
+private:
+
+  std::map<ceph_tid_t, std::unique_ptr<VersionCompletion>> version_requests;
+  ceph_tid_t version_req_id;
+  void handle_get_version_reply(MMonGetVersionReply* m);
+  md_config_t::config_callback config_cb;
+  std::function<void(void)> config_notify_cb;
+};
+
+#endif
diff --git a/src/mon/MonCommand.h b/src/mon/MonCommand.h
new file mode 100644
index 000000000..cb60d3d17
--- /dev/null
+++ b/src/mon/MonCommand.h
@@ -0,0 +1,175 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2017 John Spray <john.spray@redhat.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#pragma once
+
+#include <string>
+#include "include/encoding.h"
+
+struct MonCommand {
+  std::string cmdstring;
+  std::string helpstring;
+  std::string module;
+  std::string req_perms;
+  uint64_t flags;
+
+  // MonCommand flags
+  static const uint64_t FLAG_NONE       = 0;
+  static const uint64_t FLAG_NOFORWARD  = 1 << 0;
+  static const uint64_t FLAG_OBSOLETE   = 1 << 1;
+  static const uint64_t FLAG_DEPRECATED = 1 << 2;
+  static const uint64_t FLAG_MGR        = 1 << 3;
+  static const uint64_t FLAG_POLL       = 1 << 4;
+  static const uint64_t FLAG_HIDDEN     = 1 << 5;
+  // asok and tell commands are not forwarded, and they should not be listed
+  // in --help output.
+  static const uint64_t FLAG_TELL       = (FLAG_NOFORWARD | FLAG_HIDDEN);
+
+  bool has_flag(uint64_t flag) const { return (flags & flag) == flag; }
+  void set_flag(uint64_t flag) { flags |= flag; }
+  void unset_flag(uint64_t flag) { flags &= ~flag; }
+
+  void encode(ceph::buffer::list &bl) const {
+    ENCODE_START(1, 1, bl);
+    encode_bare(bl);
+    encode(flags, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(ceph::buffer::list::const_iterator &bl) {
+    DECODE_START(1, bl);
+    decode_bare(bl);
+    decode(flags, bl);
+    DECODE_FINISH(bl);
+  }
+
+  /**
+   * Unversioned encoding for use within encode_array.
+   */
+  void encode_bare(ceph::buffer::list &bl) const {
+    using ceph::encode;
+    encode(cmdstring, bl);
+    encode(helpstring, bl);
+    encode(module, bl);
+    encode(req_perms, bl);
+    std::string availability = "cli,rest";  // Removed field, for backward compat
+    encode(availability, bl);
+  }
+  void decode_bare(ceph::buffer::list::const_iterator &bl) {
+    using ceph::decode;
+    decode(cmdstring, bl);
+    decode(helpstring, bl);
+    decode(module, bl);
+    decode(req_perms, bl);
+    std::string availability;  // Removed field, for backward compat
+    decode(availability, bl);
+  }
+  bool is_compat(const MonCommand* o) const {
+    return cmdstring == o->cmdstring &&
+	module == o->module && req_perms == o->req_perms;
+  }
+
+  bool is_tell() const {
+    return has_flag(MonCommand::FLAG_TELL);
+  }
+
+  bool is_noforward() const {
+    return has_flag(MonCommand::FLAG_NOFORWARD);
+  }
+
+  bool is_obsolete() const {
+    return has_flag(MonCommand::FLAG_OBSOLETE);
+  }
+
+  bool is_deprecated() const {
+    return has_flag(MonCommand::FLAG_DEPRECATED);
+  }
+
+  bool is_mgr() const {
+    return has_flag(MonCommand::FLAG_MGR);
+  }
+
+  bool is_hidden() const {
+    return has_flag(MonCommand::FLAG_HIDDEN);
+  }
+
+  static void encode_array(const MonCommand *cmds, int size, ceph::buffer::list &bl) {
+    ENCODE_START(2, 1, bl);
+    uint16_t s = size;
+    encode(s, bl);
+    for (int i = 0; i < size; ++i) {
+      cmds[i].encode_bare(bl);
+    }
+    for (int i = 0; i < size; i++) {
+      encode(cmds[i].flags, bl);
+    }
+    ENCODE_FINISH(bl);
+  }
+  static void decode_array(MonCommand **cmds, int *size,
+                           ceph::buffer::list::const_iterator &bl) {
+    DECODE_START(2, bl);
+    uint16_t s = 0;
+    decode(s, bl);
+    *size = s;
+    *cmds = new MonCommand[*size];
+    for (int i = 0; i < *size; ++i) {
+      (*cmds)[i].decode_bare(bl);
+    }
+    if (struct_v >= 2) {
+      for (int i = 0; i < *size; i++)
+        decode((*cmds)[i].flags, bl);
+    } else {
+      for (int i = 0; i < *size; i++)
+        (*cmds)[i].flags = 0;
+    }
+    DECODE_FINISH(bl);
+  }
+
+  // this uses a u16 for the count, so we need a special encoder/decoder.
+  static void encode_vector(const std::vector<MonCommand>& cmds,
+			    ceph::buffer::list &bl) {
+    ENCODE_START(2, 1, bl);
+    uint16_t s = cmds.size();
+    encode(s, bl);
+    for (unsigned i = 0; i < s; ++i) {
+      cmds[i].encode_bare(bl);
+    }
+    for (unsigned i = 0; i < s; i++) {
+      encode(cmds[i].flags, bl);
+    }
+    ENCODE_FINISH(bl);
+  }
+  static void decode_vector(std::vector<MonCommand> &cmds,
+			    ceph::buffer::list::const_iterator &bl) {
+    DECODE_START(2, bl);
+    uint16_t s = 0;
+    decode(s, bl);
+    cmds.resize(s);
+    for (unsigned i = 0; i < s; ++i) {
+      cmds[i].decode_bare(bl);
+    }
+    if (struct_v >= 2) {
+      for (unsigned i = 0; i < s; i++)
+        decode(cmds[i].flags, bl);
+    } else {
+      for (unsigned i = 0; i < s; i++)
+        cmds[i].flags = 0;
+    }
+    DECODE_FINISH(bl);
+  }
+
+  bool requires_perm(char p) const {
+    return (req_perms.find(p) != std::string::npos);
+  }
+};
+WRITE_CLASS_ENCODER(MonCommand)
diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
new file mode 100644
index 000000000..f5ca47eb4
--- /dev/null
+++ b/src/mon/MonCommands.h
@@ -0,0 +1,1407 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+/* no guard; may be included multiple times */
+
+/*
+ * Define commands that are reported by the monitor's
+ * "get_command_descriptions" command, and parsed by the Python
+ * frontend 'ceph' (and perhaps by other frontends, such as a RESTful
+ * server). The format is:
+ *
+ * COMMAND(signature, helpstring, modulename, req perms, availability)
+ * where:
+ * signature:  describes the command and its parameters (more below)
+ * helpstring: displays in CLI help, API help (nice if it refers to
+ *             parameter names from signature, 40-a few hundred chars)
+ * modulename: the monitor module or daemon this applies to:
+ *             mds, osd, pg (osd), mon, auth, log, config-key, mgr
+ * req perms:  required permission in that modulename space to execute command
+ *             this also controls what type of REST command is accepted
+ *
+ * The commands describe themselves completely enough for the separate
+ * frontend(s) to be able to accept user input and validate it against
+ * the command descriptions, and generate a JSON object that contains
+ * key:value mappings of parameter names to validated parameter values.
+ *
+ * 'signature' is a space-separated list of individual command descriptors;
+ * each descriptor is either a literal string, which can contain no spaces or
+ * '=' signs (for instance, in "pg stat", both "pg" and "stat" are literal
+ * strings representing one descriptor each), or a list of key=val[,key=val...]
+ * which also includes no spaces.
+ *
+ * The key=val form describes a non-literal parameter.  Each will have at
+ * least a name= and type=, and each type can have its own type-specific
+ * parameters.  The parser is the arbiter of these types and their
+ * interpretation.  A few more non-type-specific key=val pairs exist:
+ *
+ *    req=false marks an optional parameter (default for req is 'true')
+ *    n=<n> is a repeat count for how many of this argument must be supplied.
+ *          n=1 is the default.
+ *          n=N is a special case that means "1 or more".
+ *
+ * A perhaps-incomplete list of types:
+ *
+ * CephInt: Optional: range=min[|max]
+ * CephFloat: Optional range
+ * CephString: optional badchars
+ * CephSocketpath: validation involves "is it S_ISSOCK"
+ * CephIPAddr: v4 or v6 addr with optional port, syntax validated
+ * CephEntityAddr: CephIPAddr + optional '/nonce'
+ * CephPoolname: Plainold string
+ * CephObjectname: Another plainold string
+ * CephPgid: n.xxx where n is an int > 0, xxx is a hex number > 0
+ * CephName: daemon name, '*' or '<type>.<id>' (id must be int for type osd)
+ * CephOsdName: osd name, '*' or '<id> or 'osd.<id>' (id must be int)
+ * CephChoices: strings="foo|bar" means this param can be either
+ * CephFilepath: openable file
+ * CephFragment: cephfs 'fragID': val/bits, val in hex 0xnnn, bits in dec
+ * CephUUID: uuid in text matching Python uuid.UUID()
+ * CephPrefix: special type assigned to literals
+ *
+ * Example:
+ *
+ * COMMAND("auth add "
+ *   	   "name=entity,type=CephString "
+ *   	   "name=caps,type=CephString,n=N,req=false",
+ *   	   "add auth info for <name> from input file, or random key "
+ *   	   "if no input given, and/or any caps specified in the command")
+ *
+ * defines a command "auth add" that takes a required argument "entity"
+ * of type "CephString", and from 1 to N arguments named "caps" of type
+ * CephString, at least one of which is required.  The front end will
+ * validate user input against this description.  Let's say the user
+ * enters auth add client.admin 'mon rwx' 'osd *'.  The result will be a
+ * JSON object like {"prefix":"auth add", "entity":"client.admin",
+ * "caps":["mon rwx", "osd *"]}.
+ * Note that
+ * 	- string literals are accumulated into 'prefix'
+ * 	- n=1 descriptors are given normal string or int object values
+ * 	- n=N descriptors are given array values
+ *
+ * NOTE: be careful with spaces.  Each descriptor must be separated by
+ * one space, no other characters, so if you split lines as above, be
+ * sure to close and reopen the quotes, and be careful to include the '
+ * separating spaces in the quoted string.
+ *
+ * The monitor marshals this JSON into a std::map<string, cmd_vartype>
+ * where cmd_vartype is a boost::variant type-enforcing discriminated
+ * type, so the monitor is expected to know the type of each argument.
+ * See cmdparse.cc/h for more details.
+ *
+ * The flag parameter for COMMAND_WITH_FLAGS macro must be passed using
+ * FLAG(f), where 'f' may be one of the following:
+ *
+ *  NONE      - no flag assigned
+ *  NOFORWARD - command may not be forwarded
+ *  OBSOLETE  - command is considered obsolete
+ *  DEPRECATED - command is considered deprecated
+ *  MGR       - command goes to ceph-mgr (for luminous+)
+ *  POLL      - command is intended to be called periodically by the
+ *              client (see iostat)
+ *  HIDDEN    - command is hidden (no reported by help etc)
+ *  TELL      - tell/asok command. it's an alias of (NOFORWARD | HIDDEN)
+ *
+ * A command should always be first considered DEPRECATED before being
+ * considered OBSOLETE, giving due consideration to users and conforming
+ * to any guidelines regarding deprecating commands.
+ */
+
+COMMAND("pg map name=pgid,type=CephPgid", "show mapping of pg to osds", \
+	"pg", "r")
+COMMAND("pg repeer name=pgid,type=CephPgid", "force a PG to repeer",
+	"osd", "rw")
+COMMAND("osd last-stat-seq name=id,type=CephOsdName", \
+	"get the last pg stats sequence number reported for this osd", \
+	"osd", "r")
+
+/*
+ * auth commands AuthMonitor.cc
+ */
+
+COMMAND("auth export name=entity,type=CephString,req=false", \
+       	"write keyring for requested entity, or master keyring if none given", \
+	"auth", "rx")
+COMMAND("auth get name=entity,type=CephString", \
+	"write keyring file with requested key", "auth", "rx")
+COMMAND("auth get-key name=entity,type=CephString", "display requested key", \
+	"auth", "rx")
+COMMAND("auth print-key name=entity,type=CephString", "display requested key", \
+	"auth", "rx")
+COMMAND("auth print_key name=entity,type=CephString", "display requested key", \
+	"auth", "rx")
+COMMAND_WITH_FLAG("auth list", "list authentication state", "auth", "rx",
+		  FLAG(DEPRECATED))
+COMMAND("auth ls", "list authentication state", "auth", "rx")
+COMMAND("auth import", "auth import: read keyring file from -i <file>",
+	"auth", "rwx")
+COMMAND("auth add "
+	"name=entity,type=CephString "
+	"name=caps,type=CephString,n=N,req=false",
+	"add auth info for <entity> from input file, or random key if no "
+        "input is given, and/or any caps specified in the command",
+	"auth", "rwx")
+COMMAND("auth get-or-create-key "
+	"name=entity,type=CephString "
+	"name=caps,type=CephString,n=N,req=false",
+	"get, or add, key for <name> from system/caps pairs specified in the command.  If key already exists, any given caps must match the existing caps for that key.",
+	"auth", "rwx")
+COMMAND("auth get-or-create "
+	"name=entity,type=CephString "
+	"name=caps,type=CephString,n=N,req=false",
+	"add auth info for <entity> from input file, or random key if no input given, and/or any caps specified in the command",
+	"auth", "rwx")
+COMMAND("fs authorize "
+   "name=filesystem,type=CephString "
+   "name=entity,type=CephString "
+	"name=caps,type=CephString,n=N",
+	"add auth for <entity> to access file system <filesystem> based on following directory and permissions pairs",
+	"auth", "rwx")
+COMMAND("auth caps "
+	"name=entity,type=CephString "
+	"name=caps,type=CephString,n=N",
+	"update caps for <name> from caps specified in the command",
+	"auth", "rwx")
+COMMAND_WITH_FLAG("auth del "
+	"name=entity,type=CephString",
+	"delete all caps for <name>",
+	"auth", "rwx",
+    FLAG(DEPRECATED))
+COMMAND("auth rm "
+	"name=entity,type=CephString",
+	"remove all caps for <name>",
+	"auth", "rwx")
+
+/*
+ * Monitor commands (Monitor.cc)
+ */
+COMMAND_WITH_FLAG("compact", "cause compaction of monitor's leveldb/rocksdb storage",
+	     "mon", "rw",
+             FLAG(TELL))
+COMMAND_WITH_FLAG("scrub", "scrub the monitor stores",
+             "mon", "rw",
+             FLAG(OBSOLETE))
+COMMAND("fsid", "show cluster FSID/UUID", "mon", "r")
+COMMAND("log name=logtext,type=CephString,n=N",
+	"log supplied text to the monitor log", "mon", "rw")
+COMMAND("log last "
+        "name=num,type=CephInt,range=1,req=false "
+        "name=level,type=CephChoices,strings=debug|info|sec|warn|error,req=false "
+        "name=channel,type=CephChoices,strings=*|cluster|audit|cephadm,req=false",
+	"print last few lines of the cluster log",
+	"mon", "r")
+
+COMMAND("status", "show cluster status", "mon", "r")
+COMMAND("health name=detail,type=CephChoices,strings=detail,req=false",
+	"show cluster health", "mon", "r")
+COMMAND("health mute "\
+	"name=code,type=CephString "
+	"name=ttl,type=CephString,req=false "
+	"name=sticky,type=CephBool,req=false",
+	"mute health alert", "mon", "w")
+COMMAND("health unmute "\
+	"name=code,type=CephString,req=false",
+	"unmute existing health alert mute(s)", "mon", "w")
+COMMAND("time-sync-status", "show time sync status", "mon", "r")
+COMMAND("df name=detail,type=CephChoices,strings=detail,req=false",
+	"show cluster free space stats", "mon", "r")
+COMMAND("report name=tags,type=CephString,n=N,req=false",
+	"report full status of cluster, optional title tag strings",
+	"mon", "r")
+COMMAND("features", "report of connected features",
+        "mon", "r")
+COMMAND("quorum_status", "report status of monitor quorum",
+	"mon", "r")
+COMMAND("mon ok-to-stop "
+	"name=ids,type=CephString,n=N",
+	"check whether mon(s) can be safely stopped without reducing immediate "
+	"availability",
+	"mon", "r")
+COMMAND("mon ok-to-add-offline",
+	"check whether adding a mon and not starting it would break quorum",
+	"mon", "r")
+COMMAND("mon ok-to-rm "
+	"name=id,type=CephString",
+	"check whether removing the specified mon would break quorum",
+	"mon", "r")
+
+COMMAND("tell "
+	"name=target,type=CephName "
+	"name=args,type=CephString,n=N",
+	"send a command to a specific daemon", "mon", "rw")
+COMMAND_WITH_FLAG("version", "show mon daemon version", "mon", "r",
+                  FLAG(TELL))
+
+COMMAND("node ls "
+	"name=type,type=CephChoices,strings=all|osd|mon|mds|mgr,req=false",
+	"list all nodes in cluster [type]", "mon", "r")
+/*
+ * Monitor-specific commands under module 'mon'
+ */
+COMMAND_WITH_FLAG("mon scrub",
+    "scrub the monitor stores",
+    "mon", "rw",
+    FLAG(NONE))
+COMMAND("mon metadata name=id,type=CephString,req=false",
+	"fetch metadata for mon <id>",
+	"mon", "r")
+COMMAND("mon count-metadata name=property,type=CephString",
+	"count mons by metadata field property",
+	"mon", "r")
+COMMAND("mon versions",
+	"check running versions of monitors",
+	"mon", "r")
+COMMAND("versions",
+	"check running versions of ceph daemons",
+	"mon", "r")
+
+
+
+/*
+ * MDS commands (MDSMonitor.cc)
+ */
+
+COMMAND_WITH_FLAG("mds stat", "show MDS status", "mds", "r", FLAG(HIDDEN))
+COMMAND_WITH_FLAG("mds dump "
+	"name=epoch,type=CephInt,req=false,range=0",
+	"dump legacy MDS cluster info, optionally from epoch",
+        "mds", "r", FLAG(OBSOLETE))
+COMMAND("fs dump "
+	"name=epoch,type=CephInt,req=false,range=0",
+	"dump all CephFS status, optionally from epoch", "mds", "r")
+COMMAND_WITH_FLAG("mds getmap "
+	"name=epoch,type=CephInt,req=false,range=0",
+	"get MDS map, optionally from epoch", "mds", "r", FLAG(OBSOLETE))
+COMMAND("mds metadata name=who,type=CephString,req=false",
+	"fetch metadata for mds <role>",
+	"mds", "r")
+COMMAND("mds count-metadata name=property,type=CephString",
+	"count MDSs by metadata field property",
+	"mds", "r")
+COMMAND("mds versions",
+	"check running versions of MDSs",
+	"mds", "r")
+COMMAND_WITH_FLAG("mds tell "
+	"name=who,type=CephString "
+	"name=args,type=CephString,n=N",
+	"send command to particular mds", "mds", "rw", FLAG(OBSOLETE))
+COMMAND_WITH_FLAG("mds stop name=role,type=CephString", "stop mds",
+	"mds", "rw", FLAG(OBSOLETE))
+COMMAND_WITH_FLAG("mds deactivate name=role,type=CephString",
+        "clean up specified MDS rank (use with `set max_mds` to shrink cluster)",
+	"mds", "rw", FLAG(OBSOLETE))
+COMMAND("mds ok-to-stop name=ids,type=CephString,n=N",
+	"check whether stopping the specified MDS would reduce immediate availability",
+	"mds", "r")
+COMMAND_WITH_FLAG("mds set_max_mds "
+	"name=maxmds,type=CephInt,range=0",
+	"set max MDS index", "mds", "rw", FLAG(OBSOLETE))
+COMMAND_WITH_FLAG("mds set "
+	"name=var,type=CephChoices,strings=max_mds|max_file_size|inline_data|"
+	"allow_new_snaps|allow_multimds|allow_multimds_snaps|allow_dirfrags "
+	"name=val,type=CephString "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"set mds parameter <var> to <val>", "mds", "rw", FLAG(OBSOLETE))
+COMMAND_WITH_FLAG("mds freeze name=role_or_gid,type=CephString"
+	" name=val,type=CephString",
+	"freeze MDS yes/no", "mds", "rw", FLAG(HIDDEN))
+// arbitrary limit 0-20 below; worth standing on head to make it
+// relate to actual state definitions?
+// #include "include/ceph_fs.h"
+COMMAND_WITH_FLAG("mds set_state "
+	"name=gid,type=CephInt,range=0 "
+	"name=state,type=CephInt,range=0|20",
+	"set mds state of <gid> to <numeric-state>", "mds", "rw", FLAG(HIDDEN))
+COMMAND("mds fail name=role_or_gid,type=CephString",
+	"Mark MDS failed: trigger a failover if a standby is available",
+        "mds", "rw")
+COMMAND("mds repaired name=role,type=CephString",
+	"mark a damaged MDS rank as no longer damaged", "mds", "rw")
+COMMAND("mds rm "
+	"name=gid,type=CephInt,range=0",
+	"remove nonactive mds", "mds", "rw")
+COMMAND_WITH_FLAG("mds rmfailed name=role,type=CephString "
+        "name=yes_i_really_mean_it,type=CephBool,req=false",
+	"remove failed rank", "mds", "rw", FLAG(HIDDEN))
+COMMAND_WITH_FLAG("mds cluster_down", "take MDS cluster down", "mds", "rw", FLAG(OBSOLETE))
+COMMAND_WITH_FLAG("mds cluster_up", "bring MDS cluster up", "mds", "rw", FLAG(OBSOLETE))
+COMMAND_WITH_FLAG("mds compat show", "show mds compatibility settings",
+	"mds", "r", FLAG(DEPRECATED))
+COMMAND("fs compat show "
+        "name=fs_name,type=CephString ",
+        "show fs compatibility settings",
+	"mds", "r")
+COMMAND_WITH_FLAG("mds compat rm_compat "
+	"name=feature,type=CephInt,range=0",
+	"remove compatible feature", "mds", "rw", FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("mds compat rm_incompat "
+	"name=feature,type=CephInt,range=0",
+	"remove incompatible feature", "mds", "rw", FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("mds add_data_pool "
+	"name=pool,type=CephString",
+	"add data pool <pool>", "mds", "rw", FLAG(OBSOLETE))
+COMMAND_WITH_FLAG("mds rm_data_pool "
+	"name=pool,type=CephString",
+	"remove data pool <pool>", "mds", "rw", FLAG(OBSOLETE))
+COMMAND_WITH_FLAG("mds remove_data_pool "
+	"name=pool,type=CephString",
+	"remove data pool <pool>", "mds", "rw", FLAG(OBSOLETE))
+COMMAND_WITH_FLAG("mds newfs "
+	"name=metadata,type=CephInt,range=0 "
+	"name=data,type=CephInt,range=0 "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"make new filesystem using pools <metadata> and <data>",
+	"mds", "rw", FLAG(OBSOLETE))
+COMMAND("fs new "
+	"name=fs_name,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=metadata,type=CephString "
+	"name=data,type=CephString "
+	"name=force,type=CephBool,req=false "
+	"name=allow_dangerous_metadata_overlay,type=CephBool,req=false "
+	"name=fscid,type=CephInt,range=0,req=false "
+	"name=recover,type=CephBool,req=false",
+	"make new filesystem using named pools <metadata> and <data>",
+	"fs", "rw")
+COMMAND("fs fail "
+	"name=fs_name,type=CephString ",
+	"bring the file system down and all of its ranks",
+	"fs", "rw")
+COMMAND("fs rm "
+	"name=fs_name,type=CephString "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"disable the named filesystem",
+	"fs", "rw")
+COMMAND("fs reset "
+	"name=fs_name,type=CephString "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"disaster recovery only: reset to a single-MDS map",
+	"fs", "rw")
+COMMAND("fs ls ",
+	"list filesystems",
+	"fs", "r")
+COMMAND("fs get name=fs_name,type=CephString",
+	"get info about one filesystem",
+	"fs", "r")
+COMMAND("fs set "
+	"name=fs_name,type=CephString "
+	"name=var,type=CephChoices,strings=max_mds|max_file_size"
+        "|allow_new_snaps|inline_data|cluster_down|allow_dirfrags|balancer"
+        "|standby_count_wanted|session_timeout|session_autoclose"
+        "|allow_standby_replay|down|joinable|min_compat_client "
+	"name=val,type=CephString "
+	"name=yes_i_really_mean_it,type=CephBool,req=false "
+	"name=yes_i_really_really_mean_it,type=CephBool,req=false",
+	"set fs parameter <var> to <val>", "mds", "rw")
+COMMAND("fs flag set name=flag_name,type=CephChoices,strings=enable_multiple "
+        "name=val,type=CephString "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"Set a global CephFS flag",
+	"fs", "rw")
+
+COMMAND("fs feature ls",
+        "list available cephfs features to be set/unset",
+	"mds", "r")
+
+COMMAND("fs compat "
+        "name=fs_name,type=CephString "
+        "name=subop,type=CephChoices,strings=rm_compat|rm_incompat|add_compat|add_incompat "
+        "name=feature,type=CephInt "
+        "name=feature_str,type=CephString,req=false ",
+        "manipulate compat settings", "fs", "rw")
+
+COMMAND("fs required_client_features "
+        "name=fs_name,type=CephString "
+        "name=subop,type=CephChoices,strings=add|rm "
+        "name=val,type=CephString ",
+        "add/remove required features of clients", "mds", "rw")
+
+COMMAND("fs add_data_pool name=fs_name,type=CephString "
+	"name=pool,type=CephString",
+	"add data pool <pool>", "mds", "rw")
+COMMAND("fs rm_data_pool name=fs_name,type=CephString "
+	"name=pool,type=CephString",
+	"remove data pool <pool>", "mds", "rw")
+COMMAND_WITH_FLAG("fs set_default name=fs_name,type=CephString",
+		  "set the default to the named filesystem",
+		  "fs", "rw",
+		  FLAG(DEPRECATED))
+COMMAND("fs set-default name=fs_name,type=CephString",
+	"set the default to the named filesystem",
+	"fs", "rw")
+COMMAND("fs mirror enable "
+	"name=fs_name,type=CephString ",
+	"enable mirroring for a ceph filesystem", "mds", "rw")
+COMMAND("fs mirror disable "
+	"name=fs_name,type=CephString ",
+	"disable mirroring for a ceph filesystem", "mds", "rw")
+COMMAND("fs mirror peer_add "
+	"name=fs_name,type=CephString "
+	"name=uuid,type=CephString "
+	"name=remote_cluster_spec,type=CephString "
+	"name=remote_fs_name,type=CephString",
+	"add a mirror peer for a ceph filesystem", "mds", "rw")
+COMMAND("fs mirror peer_remove "
+	"name=fs_name,type=CephString "
+	"name=uuid,type=CephString ",
+	"remove a mirror peer for a ceph filesystem", "mds", "rw")
+
+/*
+ * Monmap commands
+ */
+COMMAND("mon dump "
+	"name=epoch,type=CephInt,range=0,req=false",
+	"dump formatted monmap (optionally from epoch)",
+	"mon", "r")
+COMMAND("mon stat", "summarize monitor status", "mon", "r")
+COMMAND("mon getmap "
+	"name=epoch,type=CephInt,range=0,req=false",
+	"get monmap", "mon", "r")
+COMMAND("mon add "
+	"name=name,type=CephString "
+	"name=addr,type=CephIPAddr "
+	"name=location,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=],req=false",
+	"add new monitor named <name> at <addr>, possibly with CRUSH location <location>", "mon", "rw")
+COMMAND("mon rm "
+	"name=name,type=CephString",
+	"remove monitor named <name>", "mon", "rw")
+COMMAND_WITH_FLAG("mon remove "
+	"name=name,type=CephString",
+	"remove monitor named <name>", "mon", "rw",
+    FLAG(DEPRECATED))
+COMMAND("mon feature ls "
+        "name=with_value,type=CephChoices,strings=--with-value,req=false",
+        "list available mon map features to be set/unset",
+        "mon", "r")
+COMMAND("mon feature set "
+        "name=feature_name,type=CephString "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+        "set provided feature on mon map",
+        "mon", "rw")
+COMMAND("mon set-rank "
+	"name=name,type=CephString "
+	"name=rank,type=CephInt",
+	"set the rank for the specified mon",
+	"mon", "rw")
+COMMAND("mon set-addrs "
+	"name=name,type=CephString "
+	"name=addrs,type=CephString",
+	"set the addrs (IPs and ports) a specific monitor binds to",
+	"mon", "rw")
+COMMAND("mon set-weight "
+        "name=name,type=CephString "
+        "name=weight,type=CephInt,range=0|65535",
+        "set the weight for the specified mon",
+        "mon", "rw")
+COMMAND("mon enable-msgr2",
+	"enable the msgr2 protocol on port 3300",
+	"mon", "rw")
+COMMAND("mon set election_strategy " \
+	"name=strategy,type=CephString", \
+	"set the election strategy to use; choices classic, disallow, connectivity", \
+	"mon", "rw")
+COMMAND("mon add disallowed_leader " \
+	"name=name,type=CephString", \
+	"prevent the named mon from being a leader", \
+	"mon", "rw")
+COMMAND("mon rm disallowed_leader " \
+	"name=name,type=CephString", \
+	"allow the named mon to be a leader again", \
+	"mon", "rw")
+COMMAND("mon set_location " \
+	"name=name,type=CephString "
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]",
+	"specify location <args> for the monitor <name>, using CRUSH bucket names", \
+	"mon", "rw")
+COMMAND("mon enable_stretch_mode " \
+	"name=tiebreaker_mon,type=CephString, "
+	"name=new_crush_rule,type=CephString, "
+	"name=dividing_bucket,type=CephString, ",
+	"enable stretch mode, changing the peering rules and "
+	"failure handling on all pools with <tiebreaker_mon> "
+	"as the tiebreaker and setting <dividing_bucket> locations "
+	"as the units for stretching across",
+	"mon", "rw")
+COMMAND("mon set_new_tiebreaker " \
+	"name=name,type=CephString "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"switch the stretch tiebreaker to be the named mon", \
+	"mon", "rw")
+
+/*
+ * OSD commands
+ */
+COMMAND("osd stat", "print summary of OSD map", "osd", "r")
+COMMAND("osd dump "
+	"name=epoch,type=CephInt,range=0,req=false",
+	"print summary of OSD map", "osd", "r")
+COMMAND("osd info "
+	"name=id,type=CephOsdName,req=false",
+	"print osd's {id} information (instead of all osds from map)",
+	"osd", "r")
+COMMAND("osd tree "
+	"name=epoch,type=CephInt,range=0,req=false "
+	"name=states,type=CephChoices,strings=up|down|in|out|destroyed,n=N,req=false",
+	"print OSD tree", "osd", "r")
+COMMAND("osd tree-from "
+	"name=epoch,type=CephInt,range=0,req=false "
+	"name=bucket,type=CephString "
+	"name=states,type=CephChoices,strings=up|down|in|out|destroyed,n=N,req=false",
+	"print OSD tree in bucket", "osd", "r")
+COMMAND("osd ls "
+	"name=epoch,type=CephInt,range=0,req=false",
+	"show all OSD ids", "osd", "r")
+COMMAND("osd getmap "
+	"name=epoch,type=CephInt,range=0,req=false",
+	"get OSD map", "osd", "r")
+COMMAND("osd getcrushmap "
+	"name=epoch,type=CephInt,range=0,req=false",
+	"get CRUSH map", "osd", "r")
+COMMAND("osd getmaxosd", "show largest OSD id", "osd", "r")
+COMMAND("osd ls-tree "
+        "name=epoch,type=CephInt,range=0,req=false "
+        "name=name,type=CephString,req=true",
+        "show OSD ids under bucket <name> in the CRUSH map",
+        "osd", "r")
+COMMAND("osd find "
+	"name=id,type=CephOsdName",
+	"find osd <id> in the CRUSH map and show its location",
+	"osd", "r")
+COMMAND("osd metadata "
+	"name=id,type=CephOsdName,req=false",
+	"fetch metadata for osd {id} (default all)",
+	"osd", "r")
+COMMAND("osd count-metadata name=property,type=CephString",
+	"count OSDs by metadata field property",
+	"osd", "r")
+COMMAND("osd versions",
+	"check running versions of OSDs",
+	"osd", "r")
+COMMAND("osd numa-status",
+	"show NUMA status of OSDs",
+	"osd", "r")
+COMMAND("osd map "
+	"name=pool,type=CephPoolname "
+	"name=object,type=CephObjectname "
+	"name=nspace,type=CephString,req=false",
+	"find pg for <object> in <pool> with [namespace]", "osd", "r")
+COMMAND_WITH_FLAG("osd lspools",
+		  "list pools", "osd", "r", FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("osd crush rule list", "list crush rules", "osd", "r",
+		  FLAG(DEPRECATED))
+COMMAND("osd crush rule ls", "list crush rules", "osd", "r")
+COMMAND("osd crush rule ls-by-class "
+        "name=class,type=CephString,goodchars=[A-Za-z0-9-_.]",
+        "list all crush rules that reference the same <class>",
+        "osd", "r")
+COMMAND("osd crush rule dump "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.],req=false",
+	"dump crush rule <name> (default all)",
+	"osd", "r")
+COMMAND("osd crush dump",
+	"dump crush map",
+	"osd", "r")
+COMMAND("osd setcrushmap name=prior_version,type=CephInt,req=false",
+	"set crush map from input file",
+	"osd", "rw")
+COMMAND("osd crush set name=prior_version,type=CephInt,req=false",
+	"set crush map from input file",
+	"osd", "rw")
+COMMAND("osd crush add-bucket "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] "
+        "name=type,type=CephString "
+        "name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=],req=false",
+	"add no-parent (probably root) crush bucket <name> of type <type> "
+        "to location <args>",
+	"osd", "rw")
+COMMAND("osd crush rename-bucket "
+	"name=srcname,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=dstname,type=CephString,goodchars=[A-Za-z0-9-_.]",
+	"rename bucket <srcname> to <dstname>",
+	"osd", "rw")
+COMMAND("osd crush set "
+	"name=id,type=CephOsdName "
+	"name=weight,type=CephFloat,range=0.0 "
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]",
+	"update crushmap position and weight for <name> to <weight> with location <args>",
+	"osd", "rw")
+COMMAND("osd crush add "
+	"name=id,type=CephOsdName "
+	"name=weight,type=CephFloat,range=0.0 "
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]",
+	"add or update crushmap position and weight for <name> with <weight> and location <args>",
+	"osd", "rw")
+COMMAND("osd crush set-all-straw-buckets-to-straw2",
+        "convert all CRUSH current straw buckets to use the straw2 algorithm",
+	"osd", "rw")
+COMMAND("osd crush class create "
+        "name=class,type=CephString,goodchars=[A-Za-z0-9-_]",
+        "create crush device class <class>",
+        "osd", "rw")
+COMMAND("osd crush class rm "
+        "name=class,type=CephString,goodchars=[A-Za-z0-9-_]",
+        "remove crush device class <class>",
+        "osd", "rw")
+COMMAND("osd crush set-device-class "
+        "name=class,type=CephString "
+	"name=ids,type=CephString,n=N",
+	"set the <class> of the osd(s) <id> [<id>...],"
+        "or use <all|any> to set all.",
+	"osd", "rw")
+COMMAND("osd crush rm-device-class "
+        "name=ids,type=CephString,n=N",
+        "remove class of the osd(s) <id> [<id>...],"
+        "or use <all|any> to remove all.",
+        "osd", "rw")
+COMMAND("osd crush class rename "
+        "name=srcname,type=CephString,goodchars=[A-Za-z0-9-_] "
+        "name=dstname,type=CephString,goodchars=[A-Za-z0-9-_]",
+        "rename crush device class <srcname> to <dstname>",
+        "osd", "rw")
+COMMAND("osd crush create-or-move "
+	"name=id,type=CephOsdName "
+	"name=weight,type=CephFloat,range=0.0 "
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]",
+	"create entry or move existing entry for <name> <weight> at/to location <args>",
+	"osd", "rw")
+COMMAND("osd crush move "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]",
+	"move existing entry for <name> to location <args>",
+	"osd", "rw")
+COMMAND("osd crush swap-bucket "
+	"name=source,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=dest,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"swap existing bucket contents from (orphan) bucket <source> and <target>",
+	"osd", "rw")
+COMMAND("osd crush link "
+	"name=name,type=CephString "
+	"name=args,type=CephString,n=N,goodchars=[A-Za-z0-9-_.=]",
+	"link existing entry for <name> under location <args>",
+	"osd", "rw")
+COMMAND("osd crush rm "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=ancestor,type=CephString,req=false,goodchars=[A-Za-z0-9-_.]",
+	"remove <name> from crush map (everywhere, or just at <ancestor>)",\
+	"osd", "rw")
+COMMAND_WITH_FLAG("osd crush remove "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=ancestor,type=CephString,req=false,goodchars=[A-Za-z0-9-_.]",
+	"remove <name> from crush map (everywhere, or just at <ancestor>)",
+	"osd", "rw",
+    FLAG(DEPRECATED))
+COMMAND("osd crush unlink "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=ancestor,type=CephString,req=false,goodchars=[A-Za-z0-9-_.]",
+	"unlink <name> from crush map (everywhere, or just at <ancestor>)",
+	"osd", "rw")
+COMMAND("osd crush reweight-all",
+	"recalculate the weights for the tree to ensure they sum correctly",
+	"osd", "rw")
+COMMAND("osd crush reweight "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=weight,type=CephFloat,range=0.0",
+	"change <name>'s weight to <weight> in crush map",
+	"osd", "rw")
+COMMAND("osd crush reweight-subtree "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=weight,type=CephFloat,range=0.0",
+	"change all leaf items beneath <name> to <weight> in crush map",
+	"osd", "rw")
+COMMAND("osd crush tunables "
+	"name=profile,type=CephChoices,strings=legacy|argonaut|bobtail|firefly|hammer|jewel|optimal|default",
+	"set crush tunables values to <profile>", "osd", "rw")
+COMMAND("osd crush set-tunable "
+	"name=tunable,type=CephChoices,strings=straw_calc_version "
+	"name=value,type=CephInt",
+	"set crush tunable <tunable> to <value>",
+	"osd", "rw")
+COMMAND("osd crush get-tunable "
+	"name=tunable,type=CephChoices,strings=straw_calc_version",
+	"get crush tunable <tunable>",
+	"osd", "r")
+COMMAND("osd crush show-tunables",
+	"show current crush tunables", "osd", "r")
+COMMAND("osd crush rule create-simple "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=root,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=type,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=mode,type=CephChoices,strings=firstn|indep,req=false",
+	"create crush rule <name> to start from <root>, replicate across buckets of type <type>, using a choose mode of <firstn|indep> (default firstn; indep best for erasure pools)",
+	"osd", "rw")
+COMMAND("osd crush rule create-replicated "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=root,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=type,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=class,type=CephString,goodchars=[A-Za-z0-9-_.],req=false",
+	"create crush rule <name> for replicated pool to start from <root>, replicate across buckets of type <type>, use devices of type <class> (ssd or hdd)",
+	"osd", "rw")
+COMMAND("osd crush rule create-erasure "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=profile,type=CephString,req=false,goodchars=[A-Za-z0-9-_.=]",
+	"create crush rule <name> for erasure coded pool created with <profile> (default default)",
+	"osd", "rw")
+COMMAND("osd crush rule rm "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] ",
+	"remove crush rule <name>", "osd", "rw")
+COMMAND("osd crush rule rename "
+        "name=srcname,type=CephString,goodchars=[A-Za-z0-9-_.] "
+        "name=dstname,type=CephString,goodchars=[A-Za-z0-9-_.]",
+        "rename crush rule <srcname> to <dstname>",
+        "osd", "rw")
+COMMAND("osd crush tree "
+        "name=shadow,type=CephChoices,strings=--show-shadow,req=false",
+	"dump crush buckets and items in a tree view",
+	"osd", "r")
+COMMAND("osd crush ls name=node,type=CephString,goodchars=[A-Za-z0-9-_.]",
+	"list items beneath a node in the CRUSH tree",
+	"osd", "r")
+COMMAND("osd crush class ls",
+	"list all crush device classes",
+	"osd", "r")
+COMMAND("osd crush class ls-osd "
+        "name=class,type=CephString,goodchars=[A-Za-z0-9-_]",
+        "list all osds belonging to the specific <class>",
+        "osd", "r")
+COMMAND("osd crush get-device-class "
+        "name=ids,type=CephString,n=N",
+        "get classes of specified osd(s) <id> [<id>...]",
+        "osd", "r")
+COMMAND("osd crush weight-set ls",
+	"list crush weight sets",
+	"osd", "r")
+COMMAND("osd crush weight-set dump",
+	"dump crush weight sets",
+	"osd", "r")
+COMMAND("osd crush weight-set create-compat",
+	"create a default backward-compatible weight-set",
+	"osd", "rw")
+COMMAND("osd crush weight-set create "
+        "name=pool,type=CephPoolname "\
+        "name=mode,type=CephChoices,strings=flat|positional",
+	"create a weight-set for a given pool",
+	"osd", "rw")
+COMMAND("osd crush weight-set rm name=pool,type=CephPoolname",
+	"remove the weight-set for a given pool",
+	"osd", "rw")
+COMMAND("osd crush weight-set rm-compat",
+	"remove the backward-compatible weight-set",
+	"osd", "rw")
+COMMAND("osd crush weight-set reweight "
+        "name=pool,type=CephPoolname "
+	"name=item,type=CephString "
+        "name=weight,type=CephFloat,range=0.0,n=N",
+	"set weight for an item (bucket or osd) in a pool's weight-set",
+	"osd", "rw")
+COMMAND("osd crush weight-set reweight-compat "
+	"name=item,type=CephString "
+        "name=weight,type=CephFloat,range=0.0,n=N",
+	"set weight for an item (bucket or osd) in the backward-compatible weight-set",
+	"osd", "rw")
+COMMAND("osd setmaxosd "
+	"name=newmax,type=CephInt,range=0",
+	"set new maximum osd value", "osd", "rw")
+COMMAND("osd set-full-ratio "
+	"name=ratio,type=CephFloat,range=0.0|1.0",
+	"set usage ratio at which OSDs are marked full",
+	"osd", "rw")
+COMMAND("osd set-backfillfull-ratio "
+	"name=ratio,type=CephFloat,range=0.0|1.0",
+	"set usage ratio at which OSDs are marked too full to backfill",
+	"osd", "rw")
+COMMAND("osd set-nearfull-ratio "
+	"name=ratio,type=CephFloat,range=0.0|1.0",
+	"set usage ratio at which OSDs are marked near-full",
+	"osd", "rw")
+COMMAND("osd get-require-min-compat-client",
+        "get the minimum client version we will maintain compatibility with",
+        "osd", "r")
+COMMAND("osd set-require-min-compat-client "
+	"name=version,type=CephString "
+        "name=yes_i_really_mean_it,type=CephBool,req=false",
+	"set the minimum client version we will maintain compatibility with",
+	"osd", "rw")
+COMMAND("osd pause", "pause osd", "osd", "rw")
+COMMAND("osd unpause", "unpause osd", "osd", "rw")
+COMMAND("osd erasure-code-profile set "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=profile,type=CephString,n=N,req=false "
+	"name=force,type=CephBool,req=false",
+	"create erasure code profile <name> with [<key[=value]> ...] pairs. Add a --force at the end to override an existing profile (VERY DANGEROUS)",
+	"osd", "rw")
+COMMAND("osd erasure-code-profile get "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.]",
+	"get erasure code profile <name>",
+	"osd", "r")
+COMMAND("osd erasure-code-profile rm "
+	"name=name,type=CephString,goodchars=[A-Za-z0-9-_.]",
+	"remove erasure code profile <name>",
+	"osd", "rw")
+COMMAND("osd erasure-code-profile ls",
+	"list all erasure code profiles",
+	"osd", "r")
+COMMAND("osd set "
+	"name=key,type=CephChoices,strings=full|pause|noup|nodown|"
+	"noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|"
+	"notieragent|nosnaptrim|pglog_hardlimit "
+        "name=yes_i_really_mean_it,type=CephBool,req=false",
+	"set <key>", "osd", "rw")
+COMMAND("osd unset "
+	"name=key,type=CephChoices,strings=full|pause|noup|nodown|"\
+	"noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|"
+	"notieragent|nosnaptrim",
+	"unset <key>", "osd", "rw")
+COMMAND("osd require-osd-release "\
+	"name=release,type=CephChoices,strings=luminous|mimic|nautilus|octopus|pacific "
+        "name=yes_i_really_mean_it,type=CephBool,req=false",
+	"set the minimum allowed OSD release to participate in the cluster",
+	"osd", "rw")
+COMMAND("osd down "
+	"name=ids,type=CephString,n=N "
+	"name=definitely_dead,type=CephBool,req=false",
+	"set osd(s) <id> [<id>...] down, "
+        "or use <any|all> to set all osds down",
+        "osd", "rw")
+COMMAND("osd stop "
+        "type=CephString,name=ids,n=N",
+        "stop the corresponding osd daemons and mark them as down",
+        "osd", "rw")
+COMMAND("osd out "
+	"name=ids,type=CephString,n=N",
+	"set osd(s) <id> [<id>...] out, "
+        "or use <any|all> to set all osds out",
+        "osd", "rw")
+COMMAND("osd in "
+	"name=ids,type=CephString,n=N",
+	"set osd(s) <id> [<id>...] in, "
+        "can use <any|all> to automatically set all previously out osds in",
+        "osd", "rw")
+COMMAND_WITH_FLAG("osd rm "
+	"name=ids,type=CephString,n=N",
+	"remove osd(s) <id> [<id>...], "
+        "or use <any|all> to remove all osds",
+	"osd", "rw",
+	FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("osd add-noup "
+        "name=ids,type=CephString,n=N",
+        "mark osd(s) <id> [<id>...] as noup, "
+        "or use <all|any> to mark all osds as noup",
+        "osd", "rw",
+        FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("osd add-nodown "
+        "name=ids,type=CephString,n=N",
+        "mark osd(s) <id> [<id>...] as nodown, "
+        "or use <all|any> to mark all osds as nodown",
+        "osd", "rw",
+        FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("osd add-noin "
+        "name=ids,type=CephString,n=N",
+        "mark osd(s) <id> [<id>...] as noin, "
+        "or use <all|any> to mark all osds as noin",
+        "osd", "rw",
+        FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("osd add-noout "
+        "name=ids,type=CephString,n=N",
+        "mark osd(s) <id> [<id>...] as noout, "
+        "or use <all|any> to mark all osds as noout",
+        "osd", "rw",
+        FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("osd rm-noup "
+        "name=ids,type=CephString,n=N",
+        "allow osd(s) <id> [<id>...] to be marked up "
+        "(if they are currently marked as noup), "
+        "can use <all|any> to automatically filter out all noup osds",
+        "osd", "rw",
+        FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("osd rm-nodown "
+        "name=ids,type=CephString,n=N",
+        "allow osd(s) <id> [<id>...] to be marked down "
+        "(if they are currently marked as nodown), "
+        "can use <all|any> to automatically filter out all nodown osds",
+        "osd", "rw",
+        FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("osd rm-noin "
+        "name=ids,type=CephString,n=N",
+        "allow osd(s) <id> [<id>...] to be marked in "
+        "(if they are currently marked as noin), "
+        "can use <all|any> to automatically filter out all noin osds",
+        "osd", "rw",
+        FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("osd rm-noout "
+        "name=ids,type=CephString,n=N",
+        "allow osd(s) <id> [<id>...] to be marked out "
+        "(if they are currently marked as noout), "
+        "can use <all|any> to automatically filter out all noout osds",
+        "osd", "rw",
+        FLAG(DEPRECATED))
+COMMAND("osd set-group "
+        "name=flags,type=CephString "
+        "name=who,type=CephString,n=N",
+        "set <flags> for batch osds or crush nodes, "
+        "<flags> must be a comma-separated subset of {noup,nodown,noin,noout}",
+        "osd", "rw")
+COMMAND("osd unset-group "
+        "name=flags,type=CephString "
+        "name=who,type=CephString,n=N",
+        "unset <flags> for batch osds or crush nodes, "
+        "<flags> must be a comma-separated subset of {noup,nodown,noin,noout}",
+        "osd", "rw")
+COMMAND("osd reweight "
+	"name=id,type=CephOsdName "
+	"type=CephFloat,name=weight,range=0.0|1.0",
+	"reweight osd to 0.0 < <weight> < 1.0", "osd", "rw")
+COMMAND("osd reweightn "
+	"name=weights,type=CephString",
+	"reweight osds with {<id>: <weight>,...}",
+	"osd", "rw")
+COMMAND("osd force-create-pg "
+	"name=pgid,type=CephPgid "\
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"force creation of pg <pgid>",
+        "osd", "rw")
+COMMAND("osd pg-temp "
+	"name=pgid,type=CephPgid "
+	"name=id,type=CephOsdName,n=N,req=false",
+	"set pg_temp mapping pgid:[<id> [<id>...]] (developers only)",
+        "osd", "rw")
+COMMAND("osd pg-upmap "
+	"name=pgid,type=CephPgid "
+	"name=id,type=CephOsdName,n=N",
+	"set pg_upmap mapping <pgid>:[<id> [<id>...]] (developers only)",
+        "osd", "rw")
+COMMAND("osd rm-pg-upmap "
+	"name=pgid,type=CephPgid",
+	"clear pg_upmap mapping for <pgid> (developers only)",
+        "osd", "rw")
+COMMAND("osd pg-upmap-items "
+	"name=pgid,type=CephPgid "
+	"name=id,type=CephOsdName,n=N",
+	"set pg_upmap_items mapping <pgid>:{<id> to <id>, [...]} (developers only)",
+        "osd", "rw")
+COMMAND("osd rm-pg-upmap-items "
+	"name=pgid,type=CephPgid",
+	"clear pg_upmap_items mapping for <pgid> (developers only)",
+        "osd", "rw")
+COMMAND("osd primary-temp "
+	"name=pgid,type=CephPgid "
+	"name=id,type=CephOsdName",
+        "set primary_temp mapping pgid:<id>|-1 (developers only)",
+        "osd", "rw")
+COMMAND("osd primary-affinity "
+	"name=id,type=CephOsdName "
+	"type=CephFloat,name=weight,range=0.0|1.0",
+	"adjust osd primary-affinity from 0.0 <= <weight> <= 1.0",
+	"osd", "rw")
+COMMAND_WITH_FLAG("osd destroy-actual "
+        "name=id,type=CephOsdName "
+        "name=yes_i_really_mean_it,type=CephBool,req=false",
+        "mark osd as being destroyed. Keeps the ID intact (allowing reuse), "
+        "but removes cephx keys, config-key data and lockbox keys, "\
+        "rendering data permanently unreadable.",
+		  "osd", "rw", FLAG(HIDDEN))
+COMMAND("osd purge-new "
+        "name=id,type=CephOsdName "
+        "name=yes_i_really_mean_it,type=CephBool,req=false",
+        "purge all traces of an OSD that was partially created but never "
+	"started",
+        "osd", "rw")
+COMMAND_WITH_FLAG("osd purge-actual "
+        "name=id,type=CephOsdName "
+        "name=yes_i_really_mean_it,type=CephBool,req=false",
+        "purge all osd data from the monitors. Combines `osd destroy`, "
+        "`osd rm`, and `osd crush rm`.",
+		  "osd", "rw", FLAG(HIDDEN))
+COMMAND("osd lost "
+	"name=id,type=CephOsdName "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"mark osd as permanently lost. THIS DESTROYS DATA IF NO MORE REPLICAS EXIST, BE CAREFUL",
+	"osd", "rw")
+COMMAND_WITH_FLAG("osd create "
+	"name=uuid,type=CephUUID,req=false "
+	"name=id,type=CephOsdName,req=false",
+	"create new osd (with optional UUID and ID)", "osd", "rw",
+	FLAG(DEPRECATED))
+COMMAND("osd new "
+        "name=uuid,type=CephUUID,req=true "
+        "name=id,type=CephOsdName,req=false",
+        "Create a new OSD. If supplied, the `id` to be replaced needs to "
+        "exist and have been previously destroyed. "
+        "Reads secrets from JSON file via `-i <file>` (see man page).",
+        "osd", "rw")
+COMMAND("osd blocklist "
+	"name=range,type=CephString,goodchars=[range],req=false "
+	"name=blocklistop,type=CephChoices,strings=add|rm "
+	"name=addr,type=CephEntityAddr "
+	"name=expire,type=CephFloat,range=0.0,req=false",
+	"add (optionally until <expire> seconds from now) or remove <addr> from blocklist",
+	"osd", "rw")
+COMMAND("osd blocklist ls", "show blocklisted clients", "osd", "r")
+COMMAND("osd blocklist clear", "clear all blocklisted clients", "osd", "rw")
+
+COMMAND_WITH_FLAG("osd blacklist "
+	"name=blacklistop,type=CephChoices,strings=add|rm "
+	"name=addr,type=CephEntityAddr "
+	"name=expire,type=CephFloat,range=0.0,req=false",
+	"add (optionally until <expire> seconds from now) or remove <addr> from blacklist",
+	"osd", "rw",
+	FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("osd blacklist ls", "show blacklisted clients", "osd", "r",
+	FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("osd blacklist clear", "clear all blacklisted clients", "osd", "rw",
+	FLAG(DEPRECATED))
+
+COMMAND("osd pool mksnap "
+	"name=pool,type=CephPoolname "
+	"name=snap,type=CephString",
+	"make snapshot <snap> in <pool>", "osd", "rw")
+COMMAND("osd pool rmsnap "
+	"name=pool,type=CephPoolname "
+	"name=snap,type=CephString",
+	"remove snapshot <snap> from <pool>", "osd", "rw")
+COMMAND("osd pool ls "
+	"name=detail,type=CephChoices,strings=detail,req=false",
+	"list pools", "osd", "r")
+COMMAND("osd pool create "
+	"name=pool,type=CephPoolname "
+	"name=pg_num,type=CephInt,range=0,req=false "
+	"name=pgp_num,type=CephInt,range=0,req=false "
+        "name=pool_type,type=CephChoices,strings=replicated|erasure,req=false "
+	"name=erasure_code_profile,type=CephString,req=false,goodchars=[A-Za-z0-9-_.] "
+	"name=rule,type=CephString,req=false "
+        "name=expected_num_objects,type=CephInt,range=0,req=false "
+        "name=size,type=CephInt,range=0,req=false "
+	"name=pg_num_min,type=CephInt,range=0,req=false "
+	"name=pg_num_max,type=CephInt,range=0,req=false "
+	"name=autoscale_mode,type=CephChoices,strings=on|off|warn,req=false "
+	"name=bulk,type=CephBool,req=false "
+	"name=target_size_bytes,type=CephInt,range=0,req=false "
+	"name=target_size_ratio,type=CephFloat,range=0.0,req=false",\
+	"create pool", "osd", "rw")
+COMMAND_WITH_FLAG("osd pool delete "
+	"name=pool,type=CephPoolname "
+	"name=pool2,type=CephPoolname,req=false "
+	"name=yes_i_really_really_mean_it,type=CephBool,req=false "
+	"name=yes_i_really_really_mean_it_not_faking,type=CephBool,req=false ",
+	"delete pool",
+	"osd", "rw",
+    FLAG(DEPRECATED))
+COMMAND("osd pool rm "
+	"name=pool,type=CephPoolname "
+	"name=pool2,type=CephPoolname,req=false "
+	"name=yes_i_really_really_mean_it,type=CephBool,req=false "
+	"name=yes_i_really_really_mean_it_not_faking,type=CephBool,req=false ",
+	"remove pool",
+	"osd", "rw")
+COMMAND("osd pool rename "
+	"name=srcpool,type=CephPoolname "
+	"name=destpool,type=CephPoolname",
+	"rename <srcpool> to <destpool>", "osd", "rw")
+COMMAND("osd pool get "
+	"name=pool,type=CephPoolname "
+	"name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_objects|target_max_bytes|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|erasure_code_profile|min_read_recency_for_promote|all|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|pg_num_max|target_size_bytes|target_size_ratio|dedup_tier|dedup_chunk_algorithm|dedup_cdc_chunk_size|bulk",
+	"get pool parameter <var>", "osd", "r")
+COMMAND("osd pool set "
+	"name=pool,type=CephPoolname "
+	"name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|pgp_num_actual|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_bytes|target_max_objects|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|min_read_recency_for_promote|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|pg_num_max|target_size_bytes|target_size_ratio|dedup_tier|dedup_chunk_algorithm|dedup_cdc_chunk_size|bulk "
+	"name=val,type=CephString "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"set pool parameter <var> to <val>", "osd", "rw")
+// 'val' is a CephString because it can include a unit.  Perhaps
+// there should be a Python type for validation/conversion of strings
+// with units.
+COMMAND("osd pool set-quota "
+	"name=pool,type=CephPoolname "
+	"name=field,type=CephChoices,strings=max_objects|max_bytes "
+	"name=val,type=CephString",
+	"set object or byte limit on pool", "osd", "rw")
+COMMAND("osd pool get-quota "
+        "name=pool,type=CephPoolname ",
+        "obtain object or byte limits for pool",
+        "osd", "r")
+COMMAND("osd pool application enable "
+        "name=pool,type=CephPoolname "
+        "name=app,type=CephString,goodchars=[A-Za-z0-9-_.] "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+        "enable use of an application <app> [cephfs,rbd,rgw] on pool <poolname>",
+        "osd", "rw")
+COMMAND("osd pool application disable "
+        "name=pool,type=CephPoolname "
+        "name=app,type=CephString "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+        "disables use of an application <app> on pool <poolname>",
+        "osd", "rw")
+COMMAND("osd pool application set "
+        "name=pool,type=CephPoolname "
+        "name=app,type=CephString "
+        "name=key,type=CephString,goodchars=[A-Za-z0-9-_.] "
+        "name=value,type=CephString,goodchars=[A-Za-z0-9-_.=]",
+        "sets application <app> metadata key <key> to <value> on pool <poolname>",
+        "osd", "rw")
+COMMAND("osd pool application rm "
+        "name=pool,type=CephPoolname "
+        "name=app,type=CephString "
+        "name=key,type=CephString",
+        "removes application <app> metadata key <key> on pool <poolname>",
+        "osd", "rw")
+COMMAND("osd pool application get "
+        "name=pool,type=CephPoolname,req=fasle "
+        "name=app,type=CephString,req=false "
+        "name=key,type=CephString,req=false",
+        "get value of key <key> of application <app> on pool <poolname>",
+        "osd", "r")
+COMMAND("osd utilization",
+	"get basic pg distribution stats",
+	"osd", "r")
+COMMAND("osd force_healthy_stretch_mode " \
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"force a healthy stretch mode, requiring the full number of CRUSH buckets "
+	"to peer and letting all non-tiebreaker monitors be elected leader ",
+	"osd", "rw")
+COMMAND("osd force_recovery_stretch_mode " \
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"try and force a recovery stretch mode, increasing the "
+	"pool size to its non-failure value if currently degraded and "
+	"all monitor buckets are up",
+	"osd", "rw")
+
+
+// tiering
+COMMAND("osd tier add "
+	"name=pool,type=CephPoolname "
+	"name=tierpool,type=CephPoolname "
+	"name=force_nonempty,type=CephChoices,strings=--force-nonempty,req=false",
+	"add the tier <tierpool> (the second one) to base pool <pool> (the first one)",
+	"osd", "rw")
+COMMAND("osd tier rm "
+	"name=pool,type=CephPoolname "
+	"name=tierpool,type=CephPoolname",
+	"remove the tier <tierpool> (the second one) from base pool <pool> (the first one)",
+	"osd", "rw")
+COMMAND_WITH_FLAG("osd tier remove "
+	"name=pool,type=CephPoolname "
+	"name=tierpool,type=CephPoolname",
+	"remove the tier <tierpool> (the second one) from base pool <pool> (the first one)",
+	"osd", "rw",
+    FLAG(DEPRECATED))
+COMMAND("osd tier cache-mode "
+	"name=pool,type=CephPoolname "
+	"name=mode,type=CephChoices,strings=writeback|readproxy|readonly|none "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"specify the caching mode for cache tier <pool>", "osd", "rw")
+COMMAND("osd tier set-overlay "
+	"name=pool,type=CephPoolname "
+	"name=overlaypool,type=CephPoolname",
+	"set the overlay pool for base pool <pool> to be <overlaypool>", "osd", "rw")
+COMMAND("osd tier rm-overlay "
+	"name=pool,type=CephPoolname ",
+	"remove the overlay pool for base pool <pool>", "osd", "rw")
+COMMAND_WITH_FLAG("osd tier remove-overlay "
+	"name=pool,type=CephPoolname ",
+	"remove the overlay pool for base pool <pool>", "osd", "rw",
+    FLAG(DEPRECATED))
+
+COMMAND("osd tier add-cache "
+	"name=pool,type=CephPoolname "
+	"name=tierpool,type=CephPoolname "
+	"name=size,type=CephInt,range=0",
+	"add a cache <tierpool> (the second one) of size <size> to existing pool <pool> (the first one)",
+	"osd", "rw")
+
+/*
+ * mon/KVMonitor.cc
+ */
+
+COMMAND("config-key get "
+	"name=key,type=CephString",
+	"get <key>", "config-key", "r")
+COMMAND("config-key set "
+	"name=key,type=CephString "
+	"name=val,type=CephString,req=false",
+	"set <key> to value <val>", "config-key", "rw")
+COMMAND_WITH_FLAG("config-key put "
+		  "name=key,type=CephString "
+		  "name=val,type=CephString,req=false",
+		  "put <key>, value <val>", "config-key", "rw",
+		  FLAG(DEPRECATED))
+COMMAND_WITH_FLAG("config-key del "
+	"name=key,type=CephString",
+	"delete <key>", "config-key", "rw",
+    FLAG(DEPRECATED))
+COMMAND("config-key rm "
+	"name=key,type=CephString",
+	"rm <key>", "config-key", "rw")
+COMMAND("config-key exists "
+	"name=key,type=CephString",
+	"check for <key>'s existence", "config-key", "r")
+COMMAND_WITH_FLAG("config-key list ", "list keys", "config-key", "r",
+		  FLAG(DEPRECATED))
+COMMAND("config-key ls ", "list keys", "config-key", "r")
+COMMAND("config-key dump "
+	"name=key,type=CephString,req=false", "dump keys and values (with optional prefix)", "config-key", "r")
+
+
+/*
+ * mon/MgrMonitor.cc
+ */
+COMMAND("mgr stat",
+	"dump basic info about the mgr cluster state",
+	"mgr", "r")
+COMMAND("mgr dump "
+	"name=epoch,type=CephInt,range=0,req=false",
+	"dump the latest MgrMap",
+	"mgr", "r")
+COMMAND("mgr fail name=who,type=CephString,req=false",
+	"treat the named manager daemon as failed", "mgr", "rw")
+COMMAND("mgr module ls",
+	"list active mgr modules", "mgr", "r")
+COMMAND("mgr services",
+	"list service endpoints provided by mgr modules",
+        "mgr", "r")
+COMMAND("mgr module enable "
+	"name=module,type=CephString "
+	"name=force,type=CephChoices,strings=--force,req=false",
+	"enable mgr module", "mgr", "rw")
+COMMAND("mgr module disable "
+	"name=module,type=CephString",
+	"disable mgr module", "mgr", "rw")
+COMMAND("mgr metadata name=who,type=CephString,req=false",
+	"dump metadata for all daemons or a specific daemon",
+	"mgr", "r")
+COMMAND("mgr count-metadata name=property,type=CephString",
+	"count ceph-mgr daemons by metadata field property",
+	"mgr", "r")
+COMMAND("mgr versions",
+	"check running versions of ceph-mgr daemons",
+	"mgr", "r")
+
+// ConfigMonitor
+COMMAND("config set"
+	" name=who,type=CephString"
+	" name=name,type=CephString"
+	" name=value,type=CephString"
+	" name=force,type=CephBool,req=false",
+	"Set a configuration option for one or more entities",
+	"config", "rw")
+COMMAND("config rm"
+	" name=who,type=CephString"
+	" name=name,type=CephString",
+	"Clear a configuration option for one or more entities",
+	"config", "rw")
+COMMAND("config get "
+	"name=who,type=CephString "
+	"name=key,type=CephString,req=False",
+	"Show configuration option(s) for an entity",
+	"config", "r")
+COMMAND("config dump",
+	"Show all configuration option(s)",
+	"mon", "r")
+COMMAND("config help "
+	"name=key,type=CephString",
+	"Describe a configuration option",
+	"config", "r")
+COMMAND("config ls",
+	"List available configuration options",
+	"config", "r")
+COMMAND("config assimilate-conf",
+	"Assimilate options from a conf, and return a new, minimal conf file",
+	"config", "rw")
+COMMAND("config log name=num,type=CephInt,req=False",
+	"Show recent history of config changes",
+	"config", "r")
+COMMAND("config reset "
+	"name=num,type=CephInt,range=0",
+	"Revert configuration to a historical version specified by <num>",
+	"config", "rw")
+COMMAND("config generate-minimal-conf",
+	"Generate a minimal ceph.conf file",
+	"config", "r")
+
+
+
+
+// these are tell commands that were implemented as CLI commands in
+// the broken pre-octopus way that we want to allow to work when a
+// monitor has upgraded to octopus+ but the monmap min_mon_release is
+// still < octopus.  we exclude things that weren't well supported
+// before and that aren't implemented by the octopus mon anymore.
+//
+// the command set below matches the kludge in Monitor::handle_command
+// that shunts these off to the asok machinery.
+
+COMMAND_WITH_FLAG("injectargs "
+	    "name=injected_args,type=CephString,n=N",
+	    "inject config arguments into monitor", "mon", "rw",
+            FLAG(TELL))
+COMMAND_WITH_FLAG("smart name=devid,type=CephString,req=false",
+            "Query health metrics for underlying device",
+	    "mon", "rw",
+            FLAG(TELL))
+COMMAND_WITH_FLAG("mon_status",
+	    "report status of monitors",
+	    "mon", "r",
+            FLAG(TELL))
+COMMAND_WITH_FLAG("heap "
+            "name=heapcmd,type=CephChoices,strings=dump|start_profiler|stop_profiler|release|stats "
+            "name=value,type=CephString,req=false",
+            "show heap usage info (available only if compiled with tcmalloc)",
+	    "mon", "rw",
+            FLAG(TELL))
+COMMAND_WITH_FLAG("connection scores dump",
+		  "show the scores used in connectivity-based elections",
+		  "mon", "rwx",
+		  FLAG(TELL))
+COMMAND_WITH_FLAG("connection scores reset",
+		  "reset the scores used in connectivity-based elections",
+		  "mon", "rwx",
+		  FLAG(TELL))
+COMMAND_WITH_FLAG("sync_force "
+            "name=validate,type=CephChoices,strings=--yes-i-really-mean-it,req=false",
+            "force sync of and clear monitor store",
+            "mon", "rw",
+            FLAG(TELL))
+COMMAND_WITH_FLAG("add_bootstrap_peer_hint "
+            "name=addr,type=CephIPAddr",
+            "add peer address as potential bootstrap "
+            "peer for cluster bringup",
+            "mon", "rw",
+            FLAG(TELL))
+COMMAND_WITH_FLAG("add_bootstrap_peer_hintv "
+            "name=addrv,type=CephString",
+            "add peer address vector as potential bootstrap "
+            "peer for cluster bringup",
+            "mon", "rw",
+            FLAG(TELL))
+COMMAND_WITH_FLAG("quorum enter ",
+            "force monitor back into quorum",
+            "mon", "rw",
+            FLAG(TELL))
+COMMAND_WITH_FLAG("quorum exit",
+            "force monitor out of the quorum",
+            "mon", "rw",
+            FLAG(TELL))
+COMMAND_WITH_FLAG("ops",
+            "show the ops currently in flight",
+            "mon", "r",
+            FLAG(TELL))
+COMMAND_WITH_FLAG("sessions",
+            "list existing sessions",
+            "mon", "r",
+            FLAG(TELL))
+COMMAND_WITH_FLAG("dump_historic_ops",
+            "dump_historic_ops",
+            "mon", "r",
+            FLAG(TELL))
diff --git a/src/mon/MonMap.cc b/src/mon/MonMap.cc
new file mode 100644
index 000000000..2d14578a6
--- /dev/null
+++ b/src/mon/MonMap.cc
@@ -0,0 +1,972 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "MonMap.h"
+
+#include <algorithm>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#ifdef WITH_SEASTAR
+#include <seastar/core/fstream.hh>
+#include <seastar/core/reactor.hh>
+#include <seastar/net/dns.hh>
+#include "crimson/common/config_proxy.h"
+#endif
+
+#include "common/Formatter.h"
+
+#include "include/ceph_features.h"
+#include "include/addr_parsing.h"
+#include "common/ceph_argparse.h"
+#include "common/dns_resolve.h"
+#include "common/errno.h"
+#include "common/dout.h"
+#include "common/Clock.h"
+#include "mon/health_check.h"
+
+using std::list;
+using std::map;
+using std::ostream;
+using std::set;
+using std::string;
+using std::vector;
+
+using ceph::DNSResolver;
+using ceph::Formatter;
+
+void mon_info_t::encode(ceph::buffer::list& bl, uint64_t features) const
+{
+  uint8_t v = 5;
+  uint8_t min_v = 1;
+  if (!crush_loc.empty()) {
+    // we added crush_loc in version 5, but need to let old clients decode it
+    // so just leave the min_v at version 1. Monitors are protected
+    // from misunderstandings about location because setting it is blocked
+    // on FEATURE_PINGING
+    min_v = 1;
+  }
+  if (!HAVE_FEATURE(features, SERVER_NAUTILUS)) {
+    v = 2;
+  }
+  ENCODE_START(v, min_v, bl);
+  encode(name, bl);
+  if (v < 3) {
+    ceph_assert(min_v == 1);
+    auto a = public_addrs.legacy_addr();
+    if (a != entity_addr_t()) {
+      encode(a, bl, features);
+    } else {
+      // note: we don't have a legacy addr here, so lie so that it looks
+      // like one, just so that old clients get a valid-looking map.
+      // they won't be able to talk to the v2 mons, but that's better
+      // than nothing.
+      encode(public_addrs.as_legacy_addr(), bl, features);
+    }
+  } else {
+    encode(public_addrs, bl, features);
+  }
+  encode(priority, bl);
+  encode(weight, bl);
+  encode(crush_loc, bl);
+  ENCODE_FINISH(bl);
+}
+
+void mon_info_t::decode(ceph::buffer::list::const_iterator& p)
+{
+  DECODE_START(5, p);
+  decode(name, p);
+  decode(public_addrs, p);
+  if (struct_v >= 2) {
+    decode(priority, p);
+  }
+  if (struct_v >= 4) {
+    decode(weight, p);
+  }
+  if (struct_v >= 5) {
+    decode(crush_loc, p);
+  }
+  DECODE_FINISH(p);
+}
+
+void mon_info_t::print(ostream& out) const
+{
+  out << "mon." << name
+      << " addrs " << public_addrs
+      << " priority " << priority
+      << " weight " << weight
+      << " crush location " << crush_loc;
+}
+
+namespace {
+  struct rank_cmp {
+    bool operator()(const mon_info_t &a, const mon_info_t &b) const {
+      if (a.public_addrs.legacy_or_front_addr() == b.public_addrs.legacy_or_front_addr())
+        return a.name < b.name;
+      return a.public_addrs.legacy_or_front_addr() < b.public_addrs.legacy_or_front_addr();
+    }
+  };
+}
+
+void MonMap::calc_legacy_ranks()
+{
+  ranks.resize(mon_info.size());
+
+  // Used to order entries according to public_addr, because that's
+  // how the ranks are expected to be ordered by. We may expand this
+  // later on, according to some other criteria, by specifying a
+  // different comparator.
+  //
+  // Please note that we use a 'set' here instead of resorting to
+  // std::sort() because we need more info than that's available in
+  // the vector. The vector will thus be ordered by, e.g., public_addr
+  // while only containing the names of each individual monitor.
+  // The only way of achieving this with std::sort() would be to first
+  // insert every mon_info_t entry into a vector 'foo', std::sort() 'foo'
+  // with custom comparison functions, and then copy each invidual entry
+  // to a new vector. Unless there's a simpler way, we don't think the
+  // added complexity makes up for the additional memory usage of a 'set'.
+  set<mon_info_t, rank_cmp> tmp;
+
+  for (auto p = mon_info.begin(); p != mon_info.end(); ++p) {
+    mon_info_t &m = p->second;
+    tmp.insert(m);
+  }
+
+  // map the set to the actual ranks etc
+  unsigned i = 0;
+  for (auto p = tmp.begin(); p != tmp.end(); ++p, ++i) {
+    ranks[i] = p->name;
+  }
+}
+
+void MonMap::encode(ceph::buffer::list& blist, uint64_t con_features) const
+{
+  if ((con_features & CEPH_FEATURE_MONNAMES) == 0) {
+    using ceph::encode;
+    __u16 v = 1;
+    encode(v, blist);
+    ceph::encode_raw(fsid, blist);
+    encode(epoch, blist);
+    vector<entity_inst_t> mon_inst(ranks.size());
+    for (unsigned n = 0; n < ranks.size(); n++) {
+      mon_inst[n].name = entity_name_t::MON(n);
+      mon_inst[n].addr = get_addrs(n).legacy_addr();
+    }
+    encode(mon_inst, blist, con_features);
+    encode(last_changed, blist);
+    encode(created, blist);
+    return;
+  }
+
+  map<string,entity_addr_t> legacy_mon_addr;
+  if (!HAVE_FEATURE(con_features, MONENC) ||
+      !HAVE_FEATURE(con_features, SERVER_NAUTILUS)) {
+    for (auto& [name, info] : mon_info) {
+      legacy_mon_addr[name] = info.public_addrs.legacy_addr();
+    }
+  }
+
+  if (!HAVE_FEATURE(con_features, MONENC)) {
+    /* we keep the mon_addr map when encoding to ensure compatibility
+       * with clients and other monitors that do not yet support the 'mons'
+       * map. This map keeps its original behavior, containing a mapping of
+       * monitor id (i.e., 'foo' in 'mon.foo') to the monitor's public
+       * address -- which is obtained from the public address of each entry
+       * in the 'mons' map.
+       */
+    using ceph::encode;
+    __u16 v = 2;
+    encode(v, blist);
+    ceph::encode_raw(fsid, blist);
+    encode(epoch, blist);
+    encode(legacy_mon_addr, blist, con_features);
+    encode(last_changed, blist);
+    encode(created, blist);
+    return;
+  }
+
+  if (!HAVE_FEATURE(con_features, SERVER_NAUTILUS)) {
+    ENCODE_START(5, 3, blist);
+    ceph::encode_raw(fsid, blist);
+    encode(epoch, blist);
+    encode(legacy_mon_addr, blist, con_features);
+    encode(last_changed, blist);
+    encode(created, blist);
+    encode(persistent_features, blist);
+    encode(optional_features, blist);
+    encode(mon_info, blist, con_features);
+    ENCODE_FINISH(blist);
+    return;
+  }
+
+  ENCODE_START(9, 6, blist);
+  ceph::encode_raw(fsid, blist);
+  encode(epoch, blist);
+  encode(last_changed, blist);
+  encode(created, blist);
+  encode(persistent_features, blist);
+  encode(optional_features, blist);
+  encode(mon_info, blist, con_features);
+  encode(ranks, blist);
+  encode(min_mon_release, blist);
+  encode(removed_ranks, blist);
+  uint8_t t = strategy;
+  encode(t, blist);
+  encode(disallowed_leaders, blist);
+  encode(stretch_mode_enabled, blist);
+  encode(tiebreaker_mon, blist);
+  encode(stretch_marked_down_mons, blist);
+  ENCODE_FINISH(blist);
+}
+
+void MonMap::decode(ceph::buffer::list::const_iterator& p)
+{
+  map<string,entity_addr_t> mon_addr;
+  DECODE_START_LEGACY_COMPAT_LEN_16(9, 3, 3, p);
+  ceph::decode_raw(fsid, p);
+  decode(epoch, p);
+  if (struct_v == 1) {
+    vector<entity_inst_t> mon_inst;
+    decode(mon_inst, p);
+    for (unsigned i = 0; i < mon_inst.size(); i++) {
+      char n[2];
+      n[0] = '0' + i;
+      n[1] = 0;
+      string name = n;
+      mon_addr[name] = mon_inst[i].addr;
+    }
+  } else if (struct_v < 6) {
+    decode(mon_addr, p);
+  }
+  decode(last_changed, p);
+  decode(created, p);
+  if (struct_v >= 4) {
+    decode(persistent_features, p);
+    decode(optional_features, p);
+  }
+  if (struct_v < 5) {
+    // generate mon_info from legacy mon_addr
+    for (auto& [name, addr] : mon_addr) {
+      mon_info_t &m = mon_info[name];
+      m.name = name;
+      m.public_addrs = entity_addrvec_t(addr);
+    }
+  } else {
+    decode(mon_info, p);
+  }
+  if (struct_v < 6) {
+    calc_legacy_ranks();
+  } else {
+    decode(ranks, p);
+  }
+  if (struct_v >= 7) {
+    decode(min_mon_release, p);
+  } else {
+    min_mon_release = infer_ceph_release_from_mon_features(persistent_features);
+  }
+  if (struct_v >= 8) {
+    decode(removed_ranks, p);
+    uint8_t t;
+    decode(t, p);
+    strategy = static_cast<election_strategy>(t);
+    decode(disallowed_leaders, p);
+  }
+  if (struct_v >= 9) {
+    decode(stretch_mode_enabled, p);
+    decode(tiebreaker_mon, p);
+    decode(stretch_marked_down_mons, p);
+  } else {
+    stretch_mode_enabled = false;
+    tiebreaker_mon = "";
+    stretch_marked_down_mons.clear();
+  }
+  calc_addr_mons();
+  DECODE_FINISH(p);
+}
+
+void MonMap::generate_test_instances(list<MonMap*>& o)
+{
+  o.push_back(new MonMap);
+  o.push_back(new MonMap);
+  o.back()->epoch = 1;
+  o.back()->last_changed = utime_t(123, 456);
+  o.back()->created = utime_t(789, 101112);
+  o.back()->add("one", entity_addrvec_t());
+
+  MonMap *m = new MonMap;
+  {
+    m->epoch = 1;
+    m->last_changed = utime_t(123, 456);
+
+    entity_addrvec_t empty_addr_one = entity_addrvec_t(entity_addr_t());
+    empty_addr_one.v[0].set_nonce(1);
+    m->add("empty_addr_one", empty_addr_one);
+    entity_addrvec_t empty_addr_two = entity_addrvec_t(entity_addr_t());
+    empty_addr_two.v[0].set_nonce(2);
+    m->add("empty_addr_two", empty_addr_two);
+
+    const char *local_pub_addr_s = "127.0.1.2";
+
+    const char *end_p = local_pub_addr_s + strlen(local_pub_addr_s);
+    entity_addrvec_t local_pub_addr;
+    local_pub_addr.parse(local_pub_addr_s, &end_p);
+
+    m->add(mon_info_t("filled_pub_addr", entity_addrvec_t(local_pub_addr), 1, 1));
+
+    m->add("empty_addr_zero", entity_addrvec_t());
+  }
+  o.push_back(m);
+}
+
+// read from/write to a file
+int MonMap::write(const char *fn) 
+{
+  // encode
+  ceph::buffer::list bl;
+  encode(bl, CEPH_FEATURES_ALL);
+  
+  return bl.write_file(fn);
+}
+
+int MonMap::read(const char *fn) 
+{
+  // read
+  ceph::buffer::list bl;
+  std::string error;
+  int r = bl.read_file(fn, &error);
+  if (r < 0)
+    return r;
+  decode(bl);
+  return 0;
+}
+
+void MonMap::print_summary(ostream& out) const
+{
+  out << "e" << epoch << ": "
+      << mon_info.size() << " mons at {";
+  // the map that we used to print, as it was, no longer
+  // maps strings to the monitor's public address, but to
+  // mon_info_t instead. As such, print the map in a way
+  // that keeps the expected format.
+  bool has_printed = false;
+  for (auto p = mon_info.begin(); p != mon_info.end(); ++p) {
+    if (has_printed)
+      out << ",";
+    out << p->first << "=" << p->second.public_addrs;
+    has_printed = true;
+  }
+  out << "}" << " removed_ranks: {" << removed_ranks << "}";
+}
+ 
+void MonMap::print(ostream& out) const
+{
+  out << "epoch " << epoch << "\n";
+  out << "fsid " << fsid << "\n";
+  out << "last_changed " << last_changed << "\n";
+  out << "created " << created << "\n";
+  out << "min_mon_release " << to_integer<unsigned>(min_mon_release)
+      << " (" << min_mon_release << ")\n";
+  out << "election_strategy: " << strategy << "\n";
+  if (stretch_mode_enabled) {
+    out << "stretch_mode_enabled " << stretch_mode_enabled << "\n";
+    out << "tiebreaker_mon " << tiebreaker_mon << "\n";
+  }
+  if (stretch_mode_enabled ||
+      !disallowed_leaders.empty()) {
+    out << "disallowed_leaders " << disallowed_leaders << "\n";
+  }
+  unsigned i = 0;
+  for (auto p = ranks.begin(); p != ranks.end(); ++p) {
+    const auto &mi = mon_info.find(*p);
+    ceph_assert(mi != mon_info.end());
+    out << i++ << ": " << mi->second.public_addrs << " mon." << *p;
+    if (!mi->second.crush_loc.empty()) {
+      out << "; crush_location " << mi->second.crush_loc;
+    }
+    out << "\n";
+  }
+}
+
+void MonMap::dump(Formatter *f) const
+{
+  f->dump_unsigned("epoch", epoch);
+  f->dump_stream("fsid") <<  fsid;
+  last_changed.gmtime(f->dump_stream("modified"));
+  created.gmtime(f->dump_stream("created"));
+  f->dump_unsigned("min_mon_release", to_integer<unsigned>(min_mon_release));
+  f->dump_string("min_mon_release_name", to_string(min_mon_release));
+  f->dump_int ("election_strategy", strategy);
+  f->dump_stream("disallowed_leaders: ") << disallowed_leaders;
+  f->dump_bool("stretch_mode", stretch_mode_enabled);
+  f->dump_string("tiebreaker_mon", tiebreaker_mon);
+  f->dump_stream("removed_ranks: ") << removed_ranks;
+  f->open_object_section("features");
+  persistent_features.dump(f, "persistent");
+  optional_features.dump(f, "optional");
+  f->close_section();
+  f->open_array_section("mons");
+  int i = 0;
+  for (auto p = ranks.begin(); p != ranks.end(); ++p, ++i) {
+    f->open_object_section("mon");
+    f->dump_int("rank", i);
+    f->dump_string("name", *p);
+    f->dump_object("public_addrs", get_addrs(*p));
+    // compat: make these look like pre-nautilus entity_addr_t
+    f->dump_stream("addr") << get_addrs(*p).get_legacy_str();
+    f->dump_stream("public_addr") << get_addrs(*p).get_legacy_str();
+    f->dump_unsigned("priority", get_priority(*p));
+    f->dump_unsigned("weight", get_weight(*p));
+    const auto &mi = mon_info.find(*p);
+    // we don't need to assert this validity as all the get_* functions did
+    f->dump_stream("crush_location") << mi->second.crush_loc;
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void MonMap::dump_summary(Formatter *f) const
+{
+  f->dump_unsigned("epoch", epoch);
+  f->dump_string("min_mon_release_name", to_string(min_mon_release));
+  f->dump_unsigned("num_mons", ranks.size());
+}
+
+// an ambiguous mon addr may be legacy or may be msgr2--we aren' sure.
+// when that happens we need to try them both (unless we can
+// reasonably infer from the port number which it is).
+void MonMap::_add_ambiguous_addr(const string& name,
+                                 entity_addr_t addr,
+                                 int priority,
+                                 int weight,
+                                 bool for_mkfs)
+{
+  if (addr.get_type() != entity_addr_t::TYPE_ANY) {
+    // a v1: or v2: prefix was specified
+    if (addr.get_port() == 0) {
+      // use default port
+      if (addr.get_type() == entity_addr_t::TYPE_LEGACY) {
+	addr.set_port(CEPH_MON_PORT_LEGACY);
+      } else if (addr.get_type() == entity_addr_t::TYPE_MSGR2) {
+	addr.set_port(CEPH_MON_PORT_IANA);
+      } else {
+	// wth
+	return;
+      }
+      if (!contains(addr)) {
+	add(name, entity_addrvec_t(addr), priority, weight);
+      }
+    } else {
+      if (!contains(addr)) {
+	add(name, entity_addrvec_t(addr), priority, weight);
+      }
+    }
+  } else {
+    // no v1: or v2: prefix specified
+    if (addr.get_port() == CEPH_MON_PORT_LEGACY) {
+      // legacy port implies legacy addr
+      addr.set_type(entity_addr_t::TYPE_LEGACY);
+      if (!contains(addr)) {
+	if (!for_mkfs) {
+	  add(name + "-legacy", entity_addrvec_t(addr), priority, weight);
+	} else {
+	  add(name, entity_addrvec_t(addr), priority, weight);
+	}
+      }
+    } else if (addr.get_port() == CEPH_MON_PORT_IANA) {
+      // iana port implies msgr2 addr
+      addr.set_type(entity_addr_t::TYPE_MSGR2);
+      if (!contains(addr)) {
+	add(name, entity_addrvec_t(addr), priority, weight);
+      }
+    } else if (addr.get_port() == 0) {
+      // no port; include both msgr2 and legacy ports
+      if (!for_mkfs) {
+	addr.set_type(entity_addr_t::TYPE_MSGR2);
+	addr.set_port(CEPH_MON_PORT_IANA);
+	if (!contains(addr)) {
+	  add(name, entity_addrvec_t(addr), priority, weight);
+	}
+	addr.set_type(entity_addr_t::TYPE_LEGACY);
+	addr.set_port(CEPH_MON_PORT_LEGACY);
+	if (!contains(addr)) {
+	  add(name + "-legacy", entity_addrvec_t(addr), priority, weight);
+	}
+      } else {
+	entity_addrvec_t av;
+	addr.set_type(entity_addr_t::TYPE_MSGR2);
+	addr.set_port(CEPH_MON_PORT_IANA);
+	av.v.push_back(addr);
+	addr.set_type(entity_addr_t::TYPE_LEGACY);
+	addr.set_port(CEPH_MON_PORT_LEGACY);
+	av.v.push_back(addr);
+	if (!contains(av)) {
+	  add(name, av, priority, weight);
+	}
+      }
+    } else {
+      addr.set_type(entity_addr_t::TYPE_MSGR2);
+      if (!contains(addr)) {
+	add(name, entity_addrvec_t(addr), priority, weight);
+      }
+      if (!for_mkfs) {
+	// try legacy on same port too
+	addr.set_type(entity_addr_t::TYPE_LEGACY);
+	if (!contains(addr)) {
+	  add(name + "-legacy", entity_addrvec_t(addr), priority, weight);
+	}
+      }
+    }
+  }
+}
+
+void MonMap::init_with_addrs(const std::vector<entity_addrvec_t>& addrs,
+                             bool for_mkfs,
+                             std::string_view prefix)
+{
+  char id = 'a';
+  for (auto& addr : addrs) {
+    string name{prefix};
+    name += id++;
+    if (addr.v.size() == 1) {
+      _add_ambiguous_addr(name, addr.front(), 0, 0, for_mkfs);
+    } else {
+      // they specified an addrvec, so let's assume they also specified
+      // the addr *type* and *port*.  (we could possibly improve this?)
+      add(name, addr, 0);
+    }
+  }
+}
+
+int MonMap::init_with_ips(const std::string& ips,
+			  bool for_mkfs,
+			  std::string_view prefix)
+{
+  vector<entity_addrvec_t> addrs;
+  if (!parse_ip_port_vec(
+	ips.c_str(), addrs,
+	entity_addr_t::TYPE_ANY)) {
+    return -EINVAL;
+  }
+  if (addrs.empty())
+    return -ENOENT;
+  init_with_addrs(addrs, for_mkfs, prefix);
+  return 0;
+}
+
+int MonMap::init_with_hosts(const std::string& hostlist,
+			    bool for_mkfs,
+			    std::string_view prefix)
+{
+  // maybe they passed us a DNS-resolvable name
+  char *hosts = resolve_addrs(hostlist.c_str());
+  if (!hosts)
+    return -EINVAL;
+
+  vector<entity_addrvec_t> addrs;
+  bool success = parse_ip_port_vec(
+    hosts, addrs,
+    entity_addr_t::TYPE_ANY);
+  free(hosts);
+  if (!success)
+    return -EINVAL;
+  if (addrs.empty())
+    return -ENOENT;
+  init_with_addrs(addrs, for_mkfs, prefix);
+  calc_legacy_ranks();
+  return 0;
+}
+
+void MonMap::set_initial_members(CephContext *cct,
+				 list<std::string>& initial_members,
+				 string my_name,
+				 const entity_addrvec_t& my_addrs,
+				 set<entity_addrvec_t> *removed)
+{
+  // remove non-initial members
+  unsigned i = 0;
+  while (i < size()) {
+    string n = get_name(i);
+    if (std::find(initial_members.begin(), initial_members.end(), n)
+	!= initial_members.end()) {
+      lgeneric_dout(cct, 1) << " keeping " << n << " " << get_addrs(i) << dendl;
+      i++;
+      continue;
+    }
+
+    lgeneric_dout(cct, 1) << " removing " << get_name(i) << " " << get_addrs(i)
+			  << dendl;
+    if (removed) {
+      removed->insert(get_addrs(i));
+    }
+    remove(n);
+    ceph_assert(!contains(n));
+  }
+
+  // add missing initial members
+  for (auto& p : initial_members) {
+    if (!contains(p)) {
+      if (p == my_name) {
+	lgeneric_dout(cct, 1) << " adding self " << p << " " << my_addrs
+			      << dendl;
+	add(p, my_addrs);
+      } else {
+	entity_addr_t a;
+	a.set_type(entity_addr_t::TYPE_LEGACY);
+	a.set_family(AF_INET);
+	for (int n=1; ; n++) {
+	  a.set_nonce(n);
+	  if (!contains(a))
+	    break;
+	}
+	lgeneric_dout(cct, 1) << " adding " << p << " " << a << dendl;
+	add(p, entity_addrvec_t(a));
+      }
+      ceph_assert(contains(p));
+    }
+  }
+  calc_legacy_ranks();
+}
+
+int MonMap::init_with_config_file(const ConfigProxy& conf,
+                                  std::ostream& errout)
+{
+  std::vector<std::string> sections;
+  int ret = conf.get_all_sections(sections);
+  if (ret) {
+    errout << "Unable to find any monitors in the configuration "
+         << "file, because there was an error listing the sections. error "
+	 << ret << std::endl;
+    return -ENOENT;
+  }
+  std::vector<std::string> mon_names;
+  for (const auto& section : sections) {
+    if (section.substr(0, 4) == "mon." && section.size() > 4) {
+      mon_names.push_back(section.substr(4));
+    }
+  }
+
+  // Find an address for each monitor in the config file.
+  for (const auto& mon_name : mon_names) {
+    std::vector<std::string> sections;
+    std::string m_name("mon");
+    m_name += ".";
+    m_name += mon_name;
+    sections.push_back(m_name);
+    sections.push_back("mon");
+    sections.push_back("global");
+    std::string val;
+    int res = conf.get_val_from_conf_file(sections, "mon addr", val, true);
+    if (res) {
+      errout << "failed to get an address for mon." << mon_name
+             << ": error " << res << std::endl;
+      continue;
+    }
+    // the 'mon addr' field is a legacy field, so assume anything
+    // there on a weird port is a v1 address, and do not handle
+    // addrvecs.
+    entity_addr_t addr;
+    if (!addr.parse(val.c_str(), nullptr, entity_addr_t::TYPE_LEGACY)) {
+      errout << "unable to parse address for mon." << mon_name
+             << ": addr='" << val << "'" << std::endl;
+      continue;
+    }
+    if (addr.get_port() == 0) {
+      addr.set_port(CEPH_MON_PORT_LEGACY);
+    }
+    uint16_t priority = 0;
+    if (!conf.get_val_from_conf_file(sections, "mon priority", val, false)) {
+      try {
+        priority = std::stoul(val);
+      } catch (std::logic_error&) {
+        errout << "unable to parse priority for mon." << mon_name
+               << ": priority='" << val << "'" << std::endl;
+        continue;
+      }
+    }
+    uint16_t weight = 0;
+    if (!conf.get_val_from_conf_file(sections, "mon weight", val, false)) {
+      try {
+        weight = std::stoul(val);
+      } catch (std::logic_error&) {
+        errout << "unable to parse weight for mon." << mon_name
+               << ": weight='" << val << "'"
+               << std::endl;
+        continue;
+      }
+    }
+
+    // make sure this mon isn't already in the map
+    if (contains(addr))
+      remove(get_name(addr));
+    if (contains(mon_name))
+      remove(mon_name);
+    _add_ambiguous_addr(mon_name, addr, priority, weight, false);
+  }
+  return 0;
+}
+
+void MonMap::check_health(health_check_map_t *checks) const
+{
+  if (stretch_mode_enabled) {
+    list<string> detail;
+    for (auto& p : mon_info) {
+      if (p.second.crush_loc.empty()) {
+	ostringstream ss;
+	ss << "mon " << p.first << " has no location set while in stretch mode";
+	detail.push_back(ss.str());
+      }
+    }
+    if (!detail.empty()) {
+      ostringstream ss;
+      ss << detail.size() << " monitor(s) have no location set while in stretch mode"
+	 << "; this may cause issues with failover, OSD connections, netsplit handling, etc";
+      auto& d = checks->add("MON_LOCATION_NOT_SET", HEALTH_WARN,
+			    ss.str(), detail.size());
+      d.detail.swap(detail);
+    }
+  }
+}
+
+#ifdef WITH_SEASTAR
+
+using namespace seastar;
+
+seastar::future<> MonMap::read_monmap(const std::string& monmap)
+{
+  return open_file_dma(monmap, open_flags::ro).then([this] (file f) {
+    return f.size().then([this, f = std::move(f)](size_t s) {
+      return do_with(make_file_input_stream(f), [this, s](input_stream<char>& in) {
+        return in.read_exactly(s).then([this](temporary_buffer<char> buf) {
+          ceph::buffer::list bl;
+          bl.push_back(ceph::buffer::ptr_node::create(
+            ceph::buffer::create(std::move(buf))));
+          decode(bl);
+        });
+      });
+    });
+  });
+}
+
+seastar::future<> MonMap::init_with_dns_srv(bool for_mkfs, const std::string& name)
+{
+  string domain;
+  string service = name;
+  // check if domain is also provided and extract it from srv_name
+  size_t idx = name.find("_");
+  if (idx != name.npos) {
+    domain = name.substr(idx + 1);
+    service = name.substr(0, idx);
+  }
+  return seastar::net::dns::get_srv_records(
+      seastar::net::dns_resolver::srv_proto::tcp,
+      service, domain).then([this](seastar::net::dns_resolver::srv_records records) {
+    return parallel_for_each(records, [this](auto record) {
+      return seastar::net::dns::resolve_name(record.target).then(
+          [record,this](seastar::net::inet_address a) {
+	// the resolved address does not contain ceph specific info like nonce
+	// nonce or msgr proto (legacy, msgr2), so set entity_addr_t manually
+	entity_addr_t addr;
+	addr.set_type(entity_addr_t::TYPE_ANY);
+	addr.set_family(int(a.in_family()));
+	addr.set_port(record.port);
+	switch (a.in_family()) {
+	case seastar::net::inet_address::family::INET:
+	  addr.in4_addr().sin_addr = a;
+	  break;
+	case seastar::net::inet_address::family::INET6:
+	  addr.in6_addr().sin6_addr = a;
+	  break;
+	}
+        _add_ambiguous_addr(record.target,
+                            addr,
+                            record.priority,
+                            record.weight,
+                            false);
+      });
+    });
+  }).handle_exception_type([](const std::system_error& e) {
+    // ignore DNS failures
+    return seastar::make_ready_future<>();
+  });
+}
+
+seastar::future<> MonMap::build_monmap(const crimson::common::ConfigProxy& conf,
+				       bool for_mkfs)
+{
+  // -m foo?
+  if (const auto mon_host = conf.get_val<std::string>("mon_host");
+      !mon_host.empty()) {
+    if (auto ret = init_with_ips(mon_host, for_mkfs, "noname-"); ret == 0) {
+      return make_ready_future<>();
+    }
+    // TODO: resolve_addrs() is a blocking call
+    if (auto ret = init_with_hosts(mon_host, for_mkfs, "noname-"); ret == 0) {
+      return make_ready_future<>();
+    } else {
+      throw std::runtime_error(cpp_strerror(ret));
+    }
+  }
+
+  // What monitors are in the config file?
+  ostringstream errout;
+  if (auto ret = init_with_config_file(conf, errout); ret < 0) {
+    throw std::runtime_error(errout.str());
+  }
+  if (size() > 0) {
+    return make_ready_future<>();
+  }
+  // no info found from conf options lets try use DNS SRV records
+  const string srv_name = conf.get_val<std::string>("mon_dns_srv_name");
+  return init_with_dns_srv(for_mkfs, srv_name).then([this] {
+    if (size() == 0) {
+      throw std::runtime_error("no monitors specified to connect to.");
+    }
+  });
+}
+
+seastar::future<> MonMap::build_initial(const crimson::common::ConfigProxy& conf, bool for_mkfs)
+{
+  // file?
+  if (const auto monmap = conf.get_val<std::string>("monmap");
+      !monmap.empty()) {
+    return read_monmap(monmap);
+  } else {
+    // fsid from conf?
+    if (const auto new_fsid = conf.get_val<uuid_d>("fsid");
+        !new_fsid.is_zero()) {
+      fsid = new_fsid;
+    }
+    return build_monmap(conf, for_mkfs).then([this] {
+      created = ceph_clock_now();
+      last_changed = created;
+      calc_legacy_ranks();
+    });
+  }
+}
+
+#else  // WITH_SEASTAR
+
+int MonMap::init_with_monmap(const std::string& monmap, std::ostream& errout)
+{
+  int r;
+  try {
+    r = read(monmap.c_str());
+  } catch (ceph::buffer::error&) {
+    r = -EINVAL;
+  }
+  if (r >= 0)
+    return 0;
+  errout << "unable to read/decode monmap from " << monmap
+         << ": " << cpp_strerror(-r) << std::endl;
+  return r;
+}
+
+int MonMap::init_with_dns_srv(CephContext* cct,
+                              std::string srv_name,
+			      bool for_mkfs,
+                              std::ostream& errout)
+{
+  string domain;
+  // check if domain is also provided and extract it from srv_name
+  size_t idx = srv_name.find("_");
+  if (idx != string::npos) {
+    domain = srv_name.substr(idx + 1);
+    srv_name = srv_name.substr(0, idx);
+  }
+
+  map<string, DNSResolver::Record> records;
+  if (DNSResolver::get_instance()->resolve_srv_hosts(cct, srv_name,
+        DNSResolver::SRV_Protocol::TCP, domain, &records) != 0) {
+
+    errout << "unable to get monitor info from DNS SRV with service name: "
+           << "ceph-mon" << std::endl;
+    return -1;
+  } else {
+    for (auto& record : records) {
+      record.second.addr.set_type(entity_addr_t::TYPE_ANY);
+      _add_ambiguous_addr(record.first,
+                          record.second.addr,
+                          record.second.priority,
+                          record.second.weight,
+                          false);
+    }
+    return 0;
+  }
+}
+
+int MonMap::build_initial(CephContext *cct, bool for_mkfs, ostream& errout)
+{
+  const auto& conf = cct->_conf;
+
+  // mon_host_override?
+  auto mon_host_override = conf.get_val<std::string>("mon_host_override");
+  if (!mon_host_override.empty()) {
+    lgeneric_dout(cct, 1) << "Using mon_host_override " << mon_host_override << dendl;
+    auto ret = init_with_ips(mon_host_override, for_mkfs, "noname-");
+    if (ret == -EINVAL) {
+      ret = init_with_hosts(mon_host_override, for_mkfs, "noname-");
+    }
+    if (ret < 0) {
+      errout << "unable to parse addrs in '" << mon_host_override << "'"
+	     << std::endl;
+    }
+    return ret;
+  }
+
+  // cct?
+  auto addrs = cct->get_mon_addrs();
+  if (addrs != nullptr && (addrs->size() > 0)) {
+    init_with_addrs(*addrs, for_mkfs, "noname-");
+    return 0;
+  }
+
+  // file?
+  if (const auto monmap = conf.get_val<std::string>("monmap");
+      !monmap.empty()) {
+    return init_with_monmap(monmap, errout);
+  }
+
+  // fsid from conf?
+  if (const auto new_fsid = conf.get_val<uuid_d>("fsid");
+      !new_fsid.is_zero()) {
+    fsid = new_fsid;
+  }
+  // -m foo?
+  if (const auto mon_host = conf.get_val<std::string>("mon_host");
+      !mon_host.empty()) {
+    auto ret = init_with_ips(mon_host, for_mkfs, "noname-");
+    if (ret == -EINVAL) {
+      ret = init_with_hosts(mon_host, for_mkfs, "noname-");
+    }
+    if (ret < 0) {
+      errout << "unable to parse addrs in '" << mon_host << "'"
+	     << std::endl;
+      return ret;
+    }
+  }
+  if (size() == 0) {
+    // What monitors are in the config file?
+    if (auto ret = init_with_config_file(conf, errout); ret < 0) {
+      return ret;
+    }
+  }
+  if (size() == 0) {
+    // no info found from conf options lets try use DNS SRV records
+    string srv_name = conf.get_val<std::string>("mon_dns_srv_name");
+    if (auto ret = init_with_dns_srv(cct, srv_name, for_mkfs, errout); ret < 0) {
+      return -ENOENT;
+    }
+  }
+  if (size() == 0) {
+    errout << "no monitors specified to connect to." << std::endl;
+    return -ENOENT;
+  }
+  strategy = static_cast<election_strategy>(conf.get_val<uint64_t>("mon_election_default_strategy"));
+  created = ceph_clock_now();
+  last_changed = created;
+  calc_legacy_ranks();
+  return 0;
+}
+#endif	// WITH_SEASTAR
diff --git a/src/mon/MonMap.h b/src/mon/MonMap.h
new file mode 100644
index 000000000..02304edfd
--- /dev/null
+++ b/src/mon/MonMap.h
@@ -0,0 +1,546 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MONMAP_H
+#define CEPH_MONMAP_H
+
+#ifdef WITH_SEASTAR
+#include <seastar/core/future.hh>
+#endif
+
+#include "common/config_fwd.h"
+#include "common/ceph_releases.h"
+
+#include "include/err.h"
+#include "include/types.h"
+
+#include "mon/mon_types.h"
+#include "msg/Message.h"
+
+class health_check_map_t;
+
+#ifdef WITH_SEASTAR
+namespace crimson::common {
+  class ConfigProxy;
+}
+#endif
+
+namespace ceph {
+  class Formatter;
+}
+
+struct mon_info_t {
+  /**
+   * monitor name
+   *
+   * i.e., 'foo' in 'mon.foo'
+   */
+  std::string name;
+  /**
+   * monitor's public address(es)
+   *
+   * public facing address(es), used to communicate with all clients
+   * and with other monitors.
+   */
+  entity_addrvec_t public_addrs;
+  /**
+   * the priority of the mon, the lower value the more preferred
+   */
+  uint16_t priority{0};
+  uint16_t weight{0};
+
+  /**
+   * The location of the monitor, in CRUSH hierarchy terms
+   */
+  std::map<std::string,std::string> crush_loc;
+
+  // <REMOVE ME>
+  mon_info_t(const std::string& n, const entity_addr_t& p_addr, uint16_t p)
+    : name(n), public_addrs(p_addr), priority(p)
+  {}
+  // </REMOVE ME>
+
+  mon_info_t(const std::string& n, const entity_addrvec_t& p_addrs,
+             uint16_t p, uint16_t w)
+    : name(n), public_addrs(p_addrs), priority(p), weight(w)
+  {}
+  mon_info_t(const std::string &n, const entity_addrvec_t& p_addrs)
+    : name(n), public_addrs(p_addrs)
+  { }
+
+  mon_info_t() { }
+
+
+  void encode(ceph::buffer::list& bl, uint64_t features) const;
+  void decode(ceph::buffer::list::const_iterator& p);
+  void print(std::ostream& out) const;
+};
+WRITE_CLASS_ENCODER_FEATURES(mon_info_t)
+
+inline std::ostream& operator<<(std::ostream& out, const mon_info_t& mon) {
+  mon.print(out);
+  return out;
+}
+
+class MonMap {
+ public:
+  epoch_t epoch;       // what epoch/version of the monmap
+  uuid_d fsid;
+  utime_t last_changed;
+  utime_t created;
+
+  std::map<std::string, mon_info_t> mon_info;
+  std::map<entity_addr_t, std::string> addr_mons;
+
+  std::vector<std::string> ranks;
+  /* ranks which were removed when this map took effect.
+     There should only be one at a time, but leave support
+     for arbitrary numbers just to be safe. */
+  std::set<int> removed_ranks;
+
+  /**
+   * Persistent Features are all those features that once set on a
+   * monmap cannot, and should not, be removed. These will define the
+   * non-negotiable features that a given monitor must support to
+   * properly operate in a given quorum.
+   *
+   * Should be reserved for features that we really want to make sure
+   * are sticky, and are important enough to tolerate not being able
+   * to downgrade a monitor.
+   */
+  mon_feature_t persistent_features;
+  /**
+   * Optional Features are all those features that can be enabled or
+   * disabled following a given criteria -- e.g., user-mandated via the
+   * cli --, and act much like indicators of what the cluster currently
+   * supports.
+   *
+   * They are by no means "optional" in the sense that monitors can
+   * ignore them. Just that they are not persistent.
+   */
+  mon_feature_t optional_features;
+
+  /**
+   * Returns the set of features required by this monmap.
+   *
+   * The features required by this monmap is the union of all the
+   * currently set persistent features and the currently set optional
+   * features.
+   *
+   * @returns the set of features required by this monmap
+   */
+  mon_feature_t get_required_features() const {
+    return (persistent_features | optional_features);
+  }
+
+  // upgrade gate
+  ceph_release_t min_mon_release{ceph_release_t::unknown};
+
+  void _add_ambiguous_addr(const std::string& name,
+                           entity_addr_t addr,
+                           int priority,
+                           int weight,
+                           bool for_mkfs);
+
+  enum election_strategy {
+			  // Keep in sync with ElectionLogic.h!
+    CLASSIC = 1, // the original rank-based one
+    DISALLOW = 2, // disallow a set from being leader
+    CONNECTIVITY = 3 // includes DISALLOW, extends to prefer stronger connections
+  };
+  election_strategy strategy = CLASSIC;
+  std::set<std::string> disallowed_leaders; // can't be leader under CONNECTIVITY/DISALLOW
+  bool stretch_mode_enabled = false;
+  string tiebreaker_mon;
+  set<string> stretch_marked_down_mons; // can't be leader until fully recovered
+
+public:
+  void calc_legacy_ranks();
+  void calc_addr_mons() {
+    // populate addr_mons
+    addr_mons.clear();
+    for (auto& p : mon_info) {
+      for (auto& a : p.second.public_addrs.v) {
+	addr_mons[a] = p.first;
+      }
+    }
+  }
+
+  MonMap()
+    : epoch(0) {
+  }
+
+  uuid_d& get_fsid() { return fsid; }
+
+  unsigned size() const {
+    return mon_info.size();
+  }
+
+  unsigned min_quorum_size(unsigned total_mons=0) const {
+    if (total_mons == 0) {
+      total_mons = size();
+    }
+    return total_mons / 2 + 1;
+  }
+
+  epoch_t get_epoch() const { return epoch; }
+  void set_epoch(epoch_t e) { epoch = e; }
+
+  /**
+   * Obtain list of public facing addresses
+   *
+   * @param ls list to populate with the monitors' addresses
+   */
+  void list_addrs(std::list<entity_addr_t>& ls) const {
+    for (auto& i : mon_info) {
+      for (auto& j : i.second.public_addrs.v) {
+	ls.push_back(j);
+      }
+    }
+  }
+
+  /**
+   * Add new monitor to the monmap
+   *
+   * @param m monitor info of the new monitor
+   */
+  void add(const mon_info_t& m) {
+    ceph_assert(mon_info.count(m.name) == 0);
+    for (auto& a : m.public_addrs.v) {
+      ceph_assert(addr_mons.count(a) == 0);
+    }
+    mon_info[m.name] = m;
+    if (get_required_features().contains_all(
+	  ceph::features::mon::FEATURE_NAUTILUS)) {
+      ranks.push_back(m.name);
+      ceph_assert(ranks.size() == mon_info.size());
+    } else {
+      calc_legacy_ranks();
+    }
+    calc_addr_mons();
+  }
+
+  /**
+   * Add new monitor to the monmap
+   *
+   * @param name Monitor name (i.e., 'foo' in 'mon.foo')
+   * @param addr Monitor's public address
+   */
+  void add(const std::string &name, const entity_addrvec_t &addrv,
+	   uint16_t priority=0, uint16_t weight=0) {
+    add(mon_info_t(name, addrv, priority, weight));
+  }
+
+  /**
+   * Remove monitor from the monmap
+   *
+   * @param name Monitor name (i.e., 'foo' in 'mon.foo')
+   */
+  void remove(const std::string &name) {
+    // this must match what we do in ConnectionTracker::notify_rank_removed
+    ceph_assert(mon_info.count(name));
+    int rank = get_rank(name);
+    mon_info.erase(name);
+    disallowed_leaders.erase(name);
+    ceph_assert(mon_info.count(name) == 0);
+    if (rank >= 0 ) {
+      removed_ranks.insert(rank);
+    }
+    if (get_required_features().contains_all(
+	  ceph::features::mon::FEATURE_NAUTILUS)) {
+      ranks.erase(std::find(ranks.begin(), ranks.end(), name));
+      ceph_assert(ranks.size() == mon_info.size());
+    } else {
+      calc_legacy_ranks();
+    }
+    calc_addr_mons();
+  }
+
+  /**
+   * Rename monitor from @p oldname to @p newname
+   *
+   * @param oldname monitor's current name (i.e., 'foo' in 'mon.foo')
+   * @param newname monitor's new name (i.e., 'bar' in 'mon.bar')
+   */
+  void rename(std::string oldname, std::string newname) {
+    ceph_assert(contains(oldname));
+    ceph_assert(!contains(newname));
+    mon_info[newname] = mon_info[oldname];
+    mon_info.erase(oldname);
+    mon_info[newname].name = newname;
+    if (get_required_features().contains_all(
+	  ceph::features::mon::FEATURE_NAUTILUS)) {
+      *std::find(ranks.begin(), ranks.end(), oldname) = newname;
+      ceph_assert(ranks.size() == mon_info.size());
+    } else {
+      calc_legacy_ranks();
+    }
+    calc_addr_mons();
+  }
+
+  int set_rank(const std::string& name, int rank) {
+    int oldrank = get_rank(name);
+    if (oldrank < 0) {
+      return -ENOENT;
+    }
+    if (rank < 0 || rank >= (int)ranks.size()) {
+      return -EINVAL;
+    }
+    if (oldrank != rank) {
+      ranks.erase(ranks.begin() + oldrank);
+      ranks.insert(ranks.begin() + rank, name);
+    }
+    return 0;
+  }
+
+  bool contains(const std::string& name) const {
+    return mon_info.count(name);
+  }
+
+  /**
+   * Check if monmap contains a monitor with address @p a
+   *
+   * @note checks for all addresses a monitor may have, public or otherwise.
+   *
+   * @param a monitor address
+   * @returns true if monmap contains a monitor with address @p;
+   *          false otherwise.
+   */
+  bool contains(const entity_addr_t &a, std::string *name=nullptr) const {
+    for (auto& i : mon_info) {
+      for (auto& j : i.second.public_addrs.v) {
+	if (j == a) {
+	  if (name) {
+	    *name = i.first;
+	  }
+	  return true;
+	}
+      }
+    }
+    return false;
+  }
+  bool contains(const entity_addrvec_t &av, std::string *name=nullptr) const {
+    for (auto& i : mon_info) {
+      for (auto& j : i.second.public_addrs.v) {
+	for (auto& k : av.v) {
+	  if (j == k) {
+	    if (name) {
+	      *name = i.first;
+	    }
+	    return true;
+	  }
+	}
+      }
+    }
+    return false;
+  }
+
+  std::string get_name(unsigned n) const {
+    ceph_assert(n < ranks.size());
+    return ranks[n];
+  }
+  std::string get_name(const entity_addr_t& a) const {
+    std::map<entity_addr_t, std::string>::const_iterator p = addr_mons.find(a);
+    if (p == addr_mons.end())
+      return std::string();
+    else
+      return p->second;
+  }
+  std::string get_name(const entity_addrvec_t& av) const {
+    for (auto& i : av.v) {
+      std::map<entity_addr_t, std::string>::const_iterator p = addr_mons.find(i);
+      if (p != addr_mons.end())
+	return p->second;
+    }
+    return std::string();
+  }
+
+  int get_rank(const std::string& n) const {
+    if (auto found = std::find(ranks.begin(), ranks.end(), n);
+	found != ranks.end()) {
+      return std::distance(ranks.begin(), found);
+    } else {
+      return -1;
+    }
+  }
+  int get_rank(const entity_addr_t& a) const {
+    std::string n = get_name(a);
+    if (!n.empty()) {
+      return get_rank(n);
+    }
+    return -1;
+  }
+  int get_rank(const entity_addrvec_t& av) const {
+    std::string n = get_name(av);
+    if (!n.empty()) {
+      return get_rank(n);
+    }
+    return -1;
+  }
+  bool get_addr_name(const entity_addr_t& a, std::string& name) {
+    if (addr_mons.count(a) == 0)
+      return false;
+    name = addr_mons[a];
+    return true;
+  }
+
+  const entity_addrvec_t& get_addrs(const std::string& n) const {
+    ceph_assert(mon_info.count(n));
+    std::map<std::string,mon_info_t>::const_iterator p = mon_info.find(n);
+    return p->second.public_addrs;
+  }
+  const entity_addrvec_t& get_addrs(unsigned m) const {
+    ceph_assert(m < ranks.size());
+    return get_addrs(ranks[m]);
+  }
+  void set_addrvec(const std::string& n, const entity_addrvec_t& a) {
+    ceph_assert(mon_info.count(n));
+    mon_info[n].public_addrs = a;
+    calc_addr_mons();
+  }
+  uint16_t get_priority(const std::string& n) const {
+    auto it = mon_info.find(n);
+    ceph_assert(it != mon_info.end());
+    return it->second.priority;
+  }
+  uint16_t get_weight(const std::string& n) const {
+    auto it = mon_info.find(n);
+    ceph_assert(it != mon_info.end());
+    return it->second.weight;
+  }
+  void set_weight(const std::string& n, uint16_t v) {
+    auto it = mon_info.find(n);
+    ceph_assert(it != mon_info.end());
+    it->second.weight = v;
+  }
+
+  void encode(ceph::buffer::list& blist, uint64_t con_features) const;
+  void decode(ceph::buffer::list& blist) {
+    auto p = std::cbegin(blist);
+    decode(p);
+  }
+  void decode(ceph::buffer::list::const_iterator& p);
+
+  void generate_fsid() {
+    fsid.generate_random();
+  }
+
+  // read from/write to a file
+  int write(const char *fn);
+  int read(const char *fn);
+
+  /**
+   * build an initial bootstrap monmap from conf
+   *
+   * Build an initial bootstrap monmap from the config.  This will
+   * try, in this order:
+   *
+   *   1 monmap   -- an explicitly provided monmap
+   *   2 mon_host -- list of monitors
+   *   3 config [mon.*] sections, and 'mon addr' fields in those sections
+   *
+   * @param cct context (and associated config)
+   * @param errout std::ostream to send error messages too
+   */
+#ifdef WITH_SEASTAR
+  seastar::future<> build_initial(const crimson::common::ConfigProxy& conf, bool for_mkfs);
+#else
+  int build_initial(CephContext *cct, bool for_mkfs, std::ostream& errout);
+#endif
+  /**
+   * filter monmap given a set of initial members.
+   *
+   * Remove mons that aren't in the initial_members list.  Add missing
+   * mons and give them dummy IPs (blank IPv4, with a non-zero
+   * nonce). If the name matches my_name, then my_addr will be used in
+   * place of a dummy addr.
+   *
+   * @param initial_members list of initial member names
+   * @param my_name name of self, can be blank
+   * @param my_addr my addr
+   * @param removed optional pointer to set to insert removed mon addrs to
+   */
+  void set_initial_members(CephContext *cct,
+			   std::list<std::string>& initial_members,
+			   std::string my_name,
+			   const entity_addrvec_t& my_addrs,
+			   std::set<entity_addrvec_t> *removed);
+
+  void print(std::ostream& out) const;
+  void print_summary(std::ostream& out) const;
+  void dump(ceph::Formatter *f) const;
+  void dump_summary(ceph::Formatter *f) const;
+
+  void check_health(health_check_map_t *checks) const;
+
+  static void generate_test_instances(std::list<MonMap*>& o);
+protected:
+  /**
+   * build a monmap from a list of entity_addrvec_t's
+   *
+   * Give mons dummy names.
+   *
+   * @param addrs  list of entity_addrvec_t's
+   * @param prefix prefix to prepend to generated mon names
+   */
+  void init_with_addrs(const std::vector<entity_addrvec_t>& addrs,
+                       bool for_mkfs,
+                       std::string_view prefix);
+  /**
+   * build a monmap from a list of ips
+   *
+   * Give mons dummy names.
+   *
+   * @param hosts  list of ips, space or comma separated
+   * @param prefix prefix to prepend to generated mon names
+   * @return 0 for success, -errno on error
+   */
+  int init_with_ips(const std::string& ips,
+		    bool for_mkfs,
+		    std::string_view prefix);
+  /**
+   * build a monmap from a list of hostnames
+   *
+   * Give mons dummy names.
+   *
+   * @param hosts  list of ips, space or comma separated
+   * @param prefix prefix to prepend to generated mon names
+   * @return 0 for success, -errno on error
+   */
+  int init_with_hosts(const std::string& hostlist,
+		      bool for_mkfs,
+		      std::string_view prefix);
+  int init_with_config_file(const ConfigProxy& conf, std::ostream& errout);
+#if WITH_SEASTAR
+  seastar::future<> read_monmap(const std::string& monmap);
+  /// try to build monmap with different settings, like
+  /// mon_host, mon* sections, and mon_dns_srv_name
+  seastar::future<> build_monmap(const crimson::common::ConfigProxy& conf, bool for_mkfs);
+  /// initialize monmap by resolving given service name
+  seastar::future<> init_with_dns_srv(bool for_mkfs, const std::string& name);
+#else
+  /// read from encoded monmap file
+  int init_with_monmap(const std::string& monmap, std::ostream& errout);
+  int init_with_dns_srv(CephContext* cct, std::string srv_name, bool for_mkfs,
+			std::ostream& errout);
+#endif
+};
+WRITE_CLASS_ENCODER_FEATURES(MonMap)
+
+inline std::ostream& operator<<(std::ostream &out, const MonMap &m) {
+  m.print_summary(out);
+  return out;
+}
+
+#endif
diff --git a/src/mon/MonOpRequest.h b/src/mon/MonOpRequest.h
new file mode 100644
index 000000000..73275e81e
--- /dev/null
+++ b/src/mon/MonOpRequest.h
@@ -0,0 +1,238 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2015 Red Hat <contact@redhat.com>
+ * Copyright (C) 2015 SUSE LINUX GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#ifndef MON_OPREQUEST_H_
+#define MON_OPREQUEST_H_
+#include <iosfwd>
+#include <stdint.h>
+
+#include "common/TrackedOp.h"
+#include "mon/Session.h"
+#include "msg/Message.h"
+
+struct MonOpRequest : public TrackedOp {
+  friend class OpTracker;
+
+  void mark_dispatch() {
+    mark_event("monitor_dispatch");
+  }
+  void mark_wait_for_quorum() {
+    mark_event("wait_for_quorum");
+  }
+  void mark_zap() {
+    mark_event("monitor_zap");
+  }
+  void mark_forwarded() {
+    mark_event("forwarded");
+    forwarded_to_leader = true;
+  }
+
+  void mark_svc_event(const std::string &service, const std::string &event) {
+    std::string s = service;
+    s.append(":").append(event);
+    mark_event(s);
+  }
+
+  void mark_logmon_event(const std::string &event) {
+    mark_svc_event("logm", event);
+  }
+  void mark_osdmon_event(const std::string &event) {
+    mark_svc_event("osdmap", event);
+  }
+  void mark_pgmon_event(const std::string &event) {
+    mark_svc_event("pgmap", event);
+  }
+  void mark_mdsmon_event(const std::string &event) {
+    mark_svc_event("mdsmap", event);
+  }
+  void mark_authmon_event(const std::string &event) {
+    mark_svc_event("auth", event);
+  }
+  void mark_paxos_event(const std::string &event) {
+    mark_svc_event("paxos", event);
+  }
+
+
+  enum op_type_t {
+    OP_TYPE_NONE    = 0,      ///< no type defined (default)
+    OP_TYPE_SERVICE,          ///< belongs to a Paxos Service or similar
+    OP_TYPE_MONITOR,          ///< belongs to the Monitor class
+    OP_TYPE_ELECTION,         ///< belongs to the Elector class
+    OP_TYPE_PAXOS,            ///< refers to Paxos messages
+    OP_TYPE_COMMAND,          ///< is a command
+  };
+
+  MonOpRequest(const MonOpRequest &other) = delete;
+  MonOpRequest & operator = (const MonOpRequest &other) = delete;
+
+private:
+  Message *request;
+  utime_t dequeued_time;
+  RefCountedPtr session;
+  ConnectionRef con;
+  bool forwarded_to_leader;
+  op_type_t op_type;
+
+  MonOpRequest(Message *req, OpTracker *tracker) :
+    TrackedOp(tracker,
+      req->get_recv_stamp().is_zero() ?
+      ceph_clock_now() : req->get_recv_stamp()),
+    request(req),
+    con(NULL),
+    forwarded_to_leader(false),
+    op_type(OP_TYPE_NONE)
+  {
+    if (req) {
+      con = req->get_connection();
+      if (con) {
+        session = con->get_priv();
+      }
+    }
+  }
+
+  void _dump(ceph::Formatter *f) const override {
+    {
+      f->open_array_section("events");
+      std::lock_guard l(lock);
+    for (auto i = events.begin(); i != events.end(); ++i) {
+      f->open_object_section("event");
+      f->dump_string("event", i->str);
+      f->dump_stream("time") << i->stamp;
+
+      auto i_next = i + 1;
+
+      if (i_next < events.end()) {
+	f->dump_float("duration", i_next->stamp - i->stamp);
+      } else {
+	f->dump_float("duration", events.rbegin()->stamp - get_initiated());
+      }
+
+      f->close_section();
+    }
+      f->close_section();
+      f->open_object_section("info");
+      f->dump_int("seq", seq);
+      f->dump_bool("src_is_mon", is_src_mon());
+      f->dump_stream("source") << request->get_source_inst();
+      f->dump_bool("forwarded_to_leader", forwarded_to_leader);
+      f->close_section();
+    }
+  }
+
+protected:
+  void _dump_op_descriptor_unlocked(std::ostream& stream) const override {
+    get_req()->print(stream);
+  }
+
+public:
+  ~MonOpRequest() override {
+    request->put();
+  }
+
+  MonSession *get_session() const {
+    return static_cast<MonSession*>(session.get());
+  }
+
+  template<class T>
+  T *get_req() const { return static_cast<T*>(request); }
+
+  Message *get_req() const { return get_req<Message>(); }
+
+  int get_req_type() const {
+    if (!request)
+      return 0;
+    return request->get_type();
+  }
+
+  ConnectionRef get_connection() { return con; }
+
+  void set_session(MonSession *s) {
+    session.reset(s);
+  }
+
+  bool is_src_mon() const {
+    return (con && con->get_peer_type() & CEPH_ENTITY_TYPE_MON);
+  }
+
+  typedef boost::intrusive_ptr<MonOpRequest> Ref;
+
+  void set_op_type(op_type_t t) {
+    op_type = t;
+  }
+  void set_type_service() {
+    set_op_type(OP_TYPE_SERVICE);
+  }
+  void set_type_monitor() {
+    set_op_type(OP_TYPE_MONITOR);
+  }
+  void set_type_paxos() {
+    set_op_type(OP_TYPE_PAXOS);
+  }
+  void set_type_election_or_ping() {
+    set_op_type(OP_TYPE_ELECTION);
+  }
+  void set_type_command() {
+    set_op_type(OP_TYPE_COMMAND);
+  }
+
+  op_type_t get_op_type() {
+    return op_type;
+  }
+
+  bool is_type_service() {
+    return (get_op_type() == OP_TYPE_SERVICE);
+  }
+  bool is_type_monitor() {
+    return (get_op_type() == OP_TYPE_MONITOR);
+  }
+  bool is_type_paxos() {
+    return (get_op_type() == OP_TYPE_PAXOS);
+  }
+  bool is_type_election_or_ping() {
+    return (get_op_type() == OP_TYPE_ELECTION);
+  }
+  bool is_type_command() {
+    return (get_op_type() == OP_TYPE_COMMAND);
+  }
+};
+
+typedef MonOpRequest::Ref MonOpRequestRef;
+
+struct C_MonOp : public Context
+{
+  MonOpRequestRef op;
+
+  explicit C_MonOp(MonOpRequestRef o) :
+    op(o) { }
+
+  void finish(int r) override {
+    if (op && r == -ECANCELED) {
+      op->mark_event("callback canceled");
+    } else if (op && r == -EAGAIN) {
+      op->mark_event("callback retry");
+    } else if (op && r == 0) {
+      op->mark_event("callback finished");
+    }
+    _finish(r);
+  }
+
+  void mark_op_event(const std::string &event) {
+    if (op)
+      op->mark_event(event);
+  }
+
+  virtual void _finish(int r) = 0;
+};
+
+#endif /* MON_OPREQUEST_H_ */
diff --git a/src/mon/MonSub.cc b/src/mon/MonSub.cc
new file mode 100644
index 000000000..a2c60ba91
--- /dev/null
+++ b/src/mon/MonSub.cc
@@ -0,0 +1,114 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "MonSub.h"
+
+bool MonSub::have_new() const {
+  return !sub_new.empty();
+}
+
+bool MonSub::need_renew() const
+{
+  return ceph::coarse_mono_clock::now() > renew_after;
+}
+
+void MonSub::renewed()
+{
+  if (clock::is_zero(renew_sent)) {
+    renew_sent = clock::now();
+  }
+  // update sub_sent with sub_new
+  sub_new.insert(sub_sent.begin(), sub_sent.end());
+  std::swap(sub_new, sub_sent);
+  sub_new.clear();
+}
+
+void MonSub::acked(uint32_t interval)
+{
+  if (!clock::is_zero(renew_sent)) {
+    // NOTE: this is only needed for legacy (infernalis or older)
+    // mons; see MonClient::tick().
+    renew_after = renew_sent;
+    renew_after += ceph::make_timespan(interval / 2.0);
+    renew_sent = clock::zero();
+  }
+}
+
+bool MonSub::reload()
+{
+  for (auto& [what, sub] : sub_sent) {
+    if (sub_new.count(what) == 0) {
+      sub_new[what] = sub;
+    }
+  }
+  return have_new();
+}
+
+void MonSub::got(const std::string& what, version_t have)
+{
+  if (auto i = sub_new.find(what); i != sub_new.end()) {
+    auto& sub = i->second;
+    if (sub.start <= have) {
+      if (sub.flags & CEPH_SUBSCRIBE_ONETIME) {
+        sub_new.erase(i);
+      } else {
+        sub.start = have + 1;
+      }
+    }
+  } else if (auto i = sub_sent.find(what); i != sub_sent.end()) {
+    auto& sub = i->second;
+    if (sub.start <= have) {
+      if (sub.flags & CEPH_SUBSCRIBE_ONETIME) {
+        sub_sent.erase(i);
+      } else {
+        sub.start = have + 1;
+      }
+    }
+  }
+}
+
+bool MonSub::want(const std::string& what, version_t start, unsigned flags)
+{
+  if (auto sub = sub_new.find(what);
+      sub != sub_new.end() &&
+      sub->second.start == start &&
+      sub->second.flags == flags) {
+    return false;
+  } else if (auto sub = sub_sent.find(what);
+      sub != sub_sent.end() &&
+      sub->second.start == start &&
+      sub->second.flags == flags) {
+	return false;
+  } else {
+    sub_new[what].start = start;
+    sub_new[what].flags = flags;
+    return true;
+  }
+}
+
+bool MonSub::inc_want(const std::string& what, version_t start, unsigned flags)
+{
+  if (auto sub = sub_new.find(what); sub != sub_new.end()) {
+    if (sub->second.start >= start) {
+      return false;
+    } else {
+      sub->second.start = start;
+      sub->second.flags = flags;
+      return true;
+    }
+  } else if (auto sub = sub_sent.find(what);
+             sub == sub_sent.end() || sub->second.start < start) {
+    auto& item = sub_new[what];
+    item.start = start;
+    item.flags = flags;
+    return true;
+  } else {
+    return false;
+  }
+}
+
+void MonSub::unwant(const std::string& what)
+{
+  sub_sent.erase(what);
+  sub_new.erase(what);
+}
diff --git a/src/mon/MonSub.h b/src/mon/MonSub.h
new file mode 100644
index 000000000..8ff5a8f18
--- /dev/null
+++ b/src/mon/MonSub.h
@@ -0,0 +1,46 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <map>
+#include <string>
+
+#include "common/ceph_time.h"
+#include "include/types.h"
+
+// mon subscriptions
+class MonSub
+{
+public:
+  // @returns true if there is any "new" subscriptions
+  bool have_new() const;
+  auto get_subs() const {
+    return sub_new;
+  }
+  bool need_renew() const;
+  // change the status of "new" subscriptions to "sent"
+  void renewed();
+  // the peer acked the subscription request
+  void acked(uint32_t interval);
+  void got(const std::string& what, version_t version);
+  // revert the status of subscriptions from "sent" to "new"
+  // @returns true if there is any pending "new" subscriptions
+  bool reload();
+  // add a new subscription
+  bool want(const std::string& what, version_t start, unsigned flags);
+  // increment the requested subscription start point. If you do increase
+  // the value, apply the passed-in flags as well; otherwise do nothing.
+  bool inc_want(const std::string& what, version_t start, unsigned flags);
+  // cancel a subscription
+  void unwant(const std::string& what);
+private:
+  // my subs, and current versions
+  std::map<std::string,ceph_mon_subscribe_item> sub_sent;
+  // unsent new subs
+  std::map<std::string,ceph_mon_subscribe_item> sub_new;
+  using time_point = ceph::coarse_mono_time;
+  using clock = typename time_point::clock;
+  time_point renew_sent;
+  time_point renew_after;
+};
diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
new file mode 100644
index 000000000..ce7ec37d9
--- /dev/null
+++ b/src/mon/Monitor.cc
@@ -0,0 +1,6887 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#include <iterator>
+#include <sstream>
+#include <tuple>
+#include <stdlib.h>
+#include <signal.h>
+#include <limits.h>
+#include <cstring>
+#include <boost/scope_exit.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+
+#include "json_spirit/json_spirit_reader.h"
+#include "json_spirit/json_spirit_writer.h"
+
+#include "Monitor.h"
+#include "common/version.h"
+#include "common/blkdev.h"
+#include "common/cmdparse.h"
+#include "common/signal.h"
+
+#include "osd/OSDMap.h"
+
+#include "MonitorDBStore.h"
+
+#include "messages/PaxosServiceMessage.h"
+#include "messages/MMonMap.h"
+#include "messages/MMonGetMap.h"
+#include "messages/MMonGetVersion.h"
+#include "messages/MMonGetVersionReply.h"
+#include "messages/MGenericMessage.h"
+#include "messages/MMonCommand.h"
+#include "messages/MMonCommandAck.h"
+#include "messages/MMonSync.h"
+#include "messages/MMonScrub.h"
+#include "messages/MMonProbe.h"
+#include "messages/MMonJoin.h"
+#include "messages/MMonPaxos.h"
+#include "messages/MRoute.h"
+#include "messages/MForward.h"
+
+#include "messages/MMonSubscribe.h"
+#include "messages/MMonSubscribeAck.h"
+
+#include "messages/MCommand.h"
+#include "messages/MCommandReply.h"
+
+#include "messages/MAuthReply.h"
+
+#include "messages/MTimeCheck2.h"
+#include "messages/MPing.h"
+
+#include "common/strtol.h"
+#include "common/ceph_argparse.h"
+#include "common/Timer.h"
+#include "common/Clock.h"
+#include "common/errno.h"
+#include "common/perf_counters.h"
+#include "common/admin_socket.h"
+#include "global/signal_handler.h"
+#include "common/Formatter.h"
+#include "include/stringify.h"
+#include "include/color.h"
+#include "include/ceph_fs.h"
+#include "include/str_list.h"
+
+#include "OSDMonitor.h"
+#include "MDSMonitor.h"
+#include "MonmapMonitor.h"
+#include "LogMonitor.h"
+#include "AuthMonitor.h"
+#include "MgrMonitor.h"
+#include "MgrStatMonitor.h"
+#include "ConfigMonitor.h"
+#include "KVMonitor.h"
+#include "mon/HealthMonitor.h"
+#include "common/config.h"
+#include "common/cmdparse.h"
+#include "include/ceph_assert.h"
+#include "include/compat.h"
+#include "perfglue/heap_profiler.h"
+
+#include "auth/none/AuthNoneClientHandler.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+using namespace TOPNSPC::common;
+
+using std::cout;
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::setfill;
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+using std::unique_ptr;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::ErasureCodeInterfaceRef;
+using ceph::ErasureCodeProfile;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::make_message;
+using ceph::mono_clock;
+using ceph::mono_time;
+using ceph::timespan_str;
+
+
+static ostream& _prefix(std::ostream *_dout, const Monitor *mon) {
+  return *_dout << "mon." << mon->name << "@" << mon->rank
+		<< "(" << mon->get_state_name() << ") e" << mon->monmap->get_epoch() << " ";
+}
+
+const string Monitor::MONITOR_NAME = "monitor";
+const string Monitor::MONITOR_STORE_PREFIX = "monitor_store";
+
+
+#undef FLAG
+#undef COMMAND
+#undef COMMAND_WITH_FLAG
+#define FLAG(f) (MonCommand::FLAG_##f)
+#define COMMAND(parsesig, helptext, modulename, req_perms)	\
+  {parsesig, helptext, modulename, req_perms, FLAG(NONE)},
+#define COMMAND_WITH_FLAG(parsesig, helptext, modulename, req_perms, flags) \
+  {parsesig, helptext, modulename, req_perms, flags},
+MonCommand mon_commands[] = {
+#include <mon/MonCommands.h>
+};
+#undef COMMAND
+#undef COMMAND_WITH_FLAG
+
+Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s,
+		 Messenger *m, Messenger *mgr_m, MonMap *map) :
+  Dispatcher(cct_),
+  AuthServer(cct_),
+  name(nm),
+  rank(-1), 
+  messenger(m),
+  con_self(m ? m->get_loopback_connection() : NULL),
+  timer(cct_, lock),
+  finisher(cct_, "mon_finisher", "fin"),
+  cpu_tp(cct, "Monitor::cpu_tp", "cpu_tp", g_conf()->mon_cpu_threads),
+  has_ever_joined(false),
+  logger(NULL), cluster_logger(NULL), cluster_logger_registered(false),
+  monmap(map),
+  log_client(cct_, messenger, monmap, LogClient::FLAG_MON),
+  key_server(cct, &keyring),
+  auth_cluster_required(cct,
+			cct->_conf->auth_supported.empty() ?
+			cct->_conf->auth_cluster_required : cct->_conf->auth_supported),
+  auth_service_required(cct,
+			cct->_conf->auth_supported.empty() ?
+			cct->_conf->auth_service_required : cct->_conf->auth_supported),
+  mgr_messenger(mgr_m),
+  mgr_client(cct_, mgr_m, monmap),
+  gss_ktfile_client(cct->_conf.get_val<std::string>("gss_ktab_client_file")),
+  store(s),
+  
+  elector(this, map->strategy),
+  required_features(0),
+  leader(0),
+  quorum_con_features(0),
+  // scrub
+  scrub_version(0),
+  scrub_event(NULL),
+  scrub_timeout_event(NULL),
+
+  // sync state
+  sync_provider_count(0),
+  sync_cookie(0),
+  sync_full(false),
+  sync_start_version(0),
+  sync_timeout_event(NULL),
+  sync_last_committed_floor(0),
+
+  timecheck_round(0),
+  timecheck_acks(0),
+  timecheck_rounds_since_clean(0),
+  timecheck_event(NULL),
+
+  admin_hook(NULL),
+  routed_request_tid(0),
+  op_tracker(cct, g_conf().get_val<bool>("mon_enable_op_tracker"), 1)
+{
+  clog = log_client.create_channel(CLOG_CHANNEL_CLUSTER);
+  audit_clog = log_client.create_channel(CLOG_CHANNEL_AUDIT);
+
+  update_log_clients();
+
+  if (!gss_ktfile_client.empty()) {
+    // Assert we can export environment variable 
+    /* 
+        The default client keytab is used, if it is present and readable,
+        to automatically obtain initial credentials for GSSAPI client
+        applications. The principal name of the first entry in the client
+        keytab is used by default when obtaining initial credentials.
+        1. The KRB5_CLIENT_KTNAME environment variable.
+        2. The default_client_keytab_name profile variable in [libdefaults].
+        3. The hardcoded default, DEFCKTNAME.
+    */
+    const int32_t set_result(setenv("KRB5_CLIENT_KTNAME", 
+                                    gss_ktfile_client.c_str(), 1));
+    ceph_assert(set_result == 0);
+  }
+
+  op_tracker.set_complaint_and_threshold(
+      g_conf().get_val<std::chrono::seconds>("mon_op_complaint_time").count(),
+      g_conf().get_val<int64_t>("mon_op_log_threshold"));
+  op_tracker.set_history_size_and_duration(
+      g_conf().get_val<uint64_t>("mon_op_history_size"),
+      g_conf().get_val<std::chrono::seconds>("mon_op_history_duration").count());
+  op_tracker.set_history_slow_op_size_and_threshold(
+      g_conf().get_val<uint64_t>("mon_op_history_slow_op_size"),
+      g_conf().get_val<std::chrono::seconds>("mon_op_history_slow_op_threshold").count());
+
+  paxos = std::make_unique<Paxos>(*this, "paxos");
+
+  paxos_service[PAXOS_MDSMAP].reset(new MDSMonitor(*this, *paxos, "mdsmap"));
+  paxos_service[PAXOS_MONMAP].reset(new MonmapMonitor(*this, *paxos, "monmap"));
+  paxos_service[PAXOS_OSDMAP].reset(new OSDMonitor(cct, *this, *paxos, "osdmap"));
+  paxos_service[PAXOS_LOG].reset(new LogMonitor(*this, *paxos, "logm"));
+  paxos_service[PAXOS_AUTH].reset(new AuthMonitor(*this, *paxos, "auth"));
+  paxos_service[PAXOS_MGR].reset(new MgrMonitor(*this, *paxos, "mgr"));
+  paxos_service[PAXOS_MGRSTAT].reset(new MgrStatMonitor(*this, *paxos, "mgrstat"));
+  paxos_service[PAXOS_HEALTH].reset(new HealthMonitor(*this, *paxos, "health"));
+  paxos_service[PAXOS_CONFIG].reset(new ConfigMonitor(*this, *paxos, "config"));
+  paxos_service[PAXOS_KV].reset(new KVMonitor(*this, *paxos, "kv"));
+
+  bool r = mon_caps.parse("allow *", NULL);
+  ceph_assert(r);
+
+  exited_quorum = ceph_clock_now();
+
+  // prepare local commands
+  local_mon_commands.resize(std::size(mon_commands));
+  for (unsigned i = 0; i < std::size(mon_commands); ++i) {
+    local_mon_commands[i] = mon_commands[i];
+  }
+  MonCommand::encode_vector(local_mon_commands, local_mon_commands_bl);
+
+  prenautilus_local_mon_commands = local_mon_commands;
+  for (auto& i : prenautilus_local_mon_commands) {
+    std::string n = cmddesc_get_prenautilus_compat(i.cmdstring);
+    if (n != i.cmdstring) {
+      dout(20) << " pre-nautilus cmd " << i.cmdstring << " -> " << n << dendl;
+      i.cmdstring = n;
+    }
+  }
+  MonCommand::encode_vector(prenautilus_local_mon_commands, prenautilus_local_mon_commands_bl);
+
+  // assume our commands until we have an election.  this only means
+  // we won't reply with EINVAL before the election; any command that
+  // actually matters will wait until we have quorum etc and then
+  // retry (and revalidate).
+  leader_mon_commands = local_mon_commands;
+}
+
+Monitor::~Monitor()
+{
+  op_tracker.on_shutdown();
+
+  delete logger;
+  ceph_assert(session_map.sessions.empty());
+}
+
+
+class AdminHook : public AdminSocketHook {
+  Monitor *mon;
+public:
+  explicit AdminHook(Monitor *m) : mon(m) {}
+  int call(std::string_view command, const cmdmap_t& cmdmap,
+	   Formatter *f,
+	   std::ostream& errss,
+	   bufferlist& out) override {
+    stringstream outss;
+    int r = mon->do_admin_command(command, cmdmap, f, errss, outss);
+    out.append(outss);
+    return r;
+  }
+};
+
+int Monitor::do_admin_command(
+  std::string_view command,
+  const cmdmap_t& cmdmap,
+  Formatter *f,
+  std::ostream& err,
+  std::ostream& out)
+{
+  std::lock_guard l(lock);
+
+  int r = 0;
+  string args;
+  for (auto p = cmdmap.begin();
+       p != cmdmap.end(); ++p) {
+    if (p->first == "prefix")
+      continue;
+    if (!args.empty())
+      args += ", ";
+    args += cmd_vartype_stringify(p->second);
+  }
+  args = "[" + args + "]";
+
+  bool read_only = (command == "mon_status" ||
+                    command == "mon metadata" ||
+                    command == "quorum_status" ||
+                    command == "ops" ||
+                    command == "sessions");
+
+  (read_only ? audit_clog->debug() : audit_clog->info())
+    << "from='admin socket' entity='admin socket' "
+    << "cmd='" << command << "' args=" << args << ": dispatch";
+
+  if (command == "mon_status") {
+    get_mon_status(f);
+  } else if (command == "quorum_status") {
+    _quorum_status(f, out);
+  } else if (command == "sync_force") {
+    string validate;
+    if ((!cmd_getval(cmdmap, "validate", validate)) ||
+	(validate != "--yes-i-really-mean-it")) {
+      err << "are you SURE? this will mean the monitor store will be erased "
+	"the next time the monitor is restarted.  pass "
+	"'--yes-i-really-mean-it' if you really do.";
+      r = -EPERM;
+      goto abort;
+    }
+    sync_force(f);
+  } else if (command.compare(0, 23, "add_bootstrap_peer_hint") == 0 ||
+	     command.compare(0, 24, "add_bootstrap_peer_hintv") == 0) {
+    if (!_add_bootstrap_peer_hint(command, cmdmap, out))
+      goto abort;
+  } else if (command == "quorum enter") {
+    elector.start_participating();
+    start_election();
+    out << "started responding to quorum, initiated new election";
+  } else if (command == "quorum exit") {
+    start_election();
+    elector.stop_participating();
+    out << "stopped responding to quorum, initiated new election";
+  } else if (command == "ops") {
+    (void)op_tracker.dump_ops_in_flight(f);
+  } else if (command == "sessions") {
+    f->open_array_section("sessions");
+    for (auto p : session_map.sessions) {
+      f->dump_object("session", *p);
+    }
+    f->close_section();
+  } else if (command == "dump_historic_ops") {
+    if (!op_tracker.dump_historic_ops(f)) {
+      err << "op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. \
+        please enable \"mon_enable_op_tracker\", and the tracker will start to track new ops received afterwards.";
+    }
+  } else if (command == "dump_historic_ops_by_duration" ) {
+    if (op_tracker.dump_historic_ops(f, true)) {
+      err << "op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. \
+        please enable \"mon_enable_op_tracker\", and the tracker will start to track new ops received afterwards.";
+    }
+  } else if (command == "dump_historic_slow_ops") {
+    if (op_tracker.dump_historic_slow_ops(f, {})) {
+      err << "op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. \
+        please enable \"mon_enable_op_tracker\", and the tracker will start to track new ops received afterwards.";
+    }
+  } else if (command == "quorum") {
+    string quorumcmd;
+    cmd_getval(cmdmap, "quorumcmd", quorumcmd);
+    if (quorumcmd == "exit") {
+      start_election();
+      elector.stop_participating();
+      out << "stopped responding to quorum, initiated new election" << std::endl;
+    } else if (quorumcmd == "enter") {
+      elector.start_participating();
+      start_election();
+      out << "started responding to quorum, initiated new election" << std::endl;
+    } else {
+      err << "needs a valid 'quorum' command" << std::endl;
+    }
+  } else if (command == "connection scores dump") {
+    if (!get_quorum_mon_features().contains_all(
+				   ceph::features::mon::FEATURE_PINGING)) {
+      err << "Not all monitors support changing election strategies; \
+              please upgrade them first!";
+    }
+    elector.dump_connection_scores(f);
+  } else if (command == "connection scores reset") {
+    if (!get_quorum_mon_features().contains_all(
+				   ceph::features::mon::FEATURE_PINGING)) {
+      err << "Not all monitors support changing election strategies; \
+              please upgrade them first!";
+    }
+    elector.notify_clear_peer_state();
+  } else if (command == "smart") {
+    string want_devid;
+    cmd_getval(cmdmap, "devid", want_devid);
+
+    string devname = store->get_devname();
+    if (devname.empty()) {
+      err << "could not determine device name for " << store->get_path();
+      r = -ENOENT;
+      goto abort;
+    }
+    set<string> devnames;
+    get_raw_devices(devname, &devnames);
+    json_spirit::mObject json_map;
+    uint64_t smart_timeout = cct->_conf.get_val<uint64_t>(
+      "mon_smart_report_timeout");
+    for (auto& devname : devnames) {
+      string err;
+      string devid = get_device_id(devname, &err);
+      if (want_devid.size() && want_devid != devid) {
+	derr << "get_device_id failed on " << devname << ": " << err << dendl;
+	continue;
+      }
+      json_spirit::mValue smart_json;
+      if (block_device_get_metrics(devname, smart_timeout,
+				   &smart_json)) {
+	dout(10) << "block_device_get_metrics failed for /dev/" << devname
+		 << dendl;
+	continue;
+      }
+      json_map[devid] = smart_json;
+    }
+    json_spirit::write(json_map, out, json_spirit::pretty_print);
+  } else if (command == "heap") {
+    if (!ceph_using_tcmalloc()) {
+      err << "could not issue heap profiler command -- not using tcmalloc!";
+      r = -EOPNOTSUPP;
+      goto abort;
+    }
+    string cmd;
+    if (!cmd_getval(cmdmap, "heapcmd", cmd)) {
+      err << "unable to get value for command \"" << cmd << "\"";
+      r = -EINVAL;
+      goto abort;
+    }
+    std::vector<std::string> cmd_vec;
+    get_str_vec(cmd, cmd_vec);
+    string val;
+    if (cmd_getval(cmdmap, "value", val)) {
+      cmd_vec.push_back(val);
+    }
+    ceph_heap_profiler_handle_command(cmd_vec, out);
+  } else if (command == "compact") {
+    dout(1) << "triggering manual compaction" << dendl;
+    auto start = ceph::coarse_mono_clock::now();
+    store->compact_async();
+    auto end = ceph::coarse_mono_clock::now();
+    auto duration = ceph::to_seconds<double>(end - start);
+    dout(1) << "finished manual compaction in "
+	    << duration << " seconds" << dendl;
+    out << "compacted " << g_conf().get_val<std::string>("mon_keyvaluedb")
+	<< " in " << duration << " seconds";
+ } else {
+    ceph_abort_msg("bad AdminSocket command binding");
+  }
+  (read_only ? audit_clog->debug() : audit_clog->info())
+    << "from='admin socket' "
+    << "entity='admin socket' "
+    << "cmd=" << command << " "
+    << "args=" << args << ": finished";
+  return r;
+
+abort:
+  (read_only ? audit_clog->debug() : audit_clog->info())
+    << "from='admin socket' "
+    << "entity='admin socket' "
+    << "cmd=" << command << " "
+    << "args=" << args << ": aborted";
+  return r;
+}
+
+void Monitor::handle_signal(int signum)
+{
+  ceph_assert(signum == SIGINT || signum == SIGTERM);
+  derr << "*** Got Signal " << sig_str(signum) << " ***" << dendl;
+  shutdown();
+}
+
+CompatSet Monitor::get_initial_supported_features()
+{
+  CompatSet::FeatureSet ceph_mon_feature_compat;
+  CompatSet::FeatureSet ceph_mon_feature_ro_compat;
+  CompatSet::FeatureSet ceph_mon_feature_incompat;
+  ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_BASE);
+  ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_SINGLE_PAXOS);
+  return CompatSet(ceph_mon_feature_compat, ceph_mon_feature_ro_compat,
+		   ceph_mon_feature_incompat);
+}
+
+CompatSet Monitor::get_supported_features()
+{
+  CompatSet compat = get_initial_supported_features();
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES);
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC);
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2);
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V3);
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_KRAKEN);
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_LUMINOUS);
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_MIMIC);
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_NAUTILUS);
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OCTOPUS);
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_PACIFIC);
+  return compat;
+}
+
+CompatSet Monitor::get_legacy_features()
+{
+  CompatSet::FeatureSet ceph_mon_feature_compat;
+  CompatSet::FeatureSet ceph_mon_feature_ro_compat;
+  CompatSet::FeatureSet ceph_mon_feature_incompat;
+  ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_BASE);
+  return CompatSet(ceph_mon_feature_compat, ceph_mon_feature_ro_compat,
+		   ceph_mon_feature_incompat);
+}
+
+int Monitor::check_features(MonitorDBStore *store)
+{
+  CompatSet required = get_supported_features();
+  CompatSet ondisk;
+
+  read_features_off_disk(store, &ondisk);
+
+  if (!required.writeable(ondisk)) {
+    CompatSet diff = required.unsupported(ondisk);
+    generic_derr << "ERROR: on disk data includes unsupported features: " << diff << dendl;
+    return -EPERM;
+  }
+
+  return 0;
+}
+
+void Monitor::read_features_off_disk(MonitorDBStore *store, CompatSet *features)
+{
+  bufferlist featuresbl;
+  store->get(MONITOR_NAME, COMPAT_SET_LOC, featuresbl);
+  if (featuresbl.length() == 0) {
+    generic_dout(0) << "WARNING: mon fs missing feature list.\n"
+            << "Assuming it is old-style and introducing one." << dendl;
+    //we only want the baseline ~v.18 features assumed to be on disk.
+    //If new features are introduced this code needs to disappear or
+    //be made smarter.
+    *features = get_legacy_features();
+
+    features->encode(featuresbl);
+    auto t(std::make_shared<MonitorDBStore::Transaction>());
+    t->put(MONITOR_NAME, COMPAT_SET_LOC, featuresbl);
+    store->apply_transaction(t);
+  } else {
+    auto it = featuresbl.cbegin();
+    features->decode(it);
+  }
+}
+
+void Monitor::read_features()
+{
+  read_features_off_disk(store, &features);
+  dout(10) << "features " << features << dendl;
+
+  calc_quorum_requirements();
+  dout(10) << "required_features " << required_features << dendl;
+}
+
+void Monitor::write_features(MonitorDBStore::TransactionRef t)
+{
+  bufferlist bl;
+  features.encode(bl);
+  t->put(MONITOR_NAME, COMPAT_SET_LOC, bl);
+}
+
+const char** Monitor::get_tracked_conf_keys() const
+{
+  static const char* KEYS[] = {
+    "crushtool", // helpful for testing
+    "mon_election_timeout",
+    "mon_lease",
+    "mon_lease_renew_interval_factor",
+    "mon_lease_ack_timeout_factor",
+    "mon_accept_timeout_factor",
+    // clog & admin clog
+    "clog_to_monitors",
+    "clog_to_syslog",
+    "clog_to_syslog_facility",
+    "clog_to_syslog_level",
+    "clog_to_graylog",
+    "clog_to_graylog_host",
+    "clog_to_graylog_port",
+    "host",
+    "fsid",
+    // periodic health to clog
+    "mon_health_to_clog",
+    "mon_health_to_clog_interval",
+    "mon_health_to_clog_tick_interval",
+    // scrub interval
+    "mon_scrub_interval",
+    "mon_allow_pool_delete",
+    // osdmap pruning - observed, not handled.
+    "mon_osdmap_full_prune_enabled",
+    "mon_osdmap_full_prune_min",
+    "mon_osdmap_full_prune_interval",
+    "mon_osdmap_full_prune_txsize",
+    // debug options - observed, not handled
+    "mon_debug_extra_checks",
+    "mon_debug_block_osdmap_trim",
+    NULL
+  };
+  return KEYS;
+}
+
+void Monitor::handle_conf_change(const ConfigProxy& conf,
+                                 const std::set<std::string> &changed)
+{
+  sanitize_options();
+
+  dout(10) << __func__ << " " << changed << dendl;
+
+  if (changed.count("clog_to_monitors") ||
+      changed.count("clog_to_syslog") ||
+      changed.count("clog_to_syslog_level") ||
+      changed.count("clog_to_syslog_facility") ||
+      changed.count("clog_to_graylog") ||
+      changed.count("clog_to_graylog_host") ||
+      changed.count("clog_to_graylog_port") ||
+      changed.count("host") ||
+      changed.count("fsid")) {
+    update_log_clients();
+  }
+
+  if (changed.count("mon_health_to_clog") ||
+      changed.count("mon_health_to_clog_interval") ||
+      changed.count("mon_health_to_clog_tick_interval")) {
+    finisher.queue(new C_MonContext{this, [this, changed](int) {
+      std::lock_guard l{lock};
+      health_to_clog_update_conf(changed);
+    }});
+  }
+
+  if (changed.count("mon_scrub_interval")) {
+    auto scrub_interval =
+      conf.get_val<std::chrono::seconds>("mon_scrub_interval");
+    finisher.queue(new C_MonContext{this, [this, scrub_interval](int) {
+      std::lock_guard l{lock};
+      scrub_update_interval(scrub_interval);
+    }});
+  }
+}
+
+void Monitor::update_log_clients()
+{
+  map<string,string> log_to_monitors;
+  map<string,string> log_to_syslog;
+  map<string,string> log_channel;
+  map<string,string> log_prio;
+  map<string,string> log_to_graylog;
+  map<string,string> log_to_graylog_host;
+  map<string,string> log_to_graylog_port;
+  uuid_d fsid;
+  string host;
+
+  if (parse_log_client_options(g_ceph_context, log_to_monitors, log_to_syslog,
+			       log_channel, log_prio, log_to_graylog,
+			       log_to_graylog_host, log_to_graylog_port,
+			       fsid, host))
+    return;
+
+  clog->update_config(log_to_monitors, log_to_syslog,
+		      log_channel, log_prio, log_to_graylog,
+		      log_to_graylog_host, log_to_graylog_port,
+		      fsid, host);
+
+  audit_clog->update_config(log_to_monitors, log_to_syslog,
+			    log_channel, log_prio, log_to_graylog,
+			    log_to_graylog_host, log_to_graylog_port,
+			    fsid, host);
+}
+
+int Monitor::sanitize_options()
+{
+  int r = 0;
+
+  // mon_lease must be greater than mon_lease_renewal; otherwise we
+  // may incur in leases expiring before they are renewed.
+  if (g_conf()->mon_lease_renew_interval_factor >= 1.0) {
+    clog->error() << "mon_lease_renew_interval_factor ("
+		  << g_conf()->mon_lease_renew_interval_factor
+		  << ") must be less than 1.0";
+    r = -EINVAL;
+  }
+
+  // mon_lease_ack_timeout must be greater than mon_lease to make sure we've
+  // got time to renew the lease and get an ack for it. Having both options
+  // with the same value, for a given small vale, could mean timing out if
+  // the monitors happened to be overloaded -- or even under normal load for
+  // a small enough value.
+  if (g_conf()->mon_lease_ack_timeout_factor <= 1.0) {
+    clog->error() << "mon_lease_ack_timeout_factor ("
+		  << g_conf()->mon_lease_ack_timeout_factor
+		  << ") must be greater than 1.0";
+    r = -EINVAL;
+  }
+
+  return r;
+}
+
+int Monitor::preinit()
+{
+  std::unique_lock l(lock);
+
+  dout(1) << "preinit fsid " << monmap->fsid << dendl;
+
+  int r = sanitize_options();
+  if (r < 0) {
+    derr << "option sanitization failed!" << dendl;
+    return r;
+  }
+
+  ceph_assert(!logger);
+  {
+    PerfCountersBuilder pcb(g_ceph_context, "mon", l_mon_first, l_mon_last);
+    pcb.add_u64(l_mon_num_sessions, "num_sessions", "Open sessions", "sess",
+        PerfCountersBuilder::PRIO_USEFUL);
+    pcb.add_u64_counter(l_mon_session_add, "session_add", "Created sessions",
+        "sadd", PerfCountersBuilder::PRIO_INTERESTING);
+    pcb.add_u64_counter(l_mon_session_rm, "session_rm", "Removed sessions",
+        "srm", PerfCountersBuilder::PRIO_INTERESTING);
+    pcb.add_u64_counter(l_mon_session_trim, "session_trim", "Trimmed sessions",
+        "strm", PerfCountersBuilder::PRIO_USEFUL);
+    pcb.add_u64_counter(l_mon_num_elections, "num_elections", "Elections participated in",
+        "ecnt", PerfCountersBuilder::PRIO_USEFUL);
+    pcb.add_u64_counter(l_mon_election_call, "election_call", "Elections started",
+        "estt", PerfCountersBuilder::PRIO_INTERESTING);
+    pcb.add_u64_counter(l_mon_election_win, "election_win", "Elections won",
+        "ewon", PerfCountersBuilder::PRIO_INTERESTING);
+    pcb.add_u64_counter(l_mon_election_lose, "election_lose", "Elections lost",
+        "elst", PerfCountersBuilder::PRIO_INTERESTING);
+    logger = pcb.create_perf_counters();
+    cct->get_perfcounters_collection()->add(logger);
+  }
+
+  ceph_assert(!cluster_logger);
+  {
+    PerfCountersBuilder pcb(g_ceph_context, "cluster", l_cluster_first, l_cluster_last);
+    pcb.add_u64(l_cluster_num_mon, "num_mon", "Monitors");
+    pcb.add_u64(l_cluster_num_mon_quorum, "num_mon_quorum", "Monitors in quorum");
+    pcb.add_u64(l_cluster_num_osd, "num_osd", "OSDs");
+    pcb.add_u64(l_cluster_num_osd_up, "num_osd_up", "OSDs that are up");
+    pcb.add_u64(l_cluster_num_osd_in, "num_osd_in", "OSD in state \"in\" (they are in cluster)");
+    pcb.add_u64(l_cluster_osd_epoch, "osd_epoch", "Current epoch of OSD map");
+    pcb.add_u64(l_cluster_osd_bytes, "osd_bytes", "Total capacity of cluster", NULL, 0, unit_t(UNIT_BYTES));
+    pcb.add_u64(l_cluster_osd_bytes_used, "osd_bytes_used", "Used space", NULL, 0, unit_t(UNIT_BYTES));
+    pcb.add_u64(l_cluster_osd_bytes_avail, "osd_bytes_avail", "Available space", NULL, 0, unit_t(UNIT_BYTES));
+    pcb.add_u64(l_cluster_num_pool, "num_pool", "Pools");
+    pcb.add_u64(l_cluster_num_pg, "num_pg", "Placement groups");
+    pcb.add_u64(l_cluster_num_pg_active_clean, "num_pg_active_clean", "Placement groups in active+clean state");
+    pcb.add_u64(l_cluster_num_pg_active, "num_pg_active", "Placement groups in active state");
+    pcb.add_u64(l_cluster_num_pg_peering, "num_pg_peering", "Placement groups in peering state");
+    pcb.add_u64(l_cluster_num_object, "num_object", "Objects");
+    pcb.add_u64(l_cluster_num_object_degraded, "num_object_degraded", "Degraded (missing replicas) objects");
+    pcb.add_u64(l_cluster_num_object_misplaced, "num_object_misplaced", "Misplaced (wrong location in the cluster) objects");
+    pcb.add_u64(l_cluster_num_object_unfound, "num_object_unfound", "Unfound objects");
+    pcb.add_u64(l_cluster_num_bytes, "num_bytes", "Size of all objects", NULL, 0, unit_t(UNIT_BYTES));
+    cluster_logger = pcb.create_perf_counters();
+  }
+
+  paxos->init_logger();
+
+  // verify cluster_uuid
+  {
+    int r = check_fsid();
+    if (r == -ENOENT)
+      r = write_fsid();
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  // open compatset
+  read_features();
+
+  // have we ever joined a quorum?
+  has_ever_joined = (store->get(MONITOR_NAME, "joined") != 0);
+  dout(10) << "has_ever_joined = " << (int)has_ever_joined << dendl;
+
+  if (!has_ever_joined) {
+    // impose initial quorum restrictions?
+    list<string> initial_members;
+    get_str_list(g_conf()->mon_initial_members, initial_members);
+
+    if (!initial_members.empty()) {
+      dout(1) << " initial_members " << initial_members << ", filtering seed monmap" << dendl;
+
+      monmap->set_initial_members(
+	g_ceph_context, initial_members, name, messenger->get_myaddrs(),
+	&extra_probe_peers);
+
+      dout(10) << " monmap is " << *monmap << dendl;
+      dout(10) << " extra probe peers " << extra_probe_peers << dendl;
+    }
+  } else if (!monmap->contains(name)) {
+    derr << "not in monmap and have been in a quorum before; "
+         << "must have been removed" << dendl;
+    if (g_conf()->mon_force_quorum_join) {
+      dout(0) << "we should have died but "
+              << "'mon_force_quorum_join' is set -- allowing boot" << dendl;
+    } else {
+      derr << "commit suicide!" << dendl;
+      return -ENOENT;
+    }
+  }
+
+  {
+    // We have a potentially inconsistent store state in hands. Get rid of it
+    // and start fresh.
+    bool clear_store = false;
+    if (store->exists("mon_sync", "in_sync")) {
+      dout(1) << __func__ << " clean up potentially inconsistent store state"
+	      << dendl;
+      clear_store = true;
+    }
+
+    if (store->get("mon_sync", "force_sync") > 0) {
+      dout(1) << __func__ << " force sync by clearing store state" << dendl;
+      clear_store = true;
+    }
+
+    if (clear_store) {
+      set<string> sync_prefixes = get_sync_targets_names();
+      store->clear(sync_prefixes);
+    }
+  }
+
+  sync_last_committed_floor = store->get("mon_sync", "last_committed_floor");
+  dout(10) << "sync_last_committed_floor " << sync_last_committed_floor << dendl;
+
+  init_paxos();
+
+  if (is_keyring_required()) {
+    // we need to bootstrap authentication keys so we can form an
+    // initial quorum.
+    if (authmon()->get_last_committed() == 0) {
+      dout(10) << "loading initial keyring to bootstrap authentication for mkfs" << dendl;
+      bufferlist bl;
+      int err = store->get("mkfs", "keyring", bl);
+      if (err == 0 && bl.length() > 0) {
+        // Attempt to decode and extract keyring only if it is found.
+        KeyRing keyring;
+        auto p = bl.cbegin();
+        decode(keyring, p);
+        extract_save_mon_key(keyring);
+      }
+    }
+
+    string keyring_loc = g_conf()->mon_data + "/keyring";
+
+    r = keyring.load(cct, keyring_loc);
+    if (r < 0) {
+      EntityName mon_name;
+      mon_name.set_type(CEPH_ENTITY_TYPE_MON);
+      EntityAuth mon_key;
+      if (key_server.get_auth(mon_name, mon_key)) {
+	dout(1) << "copying mon. key from old db to external keyring" << dendl;
+	keyring.add(mon_name, mon_key);
+	bufferlist bl;
+	keyring.encode_plaintext(bl);
+	write_default_keyring(bl);
+      } else {
+	derr << "unable to load initial keyring " << g_conf()->keyring << dendl;
+	return r;
+      }
+    }
+  }
+
+  admin_hook = new AdminHook(this);
+  AdminSocket* admin_socket = cct->get_admin_socket();
+
+  // unlock while registering to avoid mon_lock -> admin socket lock dependency.
+  l.unlock();
+  // register tell/asock commands
+  for (const auto& command : local_mon_commands) {
+    if (!command.is_tell()) {
+      continue;
+    }
+    const auto prefix = cmddesc_get_prefix(command.cmdstring);
+    if (prefix == "injectargs" ||
+	prefix == "version" ||
+	prefix == "tell") {
+      // not registerd by me
+      continue;
+    }
+    r = admin_socket->register_command(command.cmdstring, admin_hook,
+				       command.helpstring);
+    ceph_assert(r == 0);
+  }
+  l.lock();
+
+  // add ourselves as a conf observer
+  g_conf().add_observer(this);
+
+  messenger->set_auth_client(this);
+  messenger->set_auth_server(this);
+  mgr_messenger->set_auth_client(this);
+
+  auth_registry.refresh_config();
+
+  return 0;
+}
+
+int Monitor::init()
+{
+  dout(2) << "init" << dendl;
+  std::lock_guard l(lock);
+
+  finisher.start();
+
+  // start ticker
+  timer.init();
+  new_tick();
+
+  cpu_tp.start();
+
+  // i'm ready!
+  messenger->add_dispatcher_tail(this);
+
+  // kickstart pet mgrclient
+  mgr_client.init();
+  mgr_messenger->add_dispatcher_tail(&mgr_client);
+  mgr_messenger->add_dispatcher_tail(this);  // for auth ms_* calls
+  mgrmon()->prime_mgr_client();
+
+  state = STATE_PROBING;
+
+  bootstrap();
+
+  if (!elector.peer_tracker_is_clean()){
+    dout(10) << "peer_tracker looks inconsistent"
+      << " previous bad logic, clearing ..." << dendl;
+    elector.notify_clear_peer_state();
+  }
+
+  // add features of myself into feature_map
+  session_map.feature_map.add_mon(con_self->get_features());
+  return 0;
+}
+
+void Monitor::init_paxos()
+{
+  dout(10) << __func__ << dendl;
+  paxos->init();
+
+  // init services
+  for (auto& svc : paxos_service) {
+    svc->init();
+  }
+
+  refresh_from_paxos(NULL);
+}
+
+void Monitor::refresh_from_paxos(bool *need_bootstrap)
+{
+  dout(10) << __func__ << dendl;
+
+  bufferlist bl;
+  int r = store->get(MONITOR_NAME, "cluster_fingerprint", bl);
+  if (r >= 0) {
+    try {
+      auto p = bl.cbegin();
+      decode(fingerprint, p);
+    }
+    catch (ceph::buffer::error& e) {
+      dout(10) << __func__ << " failed to decode cluster_fingerprint" << dendl;
+    }
+  } else {
+    dout(10) << __func__ << " no cluster_fingerprint" << dendl;
+  }
+
+  for (auto& svc : paxos_service) {
+    svc->refresh(need_bootstrap);
+  }
+  for (auto& svc : paxos_service) {
+    svc->post_refresh();
+  }
+  load_metadata();
+}
+
+void Monitor::register_cluster_logger()
+{
+  if (!cluster_logger_registered) {
+    dout(10) << "register_cluster_logger" << dendl;
+    cluster_logger_registered = true;
+    cct->get_perfcounters_collection()->add(cluster_logger);
+  } else {
+    dout(10) << "register_cluster_logger - already registered" << dendl;
+  }
+}
+
+void Monitor::unregister_cluster_logger()
+{
+  if (cluster_logger_registered) {
+    dout(10) << "unregister_cluster_logger" << dendl;
+    cluster_logger_registered = false;
+    cct->get_perfcounters_collection()->remove(cluster_logger);
+  } else {
+    dout(10) << "unregister_cluster_logger - not registered" << dendl;
+  }
+}
+
+void Monitor::update_logger()
+{
+  cluster_logger->set(l_cluster_num_mon, monmap->size());
+  cluster_logger->set(l_cluster_num_mon_quorum, quorum.size());
+}
+
+void Monitor::shutdown()
+{
+  dout(1) << "shutdown" << dendl;
+
+  lock.lock();
+
+  wait_for_paxos_write();
+
+  {
+    std::lock_guard l(auth_lock);
+    authmon()->_set_mon_num_rank(0, 0);
+  }
+
+  state = STATE_SHUTDOWN;
+
+  lock.unlock();
+  g_conf().remove_observer(this);
+  lock.lock();
+
+  if (admin_hook) {
+    cct->get_admin_socket()->unregister_commands(admin_hook);
+    delete admin_hook;
+    admin_hook = NULL;
+  }
+
+  elector.shutdown();
+
+  mgr_client.shutdown();
+
+  lock.unlock();
+  finisher.wait_for_empty();
+  finisher.stop();
+  lock.lock();
+
+  // clean up
+  paxos->shutdown();
+  for (auto& svc : paxos_service) {
+    svc->shutdown();
+  }
+
+  finish_contexts(g_ceph_context, waitfor_quorum, -ECANCELED);
+  finish_contexts(g_ceph_context, maybe_wait_for_quorum, -ECANCELED);
+
+  timer.shutdown();
+
+  cpu_tp.stop();
+
+  remove_all_sessions();
+
+  log_client.shutdown();
+
+  // unlock before msgr shutdown...
+  lock.unlock();
+
+  // shutdown messenger before removing logger from perfcounter collection, 
+  // otherwise _ms_dispatch() will try to update deleted logger
+  messenger->shutdown();
+  mgr_messenger->shutdown();
+
+  if (logger) {
+    cct->get_perfcounters_collection()->remove(logger);
+  }
+  if (cluster_logger) {
+    if (cluster_logger_registered)
+      cct->get_perfcounters_collection()->remove(cluster_logger);
+    delete cluster_logger;
+    cluster_logger = NULL;
+  }
+}
+
+void Monitor::wait_for_paxos_write()
+{
+  if (paxos->is_writing() || paxos->is_writing_previous()) {
+    dout(10) << __func__ << " flushing pending write" << dendl;
+    lock.unlock();
+    store->flush();
+    lock.lock();
+    dout(10) << __func__ << " flushed pending write" << dendl;
+  }
+}
+
+void Monitor::respawn()
+{
+  // --- WARNING TO FUTURE COPY/PASTERS ---
+  // You must also add a call like
+  //
+  //   ceph_pthread_setname(pthread_self(), "ceph-mon");
+  //
+  // to main() so that /proc/$pid/stat field 2 contains "(ceph-mon)"
+  // instead of "(exe)", so that killall (and log rotation) will work.
+
+  dout(0) << __func__ << dendl;
+
+  char *new_argv[orig_argc+1];
+  dout(1) << " e: '" << orig_argv[0] << "'" << dendl;
+  for (int i=0; i<orig_argc; i++) {
+    new_argv[i] = (char *)orig_argv[i];
+    dout(1) << " " << i << ": '" << orig_argv[i] << "'" << dendl;
+  }
+  new_argv[orig_argc] = NULL;
+
+  /* Determine the path to our executable, test if Linux /proc/self/exe exists.
+   * This allows us to exec the same executable even if it has since been
+   * unlinked.
+   */
+  char exe_path[PATH_MAX] = "";
+#ifdef PROCPREFIX
+  if (readlink(PROCPREFIX "/proc/self/exe", exe_path, PATH_MAX-1) != -1) {
+    dout(1) << "respawning with exe " << exe_path << dendl;
+    strcpy(exe_path, PROCPREFIX "/proc/self/exe");
+  } else {
+#else
+  {
+#endif
+    /* Print CWD for the user's interest */
+    char buf[PATH_MAX];
+    char *cwd = getcwd(buf, sizeof(buf));
+    ceph_assert(cwd);
+    dout(1) << " cwd " << cwd << dendl;
+
+    /* Fall back to a best-effort: just running in our CWD */
+    strncpy(exe_path, orig_argv[0], PATH_MAX-1);
+  }
+
+  dout(1) << " exe_path " << exe_path << dendl;
+
+  unblock_all_signals(NULL);
+  execv(exe_path, new_argv);
+
+  dout(0) << "respawn execv " << orig_argv[0]
+	  << " failed with " << cpp_strerror(errno) << dendl;
+
+  // We have to assert out here, because suicide() returns, and callers
+  // to respawn expect it never to return.
+  ceph_abort();
+}
+
+void Monitor::bootstrap()
+{
+  dout(10) << "bootstrap" << dendl;
+  wait_for_paxos_write();
+
+  sync_reset_requester();
+  unregister_cluster_logger();
+  cancel_probe_timeout();
+
+  if (monmap->get_epoch() == 0) {
+    dout(10) << "reverting to legacy ranks for seed monmap (epoch 0)" << dendl;
+    monmap->calc_legacy_ranks();
+  }
+  dout(10) << "monmap " << *monmap << dendl;
+  {
+    auto from_release = monmap->min_mon_release;
+    ostringstream err;
+    if (!can_upgrade_from(from_release, "min_mon_release", err)) {
+      derr << "current monmap has " << err.str() << " stopping." << dendl;
+      exit(0);
+    }
+  }
+  // note my rank
+  int newrank = monmap->get_rank(messenger->get_myaddrs());
+  if (newrank < 0 && rank >= 0) {
+    // was i ever part of the quorum?
+    if (has_ever_joined) {
+      dout(0) << " removed from monmap, suicide." << dendl;
+      exit(0);
+    }
+    elector.notify_clear_peer_state();
+  }
+  if (newrank >= 0 &&
+      monmap->get_addrs(newrank) != messenger->get_myaddrs()) {
+    dout(0) << " monmap addrs for rank " << newrank << " changed, i am "
+	    << messenger->get_myaddrs()
+	    << ", monmap is " << monmap->get_addrs(newrank) << ", respawning"
+	    << dendl;
+
+    if (monmap->get_epoch()) {
+      // store this map in temp mon_sync location so that we use it on
+      // our next startup
+      derr << " stashing newest monmap " << monmap->get_epoch()
+	   << " for next startup" << dendl;
+      bufferlist bl;
+      monmap->encode(bl, -1);
+      auto t(std::make_shared<MonitorDBStore::Transaction>());
+      t->put("mon_sync", "temp_newer_monmap", bl);
+      store->apply_transaction(t);
+    }
+
+    respawn();
+  }
+  if (newrank != rank) {
+    dout(0) << " my rank is now " << newrank << " (was " << rank << ")" << dendl;
+    messenger->set_myname(entity_name_t::MON(newrank));
+    rank = newrank;
+    elector.notify_rank_changed(rank);
+
+    // reset all connections, or else our peers will think we are someone else.
+    messenger->mark_down_all();
+  }
+
+  // reset
+  state = STATE_PROBING;
+
+  _reset();
+
+  // sync store
+  if (g_conf()->mon_compact_on_bootstrap) {
+    dout(10) << "bootstrap -- triggering compaction" << dendl;
+    store->compact();
+    dout(10) << "bootstrap -- finished compaction" << dendl;
+  }
+
+  // stretch mode bits
+  set_elector_disallowed_leaders(false);
+
+  // singleton monitor?
+  if (monmap->size() == 1 && rank == 0) {
+    win_standalone_election();
+    return;
+  }
+
+  reset_probe_timeout();
+
+  // i'm outside the quorum
+  if (monmap->contains(name))
+    outside_quorum.insert(name);
+
+  // probe monitors
+  dout(10) << "probing other monitors" << dendl;
+  for (unsigned i = 0; i < monmap->size(); i++) {
+    if ((int)i != rank)
+      send_mon_message(
+	new MMonProbe(monmap->fsid, MMonProbe::OP_PROBE, name, has_ever_joined,
+		      ceph_release()),
+	i);
+  }
+  for (auto& av : extra_probe_peers) {
+    if (av != messenger->get_myaddrs()) {
+      messenger->send_to_mon(
+	new MMonProbe(monmap->fsid, MMonProbe::OP_PROBE, name, has_ever_joined,
+		      ceph_release()),
+	av);
+    }
+  }
+}
+
+bool Monitor::_add_bootstrap_peer_hint(std::string_view cmd,
+				       const cmdmap_t& cmdmap,
+				       ostream& ss)
+{
+  if (is_leader() || is_peon()) {
+    ss << "mon already active; ignoring bootstrap hint";
+    return true;
+  }
+
+  entity_addrvec_t addrs;
+  string addrstr;
+  if (cmd_getval(cmdmap, "addr", addrstr)) {
+    dout(10) << "_add_bootstrap_peer_hint '" << cmd << "' addr '"
+	     << addrstr << "'" << dendl;
+
+    entity_addr_t addr;
+    const char *end = 0;
+    if (!addr.parse(addrstr.c_str(), &end, entity_addr_t::TYPE_ANY)) {
+      ss << "failed to parse addrs '" << addrstr
+	 << "'; syntax is 'add_bootstrap_peer_hint ip[:port]'";
+      return false;
+    }
+
+    addrs.v.push_back(addr);
+    if (addr.get_port() == 0) {
+      addrs.v[0].set_type(entity_addr_t::TYPE_MSGR2);
+      addrs.v[0].set_port(CEPH_MON_PORT_IANA);
+      addrs.v.push_back(addr);
+      addrs.v[1].set_type(entity_addr_t::TYPE_LEGACY);
+      addrs.v[1].set_port(CEPH_MON_PORT_LEGACY);
+    } else if (addr.get_type() == entity_addr_t::TYPE_ANY) {
+      if (addr.get_port() == CEPH_MON_PORT_LEGACY) {
+	addrs.v[0].set_type(entity_addr_t::TYPE_LEGACY);
+      } else {
+	addrs.v[0].set_type(entity_addr_t::TYPE_MSGR2);
+      }
+    }
+  } else if (cmd_getval(cmdmap, "addrv", addrstr)) {
+    dout(10) << "_add_bootstrap_peer_hintv '" << cmd << "' addrv '"
+	     << addrstr << "'" << dendl;
+    const char *end = 0;
+    if (!addrs.parse(addrstr.c_str(), &end)) {
+      ss << "failed to parse addrs '" << addrstr
+	 << "'; syntax is 'add_bootstrap_peer_hintv v2:ip:port[,v1:ip:port]'";
+      return false;
+    }
+  } else {
+    ss << "no addr or addrv provided";
+    return false;
+  }
+
+  extra_probe_peers.insert(addrs);
+  ss << "adding peer " << addrs << " to list: " << extra_probe_peers;
+  return true;
+}
+
+// called by bootstrap(), or on leader|peon -> electing
+void Monitor::_reset()
+{
+  dout(10) << __func__ << dendl;
+
+  // disable authentication
+  {
+    std::lock_guard l(auth_lock);
+    authmon()->_set_mon_num_rank(0, 0);
+  }
+
+  cancel_probe_timeout();
+  timecheck_finish();
+  health_events_cleanup();
+  health_check_log_times.clear();
+  scrub_event_cancel();
+
+  leader_since = utime_t();
+  quorum_since = {};
+  if (!quorum.empty()) {
+    exited_quorum = ceph_clock_now();
+  }
+  quorum.clear();
+  outside_quorum.clear();
+  quorum_feature_map.clear();
+
+  scrub_reset();
+
+  paxos->restart();
+
+  for (auto& svc : paxos_service) {
+    svc->restart();
+  }
+}
+
+
+// -----------------------------------------------------------
+// sync
+
+set<string> Monitor::get_sync_targets_names()
+{
+  set<string> targets;
+  targets.insert(paxos->get_name());
+  for (auto& svc : paxos_service) {
+    svc->get_store_prefixes(targets);
+  }
+  return targets;
+}
+
+
+void Monitor::sync_timeout()
+{
+  dout(10) << __func__ << dendl;
+  ceph_assert(state == STATE_SYNCHRONIZING);
+  bootstrap();
+}
+
+void Monitor::sync_obtain_latest_monmap(bufferlist &bl)
+{
+  dout(1) << __func__ << dendl;
+
+  MonMap latest_monmap;
+
+  // Grab latest monmap from MonmapMonitor
+  bufferlist monmon_bl;
+  int err = monmon()->get_monmap(monmon_bl);
+  if (err < 0) {
+    if (err != -ENOENT) {
+      derr << __func__
+           << " something wrong happened while reading the store: "
+           << cpp_strerror(err) << dendl;
+      ceph_abort_msg("error reading the store");
+    }
+  } else {
+    latest_monmap.decode(monmon_bl);
+  }
+
+  // Grab last backed up monmap (if any) and compare epochs
+  if (store->exists("mon_sync", "latest_monmap")) {
+    bufferlist backup_bl;
+    int err = store->get("mon_sync", "latest_monmap", backup_bl);
+    if (err < 0) {
+      derr << __func__
+           << " something wrong happened while reading the store: "
+           << cpp_strerror(err) << dendl;
+      ceph_abort_msg("error reading the store");
+    }
+    ceph_assert(backup_bl.length() > 0);
+
+    MonMap backup_monmap;
+    backup_monmap.decode(backup_bl);
+
+    if (backup_monmap.epoch > latest_monmap.epoch)
+      latest_monmap = backup_monmap;
+  }
+
+  // Check if our current monmap's epoch is greater than the one we've
+  // got so far.
+  if (monmap->epoch > latest_monmap.epoch)
+    latest_monmap = *monmap;
+
+  dout(1) << __func__ << " obtained monmap e" << latest_monmap.epoch << dendl;
+
+  latest_monmap.encode(bl, CEPH_FEATURES_ALL);
+}
+
+void Monitor::sync_reset_requester()
+{
+  dout(10) << __func__ << dendl;
+
+  if (sync_timeout_event) {
+    timer.cancel_event(sync_timeout_event);
+    sync_timeout_event = NULL;
+  }
+
+  sync_provider = entity_addrvec_t();
+  sync_cookie = 0;
+  sync_full = false;
+  sync_start_version = 0;
+}
+
+void Monitor::sync_reset_provider()
+{
+  dout(10) << __func__ << dendl;
+  sync_providers.clear();
+}
+
+void Monitor::sync_start(entity_addrvec_t &addrs, bool full)
+{
+  dout(10) << __func__ << " " << addrs << (full ? " full" : " recent") << dendl;
+
+  ceph_assert(state == STATE_PROBING ||
+	 state == STATE_SYNCHRONIZING);
+  state = STATE_SYNCHRONIZING;
+
+  // make sure are not a provider for anyone!
+  sync_reset_provider();
+
+  sync_full = full;
+
+  if (sync_full) {
+    // stash key state, and mark that we are syncing
+    auto t(std::make_shared<MonitorDBStore::Transaction>());
+    sync_stash_critical_state(t);
+    t->put("mon_sync", "in_sync", 1);
+
+    sync_last_committed_floor = std::max(sync_last_committed_floor, paxos->get_version());
+    dout(10) << __func__ << " marking sync in progress, storing sync_last_committed_floor "
+	     << sync_last_committed_floor << dendl;
+    t->put("mon_sync", "last_committed_floor", sync_last_committed_floor);
+
+    store->apply_transaction(t);
+
+    ceph_assert(g_conf()->mon_sync_requester_kill_at != 1);
+
+    // clear the underlying store
+    set<string> targets = get_sync_targets_names();
+    dout(10) << __func__ << " clearing prefixes " << targets << dendl;
+    store->clear(targets);
+
+    // make sure paxos knows it has been reset.  this prevents a
+    // bootstrap and then different probe reply order from possibly
+    // deciding a partial or no sync is needed.
+    paxos->init();
+
+    ceph_assert(g_conf()->mon_sync_requester_kill_at != 2);
+  }
+
+  // assume 'other' as the leader. We will update the leader once we receive
+  // a reply to the sync start.
+  sync_provider = addrs;
+
+  sync_reset_timeout();
+
+  MMonSync *m = new MMonSync(sync_full ? MMonSync::OP_GET_COOKIE_FULL : MMonSync::OP_GET_COOKIE_RECENT);
+  if (!sync_full)
+    m->last_committed = paxos->get_version();
+  messenger->send_to_mon(m, sync_provider);
+}
+
+void Monitor::sync_stash_critical_state(MonitorDBStore::TransactionRef t)
+{
+  dout(10) << __func__ << dendl;
+  bufferlist backup_monmap;
+  sync_obtain_latest_monmap(backup_monmap);
+  ceph_assert(backup_monmap.length() > 0);
+  t->put("mon_sync", "latest_monmap", backup_monmap);
+}
+
+void Monitor::sync_reset_timeout()
+{
+  dout(10) << __func__ << dendl;
+  if (sync_timeout_event)
+    timer.cancel_event(sync_timeout_event);
+  sync_timeout_event = timer.add_event_after(
+    g_conf()->mon_sync_timeout,
+    new C_MonContext{this, [this](int) {
+	sync_timeout();
+      }});
+}
+
+void Monitor::sync_finish(version_t last_committed)
+{
+  dout(10) << __func__ << " lc " << last_committed << " from " << sync_provider << dendl;
+
+  ceph_assert(g_conf()->mon_sync_requester_kill_at != 7);
+
+  if (sync_full) {
+    // finalize the paxos commits
+    auto tx(std::make_shared<MonitorDBStore::Transaction>());
+    paxos->read_and_prepare_transactions(tx, sync_start_version,
+					 last_committed);
+    tx->put(paxos->get_name(), "last_committed", last_committed);
+
+    dout(30) << __func__ << " final tx dump:\n";
+    JSONFormatter f(true);
+    tx->dump(&f);
+    f.flush(*_dout);
+    *_dout << dendl;
+
+    store->apply_transaction(tx);
+  }
+
+  ceph_assert(g_conf()->mon_sync_requester_kill_at != 8);
+
+  auto t(std::make_shared<MonitorDBStore::Transaction>());
+  t->erase("mon_sync", "in_sync");
+  t->erase("mon_sync", "force_sync");
+  t->erase("mon_sync", "last_committed_floor");
+  store->apply_transaction(t);
+
+  ceph_assert(g_conf()->mon_sync_requester_kill_at != 9);
+
+  init_paxos();
+
+  ceph_assert(g_conf()->mon_sync_requester_kill_at != 10);
+
+  bootstrap();
+}
+
+void Monitor::handle_sync(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonSync>();
+  dout(10) << __func__ << " " << *m << dendl;
+  switch (m->op) {
+
+    // provider ---------
+
+  case MMonSync::OP_GET_COOKIE_FULL:
+  case MMonSync::OP_GET_COOKIE_RECENT:
+    handle_sync_get_cookie(op);
+    break;
+  case MMonSync::OP_GET_CHUNK:
+    handle_sync_get_chunk(op);
+    break;
+
+    // client -----------
+
+  case MMonSync::OP_COOKIE:
+    handle_sync_cookie(op);
+    break;
+
+  case MMonSync::OP_CHUNK:
+  case MMonSync::OP_LAST_CHUNK:
+    handle_sync_chunk(op);
+    break;
+  case MMonSync::OP_NO_COOKIE:
+    handle_sync_no_cookie(op);
+    break;
+
+  default:
+    dout(0) << __func__ << " unknown op " << m->op << dendl;
+    ceph_abort_msg("unknown op");
+  }
+}
+
+// leader
+
+void Monitor::_sync_reply_no_cookie(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonSync>();
+  MMonSync *reply = new MMonSync(MMonSync::OP_NO_COOKIE, m->cookie);
+  m->get_connection()->send_message(reply);
+}
+
+void Monitor::handle_sync_get_cookie(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonSync>();
+  if (is_synchronizing()) {
+    _sync_reply_no_cookie(op);
+    return;
+  }
+
+  ceph_assert(g_conf()->mon_sync_provider_kill_at != 1);
+
+  // make sure they can understand us.
+  if ((required_features ^ m->get_connection()->get_features()) &
+      required_features) {
+    dout(5) << " ignoring peer mon." << m->get_source().num()
+	    << " has features " << std::hex
+	    << m->get_connection()->get_features()
+	    << " but we require " << required_features << std::dec << dendl;
+    return;
+  }
+
+  // make up a unique cookie.  include election epoch (which persists
+  // across restarts for the whole cluster) and a counter for this
+  // process instance.  there is no need to be unique *across*
+  // monitors, though.
+  uint64_t cookie = ((unsigned long long)elector.get_epoch() << 24) + ++sync_provider_count;
+  ceph_assert(sync_providers.count(cookie) == 0);
+
+  dout(10) << __func__ << " cookie " << cookie << " for " << m->get_source_inst() << dendl;
+
+  SyncProvider& sp = sync_providers[cookie];
+  sp.cookie = cookie;
+  sp.addrs = m->get_source_addrs();
+  sp.reset_timeout(g_ceph_context, g_conf()->mon_sync_timeout * 2);
+
+  set<string> sync_targets;
+  if (m->op == MMonSync::OP_GET_COOKIE_FULL) {
+    // full scan
+    sync_targets = get_sync_targets_names();
+    sp.last_committed = paxos->get_version();
+    sp.synchronizer = store->get_synchronizer(sp.last_key, sync_targets);
+    sp.full = true;
+    dout(10) << __func__ << " will sync prefixes " << sync_targets << dendl;
+  } else {
+    // just catch up paxos
+    sp.last_committed = m->last_committed;
+  }
+  dout(10) << __func__ << " will sync from version " << sp.last_committed << dendl;
+
+  MMonSync *reply = new MMonSync(MMonSync::OP_COOKIE, sp.cookie);
+  reply->last_committed = sp.last_committed;
+  m->get_connection()->send_message(reply);
+}
+
+void Monitor::handle_sync_get_chunk(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonSync>();
+  dout(10) << __func__ << " " << *m << dendl;
+
+  if (sync_providers.count(m->cookie) == 0) {
+    dout(10) << __func__ << " no cookie " << m->cookie << dendl;
+    _sync_reply_no_cookie(op);
+    return;
+  }
+
+  ceph_assert(g_conf()->mon_sync_provider_kill_at != 2);
+
+  SyncProvider& sp = sync_providers[m->cookie];
+  sp.reset_timeout(g_ceph_context, g_conf()->mon_sync_timeout * 2);
+
+  if (sp.last_committed < paxos->get_first_committed() &&
+      paxos->get_first_committed() > 1) {
+    dout(10) << __func__ << " sync requester fell behind paxos, their lc " << sp.last_committed
+	     << " < our fc " << paxos->get_first_committed() << dendl;
+    sync_providers.erase(m->cookie);
+    _sync_reply_no_cookie(op);
+    return;
+  }
+
+  MMonSync *reply = new MMonSync(MMonSync::OP_CHUNK, sp.cookie);
+  auto tx(std::make_shared<MonitorDBStore::Transaction>());
+
+  int bytes_left = g_conf()->mon_sync_max_payload_size;
+  int keys_left = g_conf()->mon_sync_max_payload_keys;
+  while (sp.last_committed < paxos->get_version() &&
+	 bytes_left > 0 &&
+	 keys_left > 0) {
+    bufferlist bl;
+    sp.last_committed++;
+
+    int err = store->get(paxos->get_name(), sp.last_committed, bl);
+    ceph_assert(err == 0);
+
+    tx->put(paxos->get_name(), sp.last_committed, bl);
+    bytes_left -= bl.length();
+    --keys_left;
+    dout(20) << __func__ << " including paxos state " << sp.last_committed
+	     << dendl;
+  }
+  reply->last_committed = sp.last_committed;
+
+  if (sp.full && bytes_left > 0 && keys_left > 0) {
+    sp.synchronizer->get_chunk_tx(tx, bytes_left, keys_left);
+    sp.last_key = sp.synchronizer->get_last_key();
+    reply->last_key = sp.last_key;
+  }
+
+  if ((sp.full && sp.synchronizer->has_next_chunk()) ||
+      sp.last_committed < paxos->get_version()) {
+    dout(10) << __func__ << " chunk, through version " << sp.last_committed
+	     << " key " << sp.last_key << dendl;
+  } else {
+    dout(10) << __func__ << " last chunk, through version " << sp.last_committed
+	     << " key " << sp.last_key << dendl;
+    reply->op = MMonSync::OP_LAST_CHUNK;
+
+    ceph_assert(g_conf()->mon_sync_provider_kill_at != 3);
+
+    // clean up our local state
+    sync_providers.erase(sp.cookie);
+  }
+
+  encode(*tx, reply->chunk_bl);
+
+  m->get_connection()->send_message(reply);
+}
+
+// requester
+
+void Monitor::handle_sync_cookie(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonSync>();
+  dout(10) << __func__ << " " << *m << dendl;
+  if (sync_cookie) {
+    dout(10) << __func__ << " already have a cookie, ignoring" << dendl;
+    return;
+  }
+  if (m->get_source_addrs() != sync_provider) {
+    dout(10) << __func__ << " source does not match, discarding" << dendl;
+    return;
+  }
+  sync_cookie = m->cookie;
+  sync_start_version = m->last_committed;
+
+  sync_reset_timeout();
+  sync_get_next_chunk();
+
+  ceph_assert(g_conf()->mon_sync_requester_kill_at != 3);
+}
+
+void Monitor::sync_get_next_chunk()
+{
+  dout(20) << __func__ << " cookie " << sync_cookie << " provider " << sync_provider << dendl;
+  if (g_conf()->mon_inject_sync_get_chunk_delay > 0) {
+    dout(20) << __func__ << " injecting delay of " << g_conf()->mon_inject_sync_get_chunk_delay << dendl;
+    usleep((long long)(g_conf()->mon_inject_sync_get_chunk_delay * 1000000.0));
+  }
+  MMonSync *r = new MMonSync(MMonSync::OP_GET_CHUNK, sync_cookie);
+  messenger->send_to_mon(r, sync_provider);
+
+  ceph_assert(g_conf()->mon_sync_requester_kill_at != 4);
+}
+
+void Monitor::handle_sync_chunk(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonSync>();
+  dout(10) << __func__ << " " << *m << dendl;
+
+  if (m->cookie != sync_cookie) {
+    dout(10) << __func__ << " cookie does not match, discarding" << dendl;
+    return;
+  }
+  if (m->get_source_addrs() != sync_provider) {
+    dout(10) << __func__ << " source does not match, discarding" << dendl;
+    return;
+  }
+
+  ceph_assert(state == STATE_SYNCHRONIZING);
+  ceph_assert(g_conf()->mon_sync_requester_kill_at != 5);
+
+  auto tx(std::make_shared<MonitorDBStore::Transaction>());
+  tx->append_from_encoded(m->chunk_bl);
+
+  dout(30) << __func__ << " tx dump:\n";
+  JSONFormatter f(true);
+  tx->dump(&f);
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  store->apply_transaction(tx);
+
+  ceph_assert(g_conf()->mon_sync_requester_kill_at != 6);
+
+  if (!sync_full) {
+    dout(10) << __func__ << " applying recent paxos transactions as we go" << dendl;
+    auto tx(std::make_shared<MonitorDBStore::Transaction>());
+    paxos->read_and_prepare_transactions(tx, paxos->get_version() + 1,
+					 m->last_committed);
+    tx->put(paxos->get_name(), "last_committed", m->last_committed);
+
+    dout(30) << __func__ << " tx dump:\n";
+    JSONFormatter f(true);
+    tx->dump(&f);
+    f.flush(*_dout);
+    *_dout << dendl;
+
+    store->apply_transaction(tx);
+    paxos->init();  // to refresh what we just wrote
+  }
+
+  if (m->op == MMonSync::OP_CHUNK) {
+    sync_reset_timeout();
+    sync_get_next_chunk();
+  } else if (m->op == MMonSync::OP_LAST_CHUNK) {
+    sync_finish(m->last_committed);
+  }
+}
+
+void Monitor::handle_sync_no_cookie(MonOpRequestRef op)
+{
+  dout(10) << __func__ << dendl;
+  bootstrap();
+}
+
+void Monitor::sync_trim_providers()
+{
+  dout(20) << __func__ << dendl;
+
+  utime_t now = ceph_clock_now();
+  map<uint64_t,SyncProvider>::iterator p = sync_providers.begin();
+  while (p != sync_providers.end()) {
+    if (now > p->second.timeout) {
+      dout(10) << __func__ << " expiring cookie " << p->second.cookie
+	       << " for " << p->second.addrs << dendl;
+      sync_providers.erase(p++);
+    } else {
+      ++p;
+    }
+  }
+}
+
+// ---------------------------------------------------
+// probe
+
+void Monitor::cancel_probe_timeout()
+{
+  if (probe_timeout_event) {
+    dout(10) << "cancel_probe_timeout " << probe_timeout_event << dendl;
+    timer.cancel_event(probe_timeout_event);
+    probe_timeout_event = NULL;
+  } else {
+    dout(10) << "cancel_probe_timeout (none scheduled)" << dendl;
+  }
+}
+
+void Monitor::reset_probe_timeout()
+{
+  cancel_probe_timeout();
+  probe_timeout_event = new C_MonContext{this, [this](int r) {
+      probe_timeout(r);
+    }};
+  double t = g_conf()->mon_probe_timeout;
+  if (timer.add_event_after(t, probe_timeout_event)) {
+    dout(10) << "reset_probe_timeout " << probe_timeout_event
+	     << " after " << t << " seconds" << dendl;
+  } else {
+    probe_timeout_event = nullptr;
+  }
+}
+
+void Monitor::probe_timeout(int r)
+{
+  dout(4) << "probe_timeout " << probe_timeout_event << dendl;
+  ceph_assert(is_probing() || is_synchronizing());
+  ceph_assert(probe_timeout_event);
+  probe_timeout_event = NULL;
+  bootstrap();
+}
+
+void Monitor::handle_probe(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonProbe>();
+  dout(10) << "handle_probe " << *m << dendl;
+
+  if (m->fsid != monmap->fsid) {
+    dout(0) << "handle_probe ignoring fsid " << m->fsid << " != " << monmap->fsid << dendl;
+    return;
+  }
+
+  switch (m->op) {
+  case MMonProbe::OP_PROBE:
+    handle_probe_probe(op);
+    break;
+
+  case MMonProbe::OP_REPLY:
+    handle_probe_reply(op);
+    break;
+
+  case MMonProbe::OP_MISSING_FEATURES:
+    derr << __func__ << " require release " << (int)m->mon_release << " > "
+	 << (int)ceph_release()
+	 << ", or missing features (have " << CEPH_FEATURES_ALL
+	 << ", required " << m->required_features
+	 << ", missing " << (m->required_features & ~CEPH_FEATURES_ALL) << ")"
+	 << dendl;
+    break;
+  }
+}
+
+void Monitor::handle_probe_probe(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonProbe>();
+
+  dout(10) << "handle_probe_probe " << m->get_source_inst() << *m
+	   << " features " << m->get_connection()->get_features() << dendl;
+  uint64_t missing = required_features & ~m->get_connection()->get_features();
+  if ((m->mon_release != ceph_release_t::unknown &&
+       m->mon_release < monmap->min_mon_release) ||
+      missing) {
+    dout(1) << " peer " << m->get_source_addr()
+	    << " release " << m->mon_release
+	    << " < min_mon_release " << monmap->min_mon_release
+	    << ", or missing features " << missing << dendl;
+    MMonProbe *r = new MMonProbe(monmap->fsid, MMonProbe::OP_MISSING_FEATURES,
+				 name, has_ever_joined, monmap->min_mon_release);
+    m->required_features = required_features;
+    m->get_connection()->send_message(r);
+    goto out;
+  }
+
+  if (!is_probing() && !is_synchronizing()) {
+    // If the probing mon is way ahead of us, we need to re-bootstrap.
+    // Normally we capture this case when we initially bootstrap, but
+    // it is possible we pass those checks (we overlap with
+    // quorum-to-be) but fail to join a quorum before it moves past
+    // us.  We need to be kicked back to bootstrap so we can
+    // synchonize, not keep calling elections.
+    if (paxos->get_version() + 1 < m->paxos_first_version) {
+      dout(1) << " peer " << m->get_source_addr() << " has first_committed "
+	      << "ahead of us, re-bootstrapping" << dendl;
+      bootstrap();
+      goto out;
+
+    }
+  }
+  
+  MMonProbe *r;
+  r = new MMonProbe(monmap->fsid, MMonProbe::OP_REPLY, name, has_ever_joined,
+		    ceph_release());
+  r->name = name;
+  r->quorum = quorum;
+  r->leader = leader;
+  monmap->encode(r->monmap_bl, m->get_connection()->get_features());
+  r->paxos_first_version = paxos->get_first_committed();
+  r->paxos_last_version = paxos->get_version();
+  m->get_connection()->send_message(r);
+
+  // did we discover a peer here?
+  if (!monmap->contains(m->get_source_addr())) {
+    dout(1) << " adding peer " << m->get_source_addrs()
+	    << " to list of hints" << dendl;
+    extra_probe_peers.insert(m->get_source_addrs());
+  } else {
+    elector.begin_peer_ping(monmap->get_rank(m->get_source_addr()));
+  }
+
+ out:
+  return;
+}
+
+void Monitor::handle_probe_reply(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonProbe>();
+  dout(10) << "handle_probe_reply " << m->get_source_inst()
+	   << " " << *m << dendl;
+  dout(10) << " monmap is " << *monmap << dendl;
+
+  // discover name and addrs during probing or electing states.
+  if (!is_probing() && !is_electing()) {
+    return;
+  }
+
+  // newer map, or they've joined a quorum and we haven't?
+  bufferlist mybl;
+  monmap->encode(mybl, m->get_connection()->get_features());
+  // make sure it's actually different; the checks below err toward
+  // taking the other guy's map, which could cause us to loop.
+  if (!mybl.contents_equal(m->monmap_bl)) {
+    MonMap *newmap = new MonMap;
+    newmap->decode(m->monmap_bl);
+    if (m->has_ever_joined && (newmap->get_epoch() > monmap->get_epoch() ||
+			       !has_ever_joined)) {
+      dout(10) << " got newer/committed monmap epoch " << newmap->get_epoch()
+	       << ", mine was " << monmap->get_epoch() << dendl;
+      int epoch_diff = newmap->get_epoch() - monmap->get_epoch();
+      delete newmap;
+      monmap->decode(m->monmap_bl);
+      dout(20) << "has_ever_joined: " << has_ever_joined << dendl;
+      if (epoch_diff == 1 && has_ever_joined) {
+        notify_new_monmap(false);
+      } else {
+        notify_new_monmap(false, false);
+        elector.notify_clear_peer_state();
+      }
+      bootstrap();
+      return;
+    }
+    delete newmap;
+  }
+
+  // rename peer?
+  string peer_name = monmap->get_name(m->get_source_addr());
+  if (monmap->get_epoch() == 0 && peer_name.compare(0, 7, "noname-") == 0) {
+    dout(10) << " renaming peer " << m->get_source_addr() << " "
+	     << peer_name << " -> " << m->name << " in my monmap"
+	     << dendl;
+    monmap->rename(peer_name, m->name);
+
+    if (is_electing()) {
+      bootstrap();
+      return;
+    }
+  } else if (peer_name.size()) {
+    dout(10) << " peer name is " << peer_name << dendl;
+  } else {
+    dout(10) << " peer " << m->get_source_addr() << " not in map" << dendl;
+  }
+
+  // new initial peer?
+  if (monmap->get_epoch() == 0 &&
+      monmap->contains(m->name) &&
+      monmap->get_addrs(m->name).front().is_blank_ip()) {
+    dout(1) << " learned initial mon " << m->name
+	    << " addrs " << m->get_source_addrs() << dendl;
+    monmap->set_addrvec(m->name, m->get_source_addrs());
+
+    bootstrap();
+    return;
+  }
+
+  // end discover phase
+  if (!is_probing()) {
+    return;
+  }
+
+  ceph_assert(paxos != NULL);
+
+  if (is_synchronizing()) {
+    dout(10) << " currently syncing" << dendl;
+    return;
+  }
+
+  entity_addrvec_t other = m->get_source_addrs();
+
+  if (m->paxos_last_version < sync_last_committed_floor) {
+    dout(10) << " peer paxos versions [" << m->paxos_first_version
+	     << "," << m->paxos_last_version << "] < my sync_last_committed_floor "
+	     << sync_last_committed_floor << ", ignoring"
+	     << dendl;
+  } else {
+    if (paxos->get_version() < m->paxos_first_version &&
+	m->paxos_first_version > 1) {  // no need to sync if we're 0 and they start at 1.
+      dout(10) << " peer paxos first versions [" << m->paxos_first_version
+	       << "," << m->paxos_last_version << "]"
+	       << " vs my version " << paxos->get_version()
+	       << " (too far ahead)"
+	       << dendl;
+      cancel_probe_timeout();
+      sync_start(other, true);
+      return;
+    }
+    if (paxos->get_version() + g_conf()->paxos_max_join_drift < m->paxos_last_version) {
+      dout(10) << " peer paxos last version " << m->paxos_last_version
+	       << " vs my version " << paxos->get_version()
+	       << " (too far ahead)"
+	       << dendl;
+      cancel_probe_timeout();
+      sync_start(other, false);
+      return;
+    }
+  }
+
+  // did the existing cluster complete upgrade to luminous?
+  if (osdmon()->osdmap.get_epoch()) {
+    if (osdmon()->osdmap.require_osd_release < ceph_release_t::luminous) {
+      derr << __func__ << " existing cluster has not completed upgrade to"
+	   << " luminous; 'ceph osd require_osd_release luminous' before"
+	   << " upgrading" << dendl;
+      exit(0);
+    }
+    if (!osdmon()->osdmap.test_flag(CEPH_OSDMAP_PURGED_SNAPDIRS) ||
+	!osdmon()->osdmap.test_flag(CEPH_OSDMAP_RECOVERY_DELETES)) {
+      derr << __func__ << " existing cluster has not completed a full luminous"
+	   << " scrub to purge legacy snapdir objects; please scrub before"
+	   << " upgrading beyond luminous." << dendl;
+      exit(0);
+    }
+  }
+
+  // is there an existing quorum?
+  if (m->quorum.size()) {
+    dout(10) << " existing quorum " << m->quorum << dendl;
+
+    dout(10) << " peer paxos version " << m->paxos_last_version
+             << " vs my version " << paxos->get_version()
+             << " (ok)"
+             << dendl;
+    bool in_map = false;
+    const auto my_info = monmap->mon_info.find(name);
+    const map<string,string> *map_crush_loc{nullptr};
+    if (my_info != monmap->mon_info.end()) {
+      in_map = true;
+      map_crush_loc = &my_info->second.crush_loc;
+    }
+    if (in_map &&
+	!monmap->get_addrs(name).front().is_blank_ip() &&
+	(!need_set_crush_loc || (*map_crush_loc == crush_loc))) {
+      // i'm part of the cluster; just initiate a new election
+      start_election();
+    } else {
+      dout(10) << " ready to join, but i'm not in the monmap/"
+	"my addr is blank/location is wrong, trying to join" << dendl;
+      send_mon_message(new MMonJoin(monmap->fsid, name,
+				    messenger->get_myaddrs(), crush_loc,
+				    need_set_crush_loc),
+		       m->leader);
+    }
+  } else {
+    if (monmap->contains(m->name)) {
+      dout(10) << " mon." << m->name << " is outside the quorum" << dendl;
+      outside_quorum.insert(m->name);
+    } else {
+      dout(10) << " mostly ignoring mon." << m->name << ", not part of monmap" << dendl;
+      return;
+    }
+
+    unsigned need = monmap->min_quorum_size();
+    dout(10) << " outside_quorum now " << outside_quorum << ", need " << need << dendl;
+    if (outside_quorum.size() >= need) {
+      if (outside_quorum.count(name)) {
+        dout(10) << " that's enough to form a new quorum, calling election" << dendl;
+        start_election();
+      } else {
+        dout(10) << " that's enough to form a new quorum, but it does not include me; waiting" << dendl;
+      }
+    } else {
+      dout(10) << " that's not yet enough for a new quorum, waiting" << dendl;
+    }
+  }
+}
+
+void Monitor::join_election()
+{
+  dout(10) << __func__ << dendl;
+  wait_for_paxos_write();
+  _reset();
+  state = STATE_ELECTING;
+
+  logger->inc(l_mon_num_elections);
+}
+
+void Monitor::start_election()
+{
+  dout(10) << "start_election" << dendl;
+  wait_for_paxos_write();
+  _reset();
+  state = STATE_ELECTING;
+
+  logger->inc(l_mon_num_elections);
+  logger->inc(l_mon_election_call);
+
+  clog->info() << "mon." << name << " calling monitor election";
+  elector.call_election();
+}
+
+void Monitor::win_standalone_election()
+{
+  dout(1) << "win_standalone_election" << dendl;
+
+  // bump election epoch, in case the previous epoch included other
+  // monitors; we need to be able to make the distinction.
+  elector.declare_standalone_victory();
+
+  rank = monmap->get_rank(name);
+  ceph_assert(rank == 0);
+  set<int> q;
+  q.insert(rank);
+
+  map<int,Metadata> metadata;
+  collect_metadata(&metadata[0]);
+
+  win_election(elector.get_epoch(), q,
+               CEPH_FEATURES_ALL,
+               ceph::features::mon::get_supported(),
+	       ceph_release(),
+	       metadata);
+}
+
+const utime_t& Monitor::get_leader_since() const
+{
+  ceph_assert(state == STATE_LEADER);
+  return leader_since;
+}
+
+epoch_t Monitor::get_epoch()
+{
+  return elector.get_epoch();
+}
+
+void Monitor::_finish_svc_election()
+{
+  ceph_assert(state == STATE_LEADER || state == STATE_PEON);
+
+  for (auto& svc : paxos_service) {
+    // we already called election_finished() on monmon(); avoid callig twice
+    if (state == STATE_LEADER && svc.get() == monmon())
+      continue;
+    svc->election_finished();
+  }
+}
+
+void Monitor::win_election(epoch_t epoch, const set<int>& active, uint64_t features,
+                           const mon_feature_t& mon_features,
+			   ceph_release_t min_mon_release,
+			   const map<int,Metadata>& metadata)
+{
+  dout(10) << __func__ << " epoch " << epoch << " quorum " << active
+	   << " features " << features
+           << " mon_features " << mon_features
+	   << " min_mon_release " << min_mon_release
+           << dendl;
+  ceph_assert(is_electing());
+  state = STATE_LEADER;
+  leader_since = ceph_clock_now();
+  quorum_since = mono_clock::now();
+  leader = rank;
+  quorum = active;
+  quorum_con_features = features;
+  quorum_mon_features = mon_features;
+  quorum_min_mon_release = min_mon_release;
+  pending_metadata = metadata;
+  outside_quorum.clear();
+
+  clog->info() << "mon." << name << " is new leader, mons " << get_quorum_names()
+      << " in quorum (ranks " << quorum << ")";
+
+  set_leader_commands(get_local_commands(mon_features));
+
+  paxos->leader_init();
+  // NOTE: tell monmap monitor first.  This is important for the
+  // bootstrap case to ensure that the very first paxos proposal
+  // codifies the monmap.  Otherwise any manner of chaos can ensue
+  // when monitors are call elections or participating in a paxos
+  // round without agreeing on who the participants are.
+  monmon()->election_finished();
+  _finish_svc_election();
+
+  logger->inc(l_mon_election_win);
+
+  // inject new metadata in first transaction.
+  {
+    // include previous metadata for missing mons (that aren't part of
+    // the current quorum).
+    map<int,Metadata> m = metadata;
+    for (unsigned rank = 0; rank < monmap->size(); ++rank) {
+      if (m.count(rank) == 0 &&
+	  mon_metadata.count(rank)) {
+	m[rank] = mon_metadata[rank];
+      }
+    }
+
+    // FIXME: This is a bit sloppy because we aren't guaranteed to submit
+    // a new transaction immediately after the election finishes.  We should
+    // do that anyway for other reasons, though.
+    MonitorDBStore::TransactionRef t = paxos->get_pending_transaction();
+    bufferlist bl;
+    encode(m, bl);
+    t->put(MONITOR_STORE_PREFIX, "last_metadata", bl);
+  }
+
+  finish_election();
+  if (monmap->size() > 1 &&
+      monmap->get_epoch() > 0) {
+    timecheck_start();
+    health_tick_start();
+
+    // Freshen the health status before doing health_to_clog in case
+    // our just-completed election changed the health
+    healthmon()->wait_for_active_ctx(new LambdaContext([this](int r){
+      dout(20) << "healthmon now active" << dendl;
+      healthmon()->tick();
+      if (healthmon()->is_proposing()) {
+        dout(20) << __func__ << " healthmon proposing, waiting" << dendl;
+        healthmon()->wait_for_finished_proposal(nullptr, new C_MonContext{this,
+              [this](int r){
+                ceph_assert(ceph_mutex_is_locked_by_me(lock));
+                do_health_to_clog_interval();
+              }});
+
+      } else {
+        do_health_to_clog_interval();
+      }
+    }));
+
+    scrub_event_start();
+  }
+}
+
+void Monitor::lose_election(epoch_t epoch, set<int> &q, int l,
+                            uint64_t features,
+                            const mon_feature_t& mon_features,
+			    ceph_release_t min_mon_release)
+{
+  state = STATE_PEON;
+  leader_since = utime_t();
+  quorum_since = mono_clock::now();
+  leader = l;
+  quorum = q;
+  outside_quorum.clear();
+  quorum_con_features = features;
+  quorum_mon_features = mon_features;
+  quorum_min_mon_release = min_mon_release;
+  dout(10) << "lose_election, epoch " << epoch << " leader is mon" << leader
+	   << " quorum is " << quorum << " features are " << quorum_con_features
+           << " mon_features are " << quorum_mon_features
+	   << " min_mon_release " << min_mon_release
+           << dendl;
+
+  paxos->peon_init();
+  _finish_svc_election();
+
+  logger->inc(l_mon_election_lose);
+
+  finish_election();
+}
+
+namespace {
+std::string collect_compression_algorithms()
+{
+  ostringstream os;
+  bool printed = false;
+  for (auto [name, key] : Compressor::compression_algorithms) {
+    if (printed) {
+      os << ", ";
+    } else {
+      printed = true;
+    }
+    std::ignore = key;
+    os << name;
+  }
+  return os.str();
+}
+}
+
+void Monitor::collect_metadata(Metadata *m)
+{
+  collect_sys_info(m, g_ceph_context);
+  (*m)["addrs"] = stringify(messenger->get_myaddrs());
+  (*m)["compression_algorithms"] = collect_compression_algorithms();
+
+  // infer storage device
+  string devname = store->get_devname();
+  set<string> devnames;
+  get_raw_devices(devname, &devnames);
+  map<string,string> errs;
+  get_device_metadata(devnames, m, &errs);
+  for (auto& i : errs) {
+    dout(1) << __func__ << " " << i.first << ": " << i.second << dendl;
+  }
+}
+
+void Monitor::finish_election()
+{
+  apply_quorum_to_compatset_features();
+  apply_monmap_to_compatset_features();
+  timecheck_finish();
+  exited_quorum = utime_t();
+  finish_contexts(g_ceph_context, waitfor_quorum);
+  finish_contexts(g_ceph_context, maybe_wait_for_quorum);
+  resend_routed_requests();
+  update_logger();
+  register_cluster_logger();
+
+  // enable authentication
+  {
+    std::lock_guard l(auth_lock);
+    authmon()->_set_mon_num_rank(monmap->size(), rank);
+  }
+
+  // am i named and located properly?
+  string cur_name = monmap->get_name(messenger->get_myaddrs());
+  const auto my_infop = monmap->mon_info.find(cur_name);
+  const map<string,string>& map_crush_loc = my_infop->second.crush_loc;
+  
+  if (cur_name != name ||
+      (need_set_crush_loc && map_crush_loc != crush_loc)) {
+    dout(10) << " renaming/moving myself from " << cur_name << "/"
+	     << map_crush_loc <<" -> " << name << "/" << crush_loc << dendl;
+    send_mon_message(new MMonJoin(monmap->fsid, name, messenger->get_myaddrs(),
+				  crush_loc, need_set_crush_loc),
+		     leader);
+    return;
+  }
+  do_stretch_mode_election_work();
+}
+
+void Monitor::_apply_compatset_features(CompatSet &new_features)
+{
+  if (new_features.compare(features) != 0) {
+    CompatSet diff = features.unsupported(new_features);
+    dout(1) << __func__ << " enabling new quorum features: " << diff << dendl;
+    features = new_features;
+
+    auto t = std::make_shared<MonitorDBStore::Transaction>();
+    write_features(t);
+    store->apply_transaction(t);
+
+    calc_quorum_requirements();
+  }
+}
+
+void Monitor::apply_quorum_to_compatset_features()
+{
+  CompatSet new_features(features);
+  new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES);
+  if (quorum_con_features & CEPH_FEATURE_OSDMAP_ENC) {
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC);
+  }
+  new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2);
+  new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V3);
+  dout(5) << __func__ << dendl;
+  _apply_compatset_features(new_features);
+}
+
+void Monitor::apply_monmap_to_compatset_features()
+{
+  CompatSet new_features(features);
+  mon_feature_t monmap_features = monmap->get_required_features();
+
+  /* persistent monmap features may go into the compatset.
+   * optional monmap features may not - why?
+   *   because optional monmap features may be set/unset by the admin,
+   *   and possibly by other means that haven't yet been thought out,
+   *   so we can't make the monitor enforce them on start - because they
+   *   may go away.
+   *   this, of course, does not invalidate setting a compatset feature
+   *   for an optional feature - as long as you make sure to clean it up
+   *   once you unset it.
+   */
+  if (monmap_features.contains_all(ceph::features::mon::FEATURE_KRAKEN)) {
+    ceph_assert(ceph::features::mon::get_persistent().contains_all(
+           ceph::features::mon::FEATURE_KRAKEN));
+    // this feature should only ever be set if the quorum supports it.
+    ceph_assert(HAVE_FEATURE(quorum_con_features, SERVER_KRAKEN));
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_KRAKEN);
+  }
+  if (monmap_features.contains_all(ceph::features::mon::FEATURE_LUMINOUS)) {
+    ceph_assert(ceph::features::mon::get_persistent().contains_all(
+           ceph::features::mon::FEATURE_LUMINOUS));
+    // this feature should only ever be set if the quorum supports it.
+    ceph_assert(HAVE_FEATURE(quorum_con_features, SERVER_LUMINOUS));
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_LUMINOUS);
+  }
+  if (monmap_features.contains_all(ceph::features::mon::FEATURE_MIMIC)) {
+    ceph_assert(ceph::features::mon::get_persistent().contains_all(
+           ceph::features::mon::FEATURE_MIMIC));
+    // this feature should only ever be set if the quorum supports it.
+    ceph_assert(HAVE_FEATURE(quorum_con_features, SERVER_MIMIC));
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_MIMIC);
+  }
+  if (monmap_features.contains_all(ceph::features::mon::FEATURE_NAUTILUS)) {
+    ceph_assert(ceph::features::mon::get_persistent().contains_all(
+           ceph::features::mon::FEATURE_NAUTILUS));
+    // this feature should only ever be set if the quorum supports it.
+    ceph_assert(HAVE_FEATURE(quorum_con_features, SERVER_NAUTILUS));
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_NAUTILUS);
+  }
+  if (monmap_features.contains_all(ceph::features::mon::FEATURE_OCTOPUS)) {
+    ceph_assert(ceph::features::mon::get_persistent().contains_all(
+           ceph::features::mon::FEATURE_OCTOPUS));
+    // this feature should only ever be set if the quorum supports it.
+    ceph_assert(HAVE_FEATURE(quorum_con_features, SERVER_OCTOPUS));
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OCTOPUS);
+  }
+  if (monmap_features.contains_all(ceph::features::mon::FEATURE_PACIFIC)) {
+    ceph_assert(ceph::features::mon::get_persistent().contains_all(
+           ceph::features::mon::FEATURE_PACIFIC));
+    // this feature should only ever be set if the quorum supports it.
+    ceph_assert(HAVE_FEATURE(quorum_con_features, SERVER_PACIFIC));
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_PACIFIC);
+  }
+
+  dout(5) << __func__ << dendl;
+  _apply_compatset_features(new_features);
+}
+
+void Monitor::calc_quorum_requirements()
+{
+  required_features = 0;
+
+  // compatset
+  if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC)) {
+    required_features |= CEPH_FEATURE_OSDMAP_ENC;
+  }
+  if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_KRAKEN)) {
+    required_features |= CEPH_FEATUREMASK_SERVER_KRAKEN;
+  }
+  if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_LUMINOUS)) {
+    required_features |= CEPH_FEATUREMASK_SERVER_LUMINOUS;
+  }
+  if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_MIMIC)) {
+    required_features |= CEPH_FEATUREMASK_SERVER_MIMIC;
+  }
+  if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_NAUTILUS)) {
+    required_features |= CEPH_FEATUREMASK_SERVER_NAUTILUS |
+      CEPH_FEATUREMASK_CEPHX_V2;
+  }
+  if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_OCTOPUS)) {
+    required_features |= CEPH_FEATUREMASK_SERVER_OCTOPUS;
+  }
+  if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_PACIFIC)) {
+    required_features |= CEPH_FEATUREMASK_SERVER_PACIFIC;
+  }
+
+  // monmap
+  if (monmap->get_required_features().contains_all(
+	ceph::features::mon::FEATURE_KRAKEN)) {
+    required_features |= CEPH_FEATUREMASK_SERVER_KRAKEN;
+  }
+  if (monmap->get_required_features().contains_all(
+	ceph::features::mon::FEATURE_LUMINOUS)) {
+    required_features |= CEPH_FEATUREMASK_SERVER_LUMINOUS;
+  }
+  if (monmap->get_required_features().contains_all(
+	ceph::features::mon::FEATURE_MIMIC)) {
+    required_features |= CEPH_FEATUREMASK_SERVER_MIMIC;
+  }
+  if (monmap->get_required_features().contains_all(
+	ceph::features::mon::FEATURE_NAUTILUS)) {
+    required_features |= CEPH_FEATUREMASK_SERVER_NAUTILUS |
+      CEPH_FEATUREMASK_CEPHX_V2;
+  }
+  dout(10) << __func__ << " required_features " << required_features << dendl;
+}
+
+void Monitor::get_combined_feature_map(FeatureMap *fm)
+{
+  *fm += session_map.feature_map;
+  for (auto id : quorum) {
+    if (id != rank) {
+      *fm += quorum_feature_map[id];
+    }
+  }
+}
+
+void Monitor::sync_force(Formatter *f)
+{
+  auto tx(std::make_shared<MonitorDBStore::Transaction>());
+  sync_stash_critical_state(tx);
+  tx->put("mon_sync", "force_sync", 1);
+  store->apply_transaction(tx);
+
+  f->open_object_section("sync_force");
+  f->dump_int("ret", 0);
+  f->dump_stream("msg") << "forcing store sync the next time the monitor starts";
+  f->close_section(); // sync_force
+}
+
+void Monitor::_quorum_status(Formatter *f, ostream& ss)
+{
+  bool free_formatter = false;
+
+  if (!f) {
+    // louzy/lazy hack: default to json if no formatter has been defined
+    f = new JSONFormatter();
+    free_formatter = true;
+  }
+  f->open_object_section("quorum_status");
+  f->dump_int("election_epoch", get_epoch());
+
+  f->open_array_section("quorum");
+  for (set<int>::iterator p = quorum.begin(); p != quorum.end(); ++p)
+    f->dump_int("mon", *p);
+  f->close_section(); // quorum
+
+  list<string> quorum_names = get_quorum_names();
+  f->open_array_section("quorum_names");
+  for (list<string>::iterator p = quorum_names.begin(); p != quorum_names.end(); ++p)
+    f->dump_string("mon", *p);
+  f->close_section(); // quorum_names
+
+  f->dump_string("quorum_leader_name", quorum.empty() ? string() : monmap->get_name(leader));
+
+  if (!quorum.empty()) {
+    f->dump_int(
+      "quorum_age",
+      quorum_age());
+  }
+
+  f->open_object_section("features");
+  f->dump_stream("quorum_con") << quorum_con_features;
+  quorum_mon_features.dump(f, "quorum_mon");
+  f->close_section();
+
+  f->open_object_section("monmap");
+  monmap->dump(f);
+  f->close_section(); // monmap
+
+  f->close_section(); // quorum_status
+  f->flush(ss);
+  if (free_formatter)
+    delete f;
+}
+
+void Monitor::get_mon_status(Formatter *f)
+{
+  f->open_object_section("mon_status");
+  f->dump_string("name", name);
+  f->dump_int("rank", rank);
+  f->dump_string("state", get_state_name());
+  f->dump_int("election_epoch", get_epoch());
+
+  f->open_array_section("quorum");
+  for (set<int>::iterator p = quorum.begin(); p != quorum.end(); ++p) {
+    f->dump_int("mon", *p);
+  }
+  f->close_section(); // quorum
+
+  if (!quorum.empty()) {
+    f->dump_int(
+      "quorum_age",
+      quorum_age());
+  }
+
+  f->open_object_section("features");
+  f->dump_stream("required_con") << required_features;
+  mon_feature_t req_mon_features = get_required_mon_features();
+  req_mon_features.dump(f, "required_mon");
+  f->dump_stream("quorum_con") << quorum_con_features;
+  quorum_mon_features.dump(f, "quorum_mon");
+  f->close_section(); // features
+
+  f->open_array_section("outside_quorum");
+  for (set<string>::iterator p = outside_quorum.begin(); p != outside_quorum.end(); ++p)
+    f->dump_string("mon", *p);
+  f->close_section(); // outside_quorum
+
+  f->open_array_section("extra_probe_peers");
+  for (set<entity_addrvec_t>::iterator p = extra_probe_peers.begin();
+       p != extra_probe_peers.end();
+       ++p) {
+    f->dump_object("peer", *p);
+  }
+  f->close_section(); // extra_probe_peers
+
+  f->open_array_section("sync_provider");
+  for (map<uint64_t,SyncProvider>::const_iterator p = sync_providers.begin();
+       p != sync_providers.end();
+       ++p) {
+    f->dump_unsigned("cookie", p->second.cookie);
+    f->dump_object("addrs", p->second.addrs);
+    f->dump_stream("timeout") << p->second.timeout;
+    f->dump_unsigned("last_committed", p->second.last_committed);
+    f->dump_stream("last_key") << p->second.last_key;
+  }
+  f->close_section();
+
+  if (is_synchronizing()) {
+    f->open_object_section("sync");
+    f->dump_stream("sync_provider") << sync_provider;
+    f->dump_unsigned("sync_cookie", sync_cookie);
+    f->dump_unsigned("sync_start_version", sync_start_version);
+    f->close_section();
+  }
+
+  if (g_conf()->mon_sync_provider_kill_at > 0)
+    f->dump_int("provider_kill_at", g_conf()->mon_sync_provider_kill_at);
+  if (g_conf()->mon_sync_requester_kill_at > 0)
+    f->dump_int("requester_kill_at", g_conf()->mon_sync_requester_kill_at);
+
+  f->open_object_section("monmap");
+  monmap->dump(f);
+  f->close_section();
+
+  f->dump_object("feature_map", session_map.feature_map);
+  f->dump_bool("stretch_mode", stretch_mode_engaged);
+  f->close_section(); // mon_status
+}
+
+
+// health status to clog
+
+void Monitor::health_tick_start()
+{
+  if (!cct->_conf->mon_health_to_clog ||
+      cct->_conf->mon_health_to_clog_tick_interval <= 0)
+    return;
+
+  dout(15) << __func__ << dendl;
+
+  health_tick_stop();
+  health_tick_event = timer.add_event_after(
+    cct->_conf->mon_health_to_clog_tick_interval,
+    new C_MonContext{this, [this](int r) {
+	if (r < 0)
+	  return;
+	health_tick_start();
+      }});
+}
+
+void Monitor::health_tick_stop()
+{
+  dout(15) << __func__ << dendl;
+
+  if (health_tick_event) {
+    timer.cancel_event(health_tick_event);
+    health_tick_event = NULL;
+  }
+}
+
+ceph::real_clock::time_point Monitor::health_interval_calc_next_update()
+{
+  auto now = ceph::real_clock::now();
+
+  auto secs = std::chrono::duration_cast<std::chrono::seconds>(now.time_since_epoch());
+  int remainder = secs.count() % cct->_conf->mon_health_to_clog_interval;
+  int adjustment = cct->_conf->mon_health_to_clog_interval - remainder;
+  auto next = secs + std::chrono::seconds(adjustment);
+
+  dout(20) << __func__
+    << " now: " << now << ","
+    << " next: " << next << ","
+    << " interval: " << cct->_conf->mon_health_to_clog_interval
+    << dendl;
+
+  return ceph::real_clock::time_point{next};
+}
+
+void Monitor::health_interval_start()
+{
+  dout(15) << __func__ << dendl;
+
+  if (!cct->_conf->mon_health_to_clog ||
+      cct->_conf->mon_health_to_clog_interval <= 0) {
+    return;
+  }
+
+  health_interval_stop();
+  auto next = health_interval_calc_next_update();
+  health_interval_event = new C_MonContext{this, [this](int r) {
+      if (r < 0)
+        return;
+      do_health_to_clog_interval();
+    }};
+  if (!timer.add_event_at(next, health_interval_event)) {
+    health_interval_event = nullptr;
+  }
+}
+
+void Monitor::health_interval_stop()
+{
+  dout(15) << __func__ << dendl;
+  if (health_interval_event) {
+    timer.cancel_event(health_interval_event);
+  }
+  health_interval_event = NULL;
+}
+
+void Monitor::health_events_cleanup()
+{
+  health_tick_stop();
+  health_interval_stop();
+  health_status_cache.reset();
+}
+
+void Monitor::health_to_clog_update_conf(const std::set<std::string> &changed)
+{
+  dout(20) << __func__ << dendl;
+
+  if (changed.count("mon_health_to_clog")) {
+    if (!cct->_conf->mon_health_to_clog) {
+      health_events_cleanup();
+      return;
+    } else {
+      if (!health_tick_event) {
+        health_tick_start();
+      }
+      if (!health_interval_event) {
+        health_interval_start();
+      }
+    }
+  }
+
+  if (changed.count("mon_health_to_clog_interval")) {
+    if (cct->_conf->mon_health_to_clog_interval <= 0) {
+      health_interval_stop();
+    } else {
+      health_interval_start();
+    }
+  }
+
+  if (changed.count("mon_health_to_clog_tick_interval")) {
+    if (cct->_conf->mon_health_to_clog_tick_interval <= 0) {
+      health_tick_stop();
+    } else {
+      health_tick_start();
+    }
+  }
+}
+
+void Monitor::do_health_to_clog_interval()
+{
+  // outputting to clog may have been disabled in the conf
+  // since we were scheduled.
+  if (!cct->_conf->mon_health_to_clog ||
+      cct->_conf->mon_health_to_clog_interval <= 0)
+    return;
+
+  dout(10) << __func__ << dendl;
+
+  // do we have a cached value for next_clog_update?  if not,
+  // do we know when the last update was?
+
+  do_health_to_clog(true);
+  health_interval_start();
+}
+
+void Monitor::do_health_to_clog(bool force)
+{
+  // outputting to clog may have been disabled in the conf
+  // since we were scheduled.
+  if (!cct->_conf->mon_health_to_clog ||
+      cct->_conf->mon_health_to_clog_interval <= 0)
+    return;
+
+  dout(10) << __func__ << (force ? " (force)" : "") << dendl;
+
+  string summary;
+  health_status_t level = healthmon()->get_health_status(false, nullptr, &summary);
+  if (!force &&
+      summary == health_status_cache.summary &&
+      level == health_status_cache.overall)
+    return;
+
+  if (g_conf()->mon_health_detail_to_clog &&
+      summary != health_status_cache.summary &&
+      level != HEALTH_OK) {
+    string details;
+    level = healthmon()->get_health_status(true, nullptr, &details);
+    clog->health(level) << "Health detail: " << details;
+  } else {
+    clog->health(level) << "overall " << summary;
+  }
+  health_status_cache.summary = summary;
+  health_status_cache.overall = level;
+}
+
+void Monitor::log_health(
+  const health_check_map_t& updated,
+  const health_check_map_t& previous,
+  MonitorDBStore::TransactionRef t)
+{
+  if (!g_conf()->mon_health_to_clog) {
+    return;
+  }
+
+  const utime_t now = ceph_clock_now();
+
+  // FIXME: log atomically as part of @t instead of using clog.
+  dout(10) << __func__ << " updated " << updated.checks.size()
+	   << " previous " << previous.checks.size()
+	   << dendl;
+  const auto min_log_period = g_conf().get_val<int64_t>(
+      "mon_health_log_update_period");
+  for (auto& p : updated.checks) {
+    auto q = previous.checks.find(p.first);
+    bool logged = false;
+    if (q == previous.checks.end()) {
+      // new
+      ostringstream ss;
+      ss << "Health check failed: " << p.second.summary << " ("
+         << p.first << ")";
+      clog->health(p.second.severity) << ss.str();
+
+      logged = true;
+    } else {
+      if (p.second.summary != q->second.summary ||
+	  p.second.severity != q->second.severity) {
+
+        auto status_iter = health_check_log_times.find(p.first);
+        if (status_iter != health_check_log_times.end()) {
+          if (p.second.severity == q->second.severity &&
+              now - status_iter->second.updated_at < min_log_period) {
+            // We already logged this recently and the severity is unchanged,
+            // so skip emitting an update of the summary string.
+            // We'll get an update out of tick() later if the check
+            // is still failing.
+            continue;
+          }
+        }
+
+        // summary or severity changed (ignore detail changes at this level)
+        ostringstream ss;
+        ss << "Health check update: " << p.second.summary << " (" << p.first << ")";
+        clog->health(p.second.severity) << ss.str();
+
+        logged = true;
+      }
+    }
+    // Record the time at which we last logged, so that we can check this
+    // when considering whether/when to print update messages.
+    if (logged) {
+      auto iter = health_check_log_times.find(p.first);
+      if (iter == health_check_log_times.end()) {
+        health_check_log_times.emplace(p.first, HealthCheckLogStatus(
+          p.second.severity, p.second.summary, now));
+      } else {
+        iter->second = HealthCheckLogStatus(
+          p.second.severity, p.second.summary, now);
+      }
+    }
+  }
+  for (auto& p : previous.checks) {
+    if (!updated.checks.count(p.first)) {
+      // cleared
+      ostringstream ss;
+      if (p.first == "DEGRADED_OBJECTS") {
+        clog->info() << "All degraded objects recovered";
+      } else if (p.first == "OSD_FLAGS") {
+        clog->info() << "OSD flags cleared";
+      } else {
+        clog->info() << "Health check cleared: " << p.first << " (was: "
+                     << p.second.summary << ")";
+      }
+
+      if (health_check_log_times.count(p.first)) {
+        health_check_log_times.erase(p.first);
+      }
+    }
+  }
+
+  if (previous.checks.size() && updated.checks.size() == 0) {
+    // We might be going into a fully healthy state, check
+    // other subsystems
+    bool any_checks = false;
+    for (auto& svc : paxos_service) {
+      if (&(svc->get_health_checks()) == &(previous)) {
+        // Ignore the ones we're clearing right now
+        continue;
+      }
+
+      if (svc->get_health_checks().checks.size() > 0) {
+        any_checks = true;
+        break;
+      }
+    }
+    if (!any_checks) {
+      clog->info() << "Cluster is now healthy";
+    }
+  }
+}
+
+void Monitor::update_pending_metadata()
+{
+  Metadata metadata;
+  collect_metadata(&metadata);
+  size_t version_size = mon_metadata[rank]["ceph_version_short"].size();
+  const std::string current_version = mon_metadata[rank]["ceph_version_short"];
+  const std::string pending_version = metadata["ceph_version_short"];
+
+  if (current_version.compare(0, version_size, pending_version) < 0) {
+    mgr_client.update_daemon_metadata("mon", name, metadata);
+  }
+}
+
+void Monitor::get_cluster_status(stringstream &ss, Formatter *f,
+				 MonSession *session)
+{
+  if (f)
+    f->open_object_section("status");
+
+  const auto&& fs_names = session->get_allowed_fs_names();
+
+  if (f) {
+    f->dump_stream("fsid") << monmap->get_fsid();
+    healthmon()->get_health_status(false, f, nullptr);
+    f->dump_unsigned("election_epoch", get_epoch());
+    {
+      f->open_array_section("quorum");
+      for (set<int>::iterator p = quorum.begin(); p != quorum.end(); ++p)
+	f->dump_int("rank", *p);
+      f->close_section();
+      f->open_array_section("quorum_names");
+      for (set<int>::iterator p = quorum.begin(); p != quorum.end(); ++p)
+	f->dump_string("id", monmap->get_name(*p));
+      f->close_section();
+      f->dump_int(
+	"quorum_age",
+        quorum_age());
+    }
+    f->open_object_section("monmap");
+    monmap->dump_summary(f);
+    f->close_section();
+    f->open_object_section("osdmap");
+    osdmon()->osdmap.print_summary(f, cout, string(12, ' '));
+    f->close_section();
+    f->open_object_section("pgmap");
+    mgrstatmon()->print_summary(f, NULL);
+    f->close_section();
+    f->open_object_section("fsmap");
+
+    FSMap fsmap_copy = mdsmon()->get_fsmap();
+    if (!fs_names.empty()) {
+      fsmap_copy.filter(fs_names);
+    }
+    const FSMap *fsmapp = &fsmap_copy;
+
+    fsmapp->print_summary(f, NULL);
+    f->close_section();
+    f->open_object_section("mgrmap");
+    mgrmon()->get_map().print_summary(f, nullptr);
+    f->close_section();
+
+    f->dump_object("servicemap", mgrstatmon()->get_service_map());
+
+    f->open_object_section("progress_events");
+    for (auto& i : mgrstatmon()->get_progress_events()) {
+      f->dump_object(i.first.c_str(), i.second);
+    }
+    f->close_section();
+
+    f->close_section();
+  } else {
+    ss << "  cluster:\n";
+    ss << "    id:     " << monmap->get_fsid() << "\n";
+
+    string health;
+    healthmon()->get_health_status(false, nullptr, &health,
+				   "\n            ", "\n            ");
+    ss << "    health: " << health << "\n";
+
+    ss << "\n \n  services:\n";
+    {
+      size_t maxlen = 3;
+      auto& service_map = mgrstatmon()->get_service_map();
+      for (auto& p : service_map.services) {
+	maxlen = std::max(maxlen, p.first.size());
+      }
+      string spacing(maxlen - 3, ' ');
+      const auto quorum_names = get_quorum_names();
+      const auto mon_count = monmap->mon_info.size();
+      auto mnow = ceph::mono_clock::now();
+      ss << "    mon: " << spacing << mon_count << " daemons, quorum "
+	 << quorum_names << " (age " << timespan_str(mnow - quorum_since) << ")";
+      if (quorum_names.size() != mon_count) {
+	std::list<std::string> out_of_q;
+	for (size_t i = 0; i < monmap->ranks.size(); ++i) {
+	  if (quorum.count(i) == 0) {
+	    out_of_q.push_back(monmap->ranks[i]);
+	  }
+	}
+	ss << ", out of quorum: " << joinify(out_of_q.begin(),
+					     out_of_q.end(), std::string(", "));
+      }
+      ss << "\n";
+      if (mgrmon()->in_use()) {
+	ss << "    mgr: " << spacing;
+	mgrmon()->get_map().print_summary(nullptr, &ss);
+	ss << "\n";
+      }
+
+      FSMap fsmap_copy = mdsmon()->get_fsmap();
+      if (!fs_names.empty()) {
+	fsmap_copy.filter(fs_names);
+      }
+      const FSMap *fsmapp = &fsmap_copy;
+
+      if (fsmapp->filesystem_count() > 0 and mdsmon()->should_print_status()){
+        ss << "    mds: " << spacing;
+	fsmapp->print_daemon_summary(ss);
+	ss << "\n";
+      }
+
+      ss << "    osd: " << spacing;
+      osdmon()->osdmap.print_summary(NULL, ss, string(maxlen + 6, ' '));
+      ss << "\n";
+      for (auto& p : service_map.services) {
+        const std::string &service = p.first;
+        // filter out normal ceph entity types
+        if (ServiceMap::is_normal_ceph_entity(service)) {
+          continue;
+        }
+	ss << "    " << p.first << ": " << string(maxlen - p.first.size(), ' ')
+	   << p.second.get_summary() << "\n";
+      }
+    }
+
+    if (auto& service_map = mgrstatmon()->get_service_map();
+        std::any_of(service_map.services.begin(),
+                    service_map.services.end(),
+                    [](auto& service) {
+                      return service.second.has_running_tasks();
+                    })) {
+      ss << "\n \n  task status:\n";
+      for (auto& [name, service] : service_map.services) {
+	ss << service.get_task_summary(name);
+      }
+    }
+
+    ss << "\n \n  data:\n";
+    mdsmon()->print_fs_summary(ss);
+    mgrstatmon()->print_summary(NULL, &ss);
+
+    auto& pem = mgrstatmon()->get_progress_events();
+    if (!pem.empty()) {
+      ss << "\n \n  progress:\n";
+      for (auto& i : pem) {
+	if (i.second.add_to_ceph_s){
+	ss << "    " << i.second.message << "\n";
+	}
+      }
+    }
+    ss << "\n ";
+  }
+}
+
+void Monitor::_generate_command_map(cmdmap_t& cmdmap,
+                                    map<string,string> &param_str_map)
+{
+  for (auto p = cmdmap.begin(); p != cmdmap.end(); ++p) {
+    if (p->first == "prefix")
+      continue;
+    if (p->first == "caps") {
+      vector<string> cv;
+      if (cmd_getval(cmdmap, "caps", cv) &&
+	  cv.size() % 2 == 0) {
+	for (unsigned i = 0; i < cv.size(); i += 2) {
+	  string k = string("caps_") + cv[i];
+	  param_str_map[k] = cv[i + 1];
+	}
+	continue;
+      }
+    }
+    param_str_map[p->first] = cmd_vartype_stringify(p->second);
+  }
+}
+
+const MonCommand *Monitor::_get_moncommand(
+  const string &cmd_prefix,
+  const vector<MonCommand>& cmds)
+{
+  for (auto& c : cmds) {
+    if (c.cmdstring.compare(0, cmd_prefix.size(), cmd_prefix) == 0) {
+      return &c;
+    }
+  }
+  return nullptr;
+}
+
+bool Monitor::_allowed_command(MonSession *s, const string &module,
+			       const string &prefix, const cmdmap_t& cmdmap,
+                               const map<string,string>& param_str_map,
+                               const MonCommand *this_cmd) {
+
+  bool cmd_r = this_cmd->requires_perm('r');
+  bool cmd_w = this_cmd->requires_perm('w');
+  bool cmd_x = this_cmd->requires_perm('x');
+
+  bool capable = s->caps.is_capable(
+    g_ceph_context,
+    s->entity_name,
+    module, prefix, param_str_map,
+    cmd_r, cmd_w, cmd_x,
+    s->get_peer_socket_addr());
+
+  dout(10) << __func__ << " " << (capable ? "" : "not ") << "capable" << dendl;
+  return capable;
+}
+
+void Monitor::format_command_descriptions(const std::vector<MonCommand> &commands,
+					  Formatter *f,
+					  uint64_t features,
+					  bufferlist *rdata)
+{
+  int cmdnum = 0;
+  f->open_object_section("command_descriptions");
+  for (const auto &cmd : commands) {
+    unsigned flags = cmd.flags;
+    ostringstream secname;
+    secname << "cmd" << setfill('0') << std::setw(3) << cmdnum;
+    dump_cmddesc_to_json(f, features, secname.str(),
+			 cmd.cmdstring, cmd.helpstring, cmd.module,
+			 cmd.req_perms, flags);
+    cmdnum++;
+  }
+  f->close_section();	// command_descriptions
+
+  f->flush(*rdata);
+}
+
+bool Monitor::is_keyring_required()
+{
+  return auth_cluster_required.is_supported_auth(CEPH_AUTH_CEPHX) || 
+         auth_service_required.is_supported_auth(CEPH_AUTH_CEPHX) || 
+         auth_cluster_required.is_supported_auth(CEPH_AUTH_GSS)   || 
+         auth_service_required.is_supported_auth(CEPH_AUTH_GSS);
+}
+
+struct C_MgrProxyCommand : public Context {
+  Monitor *mon;
+  MonOpRequestRef op;
+  uint64_t size;
+  bufferlist outbl;
+  string outs;
+  C_MgrProxyCommand(Monitor *mon, MonOpRequestRef op, uint64_t s)
+    : mon(mon), op(op), size(s) { }
+  void finish(int r) {
+    std::lock_guard l(mon->lock);
+    mon->mgr_proxy_bytes -= size;
+    mon->reply_command(op, r, outs, outbl, 0);
+  }
+};
+
+void Monitor::handle_tell_command(MonOpRequestRef op)
+{
+  ceph_assert(op->is_type_command());
+  MCommand *m = static_cast<MCommand*>(op->get_req());
+  if (m->fsid != monmap->fsid) {
+    dout(0) << "handle_command on fsid " << m->fsid << " != " << monmap->fsid << dendl;
+    return reply_tell_command(op, -EACCES, "wrong fsid");
+  }
+  MonSession *session = op->get_session();
+  if (!session) {
+    dout(5) << __func__ << " dropping stray message " << *m << dendl;
+    return;
+  }
+  cmdmap_t cmdmap;
+  if (stringstream ss; !cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    return reply_tell_command(op, -EINVAL, ss.str());
+  }
+  map<string,string> param_str_map;
+  _generate_command_map(cmdmap, param_str_map);
+  string prefix;
+  if (!cmd_getval(cmdmap, "prefix", prefix)) {
+    return reply_tell_command(op, -EINVAL, "no prefix");
+  }
+  if (auto cmd = _get_moncommand(prefix,
+				 get_local_commands(quorum_mon_features));
+      cmd) {
+    if (cmd->is_obsolete() ||
+	(cct->_conf->mon_debug_deprecated_as_obsolete &&
+	 cmd->is_deprecated())) {
+      return reply_tell_command(op, -ENOTSUP,
+				"command is obsolete; "
+				"please check usage and/or man page");
+    }
+  }
+  // see if command is allowed
+  if (!session->caps.is_capable(
+      g_ceph_context,
+      session->entity_name,
+      "mon", prefix, param_str_map,
+      true, true, true,
+      session->get_peer_socket_addr())) {
+    return reply_tell_command(op, -EACCES, "insufficient caps");
+  }
+  // pass it to asok
+  cct->get_admin_socket()->queue_tell_command(m);
+}
+
+void Monitor::handle_command(MonOpRequestRef op)
+{
+  ceph_assert(op->is_type_command());
+  auto m = op->get_req<MMonCommand>();
+  if (m->fsid != monmap->fsid) {
+    dout(0) << "handle_command on fsid " << m->fsid << " != " << monmap->fsid
+	    << dendl;
+    reply_command(op, -EPERM, "wrong fsid", 0);
+    return;
+  }
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    dout(5) << __func__ << " dropping stray message " << *m << dendl;
+    return;
+  }
+
+  if (m->cmd.empty()) {
+    reply_command(op, -EINVAL, "no command specified", 0);
+    return;
+  }
+
+  string prefix;
+  vector<string> fullcmd;
+  cmdmap_t cmdmap;
+  stringstream ss, ds;
+  bufferlist rdata;
+  string rs;
+  int r = -EINVAL;
+  rs = "unrecognized command";
+
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    // ss has reason for failure
+    r = -EINVAL;
+    rs = ss.str();
+    if (!m->get_source().is_mon())  // don't reply to mon->mon commands
+      reply_command(op, r, rs, 0);
+    return;
+  }
+
+  // check return value. If no prefix parameter provided,
+  // return value will be false, then return error info.
+  if (!cmd_getval(cmdmap, "prefix", prefix)) {
+    reply_command(op, -EINVAL, "command prefix not found", 0);
+    return;
+  }
+
+  // check prefix is empty
+  if (prefix.empty()) {
+    reply_command(op, -EINVAL, "command prefix must not be empty", 0);
+    return;
+  }
+
+  if (prefix == "get_command_descriptions") {
+    bufferlist rdata;
+    Formatter *f = Formatter::create("json");
+
+    std::vector<MonCommand> commands = static_cast<MgrMonitor*>(
+        paxos_service[PAXOS_MGR].get())->get_command_descs();
+
+    for (auto& c : leader_mon_commands) {
+      commands.push_back(c);
+    }
+
+    auto features = m->get_connection()->get_features();
+    format_command_descriptions(commands, f, features, &rdata);
+    delete f;
+    reply_command(op, 0, "", rdata, 0);
+    return;
+  }
+
+  dout(0) << "handle_command " << *m << dendl;
+
+  string format;
+  cmd_getval(cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+  get_str_vec(prefix, fullcmd);
+
+  // make sure fullcmd is not empty.
+  // invalid prefix will cause empty vector fullcmd.
+  // such as, prefix=";,,;"
+  if (fullcmd.empty()) {
+    reply_command(op, -EINVAL, "command requires a prefix to be valid", 0);
+    return;
+  }
+
+  std::string_view module = fullcmd[0];
+
+  // validate command is in leader map
+
+  const MonCommand *leader_cmd;
+  const auto& mgr_cmds = mgrmon()->get_command_descs();
+  const MonCommand *mgr_cmd = nullptr;
+  if (!mgr_cmds.empty()) {
+    mgr_cmd = _get_moncommand(prefix, mgr_cmds);
+  }
+  leader_cmd = _get_moncommand(prefix, leader_mon_commands);
+  if (!leader_cmd) {
+    leader_cmd = mgr_cmd;
+    if (!leader_cmd) {
+      reply_command(op, -EINVAL, "command not known", 0);
+      return;
+    }
+  }
+  // validate command is in our map & matches, or forward if it is allowed
+  const MonCommand *mon_cmd = _get_moncommand(
+    prefix,
+    get_local_commands(quorum_mon_features));
+  if (!mon_cmd) {
+    mon_cmd = mgr_cmd;
+  }
+  if (!is_leader()) {
+    if (!mon_cmd) {
+      if (leader_cmd->is_noforward()) {
+	reply_command(op, -EINVAL,
+		      "command not locally supported and not allowed to forward",
+		      0);
+	return;
+      }
+      dout(10) << "Command not locally supported, forwarding request "
+	       << m << dendl;
+      forward_request_leader(op);
+      return;
+    } else if (!mon_cmd->is_compat(leader_cmd)) {
+      if (mon_cmd->is_noforward()) {
+	reply_command(op, -EINVAL,
+		      "command not compatible with leader and not allowed to forward",
+		      0);
+	return;
+      }
+      dout(10) << "Command not compatible with leader, forwarding request "
+	       << m << dendl;
+      forward_request_leader(op);
+      return;
+    }
+  }
+
+  if (mon_cmd->is_obsolete() ||
+      (cct->_conf->mon_debug_deprecated_as_obsolete
+       && mon_cmd->is_deprecated())) {
+    reply_command(op, -ENOTSUP,
+                  "command is obsolete; please check usage and/or man page",
+                  0);
+    return;
+  }
+
+  if (session->proxy_con && mon_cmd->is_noforward()) {
+    dout(10) << "Got forward for noforward command " << m << dendl;
+    reply_command(op, -EINVAL, "forward for noforward command", rdata, 0);
+    return;
+  }
+
+  /* what we perceive as being the service the command falls under */
+  string service(mon_cmd->module);
+
+  dout(25) << __func__ << " prefix='" << prefix
+           << "' module='" << module
+           << "' service='" << service << "'" << dendl;
+
+  bool cmd_is_rw =
+    (mon_cmd->requires_perm('w') || mon_cmd->requires_perm('x'));
+
+  // validate user's permissions for requested command
+  map<string,string> param_str_map;
+
+  // Catch bad_cmd_get exception if _generate_command_map() throws it
+  try {
+    _generate_command_map(cmdmap, param_str_map);
+  }
+  catch(bad_cmd_get& e) {
+    reply_command(op, -EINVAL, e.what(), 0);
+  }
+
+  if (!_allowed_command(session, service, prefix, cmdmap,
+                        param_str_map, mon_cmd)) {
+    dout(1) << __func__ << " access denied" << dendl;
+    if (prefix != "config set" && prefix != "config-key set")
+      (cmd_is_rw ? audit_clog->info() : audit_clog->debug())
+        << "from='" << session->name << " " << session->addrs << "' "
+        << "entity='" << session->entity_name << "' "
+        << "cmd=" << m->cmd << ":  access denied";
+    reply_command(op, -EACCES, "access denied", 0);
+    return;
+  }
+
+  if (prefix != "config set" && prefix != "config-key set")
+    (cmd_is_rw ? audit_clog->info() : audit_clog->debug())
+        << "from='" << session->name << " " << session->addrs << "' "
+        << "entity='" << session->entity_name << "' "
+        << "cmd=" << m->cmd << ": dispatch";
+
+  // compat kludge for legacy clients trying to tell commands that are
+  // new.  see bottom of MonCommands.h.  we need to handle both (1)
+  // pre-octopus clients and (2) octopus clients with a mix of pre-octopus
+  // and octopus mons.
+  if ((!HAVE_FEATURE(m->get_connection()->get_features(), SERVER_OCTOPUS) ||
+       monmap->min_mon_release < ceph_release_t::octopus) &&
+      (prefix == "injectargs" ||
+       prefix == "smart" ||
+       prefix == "mon_status" ||
+       prefix == "heap")) {
+    if (m->get_connection()->get_messenger() == 0) {
+      // Prior to octopus, monitors might forward these messages
+      // around. that was broken at baseline, and if we try to process
+      // this message now, it will assert out when we try to send a
+      // message in reply from the asok/tell worker (see
+      // AnonConnection).  Just reply with an error.
+      dout(5) << __func__ << " failing forwarded command from a (presumably) "
+	      << "pre-octopus peer" << dendl;
+      reply_command(
+	op, -EBUSY,
+	"failing forwarded tell command in mixed-version mon cluster", 0);
+      return;
+    }
+    dout(5) << __func__ << " passing command to tell/asok" << dendl;
+    cct->get_admin_socket()->queue_tell_command(m);
+    return;
+  }
+
+  if (mon_cmd->is_mgr()) {
+    const auto& hdr = m->get_header();
+    uint64_t size = hdr.front_len + hdr.middle_len + hdr.data_len;
+    uint64_t max = g_conf().get_val<Option::size_t>("mon_client_bytes")
+                 * g_conf().get_val<double>("mon_mgr_proxy_client_bytes_ratio");
+    if (mgr_proxy_bytes + size > max) {
+      dout(10) << __func__ << " current mgr proxy bytes " << mgr_proxy_bytes
+	       << " + " << size << " > max " << max << dendl;
+      reply_command(op, -EAGAIN, "hit limit on proxied mgr commands", rdata, 0);
+      return;
+    }
+    mgr_proxy_bytes += size;
+    dout(10) << __func__ << " proxying mgr command (+" << size
+	     << " -> " << mgr_proxy_bytes << ")" << dendl;
+    C_MgrProxyCommand *fin = new C_MgrProxyCommand(this, op, size);
+    mgr_client.start_command(m->cmd,
+			     m->get_data(),
+			     &fin->outbl,
+			     &fin->outs,
+			     new C_OnFinisher(fin, &finisher));
+    return;
+  }
+
+  if ((module == "mds" || module == "fs")  &&
+      prefix != "fs authorize") {
+    mdsmon()->dispatch(op);
+    return;
+  }
+  if ((module == "osd" ||
+       prefix == "pg map" ||
+       prefix == "pg repeer") &&
+      prefix != "osd last-stat-seq") {
+    osdmon()->dispatch(op);
+    return;
+  }
+  if (module == "config") {
+    configmon()->dispatch(op);
+    return;
+  }
+
+  if (module == "mon" &&
+      /* Let the Monitor class handle the following commands:
+       *  'mon scrub'
+       */
+      prefix != "mon scrub" &&
+      prefix != "mon metadata" &&
+      prefix != "mon versions" &&
+      prefix != "mon count-metadata" &&
+      prefix != "mon ok-to-stop" &&
+      prefix != "mon ok-to-add-offline" &&
+      prefix != "mon ok-to-rm") {
+    monmon()->dispatch(op);
+    return;
+  }
+  if (module == "health" && prefix != "health") {
+    healthmon()->dispatch(op);
+    return;
+  }
+  if (module == "auth" || prefix == "fs authorize") {
+    authmon()->dispatch(op);
+    return;
+  }
+  if (module == "log") {
+    logmon()->dispatch(op);
+    return;
+  }
+
+  if (module == "config-key") {
+    kvmon()->dispatch(op);
+    return;
+  }
+
+  if (module == "mgr") {
+    mgrmon()->dispatch(op);
+    return;
+  }
+
+  if (prefix == "fsid") {
+    if (f) {
+      f->open_object_section("fsid");
+      f->dump_stream("fsid") << monmap->fsid;
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      ds << monmap->fsid;
+      rdata.append(ds);
+    }
+    reply_command(op, 0, "", rdata, 0);
+    return;
+  }
+
+  if (prefix == "mon scrub") {
+    wait_for_paxos_write();
+    if (is_leader()) {
+      int r = scrub_start();
+      reply_command(op, r, "", rdata, 0);
+    } else if (is_peon()) {
+      forward_request_leader(op);
+    } else {
+      reply_command(op, -EAGAIN, "no quorum", rdata, 0);
+    }
+    return;
+  }
+
+  if (prefix == "time-sync-status") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    f->open_object_section("time_sync");
+    if (!timecheck_skews.empty()) {
+      f->open_object_section("time_skew_status");
+      for (auto& i : timecheck_skews) {
+	double skew = i.second;
+	double latency = timecheck_latencies[i.first];
+	string name = monmap->get_name(i.first);
+	ostringstream tcss;
+	health_status_t tcstatus = timecheck_status(tcss, skew, latency);
+	f->open_object_section(name.c_str());
+	f->dump_float("skew", skew);
+	f->dump_float("latency", latency);
+	f->dump_stream("health") << tcstatus;
+	if (tcstatus != HEALTH_OK) {
+	  f->dump_stream("details") << tcss.str();
+	}
+	f->close_section();
+      }
+      f->close_section();
+    }
+    f->open_object_section("timechecks");
+    f->dump_unsigned("epoch", get_epoch());
+    f->dump_int("round", timecheck_round);
+    f->dump_stream("round_status") << ((timecheck_round%2) ?
+				       "on-going" : "finished");
+    f->close_section();
+    f->close_section();
+    f->flush(rdata);
+    r = 0;
+    rs = "";
+  } else if (prefix == "status" ||
+	     prefix == "health" ||
+	     prefix == "df") {
+    string detail;
+    cmd_getval(cmdmap, "detail", detail);
+
+    if (prefix == "status") {
+      // get_cluster_status handles f == NULL
+      get_cluster_status(ds, f.get(), session);
+
+      if (f) {
+        f->flush(ds);
+        ds << '\n';
+      }
+      rdata.append(ds);
+    } else if (prefix == "health") {
+      string plain;
+      healthmon()->get_health_status(detail == "detail", f.get(), f ? nullptr : &plain);
+      if (f) {
+	f->flush(rdata);
+      } else {
+	rdata.append(plain);
+      }
+    } else if (prefix == "df") {
+      bool verbose = (detail == "detail");
+      if (f)
+        f->open_object_section("stats");
+
+      mgrstatmon()->dump_cluster_stats(&ds, f.get(), verbose);
+      if (!f) {
+	ds << "\n \n";
+      }
+      mgrstatmon()->dump_pool_stats(osdmon()->osdmap, &ds, f.get(), verbose);
+
+      if (f) {
+        f->close_section();
+        f->flush(ds);
+        ds << '\n';
+      }
+    } else {
+      ceph_abort_msg("We should never get here!");
+      return;
+    }
+    rdata.append(ds);
+    rs = "";
+    r = 0;
+  } else if (prefix == "report") {
+
+    // this must be formatted, in its current form
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    f->open_object_section("report");
+    f->dump_stream("cluster_fingerprint") << fingerprint;
+    f->dump_string("version", ceph_version_to_str());
+    f->dump_string("commit", git_version_to_str());
+    f->dump_stream("timestamp") << ceph_clock_now();
+
+    vector<string> tagsvec;
+    cmd_getval(cmdmap, "tags", tagsvec);
+    string tagstr = str_join(tagsvec, " ");
+    if (!tagstr.empty())
+      tagstr = tagstr.substr(0, tagstr.find_last_of(' '));
+    f->dump_string("tag", tagstr);
+
+    healthmon()->get_health_status(true, f.get(), nullptr);
+
+    monmon()->dump_info(f.get());
+    osdmon()->dump_info(f.get());
+    mdsmon()->dump_info(f.get());
+    authmon()->dump_info(f.get());
+    mgrstatmon()->dump_info(f.get());
+
+    paxos->dump_info(f.get());
+
+    f->close_section();
+    f->flush(rdata);
+
+    ostringstream ss2;
+    ss2 << "report " << rdata.crc32c(CEPH_MON_PORT_LEGACY);
+    rs = ss2.str();
+    r = 0;
+  } else if (prefix == "osd last-stat-seq") {
+    int64_t osd = 0;
+    cmd_getval(cmdmap, "id", osd);
+    uint64_t seq = mgrstatmon()->get_last_osd_stat_seq(osd);
+    if (f) {
+      f->dump_unsigned("seq", seq);
+      f->flush(ds);
+    } else {
+      ds << seq;
+      rdata.append(ds);
+    }
+    rs = "";
+    r = 0;
+  } else if (prefix == "node ls") {
+    string node_type("all");
+    cmd_getval(cmdmap, "type", node_type);
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    if (node_type == "all") {
+      f->open_object_section("nodes");
+      print_nodes(f.get(), ds);
+      osdmon()->print_nodes(f.get());
+      mdsmon()->print_nodes(f.get());
+      mgrmon()->print_nodes(f.get());
+      f->close_section();
+    } else if (node_type == "mon") {
+      print_nodes(f.get(), ds);
+    } else if (node_type == "osd") {
+      osdmon()->print_nodes(f.get());
+    } else if (node_type == "mds") {
+      mdsmon()->print_nodes(f.get());
+    } else if (node_type == "mgr") {
+      mgrmon()->print_nodes(f.get());
+    }
+    f->flush(ds);
+    rdata.append(ds);
+    rs = "";
+    r = 0;
+  } else if (prefix == "features") {
+    if (!is_leader() && !is_peon()) {
+      dout(10) << " waiting for quorum" << dendl;
+      waitfor_quorum.push_back(new C_RetryMessage(this, op));
+      return;
+    }
+    if (!is_leader()) {
+      forward_request_leader(op);
+      return;
+    }
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    FeatureMap fm;
+    get_combined_feature_map(&fm);
+    f->dump_object("features", fm);
+    f->flush(rdata);
+    rs = "";
+    r = 0;
+  } else if (prefix == "mon metadata") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+
+    string name;
+    bool all = !cmd_getval(cmdmap, "id", name);
+    if (!all) {
+      // Dump a single mon's metadata
+      int mon = monmap->get_rank(name);
+      if (mon < 0) {
+        rs = "requested mon not found";
+        r = -ENOENT;
+        goto out;
+      }
+      f->open_object_section("mon_metadata");
+      r = get_mon_metadata(mon, f.get(), ds);
+      f->close_section();
+    } else {
+      // Dump all mons' metadata
+      r = 0;
+      f->open_array_section("mon_metadata");
+      for (unsigned int rank = 0; rank < monmap->size(); ++rank) {
+        std::ostringstream get_err;
+        f->open_object_section("mon");
+        f->dump_string("name", monmap->get_name(rank));
+        r = get_mon_metadata(rank, f.get(), get_err);
+        f->close_section();
+        if (r == -ENOENT || r == -EINVAL) {
+          dout(1) << get_err.str() << dendl;
+          // Drop error, list what metadata we do have
+          r = 0;
+        } else if (r != 0) {
+          derr << "Unexpected error from get_mon_metadata: "
+               << cpp_strerror(r) << dendl;
+          ds << get_err.str();
+          break;
+        }
+      }
+      f->close_section();
+    }
+
+    f->flush(ds);
+    rdata.append(ds);
+    rs = "";
+  } else if (prefix == "mon versions") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    count_metadata("ceph_version", f.get());
+    f->flush(ds);
+    rdata.append(ds);
+    rs = "";
+    r = 0;
+  } else if (prefix == "mon count-metadata") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    string field;
+    cmd_getval(cmdmap, "property", field);
+    count_metadata(field, f.get());
+    f->flush(ds);
+    rdata.append(ds);
+    rs = "";
+    r = 0;
+  } else if (prefix == "quorum_status") {
+    // make sure our map is readable and up to date
+    if (!is_leader() && !is_peon()) {
+      dout(10) << " waiting for quorum" << dendl;
+      waitfor_quorum.push_back(new C_RetryMessage(this, op));
+      return;
+    }
+    _quorum_status(f.get(), ds);
+    rdata.append(ds);
+    rs = "";
+    r = 0;
+  } else if (prefix == "mon ok-to-stop") {
+    vector<string> ids;
+    if (!cmd_getval(cmdmap, "ids", ids)) {
+      r = -EINVAL;
+      goto out;
+    }
+    set<string> wouldbe;
+    for (auto rank : quorum) {
+      wouldbe.insert(monmap->get_name(rank));
+    }
+    for (auto& n : ids) {
+      if (monmap->contains(n)) {
+	wouldbe.erase(n);
+      }
+    }
+    if (wouldbe.size() < monmap->min_quorum_size()) {
+      r = -EBUSY;
+      rs = "not enough monitors would be available (" + stringify(wouldbe) +
+	") after stopping mons " + stringify(ids);
+      goto out;
+    }
+    r = 0;
+    rs = "quorum should be preserved (" + stringify(wouldbe) +
+      ") after stopping " + stringify(ids);
+  } else if (prefix == "mon ok-to-add-offline") {
+    if (quorum.size() < monmap->min_quorum_size(monmap->size() + 1)) {
+      rs = "adding a monitor may break quorum (until that monitor starts)";
+      r = -EBUSY;
+      goto out;
+    }
+    rs = "adding another mon that is not yet online will not break quorum";
+    r = 0;
+  } else if (prefix == "mon ok-to-rm") {
+    string id;
+    if (!cmd_getval(cmdmap, "id", id)) {
+      r = -EINVAL;
+      rs = "must specify a monitor id";
+      goto out;
+    }
+    if (!monmap->contains(id)) {
+      r = 0;
+      rs = "mon." + id + " does not exist";
+      goto out;
+    }
+    int rank = monmap->get_rank(id);
+    if (quorum.count(rank) &&
+	quorum.size() - 1 < monmap->min_quorum_size(monmap->size() - 1)) {
+      r = -EBUSY;
+      rs = "removing mon." + id + " would break quorum";
+      goto out;
+    }
+    r = 0;
+    rs = "safe to remove mon." + id;
+  } else if (prefix == "version") {
+    if (f) {
+      f->open_object_section("version");
+      f->dump_string("version", pretty_version_to_str());
+      f->close_section();
+      f->flush(ds);
+    } else {
+      ds << pretty_version_to_str();
+    }
+    rdata.append(ds);
+    rs = "";
+    r = 0;
+  } else if (prefix == "versions") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    map<string,int> overall;
+    f->open_object_section("version");
+    map<string,int> mon, mgr, osd, mds;
+
+    count_metadata("ceph_version", &mon);
+    f->open_object_section("mon");
+    for (auto& p : mon) {
+      f->dump_int(p.first.c_str(), p.second);
+      overall[p.first] += p.second;
+    }
+    f->close_section();
+
+    mgrmon()->count_metadata("ceph_version", &mgr);
+    f->open_object_section("mgr");
+    for (auto& p : mgr) {
+      f->dump_int(p.first.c_str(), p.second);
+      overall[p.first] += p.second;
+    }
+    f->close_section();
+
+    osdmon()->count_metadata("ceph_version", &osd);
+    f->open_object_section("osd");
+    for (auto& p : osd) {
+      f->dump_int(p.first.c_str(), p.second);
+      overall[p.first] += p.second;
+    }
+    f->close_section();
+
+    mdsmon()->count_metadata("ceph_version", &mds);
+    f->open_object_section("mds");
+    for (auto& p : mds) {
+      f->dump_int(p.first.c_str(), p.second);
+      overall[p.first] += p.second;
+    }
+    f->close_section();
+
+    for (auto& p : mgrstatmon()->get_service_map().services) {
+      auto &service = p.first;
+      if (ServiceMap::is_normal_ceph_entity(service)) {
+        continue;
+      }
+      f->open_object_section(service.c_str());
+      map<string,int> m;
+      p.second.count_metadata("ceph_version", &m);
+      for (auto& q : m) {
+	f->dump_int(q.first.c_str(), q.second);
+	overall[q.first] += q.second;
+      }
+      f->close_section();
+    }
+
+    f->open_object_section("overall");
+    for (auto& p : overall) {
+      f->dump_int(p.first.c_str(), p.second);
+    }
+    f->close_section();
+    f->close_section();
+    f->flush(rdata);
+    rs = "";
+    r = 0;
+  }
+
+ out:
+  if (!m->get_source().is_mon())  // don't reply to mon->mon commands
+    reply_command(op, r, rs, rdata, 0);
+}
+
+void Monitor::reply_command(MonOpRequestRef op, int rc, const string &rs, version_t version)
+{
+  bufferlist rdata;
+  reply_command(op, rc, rs, rdata, version);
+}
+
+void Monitor::reply_command(MonOpRequestRef op, int rc, const string &rs,
+                            bufferlist& rdata, version_t version)
+{
+  auto m = op->get_req<MMonCommand>();
+  ceph_assert(m->get_type() == MSG_MON_COMMAND);
+  MMonCommandAck *reply = new MMonCommandAck(m->cmd, rc, rs, version);
+  reply->set_tid(m->get_tid());
+  reply->set_data(rdata);
+  send_reply(op, reply);
+}
+
+void Monitor::reply_tell_command(
+  MonOpRequestRef op, int rc, const string &rs)
+{
+  MCommand *m = static_cast<MCommand*>(op->get_req());
+  ceph_assert(m->get_type() == MSG_COMMAND);
+  MCommandReply *reply = new MCommandReply(rc, rs);
+  reply->set_tid(m->get_tid());
+  m->get_connection()->send_message(reply);
+}
+
+
+// ------------------------
+// request/reply routing
+//
+// a client/mds/osd will connect to a random monitor.  we need to forward any
+// messages requiring state updates to the leader, and then route any replies
+// back via the correct monitor and back to them.  (the monitor will not
+// initiate any connections.)
+
+void Monitor::forward_request_leader(MonOpRequestRef op)
+{
+  op->mark_event(__func__);
+
+  int mon = get_leader();
+  MonSession *session = op->get_session();
+  PaxosServiceMessage *req = op->get_req<PaxosServiceMessage>();
+  
+  if (req->get_source().is_mon() && req->get_source_addrs() != messenger->get_myaddrs()) {
+    dout(10) << "forward_request won't forward (non-local) mon request " << *req << dendl;
+  } else if (session->proxy_con) {
+    dout(10) << "forward_request won't double fwd request " << *req << dendl;
+  } else if (!session->closed) {
+    RoutedRequest *rr = new RoutedRequest;
+    rr->tid = ++routed_request_tid;
+    rr->con = req->get_connection();
+    rr->con_features = rr->con->get_features();
+    encode_message(req, CEPH_FEATURES_ALL, rr->request_bl);   // for my use only; use all features
+    rr->session = static_cast<MonSession *>(session->get());
+    rr->op = op;
+    routed_requests[rr->tid] = rr;
+    session->routed_request_tids.insert(rr->tid);
+    
+    dout(10) << "forward_request " << rr->tid << " request " << *req
+	     << " features " << rr->con_features << dendl;
+
+    MForward *forward = new MForward(rr->tid,
+                                     req,
+				     rr->con_features,
+				     rr->session->caps);
+    forward->set_priority(req->get_priority());
+    if (session->auth_handler) {
+      forward->entity_name = session->entity_name;
+    } else if (req->get_source().is_mon()) {
+      forward->entity_name.set_type(CEPH_ENTITY_TYPE_MON);
+    }
+    send_mon_message(forward, mon);
+    op->mark_forwarded();
+    ceph_assert(op->get_req()->get_type() != 0);
+  } else {
+    dout(10) << "forward_request no session for request " << *req << dendl;
+  }
+}
+
+// fake connection attached to forwarded messages
+struct AnonConnection : public Connection {
+  entity_addr_t socket_addr;
+
+  int send_message(Message *m) override {
+    ceph_assert(!"send_message on anonymous connection");
+  }
+  void send_keepalive() override {
+    ceph_assert(!"send_keepalive on anonymous connection");
+  }
+  void mark_down() override {
+    // silently ignore
+  }
+  void mark_disposable() override {
+    // silengtly ignore
+  }
+  bool is_connected() override { return false; }
+  entity_addr_t get_peer_socket_addr() const override {
+    return socket_addr;
+  }
+
+private:
+  FRIEND_MAKE_REF(AnonConnection);
+  explicit AnonConnection(CephContext *cct, const entity_addr_t& sa)
+    : Connection(cct, nullptr),
+      socket_addr(sa) {}
+};
+
+//extract the original message and put it into the regular dispatch function
+void Monitor::handle_forward(MonOpRequestRef op)
+{
+  auto m = op->get_req<MForward>();
+  dout(10) << "received forwarded message from "
+	   << ceph_entity_type_name(m->client_type)
+	   << " " << m->client_addrs
+	   << " via " << m->get_source_inst() << dendl;
+  MonSession *session = op->get_session();
+  ceph_assert(session);
+
+  if (!session->is_capable("mon", MON_CAP_X)) {
+    dout(0) << "forward from entity with insufficient caps! " 
+	    << session->caps << dendl;
+  } else {
+    // see PaxosService::dispatch(); we rely on this being anon
+    // (c->msgr == NULL)
+    PaxosServiceMessage *req = m->claim_message();
+    ceph_assert(req != NULL);
+
+    auto c = ceph::make_ref<AnonConnection>(cct, m->client_socket_addr);
+    MonSession *s = new MonSession(static_cast<Connection*>(c.get()));
+    s->_ident(req->get_source(),
+	      req->get_source_addrs());
+    c->set_priv(RefCountedPtr{s, false});
+    c->set_peer_addrs(m->client_addrs);
+    c->set_peer_type(m->client_type);
+    c->set_features(m->con_features);
+
+    s->authenticated = true;
+    s->caps = m->client_caps;
+    dout(10) << " caps are " << s->caps << dendl;
+    s->entity_name = m->entity_name;
+    dout(10) << " entity name '" << s->entity_name << "' type "
+             << s->entity_name.get_type() << dendl;
+    s->proxy_con = m->get_connection();
+    s->proxy_tid = m->tid;
+
+    req->set_connection(c);
+
+    // not super accurate, but better than nothing.
+    req->set_recv_stamp(m->get_recv_stamp());
+
+    /*
+     * note which election epoch this is; we will drop the message if
+     * there is a future election since our peers will resend routed
+     * requests in that case.
+     */
+    req->rx_election_epoch = get_epoch();
+
+    dout(10) << " mesg " << req << " from " << m->get_source_addr() << dendl;
+    _ms_dispatch(req);
+
+    // break the session <-> con ref loop by removing the con->session
+    // reference, which is no longer needed once the MonOpRequest is
+    // set up.
+    c->set_priv(NULL);
+  }
+}
+
+void Monitor::send_reply(MonOpRequestRef op, Message *reply)
+{
+  op->mark_event(__func__);
+
+  MonSession *session = op->get_session();
+  ceph_assert(session);
+  Message *req = op->get_req();
+  ConnectionRef con = op->get_connection();
+
+  reply->set_cct(g_ceph_context);
+  dout(2) << __func__ << " " << op << " " << reply << " " << *reply << dendl;
+
+  if (!con) {
+    dout(2) << "send_reply no connection, dropping reply " << *reply
+	    << " to " << req << " " << *req << dendl;
+    reply->put();
+    op->mark_event("reply: no connection");
+    return;
+  }
+
+  if (!session->con && !session->proxy_con) {
+    dout(2) << "send_reply no connection, dropping reply " << *reply
+	    << " to " << req << " " << *req << dendl;
+    reply->put();
+    op->mark_event("reply: no connection");
+    return;
+  }
+
+  if (session->proxy_con) {
+    dout(15) << "send_reply routing reply to " << con->get_peer_addr()
+	     << " via " << session->proxy_con->get_peer_addr()
+	     << " for request " << *req << dendl;
+    session->proxy_con->send_message(new MRoute(session->proxy_tid, reply));
+    op->mark_event("reply: send routed request");
+  } else {
+    session->con->send_message(reply);
+    op->mark_event("reply: send");
+  }
+}
+
+void Monitor::no_reply(MonOpRequestRef op)
+{
+  MonSession *session = op->get_session();
+  Message *req = op->get_req();
+
+  if (session->proxy_con) {
+    dout(10) << "no_reply to " << req->get_source_inst()
+	     << " via " << session->proxy_con->get_peer_addr()
+	     << " for request " << *req << dendl;
+    session->proxy_con->send_message(new MRoute(session->proxy_tid, NULL));
+    op->mark_event("no_reply: send routed request");
+  } else {
+    dout(10) << "no_reply to " << req->get_source_inst()
+             << " " << *req << dendl;
+    op->mark_event("no_reply");
+  }
+}
+
+void Monitor::handle_route(MonOpRequestRef op)
+{
+  auto m = op->get_req<MRoute>();
+  MonSession *session = op->get_session();
+  //check privileges
+  if (!session->is_capable("mon", MON_CAP_X)) {
+    dout(0) << "MRoute received from entity without appropriate perms! "
+	    << dendl;
+    return;
+  }
+  if (m->msg)
+    dout(10) << "handle_route tid " << m->session_mon_tid << " " << *m->msg
+	     << dendl;
+  else
+    dout(10) << "handle_route tid " << m->session_mon_tid << " null" << dendl;
+  
+  // look it up
+  if (!m->session_mon_tid) {
+    dout(10) << " not a routed request, ignoring" << dendl;
+    return;
+  }
+  auto found = routed_requests.find(m->session_mon_tid);
+  if (found == routed_requests.end()) {
+    dout(10) << " don't have routed request tid " << m->session_mon_tid << dendl;
+    return;
+  }
+  std::unique_ptr<RoutedRequest> rr{found->second};
+  // reset payload, in case encoding is dependent on target features
+  if (m->msg) {
+    m->msg->clear_payload();
+    rr->con->send_message(m->msg);
+    m->msg = NULL;
+  }
+  if (m->send_osdmap_first) {
+    dout(10) << " sending osdmaps from " << m->send_osdmap_first << dendl;
+    osdmon()->send_incremental(m->send_osdmap_first, rr->session,
+			       true, MonOpRequestRef());
+  }
+  ceph_assert(rr->tid == m->session_mon_tid && rr->session->routed_request_tids.count(m->session_mon_tid));
+  routed_requests.erase(found);
+  rr->session->routed_request_tids.erase(m->session_mon_tid);
+}
+
+void Monitor::resend_routed_requests()
+{
+  dout(10) << "resend_routed_requests" << dendl;
+  int mon = get_leader();
+  list<Context*> retry;
+  for (map<uint64_t, RoutedRequest*>::iterator p = routed_requests.begin();
+       p != routed_requests.end();
+       ++p) {
+    RoutedRequest *rr = p->second;
+
+    if (mon == rank) {
+      dout(10) << " requeue for self tid " << rr->tid << dendl;
+      rr->op->mark_event("retry routed request");
+      retry.push_back(new C_RetryMessage(this, rr->op));
+      if (rr->session) {
+        ceph_assert(rr->session->routed_request_tids.count(p->first));
+        rr->session->routed_request_tids.erase(p->first);
+      }
+      delete rr;
+    } else {
+      auto q = rr->request_bl.cbegin();
+      PaxosServiceMessage *req =
+	(PaxosServiceMessage *)decode_message(cct, 0, q);
+      rr->op->mark_event("resend forwarded message to leader");
+      dout(10) << " resend to mon." << mon << " tid " << rr->tid << " " << *req
+	       << dendl;
+      MForward *forward = new MForward(rr->tid,
+				       req,
+				       rr->con_features,
+				       rr->session->caps);
+      req->put();  // forward takes its own ref; drop ours.
+      forward->client_type = rr->con->get_peer_type();
+      forward->client_addrs = rr->con->get_peer_addrs();
+      forward->client_socket_addr = rr->con->get_peer_socket_addr();
+      forward->set_priority(req->get_priority());
+      send_mon_message(forward, mon);
+    }
+  }
+  if (mon == rank) {
+    routed_requests.clear();
+    finish_contexts(g_ceph_context, retry);
+  }
+}
+
+void Monitor::remove_session(MonSession *s)
+{
+  dout(10) << "remove_session " << s << " " << s->name << " " << s->addrs
+	   << " features 0x" << std::hex << s->con_features << std::dec << dendl;
+  ceph_assert(s->con);
+  ceph_assert(!s->closed);
+  for (set<uint64_t>::iterator p = s->routed_request_tids.begin();
+       p != s->routed_request_tids.end();
+       ++p) {
+    ceph_assert(routed_requests.count(*p));
+    RoutedRequest *rr = routed_requests[*p];
+    dout(10) << " dropping routed request " << rr->tid << dendl;
+    delete rr;
+    routed_requests.erase(*p);
+  }
+  s->routed_request_tids.clear();
+  s->con->set_priv(nullptr);
+  session_map.remove_session(s);
+  logger->set(l_mon_num_sessions, session_map.get_size());
+  logger->inc(l_mon_session_rm);
+}
+
+void Monitor::remove_all_sessions()
+{
+  std::lock_guard l(session_map_lock);
+  while (!session_map.sessions.empty()) {
+    MonSession *s = session_map.sessions.front();
+    remove_session(s);
+    logger->inc(l_mon_session_rm);
+  }
+  if (logger)
+    logger->set(l_mon_num_sessions, session_map.get_size());
+}
+
+void Monitor::send_mon_message(Message *m, int rank)
+{
+  messenger->send_to_mon(m, monmap->get_addrs(rank));
+}
+
+void Monitor::waitlist_or_zap_client(MonOpRequestRef op)
+{
+  /**
+   * Wait list the new session until we're in the quorum, assuming it's
+   * sufficiently new.
+   * tick() will periodically send them back through so we can send
+   * the client elsewhere if we don't think we're getting back in.
+   *
+   * But we allow a few sorts of messages:
+   * 1) Monitors can talk to us at any time, of course.
+   * 2) auth messages. It's unlikely to go through much faster, but
+   * it's possible we've just lost our quorum status and we want to take...
+   * 3) command messages. We want to accept these under all possible
+   * circumstances.
+   */
+  Message *m = op->get_req();
+  MonSession *s = op->get_session();
+  ConnectionRef con = op->get_connection();
+  utime_t too_old = ceph_clock_now();
+  too_old -= g_ceph_context->_conf->mon_lease;
+  if (m->get_recv_stamp() > too_old &&
+      con->is_connected()) {
+    dout(5) << "waitlisting message " << *m << dendl;
+    maybe_wait_for_quorum.push_back(new C_RetryMessage(this, op));
+    op->mark_wait_for_quorum();
+  } else {
+    dout(5) << "discarding message " << *m << " and sending client elsewhere" << dendl;
+    con->mark_down();
+    // proxied sessions aren't registered and don't have a con; don't remove
+    // those.
+    if (!s->proxy_con) {
+      std::lock_guard l(session_map_lock);
+      remove_session(s);
+    }
+    op->mark_zap();
+  }
+}
+
+void Monitor::_ms_dispatch(Message *m)
+{
+  if (is_shutdown()) {
+    m->put();
+    return;
+  }
+
+  MonOpRequestRef op = op_tracker.create_request<MonOpRequest>(m);
+  bool src_is_mon = op->is_src_mon();
+  op->mark_event("mon:_ms_dispatch");
+  MonSession *s = op->get_session();
+  if (s && s->closed) {
+    return;
+  }
+
+  if (src_is_mon && s) {
+    ConnectionRef con = m->get_connection();
+    if (con->get_messenger() && con->get_features() != s->con_features) {
+      // only update features if this is a non-anonymous connection
+      dout(10) << __func__ << " feature change for " << m->get_source_inst()
+               << " (was " << s->con_features
+               << ", now " << con->get_features() << ")" << dendl;
+      // connection features changed - recreate session.
+      if (s->con && s->con != con) {
+        dout(10) << __func__ << " connection for " << m->get_source_inst()
+                 << " changed from session; mark down and replace" << dendl;
+        s->con->mark_down();
+      }
+      if (s->item.is_on_list()) {
+        // forwarded messages' sessions are not in the sessions map and
+        // exist only while the op is being handled.
+        std::lock_guard l(session_map_lock);
+        remove_session(s);
+      }
+      s = nullptr;
+    }
+  }
+
+  if (!s) {
+    // if the sender is not a monitor, make sure their first message for a
+    // session is an MAuth.  If it is not, assume it's a stray message,
+    // and considering that we are creating a new session it is safe to
+    // assume that the sender hasn't authenticated yet, so we have no way
+    // of assessing whether we should handle it or not.
+    if (!src_is_mon && (m->get_type() != CEPH_MSG_AUTH &&
+			m->get_type() != CEPH_MSG_MON_GET_MAP &&
+			m->get_type() != CEPH_MSG_PING)) {
+      dout(1) << __func__ << " dropping stray message " << *m
+	      << " from " << m->get_source_inst() << dendl;
+      return;
+    }
+
+    ConnectionRef con = m->get_connection();
+    {
+      std::lock_guard l(session_map_lock);
+      s = session_map.new_session(m->get_source(),
+				  m->get_source_addrs(),
+				  con.get());
+    }
+    ceph_assert(s);
+    con->set_priv(RefCountedPtr{s, false});
+    dout(10) << __func__ << " new session " << s << " " << *s
+	     << " features 0x" << std::hex
+	     << s->con_features << std::dec << dendl;
+    op->set_session(s);
+
+    logger->set(l_mon_num_sessions, session_map.get_size());
+    logger->inc(l_mon_session_add);
+
+    if (src_is_mon) {
+      // give it monitor caps; the peer type has been authenticated
+      dout(5) << __func__ << " setting monitor caps on this connection" << dendl;
+      if (!s->caps.is_allow_all()) // but no need to repeatedly copy
+        s->caps = mon_caps;
+      s->authenticated = true;
+    }
+  } else {
+    dout(20) << __func__ << " existing session " << s << " for " << s->name
+	     << dendl;
+  }
+
+  ceph_assert(s);
+
+  s->session_timeout = ceph_clock_now();
+  s->session_timeout += g_conf()->mon_session_timeout;
+
+  if (s->auth_handler) {
+    s->entity_name = s->auth_handler->get_entity_name();
+    s->global_id = s->auth_handler->get_global_id();
+    s->global_id_status = s->auth_handler->get_global_id_status();
+  }
+  dout(20) << " entity_name " << s->entity_name
+	   << " global_id " << s->global_id
+	   << " (" << s->global_id_status
+	   << ") caps " << s->caps.get_str() << dendl;
+
+  if (!session_stretch_allowed(s, op)) {
+    return;
+  }
+  if ((is_synchronizing() ||
+       (!s->authenticated && !exited_quorum.is_zero())) &&
+      !src_is_mon &&
+      m->get_type() != CEPH_MSG_PING) {
+    waitlist_or_zap_client(op);
+  } else {
+    dispatch_op(op);
+  }
+  return;
+}
+
+void Monitor::dispatch_op(MonOpRequestRef op)
+{
+  op->mark_event("mon:dispatch_op");
+  MonSession *s = op->get_session();
+  ceph_assert(s);
+  if (s->closed) {
+    dout(10) << " session closed, dropping " << op->get_req() << dendl;
+    return;
+  }
+
+  /* we will consider the default type as being 'monitor' until proven wrong */
+  op->set_type_monitor();
+  /* deal with all messages that do not necessarily need caps */
+  switch (op->get_req()->get_type()) {
+    // auth
+    case MSG_MON_GLOBAL_ID:
+    case CEPH_MSG_AUTH:
+      op->set_type_service();
+      /* no need to check caps here */
+      paxos_service[PAXOS_AUTH]->dispatch(op);
+      return;
+
+    case CEPH_MSG_PING:
+      handle_ping(op);
+      return;
+    case MSG_COMMAND:
+      op->set_type_command();
+      handle_tell_command(op);
+      return;
+  }
+
+  if (!op->get_session()->authenticated) {
+    dout(5) << __func__ << " " << op->get_req()->get_source_inst()
+            << " is not authenticated, dropping " << *(op->get_req())
+            << dendl;
+    return;
+  }
+
+  // global_id_status == NONE: all sessions for auth_none and krb,
+  // mon <-> mon sessions (including proxied sessions) for cephx
+  ceph_assert(s->global_id_status == global_id_status_t::NONE ||
+              s->global_id_status == global_id_status_t::NEW_OK ||
+              s->global_id_status == global_id_status_t::NEW_NOT_EXPOSED ||
+              s->global_id_status == global_id_status_t::RECLAIM_OK ||
+              s->global_id_status == global_id_status_t::RECLAIM_INSECURE);
+
+  // let mon_getmap through for "ping" (which doesn't reconnect)
+  // and "tell" (which reconnects but doesn't attempt to preserve
+  // its global_id and stays in NEW_NOT_EXPOSED, retrying until
+  // ->send_attempts reaches 0)
+  if (cct->_conf->auth_expose_insecure_global_id_reclaim &&
+      s->global_id_status == global_id_status_t::NEW_NOT_EXPOSED &&
+      op->get_req()->get_type() != CEPH_MSG_MON_GET_MAP) {
+    dout(5) << __func__ << " " << op->get_req()->get_source_inst()
+            << " may omit old_ticket on reconnects, discarding "
+            << *op->get_req() << " and forcing reconnect" << dendl;
+    ceph_assert(s->con && !s->proxy_con);
+    s->con->mark_down();
+    {
+      std::lock_guard l(session_map_lock);
+      remove_session(s);
+    }
+    op->mark_zap();
+    return;
+  }
+
+  switch (op->get_req()->get_type()) {
+    case CEPH_MSG_MON_GET_MAP:
+      handle_mon_get_map(op);
+      return;
+
+    case MSG_GET_CONFIG:
+      configmon()->handle_get_config(op);
+      return;
+
+    case CEPH_MSG_MON_SUBSCRIBE:
+      /* FIXME: check what's being subscribed, filter accordingly */
+      handle_subscribe(op);
+      return;
+  }
+
+  /* well, maybe the op belongs to a service... */
+  op->set_type_service();
+  /* deal with all messages which caps should be checked somewhere else */
+  switch (op->get_req()->get_type()) {
+
+    // OSDs
+    case CEPH_MSG_MON_GET_OSDMAP:
+    case CEPH_MSG_POOLOP:
+    case MSG_OSD_BEACON:
+    case MSG_OSD_MARK_ME_DOWN:
+    case MSG_OSD_MARK_ME_DEAD:
+    case MSG_OSD_FULL:
+    case MSG_OSD_FAILURE:
+    case MSG_OSD_BOOT:
+    case MSG_OSD_ALIVE:
+    case MSG_OSD_PGTEMP:
+    case MSG_OSD_PG_CREATED:
+    case MSG_REMOVE_SNAPS:
+    case MSG_MON_GET_PURGED_SNAPS:
+    case MSG_OSD_PG_READY_TO_MERGE:
+      paxos_service[PAXOS_OSDMAP]->dispatch(op);
+      return;
+
+    // MDSs
+    case MSG_MDS_BEACON:
+    case MSG_MDS_OFFLOAD_TARGETS:
+      paxos_service[PAXOS_MDSMAP]->dispatch(op);
+      return;
+
+    // Mgrs
+    case MSG_MGR_BEACON:
+      paxos_service[PAXOS_MGR]->dispatch(op);
+      return;
+
+    // MgrStat
+    case MSG_MON_MGR_REPORT:
+    case CEPH_MSG_STATFS:
+    case MSG_GETPOOLSTATS:
+      paxos_service[PAXOS_MGRSTAT]->dispatch(op);
+      return;
+
+      // log
+    case MSG_LOG:
+      paxos_service[PAXOS_LOG]->dispatch(op);
+      return;
+
+    // handle_command() does its own caps checking
+    case MSG_MON_COMMAND:
+      op->set_type_command();
+      handle_command(op);
+      return;
+  }
+
+  /* nop, looks like it's not a service message; revert back to monitor */
+  op->set_type_monitor();
+
+  /* messages we, the Monitor class, need to deal with
+   * but may be sent by clients. */
+
+  if (!op->get_session()->is_capable("mon", MON_CAP_R)) {
+    dout(5) << __func__ << " " << op->get_req()->get_source_inst()
+            << " not enough caps for " << *(op->get_req()) << " -- dropping"
+            << dendl;
+    return;
+  }
+
+  switch (op->get_req()->get_type()) {
+    // misc
+    case CEPH_MSG_MON_GET_VERSION:
+      handle_get_version(op);
+      return;
+  }
+
+  if (!op->is_src_mon()) {
+    dout(1) << __func__ << " unexpected monitor message from"
+            << " non-monitor entity " << op->get_req()->get_source_inst()
+            << " " << *(op->get_req()) << " -- dropping" << dendl;
+    return;
+  }
+
+  /* messages that should only be sent by another monitor */
+  switch (op->get_req()->get_type()) {
+
+    case MSG_ROUTE:
+      handle_route(op);
+      return;
+
+    case MSG_MON_PROBE:
+      handle_probe(op);
+      return;
+
+    // Sync (i.e., the new slurp, but on steroids)
+    case MSG_MON_SYNC:
+      handle_sync(op);
+      return;
+    case MSG_MON_SCRUB:
+      handle_scrub(op);
+      return;
+
+    /* log acks are sent from a monitor we sent the MLog to, and are
+       never sent by clients to us. */
+    case MSG_LOGACK:
+      log_client.handle_log_ack((MLogAck*)op->get_req());
+      return;
+
+    // monmap
+    case MSG_MON_JOIN:
+      op->set_type_service();
+      paxos_service[PAXOS_MONMAP]->dispatch(op);
+      return;
+
+    // paxos
+    case MSG_MON_PAXOS:
+      {
+        op->set_type_paxos();
+        auto pm = op->get_req<MMonPaxos>();
+        if (!op->get_session()->is_capable("mon", MON_CAP_X)) {
+          //can't send these!
+          return;
+        }
+
+        if (state == STATE_SYNCHRONIZING) {
+          // we are synchronizing. These messages would do us no
+          // good, thus just drop them and ignore them.
+          dout(10) << __func__ << " ignore paxos msg from "
+            << pm->get_source_inst() << dendl;
+          return;
+        }
+
+        // sanitize
+        if (pm->epoch > get_epoch()) {
+          bootstrap();
+          return;
+        }
+        if (pm->epoch != get_epoch()) {
+          return;
+        }
+
+        paxos->dispatch(op);
+      }
+      return;
+
+    // elector messages
+    case MSG_MON_ELECTION:
+      op->set_type_election_or_ping();
+      //check privileges here for simplicity
+      if (!op->get_session()->is_capable("mon", MON_CAP_X)) {
+        dout(0) << "MMonElection received from entity without enough caps!"
+          << op->get_session()->caps << dendl;
+        return;;
+      }
+      if (!is_probing() && !is_synchronizing()) {
+        elector.dispatch(op);
+      }
+      return;
+
+    case MSG_MON_PING:
+      op->set_type_election_or_ping();
+      elector.dispatch(op);
+      return;
+
+    case MSG_FORWARD:
+      handle_forward(op);
+      return;
+
+    case MSG_TIMECHECK:
+      dout(5) << __func__ << " ignoring " << op << dendl;
+      return;
+    case MSG_TIMECHECK2:
+      handle_timecheck(op);
+      return;
+
+    case MSG_MON_HEALTH:
+      dout(5) << __func__ << " dropping deprecated message: "
+	      << *op->get_req() << dendl;
+      break;
+    case MSG_MON_HEALTH_CHECKS:
+      op->set_type_service();
+      paxos_service[PAXOS_HEALTH]->dispatch(op);
+      return;
+  }
+  dout(1) << "dropping unexpected " << *(op->get_req()) << dendl;
+  return;
+}
+
+void Monitor::handle_ping(MonOpRequestRef op)
+{
+  auto m = op->get_req<MPing>();
+  dout(10) << __func__ << " " << *m << dendl;
+  MPing *reply = new MPing;
+  bufferlist payload;
+  boost::scoped_ptr<Formatter> f(new JSONFormatter(true));
+  f->open_object_section("pong");
+
+  healthmon()->get_health_status(false, f.get(), nullptr);
+  get_mon_status(f.get());
+
+  f->close_section();
+  stringstream ss;
+  f->flush(ss);
+  encode(ss.str(), payload);
+  reply->set_payload(payload);
+  dout(10) << __func__ << " reply payload len " << reply->get_payload().length() << dendl;
+  m->get_connection()->send_message(reply);
+}
+
+void Monitor::timecheck_start()
+{
+  dout(10) << __func__ << dendl;
+  timecheck_cleanup();
+  if (get_quorum_mon_features().contains_all(
+	ceph::features::mon::FEATURE_NAUTILUS)) {
+    timecheck_start_round();
+  }
+}
+
+void Monitor::timecheck_finish()
+{
+  dout(10) << __func__ << dendl;
+  timecheck_cleanup();
+}
+
+void Monitor::timecheck_start_round()
+{
+  dout(10) << __func__ << " curr " << timecheck_round << dendl;
+  ceph_assert(is_leader());
+
+  if (monmap->size() == 1) {
+    ceph_abort_msg("We are alone; this shouldn't have been scheduled!");
+    return;
+  }
+
+  if (timecheck_round % 2) {
+    dout(10) << __func__ << " there's a timecheck going on" << dendl;
+    utime_t curr_time = ceph_clock_now();
+    double max = g_conf()->mon_timecheck_interval*3;
+    if (curr_time - timecheck_round_start < max) {
+      dout(10) << __func__ << " keep current round going" << dendl;
+      goto out;
+    } else {
+      dout(10) << __func__
+               << " finish current timecheck and start new" << dendl;
+      timecheck_cancel_round();
+    }
+  }
+
+  ceph_assert(timecheck_round % 2 == 0);
+  timecheck_acks = 0;
+  timecheck_round ++;
+  timecheck_round_start = ceph_clock_now();
+  dout(10) << __func__ << " new " << timecheck_round << dendl;
+
+  timecheck();
+out:
+  dout(10) << __func__ << " setting up next event" << dendl;
+  timecheck_reset_event();
+}
+
+void Monitor::timecheck_finish_round(bool success)
+{
+  dout(10) << __func__ << " curr " << timecheck_round << dendl;
+  ceph_assert(timecheck_round % 2);
+  timecheck_round ++;
+  timecheck_round_start = utime_t();
+
+  if (success) {
+    ceph_assert(timecheck_waiting.empty());
+    ceph_assert(timecheck_acks == quorum.size());
+    timecheck_report();
+    timecheck_check_skews();
+    return;
+  }
+
+  dout(10) << __func__ << " " << timecheck_waiting.size()
+           << " peers still waiting:";
+  for (auto& p : timecheck_waiting) {
+    *_dout << " mon." << p.first;
+  }
+  *_dout << dendl;
+  timecheck_waiting.clear();
+
+  dout(10) << __func__ << " finished to " << timecheck_round << dendl;
+}
+
+void Monitor::timecheck_cancel_round()
+{
+  timecheck_finish_round(false);
+}
+
+void Monitor::timecheck_cleanup()
+{
+  timecheck_round = 0;
+  timecheck_acks = 0;
+  timecheck_round_start = utime_t();
+
+  if (timecheck_event) {
+    timer.cancel_event(timecheck_event);
+    timecheck_event = NULL;
+  }
+  timecheck_waiting.clear();
+  timecheck_skews.clear();
+  timecheck_latencies.clear();
+
+  timecheck_rounds_since_clean = 0;
+}
+
+void Monitor::timecheck_reset_event()
+{
+  if (timecheck_event) {
+    timer.cancel_event(timecheck_event);
+    timecheck_event = NULL;
+  }
+
+  double delay =
+    cct->_conf->mon_timecheck_skew_interval * timecheck_rounds_since_clean;
+
+  if (delay <= 0 || delay > cct->_conf->mon_timecheck_interval) {
+    delay = cct->_conf->mon_timecheck_interval;
+  }
+
+  dout(10) << __func__ << " delay " << delay
+           << " rounds_since_clean " << timecheck_rounds_since_clean
+           << dendl;
+
+  timecheck_event = timer.add_event_after(
+    delay,
+    new C_MonContext{this, [this](int) {
+	timecheck_start_round();
+      }});
+}
+
+void Monitor::timecheck_check_skews()
+{
+  dout(10) << __func__ << dendl;
+  ceph_assert(is_leader());
+  ceph_assert((timecheck_round % 2) == 0);
+  if (monmap->size() == 1) {
+    ceph_abort_msg("We are alone; we shouldn't have gotten here!");
+    return;
+  }
+  ceph_assert(timecheck_latencies.size() == timecheck_skews.size());
+
+  bool found_skew = false;
+  for (auto& p : timecheck_skews) {
+    double abs_skew;
+    if (timecheck_has_skew(p.second, &abs_skew)) {
+      dout(10) << __func__
+               << " " << p.first << " skew " << abs_skew << dendl;
+      found_skew = true;
+    }
+  }
+
+  if (found_skew) {
+    ++timecheck_rounds_since_clean;
+    timecheck_reset_event();
+  } else if (timecheck_rounds_since_clean > 0) {
+    dout(1) << __func__
+      << " no clock skews found after " << timecheck_rounds_since_clean
+      << " rounds" << dendl;
+    // make sure the skews are really gone and not just a transient success
+    // this will run just once if not in the presence of skews again.
+    timecheck_rounds_since_clean = 1;
+    timecheck_reset_event();
+    timecheck_rounds_since_clean = 0;
+  }
+
+}
+
+void Monitor::timecheck_report()
+{
+  dout(10) << __func__ << dendl;
+  ceph_assert(is_leader());
+  ceph_assert((timecheck_round % 2) == 0);
+  if (monmap->size() == 1) {
+    ceph_abort_msg("We are alone; we shouldn't have gotten here!");
+    return;
+  }
+
+  ceph_assert(timecheck_latencies.size() == timecheck_skews.size());
+  bool do_output = true; // only output report once
+  for (set<int>::iterator q = quorum.begin(); q != quorum.end(); ++q) {
+    if (monmap->get_name(*q) == name)
+      continue;
+
+    MTimeCheck2 *m = new MTimeCheck2(MTimeCheck2::OP_REPORT);
+    m->epoch = get_epoch();
+    m->round = timecheck_round;
+
+    for (auto& it : timecheck_skews) {
+      double skew = it.second;
+      double latency = timecheck_latencies[it.first];
+
+      m->skews[it.first] = skew;
+      m->latencies[it.first] = latency;
+
+      if (do_output) {
+        dout(25) << __func__ << " mon." << it.first
+                 << " latency " << latency
+                 << " skew " << skew << dendl;
+      }
+    }
+    do_output = false;
+    dout(10) << __func__ << " send report to mon." << *q << dendl;
+    send_mon_message(m, *q);
+  }
+}
+
+void Monitor::timecheck()
+{
+  dout(10) << __func__ << dendl;
+  ceph_assert(is_leader());
+  if (monmap->size() == 1) {
+    ceph_abort_msg("We are alone; we shouldn't have gotten here!");
+    return;
+  }
+  ceph_assert(timecheck_round % 2 != 0);
+
+  timecheck_acks = 1; // we ack ourselves
+
+  dout(10) << __func__ << " start timecheck epoch " << get_epoch()
+           << " round " << timecheck_round << dendl;
+
+  // we are at the eye of the storm; the point of reference
+  timecheck_skews[rank] = 0.0;
+  timecheck_latencies[rank] = 0.0;
+
+  for (set<int>::iterator it = quorum.begin(); it != quorum.end(); ++it) {
+    if (monmap->get_name(*it) == name)
+      continue;
+
+    utime_t curr_time = ceph_clock_now();
+    timecheck_waiting[*it] = curr_time;
+    MTimeCheck2 *m = new MTimeCheck2(MTimeCheck2::OP_PING);
+    m->epoch = get_epoch();
+    m->round = timecheck_round;
+    dout(10) << __func__ << " send " << *m << " to mon." << *it << dendl;
+    send_mon_message(m, *it);
+  }
+}
+
+health_status_t Monitor::timecheck_status(ostringstream &ss,
+                                          const double skew_bound,
+                                          const double latency)
+{
+  health_status_t status = HEALTH_OK;
+  ceph_assert(latency >= 0);
+
+  double abs_skew;
+  if (timecheck_has_skew(skew_bound, &abs_skew)) {
+    status = HEALTH_WARN;
+    ss << "clock skew " << abs_skew << "s"
+       << " > max " << g_conf()->mon_clock_drift_allowed << "s";
+  }
+
+  return status;
+}
+
+void Monitor::handle_timecheck_leader(MonOpRequestRef op)
+{
+  auto m = op->get_req<MTimeCheck2>();
+  dout(10) << __func__ << " " << *m << dendl;
+  /* handles PONG's */
+  ceph_assert(m->op == MTimeCheck2::OP_PONG);
+
+  int other = m->get_source().num();
+  if (m->epoch < get_epoch()) {
+    dout(1) << __func__ << " got old timecheck epoch " << m->epoch
+            << " from " << other
+            << " curr " << get_epoch()
+            << " -- severely lagged? discard" << dendl;
+    return;
+  }
+  ceph_assert(m->epoch == get_epoch());
+
+  if (m->round < timecheck_round) {
+    dout(1) << __func__ << " got old round " << m->round
+            << " from " << other
+            << " curr " << timecheck_round << " -- discard" << dendl;
+    return;
+  }
+
+  utime_t curr_time = ceph_clock_now();
+
+  ceph_assert(timecheck_waiting.count(other) > 0);
+  utime_t timecheck_sent = timecheck_waiting[other];
+  timecheck_waiting.erase(other);
+  if (curr_time < timecheck_sent) {
+    // our clock was readjusted -- drop everything until it all makes sense.
+    dout(1) << __func__ << " our clock was readjusted --"
+            << " bump round and drop current check"
+            << dendl;
+    timecheck_cancel_round();
+    return;
+  }
+
+  /* update peer latencies */
+  double latency = (double)(curr_time - timecheck_sent);
+
+  if (timecheck_latencies.count(other) == 0)
+    timecheck_latencies[other] = latency;
+  else {
+    double avg_latency = ((timecheck_latencies[other]*0.8)+(latency*0.2));
+    timecheck_latencies[other] = avg_latency;
+  }
+
+  /*
+   * update skews
+   *
+   * some nasty thing goes on if we were to do 'a - b' between two utime_t,
+   * and 'a' happens to be lower than 'b'; so we use double instead.
+   *
+   * latency is always expected to be >= 0.
+   *
+   * delta, the difference between theirs timestamp and ours, may either be
+   * lower or higher than 0; will hardly ever be 0.
+   *
+   * The absolute skew is the absolute delta minus the latency, which is
+   * taken as a whole instead of an rtt given that there is some queueing
+   * and dispatch times involved and it's hard to assess how long exactly
+   * it took for the message to travel to the other side and be handled. So
+   * we call it a bounded skew, the worst case scenario.
+   *
+   * Now, to math!
+   *
+   * Given that the latency is always positive, we can establish that the
+   * bounded skew will be:
+   *
+   *  1. positive if the absolute delta is higher than the latency and
+   *     delta is positive
+   *  2. negative if the absolute delta is higher than the latency and
+   *     delta is negative.
+   *  3. zero if the absolute delta is lower than the latency.
+   *
+   * On 3. we make a judgement call and treat the skew as non-existent.
+   * This is because that, if the absolute delta is lower than the
+   * latency, then the apparently existing skew is nothing more than a
+   * side-effect of the high latency at work.
+   *
+   * This may not be entirely true though, as a severely skewed clock
+   * may be masked by an even higher latency, but with high latencies
+   * we probably have worse issues to deal with than just skewed clocks.
+   */
+  ceph_assert(latency >= 0);
+
+  double delta = ((double) m->timestamp) - ((double) curr_time);
+  double abs_delta = (delta > 0 ? delta : -delta);
+  double skew_bound = abs_delta - latency;
+  if (skew_bound < 0)
+    skew_bound = 0;
+  else if (delta < 0)
+    skew_bound = -skew_bound;
+
+  ostringstream ss;
+  health_status_t status = timecheck_status(ss, skew_bound, latency);
+  if (status != HEALTH_OK) {
+    clog->health(status) << other << " " << ss.str();
+  }
+
+  dout(10) << __func__ << " from " << other << " ts " << m->timestamp
+	   << " delta " << delta << " skew_bound " << skew_bound
+	   << " latency " << latency << dendl;
+
+  timecheck_skews[other] = skew_bound;
+
+  timecheck_acks++;
+  if (timecheck_acks == quorum.size()) {
+    dout(10) << __func__ << " got pongs from everybody ("
+             << timecheck_acks << " total)" << dendl;
+    ceph_assert(timecheck_skews.size() == timecheck_acks);
+    ceph_assert(timecheck_waiting.empty());
+    // everyone has acked, so bump the round to finish it.
+    timecheck_finish_round();
+  }
+}
+
+void Monitor::handle_timecheck_peon(MonOpRequestRef op)
+{
+  auto m = op->get_req<MTimeCheck2>();
+  dout(10) << __func__ << " " << *m << dendl;
+
+  ceph_assert(is_peon());
+  ceph_assert(m->op == MTimeCheck2::OP_PING || m->op == MTimeCheck2::OP_REPORT);
+
+  if (m->epoch != get_epoch()) {
+    dout(1) << __func__ << " got wrong epoch "
+            << "(ours " << get_epoch()
+            << " theirs: " << m->epoch << ") -- discarding" << dendl;
+    return;
+  }
+
+  if (m->round < timecheck_round) {
+    dout(1) << __func__ << " got old round " << m->round
+            << " current " << timecheck_round
+            << " (epoch " << get_epoch() << ") -- discarding" << dendl;
+    return;
+  }
+
+  timecheck_round = m->round;
+
+  if (m->op == MTimeCheck2::OP_REPORT) {
+    ceph_assert((timecheck_round % 2) == 0);
+    timecheck_latencies.swap(m->latencies);
+    timecheck_skews.swap(m->skews);
+    return;
+  }
+
+  ceph_assert((timecheck_round % 2) != 0);
+  MTimeCheck2 *reply = new MTimeCheck2(MTimeCheck2::OP_PONG);
+  utime_t curr_time = ceph_clock_now();
+  reply->timestamp = curr_time;
+  reply->epoch = m->epoch;
+  reply->round = m->round;
+  dout(10) << __func__ << " send " << *m
+           << " to " << m->get_source_inst() << dendl;
+  m->get_connection()->send_message(reply);
+}
+
+void Monitor::handle_timecheck(MonOpRequestRef op)
+{
+  auto m = op->get_req<MTimeCheck2>();
+  dout(10) << __func__ << " " << *m << dendl;
+
+  if (is_leader()) {
+    if (m->op != MTimeCheck2::OP_PONG) {
+      dout(1) << __func__ << " drop unexpected msg (not pong)" << dendl;
+    } else {
+      handle_timecheck_leader(op);
+    }
+  } else if (is_peon()) {
+    if (m->op != MTimeCheck2::OP_PING && m->op != MTimeCheck2::OP_REPORT) {
+      dout(1) << __func__ << " drop unexpected msg (not ping or report)" << dendl;
+    } else {
+      handle_timecheck_peon(op);
+    }
+  } else {
+    dout(1) << __func__ << " drop unexpected msg" << dendl;
+  }
+}
+
+void Monitor::handle_subscribe(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonSubscribe>();
+  dout(10) << "handle_subscribe " << *m << dendl;
+  
+  bool reply = false;
+
+  MonSession *s = op->get_session();
+  ceph_assert(s);
+
+  if (m->hostname.size()) {
+    s->remote_host = m->hostname;
+  }
+
+  for (map<string,ceph_mon_subscribe_item>::iterator p = m->what.begin();
+       p != m->what.end();
+       ++p) {
+    if (p->first == "monmap" || p->first == "config") {
+      // these require no caps
+    } else if (!s->is_capable("mon", MON_CAP_R)) {
+      dout(5) << __func__ << " " << op->get_req()->get_source_inst()
+	      << " not enough caps for " << *(op->get_req()) << " -- dropping"
+	      << dendl;
+      continue;
+    }
+
+    // if there are any non-onetime subscriptions, we need to reply to start the resubscribe timer
+    if ((p->second.flags & CEPH_SUBSCRIBE_ONETIME) == 0)
+      reply = true;
+
+    // remove conflicting subscribes
+    if (logmon()->sub_name_to_id(p->first) >= 0) {
+      for (map<string, Subscription*>::iterator it = s->sub_map.begin();
+	   it != s->sub_map.end(); ) {
+	if (it->first != p->first && logmon()->sub_name_to_id(it->first) >= 0) {
+	  std::lock_guard l(session_map_lock);
+	  session_map.remove_sub((it++)->second);
+	} else {
+	  ++it;
+	}
+      }
+    }
+
+    {
+      std::lock_guard l(session_map_lock);
+      session_map.add_update_sub(s, p->first, p->second.start,
+				 p->second.flags & CEPH_SUBSCRIBE_ONETIME,
+				 m->get_connection()->has_feature(CEPH_FEATURE_INCSUBOSDMAP));
+    }
+
+    if (p->first.compare(0, 6, "mdsmap") == 0 || p->first.compare(0, 5, "fsmap") == 0) {
+      dout(10) << __func__ << ": MDS sub '" << p->first << "'" << dendl;
+      if ((int)s->is_capable("mds", MON_CAP_R)) {
+        Subscription *sub = s->sub_map[p->first];
+        ceph_assert(sub != nullptr);
+        mdsmon()->check_sub(sub);
+      }
+    } else if (p->first == "osdmap") {
+      if ((int)s->is_capable("osd", MON_CAP_R)) {
+	if (s->osd_epoch > p->second.start) {
+	  // client needs earlier osdmaps on purpose, so reset the sent epoch
+	  s->osd_epoch = 0;
+	}
+        osdmon()->check_osdmap_sub(s->sub_map["osdmap"]);
+      }
+    } else if (p->first == "osd_pg_creates") {
+      if ((int)s->is_capable("osd", MON_CAP_W)) {
+	osdmon()->check_pg_creates_sub(s->sub_map["osd_pg_creates"]);
+      }
+    } else if (p->first == "monmap") {
+      monmon()->check_sub(s->sub_map[p->first]);
+    } else if (logmon()->sub_name_to_id(p->first) >= 0) {
+      logmon()->check_sub(s->sub_map[p->first]);
+    } else if (p->first == "mgrmap" || p->first == "mgrdigest") {
+      mgrmon()->check_sub(s->sub_map[p->first]);
+    } else if (p->first == "servicemap") {
+      mgrstatmon()->check_sub(s->sub_map[p->first]);
+    } else if (p->first == "config") {
+      configmon()->check_sub(s);
+    } else if (p->first.find("kv:") == 0) {
+      kvmon()->check_sub(s->sub_map[p->first]);
+    }
+  }
+
+  if (reply) {
+    // we only need to reply if the client is old enough to think it
+    // has to send renewals.
+    ConnectionRef con = m->get_connection();
+    if (!con->has_feature(CEPH_FEATURE_MON_STATEFUL_SUB))
+      m->get_connection()->send_message(new MMonSubscribeAck(
+	monmap->get_fsid(), (int)g_conf()->mon_subscribe_interval));
+  }
+
+}
+
+void Monitor::handle_get_version(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonGetVersion>();
+  dout(10) << "handle_get_version " << *m << dendl;
+  PaxosService *svc = NULL;
+
+  MonSession *s = op->get_session();
+  ceph_assert(s);
+
+  if (!is_leader() && !is_peon()) {
+    dout(10) << " waiting for quorum" << dendl;
+    waitfor_quorum.push_back(new C_RetryMessage(this, op));
+    goto out;
+  }
+
+  if (m->what == "mdsmap") {
+    svc = mdsmon();
+  } else if (m->what == "fsmap") {
+    svc = mdsmon();
+  } else if (m->what == "osdmap") {
+    svc = osdmon();
+  } else if (m->what == "monmap") {
+    svc = monmon();
+  } else {
+    derr << "invalid map type " << m->what << dendl;
+  }
+
+  if (svc) {
+    if (!svc->is_readable()) {
+      svc->wait_for_readable(op, new C_RetryMessage(this, op));
+      goto out;
+    }
+
+    MMonGetVersionReply *reply = new MMonGetVersionReply();
+    reply->handle = m->handle;
+    reply->version = svc->get_last_committed();
+    reply->oldest_version = svc->get_first_committed();
+    reply->set_tid(m->get_tid());
+
+    m->get_connection()->send_message(reply);
+  }
+ out:
+  return;
+}
+
+bool Monitor::ms_handle_reset(Connection *con)
+{
+  dout(10) << "ms_handle_reset " << con << " " << con->get_peer_addr() << dendl;
+
+  // ignore lossless monitor sessions
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON)
+    return false;
+
+  auto priv = con->get_priv();
+  auto s = static_cast<MonSession*>(priv.get());
+  if (!s)
+    return false;
+
+  // break any con <-> session ref cycle
+  s->con->set_priv(nullptr);
+
+  if (is_shutdown())
+    return false;
+
+  std::lock_guard l(lock);
+
+  dout(10) << "reset/close on session " << s->name << " " << s->addrs << dendl;
+  if (!s->closed && s->item.is_on_list()) {
+    std::lock_guard l(session_map_lock);
+    remove_session(s);
+  }
+  return true;
+}
+
+bool Monitor::ms_handle_refused(Connection *con)
+{
+  // just log for now...
+  dout(10) << "ms_handle_refused " << con << " " << con->get_peer_addr() << dendl;
+  return false;
+}
+
+// -----
+
+void Monitor::send_latest_monmap(Connection *con)
+{
+  bufferlist bl;
+  monmap->encode(bl, con->get_features());
+  con->send_message(new MMonMap(bl));
+}
+
+void Monitor::handle_mon_get_map(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonGetMap>();
+  dout(10) << "handle_mon_get_map" << dendl;
+  send_latest_monmap(m->get_connection().get());
+}
+
+int Monitor::load_metadata()
+{
+  bufferlist bl;
+  int r = store->get(MONITOR_STORE_PREFIX, "last_metadata", bl);
+  if (r)
+    return r;
+  auto it = bl.cbegin();
+  decode(mon_metadata, it);
+
+  pending_metadata = mon_metadata;
+  return 0;
+}
+
+int Monitor::get_mon_metadata(int mon, Formatter *f, ostream& err)
+{
+  ceph_assert(f);
+  if (!mon_metadata.count(mon)) {
+    err << "mon." << mon << " not found";
+    return -EINVAL;
+  }
+  const Metadata& m = mon_metadata[mon];
+  for (Metadata::const_iterator p = m.begin(); p != m.end(); ++p) {
+    f->dump_string(p->first.c_str(), p->second);
+  }
+  return 0;
+}
+
+void Monitor::count_metadata(const string& field, map<string,int> *out)
+{
+  for (auto& p : mon_metadata) {
+    auto q = p.second.find(field);
+    if (q == p.second.end()) {
+      (*out)["unknown"]++;
+    } else {
+      (*out)[q->second]++;
+    }
+  }
+}
+
+void Monitor::count_metadata(const string& field, Formatter *f)
+{
+  map<string,int> by_val;
+  count_metadata(field, &by_val);
+  f->open_object_section(field.c_str());
+  for (auto& p : by_val) {
+    f->dump_int(p.first.c_str(), p.second);
+  }
+  f->close_section();
+}
+
+void Monitor::get_all_versions(std::map<string, list<string> > &versions)
+{
+  // mon
+  get_versions(versions);
+  // osd
+  osdmon()->get_versions(versions);
+  // mgr
+  mgrmon()->get_versions(versions);
+  // mds
+  mdsmon()->get_versions(versions);
+  dout(20) << __func__ << " all versions=" << versions << dendl;
+}
+
+void Monitor::get_versions(std::map<string, list<string> > &versions)
+{
+  for (auto& [rank, metadata] : mon_metadata) {
+    auto q = metadata.find("ceph_version_short");
+    if (q == metadata.end()) {
+      // not likely
+      continue;
+    }
+    versions[q->second].push_back(string("mon.") + monmap->get_name(rank));
+  }
+}
+
+int Monitor::print_nodes(Formatter *f, ostream& err)
+{
+  map<string, list<string> > mons;	// hostname => mon
+  for (map<int, Metadata>::iterator it = mon_metadata.begin();
+       it != mon_metadata.end(); ++it) {
+    const Metadata& m = it->second;
+    Metadata::const_iterator hostname = m.find("hostname");
+    if (hostname == m.end()) {
+      // not likely though
+      continue;
+    }
+    mons[hostname->second].push_back(monmap->get_name(it->first));
+  }
+
+  dump_services(f, mons, "mon");
+  return 0;
+}
+
+// ----------------------------------------------
+// scrub
+
+int Monitor::scrub_start()
+{
+  dout(10) << __func__ << dendl;
+  ceph_assert(is_leader());
+
+  if (!scrub_result.empty()) {
+    clog->info() << "scrub already in progress";
+    return -EBUSY;
+  }
+
+  scrub_event_cancel();
+  scrub_result.clear();
+  scrub_state.reset(new ScrubState);
+
+  scrub();
+  return 0;
+}
+
+int Monitor::scrub()
+{
+  ceph_assert(is_leader());
+  ceph_assert(scrub_state);
+
+  scrub_cancel_timeout();
+  wait_for_paxos_write();
+  scrub_version = paxos->get_version();
+
+
+  // scrub all keys if we're the only monitor in the quorum
+  int32_t num_keys =
+    (quorum.size() == 1 ? -1 : cct->_conf->mon_scrub_max_keys);
+
+  for (set<int>::iterator p = quorum.begin();
+       p != quorum.end();
+       ++p) {
+    if (*p == rank)
+      continue;
+    MMonScrub *r = new MMonScrub(MMonScrub::OP_SCRUB, scrub_version,
+                                 num_keys);
+    r->key = scrub_state->last_key;
+    send_mon_message(r, *p);
+  }
+
+  // scrub my keys
+  bool r = _scrub(&scrub_result[rank],
+                  &scrub_state->last_key,
+                  &num_keys);
+
+  scrub_state->finished = !r;
+
+  // only after we got our scrub results do we really care whether the
+  // other monitors are late on their results.  Also, this way we avoid
+  // triggering the timeout if we end up getting stuck in _scrub() for
+  // longer than the duration of the timeout.
+  scrub_reset_timeout();
+
+  if (quorum.size() == 1) {
+    ceph_assert(scrub_state->finished == true);
+    scrub_finish();
+  }
+  return 0;
+}
+
+void Monitor::handle_scrub(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonScrub>();
+  dout(10) << __func__ << " " << *m << dendl;
+  switch (m->op) {
+  case MMonScrub::OP_SCRUB:
+    {
+      if (!is_peon())
+	break;
+
+      wait_for_paxos_write();
+
+      if (m->version != paxos->get_version())
+	break;
+
+      MMonScrub *reply = new MMonScrub(MMonScrub::OP_RESULT,
+                                       m->version,
+                                       m->num_keys);
+
+      reply->key = m->key;
+      _scrub(&reply->result, &reply->key, &reply->num_keys);
+      m->get_connection()->send_message(reply);
+    }
+    break;
+
+  case MMonScrub::OP_RESULT:
+    {
+      if (!is_leader())
+	break;
+      if (m->version != scrub_version)
+	break;
+      // reset the timeout each time we get a result
+      scrub_reset_timeout();
+
+      int from = m->get_source().num();
+      ceph_assert(scrub_result.count(from) == 0);
+      scrub_result[from] = m->result;
+
+      if (scrub_result.size() == quorum.size()) {
+        scrub_check_results();
+        scrub_result.clear();
+        if (scrub_state->finished)
+          scrub_finish();
+        else
+          scrub();
+      }
+    }
+    break;
+  }
+}
+
+bool Monitor::_scrub(ScrubResult *r,
+                     pair<string,string> *start,
+                     int *num_keys)
+{
+  ceph_assert(r != NULL);
+  ceph_assert(start != NULL);
+  ceph_assert(num_keys != NULL);
+
+  set<string> prefixes = get_sync_targets_names();
+  prefixes.erase("paxos");  // exclude paxos, as this one may have extra states for proposals, etc.
+
+  dout(10) << __func__ << " start (" << *start << ")"
+           << " num_keys " << *num_keys << dendl;
+
+  MonitorDBStore::Synchronizer it = store->get_synchronizer(*start, prefixes);
+
+  int scrubbed_keys = 0;
+  pair<string,string> last_key;
+
+  while (it->has_next_chunk()) {
+
+    if (*num_keys > 0 && scrubbed_keys == *num_keys)
+      break;
+
+    pair<string,string> k = it->get_next_key();
+    if (prefixes.count(k.first) == 0)
+      continue;
+
+    if (cct->_conf->mon_scrub_inject_missing_keys > 0.0 &&
+        (rand() % 10000 < cct->_conf->mon_scrub_inject_missing_keys*10000.0)) {
+      dout(10) << __func__ << " inject missing key, skipping (" << k << ")"
+               << dendl;
+      continue;
+    }
+
+    bufferlist bl;
+    int err = store->get(k.first, k.second, bl);
+    ceph_assert(err == 0);
+    
+    uint32_t key_crc = bl.crc32c(0);
+    dout(30) << __func__ << " " << k << " bl " << bl.length() << " bytes"
+                                     << " crc " << key_crc << dendl;
+    r->prefix_keys[k.first]++;
+    if (r->prefix_crc.count(k.first) == 0) {
+      r->prefix_crc[k.first] = 0;
+    }
+    r->prefix_crc[k.first] = bl.crc32c(r->prefix_crc[k.first]);
+
+    if (cct->_conf->mon_scrub_inject_crc_mismatch > 0.0 &&
+        (rand() % 10000 < cct->_conf->mon_scrub_inject_crc_mismatch*10000.0)) {
+      dout(10) << __func__ << " inject failure at (" << k << ")" << dendl;
+      r->prefix_crc[k.first] += 1;
+    }
+
+    ++scrubbed_keys;
+    last_key = k;
+  }
+
+  dout(20) << __func__ << " last_key (" << last_key << ")"
+                       << " scrubbed_keys " << scrubbed_keys
+                       << " has_next " << it->has_next_chunk() << dendl;
+
+  *start = last_key;
+  *num_keys = scrubbed_keys;
+
+  return it->has_next_chunk();
+}
+
+void Monitor::scrub_check_results()
+{
+  dout(10) << __func__ << dendl;
+
+  // compare
+  int errors = 0;
+  ScrubResult& mine = scrub_result[rank];
+  for (map<int,ScrubResult>::iterator p = scrub_result.begin();
+       p != scrub_result.end();
+       ++p) {
+    if (p->first == rank)
+      continue;
+    if (p->second != mine) {
+      ++errors;
+      clog->error() << "scrub mismatch";
+      clog->error() << " mon." << rank << " " << mine;
+      clog->error() << " mon." << p->first << " " << p->second;
+    }
+  }
+  if (!errors)
+    clog->debug() << "scrub ok on " << quorum << ": " << mine;
+}
+
+inline void Monitor::scrub_timeout()
+{
+  dout(1) << __func__ << " restarting scrub" << dendl;
+  scrub_reset();
+  scrub_start();
+}
+
+void Monitor::scrub_finish()
+{
+  dout(10) << __func__ << dendl;
+  scrub_reset();
+  scrub_event_start();
+}
+
+void Monitor::scrub_reset()
+{
+  dout(10) << __func__ << dendl;
+  scrub_cancel_timeout();
+  scrub_version = 0;
+  scrub_result.clear();
+  scrub_state.reset();
+}
+
+inline void Monitor::scrub_update_interval(ceph::timespan interval)
+{
+  // we don't care about changes if we are not the leader.
+  // changes will be visible if we become the leader.
+  if (!is_leader())
+    return;
+
+  dout(1) << __func__ << " new interval = " << interval << dendl;
+
+  // if scrub already in progress, all changes will already be visible during
+  // the next round.  Nothing to do.
+  if (scrub_state != NULL)
+    return;
+
+  scrub_event_cancel();
+  scrub_event_start();
+}
+
+void Monitor::scrub_event_start()
+{
+  dout(10) << __func__ << dendl;
+
+  if (scrub_event)
+    scrub_event_cancel();
+
+  auto scrub_interval =
+    cct->_conf.get_val<std::chrono::seconds>("mon_scrub_interval");
+  if (scrub_interval == std::chrono::seconds::zero()) {
+    dout(1) << __func__ << " scrub event is disabled"
+            << " (mon_scrub_interval = " << scrub_interval
+            << ")" << dendl;
+    return;
+  }
+
+  scrub_event = timer.add_event_after(
+    scrub_interval,
+    new C_MonContext{this, [this](int) {
+      scrub_start();
+      }});
+}
+
+void Monitor::scrub_event_cancel()
+{
+  dout(10) << __func__ << dendl;
+  if (scrub_event) {
+    timer.cancel_event(scrub_event);
+    scrub_event = NULL;
+  }
+}
+
+inline void Monitor::scrub_cancel_timeout()
+{
+  if (scrub_timeout_event) {
+    timer.cancel_event(scrub_timeout_event);
+    scrub_timeout_event = NULL;
+  }
+}
+
+void Monitor::scrub_reset_timeout()
+{
+  dout(15) << __func__ << " reset timeout event" << dendl;
+  scrub_cancel_timeout();
+  scrub_timeout_event = timer.add_event_after(
+    g_conf()->mon_scrub_timeout,
+    new C_MonContext{this, [this](int) {
+      scrub_timeout();
+    }});
+}
+
+/************ TICK ***************/
+void Monitor::new_tick()
+{
+  timer.add_event_after(g_conf()->mon_tick_interval, new C_MonContext{this, [this](int) {
+	tick();
+      }});
+}
+
+void Monitor::tick()
+{
+  // ok go.
+  dout(11) << "tick" << dendl;
+  const utime_t now = ceph_clock_now();
+  
+  // Check if we need to emit any delayed health check updated messages
+  if (is_leader()) {
+    const auto min_period = g_conf().get_val<int64_t>(
+                              "mon_health_log_update_period");
+    for (auto& svc : paxos_service) {
+      auto health = svc->get_health_checks();
+
+      for (const auto &i : health.checks) {
+        const std::string &code = i.first;
+        const std::string &summary = i.second.summary;
+        const health_status_t severity = i.second.severity;
+
+        auto status_iter = health_check_log_times.find(code);
+        if (status_iter == health_check_log_times.end()) {
+          continue;
+        }
+
+        auto &log_status = status_iter->second;
+        bool const changed = log_status.last_message != summary
+                             || log_status.severity != severity;
+
+        if (changed && now - log_status.updated_at > min_period) {
+          log_status.last_message = summary;
+          log_status.updated_at = now;
+          log_status.severity = severity;
+
+          ostringstream ss;
+          ss << "Health check update: " << summary << " (" << code << ")";
+          clog->health(severity) << ss.str();
+        }
+      }
+    }
+  }
+
+
+  for (auto& svc : paxos_service) {
+    svc->tick();
+    svc->maybe_trim();
+  }
+  
+  // trim sessions
+  {
+    std::lock_guard l(session_map_lock);
+    auto p = session_map.sessions.begin();
+
+    bool out_for_too_long = (!exited_quorum.is_zero() &&
+			     now > (exited_quorum + 2*g_conf()->mon_lease));
+
+    while (!p.end()) {
+      MonSession *s = *p;
+      ++p;
+    
+      // don't trim monitors
+      if (s->name.is_mon())
+	continue;
+
+      if (s->session_timeout < now && s->con) {
+	// check keepalive, too
+	s->session_timeout = s->con->get_last_keepalive();
+	s->session_timeout += g_conf()->mon_session_timeout;
+      }
+      if (s->session_timeout < now) {
+	dout(10) << " trimming session " << s->con << " " << s->name
+		 << " " << s->addrs
+		 << " (timeout " << s->session_timeout
+		 << " < now " << now << ")" << dendl;
+      } else if (out_for_too_long) {
+	// boot the client Session because we've taken too long getting back in
+	dout(10) << " trimming session " << s->con << " " << s->name
+		 << " because we've been out of quorum too long" << dendl;
+      } else {
+	continue;
+      }
+
+      s->con->mark_down();
+      remove_session(s);
+      logger->inc(l_mon_session_trim);
+    }
+  }
+  sync_trim_providers();
+
+  if (!maybe_wait_for_quorum.empty()) {
+    finish_contexts(g_ceph_context, maybe_wait_for_quorum);
+  }
+
+  if (is_leader() && paxos->is_active() && fingerprint.is_zero()) {
+    // this is only necessary on upgraded clusters.
+    MonitorDBStore::TransactionRef t = paxos->get_pending_transaction();
+    prepare_new_fingerprint(t);
+    paxos->trigger_propose();
+  }
+
+  mgr_client.update_daemon_health(get_health_metrics());
+  new_tick();
+}
+
+vector<DaemonHealthMetric> Monitor::get_health_metrics() 
+{
+  vector<DaemonHealthMetric> metrics;
+
+  utime_t oldest_secs;
+  const utime_t now = ceph_clock_now();
+  auto too_old = now;
+  too_old -= g_conf().get_val<std::chrono::seconds>("mon_op_complaint_time").count();
+  int slow = 0;
+  TrackedOpRef oldest_op;
+  auto count_slow_ops = [&](TrackedOp& op) {
+    if (op.get_initiated() < too_old) {
+      slow++;
+      if (!oldest_op || op.get_initiated() < oldest_op->get_initiated()) {
+	oldest_op = &op;
+      }
+      return true;
+    } else {
+      return false;
+    }
+  };
+  if (op_tracker.visit_ops_in_flight(&oldest_secs, count_slow_ops)) {
+    if (slow) {
+      derr << __func__ << " reporting " << slow << " slow ops, oldest is "
+	   << oldest_op->get_desc() << dendl;
+    }
+    metrics.emplace_back(daemon_metric::SLOW_OPS, slow, oldest_secs);
+  } else {
+    metrics.emplace_back(daemon_metric::SLOW_OPS, 0, 0);
+  }
+  return metrics;
+}
+
+void Monitor::prepare_new_fingerprint(MonitorDBStore::TransactionRef t)
+{
+  uuid_d nf;
+  nf.generate_random();
+  dout(10) << __func__ << " proposing cluster_fingerprint " << nf << dendl;
+
+  bufferlist bl;
+  encode(nf, bl);
+  t->put(MONITOR_NAME, "cluster_fingerprint", bl);
+}
+
+int Monitor::check_fsid()
+{
+  bufferlist ebl;
+  int r = store->get(MONITOR_NAME, "cluster_uuid", ebl);
+  if (r == -ENOENT)
+    return r;
+  ceph_assert(r == 0);
+
+  string es(ebl.c_str(), ebl.length());
+
+  // only keep the first line
+  size_t pos = es.find_first_of('\n');
+  if (pos != string::npos)
+    es.resize(pos);
+
+  dout(10) << "check_fsid cluster_uuid contains '" << es << "'" << dendl;
+  uuid_d ondisk;
+  if (!ondisk.parse(es.c_str())) {
+    derr << "error: unable to parse uuid" << dendl;
+    return -EINVAL;
+  }
+
+  if (monmap->get_fsid() != ondisk) {
+    derr << "error: cluster_uuid file exists with value " << ondisk
+	 << ", != our uuid " << monmap->get_fsid() << dendl;
+    return -EEXIST;
+  }
+
+  return 0;
+}
+
+int Monitor::write_fsid()
+{
+  auto t(std::make_shared<MonitorDBStore::Transaction>());
+  write_fsid(t);
+  int r = store->apply_transaction(t);
+  return r;
+}
+
+int Monitor::write_fsid(MonitorDBStore::TransactionRef t)
+{
+  ostringstream ss;
+  ss << monmap->get_fsid() << "\n";
+  string us = ss.str();
+
+  bufferlist b;
+  b.append(us);
+
+  t->put(MONITOR_NAME, "cluster_uuid", b);
+  return 0;
+}
+
+/*
+ * this is the closest thing to a traditional 'mkfs' for ceph.
+ * initialize the monitor state machines to their initial values.
+ */
+int Monitor::mkfs(bufferlist& osdmapbl)
+{
+  auto t(std::make_shared<MonitorDBStore::Transaction>());
+
+  // verify cluster fsid
+  int r = check_fsid();
+  if (r < 0 && r != -ENOENT)
+    return r;
+
+  bufferlist magicbl;
+  magicbl.append(CEPH_MON_ONDISK_MAGIC);
+  magicbl.append("\n");
+  t->put(MONITOR_NAME, "magic", magicbl);
+
+
+  features = get_initial_supported_features();
+  write_features(t);
+
+  // save monmap, osdmap, keyring.
+  bufferlist monmapbl;
+  monmap->encode(monmapbl, CEPH_FEATURES_ALL);
+  monmap->set_epoch(0);     // must be 0 to avoid confusing first MonmapMonitor::update_from_paxos()
+  t->put("mkfs", "monmap", monmapbl);
+
+  if (osdmapbl.length()) {
+    // make sure it's a valid osdmap
+    try {
+      OSDMap om;
+      om.decode(osdmapbl);
+    }
+    catch (ceph::buffer::error& e) {
+      derr << "error decoding provided osdmap: " << e.what() << dendl;
+      return -EINVAL;
+    }
+    t->put("mkfs", "osdmap", osdmapbl);
+  }
+
+  if (is_keyring_required()) {
+    KeyRing keyring;
+    string keyring_filename;
+
+    r = ceph_resolve_file_search(g_conf()->keyring, keyring_filename);
+    if (r) {
+      if (g_conf()->key != "") {
+	string keyring_plaintext = "[mon.]\n\tkey = " + g_conf()->key +
+	  "\n\tcaps mon = \"allow *\"\n";
+	bufferlist bl;
+	bl.append(keyring_plaintext);
+	try {
+	  auto i = bl.cbegin();
+	  keyring.decode_plaintext(i);
+	}
+	catch (const ceph::buffer::error& e) {
+	  derr << "error decoding keyring " << keyring_plaintext
+	       << ": " << e.what() << dendl;
+	  return -EINVAL;
+	}
+      } else {
+	derr << "unable to find a keyring on " << g_conf()->keyring
+	     << ": " << cpp_strerror(r) << dendl;
+	return r;
+      }
+    } else {
+      r = keyring.load(g_ceph_context, keyring_filename);
+      if (r < 0) {
+	derr << "unable to load initial keyring " << g_conf()->keyring << dendl;
+	return r;
+      }
+    }
+
+    // put mon. key in external keyring; seed with everything else.
+    extract_save_mon_key(keyring);
+
+    bufferlist keyringbl;
+    keyring.encode_plaintext(keyringbl);
+    t->put("mkfs", "keyring", keyringbl);
+  }
+  write_fsid(t);
+  store->apply_transaction(t);
+
+  return 0;
+}
+
+int Monitor::write_default_keyring(bufferlist& bl)
+{
+  ostringstream os;
+  os << g_conf()->mon_data << "/keyring";
+
+  int err = 0;
+  int fd = ::open(os.str().c_str(), O_WRONLY|O_CREAT|O_CLOEXEC, 0600);
+  if (fd < 0) {
+    err = -errno;
+    dout(0) << __func__ << " failed to open " << os.str() 
+	    << ": " << cpp_strerror(err) << dendl;
+    return err;
+  }
+
+  err = bl.write_fd(fd);
+  if (!err)
+    ::fsync(fd);
+  VOID_TEMP_FAILURE_RETRY(::close(fd));
+
+  return err;
+}
+
+void Monitor::extract_save_mon_key(KeyRing& keyring)
+{
+  EntityName mon_name;
+  mon_name.set_type(CEPH_ENTITY_TYPE_MON);
+  EntityAuth mon_key;
+  if (keyring.get_auth(mon_name, mon_key)) {
+    dout(10) << "extract_save_mon_key moving mon. key to separate keyring" << dendl;
+    KeyRing pkey;
+    pkey.add(mon_name, mon_key);
+    bufferlist bl;
+    pkey.encode_plaintext(bl);
+    write_default_keyring(bl);
+    keyring.remove(mon_name);
+  }
+}
+
+// AuthClient methods -- for mon <-> mon communication
+int Monitor::get_auth_request(
+  Connection *con,
+  AuthConnectionMeta *auth_meta,
+  uint32_t *method,
+  vector<uint32_t> *preferred_modes,
+  bufferlist *out)
+{
+  std::scoped_lock l(auth_lock);
+  if (con->get_peer_type() != CEPH_ENTITY_TYPE_MON &&
+      con->get_peer_type() != CEPH_ENTITY_TYPE_MGR) {
+    return -EACCES;
+  }
+  AuthAuthorizer *auth;
+  if (!get_authorizer(con->get_peer_type(), &auth)) {
+    return -EACCES;
+  }
+  auth_meta->authorizer.reset(auth);
+  auth_registry.get_supported_modes(con->get_peer_type(),
+				    auth->protocol,
+				    preferred_modes);
+  *method = auth->protocol;
+  *out = auth->bl;
+  return 0;
+}
+
+int Monitor::handle_auth_reply_more(
+  Connection *con,
+  AuthConnectionMeta *auth_meta,
+  const bufferlist& bl,
+  bufferlist *reply)
+{
+  std::scoped_lock l(auth_lock);
+  if (!auth_meta->authorizer) {
+    derr << __func__ << " no authorizer?" << dendl;
+    return -EACCES;
+  }
+  auth_meta->authorizer->add_challenge(cct, bl);
+  *reply = auth_meta->authorizer->bl;
+  return 0;
+}
+
+int Monitor::handle_auth_done(
+  Connection *con,
+  AuthConnectionMeta *auth_meta,
+  uint64_t global_id,
+  uint32_t con_mode,
+  const bufferlist& bl,
+  CryptoKey *session_key,
+  std::string *connection_secret)
+{
+  std::scoped_lock l(auth_lock);
+  // verify authorizer reply
+  auto p = bl.begin();
+  if (!auth_meta->authorizer->verify_reply(p, connection_secret)) {
+    dout(0) << __func__ << " failed verifying authorizer reply" << dendl;
+    return -EACCES;
+  }
+  auth_meta->session_key = auth_meta->authorizer->session_key;
+  return 0;
+}
+
+int Monitor::handle_auth_bad_method(
+  Connection *con,
+  AuthConnectionMeta *auth_meta,
+  uint32_t old_auth_method,
+  int result,
+  const std::vector<uint32_t>& allowed_methods,
+  const std::vector<uint32_t>& allowed_modes)
+{
+  derr << __func__ << " hmm, they didn't like " << old_auth_method
+       << " result " << cpp_strerror(result) << dendl;
+  return -EACCES;
+}
+
+bool Monitor::get_authorizer(int service_id, AuthAuthorizer **authorizer)
+{
+  dout(10) << "get_authorizer for " << ceph_entity_type_name(service_id)
+	   << dendl;
+
+  if (is_shutdown())
+    return false;
+
+  // we only connect to other monitors and mgr; every else connects to us.
+  if (service_id != CEPH_ENTITY_TYPE_MON &&
+      service_id != CEPH_ENTITY_TYPE_MGR)
+    return false;
+
+  if (!auth_cluster_required.is_supported_auth(CEPH_AUTH_CEPHX)) {
+    // auth_none
+    dout(20) << __func__ << " building auth_none authorizer" << dendl;
+    AuthNoneClientHandler handler{g_ceph_context};
+    handler.set_global_id(0);
+    *authorizer = handler.build_authorizer(service_id);
+    return true;
+  }
+
+  CephXServiceTicketInfo auth_ticket_info;
+  CephXSessionAuthInfo info;
+  int ret;
+
+  EntityName name;
+  name.set_type(CEPH_ENTITY_TYPE_MON);
+  auth_ticket_info.ticket.name = name;
+  auth_ticket_info.ticket.global_id = 0;
+
+  if (service_id == CEPH_ENTITY_TYPE_MON) {
+    // mon to mon authentication uses the private monitor shared key and not the
+    // rotating key
+    CryptoKey secret;
+    if (!keyring.get_secret(name, secret) &&
+	!key_server.get_secret(name, secret)) {
+      dout(0) << " couldn't get secret for mon service from keyring or keyserver"
+	      << dendl;
+      stringstream ss, ds;
+      int err = key_server.list_secrets(ds);
+      if (err < 0)
+	ss << "no installed auth entries!";
+      else
+	ss << "installed auth entries:";
+      dout(0) << ss.str() << "\n" << ds.str() << dendl;
+      return false;
+    }
+
+    ret = key_server.build_session_auth_info(
+      service_id, auth_ticket_info.ticket, secret, (uint64_t)-1, info);
+    if (ret < 0) {
+      dout(0) << __func__ << " failed to build mon session_auth_info "
+	      << cpp_strerror(ret) << dendl;
+      return false;
+    }
+  } else if (service_id == CEPH_ENTITY_TYPE_MGR) {
+    // mgr
+    ret = key_server.build_session_auth_info(
+      service_id, auth_ticket_info.ticket, info);
+    if (ret < 0) {
+      derr << __func__ << " failed to build mgr service session_auth_info "
+	   << cpp_strerror(ret) << dendl;
+      return false;
+    }
+  } else {
+    ceph_abort();  // see check at top of fn
+  }
+
+  CephXTicketBlob blob;
+  if (!cephx_build_service_ticket_blob(cct, info, blob)) {
+    dout(0) << "get_authorizer failed to build service ticket" << dendl;
+    return false;
+  }
+  bufferlist ticket_data;
+  encode(blob, ticket_data);
+
+  auto iter = ticket_data.cbegin();
+  CephXTicketHandler handler(g_ceph_context, service_id);
+  decode(handler.ticket, iter);
+
+  handler.session_key = info.session_key;
+
+  *authorizer = handler.build_authorizer(0);
+  
+  return true;
+}
+
+int Monitor::handle_auth_request(
+  Connection *con,
+  AuthConnectionMeta *auth_meta,
+  bool more,
+  uint32_t auth_method,
+  const bufferlist &payload,
+  bufferlist *reply)
+{
+  std::scoped_lock l(auth_lock);
+
+  // NOTE: be careful, the Connection hasn't fully negotiated yet, so
+  // e.g., peer_features, peer_addrs, and others are still unknown.
+
+  dout(10) << __func__ << " con " << con << (more ? " (more)":" (start)")
+	   << " method " << auth_method
+	   << " payload " << payload.length()
+	   << dendl;
+  if (!payload.length()) {
+    if (!con->is_msgr2() &&
+	con->get_peer_type() != CEPH_ENTITY_TYPE_MON) {
+      // for v1 connections, we tolerate no authorizer (from
+      // non-monitors), because authentication happens via MAuth
+      // messages.
+      return 1;
+    }
+    return -EACCES;
+  }
+  if (!more) {
+    auth_meta->auth_mode = payload[0];
+  }
+
+  if (auth_meta->auth_mode >= AUTH_MODE_AUTHORIZER &&
+      auth_meta->auth_mode <= AUTH_MODE_AUTHORIZER_MAX) {
+    AuthAuthorizeHandler *ah = get_auth_authorize_handler(con->get_peer_type(),
+							  auth_method);
+    if (!ah) {
+      lderr(cct) << __func__ << " no AuthAuthorizeHandler found for auth method "
+		 << auth_method << dendl;
+      return -EOPNOTSUPP;
+    }
+    bool was_challenge = (bool)auth_meta->authorizer_challenge;
+    bool isvalid = ah->verify_authorizer(
+      cct,
+      keyring,
+      payload,
+      auth_meta->get_connection_secret_length(),
+      reply,
+      &con->peer_name,
+      &con->peer_global_id,
+      &con->peer_caps_info,
+      &auth_meta->session_key,
+      &auth_meta->connection_secret,
+      &auth_meta->authorizer_challenge);
+    if (isvalid) {
+      ms_handle_authentication(con);
+      return 1;
+    }
+    if (!more && !was_challenge && auth_meta->authorizer_challenge) {
+      return 0;
+    }
+    dout(10) << __func__ << " bad authorizer on " << con << dendl;
+    return -EACCES;
+  } else if (auth_meta->auth_mode < AUTH_MODE_MON ||
+	     auth_meta->auth_mode > AUTH_MODE_MON_MAX) {
+    derr << __func__ << " unrecognized auth mode " << auth_meta->auth_mode
+	 << dendl;
+    return -EACCES;
+  }
+
+  // wait until we've formed an initial quorum on mkfs so that we have
+  // the initial keys (e.g., client.admin).
+  if (authmon()->get_last_committed() == 0) {
+    dout(10) << __func__ << " haven't formed initial quorum, EBUSY" << dendl;
+    return -EBUSY;
+  }
+
+  RefCountedPtr priv;
+  MonSession *s;
+  int32_t r = 0;
+  auto p = payload.begin();
+  if (!more) {
+    if (con->get_priv()) {
+      return -EACCES; // wtf
+    }
+
+    // handler?
+    unique_ptr<AuthServiceHandler> auth_handler{get_auth_service_handler(
+      auth_method, g_ceph_context, &key_server)};
+    if (!auth_handler) {
+      dout(1) << __func__ << " auth_method " << auth_method << " not supported"
+	      << dendl;
+      return -EOPNOTSUPP;
+    }
+
+    uint8_t mode;
+    EntityName entity_name;
+
+    try {
+      decode(mode, p);
+      if (mode < AUTH_MODE_MON ||
+	  mode > AUTH_MODE_MON_MAX) {
+	dout(1) << __func__ << " invalid mode " << (int)mode << dendl;
+	return -EACCES;
+      }
+      assert(mode >= AUTH_MODE_MON && mode <= AUTH_MODE_MON_MAX);
+      decode(entity_name, p);
+      decode(con->peer_global_id, p);
+    } catch (ceph::buffer::error& e) {
+      dout(1) << __func__ << " failed to decode, " << e.what() << dendl;
+      return -EACCES;
+    }
+
+    // supported method?
+    if (entity_name.get_type() == CEPH_ENTITY_TYPE_MON ||
+	entity_name.get_type() == CEPH_ENTITY_TYPE_OSD ||
+	entity_name.get_type() == CEPH_ENTITY_TYPE_MDS ||
+	entity_name.get_type() == CEPH_ENTITY_TYPE_MGR) {
+      if (!auth_cluster_required.is_supported_auth(auth_method)) {
+	dout(10) << __func__ << " entity " << entity_name << " method "
+		 << auth_method << " not among supported "
+		 << auth_cluster_required.get_supported_set() << dendl;
+	return -EOPNOTSUPP;
+      }
+    } else {
+      if (!auth_service_required.is_supported_auth(auth_method)) {
+	dout(10) << __func__ << " entity " << entity_name << " method "
+		 << auth_method << " not among supported "
+		 << auth_cluster_required.get_supported_set() << dendl;
+	return -EOPNOTSUPP;
+      }
+    }
+
+    // for msgr1 we would do some weirdness here to ensure signatures
+    // are supported by the client if we require it.  for msgr2 that
+    // is not necessary.
+
+    bool is_new_global_id = false;
+    if (!con->peer_global_id) {
+      con->peer_global_id = authmon()->_assign_global_id();
+      if (!con->peer_global_id) {
+	dout(1) << __func__ << " failed to assign global_id" << dendl;
+	return -EBUSY;
+      }
+      is_new_global_id = true;
+    }
+
+    // set up partial session
+    s = new MonSession(con);
+    s->auth_handler = auth_handler.release();
+    con->set_priv(RefCountedPtr{s, false});
+
+    r = s->auth_handler->start_session(
+      entity_name,
+      con->peer_global_id,
+      is_new_global_id,
+      reply,
+      &con->peer_caps_info);
+  } else {
+    priv = con->get_priv();
+    if (!priv) {
+      // this can happen if the async ms_handle_reset event races with
+      // the unlocked call into handle_auth_request
+      return -EACCES;
+    }
+    s = static_cast<MonSession*>(priv.get());
+    r = s->auth_handler->handle_request(
+      p,
+      auth_meta->get_connection_secret_length(),
+      reply,
+      &con->peer_caps_info,
+      &auth_meta->session_key,
+      &auth_meta->connection_secret);
+  }
+  if (r > 0 &&
+      !s->authenticated) {
+    ms_handle_authentication(con);
+  }
+
+  dout(30) << " r " << r << " reply:\n";
+  reply->hexdump(*_dout);
+  *_dout << dendl;
+  return r;
+}
+
+void Monitor::ms_handle_accept(Connection *con)
+{
+  auto priv = con->get_priv();
+  MonSession *s = static_cast<MonSession*>(priv.get());
+  if (!s) {
+    // legacy protocol v1?
+    dout(10) << __func__ << " con " << con << " no session" << dendl;
+    return;
+  }
+
+  if (s->item.is_on_list()) {
+    dout(10) << __func__ << " con " << con << " session " << s
+	     << " already on list" << dendl;
+  } else {
+    std::lock_guard l(session_map_lock);
+    if (state == STATE_SHUTDOWN) {
+      dout(10) << __func__ << " ignoring new con " << con << " (shutdown)" << dendl;
+      con->mark_down();
+      return;
+    }
+    dout(10) << __func__ << " con " << con << " session " << s
+	     << " registering session for "
+	     << con->get_peer_addrs() << dendl;
+    s->_ident(entity_name_t(con->get_peer_type(), con->get_peer_id()),
+	      con->get_peer_addrs());
+    session_map.add_session(s);
+  }
+}
+
+int Monitor::ms_handle_authentication(Connection *con)
+{
+  if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) {
+    // mon <-> mon connections need no Session, and setting one up
+    // creates an awkward ref cycle between Session and Connection.
+    return 1;
+  }
+
+  auto priv = con->get_priv();
+  MonSession *s = static_cast<MonSession*>(priv.get());
+  if (!s) {
+    // must be msgr2, otherwise dispatch would have set up the session.
+    s = session_map.new_session(
+      entity_name_t(con->get_peer_type(), -1),  // we don't know yet
+      con->get_peer_addrs(),
+      con);
+    assert(s);
+    dout(10) << __func__ << " adding session " << s << " to con " << con
+	     << dendl;
+    con->set_priv(s);
+    logger->set(l_mon_num_sessions, session_map.get_size());
+    logger->inc(l_mon_session_add);
+  }
+  dout(10) << __func__ << " session " << s << " con " << con
+	   << " addr " << s->con->get_peer_addr()
+	   << " " << *s << dendl;
+
+  AuthCapsInfo &caps_info = con->get_peer_caps_info();
+  int ret = 0;
+  if (caps_info.allow_all) {
+    s->caps.set_allow_all();
+    s->authenticated = true;
+    ret = 1;
+  } else if (caps_info.caps.length()) {
+    bufferlist::const_iterator p = caps_info.caps.cbegin();
+    string str;
+    try {
+      decode(str, p);
+    } catch (const ceph::buffer::error &err) {
+      derr << __func__ << " corrupt cap data for " << con->get_peer_entity_name()
+	   << " in auth db" << dendl;
+      str.clear();
+      ret = -EACCES;
+    }
+    if (ret >= 0) {
+      if (s->caps.parse(str, NULL)) {
+	s->authenticated = true;
+	ret = 1;
+      } else {
+	derr << __func__ << " unparseable caps '" << str << "' for "
+	     << con->get_peer_entity_name() << dendl;
+	ret = -EACCES;
+      }
+    }
+  }
+
+  return ret;
+}
+
+void Monitor::set_mon_crush_location(const string& loc)
+{
+  if (loc.empty()) {
+    return;
+  }
+  vector<string> loc_vec;
+  loc_vec.push_back(loc);
+  CrushWrapper::parse_loc_map(loc_vec, &crush_loc);
+  need_set_crush_loc = true;
+}
+
+void Monitor::notify_new_monmap(bool can_change_external_state, bool remove_rank_elector)
+{
+  if (need_set_crush_loc) {
+    auto my_info_i = monmap->mon_info.find(name);
+    if (my_info_i != monmap->mon_info.end() &&
+	my_info_i->second.crush_loc == crush_loc) {
+      need_set_crush_loc = false;
+    }
+  }
+  elector.notify_strategy_maybe_changed(monmap->strategy);
+  if (remove_rank_elector){
+    dout(10) << __func__ << " we have " << monmap->ranks.size()<< " ranks" << dendl;
+    dout(10) << __func__ << " we have " << monmap->removed_ranks.size() << " removed ranks" << dendl;
+    for (auto i = monmap->removed_ranks.rbegin();
+        i != monmap->removed_ranks.rend(); ++i) {
+      int remove_rank = *i;
+      dout(10) << __func__ << " removing rank " << remove_rank << dendl;
+      if (rank == remove_rank) {
+        dout(5) << "We are removing our own rank, probably we"
+          << " are removed from monmap before we shutdown ... dropping." << dendl;
+        continue;
+      }
+      int new_rank = monmap->get_rank(messenger->get_myaddrs());
+      if (new_rank == -1) {
+        dout(5) << "We no longer exists in the monmap! ... dropping." << dendl;
+        continue;
+      }
+      elector.notify_rank_removed(remove_rank, new_rank);
+    }
+  }
+
+  if (monmap->stretch_mode_enabled) {
+    try_engage_stretch_mode();
+  }
+
+  if (is_stretch_mode()) {
+    if (!monmap->stretch_marked_down_mons.empty()) {
+      set_degraded_stretch_mode();
+    }
+  }
+  set_elector_disallowed_leaders(can_change_external_state);
+}
+
+void Monitor::set_elector_disallowed_leaders(bool allow_election)
+{
+  set<int> dl;
+  for (auto name : monmap->disallowed_leaders) {
+    dl.insert(monmap->get_rank(name));
+  }
+  if (is_stretch_mode()) {
+    for (auto name : monmap->stretch_marked_down_mons) {
+      dl.insert(monmap->get_rank(name));
+    }
+    dl.insert(monmap->get_rank(monmap->tiebreaker_mon));
+  }
+
+  bool disallowed_changed = elector.set_disallowed_leaders(dl);
+  if (disallowed_changed && allow_election) {
+    elector.call_election();
+  }
+}
+
+struct CMonEnableStretchMode : public Context {
+  Monitor *m;
+  CMonEnableStretchMode(Monitor *mon) : m(mon) {}
+  void finish(int r) {
+    m->try_engage_stretch_mode();
+  }
+};
+void Monitor::try_engage_stretch_mode()
+{
+  dout(20) << __func__ << dendl;
+  if (stretch_mode_engaged) return;
+  if (!osdmon()->is_readable()) {
+    osdmon()->wait_for_readable_ctx(new CMonEnableStretchMode(this));
+  }
+  if (osdmon()->osdmap.stretch_mode_enabled &&
+      monmap->stretch_mode_enabled) {
+    dout(10) << "Engaging stretch mode!" << dendl;
+    stretch_mode_engaged = true;
+    int32_t stretch_divider_id = osdmon()->osdmap.stretch_mode_bucket;
+    stretch_bucket_divider = osdmon()->osdmap.
+      crush->get_type_name(stretch_divider_id);
+    disconnect_disallowed_stretch_sessions();
+  }
+}
+
+void Monitor::do_stretch_mode_election_work()
+{
+  dout(20) << __func__ << dendl;
+  if (!is_stretch_mode() ||
+      !is_leader()) return;
+  dout(20) << "checking for degraded stretch mode" << dendl;
+  map<string, set<string>> old_dead_buckets;
+  old_dead_buckets.swap(dead_mon_buckets);
+  up_mon_buckets.clear();
+  // identify if we've lost a CRUSH bucket, request OSDMonitor check for death
+  map<string,set<string>> down_mon_buckets;
+  for (unsigned i = 0; i < monmap->size(); ++i) {
+    const auto &mi = monmap->mon_info[monmap->get_name(i)];
+    auto ci = mi.crush_loc.find(stretch_bucket_divider);
+    ceph_assert(ci != mi.crush_loc.end());
+    if (quorum.count(i)) {
+      up_mon_buckets.insert(ci->second);
+    } else {
+      down_mon_buckets[ci->second].insert(mi.name);
+    }
+  }
+  dout(20) << "prior dead_mon_buckets: " << old_dead_buckets
+	   << "; down_mon_buckets: " << down_mon_buckets
+	   << "; up_mon_buckets: " << up_mon_buckets << dendl;
+  for (auto di : down_mon_buckets) {
+    if (!up_mon_buckets.count(di.first)) {
+      dead_mon_buckets[di.first] = di.second;
+    }
+  }
+  dout(20) << "new dead_mon_buckets " << dead_mon_buckets << dendl;
+
+  if (dead_mon_buckets != old_dead_buckets &&
+      dead_mon_buckets.size() >= old_dead_buckets.size()) {
+    maybe_go_degraded_stretch_mode();
+  }
+}
+
+struct CMonGoDegraded : public Context {
+  Monitor *m;
+  CMonGoDegraded(Monitor *mon) : m(mon) {}
+  void finish(int r) {
+    m->maybe_go_degraded_stretch_mode();
+  }
+};
+
+struct CMonGoRecovery : public Context {
+  Monitor *m;
+  CMonGoRecovery(Monitor *mon) : m(mon) {}
+  void finish(int r) {
+    m->go_recovery_stretch_mode();
+  }
+};
+void Monitor::go_recovery_stretch_mode()
+{
+  dout(20) << __func__ << dendl;
+  if (!is_leader()) return;
+  if (!is_degraded_stretch_mode()) return;
+  if (is_recovering_stretch_mode()) return;
+
+  if (dead_mon_buckets.size()) {
+    ceph_assert( 0 == "how did we try and do stretch recovery while we have dead monitor buckets?");
+    // we can't recover if we are missing monitors in a zone!
+    return;
+  }
+  
+  if (!osdmon()->is_readable()) {
+    osdmon()->wait_for_readable_ctx(new CMonGoRecovery(this));
+    return;
+  }
+
+  if (!osdmon()->is_writeable()) {
+    osdmon()->wait_for_writeable_ctx(new CMonGoRecovery(this));
+  }
+  osdmon()->trigger_recovery_stretch_mode();
+}
+
+void Monitor::set_recovery_stretch_mode()
+{
+  degraded_stretch_mode = true;
+  recovering_stretch_mode = true;
+  osdmon()->set_recovery_stretch_mode();
+}
+
+void Monitor::maybe_go_degraded_stretch_mode()
+{
+  dout(20) << __func__ << dendl;
+  if (is_degraded_stretch_mode()) return;
+  if (!is_leader()) return;
+  if (dead_mon_buckets.empty()) return;
+  if (!osdmon()->is_readable()) {
+    osdmon()->wait_for_readable_ctx(new CMonGoDegraded(this));
+    return;
+  }
+  ceph_assert(monmap->contains(monmap->tiebreaker_mon));
+  // filter out the tiebreaker zone and check if remaining sites are down by OSDs too
+  const auto &mi = monmap->mon_info[monmap->tiebreaker_mon];
+  auto ci = mi.crush_loc.find(stretch_bucket_divider);
+  map<string, set<string>> filtered_dead_buckets = dead_mon_buckets;
+  filtered_dead_buckets.erase(ci->second);
+
+  set<int> matched_down_buckets;
+  set<string> matched_down_mons;
+  bool dead = osdmon()->check_for_dead_crush_zones(filtered_dead_buckets,
+						   &matched_down_buckets,
+						   &matched_down_mons);
+  if (dead) {
+    if (!osdmon()->is_writeable()) {
+      osdmon()->wait_for_writeable_ctx(new CMonGoDegraded(this));
+    }
+    if (!monmon()->is_writeable()) {
+      monmon()->wait_for_writeable_ctx(new CMonGoDegraded(this));
+    }
+    trigger_degraded_stretch_mode(matched_down_mons, matched_down_buckets);
+  }
+}
+
+void Monitor::trigger_degraded_stretch_mode(const set<string>& dead_mons,
+					    const set<int>& dead_buckets)
+{
+  dout(20) << __func__ << dendl;
+  ceph_assert(osdmon()->is_writeable());
+  ceph_assert(monmon()->is_writeable());
+
+  // figure out which OSD zone(s) remains alive by removing
+  // tiebreaker mon from up_mon_buckets
+  set<string> live_zones = up_mon_buckets;
+  ceph_assert(monmap->contains(monmap->tiebreaker_mon));
+  const auto &mi = monmap->mon_info[monmap->tiebreaker_mon];
+  auto ci = mi.crush_loc.find(stretch_bucket_divider);
+  live_zones.erase(ci->second);
+  ceph_assert(live_zones.size() == 1); // only support 2 zones right now
+  
+  osdmon()->trigger_degraded_stretch_mode(dead_buckets, live_zones);
+  monmon()->trigger_degraded_stretch_mode(dead_mons);
+  set_degraded_stretch_mode();
+}
+
+void Monitor::set_degraded_stretch_mode()
+{
+  degraded_stretch_mode = true;
+  recovering_stretch_mode = false;
+  osdmon()->set_degraded_stretch_mode();
+}
+
+struct CMonGoHealthy : public Context {
+  Monitor *m;
+  CMonGoHealthy(Monitor *mon) : m(mon) {}
+  void finish(int r) {
+    m->trigger_healthy_stretch_mode();
+  }
+};
+
+
+void Monitor::trigger_healthy_stretch_mode()
+{
+  dout(20) << __func__ << dendl;
+  if (!is_degraded_stretch_mode()) return;
+  if (!is_leader()) return;
+  if (!osdmon()->is_writeable()) {
+    osdmon()->wait_for_writeable_ctx(new CMonGoHealthy(this));
+  }
+  if (!monmon()->is_writeable()) {
+    monmon()->wait_for_writeable_ctx(new CMonGoHealthy(this));
+  }
+
+  ceph_assert(osdmon()->osdmap.recovering_stretch_mode);
+  osdmon()->trigger_healthy_stretch_mode();
+  monmon()->trigger_healthy_stretch_mode();
+}
+
+void Monitor::set_healthy_stretch_mode()
+{
+  degraded_stretch_mode = false;
+  recovering_stretch_mode = false;
+  osdmon()->set_healthy_stretch_mode();
+}
+
+bool Monitor::session_stretch_allowed(MonSession *s, MonOpRequestRef& op)
+{
+  if (!is_stretch_mode()) return true;
+  if (s->proxy_con) return true;
+  if (s->validated_stretch_connection) return true;
+  if (!s->con) return true;
+  if (s->con->peer_is_osd()) {
+    dout(20) << __func__ << "checking OSD session" << s << dendl;
+    // okay, check the crush location
+    int barrier_id;
+    int retval = osdmon()->osdmap.crush->get_validated_type_id(stretch_bucket_divider,
+							       &barrier_id);
+    ceph_assert(retval >= 0);
+    int osd_bucket_id = osdmon()->osdmap.crush->get_parent_of_type(s->con->peer_id,
+								   barrier_id);
+    const auto &mi = monmap->mon_info.find(name);
+    ceph_assert(mi != monmap->mon_info.end());
+    auto ci = mi->second.crush_loc.find(stretch_bucket_divider);
+    ceph_assert(ci != mi->second.crush_loc.end());
+    int mon_bucket_id = osdmon()->osdmap.crush->get_item_id(ci->second);
+    
+    if (osd_bucket_id != mon_bucket_id) {
+      dout(5) << "discarding session " << *s
+	      << " and sending OSD to matched zone" << dendl;
+      s->con->mark_down();
+      std::lock_guard l(session_map_lock);
+      remove_session(s);
+      if (op) {
+	op->mark_zap();
+      }
+      return false;
+    }
+  }
+
+  s->validated_stretch_connection = true;
+  return true;
+}
+
+void Monitor::disconnect_disallowed_stretch_sessions()
+{
+  dout(20) << __func__ << dendl;
+  MonOpRequestRef blank;
+  auto i = session_map.sessions.begin();
+  while (i != session_map.sessions.end()) {
+    auto j = i;
+    ++i;
+    session_stretch_allowed(*j, blank);
+  }
+}
diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
new file mode 100644
index 000000000..1093649bb
--- /dev/null
+++ b/src/mon/Monitor.h
@@ -0,0 +1,1148 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+/*
+ * This is the top level monitor. It runs on each machine in the Monitor
+ * Cluster. The election of a leader for the paxos algorithm only happens
+ * once per machine via the elector. There is a separate paxos instance (state)
+ * kept for each of the system components: Object Store Device (OSD) Monitor,
+ * Placement Group (PG) Monitor, Metadata Server (MDS) Monitor, and Client Monitor.
+ */
+
+#ifndef CEPH_MONITOR_H
+#define CEPH_MONITOR_H
+
+#include <errno.h>
+#include <cmath>
+#include <string>
+#include <array>
+
+#include "include/types.h"
+#include "include/health.h"
+#include "msg/Messenger.h"
+
+#include "common/Timer.h"
+
+#include "health_check.h"
+#include "MonMap.h"
+#include "Elector.h"
+#include "Paxos.h"
+#include "Session.h"
+#include "MonCommand.h"
+
+
+#include "common/config_obs.h"
+#include "common/LogClient.h"
+#include "auth/AuthClient.h"
+#include "auth/AuthServer.h"
+#include "auth/cephx/CephxKeyServer.h"
+#include "auth/AuthMethodList.h"
+#include "auth/KeyRing.h"
+#include "include/common_fwd.h"
+#include "messages/MMonCommand.h"
+#include "mon/MonitorDBStore.h"
+#include "mgr/MgrClient.h"
+
+#include "mon/MonOpRequest.h"
+#include "common/WorkQueue.h"
+
+using namespace TOPNSPC::common;
+
+#define CEPH_MON_PROTOCOL     13 /* cluster internal */
+
+
+enum {
+  l_cluster_first = 555000,
+  l_cluster_num_mon,
+  l_cluster_num_mon_quorum,
+  l_cluster_num_osd,
+  l_cluster_num_osd_up,
+  l_cluster_num_osd_in,
+  l_cluster_osd_epoch,
+  l_cluster_osd_bytes,
+  l_cluster_osd_bytes_used,
+  l_cluster_osd_bytes_avail,
+  l_cluster_num_pool,
+  l_cluster_num_pg,
+  l_cluster_num_pg_active_clean,
+  l_cluster_num_pg_active,
+  l_cluster_num_pg_peering,
+  l_cluster_num_object,
+  l_cluster_num_object_degraded,
+  l_cluster_num_object_misplaced,
+  l_cluster_num_object_unfound,
+  l_cluster_num_bytes,
+  l_cluster_last,
+};
+
+enum {
+  l_mon_first = 456000,
+  l_mon_num_sessions,
+  l_mon_session_add,
+  l_mon_session_rm,
+  l_mon_session_trim,
+  l_mon_num_elections,
+  l_mon_election_call,
+  l_mon_election_win,
+  l_mon_election_lose,
+  l_mon_last,
+};
+
+class PaxosService;
+
+class AdminSocketHook;
+
+#define COMPAT_SET_LOC "feature_set"
+
+class Monitor : public Dispatcher,
+		public AuthClient,
+		public AuthServer,
+                public md_config_obs_t {
+public:
+  int orig_argc = 0;
+  const char **orig_argv = nullptr;
+
+  // me
+  std::string name;
+  int rank;
+  Messenger *messenger;
+  ConnectionRef con_self;
+  ceph::mutex lock = ceph::make_mutex("Monitor::lock");
+  SafeTimer timer;
+  Finisher finisher;
+  ThreadPool cpu_tp;  ///< threadpool for CPU intensive work
+
+  ceph::mutex auth_lock = ceph::make_mutex("Monitor::auth_lock");
+
+  /// true if we have ever joined a quorum.  if false, we are either a
+  /// new cluster, a newly joining monitor, or a just-upgraded
+  /// monitor.
+  bool has_ever_joined;
+
+  PerfCounters *logger, *cluster_logger;
+  bool cluster_logger_registered;
+
+  void register_cluster_logger();
+  void unregister_cluster_logger();
+
+  MonMap *monmap;
+  uuid_d fingerprint;
+
+  std::set<entity_addrvec_t> extra_probe_peers;
+
+  LogClient log_client;
+  LogChannelRef clog;
+  LogChannelRef audit_clog;
+  KeyRing keyring;
+  KeyServer key_server;
+
+  AuthMethodList auth_cluster_required;
+  AuthMethodList auth_service_required;
+
+  CompatSet features;
+
+  std::vector<MonCommand> leader_mon_commands; // quorum leader's commands
+  std::vector<MonCommand> local_mon_commands;  // commands i support
+  ceph::buffer::list local_mon_commands_bl;       // encoded version of above
+
+  std::vector<MonCommand> prenautilus_local_mon_commands;
+  ceph::buffer::list prenautilus_local_mon_commands_bl;
+
+  Messenger *mgr_messenger;
+  MgrClient mgr_client;
+  uint64_t mgr_proxy_bytes = 0;  // in-flight proxied mgr command message bytes
+  std::string gss_ktfile_client{};
+
+private:
+  void new_tick();
+
+  // -- local storage --
+public:
+  MonitorDBStore *store;
+  static const std::string MONITOR_NAME;
+  static const std::string MONITOR_STORE_PREFIX;
+
+  // -- monitor state --
+private:
+  enum {
+    STATE_INIT = 1,
+    STATE_PROBING,
+    STATE_SYNCHRONIZING,
+    STATE_ELECTING,
+    STATE_LEADER,
+    STATE_PEON,
+    STATE_SHUTDOWN
+  };
+  int state = STATE_INIT;
+
+public:
+  static const char *get_state_name(int s) {
+    switch (s) {
+    case STATE_PROBING: return "probing";
+    case STATE_SYNCHRONIZING: return "synchronizing";
+    case STATE_ELECTING: return "electing";
+    case STATE_LEADER: return "leader";
+    case STATE_PEON: return "peon";
+    case STATE_SHUTDOWN: return "shutdown";
+    default: return "???";
+    }
+  }
+  const char *get_state_name() const {
+    return get_state_name(state);
+  }
+
+  bool is_init() const { return state == STATE_INIT; }
+  bool is_shutdown() const { return state == STATE_SHUTDOWN; }
+  bool is_probing() const { return state == STATE_PROBING; }
+  bool is_synchronizing() const { return state == STATE_SYNCHRONIZING; }
+  bool is_electing() const { return state == STATE_ELECTING; }
+  bool is_leader() const { return state == STATE_LEADER; }
+  bool is_peon() const { return state == STATE_PEON; }
+
+  const utime_t &get_leader_since() const;
+
+  void prepare_new_fingerprint(MonitorDBStore::TransactionRef t);
+
+  std::vector<DaemonHealthMetric> get_health_metrics();
+
+  int quorum_age() const {
+    auto age = std::chrono::duration_cast<std::chrono::seconds>(
+      ceph::mono_clock::now() - quorum_since);
+    return age.count();
+  }
+
+  bool is_mon_down() const {
+    int max = monmap->size();
+    int actual = get_quorum().size();
+    auto now = ceph::real_clock::now();
+    return actual < max && now > monmap->created.to_real_time();
+  }
+
+  // -- elector --
+private:
+  std::unique_ptr<Paxos> paxos;
+  Elector elector;
+  friend class Elector;
+
+  /// features we require of peers (based on on-disk compatset)
+  uint64_t required_features;
+  
+  int leader;            // current leader (to best of knowledge)
+  std::set<int> quorum;       // current active set of monitors (if !starting)
+  ceph::mono_clock::time_point quorum_since;  // when quorum formed
+  utime_t leader_since;  // when this monitor became the leader, if it is the leader
+  utime_t exited_quorum; // time detected as not in quorum; 0 if in
+
+  // map of counts of connected clients, by type and features, for
+  // each quorum mon
+  std::map<int,FeatureMap> quorum_feature_map;
+
+  /**
+   * Intersection of quorum member's connection feature bits.
+   */
+  uint64_t quorum_con_features;
+  /**
+   * Intersection of quorum members mon-specific feature bits
+   */
+  mon_feature_t quorum_mon_features;
+
+  ceph_release_t quorum_min_mon_release{ceph_release_t::unknown};
+
+  std::set<std::string> outside_quorum;
+
+  bool stretch_mode_engaged{false};
+  bool degraded_stretch_mode{false};
+  bool recovering_stretch_mode{false};
+  string stretch_bucket_divider;
+  map<string, set<string>> dead_mon_buckets; // bucket->mon ranks, locations with no live mons
+  set<string> up_mon_buckets; // locations with a live mon
+  void do_stretch_mode_election_work();
+
+  bool session_stretch_allowed(MonSession *s, MonOpRequestRef& op);
+  void disconnect_disallowed_stretch_sessions();
+  void set_elector_disallowed_leaders(bool allow_election);
+
+  map <string,string> crush_loc;
+  bool need_set_crush_loc{false};
+public:
+  bool is_stretch_mode() { return stretch_mode_engaged; }
+  bool is_degraded_stretch_mode() { return degraded_stretch_mode; }
+  bool is_recovering_stretch_mode() { return recovering_stretch_mode; }
+
+  /**
+   * This set of functions maintains the in-memory stretch state
+   * and sets up transitions of the map states by calling in to
+   * MonmapMonitor and OSDMonitor.
+   *
+   * The [maybe_]go_* functions are called on the leader to
+   * decide if transitions should happen; the trigger_* functions
+   * set up the map transitions; and the set_* functions actually
+   * change the memory state -- but these are only called
+   * via OSDMonitor::update_from_paxos, to guarantee consistent
+   * updates across the entire cluster.
+   */
+  void try_engage_stretch_mode();
+  void maybe_go_degraded_stretch_mode();
+  void trigger_degraded_stretch_mode(const set<string>& dead_mons,
+				     const set<int>& dead_buckets);
+  void set_degraded_stretch_mode();
+  void go_recovery_stretch_mode();
+  void set_recovery_stretch_mode();
+  void trigger_healthy_stretch_mode();
+  void set_healthy_stretch_mode();
+  void enable_stretch_mode();
+  void set_mon_crush_location(const string& loc);
+
+  
+private:
+
+  /**
+   * @defgroup Monitor_h_scrub
+   * @{
+   */
+  version_t scrub_version;            ///< paxos version we are scrubbing
+  std::map<int,ScrubResult> scrub_result;  ///< results so far
+
+  /**
+   * trigger a cross-mon scrub
+   *
+   * Verify all mons are storing identical content
+   */
+  int scrub_start();
+  int scrub();
+  void handle_scrub(MonOpRequestRef op);
+  bool _scrub(ScrubResult *r,
+              std::pair<std::string,std::string> *start,
+              int *num_keys);
+  void scrub_check_results();
+  void scrub_timeout();
+  void scrub_finish();
+  void scrub_reset();
+  void scrub_update_interval(ceph::timespan interval);
+
+  Context *scrub_event;       ///< periodic event to trigger scrub (leader)
+  Context *scrub_timeout_event;  ///< scrub round timeout (leader)
+  void scrub_event_start();
+  void scrub_event_cancel();
+  void scrub_reset_timeout();
+  void scrub_cancel_timeout();
+
+  struct ScrubState {
+    std::pair<std::string,std::string> last_key; ///< last scrubbed key
+    bool finished;
+
+    ScrubState() : finished(false) { }
+    virtual ~ScrubState() { }
+  };
+  std::shared_ptr<ScrubState> scrub_state; ///< keeps track of current scrub
+
+  /**
+   * @defgroup Monitor_h_sync Synchronization
+   * @{
+   */
+  /**
+   * @} // provider state
+   */
+  struct SyncProvider {
+    entity_addrvec_t addrs;
+    uint64_t cookie;       ///< unique cookie for this sync attempt
+    utime_t timeout;       ///< when we give up and expire this attempt
+    version_t last_committed; ///< last paxos version on peer
+    std::pair<std::string,std::string> last_key; ///< last key sent to (or on) peer
+    bool full;             ///< full scan?
+    MonitorDBStore::Synchronizer synchronizer;   ///< iterator
+
+    SyncProvider() : cookie(0), last_committed(0), full(false) {}
+
+    void reset_timeout(CephContext *cct, int grace) {
+      timeout = ceph_clock_now();
+      timeout += grace;
+    }
+  };
+
+  std::map<std::uint64_t, SyncProvider> sync_providers;  ///< cookie -> SyncProvider for those syncing from us
+  uint64_t sync_provider_count;   ///< counter for issued cookies to keep them unique
+
+  /**
+   * @} // requester state
+   */
+  entity_addrvec_t sync_provider;  ///< who we are syncing from
+  uint64_t sync_cookie;          ///< 0 if we are starting, non-zero otherwise
+  bool sync_full;                ///< true if we are a full sync, false for recent catch-up
+  version_t sync_start_version;  ///< last_committed at sync start
+  Context *sync_timeout_event;   ///< timeout event
+
+  /**
+   * floor for sync source
+   *
+   * When we sync we forget about our old last_committed value which
+   * can be dangerous.  For example, if we have a cluster of:
+   *
+   *   mon.a: lc 100
+   *   mon.b: lc 80
+   *   mon.c: lc 100 (us)
+   *
+   * If something forces us to sync (say, corruption, or manual
+   * intervention, or bug), we forget last_committed, and might abort.
+   * If mon.a happens to be down when we come back, we will see:
+   *
+   *   mon.b: lc 80
+   *   mon.c: lc 0 (us)
+   *
+   * and sync from mon.b, at which point a+b will both have lc 80 and
+   * come online with a majority holding out of date commits.
+   *
+   * Avoid this by preserving our old last_committed value prior to
+   * sync and never going backwards.
+   */
+  version_t sync_last_committed_floor;
+
+  /**
+   * Obtain the synchronization target prefixes in set form.
+   *
+   * We consider a target prefix all those that are relevant when
+   * synchronizing two stores. That is, all those that hold paxos service's
+   * versions, as well as paxos versions, or any control keys such as the
+   * first or last committed version.
+   *
+   * Given the current design, this function should return the name of all and
+   * any available paxos service, plus the paxos name.
+   *
+   * @returns a set of strings referring to the prefixes being synchronized
+   */
+  std::set<std::string> get_sync_targets_names();
+
+  /**
+   * Reset the monitor's sync-related data structures for syncing *from* a peer
+   */
+  void sync_reset_requester();
+
+  /**
+   * Reset sync state related to allowing others to sync from us
+   */
+  void sync_reset_provider();
+
+  /**
+   * Caled when a sync attempt times out (requester-side)
+   */
+  void sync_timeout();
+
+  /**
+   * Get the latest monmap for backup purposes during sync
+   */
+  void sync_obtain_latest_monmap(ceph::buffer::list &bl);
+
+  /**
+   * Start sync process
+   *
+   * Start pulling committed state from another monitor.
+   *
+   * @param entity where to pull committed state from
+   * @param full whether to do a full sync or just catch up on recent paxos
+   */
+  void sync_start(entity_addrvec_t &addrs, bool full);
+
+public:
+  /**
+   * force a sync on next mon restart
+   */
+  void sync_force(ceph::Formatter *f);
+
+private:
+  /**
+   * store critical state for safekeeping during sync
+   *
+   * We store a few things on the side that we don't want to get clobbered by sync.  This
+   * includes the latest monmap and a lower bound on last_committed.
+   */
+  void sync_stash_critical_state(MonitorDBStore::TransactionRef tx);
+
+  /**
+   * reset the sync timeout
+   *
+   * This is used on the client to restart if things aren't progressing
+   */
+  void sync_reset_timeout();
+
+  /**
+   * trim stale sync provider state
+   *
+   * If someone is syncing from us and hasn't talked to us recently, expire their state.
+   */
+  void sync_trim_providers();
+
+  /**
+   * Complete a sync
+   *
+   * Finish up a sync after we've gotten all of the chunks.
+   *
+   * @param last_committed final last_committed value from provider
+   */
+  void sync_finish(version_t last_committed);
+
+  /**
+   * request the next chunk from the provider
+   */
+  void sync_get_next_chunk();
+
+  /**
+   * handle sync message
+   *
+   * @param m Sync message with operation type MMonSync::OP_START_CHUNKS
+   */
+  void handle_sync(MonOpRequestRef op);
+
+  void _sync_reply_no_cookie(MonOpRequestRef op);
+
+  void handle_sync_get_cookie(MonOpRequestRef op);
+  void handle_sync_get_chunk(MonOpRequestRef op);
+  void handle_sync_finish(MonOpRequestRef op);
+
+  void handle_sync_cookie(MonOpRequestRef op);
+  void handle_sync_forward(MonOpRequestRef op);
+  void handle_sync_chunk(MonOpRequestRef op);
+  void handle_sync_no_cookie(MonOpRequestRef op);
+
+  /**
+   * @} // Synchronization
+   */
+
+  std::list<Context*> waitfor_quorum;
+  std::list<Context*> maybe_wait_for_quorum;
+
+  /**
+   * @defgroup Monitor_h_TimeCheck Monitor Clock Drift Early Warning System
+   * @{
+   *
+   * We use time checks to keep track of any clock drifting going on in the
+   * cluster. This is accomplished by periodically ping each monitor in the
+   * quorum and register its response time on a map, assessing how much its
+   * clock has drifted. We also take this opportunity to assess the latency
+   * on response.
+   *
+   * This mechanism works as follows:
+   *
+   *  - Leader sends out a 'PING' message to each other monitor in the quorum.
+   *    The message is timestamped with the leader's current time. The leader's
+   *    current time is recorded in a map, associated with each peon's
+   *    instance.
+   *  - The peon replies to the leader with a timestamped 'PONG' message.
+   *  - The leader calculates a delta between the peon's timestamp and its
+   *    current time and stashes it.
+   *  - The leader also calculates the time it took to receive the 'PONG'
+   *    since the 'PING' was sent, and stashes an approximate latency estimate.
+   *  - Once all the quorum members have pong'ed, the leader will share the
+   *    clock skew and latency maps with all the monitors in the quorum.
+   */
+  std::map<int, utime_t> timecheck_waiting;
+  std::map<int, double> timecheck_skews;
+  std::map<int, double> timecheck_latencies;
+  // odd value means we are mid-round; even value means the round has
+  // finished.
+  version_t timecheck_round;
+  unsigned int timecheck_acks;
+  utime_t timecheck_round_start;
+  friend class HealthMonitor;
+  /* When we hit a skew we will start a new round based off of
+   * 'mon_timecheck_skew_interval'. Each new round will be backed off
+   * until we hit 'mon_timecheck_interval' -- which is the typical
+   * interval when not in the presence of a skew.
+   *
+   * This variable tracks the number of rounds with skews since last clean
+   * so that we can report to the user and properly adjust the backoff.
+   */
+  uint64_t timecheck_rounds_since_clean;
+  /**
+   * Time Check event.
+   */
+  Context *timecheck_event;
+
+  void timecheck_start();
+  void timecheck_finish();
+  void timecheck_start_round();
+  void timecheck_finish_round(bool success = true);
+  void timecheck_cancel_round();
+  void timecheck_cleanup();
+  void timecheck_reset_event();
+  void timecheck_check_skews();
+  void timecheck_report();
+  void timecheck();
+  health_status_t timecheck_status(std::ostringstream &ss,
+                                   const double skew_bound,
+                                   const double latency);
+  void handle_timecheck_leader(MonOpRequestRef op);
+  void handle_timecheck_peon(MonOpRequestRef op);
+  void handle_timecheck(MonOpRequestRef op);
+
+  /**
+   * Returns 'true' if this is considered to be a skew; 'false' otherwise.
+   */
+  bool timecheck_has_skew(const double skew_bound, double *abs) const {
+    double abs_skew = std::fabs(skew_bound);
+    if (abs)
+      *abs = abs_skew;
+    return (abs_skew > g_conf()->mon_clock_drift_allowed);
+  }
+
+  /**
+   * @}
+   */
+  /**
+   * Handle ping messages from others.
+   */
+  void handle_ping(MonOpRequestRef op);
+
+  Context *probe_timeout_event = nullptr;  // for probing
+
+  void reset_probe_timeout();
+  void cancel_probe_timeout();
+  void probe_timeout(int r);
+
+  void _apply_compatset_features(CompatSet &new_features);
+
+public:
+  epoch_t get_epoch();
+  int get_leader() const { return leader; }
+  std::string get_leader_name() {
+    return quorum.empty() ? std::string() : monmap->get_name(leader);
+  }
+  const std::set<int>& get_quorum() const { return quorum; }
+  std::list<std::string> get_quorum_names() {
+    std::list<std::string> q;
+    for (auto p = quorum.begin(); p != quorum.end(); ++p)
+      q.push_back(monmap->get_name(*p));
+    return q;
+  }
+  uint64_t get_quorum_con_features() const {
+    return quorum_con_features;
+  }
+  mon_feature_t get_quorum_mon_features() const {
+    return quorum_mon_features;
+  }
+  uint64_t get_required_features() const {
+    return required_features;
+  }
+  mon_feature_t get_required_mon_features() const {
+    return monmap->get_required_features();
+  }
+  void apply_quorum_to_compatset_features();
+  void apply_monmap_to_compatset_features();
+  void calc_quorum_requirements();
+
+  void get_combined_feature_map(FeatureMap *fm);
+
+private:
+  void _reset();   ///< called from bootstrap, start_, or join_election
+  void wait_for_paxos_write();
+  void _finish_svc_election(); ///< called by {win,lose}_election
+  void respawn();
+public:
+  void bootstrap();
+  void join_election();
+  void start_election();
+  void win_standalone_election();
+  // end election (called by Elector)
+  void win_election(epoch_t epoch, const std::set<int>& q,
+		    uint64_t features,
+                    const mon_feature_t& mon_features,
+		    ceph_release_t min_mon_release,
+		    const std::map<int,Metadata>& metadata);
+  void lose_election(epoch_t epoch, std::set<int>& q, int l,
+		     uint64_t features,
+                     const mon_feature_t& mon_features,
+		     ceph_release_t min_mon_release);
+  // end election (called by Elector)
+  void finish_election();
+
+  void update_logger();
+
+  /**
+   * Vector holding the Services serviced by this Monitor.
+   */
+  std::array<std::unique_ptr<PaxosService>, PAXOS_NUM> paxos_service;
+
+  class MDSMonitor *mdsmon() {
+    return (class MDSMonitor *)paxos_service[PAXOS_MDSMAP].get();
+  }
+
+  class MonmapMonitor *monmon() {
+    return (class MonmapMonitor *)paxos_service[PAXOS_MONMAP].get();
+  }
+
+  class OSDMonitor *osdmon() {
+    return (class OSDMonitor *)paxos_service[PAXOS_OSDMAP].get();
+  }
+
+  class AuthMonitor *authmon() {
+    return (class AuthMonitor *)paxos_service[PAXOS_AUTH].get();
+  }
+
+  class LogMonitor *logmon() {
+    return (class LogMonitor*) paxos_service[PAXOS_LOG].get();
+  }
+
+  class MgrMonitor *mgrmon() {
+    return (class MgrMonitor*) paxos_service[PAXOS_MGR].get();
+  }
+
+  class MgrStatMonitor *mgrstatmon() {
+    return (class MgrStatMonitor*) paxos_service[PAXOS_MGRSTAT].get();
+  }
+
+  class HealthMonitor *healthmon() {
+    return (class HealthMonitor*) paxos_service[PAXOS_HEALTH].get();
+  }
+
+  class ConfigMonitor *configmon() {
+    return (class ConfigMonitor*) paxos_service[PAXOS_CONFIG].get();
+  }
+
+  class KVMonitor *kvmon() {
+    return (class KVMonitor*) paxos_service[PAXOS_KV].get();
+  }
+
+  friend class Paxos;
+  friend class OSDMonitor;
+  friend class MDSMonitor;
+  friend class MonmapMonitor;
+  friend class LogMonitor;
+  friend class KVMonitor;
+
+  // -- sessions --
+  MonSessionMap session_map;
+  ceph::mutex session_map_lock = ceph::make_mutex("Monitor::session_map_lock");
+  AdminSocketHook *admin_hook;
+
+  template<typename Func, typename...Args>
+  void with_session_map(Func&& func) {
+    std::lock_guard l(session_map_lock);
+    std::forward<Func>(func)(session_map);
+  }
+  void send_latest_monmap(Connection *con);
+
+  // messages
+  void handle_get_version(MonOpRequestRef op);
+  void handle_subscribe(MonOpRequestRef op);
+  void handle_mon_get_map(MonOpRequestRef op);
+
+  static void _generate_command_map(cmdmap_t& cmdmap,
+                                    std::map<std::string,std::string> &param_str_map);
+  static const MonCommand *_get_moncommand(
+    const std::string &cmd_prefix,
+    const std::vector<MonCommand>& cmds);
+  bool _allowed_command(MonSession *s, const std::string& module,
+			const std::string& prefix,
+                        const cmdmap_t& cmdmap,
+                        const std::map<std::string,std::string>& param_str_map,
+                        const MonCommand *this_cmd);
+  void get_mon_status(ceph::Formatter *f);
+  void _quorum_status(ceph::Formatter *f, std::ostream& ss);
+  bool _add_bootstrap_peer_hint(std::string_view cmd, const cmdmap_t& cmdmap,
+				std::ostream& ss);
+  void handle_tell_command(MonOpRequestRef op);
+  void handle_command(MonOpRequestRef op);
+  void handle_route(MonOpRequestRef op);
+
+  int get_mon_metadata(int mon, ceph::Formatter *f, std::ostream& err);
+  int print_nodes(ceph::Formatter *f, std::ostream& err);
+
+  // track metadata reported by win_election()
+  std::map<int, Metadata> mon_metadata;
+  std::map<int, Metadata> pending_metadata;
+
+  /**
+   *
+   */
+  struct health_cache_t {
+    health_status_t overall;
+    std::string summary;
+
+    void reset() {
+      // health_status_t doesn't really have a NONE value and we're not
+      // okay with setting something else (say, HEALTH_ERR).  so just
+      // leave it be.
+      summary.clear();
+    }
+  } health_status_cache;
+
+  Context *health_tick_event = nullptr;
+  Context *health_interval_event = nullptr;
+
+  void health_tick_start();
+  void health_tick_stop();
+  ceph::real_clock::time_point health_interval_calc_next_update();
+  void health_interval_start();
+  void health_interval_stop();
+  void health_events_cleanup();
+
+  void health_to_clog_update_conf(const std::set<std::string> &changed);
+
+  void do_health_to_clog_interval();
+  void do_health_to_clog(bool force = false);
+
+  void log_health(
+    const health_check_map_t& updated,
+    const health_check_map_t& previous,
+    MonitorDBStore::TransactionRef t);
+
+  void update_pending_metadata();
+
+protected:
+
+  class HealthCheckLogStatus {
+    public:
+    health_status_t severity;
+    std::string last_message;
+    utime_t updated_at = 0;
+    HealthCheckLogStatus(health_status_t severity_,
+                         const std::string &last_message_,
+                         utime_t updated_at_)
+      : severity(severity_),
+        last_message(last_message_),
+        updated_at(updated_at_)
+    {}
+  };
+  std::map<std::string, HealthCheckLogStatus> health_check_log_times;
+
+public:
+
+  void get_cluster_status(std::stringstream &ss, ceph::Formatter *f,
+			  MonSession *session);
+
+  void reply_command(MonOpRequestRef op, int rc, const std::string &rs, version_t version);
+  void reply_command(MonOpRequestRef op, int rc, const std::string &rs, ceph::buffer::list& rdata, version_t version);
+
+  void reply_tell_command(MonOpRequestRef op, int rc, const std::string &rs);
+
+
+
+  void handle_probe(MonOpRequestRef op);
+  /**
+   * Handle a Probe Operation, replying with our name, quorum and known versions.
+   *
+   * We use the MMonProbe message class for anything and everything related with
+   * Monitor probing. One of the operations relates directly with the probing
+   * itself, in which we receive a probe request and to which we reply with
+   * our name, our quorum and the known versions for each Paxos service. Thus the
+   * redundant function name. This reply will obviously be sent to the one
+   * probing/requesting these infos.
+   *
+   * @todo Add @pre and @post
+   *
+   * @param m A Probe message, with an operation of type Probe.
+   */
+  void handle_probe_probe(MonOpRequestRef op);
+  void handle_probe_reply(MonOpRequestRef op);
+
+  // request routing
+  struct RoutedRequest {
+    uint64_t tid;
+    ceph::buffer::list request_bl;
+    MonSession *session;
+    ConnectionRef con;
+    uint64_t con_features;
+    MonOpRequestRef op;
+
+    RoutedRequest() : tid(0), session(NULL), con_features(0) {}
+    ~RoutedRequest() {
+      if (session)
+	session->put();
+    }
+  };
+  uint64_t routed_request_tid;
+  std::map<uint64_t, RoutedRequest*> routed_requests;
+
+  void forward_request_leader(MonOpRequestRef op);
+  void handle_forward(MonOpRequestRef op);
+  void send_reply(MonOpRequestRef op, Message *reply);
+  void no_reply(MonOpRequestRef op);
+  void resend_routed_requests();
+  void remove_session(MonSession *s);
+  void remove_all_sessions();
+  void waitlist_or_zap_client(MonOpRequestRef op);
+
+  void send_mon_message(Message *m, int rank);
+  /** can_change_external_state if we can do things like
+   *  call elections as a result of the new map.
+   */
+  void notify_new_monmap(bool can_change_external_state=false, bool remove_rank_elector=true);
+
+public:
+  struct C_Command : public C_MonOp {
+    Monitor &mon;
+    int rc;
+    std::string rs;
+    ceph::buffer::list rdata;
+    version_t version;
+    C_Command(Monitor &_mm, MonOpRequestRef _op, int r, std::string s, version_t v) :
+      C_MonOp(_op), mon(_mm), rc(r), rs(s), version(v){}
+    C_Command(Monitor &_mm, MonOpRequestRef _op, int r, std::string s, ceph::buffer::list rd, version_t v) :
+      C_MonOp(_op), mon(_mm), rc(r), rs(s), rdata(rd), version(v){}
+
+    void _finish(int r) override {
+      auto m = op->get_req<MMonCommand>();
+      if (r >= 0) {
+	std::ostringstream ss;
+        if (!op->get_req()->get_connection()) {
+          ss << "connection dropped for command ";
+        } else {
+          MonSession *s = op->get_session();
+
+          // if client drops we may not have a session to draw information from.
+          if (s) {
+            ss << "from='" << s->name << " " << s->addrs << "' "
+              << "entity='" << s->entity_name << "' ";
+          } else {
+            ss << "session dropped for command ";
+          }
+        }
+        cmdmap_t cmdmap;
+        std::ostringstream ds;
+        string prefix;
+        cmdmap_from_json(m->cmd, &cmdmap, ds);
+        cmd_getval(cmdmap, "prefix", prefix);
+        if (prefix != "config set" && prefix != "config-key set")
+          ss << "cmd='" << m->cmd << "': finished";
+
+        mon.audit_clog->info() << ss.str();
+        mon.reply_command(op, rc, rs, rdata, version);
+      }
+      else if (r == -ECANCELED)
+        return;
+      else if (r == -EAGAIN)
+        mon.dispatch_op(op);
+      else
+	ceph_abort_msg("bad C_Command return value");
+    }
+  };
+
+ private:
+  class C_RetryMessage : public C_MonOp {
+    Monitor *mon;
+  public:
+    C_RetryMessage(Monitor *m, MonOpRequestRef op) :
+      C_MonOp(op), mon(m) { }
+
+    void _finish(int r) override {
+      if (r == -EAGAIN || r >= 0)
+        mon->dispatch_op(op);
+      else if (r == -ECANCELED)
+        return;
+      else
+	ceph_abort_msg("bad C_RetryMessage return value");
+    }
+  };
+
+  //ms_dispatch handles a lot of logic and we want to reuse it
+  //on forwarded messages, so we create a non-locking version for this class
+  void _ms_dispatch(Message *m);
+  bool ms_dispatch(Message *m) override {
+    std::lock_guard l{lock};
+    _ms_dispatch(m);
+    return true;
+  }
+  void dispatch_op(MonOpRequestRef op);
+  //mon_caps is used for un-connected messages from monitors
+  MonCap mon_caps;
+  bool get_authorizer(int dest_type, AuthAuthorizer **authorizer);
+public: // for AuthMonitor msgr1:
+  int ms_handle_authentication(Connection *con) override;
+private:
+  void ms_handle_accept(Connection *con) override;
+  bool ms_handle_reset(Connection *con) override;
+  void ms_handle_remote_reset(Connection *con) override {}
+  bool ms_handle_refused(Connection *con) override;
+
+  // AuthClient
+  int get_auth_request(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    uint32_t *method,
+    std::vector<uint32_t> *preferred_modes,
+    ceph::buffer::list *out) override;
+  int handle_auth_reply_more(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+   const ceph::buffer::list& bl,
+    ceph::buffer::list *reply) override;
+  int handle_auth_done(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    uint64_t global_id,
+    uint32_t con_mode,
+    const ceph::buffer::list& bl,
+    CryptoKey *session_key,
+    std::string *connection_secret) override;
+  int handle_auth_bad_method(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    uint32_t old_auth_method,
+    int result,
+    const std::vector<uint32_t>& allowed_methods,
+    const std::vector<uint32_t>& allowed_modes) override;
+  // /AuthClient
+  // AuthServer
+  int handle_auth_request(
+    Connection *con,
+    AuthConnectionMeta *auth_meta,
+    bool more,
+    uint32_t auth_method,
+    const ceph::buffer::list& bl,
+    ceph::buffer::list *reply) override;
+  // /AuthServer
+
+  int write_default_keyring(ceph::buffer::list& bl);
+  void extract_save_mon_key(KeyRing& keyring);
+
+  void collect_metadata(Metadata *m);
+  int load_metadata();
+  void count_metadata(const std::string& field, ceph::Formatter *f);
+  void count_metadata(const std::string& field, std::map<std::string,int> *out);
+  // get_all_versions() gathers version information from daemons for health check
+  void get_all_versions(std::map<string, std::list<std::string>> &versions);
+  void get_versions(std::map<string, std::list<std::string>> &versions);
+
+  // features
+  static CompatSet get_initial_supported_features();
+  static CompatSet get_supported_features();
+  static CompatSet get_legacy_features();
+  /// read the ondisk features into the CompatSet pointed to by read_features
+  static void read_features_off_disk(MonitorDBStore *store, CompatSet *read_features);
+  void read_features();
+  void write_features(MonitorDBStore::TransactionRef t);
+
+  OpTracker op_tracker;
+
+ public:
+  Monitor(CephContext *cct_, std::string nm, MonitorDBStore *s,
+	  Messenger *m, Messenger *mgr_m, MonMap *map);
+  ~Monitor() override;
+
+  static int check_features(MonitorDBStore *store);
+
+  // config observer
+  const char** get_tracked_conf_keys() const override;
+  void handle_conf_change(const ConfigProxy& conf,
+                          const std::set<std::string> &changed) override;
+
+  void update_log_clients();
+  int sanitize_options();
+  int preinit();
+  int init();
+  void init_paxos();
+  void refresh_from_paxos(bool *need_bootstrap);
+  void shutdown();
+  void tick();
+
+  void handle_signal(int sig);
+
+  int mkfs(ceph::buffer::list& osdmapbl);
+
+  /**
+   * check cluster_fsid file
+   *
+   * @return EEXIST if file exists and doesn't match, 0 on match, or negative error code
+   */
+  int check_fsid();
+
+  /**
+   * write cluster_fsid file
+   *
+   * @return 0 on success, or negative error code
+   */
+  int write_fsid();
+  int write_fsid(MonitorDBStore::TransactionRef t);
+
+  int do_admin_command(std::string_view command, const cmdmap_t& cmdmap,
+		       ceph::Formatter *f,
+		       std::ostream& err,
+		       std::ostream& out);
+
+private:
+  // don't allow copying
+  Monitor(const Monitor& rhs);
+  Monitor& operator=(const Monitor &rhs);
+
+public:
+  static void format_command_descriptions(const std::vector<MonCommand> &commands,
+					  ceph::Formatter *f,
+					  uint64_t features,
+					  ceph::buffer::list *rdata);
+
+  const std::vector<MonCommand> &get_local_commands(mon_feature_t f) {
+    if (f.contains_all(ceph::features::mon::FEATURE_NAUTILUS)) {
+      return local_mon_commands;
+    } else {
+      return prenautilus_local_mon_commands;
+    }
+  }
+  const ceph::buffer::list& get_local_commands_bl(mon_feature_t f) {
+    if (f.contains_all(ceph::features::mon::FEATURE_NAUTILUS)) {
+      return local_mon_commands_bl;
+    } else {
+      return prenautilus_local_mon_commands_bl;
+    }
+  }
+  void set_leader_commands(const std::vector<MonCommand>& cmds) {
+    leader_mon_commands = cmds;
+  }
+
+  bool is_keyring_required();
+};
+
+#define CEPH_MON_FEATURE_INCOMPAT_BASE CompatSet::Feature (1, "initial feature set (~v.18)")
+#define CEPH_MON_FEATURE_INCOMPAT_GV CompatSet::Feature (2, "global version sequencing (v0.52)")
+#define CEPH_MON_FEATURE_INCOMPAT_SINGLE_PAXOS CompatSet::Feature (3, "single paxos with k/v store (v0.\?)")
+#define CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES CompatSet::Feature(4, "support erasure code pools")
+#define CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC CompatSet::Feature(5, "new-style osdmap encoding")
+#define CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2 CompatSet::Feature(6, "support isa/lrc erasure code")
+#define CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V3 CompatSet::Feature(7, "support shec erasure code")
+#define CEPH_MON_FEATURE_INCOMPAT_KRAKEN CompatSet::Feature(8, "support monmap features")
+#define CEPH_MON_FEATURE_INCOMPAT_LUMINOUS CompatSet::Feature(9, "luminous ondisk layout")
+#define CEPH_MON_FEATURE_INCOMPAT_MIMIC CompatSet::Feature(10, "mimic ondisk layout")
+#define CEPH_MON_FEATURE_INCOMPAT_NAUTILUS CompatSet::Feature(11, "nautilus ondisk layout")
+#define CEPH_MON_FEATURE_INCOMPAT_OCTOPUS CompatSet::Feature(12, "octopus ondisk layout")
+#define CEPH_MON_FEATURE_INCOMPAT_PACIFIC CompatSet::Feature(13, "pacific ondisk layout")
+// make sure you add your feature to Monitor::get_supported_features
+
+
+/* Callers use:
+ *
+ *      new C_MonContext{...}
+ *
+ * instead of
+ *
+ *      new C_MonContext(...)
+ *
+ * because of gcc bug [1].
+ *
+ * [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85883
+ */
+template<typename T>
+class C_MonContext : public LambdaContext<T> {
+public:
+  C_MonContext(const Monitor* m, T&& f) :
+      LambdaContext<T>(std::forward<T>(f)),
+      mon(m)
+  {}
+  void finish(int r) override {
+    if (mon->is_shutdown())
+      return;
+    LambdaContext<T>::finish(r);
+  }
+private:
+  const Monitor* mon;
+};
+
+#endif
diff --git a/src/mon/MonitorDBStore.h b/src/mon/MonitorDBStore.h
new file mode 100644
index 000000000..c33d35e48
--- /dev/null
+++ b/src/mon/MonitorDBStore.h
@@ -0,0 +1,814 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 Inktank, Inc.
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#ifndef CEPH_MONITOR_DB_STORE_H
+#define CEPH_MONITOR_DB_STORE_H
+
+#include "include/types.h"
+#include "include/buffer.h"
+#include <set>
+#include <map>
+#include <string>
+#include <boost/scoped_ptr.hpp>
+#include <sstream>
+#include <fstream>
+#include "kv/KeyValueDB.h"
+
+#include "include/ceph_assert.h"
+#include "common/Formatter.h"
+#include "common/Finisher.h"
+#include "common/errno.h"
+#include "common/debug.h"
+#include "common/safe_io.h"
+#include "common/blkdev.h"
+#include "common/PriorityCache.h"
+
+#define dout_context g_ceph_context
+
+class MonitorDBStore
+{
+  std::string path;
+  boost::scoped_ptr<KeyValueDB> db;
+  bool do_dump;
+  int dump_fd_binary;
+  std::ofstream dump_fd_json;
+  ceph::JSONFormatter dump_fmt;
+  
+
+  Finisher io_work;
+
+  bool is_open;
+
+ public:
+
+  std::string get_devname() {
+    char devname[4096] = {0}, partition[4096];
+    get_device_by_path(path.c_str(), partition, devname,
+		       sizeof(devname));
+    return devname;
+  }
+
+  std::string get_path() {
+    return path;
+  }
+
+  std::shared_ptr<PriorityCache::PriCache> get_priority_cache() const {
+    return db->get_priority_cache();
+  }
+
+  struct Op {
+    uint8_t type;
+    std::string prefix;
+    std::string key, endkey;
+    ceph::buffer::list bl;
+
+    Op()
+      : type(0) { }
+    Op(int t, const std::string& p, const std::string& k)
+      : type(t), prefix(p), key(k) { }
+    Op(int t, const std::string& p, const std::string& k, const ceph::buffer::list& b)
+      : type(t), prefix(p), key(k), bl(b) { }
+    Op(int t, const std::string& p, const std::string& start, const std::string& end)
+      : type(t), prefix(p), key(start), endkey(end) { }
+
+    void encode(ceph::buffer::list& encode_bl) const {
+      ENCODE_START(2, 1, encode_bl);
+      encode(type, encode_bl);
+      encode(prefix, encode_bl);
+      encode(key, encode_bl);
+      encode(bl, encode_bl);
+      encode(endkey, encode_bl);
+      ENCODE_FINISH(encode_bl);
+    }
+
+    void decode(ceph::buffer::list::const_iterator& decode_bl) {
+      DECODE_START(2, decode_bl);
+      decode(type, decode_bl);
+      decode(prefix, decode_bl);
+      decode(key, decode_bl);
+      decode(bl, decode_bl);
+      if (struct_v >= 2)
+	decode(endkey, decode_bl);
+      DECODE_FINISH(decode_bl);
+    }
+
+    void dump(ceph::Formatter *f) const {
+      f->dump_int("type", type);
+      f->dump_string("prefix", prefix);
+      f->dump_string("key", key);
+      if (endkey.length()) {
+	f->dump_string("endkey", endkey);
+      }
+    }
+
+    int approx_size() const {
+      return 6 + 1 +
+	4 + prefix.size() +
+	4 + key.size() +
+	4 + endkey.size() +
+	4 + bl.length();
+    }
+
+    static void generate_test_instances(std::list<Op*>& ls) {
+      ls.push_back(new Op);
+      // we get coverage here from the Transaction instances
+    }
+  };
+
+  struct Transaction;
+  typedef std::shared_ptr<Transaction> TransactionRef;
+  struct Transaction {
+    std::list<Op> ops;
+    uint64_t bytes, keys;
+
+    Transaction() : bytes(6 + 4 + 8*2), keys(0) {}
+
+    enum {
+      OP_PUT	= 1,
+      OP_ERASE	= 2,
+      OP_COMPACT = 3,
+      OP_ERASE_RANGE = 4,
+    };
+
+    void put(const std::string& prefix, const std::string& key, const ceph::buffer::list& bl) {
+      ops.push_back(Op(OP_PUT, prefix, key, bl));
+      ++keys;
+      bytes += ops.back().approx_size();
+    }
+
+    void put(const std::string& prefix, version_t ver, const ceph::buffer::list& bl) {
+      std::ostringstream os;
+      os << ver;
+      put(prefix, os.str(), bl);
+    }
+
+    void put(const std::string& prefix, const std::string& key, version_t ver) {
+      using ceph::encode;
+      ceph::buffer::list bl;
+      encode(ver, bl);
+      put(prefix, key, bl);
+    }
+
+    void erase(const std::string& prefix, const std::string& key) {
+      ops.push_back(Op(OP_ERASE, prefix, key));
+      ++keys;
+      bytes += ops.back().approx_size();
+    }
+
+    void erase(const std::string& prefix, version_t ver) {
+      std::ostringstream os;
+      os << ver;
+      erase(prefix, os.str());
+    }
+
+    void erase_range(const std::string& prefix, const std::string& begin,
+		     const std::string& end) {
+      ops.push_back(Op(OP_ERASE_RANGE, prefix, begin, end));
+      ++keys;
+      bytes += ops.back().approx_size();
+    }
+
+    void compact_prefix(const std::string& prefix) {
+      ops.push_back(Op(OP_COMPACT, prefix, {}));
+    }
+
+    void compact_range(const std::string& prefix, const std::string& start,
+		       const std::string& end) {
+      ops.push_back(Op(OP_COMPACT, prefix, start, end));
+    }
+
+    void encode(ceph::buffer::list& bl) const {
+      ENCODE_START(2, 1, bl);
+      encode(ops, bl);
+      encode(bytes, bl);
+      encode(keys, bl);
+      ENCODE_FINISH(bl);
+    }
+
+    void decode(ceph::buffer::list::const_iterator& bl) {
+      DECODE_START(2, bl);
+      decode(ops, bl);
+      if (struct_v >= 2) {
+	decode(bytes, bl);
+	decode(keys, bl);
+      }
+      DECODE_FINISH(bl);
+    }
+
+    static void generate_test_instances(std::list<Transaction*>& ls) {
+      ls.push_back(new Transaction);
+      ls.push_back(new Transaction);
+      ceph::buffer::list bl;
+      bl.append("value");
+      ls.back()->put("prefix", "key", bl);
+      ls.back()->erase("prefix2", "key2");
+      ls.back()->erase_range("prefix3", "key3", "key4");
+      ls.back()->compact_prefix("prefix3");
+      ls.back()->compact_range("prefix4", "from", "to");
+    }
+
+    void append(TransactionRef other) {
+      ops.splice(ops.end(), other->ops);
+      keys += other->keys;
+      bytes += other->bytes;
+    }
+
+    void append_from_encoded(ceph::buffer::list& bl) {
+      auto other(std::make_shared<Transaction>());
+      auto it = bl.cbegin();
+      other->decode(it);
+      append(other);
+    }
+
+    bool empty() {
+      return (size() == 0);
+    }
+
+    size_t size() const {
+      return ops.size();
+    }
+    uint64_t get_keys() const {
+      return keys;
+    }
+    uint64_t get_bytes() const {
+      return bytes;
+    }
+
+    void dump(ceph::Formatter *f, bool dump_val=false) const {
+      f->open_object_section("transaction");
+      f->open_array_section("ops");
+      int op_num = 0;
+      for (auto it = ops.begin(); it != ops.end(); ++it) {
+	const Op& op = *it;
+	f->open_object_section("op");
+	f->dump_int("op_num", op_num++);
+	switch (op.type) {
+	case OP_PUT:
+	  {
+	    f->dump_string("type", "PUT");
+	    f->dump_string("prefix", op.prefix);
+	    f->dump_string("key", op.key);
+	    f->dump_unsigned("length", op.bl.length());
+	    if (dump_val) {
+	      std::ostringstream os;
+	      op.bl.hexdump(os);
+	      f->dump_string("bl", os.str());
+	    }
+	  }
+	  break;
+	case OP_ERASE:
+	  {
+	    f->dump_string("type", "ERASE");
+	    f->dump_string("prefix", op.prefix);
+	    f->dump_string("key", op.key);
+	  }
+	  break;
+	case OP_ERASE_RANGE:
+	  {
+	    f->dump_string("type", "ERASE_RANGE");
+	    f->dump_string("prefix", op.prefix);
+	    f->dump_string("start", op.key);
+	    f->dump_string("end", op.endkey);
+	  }
+	  break;
+	case OP_COMPACT:
+	  {
+	    f->dump_string("type", "COMPACT");
+	    f->dump_string("prefix", op.prefix);
+	    f->dump_string("start", op.key);
+	    f->dump_string("end", op.endkey);
+	  }
+	  break;
+	default:
+	  {
+	    f->dump_string("type", "unknown");
+	    f->dump_unsigned("op_code", op.type);
+	    break;
+	  }
+	}
+	f->close_section();
+      }
+      f->close_section();
+      f->dump_unsigned("num_keys", keys);
+      f->dump_unsigned("num_bytes", bytes);
+      f->close_section();
+    }
+  };
+
+  int apply_transaction(MonitorDBStore::TransactionRef t) {
+    KeyValueDB::Transaction dbt = db->get_transaction();
+
+    if (do_dump) {
+      if (!g_conf()->mon_debug_dump_json) {
+        ceph::buffer::list bl;
+        t->encode(bl);
+        bl.write_fd(dump_fd_binary);
+      } else {
+        t->dump(&dump_fmt, true);
+        dump_fmt.flush(dump_fd_json);
+        dump_fd_json.flush();
+      }
+    }
+
+    std::list<std::pair<std::string, std::pair<std::string,std::string>>> compact;
+    for (auto it = t->ops.begin(); it != t->ops.end(); ++it) {
+      const Op& op = *it;
+      switch (op.type) {
+      case Transaction::OP_PUT:
+	dbt->set(op.prefix, op.key, op.bl);
+	break;
+      case Transaction::OP_ERASE:
+	dbt->rmkey(op.prefix, op.key);
+	break;
+      case Transaction::OP_ERASE_RANGE:
+	dbt->rm_range_keys(op.prefix, op.key, op.endkey);
+	break;
+      case Transaction::OP_COMPACT:
+	compact.push_back(make_pair(op.prefix, make_pair(op.key, op.endkey)));
+	break;
+      default:
+	derr << __func__ << " unknown op type " << op.type << dendl;
+	ceph_abort();
+	break;
+      }
+    }
+    int r = db->submit_transaction_sync(dbt);
+    if (r >= 0) {
+      while (!compact.empty()) {
+	if (compact.front().second.first == std::string() &&
+	    compact.front().second.second == std::string())
+	  db->compact_prefix_async(compact.front().first);
+	else
+	  db->compact_range_async(compact.front().first, compact.front().second.first, compact.front().second.second);
+	compact.pop_front();
+      }
+    } else {
+      ceph_abort_msg("failed to write to db");
+    }
+    return r;
+  }
+
+  struct C_DoTransaction : public Context {
+    MonitorDBStore *store;
+    MonitorDBStore::TransactionRef t;
+    Context *oncommit;
+    C_DoTransaction(MonitorDBStore *s, MonitorDBStore::TransactionRef t,
+		    Context *f)
+      : store(s), t(t), oncommit(f)
+    {}
+    void finish(int r) override {
+      /* The store serializes writes.  Each transaction is handled
+       * sequentially by the io_work Finisher.  If a transaction takes longer
+       * to apply its state to permanent storage, then no other transaction
+       * will be handled meanwhile.
+       *
+       * We will now randomly inject random delays.  We can safely sleep prior
+       * to applying the transaction as it won't break the model.
+       */
+      double delay_prob = g_conf()->mon_inject_transaction_delay_probability;
+      if (delay_prob && (rand() % 10000 < delay_prob * 10000.0)) {
+        utime_t delay;
+        double delay_max = g_conf()->mon_inject_transaction_delay_max;
+        delay.set_from_double(delay_max * (double)(rand() % 10000) / 10000.0);
+        lsubdout(g_ceph_context, mon, 1)
+          << "apply_transaction will be delayed for " << delay
+          << " seconds" << dendl;
+        delay.sleep();
+      }
+      int ret = store->apply_transaction(t);
+      oncommit->complete(ret);
+    }
+  };
+
+  /**
+   * queue transaction
+   *
+   * Queue a transaction to commit asynchronously.  Trigger a context
+   * on completion (without any locks held).
+   */
+  void queue_transaction(MonitorDBStore::TransactionRef t,
+			 Context *oncommit) {
+    io_work.queue(new C_DoTransaction(this, t, oncommit));
+  }
+
+  /**
+   * block and flush all io activity
+   */
+  void flush() {
+    io_work.wait_for_empty();
+  }
+
+  class StoreIteratorImpl {
+  protected:
+    bool done;
+    std::pair<std::string,std::string> last_key;
+    ceph::buffer::list crc_bl;
+
+    StoreIteratorImpl() : done(false) { }
+    virtual ~StoreIteratorImpl() { }
+
+    virtual bool _is_valid() = 0;
+
+  public:
+    __u32 crc() {
+      if (g_conf()->mon_sync_debug)
+	return crc_bl.crc32c(0);
+      return 0;
+    }
+    std::pair<std::string,std::string> get_last_key() {
+      return last_key;
+    }
+    virtual bool has_next_chunk() {
+      return !done && _is_valid();
+    }
+    virtual void get_chunk_tx(TransactionRef tx, uint64_t max_bytes,
+			      uint64_t max_keys) = 0;
+    virtual std::pair<std::string,std::string> get_next_key() = 0;
+  };
+  typedef std::shared_ptr<StoreIteratorImpl> Synchronizer;
+
+  class WholeStoreIteratorImpl : public StoreIteratorImpl {
+    KeyValueDB::WholeSpaceIterator iter;
+    std::set<std::string> sync_prefixes;
+
+  public:
+    WholeStoreIteratorImpl(KeyValueDB::WholeSpaceIterator iter,
+			   std::set<std::string> &prefixes)
+      : StoreIteratorImpl(),
+	iter(iter),
+	sync_prefixes(prefixes)
+    { }
+
+    ~WholeStoreIteratorImpl() override { }
+
+    /**
+     * Obtain a chunk of the store
+     *
+     * @param bl	    Encoded transaction that will recreate the chunk
+     * @param first_key	    Pair containing the first key to obtain, and that
+     *			    will contain the first key in the chunk (that may
+     *			    differ from the one passed on to the function)
+     * @param last_key[out] Last key in the chunk
+     */
+    void get_chunk_tx(TransactionRef tx, uint64_t max_bytes,
+		      uint64_t max_keys) override {
+      using ceph::encode;
+      ceph_assert(done == false);
+      ceph_assert(iter->valid() == true);
+
+      while (iter->valid()) {
+	std::string prefix(iter->raw_key().first);
+	std::string key(iter->raw_key().second);
+	if (sync_prefixes.count(prefix)) {
+	  ceph::buffer::list value = iter->value();
+	  if (tx->empty() ||
+	      (tx->get_bytes() + value.length() + key.size() +
+	       prefix.size() < max_bytes &&
+	       tx->get_keys() < max_keys)) {
+	    // NOTE: putting every key in a separate transaction is
+	    // questionable as far as efficiency goes
+	    auto tmp(std::make_shared<Transaction>());
+	    tmp->put(prefix, key, value);
+	    tx->append(tmp);
+	    if (g_conf()->mon_sync_debug) {
+	      encode(prefix, crc_bl);
+	      encode(key, crc_bl);
+	      encode(value, crc_bl);
+	    }
+	  } else {
+	    last_key.first = prefix;
+	    last_key.second = key;
+	    return;
+	  }
+	}
+	iter->next();
+      }
+      ceph_assert(iter->valid() == false);
+      done = true;
+    }
+
+    std::pair<std::string,std::string> get_next_key() override {
+      ceph_assert(iter->valid());
+
+      for (; iter->valid(); iter->next()) {
+	std::pair<std::string,std::string> r = iter->raw_key();
+        if (sync_prefixes.count(r.first) > 0) {
+          iter->next();
+          return r;
+        }
+      }
+      return std::pair<std::string,std::string>();
+    }
+
+    bool _is_valid() override {
+      return iter->valid();
+    }
+  };
+
+  Synchronizer get_synchronizer(std::pair<std::string,std::string> &key,
+				std::set<std::string> &prefixes) {
+    KeyValueDB::WholeSpaceIterator iter;
+    iter = db->get_wholespace_iterator();
+
+    if (!key.first.empty() && !key.second.empty())
+      iter->upper_bound(key.first, key.second);
+    else
+      iter->seek_to_first();
+
+    return std::shared_ptr<StoreIteratorImpl>(
+	new WholeStoreIteratorImpl(iter, prefixes)
+    );
+  }
+
+  KeyValueDB::Iterator get_iterator(const std::string &prefix) {
+    ceph_assert(!prefix.empty());
+    KeyValueDB::Iterator iter = db->get_iterator(prefix);
+    iter->seek_to_first();
+    return iter;
+  }
+
+  KeyValueDB::WholeSpaceIterator get_iterator() {
+    KeyValueDB::WholeSpaceIterator iter;
+    iter = db->get_wholespace_iterator();
+    iter->seek_to_first();
+    return iter;
+  }
+
+  int get(const std::string& prefix, const std::string& key, ceph::buffer::list& bl) {
+    ceph_assert(bl.length() == 0);
+    return db->get(prefix, key, &bl);
+  }
+
+  int get(const std::string& prefix, const version_t ver, ceph::buffer::list& bl) {
+    std::ostringstream os;
+    os << ver;
+    return get(prefix, os.str(), bl);
+  }
+
+  version_t get(const std::string& prefix, const std::string& key) {
+    using ceph::decode;
+    ceph::buffer::list bl;
+    int err = get(prefix, key, bl);
+    if (err < 0) {
+      if (err == -ENOENT) // if key doesn't exist, assume its value is 0
+        return 0;
+      // we're not expecting any other negative return value, and we can't
+      // just return a negative value if we're returning a version_t
+      generic_dout(0) << "MonitorDBStore::get() error obtaining"
+                      << " (" << prefix << ":" << key << "): "
+                      << cpp_strerror(err) << dendl;
+      ceph_abort_msg("error obtaining key");
+    }
+
+    ceph_assert(bl.length());
+    version_t ver;
+    auto p = bl.cbegin();
+    decode(ver, p);
+    return ver;
+  }
+
+  bool exists(const std::string& prefix, const std::string& key) {
+    KeyValueDB::Iterator it = db->get_iterator(prefix);
+    int err = it->lower_bound(key);
+    if (err < 0)
+      return false;
+
+    return (it->valid() && it->key() == key);
+  }
+
+  bool exists(const std::string& prefix, version_t ver) {
+    std::ostringstream os;
+    os << ver;
+    return exists(prefix, os.str());
+  }
+
+  std::string combine_strings(const std::string& prefix, const std::string& value) {
+    std::string out = prefix;
+    out.push_back('_');
+    out.append(value);
+    return out;
+  }
+
+  std::string combine_strings(const std::string& prefix, const version_t ver) {
+    std::ostringstream os;
+    os << ver;
+    return combine_strings(prefix, os.str());
+  }
+
+  void clear(std::set<std::string>& prefixes) {
+    KeyValueDB::Transaction dbt = db->get_transaction();
+
+    for (auto iter = prefixes.begin(); iter != prefixes.end(); ++iter) {
+      dbt->rmkeys_by_prefix((*iter));
+    }
+    int r = db->submit_transaction_sync(dbt);
+    ceph_assert(r >= 0);
+  }
+
+  void _open(const std::string& kv_type) {
+    int pos = 0;
+    for (auto rit = path.rbegin(); rit != path.rend(); ++rit, ++pos) {
+      if (*rit != '/')
+	break;
+    }
+    std::ostringstream os;
+    os << path.substr(0, path.size() - pos) << "/store.db";
+    std::string full_path = os.str();
+
+    KeyValueDB *db_ptr = KeyValueDB::create(g_ceph_context,
+					    kv_type,
+					    full_path);
+    if (!db_ptr) {
+      derr << __func__ << " error initializing "
+	   << kv_type << " db back storage in "
+	   << full_path << dendl;
+      ceph_abort_msg("MonitorDBStore: error initializing keyvaluedb back storage");
+    }
+    db.reset(db_ptr);
+
+    if (g_conf()->mon_debug_dump_transactions) {
+      if (!g_conf()->mon_debug_dump_json) {
+        dump_fd_binary = ::open(
+          g_conf()->mon_debug_dump_location.c_str(),
+          O_CREAT|O_APPEND|O_WRONLY|O_CLOEXEC, 0644);
+        if (dump_fd_binary < 0) {
+          dump_fd_binary = -errno;
+          derr << "Could not open log file, got "
+               << cpp_strerror(dump_fd_binary) << dendl;
+        }
+      } else {
+        dump_fmt.reset();
+        dump_fmt.open_array_section("dump");
+        dump_fd_json.open(g_conf()->mon_debug_dump_location.c_str());
+      }
+      do_dump = true;
+    }
+    if (kv_type == "rocksdb")
+      db->init(g_conf()->mon_rocksdb_options);
+    else
+      db->init();
+
+
+  }
+
+  int open(std::ostream &out) {
+    std::string kv_type;
+    int r = read_meta("kv_backend", &kv_type);
+    if (r < 0 || kv_type.empty()) {
+      // assume old monitors that did not mark the type were leveldb.
+      kv_type = "leveldb";
+      r = write_meta("kv_backend", kv_type);
+      if (r < 0)
+	return r;
+    }
+    _open(kv_type);
+    r = db->open(out);
+    if (r < 0)
+      return r;
+
+    // Monitors are few in number, so the resource cost of exposing 
+    // very detailed stats is low: ramp up the priority of all the
+    // KV store's perf counters.  Do this after open, because backend may
+    // not have constructed PerfCounters earlier.
+    if (db->get_perf_counters()) {
+      db->get_perf_counters()->set_prio_adjust(
+          PerfCountersBuilder::PRIO_USEFUL - PerfCountersBuilder::PRIO_DEBUGONLY);
+    }
+
+    io_work.start();
+    is_open = true;
+    return 0;
+  }
+
+  int create_and_open(std::ostream &out) {
+    // record the type before open
+    std::string kv_type;
+    int r = read_meta("kv_backend", &kv_type);
+    if (r < 0) {
+      kv_type = g_conf()->mon_keyvaluedb;
+      r = write_meta("kv_backend", kv_type);
+      if (r < 0)
+	return r;
+    }
+    _open(kv_type);
+    r = db->create_and_open(out);
+    if (r < 0)
+      return r;
+    io_work.start();
+    is_open = true;
+    return 0;
+  }
+
+  void close() {
+    // there should be no work queued!
+    io_work.stop();
+    is_open = false;
+    db.reset(NULL);
+  }
+
+  void compact() {
+    db->compact();
+  }
+
+  void compact_async() {
+    db->compact_async();
+  }
+
+  void compact_prefix(const std::string& prefix) {
+    db->compact_prefix(prefix);
+  }
+
+  uint64_t get_estimated_size(std::map<std::string, uint64_t> &extras) {
+    return db->get_estimated_size(extras);
+  }
+
+  /**
+   * write_meta - write a simple configuration key out-of-band
+   *
+   * Write a simple key/value pair for basic store configuration
+   * (e.g., a uuid or magic number) to an unopened/unmounted store.
+   * The default implementation writes this to a plaintext file in the
+   * path.
+   *
+   * A newline is appended.
+   *
+   * @param key key name (e.g., "fsid")
+   * @param value value (e.g., a uuid rendered as a string)
+   * @returns 0 for success, or an error code
+   */
+  int write_meta(const std::string& key,
+		 const std::string& value) const {
+    std::string v = value;
+    v += "\n";
+    int r = safe_write_file(path.c_str(), key.c_str(),
+			    v.c_str(), v.length(),
+			    0600);
+    if (r < 0)
+      return r;
+    return 0;
+  }
+
+  /**
+   * read_meta - read a simple configuration key out-of-band
+   *
+   * Read a simple key value to an unopened/mounted store.
+   *
+   * Trailing whitespace is stripped off.
+   *
+   * @param key key name
+   * @param value pointer to value string
+   * @returns 0 for success, or an error code
+   */
+  int read_meta(const std::string& key,
+		std::string *value) const {
+    char buf[4096];
+    int r = safe_read_file(path.c_str(), key.c_str(),
+			   buf, sizeof(buf));
+    if (r <= 0)
+      return r;
+    // drop trailing newlines
+    while (r && isspace(buf[r-1])) {
+      --r;
+    }
+    *value = std::string(buf, r);
+    return 0;
+  }
+
+  explicit MonitorDBStore(const std::string& path)
+    : path(path),
+      db(0),
+      do_dump(false),
+      dump_fd_binary(-1),
+      dump_fmt(true),
+      io_work(g_ceph_context, "monstore", "fn_monstore"),
+      is_open(false) {
+  }
+  ~MonitorDBStore() {
+    ceph_assert(!is_open);
+    if (do_dump) {
+      if (!g_conf()->mon_debug_dump_json) {
+        ::close(dump_fd_binary);
+      } else {
+        dump_fmt.close_section();
+        dump_fmt.flush(dump_fd_json);
+        dump_fd_json.flush();
+        dump_fd_json.close();
+      }
+    }
+  }
+
+};
+
+WRITE_CLASS_ENCODER(MonitorDBStore::Op)
+WRITE_CLASS_ENCODER(MonitorDBStore::Transaction)
+
+#endif /* CEPH_MONITOR_DB_STORE_H */
diff --git a/src/mon/MonmapMonitor.cc b/src/mon/MonmapMonitor.cc
new file mode 100644
index 000000000..91d9021c2
--- /dev/null
+++ b/src/mon/MonmapMonitor.cc
@@ -0,0 +1,1470 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2009 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "MonmapMonitor.h"
+#include "Monitor.h"
+#include "OSDMonitor.h"
+#include "messages/MMonCommand.h"
+#include "messages/MMonJoin.h"
+
+#include "common/ceph_argparse.h"
+#include "common/errno.h"
+#include <sstream>
+#include "common/config.h"
+#include "common/cmdparse.h"
+
+#include "include/ceph_assert.h"
+#include "include/stringify.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon)
+using namespace TOPNSPC::common;
+
+using std::cout;
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::setfill;
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+using std::unique_ptr;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::make_message;
+using ceph::mono_clock;
+using ceph::mono_time;
+using ceph::timespan_str;
+static ostream& _prefix(std::ostream *_dout, Monitor &mon) {
+  return *_dout << "mon." << mon.name << "@" << mon.rank
+		<< "(" << mon.get_state_name()
+		<< ").monmap v" << mon.monmap->epoch << " ";
+}
+
+void MonmapMonitor::create_initial()
+{
+  dout(10) << __func__ << " using current monmap" << dendl;
+  pending_map = *mon.monmap;
+  pending_map.epoch = 1;
+
+  if (g_conf()->mon_debug_no_initial_persistent_features) {
+    derr << __func__ << " mon_debug_no_initial_persistent_features=true"
+	 << dendl;
+  } else {
+    // initialize with default persistent features for new clusters
+    pending_map.persistent_features = ceph::features::mon::get_persistent();
+    pending_map.min_mon_release = ceph_release();
+  }
+}
+
+void MonmapMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  version_t version = get_last_committed();
+  if (version <= mon.monmap->get_epoch())
+    return;
+
+  dout(10) << __func__ << " version " << version
+	   << ", my v " << mon.monmap->epoch << dendl;
+  
+  if (need_bootstrap && version != mon.monmap->get_epoch()) {
+    dout(10) << " signaling that we need a bootstrap" << dendl;
+    *need_bootstrap = true;
+  }
+
+  // read and decode
+  monmap_bl.clear();
+  int ret = get_version(version, monmap_bl);
+  ceph_assert(ret == 0);
+  ceph_assert(monmap_bl.length());
+
+  dout(10) << __func__ << " got " << version << dendl;
+  mon.monmap->decode(monmap_bl);
+
+  if (mon.store->exists("mkfs", "monmap")) {
+    auto t(std::make_shared<MonitorDBStore::Transaction>());
+    t->erase("mkfs", "monmap");
+    mon.store->apply_transaction(t);
+  }
+
+  check_subs();
+
+  // make sure we've recorded min_mon_release
+  string val;
+  if (mon.store->read_meta("min_mon_release", &val) < 0 ||
+      val.size() == 0 ||
+      atoi(val.c_str()) != (int)ceph_release()) {
+    dout(10) << __func__ << " updating min_mon_release meta" << dendl;
+    mon.store->write_meta("min_mon_release",
+			   stringify(ceph_release()));
+  }
+
+  mon.notify_new_monmap(true);
+}
+
+void MonmapMonitor::create_pending()
+{
+  pending_map = *mon.monmap;
+  pending_map.epoch++;
+  pending_map.last_changed = ceph_clock_now();
+  pending_map.removed_ranks.clear();
+}
+
+void MonmapMonitor::encode_pending(MonitorDBStore::TransactionRef t)
+{
+  dout(10) << __func__ << " epoch " << pending_map.epoch << dendl;
+
+  ceph_assert(mon.monmap->epoch + 1 == pending_map.epoch ||
+	 pending_map.epoch == 1);  // special case mkfs!
+  bufferlist bl;
+  pending_map.encode(bl, mon.get_quorum_con_features());
+
+  put_version(t, pending_map.epoch, bl);
+  put_last_committed(t, pending_map.epoch);
+
+  // generate a cluster fingerprint, too?
+  if (pending_map.epoch == 1) {
+    mon.prepare_new_fingerprint(t);
+  }
+
+  //health
+  health_check_map_t next;
+  pending_map.check_health(&next);
+  encode_health(next, t);
+}
+
+class C_ApplyFeatures : public Context {
+  MonmapMonitor *svc;
+  mon_feature_t features;
+  ceph_release_t min_mon_release;
+public:
+  C_ApplyFeatures(MonmapMonitor *s, const mon_feature_t& f, ceph_release_t mmr) :
+    svc(s), features(f), min_mon_release(mmr) { }
+  void finish(int r) override {
+    if (r >= 0) {
+      svc->apply_mon_features(features, min_mon_release);
+    } else if (r == -EAGAIN || r == -ECANCELED) {
+      // discard features if we're no longer on the quorum that
+      // established them in the first place.
+      return;
+    } else {
+      ceph_abort_msg("bad C_ApplyFeatures return value");
+    }
+  }
+};
+
+void MonmapMonitor::apply_mon_features(const mon_feature_t& features,
+				       ceph_release_t min_mon_release)
+{
+  if (!is_writeable()) {
+    dout(5) << __func__ << " wait for service to be writeable" << dendl;
+    wait_for_writeable_ctx(new C_ApplyFeatures(this, features, min_mon_release));
+    return;
+  }
+
+  // do nothing here unless we have a full quorum
+  if (mon.get_quorum().size() < mon.monmap->size()) {
+    return;
+  }
+
+  ceph_assert(is_writeable());
+  ceph_assert(features.contains_all(pending_map.persistent_features));
+  // we should never hit this because `features` should be the result
+  // of the quorum's supported features. But if it happens, die.
+  ceph_assert(ceph::features::mon::get_supported().contains_all(features));
+
+  mon_feature_t new_features =
+    (pending_map.persistent_features ^
+     (features & ceph::features::mon::get_persistent()));
+
+  if (new_features.empty() &&
+      pending_map.min_mon_release == min_mon_release) {
+    dout(10) << __func__ << " min_mon_release (" << (int)min_mon_release
+	     << ") and features (" << features << ") match" << dendl;
+    return;
+  }
+
+  if (!new_features.empty()) {
+    dout(1) << __func__ << " applying new features "
+	    << new_features << ", had " << pending_map.persistent_features
+	    << ", will have "
+	    << (new_features | pending_map.persistent_features)
+	    << dendl;
+    pending_map.persistent_features |= new_features;
+  }
+  if (min_mon_release > pending_map.min_mon_release) {
+    dout(1) << __func__ << " increasing min_mon_release to "
+	    << to_integer<int>(min_mon_release) << " (" << min_mon_release
+	    << ")" << dendl;
+    pending_map.min_mon_release = min_mon_release;
+  }
+
+  propose_pending();
+}
+
+void MonmapMonitor::on_active()
+{
+  if (get_last_committed() >= 1 && !mon.has_ever_joined) {
+    // make note of the fact that i was, once, part of the quorum.
+    dout(10) << "noting that i was, once, part of an active quorum." << dendl;
+
+    /* This is some form of nasty in-breeding we have between the MonmapMonitor
+       and the Monitor itself. We should find a way to get rid of it given our
+       new architecture. Until then, stick with it since we are a
+       single-threaded process and, truth be told, no one else relies on this
+       thing besides us.
+     */
+    auto t(std::make_shared<MonitorDBStore::Transaction>());
+    t->put(Monitor::MONITOR_NAME, "joined", 1);
+    mon.store->apply_transaction(t);
+    mon.has_ever_joined = true;
+  }
+
+  if (mon.is_leader()) {
+    mon.clog->debug() << "monmap " << *mon.monmap;
+  }
+
+  apply_mon_features(mon.get_quorum_mon_features(),
+		     mon.quorum_min_mon_release);
+
+  mon.update_pending_metadata();
+}
+
+bool MonmapMonitor::preprocess_query(MonOpRequestRef op)
+{
+  auto m = op->get_req<PaxosServiceMessage>();
+  switch (m->get_type()) {
+    // READs
+  case MSG_MON_COMMAND:
+    try {
+      return preprocess_command(op);
+    }
+    catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+  case MSG_MON_JOIN:
+    return preprocess_join(op);
+  default:
+    ceph_abort();
+    return true;
+  }
+}
+
+void MonmapMonitor::dump_info(Formatter *f)
+{
+  f->dump_unsigned("monmap_first_committed", get_first_committed());
+  f->dump_unsigned("monmap_last_committed", get_last_committed());
+  f->open_object_section("monmap");
+  mon.monmap->dump(f);
+  f->close_section();
+  f->open_array_section("quorum");
+  for (set<int>::iterator q = mon.get_quorum().begin(); q != mon.get_quorum().end(); ++q)
+    f->dump_int("mon", *q);
+  f->close_section();
+}
+
+bool MonmapMonitor::preprocess_command(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonCommand>();
+  int r = -1;
+  bufferlist rdata;
+  stringstream ss;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    mon.reply_command(op, -EACCES, "access denied", get_last_committed());
+    return true;
+  }
+
+  string format;
+  cmd_getval(cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+  if (prefix == "mon stat") {
+    if (f) {
+      f->open_object_section("monmap");
+      mon.monmap->dump_summary(f.get());
+      f->dump_string("leader", mon.get_leader_name());
+      f->open_array_section("quorum");
+      for (auto rank: mon.get_quorum()) {
+        std::string name = mon.monmap->get_name(rank);
+        f->open_object_section("mon");
+        f->dump_int("rank", rank);
+        f->dump_string("name", name);
+        f->close_section();  // mon
+      }
+      f->close_section();  // quorum
+      f->close_section();  // monmap
+      f->flush(ss);
+    } else {
+      mon.monmap->print_summary(ss);
+      ss << ", election epoch " << mon.get_epoch() << ", leader "
+         << mon.get_leader() << " " << mon.get_leader_name()
+         << ", quorum " << mon.get_quorum()
+         << " " << mon.get_quorum_names();
+    }
+
+    rdata.append(ss);
+    ss.str("");
+    r = 0;
+
+  } else if (prefix == "mon getmap" ||
+             prefix == "mon dump") {
+
+    epoch_t epoch;
+    int64_t epochnum;
+    cmd_getval(cmdmap, "epoch", epochnum, (int64_t)0);
+    epoch = epochnum;
+
+    MonMap *p = mon.monmap;
+    if (epoch) {
+      bufferlist bl;
+      r = get_version(epoch, bl);
+      if (r == -ENOENT) {
+        ss << "there is no map for epoch " << epoch;
+        goto reply;
+      }
+      ceph_assert(r == 0);
+      ceph_assert(bl.length() > 0);
+      p = new MonMap;
+      p->decode(bl);
+    }
+
+    ceph_assert(p);
+
+    if (prefix == "mon getmap") {
+      p->encode(rdata, m->get_connection()->get_features());
+      r = 0;
+      ss << "got monmap epoch " << p->get_epoch();
+    } else if (prefix == "mon dump") {
+      stringstream ds;
+      if (f) {
+        f->open_object_section("monmap");
+        p->dump(f.get());
+        f->open_array_section("quorum");
+        for (set<int>::iterator q = mon.get_quorum().begin();
+            q != mon.get_quorum().end(); ++q) {
+          f->dump_int("mon", *q);
+        }
+        f->close_section();
+        f->close_section();
+        f->flush(ds);
+        r = 0;
+      } else {
+        p->print(ds);
+        r = 0;
+      }
+      rdata.append(ds);
+      ss << "dumped monmap epoch " << p->get_epoch();
+    }
+    if (p != mon.monmap) {
+       delete p;
+       p = nullptr;
+    }
+
+  } else if (prefix == "mon feature ls") {
+   
+    bool list_with_value = false;
+    string with_value;
+    if (cmd_getval(cmdmap, "with_value", with_value) &&
+        with_value == "--with-value") {
+      list_with_value = true;
+    }
+
+    MonMap *p = mon.monmap;
+
+    // list features
+    mon_feature_t supported = ceph::features::mon::get_supported();
+    mon_feature_t persistent = ceph::features::mon::get_persistent();
+    mon_feature_t required = p->get_required_features();
+
+    stringstream ds;
+    auto print_feature = [&](mon_feature_t& m_features, const char* m_str) {
+      if (f) {
+        if (list_with_value)
+          m_features.dump_with_value(f.get(), m_str);
+        else
+          m_features.dump(f.get(), m_str);
+      } else {
+        if (list_with_value)
+          m_features.print_with_value(ds);
+        else
+          m_features.print(ds);
+      }
+    };
+
+    if (f) {
+      f->open_object_section("features");
+
+      f->open_object_section("all");
+      print_feature(supported, "supported");
+      print_feature(persistent, "persistent");
+      f->close_section(); // all
+
+      f->open_object_section("monmap");
+      print_feature(p->persistent_features, "persistent");
+      print_feature(p->optional_features, "optional");
+      print_feature(required, "required");
+      f->close_section(); // monmap 
+
+      f->close_section(); // features
+      f->flush(ds);
+
+    } else {
+      ds << "all features" << std::endl
+        << "\tsupported: ";
+      print_feature(supported, nullptr);
+      ds << std::endl
+        << "\tpersistent: ";
+      print_feature(persistent, nullptr);
+      ds << std::endl
+        << std::endl;
+
+      ds << "on current monmap (epoch "
+         << p->get_epoch() << ")" << std::endl
+         << "\tpersistent: ";
+      print_feature(p->persistent_features, nullptr);
+      ds << std::endl
+        // omit optional features in plain-text
+        // makes it easier to read, and they're, currently, empty.
+	 << "\trequired: ";
+      print_feature(required, nullptr);
+      ds << std::endl;
+    }
+    rdata.append(ds);
+    r = 0;
+  }
+
+reply:
+  if (r != -1) {
+    string rs;
+    getline(ss, rs);
+
+    mon.reply_command(op, r, rs, rdata, get_last_committed());
+    return true;
+  } else
+    return false;
+}
+
+
+bool MonmapMonitor::prepare_update(MonOpRequestRef op)
+{
+  auto m = op->get_req<PaxosServiceMessage>();
+  dout(7) << __func__ << " " << *m << " from " << m->get_orig_source_inst() << dendl;
+  
+  switch (m->get_type()) {
+  case MSG_MON_COMMAND:
+    try {
+      return prepare_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+  case MSG_MON_JOIN:
+    return prepare_join(op);
+  default:
+    ceph_abort();
+  }
+
+  return false;
+}
+
+bool MonmapMonitor::prepare_command(MonOpRequestRef op)
+{
+  auto m = op->get_req<MMonCommand>();
+  stringstream ss;
+  string rs;
+  int err = -EINVAL;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    mon.reply_command(op, -EACCES, "access denied", get_last_committed());
+    return true;
+  }
+
+  /* We should follow the following rules:
+   *
+   * - 'monmap' is the current, consistent version of the monmap
+   * - 'pending_map' is the uncommitted version of the monmap
+   *
+   * All checks for the current state must be made against 'monmap'.
+   * All changes are made against 'pending_map'.
+   *
+   * If there are concurrent operations modifying 'pending_map', please
+   * follow the following rules.
+   *
+   * - if pending_map has already been changed, the second operation must
+   *   wait for the proposal to finish and be run again; This is the easiest
+   *   path to guarantee correctness but may impact performance (i.e., it
+   *   will take longer for the user to get a reply).
+   *
+   * - if the result of the second operation can be guaranteed to be
+   *   idempotent, the operation may reply to the user once the proposal
+   *   finishes; still needs to wait for the proposal to finish.
+   *
+   * - An operation _NEVER_ returns to the user based on pending state.
+   *
+   * If an operation does not modify current stable monmap, it may be
+   * serialized before current pending map, regardless of any change that
+   * has been made to the pending map -- remember, pending is uncommitted
+   * state, thus we are not bound by it.
+   */
+
+  ceph_assert(mon.monmap);
+  MonMap &monmap = *mon.monmap;
+
+
+  /* Please note:
+   *
+   * Adding or removing monitors may lead to loss of quorum.
+   *
+   * Because quorum may be lost, it's important to reply something
+   * to the user, lest she end up waiting forever for a reply. And
+   * no reply will ever be sent until quorum is formed again.
+   *
+   * On the other hand, this means we're leaking uncommitted state
+   * to the user. As such, please be mindful of the reply message.
+   *
+   * e.g., 'adding monitor mon.foo' is okay ('adding' is an on-going
+   * operation and conveys its not-yet-permanent nature); whereas
+   * 'added monitor mon.foo' presumes the action has successfully
+   * completed and state has been committed, which may not be true.
+   */
+
+
+  bool propose = false;
+  if (prefix == "mon add") {
+    string name;
+    cmd_getval(cmdmap, "name", name);
+    string addrstr;
+    cmd_getval(cmdmap, "addr", addrstr);
+    entity_addr_t addr;
+    bufferlist rdata;
+
+    if (!addr.parse(addrstr.c_str())) {
+      err = -EINVAL;
+      ss << "addr " << addrstr << "does not parse";
+      goto reply;
+    }
+
+    vector<string> locationvec;
+    map<string, string> loc;
+    cmd_getval(cmdmap, "location", locationvec);
+    CrushWrapper::parse_loc_map(locationvec, &loc);
+    if (locationvec.size() &&
+	!mon.get_quorum_mon_features().contains_all(
+				        ceph::features::mon::FEATURE_PINGING)) {
+      err = -ENOTSUP;
+      ss << "Not all monitors support adding monitors with a location; please upgrade first!";
+      goto reply;
+    }
+    if (locationvec.size() && !loc.size()) {
+      ss << "We could not parse your input location to anything real; " << locationvec
+	 << " turned into an empty map!";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    dout(10) << "mon add setting location for " << name << " to " << loc << dendl;
+
+    // TODO: validate location in crush map
+    if (monmap.stretch_mode_enabled && !loc.size()) {
+      ss << "We are in stretch mode and new monitors must have a location, but "
+	 << "could not parse your input location to anything real; " << locationvec
+	 << " turned into an empty map!";
+      err = -EINVAL;
+      goto reply;
+    }
+    // TODO: validate location against any existing stretch config
+
+    entity_addrvec_t addrs;
+    if (monmap.persistent_features.contains_all(
+	  ceph::features::mon::FEATURE_NAUTILUS)) {
+      if (addr.get_port() == CEPH_MON_PORT_IANA) {
+	addr.set_type(entity_addr_t::TYPE_MSGR2);
+      }
+      if (addr.get_port() == CEPH_MON_PORT_LEGACY) {
+	// if they specified the *old* default they probably don't care
+	addr.set_port(0);
+      }
+      if (addr.get_port()) {
+	addrs.v.push_back(addr);
+      } else {
+	addr.set_type(entity_addr_t::TYPE_MSGR2);
+	addr.set_port(CEPH_MON_PORT_IANA);
+	addrs.v.push_back(addr);
+	addr.set_type(entity_addr_t::TYPE_LEGACY);
+	addr.set_port(CEPH_MON_PORT_LEGACY);
+	addrs.v.push_back(addr);
+      }
+    } else {
+      if (addr.get_port() == 0) {
+	addr.set_port(CEPH_MON_PORT_LEGACY);
+      }
+      addr.set_type(entity_addr_t::TYPE_LEGACY);
+      addrs.v.push_back(addr);
+    }
+    dout(20) << __func__ << " addr " << addr << " -> addrs " << addrs << dendl;
+
+    /**
+     * If we have a monitor with the same name and different addr, then EEXIST
+     * If we have a monitor with the same addr and different name, then EEXIST
+     * If we have a monitor with the same addr and same name, then wait for
+     * the proposal to finish and return success.
+     * If we don't have the monitor, add it.
+     */
+
+    err = 0;
+    if (!ss.str().empty())
+      ss << "; ";
+
+    do {
+      if (monmap.contains(name)) {
+        if (monmap.get_addrs(name) == addrs) {
+          // stable map contains monitor with the same name at the same address.
+          // serialize before current pending map.
+          err = 0; // for clarity; this has already been set above.
+          ss << "mon." << name << " at " << addrs << " already exists";
+          goto reply;
+        } else {
+          ss << "mon." << name
+             << " already exists at address " << monmap.get_addrs(name);
+        }
+      } else if (monmap.contains(addrs)) {
+        // we established on the previous branch that name is different
+        ss << "mon." << monmap.get_name(addrs)
+           << " already exists at address " << addr;
+      } else {
+        // go ahead and add
+        break;
+      }
+      err = -EEXIST;
+      goto reply;
+    } while (false);
+
+    if (pending_map.stretch_mode_enabled) {
+      
+    }
+    
+    /* Given there's no delay between proposals on the MonmapMonitor (see
+     * MonmapMonitor::should_propose()), there is no point in checking for
+     * a mismatch between name and addr on pending_map.
+     *
+     * Once we established the monitor does not exist in the committed state,
+     * we can simply go ahead and add the monitor.
+     */
+
+    pending_map.add(name, addrs);
+    pending_map.mon_info[name].crush_loc = loc;
+    pending_map.last_changed = ceph_clock_now();
+    ss << "adding mon." << name << " at " << addrs;
+    propose = true;
+    dout(0) << __func__ << " proposing new mon." << name << dendl;
+
+  } else if (prefix == "mon remove" ||
+             prefix == "mon rm") {
+    string name;
+    cmd_getval(cmdmap, "name", name);
+    if (!monmap.contains(name)) {
+      err = 0;
+      ss << "mon." << name << " does not exist or has already been removed";
+      goto reply;
+    }
+
+    if (monmap.size() == 1) {
+      err = -EINVAL;
+      ss << "error: refusing removal of last monitor " << name;
+      goto reply;
+    }
+
+    if (pending_map.stretch_mode_enabled &&
+	name == pending_map.tiebreaker_mon) {
+      err = -EINVAL;
+      ss << "you cannot remove stretch mode's tiebreaker monitor";
+      goto reply;
+    }
+    /* At the time of writing, there is no risk of races when multiple clients
+     * attempt to use the same name. The reason is simple but may not be
+     * obvious.
+     *
+     * In a nutshell, we do not collate proposals on the MonmapMonitor. As
+     * soon as we return 'true' below, PaxosService::dispatch() will check if
+     * the service should propose, and - if so - the service will be marked as
+     * 'proposing' and a proposal will be triggered. The PaxosService class
+     * guarantees that once a service is marked 'proposing' no further writes
+     * will be handled.
+     *
+     * The decision on whether the service should propose or not is, in this
+     * case, made by MonmapMonitor::should_propose(), which always considers
+     * the proposal delay being 0.0 seconds. This is key for PaxosService to
+     * trigger the proposal immediately.
+     * 0.0 seconds of delay.
+     *
+     * From the above, there's no point in performing further checks on the
+     * pending_map, as we don't ever have multiple proposals in-flight in
+     * this service. As we've established the committed state contains the
+     * monitor, we can simply go ahead and remove it.
+     *
+     * Please note that the code hinges on all of the above to be true. It
+     * has been true since time immemorial and we don't see a good reason
+     * to make it sturdier at this time - mainly because we don't think it's
+     * going to change any time soon, lest for any bug that may be unwillingly
+     * introduced.
+     */
+
+    entity_addrvec_t addrs = pending_map.get_addrs(name);
+    pending_map.remove(name);
+    pending_map.disallowed_leaders.erase(name);
+    pending_map.last_changed = ceph_clock_now();
+    propose = true;
+    err = 0;
+
+  } else if (prefix == "mon feature set") {
+
+    /* PLEASE NOTE:
+     *
+     * We currently only support setting/unsetting persistent features.
+     * This is by design, given at the moment we still don't have optional
+     * features, and, as such, there is no point introducing an interface
+     * to manipulate them. This allows us to provide a cleaner, more
+     * intuitive interface to the user, modifying solely persistent
+     * features.
+     *
+     * In the future we should consider adding another interface to handle
+     * optional features/flags; e.g., 'mon feature flag set/unset', or
+     * 'mon flag set/unset'.
+     */
+    string feature_name;
+    if (!cmd_getval(cmdmap, "feature_name", feature_name)) {
+      ss << "missing required feature name";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    mon_feature_t feature;
+    feature = ceph::features::mon::get_feature_by_name(feature_name);
+    if (feature == ceph::features::mon::FEATURE_NONE) {
+      ss << "unknown feature '" << feature_name << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+    if (!sure) {
+      ss << "please specify '--yes-i-really-mean-it' if you "
+         << "really, **really** want to set feature '"
+         << feature << "' in the monmap.";
+      err = -EPERM;
+      goto reply;
+    }
+
+    if (!mon.get_quorum_mon_features().contains_all(feature)) {
+      ss << "current quorum does not support feature '" << feature
+         << "'; supported features: "
+         << mon.get_quorum_mon_features();
+      err = -EINVAL;
+      goto reply;
+    }
+
+    ss << "setting feature '" << feature << "'";
+
+    err = 0;
+    if (monmap.persistent_features.contains_all(feature)) {
+      dout(10) << __func__ << " feature '" << feature
+               << "' already set on monmap; no-op." << dendl;
+      goto reply;
+    }
+
+    pending_map.persistent_features.set_feature(feature);
+    pending_map.last_changed = ceph_clock_now();
+    propose = true;
+
+    dout(1) << __func__ << " " << ss.str() << "; new features will be: "
+            << "persistent = " << pending_map.persistent_features
+            // output optional nevertheless, for auditing purposes.
+            << ", optional = " << pending_map.optional_features << dendl;
+
+  } else if (prefix == "mon set-rank") {
+    string name;
+    int64_t rank;
+    if (!cmd_getval(cmdmap, "name", name) ||
+	!cmd_getval(cmdmap, "rank", rank)) {
+      err = -EINVAL;
+      goto reply;
+    }
+    int oldrank = pending_map.get_rank(name);
+    if (oldrank < 0) {
+      ss << "mon." << name << " does not exist in monmap";
+      err = -ENOENT;
+      goto reply;
+    }
+    err = 0;
+    pending_map.set_rank(name, rank);
+    pending_map.last_changed = ceph_clock_now();
+    propose = true;
+  } else if (prefix == "mon set-addrs") {
+    string name;
+    string addrs;
+    if (!cmd_getval(cmdmap, "name", name) ||
+	!cmd_getval(cmdmap, "addrs", addrs)) {
+      err = -EINVAL;
+      goto reply;
+    }
+    if (!pending_map.contains(name)) {
+      ss << "mon." << name << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+    entity_addrvec_t av;
+    if (!av.parse(addrs.c_str(), nullptr)) {
+      ss << "failed to parse addrs '" << addrs << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    for (auto& a : av.v) {
+      a.set_nonce(0);
+      if (!a.get_port()) {
+	ss << "monitor must bind to a non-zero port, not " << a;
+	err = -EINVAL;
+	goto reply;
+      }
+    }
+    err = 0;
+    pending_map.set_addrvec(name, av);
+    pending_map.last_changed = ceph_clock_now();
+    propose = true;
+  } else if (prefix == "mon set-weight") {
+    string name;
+    int64_t weight;
+    if (!cmd_getval(cmdmap, "name", name) ||
+        !cmd_getval(cmdmap, "weight", weight)) {
+      err = -EINVAL;
+      goto reply;
+    }
+    if (!pending_map.contains(name)) {
+      ss << "mon." << name << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+    err = 0;
+    pending_map.set_weight(name, weight);
+    pending_map.last_changed = ceph_clock_now();
+    propose = true;
+  } else if (prefix == "mon enable-msgr2") {
+    if (!monmap.get_required_features().contains_all(
+	  ceph::features::mon::FEATURE_NAUTILUS)) {
+      err = -EACCES;
+      ss << "all monitors must be running nautilus to enable v2";
+      goto reply;
+    }
+    for (auto& i : pending_map.mon_info) {
+      if (i.second.public_addrs.v.size() == 1 &&
+	  i.second.public_addrs.front().is_legacy() &&
+	  i.second.public_addrs.front().get_port() == CEPH_MON_PORT_LEGACY) {
+	entity_addrvec_t av;
+	entity_addr_t a = i.second.public_addrs.front();
+	a.set_type(entity_addr_t::TYPE_MSGR2);
+	a.set_port(CEPH_MON_PORT_IANA);
+	av.v.push_back(a);
+	av.v.push_back(i.second.public_addrs.front());
+	dout(10) << " setting mon." << i.first
+		 << " addrs " << i.second.public_addrs
+		 << " -> " << av << dendl;
+	pending_map.set_addrvec(i.first, av);
+	propose = true;
+	pending_map.last_changed = ceph_clock_now();
+      }
+    }
+    err = 0;
+  } else if (prefix == "mon set election_strategy") {
+    if (!mon.get_quorum_mon_features().contains_all(
+				        ceph::features::mon::FEATURE_PINGING)) {
+      err = -ENOTSUP;
+      ss << "Not all monitors support changing election strategies; please upgrade first!";
+      goto reply;
+    }
+    string strat;
+    MonMap::election_strategy strategy;
+    if (!cmd_getval(cmdmap, "strategy", strat)) {
+      err = -EINVAL;
+      goto reply;
+    }
+    if (strat == "classic") {
+      strategy = MonMap::CLASSIC;
+    } else if (strat == "disallow") {
+      strategy = MonMap::DISALLOW;
+    } else if (strat == "connectivity") {
+      strategy = MonMap::CONNECTIVITY;
+    } else {
+      err = -EINVAL;
+      goto reply;
+    }
+    err = 0;
+    pending_map.strategy = strategy;
+    pending_map.last_changed = ceph_clock_now();
+    propose = true;
+  } else if (prefix == "mon add disallowed_leader") {
+    if (!mon.get_quorum_mon_features().contains_all(
+				        ceph::features::mon::FEATURE_PINGING)) {
+      err = -ENOTSUP;
+      ss << "Not all monitors support changing election strategies; please upgrade first!";
+      goto reply;
+    }
+    string name;
+    if (!cmd_getval(cmdmap, "name", name)) {
+      err = -EINVAL;
+      goto reply;
+    }
+    if (pending_map.strategy != MonMap::DISALLOW &&
+	pending_map.strategy != MonMap::CONNECTIVITY) {
+      ss << "You cannot disallow monitors in your current election mode";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (!pending_map.contains(name)) {
+      ss << "mon." << name << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+    if (pending_map.disallowed_leaders.count(name)) {
+      ss << "mon." << name << " is already disallowed";
+      err = 0;
+      goto reply;
+    }
+    if (pending_map.disallowed_leaders.size() == pending_map.size() - 1) {
+      ss << "mon." << name << " is the only remaining allowed leader!";
+      err = -EINVAL;
+      goto reply;
+    }
+    pending_map.disallowed_leaders.insert(name);
+    pending_map.last_changed = ceph_clock_now();
+    err = 0;
+    propose = true;
+  } else if (prefix == "mon rm disallowed_leader") {
+    if (!mon.get_quorum_mon_features().contains_all(
+				        ceph::features::mon::FEATURE_PINGING)) {
+      err = -ENOTSUP;
+      ss << "Not all monitors support changing election strategies; please upgrade first!";
+      goto reply;
+    }
+    string name;
+    if (!cmd_getval(cmdmap, "name", name)) {
+      err = -EINVAL;
+      goto reply;
+    }
+    if (pending_map.strategy != MonMap::DISALLOW &&
+	pending_map.strategy != MonMap::CONNECTIVITY) {
+      ss << "You cannot disallow monitors in your current election mode";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (!pending_map.contains(name)) {
+      ss << "mon." << name << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+    if (!pending_map.disallowed_leaders.count(name)) {
+      ss << "mon." << name << " is already allowed";
+      err = 0;
+      goto reply;
+    }
+    pending_map.disallowed_leaders.erase(name);
+    pending_map.last_changed = ceph_clock_now();
+    err = 0;
+    propose = true;
+  } else if (prefix == "mon set_location") {
+    if (!mon.get_quorum_mon_features().contains_all(
+				        ceph::features::mon::FEATURE_PINGING)) {
+      err = -ENOTSUP;
+      ss << "Not all monitors support monitor locations; please upgrade first!";
+      goto reply;
+    }
+    string name;
+    if (!cmd_getval(cmdmap, "name", name)) {
+      err = -EINVAL;
+      goto reply;
+    }
+    if (!pending_map.contains(name)) {
+      ss << "mon." << name << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+
+    vector<string> argvec;
+    map<string, string> loc;
+    cmd_getval(cmdmap, "args", argvec);
+    CrushWrapper::parse_loc_map(argvec, &loc);
+
+    dout(10) << "mon set_location for " << name << " to " << loc << dendl;
+
+    // TODO: validate location in crush map
+    if (!loc.size()) {
+      ss << "We could not parse your input location to anything real; " << argvec
+	 << " turned into an empty map!";
+      err = -EINVAL;
+      goto reply;
+    }
+    // TODO: validate location against any existing stretch config
+    pending_map.mon_info[name].crush_loc = loc;
+    pending_map.last_changed = ceph_clock_now();
+    err = 0;
+    propose = true;
+  } else if (prefix == "mon set_new_tiebreaker") {
+    if (!pending_map.stretch_mode_enabled) {
+      err = -EINVAL;
+      ss << "Stretch mode is not enabled, so there is no tiebreaker";
+      goto reply;
+    }
+    string name;
+    if (!cmd_getval(cmdmap, "name", name)) {
+      err = -EINVAL;
+      goto reply;
+    }
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+
+    const auto &existing_tiebreaker_info_i = pending_map.mon_info.find(pending_map.tiebreaker_mon);
+    const auto &new_tiebreaker_info_i = pending_map.mon_info.find(name);
+    if (new_tiebreaker_info_i == pending_map.mon_info.end()) {
+      ss << "mon." << name << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+    const auto& new_info = new_tiebreaker_info_i->second;
+    if (new_info.crush_loc.empty()) {
+      ss << "mon." << name << " does not have a location specified";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    if (!mon.osdmon()->is_readable()) {
+      dout(10) << __func__
+	       << ": waiting for osdmon readable to inspect crush barrier"
+	       << dendl;
+      mon.osdmon()->wait_for_readable(op, new Monitor::C_RetryMessage(&mon, op));
+      return false;
+    }
+    int32_t stretch_divider_id = mon.osdmon()->osdmap.stretch_mode_bucket;
+    string stretch_bucket_divider = mon.osdmon()->osdmap.crush->
+      get_type_name(stretch_divider_id);
+
+    const auto& new_loc_i = new_info.crush_loc.find(stretch_bucket_divider);
+    if (new_loc_i == new_info.crush_loc.end()) {
+      ss << "mon." << name << " has a specificed location, but not a "
+	 << stretch_bucket_divider << ", which is the stretch divider";
+      err = -EINVAL;
+      goto reply;
+    }
+    const string& new_loc = new_loc_i->second;
+    set<string> matching_mons;
+    for (const auto& mii : pending_map.mon_info) {
+      const auto& other_loc_i = mii.second.crush_loc.find(stretch_bucket_divider);
+      if (mii.first == name) {
+	continue;
+      }
+      if (other_loc_i == mii.second.crush_loc.end()) { // huh
+	continue;
+      }
+      const string& other_loc = other_loc_i->second;
+      if (other_loc == new_loc &&
+	  mii.first != existing_tiebreaker_info_i->first) {
+	matching_mons.insert(mii.first);
+      }
+    }
+    if (!matching_mons.empty()) {
+      ss << "mon." << name << " has location " << new_loc_i->second
+	 << ", which matches mons " << matching_mons << " on the "
+	 << stretch_bucket_divider << " dividing bucket for stretch mode. "
+	"Pass --yes-i-really-mean-it if you're sure you want to do this."
+	"(You really don't.)";
+      err = -EINVAL;
+      goto reply;
+    }
+    pending_map.tiebreaker_mon = name;
+    pending_map.disallowed_leaders.insert(name);
+    pending_map.last_changed = ceph_clock_now();
+    err = 0;
+    propose = true;
+  } else if (prefix == "mon enable_stretch_mode") {
+    if (!mon.osdmon()->is_writeable()) {
+      dout(10) << __func__
+	      << ":  waiting for osdmon writeable for stretch mode" << dendl;
+      mon.osdmon()->wait_for_writeable(op, new Monitor::C_RetryMessage(&mon, op));
+      return false;
+    }
+    {
+      if (monmap.stretch_mode_enabled) {
+	ss << "stretch mode is already engaged";
+	err = -EINVAL;
+	goto reply;
+      }
+      if (pending_map.stretch_mode_enabled) {
+	ss << "stretch mode currently committing";
+	err = 0;
+	goto reply;
+      }
+      string tiebreaker_mon;
+      if (!cmd_getval(cmdmap, "tiebreaker_mon", tiebreaker_mon)) {
+	ss << "must specify a tiebreaker monitor";
+	err = -EINVAL;
+	goto reply;
+      }
+      string new_crush_rule;
+      if (!cmd_getval(cmdmap, "new_crush_rule", new_crush_rule)) {
+	ss << "must specify a new crush rule that spreads out copies over multiple sites";
+	err = -EINVAL;
+	goto reply;
+      }
+      string dividing_bucket;
+      if (!cmd_getval(cmdmap, "dividing_bucket", dividing_bucket)) {
+	ss << "must specify a dividing bucket";
+	err = -EINVAL;
+	goto reply;
+      }
+      //okay, initial arguments make sense, check pools and cluster state
+      err = mon.osdmon()->check_cluster_features(CEPH_FEATUREMASK_STRETCH_MODE, ss);
+      if (err)
+	goto reply;
+      struct Plugger {
+	Paxos &p;
+	Plugger(Paxos &p) : p(p) { p.plug(); }
+	~Plugger() { p.unplug(); }
+      } plugger(paxos);
+
+      set<pg_pool_t*> pools;
+      bool okay = false;
+      int errcode = 0;
+
+      mon.osdmon()->try_enable_stretch_mode_pools(ss, &okay, &errcode,
+						   &pools, new_crush_rule);
+      if (!okay) {
+	err = errcode;
+	goto reply;
+      }
+      try_enable_stretch_mode(ss, &okay, &errcode, false,
+			      tiebreaker_mon, dividing_bucket);
+      if (!okay) {
+	err = errcode;
+	goto reply;
+      }
+      mon.osdmon()->try_enable_stretch_mode(ss, &okay, &errcode, false,
+					     dividing_bucket, 2, pools, new_crush_rule);
+      if (!okay) {
+	err = errcode;
+	goto reply;
+      }
+      // everything looks good, actually commit the changes!
+      try_enable_stretch_mode(ss, &okay, &errcode, true,
+			      tiebreaker_mon, dividing_bucket);
+      mon.osdmon()->try_enable_stretch_mode(ss, &okay, &errcode, true,
+					     dividing_bucket,
+					     2, // right now we only support 2 sites
+					     pools, new_crush_rule);
+      ceph_assert(okay == true);
+    }
+    request_proposal(mon.osdmon());
+    err = 0;
+    propose = true;
+  } else {
+    ss << "unknown command " << prefix;
+    err = -EINVAL;
+  }
+
+reply:
+  getline(ss, rs);
+  mon.reply_command(op, err, rs, get_last_committed());
+  // we are returning to the user; do not propose.
+  return propose;
+}
+
+void MonmapMonitor::try_enable_stretch_mode(stringstream& ss, bool *okay,
+					    int *errcode, bool commit,
+					    const string& tiebreaker_mon,
+					    const string& dividing_bucket)
+{
+  dout(20) << __func__ << dendl;
+  *okay = false;
+  if (pending_map.strategy != MonMap::CONNECTIVITY) {
+    ss << "Monitors must use the connectivity strategy to enable stretch mode";
+    *errcode = -EINVAL;
+    ceph_assert(!commit);
+    return;
+  }
+  if (!pending_map.contains(tiebreaker_mon)) {
+    ss << "mon " << tiebreaker_mon << "does not seem to exist";
+    *errcode = -ENOENT;
+    ceph_assert(!commit);
+    return;
+  }
+  map<string,string> buckets;
+  for (const auto&mii : mon.monmap->mon_info) {
+    const auto& mi = mii.second;
+    const auto& bi = mi.crush_loc.find(dividing_bucket);
+    if (bi == mi.crush_loc.end()) {
+      ss << "Could not find location entry for " << dividing_bucket
+	 << " on monitor " << mi.name;
+      *errcode = -EINVAL;
+      ceph_assert(!commit);
+      return;
+    }
+    buckets[mii.first] = bi->second;
+  }
+  string bucket1, bucket2, tiebreaker_bucket;
+  for (auto& i : buckets) {
+    if (i.first == tiebreaker_mon) {
+      tiebreaker_bucket = i.second;
+      continue;
+    }
+    if (bucket1.empty()) {
+      bucket1 = i.second;
+    }
+    if (bucket1 != i.second &&
+	bucket2.empty()) {
+      bucket2 = i.second;
+    }
+    if (bucket1 != i.second &&
+	bucket2 != i.second) {
+      ss << "There are too many monitor buckets for stretch mode, found "
+	 << bucket1 << "," << bucket2 << "," << i.second;
+      *errcode = -EINVAL;
+      ceph_assert(!commit);
+      return;
+    }
+  }
+  if (bucket1.empty() || bucket2.empty()) {
+    ss << "There are not enough monitor buckets for stretch mode;"
+       << " must have at least 2 plus the tiebreaker but only found "
+       << (bucket1.empty() ? bucket1 : bucket2);
+    *errcode = -EINVAL;
+    ceph_assert(!commit);
+    return;
+  }
+  if (tiebreaker_bucket == bucket1 ||
+      tiebreaker_bucket == bucket2) {
+    ss << "The named tiebreaker monitor " << tiebreaker_mon
+       << " is in the same CRUSH bucket " << tiebreaker_bucket
+       << " as other monitors";
+    *errcode = -EINVAL;
+    ceph_assert(!commit);
+    return;
+  }
+  if (commit) {
+    pending_map.disallowed_leaders.insert(tiebreaker_mon);
+    pending_map.tiebreaker_mon = tiebreaker_mon;
+    pending_map.stretch_mode_enabled = true;
+  }
+  *okay = true;
+}
+
+void MonmapMonitor::trigger_degraded_stretch_mode(const set<string>& dead_mons)
+{
+  dout(20) << __func__ << dendl;
+  pending_map.stretch_marked_down_mons.insert(dead_mons.begin(), dead_mons.end());
+  propose_pending();
+}
+
+void MonmapMonitor::trigger_healthy_stretch_mode()
+{
+  dout(20) << __func__ << dendl;
+  pending_map.stretch_marked_down_mons.clear();
+  propose_pending();
+}
+
+bool MonmapMonitor::preprocess_join(MonOpRequestRef op)
+{
+  auto join = op->get_req<MMonJoin>();
+  dout(10) << __func__ << " " << join->name << " at " << join->addrs << dendl;
+
+  MonSession *session = op->get_session();
+  if (!session ||
+      !session->is_capable("mon", MON_CAP_W | MON_CAP_X)) {
+    dout(10) << " insufficient caps" << dendl;
+    return true;
+  }
+
+  const auto name_info_i = pending_map.mon_info.find(join->name);
+  if (name_info_i != pending_map.mon_info.end() &&
+      !name_info_i->second.public_addrs.front().is_blank_ip() &&
+      (!join->force_loc || join->crush_loc == name_info_i->second.crush_loc)) {
+    dout(10) << " already have " << join->name << dendl;
+    return true;
+  }
+  string addr_name;
+  if (pending_map.contains(join->addrs)) {
+    addr_name = pending_map.get_name(join->addrs);
+  }
+  if (!addr_name.empty() &&
+      addr_name == join->name &&
+      (!join->force_loc || join->crush_loc.empty() ||
+       pending_map.mon_info[addr_name].crush_loc == join->crush_loc)) {
+    dout(10) << " already have " << join->addrs << dendl;
+    return true;
+  }
+  if (pending_map.stretch_mode_enabled &&
+      join->crush_loc.empty() &&
+      (addr_name.empty() ||
+       pending_map.mon_info[addr_name].crush_loc.empty())) {
+    dout(10) << "stretch mode engaged but no source of crush_loc" << dendl;
+    mon.clog->info() << join->name << " attempted to join from " << join->name
+		      << ' ' << join->addrs
+		      << "; but lacks a crush_location for stretch mode";
+    return true;
+  }
+  return false;
+}
+
+bool MonmapMonitor::prepare_join(MonOpRequestRef op)
+{
+  auto join = op->get_req<MMonJoin>();
+  dout(0) << "adding/updating " << join->name
+	  << " at " << join->addrs << " to monitor cluster" << dendl;
+  map<string,string> existing_loc;
+  if (pending_map.contains(join->addrs)) {
+    string name = pending_map.get_name(join->addrs);
+    existing_loc = pending_map.mon_info[name].crush_loc;
+    pending_map.remove(name);
+  }
+  if (pending_map.contains(join->name))
+    pending_map.remove(join->name);
+  pending_map.add(join->name, join->addrs);
+  pending_map.mon_info[join->name].crush_loc =
+    ((join->force_loc || existing_loc.empty()) ?
+     join->crush_loc : existing_loc);
+  pending_map.last_changed = ceph_clock_now();
+  return true;
+}
+
+bool MonmapMonitor::should_propose(double& delay)
+{
+  delay = 0.0;
+  return true;
+}
+
+int MonmapMonitor::get_monmap(bufferlist &bl)
+{
+  version_t latest_ver = get_last_committed();
+  dout(10) << __func__ << " ver " << latest_ver << dendl;
+
+  if (!mon.store->exists(get_service_name(), stringify(latest_ver)))
+    return -ENOENT;
+
+  int err = get_version(latest_ver, bl);
+  if (err < 0) {
+    dout(1) << __func__ << " error obtaining monmap: "
+            << cpp_strerror(err) << dendl;
+    return err;
+  }
+  return 0;
+}
+
+void MonmapMonitor::check_subs()
+{
+  const string type = "monmap";
+  mon.with_session_map([this, &type](const MonSessionMap& session_map) {
+      auto subs = session_map.subs.find(type);
+      if (subs == session_map.subs.end())
+	return;
+      for (auto sub : *subs->second) {
+	check_sub(sub);
+      }
+    });
+}
+
+void MonmapMonitor::check_sub(Subscription *sub)
+{
+  const auto epoch = mon.monmap->get_epoch();
+  dout(10) << __func__
+	   << " monmap next " << sub->next
+	   << " have " << epoch << dendl;
+  if (sub->next <= epoch) {
+    mon.send_latest_monmap(sub->session->con.get());
+    if (sub->onetime) {
+      mon.with_session_map([sub](MonSessionMap& session_map) {
+	  session_map.remove_sub(sub);
+	});
+    } else {
+      sub->next = epoch + 1;
+    }
+  }
+}
+
+void MonmapMonitor::tick()
+{
+  if (!is_active() ||
+      !mon.is_leader()) {
+    return;
+  }
+
+  if (mon.monmap->created.is_zero()) {
+    dout(10) << __func__ << " detected empty created stamp" << dendl;
+    utime_t ctime;
+    for (version_t v = 1; v <= get_last_committed(); v++) {
+      bufferlist bl;
+      int r = get_version(v, bl);
+      if (r < 0) {
+	continue;
+      }
+      MonMap m;
+      auto p = bl.cbegin();
+      decode(m, p);
+      if (!m.last_changed.is_zero()) {
+	dout(10) << __func__ << " first monmap with last_changed is "
+		 << v << " with " << m.last_changed << dendl;
+	ctime = m.last_changed;
+	break;
+      }
+    }
+    if (ctime.is_zero()) {
+      ctime = ceph_clock_now();
+    }
+    dout(10) << __func__ << " updating created stamp to " << ctime << dendl;
+    pending_map.created = ctime;
+    propose_pending();
+  }
+}
diff --git a/src/mon/MonmapMonitor.h b/src/mon/MonmapMonitor.h
new file mode 100644
index 000000000..cf22ae9f8
--- /dev/null
+++ b/src/mon/MonmapMonitor.h
@@ -0,0 +1,111 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2009 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+/*
+ * The Monmap Monitor is used to track the monitors in the cluster.
+ */
+
+#ifndef CEPH_MONMAPMONITOR_H
+#define CEPH_MONMAPMONITOR_H
+
+#include <map>
+#include <set>
+
+#include "include/types.h"
+#include "msg/Messenger.h"
+
+#include "PaxosService.h"
+#include "MonMap.h"
+#include "MonitorDBStore.h"
+
+class MonmapMonitor : public PaxosService {
+ public:
+  MonmapMonitor(Monitor &mn, Paxos &p, const std::string& service_name)
+    : PaxosService(mn, p, service_name)
+  {
+  }
+  MonMap pending_map; //the pending map awaiting passage
+
+  void create_initial() override;
+
+  void update_from_paxos(bool *need_bootstrap) override;
+
+  void create_pending() override;
+
+  void encode_pending(MonitorDBStore::TransactionRef t) override;
+  // we always encode the full map; we have no use for full versions
+  void encode_full(MonitorDBStore::TransactionRef t) override { }
+
+  void on_active() override;
+  void apply_mon_features(const mon_feature_t& features,
+			  ceph_release_t min_mon_release);
+
+  void dump_info(ceph::Formatter *f);
+
+  bool preprocess_query(MonOpRequestRef op) override;
+  bool prepare_update(MonOpRequestRef op) override;
+
+  bool preprocess_join(MonOpRequestRef op);
+  bool prepare_join(MonOpRequestRef op);
+
+  bool preprocess_command(MonOpRequestRef op);
+  bool prepare_command(MonOpRequestRef op);
+
+  int get_monmap(ceph::buffer::list &bl);
+
+  /*
+   * Since monitors are pretty
+   * important, this implementation will just write 0.0.
+   */
+  bool should_propose(double& delay) override;
+
+  void check_sub(Subscription *sub);
+
+  void tick() override;
+
+private:
+  void check_subs();
+  ceph::buffer::list monmap_bl;
+  /**
+   * Check validity of inputs and monitor state to
+   * engage stretch mode. Designed to be used with
+   * OSDMonitor::try_enable_stretch_mode() where we call both twice,
+   * first with commit=false to validate.
+   * @param ss: a stringstream to write errors into
+   * @param okay: Filled to true if okay, false if validation fails
+   * @param errcode: filled with -errno if there's a problem
+   * @param commit: true if we should commit the change, false if just testing
+   * @param tiebreaker_mon: the name of the monitor to declare tiebreaker
+   * @param dividing_bucket: the bucket type (eg 'dc') that divides the cluster
+   */
+  void try_enable_stretch_mode(stringstream& ss, bool *okay,
+			       int *errcode, bool commit,
+			       const string& tiebreaker_mon,
+			       const string& dividing_bucket);
+
+public:
+  /**
+   * Set us to degraded stretch mode. Put the dead_mons in
+   * the MonMap.
+   */
+  void trigger_degraded_stretch_mode(const set<string>& dead_mons);
+  /**
+   * Set us to healthy stretch mode: clear out the
+   * down list to allow any non-tiebreaker mon to be the leader again.
+   */
+  void trigger_healthy_stretch_mode();
+};
+
+
+#endif
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
new file mode 100644
index 000000000..3191ed5bf
--- /dev/null
+++ b/src/mon/OSDMonitor.cc
@@ -0,0 +1,14832 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <algorithm>
+#include <boost/algorithm/string.hpp>
+#include <experimental/iterator>
+#include <locale>
+#include <sstream>
+
+#include "mon/OSDMonitor.h"
+#include "mon/Monitor.h"
+#include "mon/MDSMonitor.h"
+#include "mon/MgrStatMonitor.h"
+#include "mon/AuthMonitor.h"
+#include "mon/KVMonitor.h"
+
+#include "mon/MonitorDBStore.h"
+#include "mon/Session.h"
+
+#include "crush/CrushWrapper.h"
+#include "crush/CrushTester.h"
+#include "crush/CrushTreeDumper.h"
+
+#include "messages/MOSDBeacon.h"
+#include "messages/MOSDFailure.h"
+#include "messages/MOSDMarkMeDown.h"
+#include "messages/MOSDMarkMeDead.h"
+#include "messages/MOSDFull.h"
+#include "messages/MOSDMap.h"
+#include "messages/MMonGetOSDMap.h"
+#include "messages/MOSDBoot.h"
+#include "messages/MOSDAlive.h"
+#include "messages/MPoolOp.h"
+#include "messages/MPoolOpReply.h"
+#include "messages/MOSDPGCreate.h"
+#include "messages/MOSDPGCreate2.h"
+#include "messages/MOSDPGCreated.h"
+#include "messages/MOSDPGTemp.h"
+#include "messages/MOSDPGReadyToMerge.h"
+#include "messages/MMonCommand.h"
+#include "messages/MRemoveSnaps.h"
+#include "messages/MOSDScrub.h"
+#include "messages/MRoute.h"
+#include "messages/MMonGetPurgedSnaps.h"
+#include "messages/MMonGetPurgedSnapsReply.h"
+
+#include "common/TextTable.h"
+#include "common/Timer.h"
+#include "common/ceph_argparse.h"
+#include "common/perf_counters.h"
+#include "common/PriorityCache.h"
+#include "common/strtol.h"
+#include "common/numa.h"
+
+#include "common/config.h"
+#include "common/errno.h"
+
+#include "erasure-code/ErasureCodePlugin.h"
+#include "compressor/Compressor.h"
+#include "common/Checksummer.h"
+
+#include "include/compat.h"
+#include "include/ceph_assert.h"
+#include "include/stringify.h"
+#include "include/util.h"
+#include "common/cmdparse.h"
+#include "include/str_list.h"
+#include "include/str_map.h"
+#include "include/scope_guard.h"
+#include "perfglue/heap_profiler.h"
+
+#include "auth/cephx/CephxKeyServer.h"
+#include "osd/OSDCap.h"
+
+#include "json_spirit/json_spirit_reader.h"
+
+#include <boost/algorithm/string/predicate.hpp>
+
+using std::dec;
+using std::hex;
+using std::list;
+using std::map;
+using std::make_pair;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+
+using ceph::bufferlist;
+using ceph::decode;
+using ceph::encode;
+using ceph::ErasureCodeInterfaceRef;
+using ceph::ErasureCodePluginRegistry;
+using ceph::ErasureCodeProfile;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::make_message;
+
+#define dout_subsys ceph_subsys_mon
+static const string OSD_PG_CREATING_PREFIX("osd_pg_creating");
+static const string OSD_METADATA_PREFIX("osd_metadata");
+static const string OSD_SNAP_PREFIX("osd_snap");
+
+/*
+
+  OSD snapshot metadata
+  ---------------------
+
+  -- starting with mimic, removed in octopus --
+
+  "removed_epoch_%llu_%08lx" % (pool, epoch)
+   -> interval_set<snapid_t>
+
+  "removed_snap_%llu_%016llx" % (pool, last_snap)
+   -> { first_snap, end_snap, epoch }   (last_snap = end_snap - 1)
+
+
+  -- starting with mimic --
+
+  "purged_snap_%llu_%016llx" % (pool, last_snap)
+   -> { first_snap, end_snap, epoch }   (last_snap = end_snap - 1)
+
+  - note that the {removed,purged}_snap put the last snap in they key so
+    that we can use forward iteration only to search for an epoch in an
+    interval.  e.g., to test if epoch N is removed/purged, we'll find a key
+    >= N that either does or doesn't contain the given snap.
+
+
+  -- starting with octopus --
+
+  "purged_epoch_%08lx" % epoch
+  -> map<int64_t,interval_set<snapid_t>>
+
+  */
+using namespace TOPNSPC::common;
+namespace {
+
+struct OSDMemCache : public PriorityCache::PriCache {
+  OSDMonitor *osdmon;
+  int64_t cache_bytes[PriorityCache::Priority::LAST+1] = {0};
+  int64_t committed_bytes = 0;
+  double cache_ratio = 0;
+
+  OSDMemCache(OSDMonitor *m) : osdmon(m) {};
+
+  virtual uint64_t _get_used_bytes() const = 0;
+
+  virtual int64_t request_cache_bytes(
+      PriorityCache::Priority pri, uint64_t total_cache) const {
+    int64_t assigned = get_cache_bytes(pri);
+
+    switch (pri) {
+    // All cache items are currently set to have PRI1 priority
+    case PriorityCache::Priority::PRI1:
+      {
+        int64_t request = _get_used_bytes();
+        return (request > assigned) ? request - assigned : 0;
+      }
+    default:
+      break;
+    }
+    return -EOPNOTSUPP;
+  }
+
+  virtual int64_t get_cache_bytes(PriorityCache::Priority pri) const {
+      return cache_bytes[pri];
+  }
+
+  virtual int64_t get_cache_bytes() const {
+    int64_t total = 0;
+
+    for (int i = 0; i < PriorityCache::Priority::LAST + 1; i++) {
+      PriorityCache::Priority pri = static_cast<PriorityCache::Priority>(i);
+      total += get_cache_bytes(pri);
+    }
+    return total;
+  }
+
+  virtual void set_cache_bytes(PriorityCache::Priority pri, int64_t bytes) {
+    cache_bytes[pri] = bytes;
+  }
+  virtual void add_cache_bytes(PriorityCache::Priority pri, int64_t bytes) {
+    cache_bytes[pri] += bytes;
+  }
+  virtual int64_t commit_cache_size(uint64_t total_cache) {
+    committed_bytes = PriorityCache::get_chunk(
+        get_cache_bytes(), total_cache);
+    return committed_bytes;
+  }
+  virtual int64_t get_committed_size() const {
+    return committed_bytes;
+  }
+  virtual double get_cache_ratio() const {
+    return cache_ratio;
+  }
+  virtual void set_cache_ratio(double ratio) {
+    cache_ratio = ratio;
+  }
+  virtual string get_cache_name() const = 0;
+};
+
+struct IncCache : public OSDMemCache {
+  IncCache(OSDMonitor *m) : OSDMemCache(m) {};
+
+  virtual uint64_t _get_used_bytes() const {
+    return osdmon->inc_osd_cache.get_bytes();
+  }
+
+  virtual string get_cache_name() const {
+    return "OSDMap Inc Cache";
+  }
+
+  uint64_t _get_num_osdmaps() const {
+    return osdmon->inc_osd_cache.get_size();
+  }
+};
+
+struct FullCache : public OSDMemCache {
+  FullCache(OSDMonitor *m) : OSDMemCache(m) {};
+
+  virtual uint64_t _get_used_bytes() const {
+    return osdmon->full_osd_cache.get_bytes();
+  }
+
+  virtual string get_cache_name() const {
+    return "OSDMap Full Cache";
+  }
+
+  uint64_t _get_num_osdmaps() const {
+    return osdmon->full_osd_cache.get_size();
+  }
+};
+
+std::shared_ptr<IncCache> inc_cache;
+std::shared_ptr<FullCache> full_cache;
+
+const uint32_t MAX_POOL_APPLICATIONS = 4;
+const uint32_t MAX_POOL_APPLICATION_KEYS = 64;
+const uint32_t MAX_POOL_APPLICATION_LENGTH = 128;
+
+bool is_osd_writable(const OSDCapGrant& grant, const std::string* pool_name) {
+  // Note: this doesn't include support for the application tag match
+  if ((grant.spec.allow & OSD_CAP_W) != 0) {
+    auto& match = grant.match;
+    if (match.is_match_all()) {
+      return true;
+    } else if (pool_name != nullptr &&
+               !match.pool_namespace.pool_name.empty() &&
+               match.pool_namespace.pool_name == *pool_name) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool is_unmanaged_snap_op_permitted(CephContext* cct,
+                                    const KeyServer& key_server,
+                                    const EntityName& entity_name,
+                                    const MonCap& mon_caps,
+				    const entity_addr_t& peer_socket_addr,
+                                    const std::string* pool_name)
+{
+  typedef std::map<std::string, std::string> CommandArgs;
+
+  if (mon_caps.is_capable(
+	cct, entity_name, "osd",
+	"osd pool op unmanaged-snap",
+	(pool_name == nullptr ?
+	 CommandArgs{} /* pool DNE, require unrestricted cap */ :
+	 CommandArgs{{"poolname", *pool_name}}),
+	false, true, false,
+	peer_socket_addr)) {
+    return true;
+  }
+
+  AuthCapsInfo caps_info;
+  if (!key_server.get_service_caps(entity_name, CEPH_ENTITY_TYPE_OSD,
+                                   caps_info)) {
+    dout(10) << "unable to locate OSD cap data for " << entity_name
+             << " in auth db" << dendl;
+    return false;
+  }
+
+  string caps_str;
+  if (caps_info.caps.length() > 0) {
+    auto p = caps_info.caps.cbegin();
+    try {
+      decode(caps_str, p);
+    } catch (const ceph::buffer::error &err) {
+      derr << "corrupt OSD cap data for " << entity_name << " in auth db"
+           << dendl;
+      return false;
+    }
+  }
+
+  OSDCap osd_cap;
+  if (!osd_cap.parse(caps_str, nullptr)) {
+    dout(10) << "unable to parse OSD cap data for " << entity_name
+             << " in auth db" << dendl;
+    return false;
+  }
+
+  // if the entity has write permissions in one or all pools, permit
+  // usage of unmanaged-snapshots
+  if (osd_cap.allow_all()) {
+    return true;
+  }
+
+  for (auto& grant : osd_cap.grants) {
+    if (grant.profile.is_valid()) {
+      for (auto& profile_grant : grant.profile_grants) {
+        if (is_osd_writable(profile_grant, pool_name)) {
+          return true;
+        }
+      }
+    } else if (is_osd_writable(grant, pool_name)) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+} // anonymous namespace
+
+void LastEpochClean::Lec::report(unsigned pg_num, ps_t ps,
+				 epoch_t last_epoch_clean)
+{
+  if (ps >= pg_num) {
+    // removed PG
+    return;
+  }
+  epoch_by_pg.resize(pg_num, 0);
+  const auto old_lec = epoch_by_pg[ps];
+  if (old_lec >= last_epoch_clean) {
+    // stale lec
+    return;
+  }
+  epoch_by_pg[ps] = last_epoch_clean;
+  if (last_epoch_clean < floor) {
+    floor = last_epoch_clean;
+  } else if (last_epoch_clean > floor) {
+    if (old_lec == floor) {
+      // probably should increase floor?
+      auto new_floor = std::min_element(std::begin(epoch_by_pg),
+					std::end(epoch_by_pg));
+      floor = *new_floor;
+    }
+  }
+  if (ps != next_missing) {
+    return;
+  }
+  for (; next_missing < epoch_by_pg.size(); next_missing++) {
+    if (epoch_by_pg[next_missing] == 0) {
+      break;
+    }
+  }
+}
+
+void LastEpochClean::remove_pool(uint64_t pool)
+{
+  report_by_pool.erase(pool);
+}
+
+void LastEpochClean::report(unsigned pg_num, const pg_t& pg,
+			    epoch_t last_epoch_clean)
+{
+  auto& lec = report_by_pool[pg.pool()];
+  return lec.report(pg_num, pg.ps(), last_epoch_clean);
+}
+
+epoch_t LastEpochClean::get_lower_bound(const OSDMap& latest) const
+{
+  auto floor = latest.get_epoch();
+  for (auto& pool : latest.get_pools()) {
+    auto reported = report_by_pool.find(pool.first);
+    if (reported == report_by_pool.end()) {
+      return 0;
+    }
+    if (reported->second.next_missing < pool.second.get_pg_num()) {
+      return 0;
+    }
+    if (reported->second.floor < floor) {
+      floor = reported->second.floor;
+    }
+  }
+  return floor;
+}
+
+void LastEpochClean::dump(Formatter *f) const
+{
+  f->open_array_section("per_pool");
+
+  for (auto& [pool, lec] : report_by_pool) {
+    f->open_object_section("pool");
+    f->dump_unsigned("poolid", pool);
+    f->dump_unsigned("floor", lec.floor);
+    f->close_section();
+  }
+
+  f->close_section();
+}
+
+class C_UpdateCreatingPGs : public Context {
+public:
+  OSDMonitor *osdmon;
+  utime_t start;
+  epoch_t epoch;
+  C_UpdateCreatingPGs(OSDMonitor *osdmon, epoch_t e) :
+    osdmon(osdmon), start(ceph_clock_now()), epoch(e) {}
+  void finish(int r) override {
+    if (r >= 0) {
+      utime_t end = ceph_clock_now();
+      dout(10) << "osdmap epoch " << epoch << " mapping took "
+	       << (end - start) << " seconds" << dendl;
+      osdmon->update_creating_pgs();
+      osdmon->check_pg_creates_subs();
+    }
+  }
+};
+
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, osdmap)
+static ostream& _prefix(std::ostream *_dout, Monitor &mon, const OSDMap& osdmap) {
+  return *_dout << "mon." << mon.name << "@" << mon.rank
+		<< "(" << mon.get_state_name()
+		<< ").osd e" << osdmap.get_epoch() << " ";
+}
+
+OSDMonitor::OSDMonitor(
+  CephContext *cct,
+  Monitor &mn,
+  Paxos &p,
+  const string& service_name)
+ : PaxosService(mn, p, service_name),
+   cct(cct),
+   inc_osd_cache(g_conf()->mon_osd_cache_size),
+   full_osd_cache(g_conf()->mon_osd_cache_size),
+   has_osdmap_manifest(false),
+   mapper(mn.cct, &mn.cpu_tp)
+{
+  inc_cache = std::make_shared<IncCache>(this);
+  full_cache = std::make_shared<FullCache>(this);
+  cct->_conf.add_observer(this);
+  int r = _set_cache_sizes();
+  if (r < 0) {
+    derr << __func__ << " using default osd cache size - mon_osd_cache_size ("
+         << g_conf()->mon_osd_cache_size
+         << ") without priority cache management"
+         << dendl;
+  }
+}
+
+const char **OSDMonitor::get_tracked_conf_keys() const
+{
+  static const char* KEYS[] = {
+    "mon_memory_target",
+    "mon_memory_autotune",
+    "rocksdb_cache_size",
+    NULL
+  };
+  return KEYS;
+}
+
+void OSDMonitor::handle_conf_change(const ConfigProxy& conf,
+                                    const std::set<std::string> &changed)
+{
+  dout(10) << __func__ << " " << changed << dendl;
+
+  if (changed.count("mon_memory_autotune")) {
+    _set_cache_autotuning();
+  }
+  if (changed.count("mon_memory_target") ||
+      changed.count("rocksdb_cache_size")) {
+    int r = _update_mon_cache_settings();
+    if (r < 0) {
+      derr << __func__ << " mon_memory_target:"
+           << g_conf()->mon_memory_target
+           << " rocksdb_cache_size:"
+           << g_conf()->rocksdb_cache_size
+           << ". Unable to update cache size."
+           << dendl;
+    }
+  }
+}
+
+void OSDMonitor::_set_cache_autotuning()
+{
+  if (!g_conf()->mon_memory_autotune && pcm != nullptr) {
+    // Disable cache autotuning
+    std::lock_guard l(balancer_lock);
+    pcm = nullptr;
+  }
+
+  if (g_conf()->mon_memory_autotune && pcm == nullptr) {
+    int r = register_cache_with_pcm();
+    if (r < 0) {
+      dout(10) << __func__
+               << " Error while registering osdmon caches with pcm."
+               << " Cache auto tuning not enabled."
+               << dendl;
+      mon_memory_autotune = false;
+    } else {
+      mon_memory_autotune = true;
+    }
+  }
+}
+
+int OSDMonitor::_update_mon_cache_settings()
+{
+  if (g_conf()->mon_memory_target <= 0 ||
+      g_conf()->mon_memory_target < mon_memory_min ||
+      g_conf()->rocksdb_cache_size <= 0) {
+    return -EINVAL;
+  }
+
+  if (pcm == nullptr && rocksdb_binned_kv_cache == nullptr) {
+    derr << __func__ << " not using pcm and rocksdb" << dendl;
+    return -EINVAL;
+  }
+
+  uint64_t old_mon_memory_target = mon_memory_target;
+  uint64_t old_rocksdb_cache_size = rocksdb_cache_size;
+
+  // Set the new pcm memory cache sizes
+  mon_memory_target = g_conf()->mon_memory_target;
+  rocksdb_cache_size = g_conf()->rocksdb_cache_size;
+
+  uint64_t base = mon_memory_base;
+  double fragmentation = mon_memory_fragmentation;
+  uint64_t target = mon_memory_target;
+  uint64_t min = mon_memory_min;
+  uint64_t max = min;
+
+  uint64_t ltarget = (1.0 - fragmentation) * target;
+  if (ltarget > base + min) {
+    max = ltarget - base;
+  }
+
+  int r = _set_cache_ratios();
+  if (r < 0) {
+    derr << __func__ << " Cache ratios for pcm could not be set."
+         << " Review the kv (rocksdb) and mon_memory_target sizes."
+         << dendl;
+    mon_memory_target = old_mon_memory_target;
+    rocksdb_cache_size = old_rocksdb_cache_size;
+    return -EINVAL;
+  }
+
+  if (mon_memory_autotune && pcm != nullptr) {
+    std::lock_guard l(balancer_lock);
+    // set pcm cache levels
+    pcm->set_target_memory(target);
+    pcm->set_min_memory(min);
+    pcm->set_max_memory(max);
+    // tune memory based on new values
+    pcm->tune_memory();
+    pcm->balance();
+    _set_new_cache_sizes();
+    dout(1) << __func__ << " Updated mon cache setting."
+             << " target: " << target
+             << " min: " << min
+             << " max: " << max
+             << dendl;
+  }
+  return 0;
+}
+
+int OSDMonitor::_set_cache_sizes()
+{
+  if (g_conf()->mon_memory_autotune) {
+    // set the new osdmon cache targets to be managed by pcm
+    mon_osd_cache_size = g_conf()->mon_osd_cache_size;
+    rocksdb_cache_size = g_conf()->rocksdb_cache_size;
+    mon_memory_base = cct->_conf.get_val<Option::size_t>("osd_memory_base");
+    mon_memory_fragmentation = cct->_conf.get_val<double>("osd_memory_expected_fragmentation");
+    mon_memory_target = g_conf()->mon_memory_target;
+    mon_memory_min = g_conf()->mon_osd_cache_size_min;
+    if (mon_memory_target <= 0 || mon_memory_min <= 0) {
+      derr << __func__ << " mon_memory_target:" << mon_memory_target
+           << " mon_memory_min:" << mon_memory_min
+           << ". Invalid size option(s) provided."
+           << dendl;
+      return -EINVAL;
+    }
+    // Set the initial inc and full LRU cache sizes
+    inc_osd_cache.set_bytes(mon_memory_min);
+    full_osd_cache.set_bytes(mon_memory_min);
+    mon_memory_autotune = g_conf()->mon_memory_autotune;
+  }
+  return 0;
+}
+
+bool OSDMonitor::_have_pending_crush()
+{
+  return pending_inc.crush.length() > 0;
+}
+
+CrushWrapper &OSDMonitor::_get_stable_crush()
+{
+  return *osdmap.crush;
+}
+
+void OSDMonitor::_get_pending_crush(CrushWrapper& newcrush)
+{
+  bufferlist bl;
+  if (pending_inc.crush.length())
+    bl = pending_inc.crush;
+  else
+    osdmap.crush->encode(bl, CEPH_FEATURES_SUPPORTED_DEFAULT);
+
+  auto p = bl.cbegin();
+  newcrush.decode(p);
+}
+
+void OSDMonitor::create_initial()
+{
+  dout(10) << "create_initial for " << mon.monmap->fsid << dendl;
+
+  OSDMap newmap;
+
+  bufferlist bl;
+  mon.store->get("mkfs", "osdmap", bl);
+
+  if (bl.length()) {
+    newmap.decode(bl);
+    newmap.set_fsid(mon.monmap->fsid);
+  } else {
+    newmap.build_simple(cct, 0, mon.monmap->fsid, 0);
+  }
+  newmap.set_epoch(1);
+  newmap.created = newmap.modified = ceph_clock_now();
+
+  // new clusters should sort bitwise by default.
+  newmap.set_flag(CEPH_OSDMAP_SORTBITWISE);
+
+  newmap.flags |=
+    CEPH_OSDMAP_RECOVERY_DELETES |
+    CEPH_OSDMAP_PURGED_SNAPDIRS |
+    CEPH_OSDMAP_PGLOG_HARDLIMIT;
+  newmap.full_ratio = g_conf()->mon_osd_full_ratio;
+  if (newmap.full_ratio > 1.0) newmap.full_ratio /= 100;
+  newmap.backfillfull_ratio = g_conf()->mon_osd_backfillfull_ratio;
+  if (newmap.backfillfull_ratio > 1.0) newmap.backfillfull_ratio /= 100;
+  newmap.nearfull_ratio = g_conf()->mon_osd_nearfull_ratio;
+  if (newmap.nearfull_ratio > 1.0) newmap.nearfull_ratio /= 100;
+
+  // new cluster should require latest by default
+  if (g_conf().get_val<bool>("mon_debug_no_require_pacific")) {
+    if (g_conf().get_val<bool>("mon_debug_no_require_octopus")) {
+      derr << __func__ << " mon_debug_no_require_pacific and octopus=true" << dendl;
+      newmap.require_osd_release = ceph_release_t::nautilus;
+    } else {
+      derr << __func__ << " mon_debug_no_require_pacific=true" << dendl;
+      newmap.require_osd_release = ceph_release_t::octopus;
+    }
+  } else {
+    newmap.require_osd_release = ceph_release_t::pacific;
+  }
+
+  if (newmap.require_osd_release >= ceph_release_t::octopus) {
+    ceph_release_t r = ceph_release_from_name(
+      g_conf()->mon_osd_initial_require_min_compat_client);
+    if (!r) {
+      ceph_abort_msg("mon_osd_initial_require_min_compat_client is not valid");
+    }
+    newmap.require_min_compat_client = r;
+  }
+
+  // encode into pending incremental
+  uint64_t features = newmap.get_encoding_features();
+  newmap.encode(pending_inc.fullmap,
+                features | CEPH_FEATURE_RESERVED);
+  pending_inc.full_crc = newmap.get_crc();
+  dout(20) << " full crc " << pending_inc.full_crc << dendl;
+}
+
+void OSDMonitor::get_store_prefixes(std::set<string>& s) const
+{
+  s.insert(service_name);
+  s.insert(OSD_PG_CREATING_PREFIX);
+  s.insert(OSD_METADATA_PREFIX);
+  s.insert(OSD_SNAP_PREFIX);
+}
+
+void OSDMonitor::update_from_paxos(bool *need_bootstrap)
+{
+  // we really don't care if the version has been updated, because we may
+  // have trimmed without having increased the last committed; yet, we may
+  // need to update the in-memory manifest.
+  load_osdmap_manifest();
+
+  version_t version = get_last_committed();
+  if (version == osdmap.epoch)
+    return;
+  ceph_assert(version > osdmap.epoch);
+
+  dout(15) << "update_from_paxos paxos e " << version
+	   << ", my e " << osdmap.epoch << dendl;
+
+  int prev_num_up_osd = osdmap.num_up_osd;
+
+  if (mapping_job) {
+    if (!mapping_job->is_done()) {
+      dout(1) << __func__ << " mapping job "
+	      << mapping_job.get() << " did not complete, "
+	      << mapping_job->shards << " left, canceling" << dendl;
+      mapping_job->abort();
+    }
+    mapping_job.reset();
+  }
+
+  load_health();
+
+  /*
+   * We will possibly have a stashed latest that *we* wrote, and we will
+   * always be sure to have the oldest full map in the first..last range
+   * due to encode_trim_extra(), which includes the oldest full map in the trim
+   * transaction.
+   *
+   * encode_trim_extra() does not however write the full map's
+   * version to 'full_latest'.  This is only done when we are building the
+   * full maps from the incremental versions.  But don't panic!  We make sure
+   * that the following conditions find whichever full map version is newer.
+   */
+  version_t latest_full = get_version_latest_full();
+  if (latest_full == 0 && get_first_committed() > 1)
+    latest_full = get_first_committed();
+
+  if (get_first_committed() > 1 &&
+      latest_full < get_first_committed()) {
+    // the monitor could be just sync'ed with its peer, and the latest_full key
+    // is not encoded in the paxos commits in encode_pending(), so we need to
+    // make sure we get it pointing to a proper version.
+    version_t lc = get_last_committed();
+    version_t fc = get_first_committed();
+
+    dout(10) << __func__ << " looking for valid full map in interval"
+	     << " [" << fc << ", " << lc << "]" << dendl;
+
+    latest_full = 0;
+    for (version_t v = lc; v >= fc; v--) {
+      string full_key = "full_" + stringify(v);
+      if (mon.store->exists(get_service_name(), full_key)) {
+        dout(10) << __func__ << " found latest full map v " << v << dendl;
+        latest_full = v;
+        break;
+      }
+    }
+
+    ceph_assert(latest_full > 0);
+    auto t(std::make_shared<MonitorDBStore::Transaction>());
+    put_version_latest_full(t, latest_full);
+    mon.store->apply_transaction(t);
+    dout(10) << __func__ << " updated the on-disk full map version to "
+             << latest_full << dendl;
+  }
+
+  if ((latest_full > 0) && (latest_full > osdmap.epoch)) {
+    bufferlist latest_bl;
+    get_version_full(latest_full, latest_bl);
+    ceph_assert(latest_bl.length() != 0);
+    dout(7) << __func__ << " loading latest full map e" << latest_full << dendl;
+    osdmap = OSDMap();
+    osdmap.decode(latest_bl);
+  }
+
+  bufferlist bl;
+  if (!mon.store->get(OSD_PG_CREATING_PREFIX, "creating", bl)) {
+    auto p = bl.cbegin();
+    std::lock_guard<std::mutex> l(creating_pgs_lock);
+    creating_pgs.decode(p);
+    dout(7) << __func__ << " loading creating_pgs last_scan_epoch "
+	    << creating_pgs.last_scan_epoch
+	    << " with " << creating_pgs.pgs.size() << " pgs" << dendl;
+  } else {
+    dout(1) << __func__ << " missing creating pgs; upgrade from post-kraken?"
+	    << dendl;
+  }
+
+  // walk through incrementals
+  MonitorDBStore::TransactionRef t;
+  size_t tx_size = 0;
+  while (version > osdmap.epoch) {
+    bufferlist inc_bl;
+    int err = get_version(osdmap.epoch+1, inc_bl);
+    ceph_assert(err == 0);
+    ceph_assert(inc_bl.length());
+    // set priority cache manager levels if the osdmap is
+    // being populated for the first time.
+    if (mon_memory_autotune && pcm == nullptr) {
+      int r = register_cache_with_pcm();
+      if (r < 0) {
+        dout(10) << __func__
+                 << " Error while registering osdmon caches with pcm."
+                 << " Proceeding without cache auto tuning."
+                 << dendl;
+      }
+    }
+
+    dout(7) << "update_from_paxos  applying incremental " << osdmap.epoch+1
+	    << dendl;
+    OSDMap::Incremental inc(inc_bl);
+    err = osdmap.apply_incremental(inc);
+    ceph_assert(err == 0);
+
+    if (!t)
+      t.reset(new MonitorDBStore::Transaction);
+
+    // Write out the full map for all past epochs.  Encode the full
+    // map with the same features as the incremental.  If we don't
+    // know, use the quorum features.  If we don't know those either,
+    // encode with all features.
+    uint64_t f = inc.encode_features;
+    if (!f)
+      f = mon.get_quorum_con_features();
+    if (!f)
+      f = -1;
+    bufferlist full_bl;
+    osdmap.encode(full_bl, f | CEPH_FEATURE_RESERVED);
+    tx_size += full_bl.length();
+
+    bufferlist orig_full_bl;
+    get_version_full(osdmap.epoch, orig_full_bl);
+    if (orig_full_bl.length()) {
+      // the primary provided the full map
+      ceph_assert(inc.have_crc);
+      if (inc.full_crc != osdmap.crc) {
+	// This will happen if the mons were running mixed versions in
+	// the past or some other circumstance made the full encoded
+	// maps divergent.  Reloading here will bring us back into
+	// sync with the primary for this and all future maps.  OSDs
+	// will also be brought back into sync when they discover the
+	// crc mismatch and request a full map from a mon.
+	derr << __func__ << " full map CRC mismatch, resetting to canonical"
+	     << dendl;
+
+	dout(20) << __func__ << " my (bad) full osdmap:\n";
+	JSONFormatter jf(true);
+	jf.dump_object("osdmap", osdmap);
+	jf.flush(*_dout);
+	*_dout << "\nhexdump:\n";
+	full_bl.hexdump(*_dout);
+	*_dout << dendl;
+
+	osdmap = OSDMap();
+	osdmap.decode(orig_full_bl);
+
+	dout(20) << __func__ << " canonical full osdmap:\n";
+	JSONFormatter jf(true);
+	jf.dump_object("osdmap", osdmap);
+	jf.flush(*_dout);
+	*_dout << "\nhexdump:\n";
+	orig_full_bl.hexdump(*_dout);
+	*_dout << dendl;
+      }
+    } else {
+      ceph_assert(!inc.have_crc);
+      put_version_full(t, osdmap.epoch, full_bl);
+    }
+    put_version_latest_full(t, osdmap.epoch);
+
+    // share
+    dout(1) << osdmap << dendl;
+
+    if (osdmap.epoch == 1) {
+      t->erase("mkfs", "osdmap");
+    }
+
+    if (tx_size > g_conf()->mon_sync_max_payload_size*2) {
+      mon.store->apply_transaction(t);
+      t = MonitorDBStore::TransactionRef();
+      tx_size = 0;
+    }
+    for (const auto [osd, state] : inc.new_state) {
+      if (state & CEPH_OSD_UP) {
+	// could be marked up *or* down, but we're too lazy to check which
+	last_osd_report.erase(osd);
+      }
+      if (state & CEPH_OSD_OUT) {
+        // could be marked in *or* out, but we can safely drop it
+        osd_epochs.erase(osd);
+      }
+    }
+    for (const auto [osd, weight] : inc.new_weight) {
+      if (weight == CEPH_OSD_OUT) {
+        // manually marked out, so drop it
+        osd_epochs.erase(osd);
+      }
+    }
+  }
+
+  if (t) {
+    mon.store->apply_transaction(t);
+  }
+
+  bool marked_osd_down = false;
+  for (int o = 0; o < osdmap.get_max_osd(); o++) {
+    if (osdmap.is_out(o))
+      continue;
+    auto found = down_pending_out.find(o);
+    if (osdmap.is_down(o)) {
+      // populate down -> out map
+      if (found == down_pending_out.end()) {
+        dout(10) << " adding osd." << o << " to down_pending_out map" << dendl;
+        down_pending_out[o] = ceph_clock_now();
+	marked_osd_down = true;
+      }
+    } else {
+      if (found != down_pending_out.end()) {
+        dout(10) << " removing osd." << o << " from down_pending_out map" << dendl;
+        down_pending_out.erase(found);
+      }
+    }
+  }
+  // XXX: need to trim MonSession connected with a osd whose id > max_osd?
+
+  check_osdmap_subs();
+  check_pg_creates_subs();
+
+  share_map_with_random_osd();
+  update_logger();
+  process_failures();
+
+  // make sure our feature bits reflect the latest map
+  update_msgr_features();
+
+  if (!mon.is_leader()) {
+    // will be called by on_active() on the leader, avoid doing so twice
+    start_mapping();
+  }
+  if (osdmap.stretch_mode_enabled) {
+    dout(20) << "Stretch mode enabled in this map" << dendl;
+    mon.try_engage_stretch_mode();
+    if (osdmap.degraded_stretch_mode) {
+      dout(20) << "Degraded stretch mode set in this map" << dendl;
+      if (!osdmap.recovering_stretch_mode) {
+	mon.set_degraded_stretch_mode();
+	if (prev_num_up_osd < osdmap.num_up_osd &&
+	    (osdmap.num_up_osd / (double)osdmap.num_osd) >
+	    cct->_conf.get_val<double>("mon_stretch_cluster_recovery_ratio")) {
+	  // TODO: This works for 2-site clusters when the OSD maps are appropriately
+	  // trimmed and everything is "normal" but not if you have a lot of out OSDs
+	  // you're ignoring or in some really degenerate failure cases
+	  dout(10) << "Enabling recovery stretch mode in this map" << dendl;
+	  mon.go_recovery_stretch_mode();
+	}
+      } else {
+	mon.set_recovery_stretch_mode();
+      }
+    } else {
+      mon.set_healthy_stretch_mode();
+    }
+    if (marked_osd_down &&
+	(!osdmap.degraded_stretch_mode || osdmap.recovering_stretch_mode)) {
+      dout(20) << "Checking degraded stretch mode due to osd changes" << dendl;
+      mon.maybe_go_degraded_stretch_mode();
+    }
+  }
+}
+
+int OSDMonitor::register_cache_with_pcm()
+{
+  if (mon_memory_target <= 0 || mon_memory_min <= 0) {
+    derr << __func__ << " Invalid memory size specified for mon caches."
+         << " Caches will not be auto-tuned."
+         << dendl;
+    return -EINVAL;
+  }
+  uint64_t base = mon_memory_base;
+  double fragmentation = mon_memory_fragmentation;
+  // For calculating total target memory, consider rocksdb cache size.
+  uint64_t target = mon_memory_target;
+  uint64_t min = mon_memory_min;
+  uint64_t max = min;
+
+  // Apply the same logic as in bluestore to set the max amount
+  // of memory to use for cache. Assume base memory for OSDMaps
+  // and then add in some overhead for fragmentation.
+  uint64_t ltarget = (1.0 - fragmentation) * target;
+  if (ltarget > base + min) {
+    max = ltarget - base;
+  }
+
+  rocksdb_binned_kv_cache = mon.store->get_priority_cache();
+  if (!rocksdb_binned_kv_cache) {
+    derr << __func__ << " not using rocksdb" << dendl;
+    return -EINVAL;
+  }
+
+  int r = _set_cache_ratios();
+  if (r < 0) {
+    derr << __func__ << " Cache ratios for pcm could not be set."
+         << " Review the kv (rocksdb) and mon_memory_target sizes."
+         << dendl;
+    return -EINVAL;
+  }
+
+  pcm = std::make_shared<PriorityCache::Manager>(
+      cct, min, max, target, true);
+  pcm->insert("kv", rocksdb_binned_kv_cache, true);
+  pcm->insert("inc", inc_cache, true);
+  pcm->insert("full", full_cache, true);
+  dout(1) << __func__ << " pcm target: " << target
+           << " pcm max: " << max
+           << " pcm min: " << min
+           << " inc_osd_cache size: " << inc_osd_cache.get_size()
+           << dendl;
+  return 0;
+}
+
+int OSDMonitor::_set_cache_ratios()
+{
+  double old_cache_kv_ratio = cache_kv_ratio;
+
+  // Set the cache ratios for kv(rocksdb), inc and full caches
+  cache_kv_ratio = (double)rocksdb_cache_size / (double)mon_memory_target;
+  if (cache_kv_ratio >= 1.0) {
+    derr << __func__ << " Cache kv ratio (" << cache_kv_ratio
+         << ") must be in range [0,<1.0]."
+         << dendl;
+    cache_kv_ratio = old_cache_kv_ratio;
+    return -EINVAL;
+  }
+  rocksdb_binned_kv_cache->set_cache_ratio(cache_kv_ratio);
+  cache_inc_ratio = cache_full_ratio = (1.0 - cache_kv_ratio) / 2;
+  inc_cache->set_cache_ratio(cache_inc_ratio);
+  full_cache->set_cache_ratio(cache_full_ratio);
+
+  dout(1) << __func__ << " kv ratio " << cache_kv_ratio
+           << " inc ratio " << cache_inc_ratio
+           << " full ratio " << cache_full_ratio
+           << dendl;
+  return 0;
+}
+
+void OSDMonitor::start_mapping()
+{
+  // initiate mapping job
+  if (mapping_job) {
+    dout(10) << __func__ << " canceling previous mapping_job " << mapping_job.get()
+	     << dendl;
+    mapping_job->abort();
+  }
+  if (!osdmap.get_pools().empty()) {
+    auto fin = new C_UpdateCreatingPGs(this, osdmap.get_epoch());
+    mapping_job = mapping.start_update(osdmap, mapper,
+				       g_conf()->mon_osd_mapping_pgs_per_chunk);
+    dout(10) << __func__ << " started mapping job " << mapping_job.get()
+	     << " at " << fin->start << dendl;
+    mapping_job->set_finish_event(fin);
+  } else {
+    dout(10) << __func__ << " no pools, no mapping job" << dendl;
+    mapping_job = nullptr;
+  }
+}
+
+void OSDMonitor::update_msgr_features()
+{
+  const int types[] = {
+    entity_name_t::TYPE_OSD,
+    entity_name_t::TYPE_CLIENT,
+    entity_name_t::TYPE_MDS,
+    entity_name_t::TYPE_MON
+  };
+  for (int type : types) {
+    uint64_t mask;
+    uint64_t features = osdmap.get_features(type, &mask);
+    if ((mon.messenger->get_policy(type).features_required & mask) != features) {
+      dout(0) << "crush map has features " << features << ", adjusting msgr requires" << dendl;
+      ceph::net::Policy p = mon.messenger->get_policy(type);
+      p.features_required = (p.features_required & ~mask) | features;
+      mon.messenger->set_policy(type, p);
+    }
+  }
+}
+
+void OSDMonitor::on_active()
+{
+  update_logger();
+
+  if (mon.is_leader()) {
+    mon.clog->debug() << "osdmap " << osdmap;
+    if (!priority_convert) {
+      // Only do this once at start-up
+      convert_pool_priorities();
+      priority_convert = true;
+    }
+  } else {
+    list<MonOpRequestRef> ls;
+    take_all_failures(ls);
+    while (!ls.empty()) {
+      MonOpRequestRef op = ls.front();
+      op->mark_osdmon_event(__func__);
+      dispatch(op);
+      ls.pop_front();
+    }
+  }
+  start_mapping();
+}
+
+void OSDMonitor::on_restart()
+{
+  last_osd_report.clear();
+}
+
+void OSDMonitor::on_shutdown()
+{
+  dout(10) << __func__ << dendl;
+  if (mapping_job) {
+    dout(10) << __func__ << " canceling previous mapping_job " << mapping_job.get()
+	     << dendl;
+    mapping_job->abort();
+  }
+
+  // discard failure info, waiters
+  list<MonOpRequestRef> ls;
+  take_all_failures(ls);
+  ls.clear();
+}
+
+void OSDMonitor::update_logger()
+{
+  dout(10) << "update_logger" << dendl;
+
+  mon.cluster_logger->set(l_cluster_num_osd, osdmap.get_num_osds());
+  mon.cluster_logger->set(l_cluster_num_osd_up, osdmap.get_num_up_osds());
+  mon.cluster_logger->set(l_cluster_num_osd_in, osdmap.get_num_in_osds());
+  mon.cluster_logger->set(l_cluster_osd_epoch, osdmap.get_epoch());
+}
+
+void OSDMonitor::create_pending()
+{
+  pending_inc = OSDMap::Incremental(osdmap.epoch+1);
+  pending_inc.fsid = mon.monmap->fsid;
+  pending_metadata.clear();
+  pending_metadata_rm.clear();
+  pending_pseudo_purged_snaps.clear();
+
+  dout(10) << "create_pending e " << pending_inc.epoch << dendl;
+
+  // safety checks (this shouldn't really happen)
+  {
+    if (osdmap.backfillfull_ratio <= 0) {
+      pending_inc.new_backfillfull_ratio = g_conf()->mon_osd_backfillfull_ratio;
+      if (pending_inc.new_backfillfull_ratio > 1.0)
+	pending_inc.new_backfillfull_ratio /= 100;
+      dout(1) << __func__ << " setting backfillfull_ratio = "
+	      << pending_inc.new_backfillfull_ratio << dendl;
+    }
+    if (osdmap.full_ratio <= 0) {
+      pending_inc.new_full_ratio = g_conf()->mon_osd_full_ratio;
+      if (pending_inc.new_full_ratio > 1.0)
+        pending_inc.new_full_ratio /= 100;
+      dout(1) << __func__ << " setting full_ratio = "
+	      << pending_inc.new_full_ratio << dendl;
+    }
+    if (osdmap.nearfull_ratio <= 0) {
+      pending_inc.new_nearfull_ratio = g_conf()->mon_osd_nearfull_ratio;
+      if (pending_inc.new_nearfull_ratio > 1.0)
+        pending_inc.new_nearfull_ratio /= 100;
+      dout(1) << __func__ << " setting nearfull_ratio = "
+	      << pending_inc.new_nearfull_ratio << dendl;
+    }
+  }
+
+  // Rewrite CRUSH rule IDs if they are using legacy "ruleset"
+  // structure.
+  if (osdmap.crush->has_legacy_rule_ids()) {
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    // First, for all pools, work out which rule they really used
+    // by resolving ruleset to rule.
+    for (const auto &i : osdmap.get_pools()) {
+      const auto pool_id = i.first;
+      const auto &pool = i.second;
+      int new_rule_id = newcrush.find_rule(pool.crush_rule,
+					   pool.type, pool.size);
+
+      dout(1) << __func__ << " rewriting pool "
+	      << osdmap.get_pool_name(pool_id) << " crush ruleset "
+	      << pool.crush_rule << " -> rule id " << new_rule_id << dendl;
+      if (pending_inc.new_pools.count(pool_id) == 0) {
+	pending_inc.new_pools[pool_id] = pool;
+      }
+      pending_inc.new_pools[pool_id].crush_rule = new_rule_id;
+    }
+
+    // Now, go ahead and renumber all the rules so that their
+    // rule_id field corresponds to their position in the array
+    auto old_to_new = newcrush.renumber_rules();
+    dout(1) << __func__ << " Rewrote " << old_to_new << " crush IDs:" << dendl;
+    for (const auto &i : old_to_new) {
+      dout(1) << __func__ << " " << i.first << " -> " << i.second << dendl;
+    }
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+  }
+}
+
+creating_pgs_t
+OSDMonitor::update_pending_pgs(const OSDMap::Incremental& inc,
+			       const OSDMap& nextmap)
+{
+  dout(10) << __func__ << dendl;
+  creating_pgs_t pending_creatings;
+  {
+    std::lock_guard<std::mutex> l(creating_pgs_lock);
+    pending_creatings = creating_pgs;
+  }
+  // check for new or old pools
+  if (pending_creatings.last_scan_epoch < inc.epoch) {
+    unsigned queued = 0;
+    queued += scan_for_creating_pgs(osdmap.get_pools(),
+				    inc.old_pools,
+				    inc.modified,
+				    &pending_creatings);
+    queued += scan_for_creating_pgs(inc.new_pools,
+				    inc.old_pools,
+				    inc.modified,
+				    &pending_creatings);
+    dout(10) << __func__ << " " << queued << " pools queued" << dendl;
+    for (auto deleted_pool : inc.old_pools) {
+      auto removed = pending_creatings.remove_pool(deleted_pool);
+      dout(10) << __func__ << " " << removed
+               << " pg removed because containing pool deleted: "
+               << deleted_pool << dendl;
+      last_epoch_clean.remove_pool(deleted_pool);
+    }
+    // pgmon updates its creating_pgs in check_osd_map() which is called by
+    // on_active() and check_osd_map() could be delayed if lease expires, so its
+    // creating_pgs could be stale in comparison with the one of osdmon. let's
+    // trim them here. otherwise, they will be added back after being erased.
+    unsigned removed = 0;
+    for (auto& pg : pending_created_pgs) {
+      dout(20) << __func__ << " noting created pg " << pg << dendl;
+      pending_creatings.created_pools.insert(pg.pool());
+      removed += pending_creatings.pgs.erase(pg);
+    }
+    pending_created_pgs.clear();
+    dout(10) << __func__ << " " << removed
+	     << " pgs removed because they're created" << dendl;
+    pending_creatings.last_scan_epoch = osdmap.get_epoch();
+  }
+
+  // filter out any pgs that shouldn't exist.
+  {
+    auto i = pending_creatings.pgs.begin();
+    while (i != pending_creatings.pgs.end()) {
+      if (!nextmap.pg_exists(i->first)) {
+	dout(10) << __func__ << " removing pg " << i->first
+		 << " which should not exist" << dendl;
+	i = pending_creatings.pgs.erase(i);
+      } else {
+	++i;
+      }
+    }
+  }
+
+  // process queue
+  unsigned max = std::max<int64_t>(1, g_conf()->mon_osd_max_creating_pgs);
+  const auto total = pending_creatings.pgs.size();
+  while (pending_creatings.pgs.size() < max &&
+	 !pending_creatings.queue.empty()) {
+    auto p = pending_creatings.queue.begin();
+    int64_t poolid = p->first;
+    dout(10) << __func__ << " pool " << poolid
+	     << " created " << p->second.created
+	     << " modified " << p->second.modified
+	     << " [" << p->second.start << "-" << p->second.end << ")"
+	     << dendl;
+    int64_t n = std::min<int64_t>(max - pending_creatings.pgs.size(),
+				  p->second.end - p->second.start);
+    ps_t first = p->second.start;
+    ps_t end = first + n;
+    for (ps_t ps = first; ps < end; ++ps) {
+      const pg_t pgid{ps, static_cast<uint64_t>(poolid)};
+      // NOTE: use the *current* epoch as the PG creation epoch so that the
+      // OSD does not have to generate a long set of PastIntervals.
+      pending_creatings.pgs.emplace(
+	pgid,
+	creating_pgs_t::pg_create_info(inc.epoch,
+				       p->second.modified));
+      dout(10) << __func__ << " adding " << pgid << dendl;
+    }
+    p->second.start = end;
+    if (p->second.done()) {
+      dout(10) << __func__ << " done with queue for " << poolid << dendl;
+      pending_creatings.queue.erase(p);
+    } else {
+      dout(10) << __func__ << " pool " << poolid
+	       << " now [" << p->second.start << "-" << p->second.end << ")"
+	       << dendl;
+    }
+  }
+  dout(10) << __func__ << " queue remaining: " << pending_creatings.queue.size()
+	   << " pools" << dendl;
+
+  if (mon.monmap->min_mon_release >= ceph_release_t::octopus) {
+    // walk creating pgs' history and past_intervals forward
+    for (auto& i : pending_creatings.pgs) {
+      // this mirrors PG::start_peering_interval()
+      pg_t pgid = i.first;
+
+      // this is a bit imprecise, but sufficient?
+      struct min_size_predicate_t : public IsPGRecoverablePredicate {
+	const pg_pool_t *pi;
+	bool operator()(const set<pg_shard_t> &have) const {
+	  return have.size() >= pi->min_size;
+	}
+	explicit min_size_predicate_t(const pg_pool_t *i) : pi(i) {}
+      } min_size_predicate(nextmap.get_pg_pool(pgid.pool()));
+
+      vector<int> up, acting;
+      int up_primary, acting_primary;
+      nextmap.pg_to_up_acting_osds(
+	pgid, &up, &up_primary, &acting, &acting_primary);
+      if (i.second.history.epoch_created == 0) {
+	// new pg entry, set it up
+	i.second.up = up;
+	i.second.acting = acting;
+	i.second.up_primary = up_primary;
+	i.second.acting_primary = acting_primary;
+	i.second.history = pg_history_t(i.second.create_epoch,
+					i.second.create_stamp);
+	dout(10) << __func__ << "  pg " << pgid << " just added, "
+		 << " up " << i.second.up
+		 << " p " << i.second.up_primary
+		 << " acting " << i.second.acting
+		 << " p " << i.second.acting_primary
+		 << " history " << i.second.history
+		 << " past_intervals " << i.second.past_intervals
+		 << dendl;
+     } else {
+	std::stringstream debug;
+	if (PastIntervals::check_new_interval(
+	      i.second.acting_primary, acting_primary,
+	      i.second.acting, acting,
+	      i.second.up_primary, up_primary,
+	      i.second.up, up,
+	      i.second.history.same_interval_since,
+	      i.second.history.last_epoch_clean,
+	      &nextmap,
+	      &osdmap,
+	      pgid,
+	      min_size_predicate,
+	      &i.second.past_intervals,
+	      &debug)) {
+	  epoch_t e = inc.epoch;
+	  i.second.history.same_interval_since = e;
+	  if (i.second.up != up) {
+	    i.second.history.same_up_since = e;
+	  }
+	  if (i.second.acting_primary != acting_primary) {
+	    i.second.history.same_primary_since = e;
+	  }
+	  if (pgid.is_split(
+		osdmap.get_pg_num(pgid.pool()),
+		nextmap.get_pg_num(pgid.pool()),
+		nullptr)) {
+	    i.second.history.last_epoch_split = e;
+	  }
+	  dout(10) << __func__ << "  pg " << pgid << " new interval,"
+		   << " up " << i.second.up << " -> " << up
+		   << " p " << i.second.up_primary << " -> " << up_primary
+		   << " acting " << i.second.acting << " -> " << acting
+		   << " p " << i.second.acting_primary << " -> "
+		   << acting_primary
+		   << " history " << i.second.history
+		   << " past_intervals " << i.second.past_intervals
+		   << dendl;
+	  dout(20) << "  debug: " << debug.str() << dendl;
+	  i.second.up = up;
+	  i.second.acting = acting;
+	  i.second.up_primary = up_primary;
+	  i.second.acting_primary = acting_primary;
+	}
+      }
+    }
+  }
+  dout(10) << __func__
+	   << " " << (pending_creatings.pgs.size() - total)
+	   << "/" << pending_creatings.pgs.size()
+	   << " pgs added from queued pools" << dendl;
+  return pending_creatings;
+}
+
+void OSDMonitor::maybe_prime_pg_temp()
+{
+  bool all = false;
+  if (pending_inc.crush.length()) {
+    dout(10) << __func__ << " new crush map, all" << dendl;
+    all = true;
+  }
+
+  if (!pending_inc.new_up_client.empty()) {
+    dout(10) << __func__ << " new up osds, all" << dendl;
+    all = true;
+  }
+
+  // check for interesting OSDs
+  set<int> osds;
+  for (auto p = pending_inc.new_state.begin();
+       !all && p != pending_inc.new_state.end();
+       ++p) {
+    if ((p->second & CEPH_OSD_UP) &&
+	osdmap.is_up(p->first)) {
+      osds.insert(p->first);
+    }
+  }
+  for (auto p = pending_inc.new_weight.begin();
+       !all && p != pending_inc.new_weight.end();
+       ++p) {
+    if (osdmap.exists(p->first) && p->second < osdmap.get_weight(p->first)) {
+      // weight reduction
+      osds.insert(p->first);
+    } else {
+      dout(10) << __func__ << " osd." << p->first << " weight increase, all"
+	       << dendl;
+      all = true;
+    }
+  }
+
+  if (!all && osds.empty())
+    return;
+
+  if (!all) {
+    unsigned estimate =
+      mapping.get_osd_acting_pgs(*osds.begin()).size() * osds.size();
+    if (estimate > mapping.get_num_pgs() *
+	g_conf()->mon_osd_prime_pg_temp_max_estimate) {
+      dout(10) << __func__ << " estimate " << estimate << " pgs on "
+	       << osds.size() << " osds >= "
+	       << g_conf()->mon_osd_prime_pg_temp_max_estimate << " of total "
+	       << mapping.get_num_pgs() << " pgs, all"
+	       << dendl;
+      all = true;
+    } else {
+      dout(10) << __func__ << " estimate " << estimate << " pgs on "
+	       << osds.size() << " osds" << dendl;
+    }
+  }
+
+  OSDMap next;
+  next.deepish_copy_from(osdmap);
+  next.apply_incremental(pending_inc);
+
+  if (next.get_pools().empty()) {
+    dout(10) << __func__ << " no pools, no pg_temp priming" << dendl;
+  } else if (all) {
+    PrimeTempJob job(next, this);
+    mapper.queue(&job, g_conf()->mon_osd_mapping_pgs_per_chunk, {});
+    if (job.wait_for(g_conf()->mon_osd_prime_pg_temp_max_time)) {
+      dout(10) << __func__ << " done in " << job.get_duration() << dendl;
+    } else {
+      dout(10) << __func__ << " did not finish in "
+	       << g_conf()->mon_osd_prime_pg_temp_max_time
+	       << ", stopping" << dendl;
+      job.abort();
+    }
+  } else {
+    dout(10) << __func__ << " " << osds.size() << " interesting osds" << dendl;
+    utime_t stop = ceph_clock_now();
+    stop += g_conf()->mon_osd_prime_pg_temp_max_time;
+    const int chunk = 1000;
+    int n = chunk;
+    std::unordered_set<pg_t> did_pgs;
+    for (auto osd : osds) {
+      auto& pgs = mapping.get_osd_acting_pgs(osd);
+      dout(20) << __func__ << " osd." << osd << " " << pgs << dendl;
+      for (auto pgid : pgs) {
+	if (!did_pgs.insert(pgid).second) {
+	  continue;
+	}
+	prime_pg_temp(next, pgid);
+	if (--n <= 0) {
+	  n = chunk;
+	  if (ceph_clock_now() > stop) {
+	    dout(10) << __func__ << " consumed more than "
+		     << g_conf()->mon_osd_prime_pg_temp_max_time
+		     << " seconds, stopping"
+		     << dendl;
+	    return;
+	  }
+	}
+      }
+    }
+  }
+}
+
+void OSDMonitor::prime_pg_temp(
+  const OSDMap& next,
+  pg_t pgid)
+{
+  // TODO: remove this creating_pgs direct access?
+  if (creating_pgs.pgs.count(pgid)) {
+    return;
+  }
+  if (!osdmap.pg_exists(pgid)) {
+    return;
+  }
+
+  vector<int> up, acting;
+  mapping.get(pgid, &up, nullptr, &acting, nullptr);
+
+  vector<int> next_up, next_acting;
+  int next_up_primary, next_acting_primary;
+  next.pg_to_up_acting_osds(pgid, &next_up, &next_up_primary,
+			    &next_acting, &next_acting_primary);
+  if (acting == next_acting &&
+      !(up != acting && next_up == next_acting))
+    return;  // no change since last epoch
+
+  if (acting.empty())
+    return;  // if previously empty now we can be no worse off
+  const pg_pool_t *pool = next.get_pg_pool(pgid.pool());
+  if (pool && acting.size() < pool->min_size)
+    return;  // can be no worse off than before
+
+  if (next_up == next_acting) {
+    acting.clear();
+    dout(20) << __func__ << " next_up == next_acting now, clear pg_temp"
+	     << dendl;
+  }
+
+  dout(20) << __func__ << " " << pgid << " " << up << "/" << acting
+	   << " -> " << next_up << "/" << next_acting
+	   << ", priming " << acting
+	   << dendl;
+  {
+    std::lock_guard l(prime_pg_temp_lock);
+    // do not touch a mapping if a change is pending
+    pending_inc.new_pg_temp.emplace(
+      pgid,
+      mempool::osdmap::vector<int>(acting.begin(), acting.end()));
+  }
+}
+
+/**
+ * @note receiving a transaction in this function gives a fair amount of
+ * freedom to the service implementation if it does need it. It shouldn't.
+ */
+void OSDMonitor::encode_pending(MonitorDBStore::TransactionRef t)
+{
+  dout(10) << "encode_pending e " << pending_inc.epoch
+	   << dendl;
+
+  if (do_prune(t)) {
+    dout(1) << __func__ << " osdmap full prune encoded e"
+            << pending_inc.epoch << dendl;
+  }
+
+  // finalize up pending_inc
+  pending_inc.modified = ceph_clock_now();
+
+  int r = pending_inc.propagate_base_properties_to_tiers(cct, osdmap);
+  ceph_assert(r == 0);
+
+  if (mapping_job) {
+    if (!mapping_job->is_done()) {
+      dout(1) << __func__ << " skipping prime_pg_temp; mapping job "
+	      << mapping_job.get() << " did not complete, "
+	      << mapping_job->shards << " left" << dendl;
+      mapping_job->abort();
+    } else if (mapping.get_epoch() < osdmap.get_epoch()) {
+      dout(1) << __func__ << " skipping prime_pg_temp; mapping job "
+	      << mapping_job.get() << " is prior epoch "
+	      << mapping.get_epoch() << dendl;
+    } else {
+      if (g_conf()->mon_osd_prime_pg_temp) {
+	maybe_prime_pg_temp();
+      }
+    } 
+  } else if (g_conf()->mon_osd_prime_pg_temp) {
+    dout(1) << __func__ << " skipping prime_pg_temp; mapping job did not start"
+	    << dendl;
+  }
+  mapping_job.reset();
+
+  // ensure we don't have blank new_state updates.  these are interrpeted as
+  // CEPH_OSD_UP (and almost certainly not what we want!).
+  auto p = pending_inc.new_state.begin();
+  while (p != pending_inc.new_state.end()) {
+    if (p->second == 0) {
+      dout(10) << "new_state for osd." << p->first << " is 0, removing" << dendl;
+      p = pending_inc.new_state.erase(p);
+    } else {
+      if (p->second & CEPH_OSD_UP) {
+	pending_inc.new_last_up_change = pending_inc.modified;
+      }
+      ++p;
+    }
+  }
+  if (!pending_inc.new_up_client.empty()) {
+    pending_inc.new_last_up_change = pending_inc.modified;
+  }
+  for (auto& i : pending_inc.new_weight) {
+    if (i.first >= osdmap.max_osd) {
+      if (i.second) {
+	// new osd is already marked in
+	pending_inc.new_last_in_change = pending_inc.modified;
+        break;
+      }
+    } else if (!!i.second != !!osdmap.osd_weight[i.first]) {
+      // existing osd marked in or out
+      pending_inc.new_last_in_change = pending_inc.modified;
+      break;
+    }
+  }
+
+  {
+    OSDMap tmp;
+    tmp.deepish_copy_from(osdmap);
+    tmp.apply_incremental(pending_inc);
+
+    // clean pg_temp mappings
+    OSDMap::clean_temps(cct, osdmap, tmp, &pending_inc);
+
+    // clean inappropriate pg_upmap/pg_upmap_items (if any)
+    {
+      // check every upmapped pg for now
+      // until we could reliably identify certain cases to ignore,
+      // which is obviously the hard part TBD..
+      vector<pg_t> pgs_to_check;
+      tmp.get_upmap_pgs(&pgs_to_check);
+      if (pgs_to_check.size() <
+	  static_cast<uint64_t>(g_conf()->mon_clean_pg_upmaps_per_chunk * 2)) {
+        // not enough pgs, do it inline
+        tmp.clean_pg_upmaps(cct, &pending_inc);
+      } else {
+        CleanUpmapJob job(cct, tmp, pending_inc);
+        mapper.queue(&job, g_conf()->mon_clean_pg_upmaps_per_chunk, pgs_to_check);
+        job.wait();
+      }
+    }
+
+    // update creating pgs first so that we can remove the created pgid and
+    // process the pool flag removal below in the same osdmap epoch.
+    auto pending_creatings = update_pending_pgs(pending_inc, tmp);
+    bufferlist creatings_bl;
+    uint64_t features = CEPH_FEATURES_ALL;
+    if (mon.monmap->min_mon_release < ceph_release_t::octopus) {
+      dout(20) << __func__ << " encoding pending pgs without octopus features"
+	       << dendl;
+      features &= ~CEPH_FEATURE_SERVER_OCTOPUS;
+    }
+    encode(pending_creatings, creatings_bl, features);
+    t->put(OSD_PG_CREATING_PREFIX, "creating", creatings_bl);
+
+    // remove any old (or incompat) POOL_CREATING flags
+    for (auto& i : tmp.get_pools()) {
+      if (tmp.require_osd_release < ceph_release_t::nautilus) {
+	// pre-nautilus OSDMaps shouldn't get this flag.
+	if (pending_inc.new_pools.count(i.first)) {
+	  pending_inc.new_pools[i.first].flags &= ~pg_pool_t::FLAG_CREATING;
+	}
+      }
+      if (i.second.has_flag(pg_pool_t::FLAG_CREATING) &&
+	  !pending_creatings.still_creating_pool(i.first)) {
+	dout(10) << __func__ << " done creating pool " << i.first
+		 << ", clearing CREATING flag" << dendl;
+	if (pending_inc.new_pools.count(i.first) == 0) {
+	  pending_inc.new_pools[i.first] = i.second;
+	}
+	pending_inc.new_pools[i.first].flags &= ~pg_pool_t::FLAG_CREATING;
+      }
+    }
+
+    // collect which pools are currently affected by
+    // the near/backfill/full osd(s),
+    // and set per-pool near/backfill/full flag instead
+    set<int64_t> full_pool_ids;
+    set<int64_t> backfillfull_pool_ids;
+    set<int64_t> nearfull_pool_ids;
+    tmp.get_full_pools(cct,
+		       &full_pool_ids,
+		       &backfillfull_pool_ids,
+                         &nearfull_pool_ids);
+    if (full_pool_ids.empty() ||
+	backfillfull_pool_ids.empty() ||
+	nearfull_pool_ids.empty()) {
+      // normal case - no nearfull, backfillfull or full osds
+        // try cancel any improper nearfull/backfillfull/full pool
+        // flags first
+      for (auto &pool: tmp.get_pools()) {
+	auto p = pool.first;
+	if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_NEARFULL) &&
+	    nearfull_pool_ids.empty()) {
+	  dout(10) << __func__ << " clearing pool '" << tmp.pool_name[p]
+		   << "'s nearfull flag" << dendl;
+	  if (pending_inc.new_pools.count(p) == 0) {
+	    // load original pool info first!
+	    pending_inc.new_pools[p] = pool.second;
+	  }
+	  pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_NEARFULL;
+	}
+	if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_BACKFILLFULL) &&
+	    backfillfull_pool_ids.empty()) {
+	  dout(10) << __func__ << " clearing pool '" << tmp.pool_name[p]
+		   << "'s backfillfull flag" << dendl;
+	  if (pending_inc.new_pools.count(p) == 0) {
+	    pending_inc.new_pools[p] = pool.second;
+	  }
+	  pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_BACKFILLFULL;
+	}
+	if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL) &&
+	    full_pool_ids.empty()) {
+	  if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL_QUOTA)) {
+	    // set by EQUOTA, skipping
+	    continue;
+	  }
+	  dout(10) << __func__ << " clearing pool '" << tmp.pool_name[p]
+		   << "'s full flag" << dendl;
+	  if (pending_inc.new_pools.count(p) == 0) {
+	    pending_inc.new_pools[p] = pool.second;
+	  }
+	  pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_FULL;
+	}
+      }
+    }
+    if (!full_pool_ids.empty()) {
+      dout(10) << __func__ << " marking pool(s) " << full_pool_ids
+	       << " as full" << dendl;
+      for (auto &p: full_pool_ids) {
+	if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL)) {
+	  continue;
+	}
+	if (pending_inc.new_pools.count(p) == 0) {
+	  pending_inc.new_pools[p] = tmp.pools[p];
+	}
+	pending_inc.new_pools[p].flags |= pg_pool_t::FLAG_FULL;
+	pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_BACKFILLFULL;
+	pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_NEARFULL;
+      }
+      // cancel FLAG_FULL for pools which are no longer full too
+      for (auto &pool: tmp.get_pools()) {
+	auto p = pool.first;
+	if (full_pool_ids.count(p)) {
+	  // skip pools we have just marked as full above
+	  continue;
+	}
+	if (!tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL) ||
+	    tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL_QUOTA)) {
+	  // don't touch if currently is not full
+	  // or is running out of quota (and hence considered as full)
+	  continue;
+	}
+	dout(10) << __func__ << " clearing pool '" << tmp.pool_name[p]
+		 << "'s full flag" << dendl;
+	if (pending_inc.new_pools.count(p) == 0) {
+	  pending_inc.new_pools[p] = pool.second;
+	}
+	pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_FULL;
+      }
+    }
+    if (!backfillfull_pool_ids.empty()) {
+      for (auto &p: backfillfull_pool_ids) {
+	if (full_pool_ids.count(p)) {
+	  // skip pools we have already considered as full above
+	  continue;
+	}
+	if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL_QUOTA)) {
+	  // make sure FLAG_FULL is truly set, so we are safe not
+	  // to set a extra (redundant) FLAG_BACKFILLFULL flag
+	  ceph_assert(tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL));
+	  continue;
+	}
+	if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_BACKFILLFULL)) {
+	  // don't bother if pool is already marked as backfillfull
+	  continue;
+	}
+	dout(10) << __func__ << " marking pool '" << tmp.pool_name[p]
+		 << "'s as backfillfull" << dendl;
+	if (pending_inc.new_pools.count(p) == 0) {
+	  pending_inc.new_pools[p] = tmp.pools[p];
+	}
+	pending_inc.new_pools[p].flags |= pg_pool_t::FLAG_BACKFILLFULL;
+	pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_NEARFULL;
+      }
+      // cancel FLAG_BACKFILLFULL for pools
+      // which are no longer backfillfull too
+      for (auto &pool: tmp.get_pools()) {
+	auto p = pool.first;
+	if (full_pool_ids.count(p) || backfillfull_pool_ids.count(p)) {
+	  // skip pools we have just marked as backfillfull/full above
+	  continue;
+	}
+	if (!tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_BACKFILLFULL)) {
+	  // and don't touch if currently is not backfillfull
+	  continue;
+	}
+	dout(10) << __func__ << " clearing pool '" << tmp.pool_name[p]
+		 << "'s backfillfull flag" << dendl;
+	if (pending_inc.new_pools.count(p) == 0) {
+	  pending_inc.new_pools[p] = pool.second;
+	}
+	pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_BACKFILLFULL;
+      }
+    }
+    if (!nearfull_pool_ids.empty()) {
+      for (auto &p: nearfull_pool_ids) {
+	if (full_pool_ids.count(p) || backfillfull_pool_ids.count(p)) {
+	  continue;
+	}
+	if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL_QUOTA)) {
+	  // make sure FLAG_FULL is truly set, so we are safe not
+	  // to set a extra (redundant) FLAG_NEARFULL flag
+	  ceph_assert(tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_FULL));
+	  continue;
+	}
+	if (tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_NEARFULL)) {
+	  // don't bother if pool is already marked as nearfull
+	  continue;
+	}
+	dout(10) << __func__ << " marking pool '" << tmp.pool_name[p]
+		 << "'s as nearfull" << dendl;
+	if (pending_inc.new_pools.count(p) == 0) {
+	  pending_inc.new_pools[p] = tmp.pools[p];
+	}
+	pending_inc.new_pools[p].flags |= pg_pool_t::FLAG_NEARFULL;
+      }
+      // cancel FLAG_NEARFULL for pools
+      // which are no longer nearfull too
+      for (auto &pool: tmp.get_pools()) {
+	auto p = pool.first;
+	if (full_pool_ids.count(p) ||
+	    backfillfull_pool_ids.count(p) ||
+	    nearfull_pool_ids.count(p)) {
+	  // skip pools we have just marked as
+	  // nearfull/backfillfull/full above
+	  continue;
+	}
+	if (!tmp.get_pg_pool(p)->has_flag(pg_pool_t::FLAG_NEARFULL)) {
+	  // and don't touch if currently is not nearfull
+	  continue;
+	}
+	dout(10) << __func__ << " clearing pool '" << tmp.pool_name[p]
+		 << "'s nearfull flag" << dendl;
+	if (pending_inc.new_pools.count(p) == 0) {
+	  pending_inc.new_pools[p] = pool.second;
+	}
+	pending_inc.new_pools[p].flags &= ~pg_pool_t::FLAG_NEARFULL;
+      }
+    }
+
+    // min_compat_client?
+    if (!tmp.require_min_compat_client) {
+      auto mv = tmp.get_min_compat_client();
+      dout(1) << __func__ << " setting require_min_compat_client to currently "
+	      << "required " << mv << dendl;
+      mon.clog->info() << "setting require_min_compat_client to currently "
+			<< "required " << mv;
+      pending_inc.new_require_min_compat_client = mv;
+    }
+
+    if (osdmap.require_osd_release < ceph_release_t::nautilus &&
+	tmp.require_osd_release >= ceph_release_t::nautilus) {
+      dout(10) << __func__ << " first nautilus+ epoch" << dendl;
+      // add creating flags?
+      for (auto& i : tmp.get_pools()) {
+	if (pending_creatings.still_creating_pool(i.first)) {
+	  dout(10) << __func__ << " adding CREATING flag to pool " << i.first
+		   << dendl;
+	  if (pending_inc.new_pools.count(i.first) == 0) {
+	    pending_inc.new_pools[i.first] = i.second;
+	  }
+	  pending_inc.new_pools[i.first].flags |= pg_pool_t::FLAG_CREATING;
+	}
+      }
+      // adjust blocklist items to all be TYPE_ANY
+      for (auto& i : tmp.blocklist) {
+	auto a = i.first;
+	a.set_type(entity_addr_t::TYPE_ANY);
+	pending_inc.new_blocklist[a] = i.second;
+	pending_inc.old_blocklist.push_back(i.first);
+      }
+    }
+
+    if (osdmap.require_osd_release < ceph_release_t::octopus &&
+	tmp.require_osd_release >= ceph_release_t::octopus) {
+      dout(10) << __func__ << " first octopus+ epoch" << dendl;
+
+      // adjust obsoleted cache modes
+      for (auto& [poolid, pi] : tmp.pools) {
+	if (pi.cache_mode == pg_pool_t::CACHEMODE_FORWARD) {
+	  if (pending_inc.new_pools.count(poolid) == 0) {
+	    pending_inc.new_pools[poolid] = pi;
+	  }
+	  dout(10) << __func__ << " switching pool " << poolid
+		   << " cachemode from forward -> proxy" << dendl;
+	  pending_inc.new_pools[poolid].cache_mode = pg_pool_t::CACHEMODE_PROXY;
+	}
+	if (pi.cache_mode == pg_pool_t::CACHEMODE_READFORWARD) {
+	  if (pending_inc.new_pools.count(poolid) == 0) {
+	    pending_inc.new_pools[poolid] = pi;
+	  }
+	  dout(10) << __func__ << " switching pool " << poolid
+		   << " cachemode from readforward -> readproxy" << dendl;
+	  pending_inc.new_pools[poolid].cache_mode =
+	    pg_pool_t::CACHEMODE_READPROXY;
+	}
+      }
+
+      // clear removed_snaps for every pool
+      for (auto& [poolid, pi] : tmp.pools) {
+	if (pi.removed_snaps.empty()) {
+	  continue;
+	}
+	if (pending_inc.new_pools.count(poolid) == 0) {
+	  pending_inc.new_pools[poolid] = pi;
+	}
+	dout(10) << __func__ << " clearing pool " << poolid << " removed_snaps"
+		 << dendl;
+	pending_inc.new_pools[poolid].removed_snaps.clear();
+      }
+
+      // create a combined purged snap epoch key for all purged snaps
+      // prior to this epoch, and store it in the current epoch (i.e.,
+      // the last pre-octopus epoch, just prior to the one we're
+      // encoding now).
+      auto it = mon.store->get_iterator(OSD_SNAP_PREFIX);
+      it->lower_bound("purged_snap_");
+      map<int64_t,snap_interval_set_t> combined;
+      while (it->valid()) {
+	if (it->key().find("purged_snap_") != 0) {
+	  break;
+	}
+	string k = it->key();
+	long long unsigned pool;
+	int n = sscanf(k.c_str(), "purged_snap_%llu_", &pool);
+	if (n != 1) {
+	  derr << __func__ << " invalid purged_snaps key '" << k << "'" << dendl;
+	} else {
+	  bufferlist v = it->value();
+	  auto p = v.cbegin();
+	  snapid_t begin, end;
+	  ceph::decode(begin, p);
+	  ceph::decode(end, p);
+	  combined[pool].insert(begin, end - begin);
+	}
+	it->next();
+      }
+      if (!combined.empty()) {
+	string k = make_purged_snap_epoch_key(pending_inc.epoch - 1);
+	bufferlist v;
+	ceph::encode(combined, v);
+	t->put(OSD_SNAP_PREFIX, k, v);
+	dout(10) << __func__ << " recording pre-octopus purged_snaps in epoch "
+		 << (pending_inc.epoch - 1) << ", " << v.length() << " bytes"
+		 << dendl;
+      } else {
+	dout(10) << __func__ << " there were no pre-octopus purged snaps"
+		 << dendl;
+      }
+
+      // clean out the old removed_snap_ and removed_epoch keys
+      // ('`' is ASCII '_' + 1)
+      t->erase_range(OSD_SNAP_PREFIX, "removed_snap_", "removed_snap`");
+      t->erase_range(OSD_SNAP_PREFIX, "removed_epoch_", "removed_epoch`");
+    }
+  }
+
+  // tell me about it
+  for (auto i = pending_inc.new_state.begin();
+       i != pending_inc.new_state.end();
+       ++i) {
+    int s = i->second ? i->second : CEPH_OSD_UP;
+    if (s & CEPH_OSD_UP) {
+      dout(2) << " osd." << i->first << " DOWN" << dendl;
+      // Reset laggy parameters if failure interval exceeds a threshold.
+      const osd_xinfo_t& xi = osdmap.get_xinfo(i->first);
+      if ((xi.laggy_probability || xi.laggy_interval) && xi.down_stamp.sec()) {
+        int last_failure_interval = pending_inc.modified.sec() - xi.down_stamp.sec();
+        if (grace_interval_threshold_exceeded(last_failure_interval)) {
+          set_default_laggy_params(i->first);
+        }
+      }
+    }
+    if (s & CEPH_OSD_EXISTS)
+      dout(2) << " osd." << i->first << " DNE" << dendl;
+  }
+  for (auto i = pending_inc.new_up_client.begin();
+       i != pending_inc.new_up_client.end();
+       ++i) {
+    //FIXME: insert cluster addresses too
+    dout(2) << " osd." << i->first << " UP " << i->second << dendl;
+  }
+  for (map<int32_t,uint32_t>::iterator i = pending_inc.new_weight.begin();
+       i != pending_inc.new_weight.end();
+       ++i) {
+    if (i->second == CEPH_OSD_OUT) {
+      dout(2) << " osd." << i->first << " OUT" << dendl;
+    } else if (i->second == CEPH_OSD_IN) {
+      dout(2) << " osd." << i->first << " IN" << dendl;
+    } else {
+      dout(2) << " osd." << i->first << " WEIGHT " << hex << i->second << dec << dendl;
+    }
+  }
+
+  // features for osdmap and its incremental
+  uint64_t features;
+
+  // encode full map and determine its crc
+  OSDMap tmp;
+  {
+    tmp.deepish_copy_from(osdmap);
+    tmp.apply_incremental(pending_inc);
+
+    // determine appropriate features
+    features = tmp.get_encoding_features();
+    dout(10) << __func__ << " encoding full map with "
+	     << tmp.require_osd_release
+	     << " features " << features << dendl;
+
+    // the features should be a subset of the mon quorum's features!
+    ceph_assert((features & ~mon.get_quorum_con_features()) == 0);
+
+    bufferlist fullbl;
+    encode(tmp, fullbl, features | CEPH_FEATURE_RESERVED);
+    pending_inc.full_crc = tmp.get_crc();
+
+    // include full map in the txn.  note that old monitors will
+    // overwrite this.  new ones will now skip the local full map
+    // encode and reload from this.
+    put_version_full(t, pending_inc.epoch, fullbl);
+  }
+
+  // encode
+  ceph_assert(get_last_committed() + 1 == pending_inc.epoch);
+  bufferlist bl;
+  encode(pending_inc, bl, features | CEPH_FEATURE_RESERVED);
+
+  dout(20) << " full_crc " << tmp.get_crc()
+	   << " inc_crc " << pending_inc.inc_crc << dendl;
+
+  /* put everything in the transaction */
+  put_version(t, pending_inc.epoch, bl);
+  put_last_committed(t, pending_inc.epoch);
+
+  // metadata, too!
+  for (map<int,bufferlist>::iterator p = pending_metadata.begin();
+       p != pending_metadata.end();
+       ++p)
+    t->put(OSD_METADATA_PREFIX, stringify(p->first), p->second);
+  for (set<int>::iterator p = pending_metadata_rm.begin();
+       p != pending_metadata_rm.end();
+       ++p)
+    t->erase(OSD_METADATA_PREFIX, stringify(*p));
+  pending_metadata.clear();
+  pending_metadata_rm.clear();
+
+  // purged_snaps
+  if (tmp.require_osd_release >= ceph_release_t::octopus &&
+      !pending_inc.new_purged_snaps.empty()) {
+    // all snaps purged this epoch (across all pools)
+    string k = make_purged_snap_epoch_key(pending_inc.epoch);
+    bufferlist v;
+    encode(pending_inc.new_purged_snaps, v);
+    t->put(OSD_SNAP_PREFIX, k, v);
+  }
+  for (auto& i : pending_inc.new_purged_snaps) {
+    for (auto q = i.second.begin();
+	 q != i.second.end();
+	 ++q) {
+      insert_purged_snap_update(i.first, q.get_start(), q.get_end(),
+				pending_inc.epoch,
+				t);
+    }
+  }
+  for (auto& [pool, snaps] : pending_pseudo_purged_snaps) {
+    for (auto snap : snaps) {
+      insert_purged_snap_update(pool, snap, snap + 1,
+				pending_inc.epoch,
+				t);
+    }
+  }
+
+  // health
+  health_check_map_t next;
+  tmp.check_health(cct, &next);
+  encode_health(next, t);
+}
+
+int OSDMonitor::load_metadata(int osd, map<string, string>& m, ostream *err)
+{
+  bufferlist bl;
+  int r = mon.store->get(OSD_METADATA_PREFIX, stringify(osd), bl);
+  if (r < 0)
+    return r;
+  try {
+    auto p = bl.cbegin();
+    decode(m, p);
+  }
+  catch (ceph::buffer::error& e) {
+    if (err)
+      *err << "osd." << osd << " metadata is corrupt";
+    return -EIO;
+  }
+  return 0;
+}
+
+void OSDMonitor::count_metadata(const string& field, map<string,int> *out)
+{
+  for (int osd = 0; osd < osdmap.get_max_osd(); ++osd) {
+    if (osdmap.is_up(osd)) {
+      map<string,string> meta;
+      load_metadata(osd, meta, nullptr);
+      auto p = meta.find(field);
+      if (p == meta.end()) {
+	(*out)["unknown"]++;
+      } else {
+	(*out)[p->second]++;
+      }
+    }
+  }
+}
+
+void OSDMonitor::count_metadata(const string& field, Formatter *f)
+{
+  map<string,int> by_val;
+  count_metadata(field, &by_val);
+  f->open_object_section(field.c_str());
+  for (auto& p : by_val) {
+    f->dump_int(p.first.c_str(), p.second);
+  }
+  f->close_section();
+}
+
+void OSDMonitor::get_versions(std::map<string, list<string>> &versions)
+{
+  for (int osd = 0; osd < osdmap.get_max_osd(); ++osd) {
+    if (osdmap.is_up(osd)) {
+      map<string,string> meta;
+      load_metadata(osd, meta, nullptr);
+      auto p = meta.find("ceph_version_short");
+      if (p == meta.end()) continue;
+      versions[p->second].push_back(string("osd.") + stringify(osd));
+    }
+  }
+}
+
+int OSDMonitor::get_osd_objectstore_type(int osd, string *type)
+{
+  map<string, string> metadata;
+  int r = load_metadata(osd, metadata, nullptr);
+  if (r < 0)
+    return r;
+
+  auto it = metadata.find("osd_objectstore");
+  if (it == metadata.end())
+    return -ENOENT;
+  *type = it->second;
+  return 0;
+}
+
+bool OSDMonitor::is_pool_currently_all_bluestore(int64_t pool_id,
+						 const pg_pool_t &pool,
+						 ostream *err)
+{
+  // just check a few pgs for efficiency - this can't give a guarantee anyway,
+  // since filestore osds could always join the pool later
+  set<int> checked_osds;
+  for (unsigned ps = 0; ps < std::min(8u, pool.get_pg_num()); ++ps) {
+    vector<int> up, acting;
+    pg_t pgid(ps, pool_id);
+    osdmap.pg_to_up_acting_osds(pgid, up, acting);
+    for (int osd : up) {
+      if (checked_osds.find(osd) != checked_osds.end())
+	continue;
+      string objectstore_type;
+      int r = get_osd_objectstore_type(osd, &objectstore_type);
+      // allow with missing metadata, e.g. due to an osd never booting yet
+      if (r < 0 || objectstore_type == "bluestore") {
+	checked_osds.insert(osd);
+	continue;
+      }
+      *err << "osd." << osd << " uses " << objectstore_type;
+      return false;
+    }
+  }
+  return true;
+}
+
+int OSDMonitor::dump_osd_metadata(int osd, Formatter *f, ostream *err)
+{
+  map<string,string> m;
+  if (int r = load_metadata(osd, m, err))
+    return r;
+  for (map<string,string>::iterator p = m.begin(); p != m.end(); ++p)
+    f->dump_string(p->first.c_str(), p->second);
+  return 0;
+}
+
+void OSDMonitor::print_nodes(Formatter *f)
+{
+  // group OSDs by their hosts
+  map<string, list<int> > osds; // hostname => osd
+  for (int osd = 0; osd < osdmap.get_max_osd(); osd++) {
+    map<string, string> m;
+    if (load_metadata(osd, m, NULL)) {
+      continue;
+    }
+    map<string, string>::iterator hostname = m.find("hostname");
+    if (hostname == m.end()) {
+      // not likely though
+      continue;
+    }
+    osds[hostname->second].push_back(osd);
+  }
+
+  dump_services(f, osds, "osd");
+}
+
+void OSDMonitor::share_map_with_random_osd()
+{
+  if (osdmap.get_num_up_osds() == 0) {
+    dout(10) << __func__ << " no up osds, don't share with anyone" << dendl;
+    return;
+  }
+
+  MonSession *s = mon.session_map.get_random_osd_session(&osdmap);
+  if (!s) {
+    dout(10) << __func__ << " no up osd on our session map" << dendl;
+    return;
+  }
+
+  dout(10) << "committed, telling random " << s->name
+	   << " all about it" << dendl;
+
+  // get feature of the peer
+  // use quorum_con_features, if it's an anonymous connection.
+  uint64_t features = s->con_features ? s->con_features :
+                                        mon.get_quorum_con_features();
+  // whatev, they'll request more if they need it
+  MOSDMap *m = build_incremental(osdmap.get_epoch() - 1, osdmap.get_epoch(), features);
+  s->con->send_message(m);
+  // NOTE: do *not* record osd has up to this epoch (as we do
+  // elsewhere) as they may still need to request older values.
+}
+
+version_t OSDMonitor::get_trim_to() const
+{
+  if (mon.get_quorum().empty()) {
+    dout(10) << __func__ << " quorum not formed, trim_to = 0" << dendl;
+    return 0;
+  }
+
+  {
+    std::lock_guard<std::mutex> l(creating_pgs_lock);
+    if (!creating_pgs.pgs.empty()) {
+      dout(10) << __func__ << " pgs creating, trim_to = 0" << dendl;
+      return 0;
+    }
+  }
+
+  if (g_conf().get_val<bool>("mon_debug_block_osdmap_trim")) {
+    dout(0) << __func__
+            << " blocking osdmap trim"
+            << " ('mon_debug_block_osdmap_trim' set to 'true')"
+            << " trim_to = 0" << dendl;
+    return 0;
+  }
+
+  {
+    epoch_t floor = get_min_last_epoch_clean();
+    dout(10) << " min_last_epoch_clean " << floor << dendl;
+    if (g_conf()->mon_osd_force_trim_to > 0 &&
+	g_conf()->mon_osd_force_trim_to < (int)get_last_committed()) {
+      floor = g_conf()->mon_osd_force_trim_to;
+      dout(10) << __func__
+               << " explicit mon_osd_force_trim_to = " << floor << dendl;
+    }
+    unsigned min = g_conf()->mon_min_osdmap_epochs;
+    if (floor + min > get_last_committed()) {
+      if (min < get_last_committed())
+	floor = get_last_committed() - min;
+      else
+	floor = 0;
+    }
+    if (floor > get_first_committed()) {
+      dout(10) << __func__ << " trim_to = " << floor << dendl;
+      return floor;
+    }
+  }
+  dout(10) << __func__ << " trim_to = 0" << dendl;
+  return 0;
+}
+
+epoch_t OSDMonitor::get_min_last_epoch_clean() const
+{
+  auto floor = last_epoch_clean.get_lower_bound(osdmap);
+  // also scan osd epochs
+  // don't trim past the oldest reported osd epoch
+  for (auto [osd, epoch] : osd_epochs) {
+    if (epoch < floor) {
+      floor = epoch;
+    }
+  }
+  return floor;
+}
+
+void OSDMonitor::encode_trim_extra(MonitorDBStore::TransactionRef tx,
+				   version_t first)
+{
+  dout(10) << __func__ << " including full map for e " << first << dendl;
+  bufferlist bl;
+  get_version_full(first, bl);
+  put_version_full(tx, first, bl);
+
+  if (has_osdmap_manifest &&
+      first > osdmap_manifest.get_first_pinned()) {
+    _prune_update_trimmed(tx, first);
+  }
+}
+
+
+/* full osdmap prune
+ *
+ * for more information, please refer to doc/dev/mon-osdmap-prune.rst
+ */
+
+void OSDMonitor::load_osdmap_manifest()
+{
+  bool store_has_manifest =
+    mon.store->exists(get_service_name(), "osdmap_manifest");
+
+  if (!store_has_manifest) {
+    if (!has_osdmap_manifest) {
+      return;
+    }
+
+    dout(20) << __func__
+             << " dropping osdmap manifest from memory." << dendl;
+    osdmap_manifest = osdmap_manifest_t();
+    has_osdmap_manifest = false;
+    return;
+  }
+
+  dout(20) << __func__
+           << " osdmap manifest detected in store; reload." << dendl;
+
+  bufferlist manifest_bl;
+  int r = get_value("osdmap_manifest", manifest_bl);
+  if (r < 0) {
+    derr << __func__ << " unable to read osdmap version manifest" << dendl;
+    ceph_abort_msg("error reading manifest");
+  }
+  osdmap_manifest.decode(manifest_bl);
+  has_osdmap_manifest = true;
+
+  dout(10) << __func__ << " store osdmap manifest pinned ("
+           << osdmap_manifest.get_first_pinned()
+           << " .. "
+           << osdmap_manifest.get_last_pinned()
+           << ")"
+           << dendl;
+}
+
+bool OSDMonitor::should_prune() const
+{
+  version_t first = get_first_committed();
+  version_t last = get_last_committed();
+  version_t min_osdmap_epochs =
+    g_conf().get_val<int64_t>("mon_min_osdmap_epochs");
+  version_t prune_min =
+    g_conf().get_val<uint64_t>("mon_osdmap_full_prune_min");
+  version_t prune_interval =
+    g_conf().get_val<uint64_t>("mon_osdmap_full_prune_interval");
+  version_t last_pinned = osdmap_manifest.get_last_pinned();
+  version_t last_to_pin = last - min_osdmap_epochs;
+
+  // Make it or break it constraints.
+  //
+  // If any of these conditions fails, we will not prune, regardless of
+  // whether we have an on-disk manifest with an on-going pruning state.
+  //
+  if ((last - first) <= min_osdmap_epochs) {
+    // between the first and last committed epochs, we don't have
+    // enough epochs to trim, much less to prune.
+    dout(10) << __func__
+             << " currently holding only " << (last - first)
+             << " epochs (min osdmap epochs: " << min_osdmap_epochs
+             << "); do not prune."
+             << dendl;
+    return false;
+
+  } else if ((last_to_pin - first) < prune_min) {
+    // between the first committed epoch and the last epoch we would prune,
+    // we simply don't have enough versions over the minimum to prune maps.
+    dout(10) << __func__
+             << " could only prune " << (last_to_pin - first)
+             << " epochs (" << first << ".." << last_to_pin << "), which"
+                " is less than the required minimum (" << prune_min << ")"
+             << dendl;
+    return false;
+
+  } else if (has_osdmap_manifest && last_pinned >= last_to_pin) {
+    dout(10) << __func__
+             << " we have pruned as far as we can; do not prune."
+             << dendl;
+    return false;
+
+  } else if (last_pinned + prune_interval > last_to_pin) {
+    dout(10) << __func__
+             << " not enough epochs to form an interval (last pinned: "
+             << last_pinned << ", last to pin: "
+             << last_to_pin << ", interval: " << prune_interval << ")"
+             << dendl;
+    return false;
+  }
+
+  dout(15) << __func__
+           << " should prune (" << last_pinned << ".." << last_to_pin << ")"
+           << " lc (" << first << ".." << last << ")"
+           << dendl;
+  return true;
+}
+
+void OSDMonitor::_prune_update_trimmed(
+    MonitorDBStore::TransactionRef tx,
+    version_t first)
+{
+  dout(10) << __func__
+           << " first " << first
+           << " last_pinned " << osdmap_manifest.get_last_pinned()
+           << dendl;
+
+  osdmap_manifest_t manifest = osdmap_manifest;
+
+  if (!manifest.is_pinned(first)) {
+    manifest.pin(first);
+  }
+
+  set<version_t>::iterator p_end = manifest.pinned.find(first);
+  set<version_t>::iterator p = manifest.pinned.begin();
+  manifest.pinned.erase(p, p_end);
+  ceph_assert(manifest.get_first_pinned() == first);
+
+  if (manifest.get_last_pinned() == first+1 ||
+      manifest.pinned.size() == 1) {
+    // we reached the end of the line, as pinned maps go; clean up our
+    // manifest, and let `should_prune()` decide whether we should prune
+    // again.
+    tx->erase(get_service_name(), "osdmap_manifest");
+    return;
+  }
+
+  bufferlist bl;
+  manifest.encode(bl);
+  tx->put(get_service_name(), "osdmap_manifest", bl);
+}
+
+void OSDMonitor::prune_init(osdmap_manifest_t& manifest)
+{
+  dout(1) << __func__ << dendl;
+
+  version_t pin_first;
+
+  // verify constrainsts on stable in-memory state
+  if (!has_osdmap_manifest) {
+    // we must have never pruned, OR if we pruned the state must no longer
+    // be relevant (i.e., the state must have been removed alongside with
+    // the trim that *must* have removed past the last pinned map in a
+    // previous prune).
+    ceph_assert(osdmap_manifest.pinned.empty());
+    ceph_assert(!mon.store->exists(get_service_name(), "osdmap_manifest"));
+    pin_first = get_first_committed();
+
+  } else {
+    // we must have pruned in the past AND its state is still relevant
+    // (i.e., even if we trimmed, we still hold pinned maps in the manifest,
+    // and thus we still hold a manifest in the store).
+    ceph_assert(!osdmap_manifest.pinned.empty());
+    ceph_assert(osdmap_manifest.get_first_pinned() == get_first_committed());
+    ceph_assert(osdmap_manifest.get_last_pinned() < get_last_committed());
+
+    dout(10) << __func__
+             << " first_pinned " << osdmap_manifest.get_first_pinned()
+             << " last_pinned " << osdmap_manifest.get_last_pinned()
+             << dendl;
+
+    pin_first = osdmap_manifest.get_last_pinned();
+  }
+
+  manifest.pin(pin_first);
+}
+
+bool OSDMonitor::_prune_sanitize_options() const
+{
+  uint64_t prune_interval =
+    g_conf().get_val<uint64_t>("mon_osdmap_full_prune_interval");
+  uint64_t prune_min =
+    g_conf().get_val<uint64_t>("mon_osdmap_full_prune_min");
+  uint64_t txsize =
+    g_conf().get_val<uint64_t>("mon_osdmap_full_prune_txsize");
+
+  bool r = true;
+
+  if (prune_interval == 0) {
+    derr << __func__
+         << " prune is enabled BUT prune interval is zero; abort."
+         << dendl;
+    r = false;
+  } else if (prune_interval == 1) {
+    derr << __func__
+         << " prune interval is equal to one, which essentially means"
+            " no pruning; abort."
+         << dendl;
+    r = false;
+  }
+  if (prune_min == 0) {
+    derr << __func__
+         << " prune is enabled BUT prune min is zero; abort."
+         << dendl;
+    r = false;
+  }
+  if (prune_interval > prune_min) {
+    derr << __func__
+         << " impossible to ascertain proper prune interval because"
+         << " it is greater than the minimum prune epochs"
+         << " (min: " << prune_min << ", interval: " << prune_interval << ")"
+         << dendl;
+    r = false;
+  }
+
+  if (txsize < prune_interval - 1) {
+    derr << __func__
+         << " 'mon_osdmap_full_prune_txsize' (" << txsize
+         << ") < 'mon_osdmap_full_prune_interval-1' (" << prune_interval - 1
+         << "); abort." << dendl;
+    r = false;
+  }
+  return r;
+}
+
+bool OSDMonitor::is_prune_enabled() const {
+  return g_conf().get_val<bool>("mon_osdmap_full_prune_enabled");
+}
+
+bool OSDMonitor::is_prune_supported() const {
+  return mon.get_required_mon_features().contains_any(
+      ceph::features::mon::FEATURE_OSDMAP_PRUNE);
+}
+
+/** do_prune
+ *
+ * @returns true if has side-effects; false otherwise.
+ */
+bool OSDMonitor::do_prune(MonitorDBStore::TransactionRef tx)
+{
+  bool enabled = is_prune_enabled();
+
+  dout(1) << __func__ << " osdmap full prune "
+          << ( enabled ? "enabled" : "disabled")
+          << dendl;
+
+  if (!enabled || !_prune_sanitize_options() || !should_prune()) {
+    return false;
+  }
+
+  // we are beyond the minimum prune versions, we need to remove maps because
+  // otherwise the store will grow unbounded and we may end up having issues
+  // with available disk space or store hangs.
+
+  // we will not pin all versions. We will leave a buffer number of versions.
+  // this allows us the monitor to trim maps without caring too much about
+  // pinned maps, and then allow us to use another ceph-mon without these
+  // capabilities, without having to repair the store.
+
+  osdmap_manifest_t manifest = osdmap_manifest;
+
+  version_t first = get_first_committed();
+  version_t last = get_last_committed();
+
+  version_t last_to_pin = last - g_conf()->mon_min_osdmap_epochs;
+  version_t last_pinned = manifest.get_last_pinned();
+  uint64_t prune_interval =
+    g_conf().get_val<uint64_t>("mon_osdmap_full_prune_interval");
+  uint64_t txsize =
+    g_conf().get_val<uint64_t>("mon_osdmap_full_prune_txsize");
+
+  prune_init(manifest);
+
+  // we need to get rid of some osdmaps
+
+  dout(5) << __func__
+          << " lc (" << first << " .. " << last << ")"
+          << " last_pinned " << last_pinned
+          << " interval " << prune_interval
+          << " last_to_pin " << last_to_pin
+          << dendl;
+
+  // We will be erasing maps as we go.
+  //
+  // We will erase all maps between `last_pinned` and the `next_to_pin`.
+  //
+  // If `next_to_pin` happens to be greater than `last_to_pin`, then
+  // we stop pruning. We could prune the maps between `next_to_pin` and
+  // `last_to_pin`, but by not doing it we end up with neater pruned
+  // intervals, aligned with `prune_interval`. Besides, this should not be a
+  // problem as long as `prune_interval` is set to a sane value, instead of
+  // hundreds or thousands of maps.
+
+  auto map_exists = [this](version_t v) {
+    string k = mon.store->combine_strings("full", v);
+    return mon.store->exists(get_service_name(), k);
+  };
+
+  // 'interval' represents the number of maps from the last pinned
+  // i.e., if we pinned version 1 and have an interval of 10, we're pinning
+  // version 11 next; all intermediate versions will be removed.
+  //
+  // 'txsize' represents the maximum number of versions we'll be removing in
+  // this iteration. If 'txsize' is large enough to perform multiple passes
+  // pinning and removing maps, we will do so; if not, we'll do at least one
+  // pass. We are quite relaxed about honouring 'txsize', but we'll always
+  // ensure that we never go *over* the maximum.
+
+  // e.g., if we pin 1 and 11, we're removing versions [2..10]; i.e., 9 maps.
+  uint64_t removal_interval = prune_interval - 1;
+
+  if (txsize < removal_interval) {
+    dout(5) << __func__
+	    << " setting txsize to removal interval size ("
+	    << removal_interval << " versions"
+	    << dendl;
+    txsize = removal_interval;
+  }
+  ceph_assert(removal_interval > 0);
+
+  uint64_t num_pruned = 0;
+  while (num_pruned + removal_interval <= txsize) { 
+    last_pinned = manifest.get_last_pinned();
+
+    if (last_pinned + prune_interval > last_to_pin) {
+      break;
+    }
+    ceph_assert(last_pinned < last_to_pin);
+
+    version_t next_pinned = last_pinned + prune_interval;
+    ceph_assert(next_pinned <= last_to_pin);
+    manifest.pin(next_pinned);
+
+    dout(20) << __func__
+	     << " last_pinned " << last_pinned
+	     << " next_pinned " << next_pinned
+	     << " num_pruned " << num_pruned
+	     << " removal interval (" << (last_pinned+1)
+	     << ".." << (next_pinned-1) << ")"
+	     << " txsize " << txsize << dendl;
+
+    ceph_assert(map_exists(last_pinned));
+    ceph_assert(map_exists(next_pinned));
+
+    for (version_t v = last_pinned+1; v < next_pinned; ++v) {
+      ceph_assert(!manifest.is_pinned(v));
+
+      dout(20) << __func__ << "   pruning full osdmap e" << v << dendl;
+      string full_key = mon.store->combine_strings("full", v);
+      tx->erase(get_service_name(), full_key);
+      ++num_pruned;
+    }
+  }
+
+  ceph_assert(num_pruned > 0);
+
+  bufferlist bl;
+  manifest.encode(bl);
+  tx->put(get_service_name(), "osdmap_manifest", bl);
+
+  return true;
+}
+
+
+// -------------
+
+bool OSDMonitor::preprocess_query(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  Message *m = op->get_req();
+  dout(10) << "preprocess_query " << *m << " from " << m->get_orig_source_inst() << dendl;
+
+  switch (m->get_type()) {
+    // READs
+  case MSG_MON_COMMAND:
+    try {
+      return preprocess_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+  case CEPH_MSG_MON_GET_OSDMAP:
+    return preprocess_get_osdmap(op);
+
+    // damp updates
+  case MSG_OSD_MARK_ME_DOWN:
+    return preprocess_mark_me_down(op);
+  case MSG_OSD_MARK_ME_DEAD:
+    return preprocess_mark_me_dead(op);
+  case MSG_OSD_FULL:
+    return preprocess_full(op);
+  case MSG_OSD_FAILURE:
+    return preprocess_failure(op);
+  case MSG_OSD_BOOT:
+    return preprocess_boot(op);
+  case MSG_OSD_ALIVE:
+    return preprocess_alive(op);
+  case MSG_OSD_PG_CREATED:
+    return preprocess_pg_created(op);
+  case MSG_OSD_PG_READY_TO_MERGE:
+    return preprocess_pg_ready_to_merge(op);
+  case MSG_OSD_PGTEMP:
+    return preprocess_pgtemp(op);
+  case MSG_OSD_BEACON:
+    return preprocess_beacon(op);
+
+  case CEPH_MSG_POOLOP:
+    return preprocess_pool_op(op);
+
+  case MSG_REMOVE_SNAPS:
+    return preprocess_remove_snaps(op);
+
+  case MSG_MON_GET_PURGED_SNAPS:
+    return preprocess_get_purged_snaps(op);
+
+  default:
+    ceph_abort();
+    return true;
+  }
+}
+
+bool OSDMonitor::prepare_update(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  Message *m = op->get_req();
+  dout(7) << "prepare_update " << *m << " from " << m->get_orig_source_inst() << dendl;
+
+  switch (m->get_type()) {
+    // damp updates
+  case MSG_OSD_MARK_ME_DOWN:
+    return prepare_mark_me_down(op);
+  case MSG_OSD_MARK_ME_DEAD:
+    return prepare_mark_me_dead(op);
+  case MSG_OSD_FULL:
+    return prepare_full(op);
+  case MSG_OSD_FAILURE:
+    return prepare_failure(op);
+  case MSG_OSD_BOOT:
+    return prepare_boot(op);
+  case MSG_OSD_ALIVE:
+    return prepare_alive(op);
+  case MSG_OSD_PG_CREATED:
+    return prepare_pg_created(op);
+  case MSG_OSD_PGTEMP:
+    return prepare_pgtemp(op);
+  case MSG_OSD_PG_READY_TO_MERGE:
+    return prepare_pg_ready_to_merge(op);
+  case MSG_OSD_BEACON:
+    return prepare_beacon(op);
+
+  case MSG_MON_COMMAND:
+    try {
+      return prepare_command(op);
+    } catch (const bad_cmd_get& e) {
+      bufferlist bl;
+      mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+      return true;
+    }
+
+  case CEPH_MSG_POOLOP:
+    return prepare_pool_op(op);
+
+  case MSG_REMOVE_SNAPS:
+    return prepare_remove_snaps(op);
+
+
+  default:
+    ceph_abort();
+  }
+
+  return false;
+}
+
+bool OSDMonitor::should_propose(double& delay)
+{
+  dout(10) << "should_propose" << dendl;
+
+  // if full map, propose immediately!  any subsequent changes will be clobbered.
+  if (pending_inc.fullmap.length())
+    return true;
+
+  // adjust osd weights?
+  if (!osd_weight.empty() &&
+      osd_weight.size() == (unsigned)osdmap.get_max_osd()) {
+    dout(0) << " adjusting osd weights based on " << osd_weight << dendl;
+    osdmap.adjust_osd_weights(osd_weight, pending_inc);
+    delay = 0.0;
+    osd_weight.clear();
+    return true;
+  }
+
+  return PaxosService::should_propose(delay);
+}
+
+
+
+// ---------------------------
+// READs
+
+bool OSDMonitor::preprocess_get_osdmap(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MMonGetOSDMap>();
+
+  uint64_t features = mon.get_quorum_con_features();
+  if (op->get_session() && op->get_session()->con_features)
+    features = op->get_session()->con_features;
+
+  dout(10) << __func__ << " " << *m << dendl;
+  MOSDMap *reply = new MOSDMap(mon.monmap->fsid, features);
+  epoch_t first = get_first_committed();
+  epoch_t last = osdmap.get_epoch();
+  int max = g_conf()->osd_map_message_max;
+  ssize_t max_bytes = g_conf()->osd_map_message_max_bytes;
+  for (epoch_t e = std::max(first, m->get_full_first());
+       e <= std::min(last, m->get_full_last()) && max > 0 && max_bytes > 0;
+       ++e, --max) {
+    bufferlist& bl = reply->maps[e];
+    int r = get_version_full(e, features, bl);
+    ceph_assert(r >= 0);
+    max_bytes -= bl.length();
+  }
+  for (epoch_t e = std::max(first, m->get_inc_first());
+       e <= std::min(last, m->get_inc_last()) && max > 0 && max_bytes > 0;
+       ++e, --max) {
+    bufferlist& bl = reply->incremental_maps[e];
+    int r = get_version(e, features, bl);
+    ceph_assert(r >= 0);
+    max_bytes -= bl.length();
+  }
+  reply->oldest_map = first;
+  reply->newest_map = last;
+  mon.send_reply(op, reply);
+  return true;
+}
+
+
+// ---------------------------
+// UPDATEs
+
+// failure --
+
+bool OSDMonitor::check_source(MonOpRequestRef op, uuid_d fsid) {
+  // check permissions
+  MonSession *session = op->get_session();
+  if (!session)
+    return true;
+  if (!session->is_capable("osd", MON_CAP_X)) {
+    dout(0) << "got MOSDFailure from entity with insufficient caps "
+	    << session->caps << dendl;
+    return true;
+  }
+  if (fsid != mon.monmap->fsid) {
+    dout(0) << "check_source: on fsid " << fsid
+	    << " != " << mon.monmap->fsid << dendl;
+    return true;
+  }
+  return false;
+}
+
+
+bool OSDMonitor::preprocess_failure(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDFailure>();
+  // who is target_osd
+  int badboy = m->get_target_osd();
+
+  // check permissions
+  if (check_source(op, m->fsid))
+    goto didit;
+
+  // first, verify the reporting host is valid
+  if (m->get_orig_source().is_osd()) {
+    int from = m->get_orig_source().num();
+    if (!osdmap.exists(from) ||
+	!osdmap.get_addrs(from).legacy_equals(m->get_orig_source_addrs()) ||
+	(osdmap.is_down(from) && m->if_osd_failed())) {
+      dout(5) << "preprocess_failure from dead osd." << from
+	      << ", ignoring" << dendl;
+      send_incremental(op, m->get_epoch()+1);
+      goto didit;
+    }
+  }
+
+
+  // weird?
+  if (osdmap.is_down(badboy)) {
+    dout(5) << "preprocess_failure dne(/dup?): osd." << m->get_target_osd()
+	    << " " << m->get_target_addrs()
+	    << ", from " << m->get_orig_source() << dendl;
+    if (m->get_epoch() < osdmap.get_epoch())
+      send_incremental(op, m->get_epoch()+1);
+    goto didit;
+  }
+  if (osdmap.get_addrs(badboy) != m->get_target_addrs()) {
+    dout(5) << "preprocess_failure wrong osd: report osd." << m->get_target_osd()
+	    << " " << m->get_target_addrs()
+	    << " != map's " << osdmap.get_addrs(badboy)
+	    << ", from " << m->get_orig_source() << dendl;
+    if (m->get_epoch() < osdmap.get_epoch())
+      send_incremental(op, m->get_epoch()+1);
+    goto didit;
+  }
+
+  // already reported?
+  if (osdmap.is_down(badboy) ||
+      osdmap.get_up_from(badboy) > m->get_epoch()) {
+    dout(5) << "preprocess_failure dup/old: osd." << m->get_target_osd()
+	    << " " << m->get_target_addrs()
+	    << ", from " << m->get_orig_source() << dendl;
+    if (m->get_epoch() < osdmap.get_epoch())
+      send_incremental(op, m->get_epoch()+1);
+    goto didit;
+  }
+
+  if (!can_mark_down(badboy)) {
+    dout(5) << "preprocess_failure ignoring report of osd."
+	    << m->get_target_osd() << " " << m->get_target_addrs()
+	    << " from " << m->get_orig_source() << dendl;
+    goto didit;
+  }
+
+  dout(10) << "preprocess_failure new: osd." << m->get_target_osd()
+	   << " " << m->get_target_addrs()
+	   << ", from " << m->get_orig_source() << dendl;
+  return false;
+
+ didit:
+  mon.no_reply(op);
+  return true;
+}
+
+class C_AckMarkedDown : public C_MonOp {
+  OSDMonitor *osdmon;
+public:
+  C_AckMarkedDown(
+    OSDMonitor *osdmon,
+    MonOpRequestRef op)
+    : C_MonOp(op), osdmon(osdmon) {}
+
+  void _finish(int r) override {
+    if (r == 0) {
+      auto m = op->get_req<MOSDMarkMeDown>();
+      osdmon->mon.send_reply(
+        op,
+        new MOSDMarkMeDown(
+          m->fsid,
+          m->target_osd,
+          m->target_addrs,
+          m->get_epoch(),
+          false));   // ACK itself does not request an ack
+    } else if (r == -EAGAIN) {
+        osdmon->dispatch(op);
+    } else {
+        ceph_abort_msgf("C_AckMarkedDown: unknown result %d", r);
+    }
+  }
+  ~C_AckMarkedDown() override {
+  }
+};
+
+bool OSDMonitor::preprocess_mark_me_down(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDMarkMeDown>();
+  int from = m->target_osd;
+
+  // check permissions
+  if (check_source(op, m->fsid))
+    goto reply;
+
+  // first, verify the reporting host is valid
+  if (!m->get_orig_source().is_osd())
+    goto reply;
+
+  if (!osdmap.exists(from) ||
+      osdmap.is_down(from) ||
+      osdmap.get_addrs(from) != m->target_addrs) {
+    dout(5) << "preprocess_mark_me_down from dead osd."
+	    << from << ", ignoring" << dendl;
+    send_incremental(op, m->get_epoch()+1);
+    goto reply;
+  }
+
+  // no down might be set
+  if (!can_mark_down(from))
+    goto reply;
+
+  dout(10) << "MOSDMarkMeDown for: " << m->get_orig_source()
+	   << " " << m->target_addrs << dendl;
+  return false;
+
+ reply:
+  if (m->request_ack) {
+    Context *c(new C_AckMarkedDown(this, op));
+    c->complete(0);
+  }
+  return true;
+}
+
+bool OSDMonitor::prepare_mark_me_down(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDMarkMeDown>();
+  int target_osd = m->target_osd;
+
+  ceph_assert(osdmap.is_up(target_osd));
+  ceph_assert(osdmap.get_addrs(target_osd) == m->target_addrs);
+
+  mon.clog->info() << "osd." << target_osd << " marked itself " << ((m->down_and_dead) ? "down and dead" : "down");
+  pending_inc.new_state[target_osd] = CEPH_OSD_UP;
+  if (m->down_and_dead) {
+    if (!pending_inc.new_xinfo.count(target_osd)) {
+      pending_inc.new_xinfo[target_osd] = osdmap.osd_xinfo[target_osd];
+    }
+    pending_inc.new_xinfo[target_osd].dead_epoch = m->get_epoch();
+  }
+  if (m->request_ack)
+    wait_for_finished_proposal(op, new C_AckMarkedDown(this, op));
+  return true;
+}
+
+bool OSDMonitor::preprocess_mark_me_dead(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDMarkMeDead>();
+  int from = m->target_osd;
+
+  // check permissions
+  if (check_source(op, m->fsid)) {
+    mon.no_reply(op);
+    return true;
+  }
+
+  // first, verify the reporting host is valid
+  if (!m->get_orig_source().is_osd()) {
+    mon.no_reply(op);
+    return true;
+  }
+
+  if (!osdmap.exists(from) ||
+      !osdmap.is_down(from)) {
+    dout(5) << __func__ << " from nonexistent or up osd." << from
+	    << ", ignoring" << dendl;
+    send_incremental(op, m->get_epoch()+1);
+    mon.no_reply(op);
+    return true;
+  }
+
+  return false;
+}
+
+bool OSDMonitor::prepare_mark_me_dead(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDMarkMeDead>();
+  int target_osd = m->target_osd;
+
+  ceph_assert(osdmap.is_down(target_osd));
+
+  mon.clog->info() << "osd." << target_osd << " marked itself dead as of e"
+		    << m->get_epoch();
+  if (!pending_inc.new_xinfo.count(target_osd)) {
+    pending_inc.new_xinfo[target_osd] = osdmap.osd_xinfo[target_osd];
+  }
+  pending_inc.new_xinfo[target_osd].dead_epoch = m->get_epoch();
+  wait_for_finished_proposal(
+    op,
+    new LambdaContext(
+      [op, this] (int r) {
+	if (r >= 0) {
+	  mon.no_reply(op);	  // ignore on success
+	}
+      }
+      ));
+  return true;
+}
+
+bool OSDMonitor::can_mark_down(int i)
+{
+  if (osdmap.is_nodown(i)) {
+    dout(5) << __func__ << " osd." << i << " is marked as nodown, "
+            << "will not mark it down" << dendl;
+    return false;
+  }
+
+  int num_osds = osdmap.get_num_osds();
+  if (num_osds == 0) {
+    dout(5) << __func__ << " no osds" << dendl;
+    return false;
+  }
+  int up = osdmap.get_num_up_osds() - pending_inc.get_net_marked_down(&osdmap);
+  float up_ratio = (float)up / (float)num_osds;
+  if (up_ratio < g_conf()->mon_osd_min_up_ratio) {
+    dout(2) << __func__ << " current up_ratio " << up_ratio << " < min "
+	    << g_conf()->mon_osd_min_up_ratio
+	    << ", will not mark osd." << i << " down" << dendl;
+    return false;
+  }
+  return true;
+}
+
+bool OSDMonitor::can_mark_up(int i)
+{
+  if (osdmap.is_noup(i)) {
+    dout(5) << __func__ << " osd." << i << " is marked as noup, "
+            << "will not mark it up" << dendl;
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * @note the parameter @p i apparently only exists here so we can output the
+ *	 osd's id on messages.
+ */
+bool OSDMonitor::can_mark_out(int i)
+{
+  if (osdmap.is_noout(i)) {
+    dout(5) << __func__ << " osd." << i << " is marked as noout, "
+            << "will not mark it out" << dendl;
+    return false;
+  }
+
+  int num_osds = osdmap.get_num_osds();
+  if (num_osds == 0) {
+    dout(5) << __func__ << " no osds" << dendl;
+    return false;
+  }
+  int in = osdmap.get_num_in_osds() - pending_inc.get_net_marked_out(&osdmap);
+  float in_ratio = (float)in / (float)num_osds;
+  if (in_ratio < g_conf()->mon_osd_min_in_ratio) {
+    if (i >= 0)
+      dout(5) << __func__ << " current in_ratio " << in_ratio << " < min "
+	      << g_conf()->mon_osd_min_in_ratio
+	      << ", will not mark osd." << i << " out" << dendl;
+    else
+      dout(5) << __func__ << " current in_ratio " << in_ratio << " < min "
+	      << g_conf()->mon_osd_min_in_ratio
+	      << ", will not mark osds out" << dendl;
+    return false;
+  }
+
+  return true;
+}
+
+bool OSDMonitor::can_mark_in(int i)
+{
+  if (osdmap.is_noin(i)) {
+    dout(5) << __func__ << " osd." << i << " is marked as noin, "
+            << "will not mark it in" << dendl;
+    return false;
+  }
+
+  return true;
+}
+
+bool OSDMonitor::check_failures(utime_t now)
+{
+  bool found_failure = false;
+  auto p = failure_info.begin();
+  while (p != failure_info.end()) {
+    auto& [target_osd, fi] = *p;
+    if (can_mark_down(target_osd) &&
+	check_failure(now, target_osd, fi)) {
+      found_failure = true;
+      ++p;
+    } else if (is_failure_stale(now, fi)) {
+      dout(10) << " dropping stale failure_info for osd." << target_osd
+	       << " from " << fi.reporters.size() << " reporters"
+	       << dendl;
+      p = failure_info.erase(p);
+    } else {
+      ++p;
+    }
+  }
+  return found_failure;
+}
+
+utime_t OSDMonitor::get_grace_time(utime_t now,
+				   int target_osd,
+				   failure_info_t& fi) const
+{
+  utime_t orig_grace(g_conf()->osd_heartbeat_grace, 0);
+  if (!g_conf()->mon_osd_adjust_heartbeat_grace) {
+    return orig_grace;
+  }
+  utime_t grace = orig_grace;
+  double halflife = (double)g_conf()->mon_osd_laggy_halflife;
+  double decay_k = ::log(.5) / halflife;
+
+  // scale grace period based on historical probability of 'lagginess'
+  // (false positive failures due to slowness).
+  const osd_xinfo_t& xi = osdmap.get_xinfo(target_osd);
+  const utime_t failed_for = now - fi.get_failed_since();
+  double decay = exp((double)failed_for * decay_k);
+  dout(20) << " halflife " << halflife << " decay_k " << decay_k
+	   << " failed_for " << failed_for << " decay " << decay << dendl;
+  double my_grace = decay * (double)xi.laggy_interval * xi.laggy_probability;
+  grace += my_grace;
+
+  // consider the peers reporting a failure a proxy for a potential
+  // 'subcluster' over the overall cluster that is similarly
+  // laggy.  this is clearly not true in all cases, but will sometimes
+  // help us localize the grace correction to a subset of the system
+  // (say, a rack with a bad switch) that is unhappy.
+  double peer_grace = 0;
+  for (auto& [reporter, report] : fi.reporters) {
+    if (osdmap.exists(reporter)) {
+      const osd_xinfo_t& xi = osdmap.get_xinfo(reporter);
+      utime_t elapsed = now - xi.down_stamp;
+      double decay = exp((double)elapsed * decay_k);
+      peer_grace += decay * (double)xi.laggy_interval * xi.laggy_probability;
+    }
+  }
+  peer_grace /= (double)fi.reporters.size();
+  grace += peer_grace;
+  dout(10) << " osd." << target_osd << " has "
+	   << fi.reporters.size() << " reporters, "
+	   << grace << " grace (" << orig_grace << " + " << my_grace
+	   << " + " << peer_grace << "), max_failed_since " << fi.get_failed_since()
+	   << dendl;
+
+  return grace;
+}
+
+bool OSDMonitor::check_failure(utime_t now, int target_osd, failure_info_t& fi)
+{
+  // already pending failure?
+  if (pending_inc.new_state.count(target_osd) &&
+      pending_inc.new_state[target_osd] & CEPH_OSD_UP) {
+    dout(10) << " already pending failure" << dendl;
+    return true;
+  }
+
+  set<string> reporters_by_subtree;
+  auto reporter_subtree_level = g_conf().get_val<string>("mon_osd_reporter_subtree_level");
+  ceph_assert(fi.reporters.size());
+  for (auto p = fi.reporters.begin(); p != fi.reporters.end();) {
+    // get the parent bucket whose type matches with "reporter_subtree_level".
+    // fall back to OSD if the level doesn't exist.
+    if (osdmap.exists(p->first)) {
+      auto reporter_loc = osdmap.crush->get_full_location(p->first);
+      if (auto iter = reporter_loc.find(reporter_subtree_level);
+          iter == reporter_loc.end()) {
+        reporters_by_subtree.insert("osd." + to_string(p->first));
+      } else {
+        reporters_by_subtree.insert(iter->second);
+      }
+      ++p;
+    } else {
+      fi.cancel_report(p->first);;
+      p = fi.reporters.erase(p);
+    }
+  }
+  if (reporters_by_subtree.size() < g_conf().get_val<uint64_t>("mon_osd_min_down_reporters")) {
+    return false;
+  }
+  const utime_t failed_for = now - fi.get_failed_since();
+  const utime_t grace = get_grace_time(now, target_osd, fi);
+  if (failed_for >= grace) {
+    dout(1) << " we have enough reporters to mark osd." << target_osd
+	    << " down" << dendl;
+    pending_inc.new_state[target_osd] = CEPH_OSD_UP;
+
+    mon.clog->info() << "osd." << target_osd << " failed ("
+		      << osdmap.crush->get_full_location_ordered_string(
+			target_osd)
+		      << ") ("
+		      << (int)reporters_by_subtree.size()
+		      << " reporters from different "
+		      << reporter_subtree_level << " after "
+		      << failed_for << " >= grace " << grace << ")";
+    return true;
+  }
+  return false;
+}
+
+bool OSDMonitor::is_failure_stale(utime_t now, failure_info_t& fi) const
+{
+  // if it takes too long to either cancel the report to mark the osd down,
+  // some reporters must have failed to cancel their reports. let's just
+  // forget these reports.
+  const utime_t failed_for = now - fi.get_failed_since();
+  auto heartbeat_grace = cct->_conf.get_val<int64_t>("osd_heartbeat_grace");
+  auto heartbeat_stale = cct->_conf.get_val<int64_t>("osd_heartbeat_stale");
+  return failed_for >= (heartbeat_grace + heartbeat_stale);
+}
+
+void OSDMonitor::force_failure(int target_osd, int by)
+{
+  // already pending failure?
+  if (pending_inc.new_state.count(target_osd) &&
+      pending_inc.new_state[target_osd] & CEPH_OSD_UP) {
+    dout(10) << " already pending failure" << dendl;
+    return;
+  }
+
+  dout(1) << " we're forcing failure of osd." << target_osd << dendl;
+  pending_inc.new_state[target_osd] = CEPH_OSD_UP;
+  if (!pending_inc.new_xinfo.count(target_osd)) {
+    pending_inc.new_xinfo[target_osd] = osdmap.osd_xinfo[target_osd];
+  }
+  pending_inc.new_xinfo[target_osd].dead_epoch = pending_inc.epoch;
+
+  mon.clog->info() << "osd." << target_osd << " failed ("
+		    << osdmap.crush->get_full_location_ordered_string(target_osd)
+		    << ") (connection refused reported by osd." << by << ")";
+  return;
+}
+
+bool OSDMonitor::prepare_failure(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDFailure>();
+  dout(1) << "prepare_failure osd." << m->get_target_osd()
+	  << " " << m->get_target_addrs()
+	  << " from " << m->get_orig_source()
+          << " is reporting failure:" << m->if_osd_failed() << dendl;
+
+  int target_osd = m->get_target_osd();
+  int reporter = m->get_orig_source().num();
+  ceph_assert(osdmap.is_up(target_osd));
+  ceph_assert(osdmap.get_addrs(target_osd) == m->get_target_addrs());
+
+  mon.no_reply(op);
+
+  if (m->if_osd_failed()) {
+    // calculate failure time
+    utime_t now = ceph_clock_now();
+    utime_t failed_since =
+      m->get_recv_stamp() - utime_t(m->failed_for, 0);
+
+    // add a report
+    if (m->is_immediate()) {
+      mon.clog->debug() << "osd." << m->get_target_osd()
+			 << " reported immediately failed by "
+			 << m->get_orig_source();
+      force_failure(target_osd, reporter);
+      return true;
+    }
+    mon.clog->debug() << "osd." << m->get_target_osd() << " reported failed by "
+		      << m->get_orig_source();
+
+    failure_info_t& fi = failure_info[target_osd];
+    fi.add_report(reporter, failed_since, op);
+    return check_failure(now, target_osd, fi);
+  } else {
+    // remove the report
+    mon.clog->debug() << "osd." << m->get_target_osd()
+		       << " failure report canceled by "
+		       << m->get_orig_source();
+    if (failure_info.count(target_osd)) {
+      failure_info_t& fi = failure_info[target_osd];
+      fi.cancel_report(reporter);
+      if (fi.reporters.empty()) {
+	dout(10) << " removing last failure_info for osd." << target_osd
+		 << dendl;
+	failure_info.erase(target_osd);
+      } else {
+	dout(10) << " failure_info for osd." << target_osd << " now "
+		 << fi.reporters.size() << " reporters" << dendl;
+      }
+    } else {
+      dout(10) << " no failure_info for osd." << target_osd << dendl;
+    }
+  }
+
+  return false;
+}
+
+void OSDMonitor::process_failures()
+{
+  map<int,failure_info_t>::iterator p = failure_info.begin();
+  while (p != failure_info.end()) {
+    if (osdmap.is_up(p->first)) {
+      ++p;
+    } else {
+      dout(10) << "process_failures osd." << p->first << dendl;
+      list<MonOpRequestRef> ls;
+      p->second.take_report_messages(ls);
+      failure_info.erase(p++);
+
+      while (!ls.empty()) {
+        MonOpRequestRef o = ls.front();
+        if (o) {
+          o->mark_event(__func__);
+          MOSDFailure *m = o->get_req<MOSDFailure>();
+          send_latest(o, m->get_epoch());
+	  mon.no_reply(o);
+        }
+	ls.pop_front();
+      }
+    }
+  }
+}
+
+void OSDMonitor::take_all_failures(list<MonOpRequestRef>& ls)
+{
+  dout(10) << __func__ << " on " << failure_info.size() << " osds" << dendl;
+
+  for (map<int,failure_info_t>::iterator p = failure_info.begin();
+       p != failure_info.end();
+       ++p) {
+    p->second.take_report_messages(ls);
+  }
+  failure_info.clear();
+}
+
+int OSDMonitor::get_grace_interval_threshold()
+{
+  int halflife = g_conf()->mon_osd_laggy_halflife;
+  // Scale the halflife period (default: 1_hr) by
+  // a factor (48) to calculate the threshold.
+  int grace_threshold_factor = 48;
+  return halflife * grace_threshold_factor;
+}
+
+bool OSDMonitor::grace_interval_threshold_exceeded(int last_failed_interval)
+{
+  int grace_interval_threshold_secs = get_grace_interval_threshold();
+  if (last_failed_interval > grace_interval_threshold_secs) {
+    dout(1) << " last_failed_interval " << last_failed_interval
+            << " > grace_interval_threshold_secs " << grace_interval_threshold_secs
+            << dendl;
+    return true;
+  }
+  return false;
+}
+
+void OSDMonitor::set_default_laggy_params(int target_osd)
+{
+  if (pending_inc.new_xinfo.count(target_osd) == 0) {
+    pending_inc.new_xinfo[target_osd] = osdmap.osd_xinfo[target_osd];
+  }
+  osd_xinfo_t& xi = pending_inc.new_xinfo[target_osd];
+  xi.down_stamp = pending_inc.modified;
+  xi.laggy_probability = 0.0;
+  xi.laggy_interval = 0;
+  dout(20) << __func__ << " reset laggy, now xi " << xi << dendl;
+}
+
+
+// boot --
+
+bool OSDMonitor::preprocess_boot(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDBoot>();
+  int from = m->get_orig_source_inst().name.num();
+
+  // check permissions, ignore if failed (no response expected)
+  MonSession *session = op->get_session();
+  if (!session)
+    goto ignore;
+  if (!session->is_capable("osd", MON_CAP_X)) {
+    dout(0) << "got preprocess_boot message from entity with insufficient caps"
+	    << session->caps << dendl;
+    goto ignore;
+  }
+
+  if (m->sb.cluster_fsid != mon.monmap->fsid) {
+    dout(0) << "preprocess_boot on fsid " << m->sb.cluster_fsid
+	    << " != " << mon.monmap->fsid << dendl;
+    goto ignore;
+  }
+
+  if (m->get_orig_source_inst().addr.is_blank_ip()) {
+    dout(0) << "preprocess_boot got blank addr for " << m->get_orig_source_inst() << dendl;
+    goto ignore;
+  }
+
+  ceph_assert(m->get_orig_source_inst().name.is_osd());
+
+  // force all osds to have gone through luminous prior to upgrade to nautilus
+  {
+    vector<string> missing;
+    if (!HAVE_FEATURE(m->osd_features, SERVER_LUMINOUS)) {
+      missing.push_back("CEPH_FEATURE_SERVER_LUMINOUS");
+    }
+    if (!HAVE_FEATURE(m->osd_features, SERVER_JEWEL)) {
+      missing.push_back("CEPH_FEATURE_SERVER_JEWEL");
+    }
+    if (!HAVE_FEATURE(m->osd_features, SERVER_KRAKEN)) {
+      missing.push_back("CEPH_FEATURE_SERVER_KRAKEN");
+    }
+    if (!HAVE_FEATURE(m->osd_features, OSD_RECOVERY_DELETES)) {
+      missing.push_back("CEPH_FEATURE_OSD_RECOVERY_DELETES");
+    }
+
+    if (!missing.empty()) {
+      using std::experimental::make_ostream_joiner;
+
+      stringstream ss;
+      copy(begin(missing), end(missing), make_ostream_joiner(ss, ";"));
+
+      mon.clog->info() << "disallowing boot of OSD "
+			<< m->get_orig_source_inst()
+			<< " because the osd lacks " << ss.str();
+      goto ignore;
+    }
+  }
+
+  // make sure osd versions do not span more than 3 releases
+  if (HAVE_FEATURE(m->osd_features, SERVER_OCTOPUS) &&
+      osdmap.require_osd_release < ceph_release_t::mimic) {
+    mon.clog->info() << "disallowing boot of octopus+ OSD "
+		      << m->get_orig_source_inst()
+		      << " because require_osd_release < mimic";
+    goto ignore;
+  }
+  if (HAVE_FEATURE(m->osd_features, SERVER_PACIFIC) &&
+      osdmap.require_osd_release < ceph_release_t::nautilus) {
+    mon.clog->info() << "disallowing boot of pacific+ OSD "
+		      << m->get_orig_source_inst()
+		      << " because require_osd_release < nautilus";
+    goto ignore;
+  }
+
+  // The release check here is required because for OSD_PGLOG_HARDLIMIT,
+  // we are reusing a jewel feature bit that was retired in luminous.
+  if (osdmap.require_osd_release >= ceph_release_t::luminous &&
+      osdmap.test_flag(CEPH_OSDMAP_PGLOG_HARDLIMIT) &&
+      !(m->osd_features & CEPH_FEATURE_OSD_PGLOG_HARDLIMIT)) {
+    mon.clog->info() << "disallowing boot of OSD "
+		      << m->get_orig_source_inst()
+		      << " because 'pglog_hardlimit' osdmap flag is set and OSD lacks the OSD_PGLOG_HARDLIMIT feature";
+    goto ignore;
+  }
+
+  if (osdmap.stretch_mode_enabled &&
+      !(m->osd_features & CEPH_FEATUREMASK_STRETCH_MODE)) {
+    mon.clog->info() << "disallowing boot of OSD "
+		      << m->get_orig_source_inst()
+		      << " because stretch mode is on and OSD lacks support";
+    goto ignore;
+  }
+
+  // already booted?
+  if (osdmap.is_up(from) &&
+      osdmap.get_addrs(from).legacy_equals(m->get_orig_source_addrs()) &&
+      osdmap.get_cluster_addrs(from).legacy_equals(m->cluster_addrs)) {
+    // yup.
+    dout(7) << "preprocess_boot dup from " << m->get_orig_source()
+	    << " " << m->get_orig_source_addrs()
+	    << " =~ " << osdmap.get_addrs(from) << dendl;
+    _booted(op, false);
+    return true;
+  }
+
+  if (osdmap.exists(from) &&
+      !osdmap.get_uuid(from).is_zero() &&
+      osdmap.get_uuid(from) != m->sb.osd_fsid) {
+    dout(7) << __func__ << " from " << m->get_orig_source_inst()
+            << " clashes with existing osd: different fsid"
+            << " (ours: " << osdmap.get_uuid(from)
+            << " ; theirs: " << m->sb.osd_fsid << ")" << dendl;
+    goto ignore;
+  }
+
+  if (osdmap.exists(from) &&
+      osdmap.get_info(from).up_from > m->version &&
+      osdmap.get_most_recent_addrs(from).legacy_equals(
+	m->get_orig_source_addrs())) {
+    dout(7) << "prepare_boot msg from before last up_from, ignoring" << dendl;
+    send_latest(op, m->sb.current_epoch+1);
+    return true;
+  }
+
+  // noup?
+  if (!can_mark_up(from)) {
+    dout(7) << "preprocess_boot ignoring boot from " << m->get_orig_source_inst() << dendl;
+    send_latest(op, m->sb.current_epoch+1);
+    return true;
+  }
+
+  dout(10) << "preprocess_boot from " << m->get_orig_source_inst() << dendl;
+  return false;
+
+ ignore:
+  return true;
+}
+
+bool OSDMonitor::prepare_boot(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDBoot>();
+  dout(7) << __func__ << " from " << m->get_source()
+	  << " sb " << m->sb
+	  << " client_addrs" << m->get_connection()->get_peer_addrs()
+	  << " cluster_addrs " << m->cluster_addrs
+	  << " hb_back_addrs " << m->hb_back_addrs
+	  << " hb_front_addrs " << m->hb_front_addrs
+	  << dendl;
+
+  ceph_assert(m->get_orig_source().is_osd());
+  int from = m->get_orig_source().num();
+
+  // does this osd exist?
+  if (from >= osdmap.get_max_osd()) {
+    dout(1) << "boot from osd." << from << " >= max_osd "
+	    << osdmap.get_max_osd() << dendl;
+    return false;
+  }
+
+  int oldstate = osdmap.exists(from) ? osdmap.get_state(from) : CEPH_OSD_NEW;
+  if (pending_inc.new_state.count(from))
+    oldstate ^= pending_inc.new_state[from];
+
+  // already up?  mark down first?
+  if (osdmap.is_up(from)) {
+    dout(7) << __func__ << " was up, first marking down osd." << from << " "
+	    << osdmap.get_addrs(from) << dendl;
+    // preprocess should have caught these;  if not, assert.
+    ceph_assert(!osdmap.get_addrs(from).legacy_equals(
+		  m->get_orig_source_addrs()) ||
+		!osdmap.get_cluster_addrs(from).legacy_equals(m->cluster_addrs));
+    ceph_assert(osdmap.get_uuid(from) == m->sb.osd_fsid);
+
+    if (pending_inc.new_state.count(from) == 0 ||
+	(pending_inc.new_state[from] & CEPH_OSD_UP) == 0) {
+      // mark previous guy down
+      pending_inc.new_state[from] = CEPH_OSD_UP;
+    }
+    wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+  } else if (pending_inc.new_up_client.count(from)) {
+    // already prepared, just wait
+    dout(7) << __func__ << " already prepared, waiting on "
+	    << m->get_orig_source_addr() << dendl;
+    wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+  } else {
+    // mark new guy up.
+    pending_inc.new_up_client[from] = m->get_orig_source_addrs();
+    pending_inc.new_up_cluster[from] = m->cluster_addrs;
+    pending_inc.new_hb_back_up[from] = m->hb_back_addrs;
+    pending_inc.new_hb_front_up[from] = m->hb_front_addrs;
+
+    down_pending_out.erase(from);  // if any
+
+    if (m->sb.weight)
+      osd_weight[from] = m->sb.weight;
+
+    // set uuid?
+    dout(10) << " setting osd." << from << " uuid to " << m->sb.osd_fsid
+	     << dendl;
+    if (!osdmap.exists(from) || osdmap.get_uuid(from) != m->sb.osd_fsid) {
+      // preprocess should have caught this;  if not, assert.
+      ceph_assert(!osdmap.exists(from) || osdmap.get_uuid(from).is_zero());
+      pending_inc.new_uuid[from] = m->sb.osd_fsid;
+    }
+
+    // fresh osd?
+    if (m->sb.newest_map == 0 && osdmap.exists(from)) {
+      const osd_info_t& i = osdmap.get_info(from);
+      if (i.up_from > i.lost_at) {
+	dout(10) << " fresh osd; marking lost_at too" << dendl;
+	pending_inc.new_lost[from] = osdmap.get_epoch();
+      }
+    }
+
+    // metadata
+    bufferlist osd_metadata;
+    encode(m->metadata, osd_metadata);
+    pending_metadata[from] = osd_metadata;
+    pending_metadata_rm.erase(from);
+
+    // adjust last clean unmount epoch?
+    const osd_info_t& info = osdmap.get_info(from);
+    dout(10) << " old osd_info: " << info << dendl;
+    if (m->sb.mounted > info.last_clean_begin ||
+	(m->sb.mounted == info.last_clean_begin &&
+	 m->sb.clean_thru > info.last_clean_end)) {
+      epoch_t begin = m->sb.mounted;
+      epoch_t end = m->sb.clean_thru;
+
+      dout(10) << __func__ << " osd." << from << " last_clean_interval "
+	       << "[" << info.last_clean_begin << "," << info.last_clean_end
+	       << ") -> [" << begin << "-" << end << ")"
+	       << dendl;
+      pending_inc.new_last_clean_interval[from] =
+	pair<epoch_t,epoch_t>(begin, end);
+    }
+
+    if (pending_inc.new_xinfo.count(from) == 0)
+      pending_inc.new_xinfo[from] = osdmap.osd_xinfo[from];
+    osd_xinfo_t& xi = pending_inc.new_xinfo[from];
+    if (m->boot_epoch == 0) {
+      xi.laggy_probability *= (1.0 - g_conf()->mon_osd_laggy_weight);
+      xi.laggy_interval *= (1.0 - g_conf()->mon_osd_laggy_weight);
+      dout(10) << " not laggy, new xi " << xi << dendl;
+    } else {
+      if (xi.down_stamp.sec()) {
+        int interval = ceph_clock_now().sec() -
+	  xi.down_stamp.sec();
+        if (g_conf()->mon_osd_laggy_max_interval &&
+	    (interval > g_conf()->mon_osd_laggy_max_interval)) {
+          interval =  g_conf()->mon_osd_laggy_max_interval;
+        }
+        xi.laggy_interval =
+	  interval * g_conf()->mon_osd_laggy_weight +
+	  xi.laggy_interval * (1.0 - g_conf()->mon_osd_laggy_weight);
+      }
+      xi.laggy_probability =
+	g_conf()->mon_osd_laggy_weight +
+	xi.laggy_probability * (1.0 - g_conf()->mon_osd_laggy_weight);
+      dout(10) << " laggy, now xi " << xi << dendl;
+    }
+
+    // set features shared by the osd
+    if (m->osd_features)
+      xi.features = m->osd_features;
+    else
+      xi.features = m->get_connection()->get_features();
+
+    // mark in?
+    if ((g_conf()->mon_osd_auto_mark_auto_out_in &&
+	 (oldstate & CEPH_OSD_AUTOOUT)) ||
+	(g_conf()->mon_osd_auto_mark_new_in && (oldstate & CEPH_OSD_NEW)) ||
+	(g_conf()->mon_osd_auto_mark_in)) {
+      if (can_mark_in(from)) {
+	if (xi.old_weight > 0) {
+	  pending_inc.new_weight[from] = xi.old_weight;
+	  xi.old_weight = 0;
+	} else {
+	  pending_inc.new_weight[from] = CEPH_OSD_IN;
+	}
+      } else {
+	dout(7) << __func__ << " NOIN set, will not mark in "
+		<< m->get_orig_source_addr() << dendl;
+      }
+    }
+
+    // wait
+    wait_for_finished_proposal(op, new C_Booted(this, op));
+  }
+  return true;
+}
+
+void OSDMonitor::_booted(MonOpRequestRef op, bool logit)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDBoot>();
+  dout(7) << "_booted " << m->get_orig_source_inst() 
+	  << " w " << m->sb.weight << " from " << m->sb.current_epoch << dendl;
+
+  if (logit) {
+    mon.clog->info() << m->get_source() << " " << m->get_orig_source_addrs()
+		      << " boot";
+  }
+
+  send_latest(op, m->sb.current_epoch+1);
+}
+
+
+// -------------
+// full
+
+bool OSDMonitor::preprocess_full(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDFull>();
+  int from = m->get_orig_source().num();
+  set<string> state;
+  unsigned mask = CEPH_OSD_NEARFULL | CEPH_OSD_BACKFILLFULL | CEPH_OSD_FULL;
+
+  // check permissions, ignore if failed
+  MonSession *session = op->get_session();
+  if (!session)
+    goto ignore;
+  if (!session->is_capable("osd", MON_CAP_X)) {
+    dout(0) << "MOSDFull from entity with insufficient privileges:"
+	    << session->caps << dendl;
+    goto ignore;
+  }
+
+  // ignore a full message from the osd instance that already went down
+  if (!osdmap.exists(from)) {
+    dout(7) << __func__ << " ignoring full message from nonexistent "
+	    << m->get_orig_source_inst() << dendl;
+    goto ignore;
+  }
+  if ((!osdmap.is_up(from) &&
+       osdmap.get_most_recent_addrs(from).legacy_equals(
+	 m->get_orig_source_addrs())) ||
+      (osdmap.is_up(from) &&
+       !osdmap.get_addrs(from).legacy_equals(m->get_orig_source_addrs()))) {
+    dout(7) << __func__ << " ignoring full message from down "
+	    << m->get_orig_source_inst() << dendl;
+    goto ignore;
+  }
+
+  OSDMap::calc_state_set(osdmap.get_state(from), state);
+
+  if ((osdmap.get_state(from) & mask) == m->state) {
+    dout(7) << __func__ << " state already " << state << " for osd." << from
+	    << " " << m->get_orig_source_inst() << dendl;
+    _reply_map(op, m->version);
+    goto ignore;
+  }
+
+  dout(10) << __func__ << " want state " << state << " for osd." << from
+	   << " " << m->get_orig_source_inst() << dendl;
+  return false;
+
+ ignore:
+  return true;
+}
+
+bool OSDMonitor::prepare_full(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDFull>();
+  const int from = m->get_orig_source().num();
+
+  const unsigned mask = CEPH_OSD_NEARFULL | CEPH_OSD_BACKFILLFULL | CEPH_OSD_FULL;
+  const unsigned want_state = m->state & mask;  // safety first
+
+  unsigned cur_state = osdmap.get_state(from);
+  auto p = pending_inc.new_state.find(from);
+  if (p != pending_inc.new_state.end()) {
+    cur_state ^= p->second;
+  }
+  cur_state &= mask;
+
+  set<string> want_state_set, cur_state_set;
+  OSDMap::calc_state_set(want_state, want_state_set);
+  OSDMap::calc_state_set(cur_state, cur_state_set);
+
+  if (cur_state != want_state) {
+    if (p != pending_inc.new_state.end()) {
+      p->second &= ~mask;
+    } else {
+      pending_inc.new_state[from] = 0;
+    }
+    pending_inc.new_state[from] |= (osdmap.get_state(from) & mask) ^ want_state;
+    dout(7) << __func__ << " osd." << from << " " << cur_state_set
+	    << " -> " << want_state_set << dendl;
+  } else {
+    dout(7) << __func__ << " osd." << from << " " << cur_state_set
+	    << " = wanted " << want_state_set << ", just waiting" << dendl;
+  }
+
+  wait_for_finished_proposal(op, new C_ReplyMap(this, op, m->version));
+  return true;
+}
+
+// -------------
+// alive
+
+bool OSDMonitor::preprocess_alive(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDAlive>();
+  int from = m->get_orig_source().num();
+
+  // check permissions, ignore if failed
+  MonSession *session = op->get_session();
+  if (!session)
+    goto ignore;
+  if (!session->is_capable("osd", MON_CAP_X)) {
+    dout(0) << "attempt to send MOSDAlive from entity with insufficient privileges:"
+	    << session->caps << dendl;
+    goto ignore;
+  }
+
+  if (!osdmap.is_up(from) ||
+      !osdmap.get_addrs(from).legacy_equals(m->get_orig_source_addrs())) {
+    dout(7) << "preprocess_alive ignoring alive message from down "
+	    << m->get_orig_source() << " " << m->get_orig_source_addrs()
+	    << dendl;
+    goto ignore;
+  }
+
+  if (osdmap.get_up_thru(from) >= m->want) {
+    // yup.
+    dout(7) << "preprocess_alive want up_thru " << m->want << " dup from " << m->get_orig_source_inst() << dendl;
+    _reply_map(op, m->version);
+    return true;
+  }
+
+  dout(10) << "preprocess_alive want up_thru " << m->want
+	   << " from " << m->get_orig_source_inst() << dendl;
+  return false;
+
+ ignore:
+  return true;
+}
+
+bool OSDMonitor::prepare_alive(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDAlive>();
+  int from = m->get_orig_source().num();
+
+  if (0) {  // we probably don't care much about these
+    mon.clog->debug() << m->get_orig_source_inst() << " alive";
+  }
+
+  dout(7) << "prepare_alive want up_thru " << m->want << " have " << m->version
+	  << " from " << m->get_orig_source_inst() << dendl;
+
+  update_up_thru(from, m->version); // set to the latest map the OSD has
+  wait_for_finished_proposal(op, new C_ReplyMap(this, op, m->version));
+  return true;
+}
+
+void OSDMonitor::_reply_map(MonOpRequestRef op, epoch_t e)
+{
+  op->mark_osdmon_event(__func__);
+  dout(7) << "_reply_map " << e
+	  << " from " << op->get_req()->get_orig_source_inst()
+	  << dendl;
+  send_latest(op, e);
+}
+
+// pg_created
+bool OSDMonitor::preprocess_pg_created(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m  = op->get_req<MOSDPGCreated>();
+  dout(10) << __func__ << " " << *m << dendl;
+  auto session = op->get_session();
+  mon.no_reply(op);
+  if (!session) {
+    dout(10) << __func__ << ": no monitor session!" << dendl;
+    return true;
+  }
+  if (!session->is_capable("osd", MON_CAP_X)) {
+    derr << __func__ << " received from entity "
+         << "with insufficient privileges " << session->caps << dendl;
+    return true;
+  }
+  // always forward the "created!" to the leader
+  return false;
+}
+
+bool OSDMonitor::prepare_pg_created(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDPGCreated>();
+  dout(10) << __func__ << " " << *m << dendl;
+  auto src = m->get_orig_source();
+  auto from = src.num();
+  if (!src.is_osd() ||
+      !mon.osdmon()->osdmap.is_up(from) ||
+      !mon.osdmon()->osdmap.get_addrs(from).legacy_equals(
+	m->get_orig_source_addrs())) {
+    dout(1) << __func__ << " ignoring stats from non-active osd." << dendl;
+    return false;
+  }
+  pending_created_pgs.push_back(m->pgid);
+  return true;
+}
+
+bool OSDMonitor::preprocess_pg_ready_to_merge(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDPGReadyToMerge>();
+  dout(10) << __func__ << " " << *m << dendl;
+  const pg_pool_t *pi;
+  auto session = op->get_session();
+  if (!session) {
+    dout(10) << __func__ << ": no monitor session!" << dendl;
+    goto ignore;
+  }
+  if (!session->is_capable("osd", MON_CAP_X)) {
+    derr << __func__ << " received from entity "
+         << "with insufficient privileges " << session->caps << dendl;
+    goto ignore;
+  }
+  pi = osdmap.get_pg_pool(m->pgid.pool());
+  if (!pi) {
+    derr << __func__ << " pool for " << m->pgid << " dne" << dendl;
+    goto ignore;
+  }
+  if (pi->get_pg_num() <= m->pgid.ps()) {
+    dout(20) << " pg_num " << pi->get_pg_num() << " already < " << m->pgid << dendl;
+    goto ignore;
+  }
+  if (pi->get_pg_num() != m->pgid.ps() + 1) {
+    derr << " OSD trying to merge wrong pgid " << m->pgid << dendl;
+    goto ignore;
+  }
+  if (pi->get_pg_num_pending() > m->pgid.ps()) {
+    dout(20) << " pg_num_pending " << pi->get_pg_num_pending() << " > " << m->pgid << dendl;
+    goto ignore;
+  }
+  return false;
+
+ ignore:
+  mon.no_reply(op);
+  return true;
+}
+
+bool OSDMonitor::prepare_pg_ready_to_merge(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m  = op->get_req<MOSDPGReadyToMerge>();
+  dout(10) << __func__ << " " << *m << dendl;
+  pg_pool_t p;
+  if (pending_inc.new_pools.count(m->pgid.pool()))
+    p = pending_inc.new_pools[m->pgid.pool()];
+  else
+    p = *osdmap.get_pg_pool(m->pgid.pool());
+  if (p.get_pg_num() != m->pgid.ps() + 1 ||
+      p.get_pg_num_pending() > m->pgid.ps()) {
+    dout(10) << __func__
+	     << " race with concurrent pg_num[_pending] update, will retry"
+	     << dendl;
+    wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+    return true;
+  }
+
+  if (m->ready) {
+    p.dec_pg_num(m->pgid,
+		 pending_inc.epoch,
+		 m->source_version,
+		 m->target_version,
+		 m->last_epoch_started,
+		 m->last_epoch_clean);
+    p.last_change = pending_inc.epoch;
+  } else {
+    // back off the merge attempt!
+    p.set_pg_num_pending(p.get_pg_num());
+  }
+
+  // force pre-nautilus clients to resend their ops, since they
+  // don't understand pg_num_pending changes form a new interval
+  p.last_force_op_resend_prenautilus = pending_inc.epoch;
+
+  pending_inc.new_pools[m->pgid.pool()] = p;
+
+  auto prob = g_conf().get_val<double>("mon_inject_pg_merge_bounce_probability");
+  if (m->ready &&
+      prob > 0 &&
+      prob > (double)(rand() % 1000)/1000.0) {
+    derr << __func__ << " injecting pg merge pg_num bounce" << dendl;
+    auto n = new MMonCommand(mon.monmap->get_fsid());
+    n->set_connection(m->get_connection());
+    n->cmd = { "{\"prefix\":\"osd pool set\", \"pool\": \"" +
+	       osdmap.get_pool_name(m->pgid.pool()) +
+	       "\", \"var\": \"pg_num_actual\", \"val\": \"" +
+	       stringify(m->pgid.ps() + 1) + "\"}" };
+    MonOpRequestRef nop = mon.op_tracker.create_request<MonOpRequest>(n);
+    nop->set_type_service();
+    wait_for_finished_proposal(op, new C_RetryMessage(this, nop));
+  } else {
+    wait_for_finished_proposal(op, new C_ReplyMap(this, op, m->version));
+  }
+  return true;
+}
+
+
+// -------------
+// pg_temp changes
+
+bool OSDMonitor::preprocess_pgtemp(MonOpRequestRef op)
+{
+  auto m = op->get_req<MOSDPGTemp>();
+  dout(10) << "preprocess_pgtemp " << *m << dendl;
+  mempool::osdmap::vector<int> empty;
+  int from = m->get_orig_source().num();
+  size_t ignore_cnt = 0;
+
+  // check caps
+  MonSession *session = op->get_session();
+  if (!session)
+    goto ignore;
+  if (!session->is_capable("osd", MON_CAP_X)) {
+    dout(0) << "attempt to send MOSDPGTemp from entity with insufficient caps "
+	    << session->caps << dendl;
+    goto ignore;
+  }
+
+  if (!osdmap.is_up(from) ||
+      !osdmap.get_addrs(from).legacy_equals(m->get_orig_source_addrs())) {
+    dout(7) << "ignoring pgtemp message from down "
+	    << m->get_orig_source() << " " << m->get_orig_source_addrs()
+	    << dendl;
+    goto ignore;
+  }
+
+  if (m->forced) {
+    return false;
+  }
+
+  for (auto p = m->pg_temp.begin(); p != m->pg_temp.end(); ++p) {
+    dout(20) << " " << p->first
+	     << (osdmap.pg_temp->count(p->first) ? osdmap.pg_temp->get(p->first) : empty)
+             << " -> " << p->second << dendl;
+
+    // does the pool exist?
+    if (!osdmap.have_pg_pool(p->first.pool())) {
+      /*
+       * 1. If the osdmap does not have the pool, it means the pool has been
+       *    removed in-between the osd sending this message and us handling it.
+       * 2. If osdmap doesn't have the pool, it is safe to assume the pool does
+       *    not exist in the pending either, as the osds would not send a
+       *    message about a pool they know nothing about (yet).
+       * 3. However, if the pool does exist in the pending, then it must be a
+       *    new pool, and not relevant to this message (see 1).
+       */
+      dout(10) << __func__ << " ignore " << p->first << " -> " << p->second
+               << ": pool has been removed" << dendl;
+      ignore_cnt++;
+      continue;
+    }
+
+    int acting_primary = -1;
+    osdmap.pg_to_up_acting_osds(
+      p->first, nullptr, nullptr, nullptr, &acting_primary);
+    if (acting_primary != from) {
+      /* If the source isn't the primary based on the current osdmap, we know
+       * that the interval changed and that we can discard this message.
+       * Indeed, we must do so to avoid 16127 since we can't otherwise determine
+       * which of two pg temp mappings on the same pg is more recent.
+       */
+      dout(10) << __func__ << " ignore " << p->first << " -> " << p->second
+	       << ": primary has changed" << dendl;
+      ignore_cnt++;
+      continue;
+    }
+
+    // removal?
+    if (p->second.empty() && (osdmap.pg_temp->count(p->first) ||
+			      osdmap.primary_temp->count(p->first)))
+      return false;
+    // change?
+    //  NOTE: we assume that this will clear pg_primary, so consider
+    //        an existing pg_primary field to imply a change
+    if (p->second.size() &&
+	(osdmap.pg_temp->count(p->first) == 0 ||
+	 osdmap.pg_temp->get(p->first) != p->second ||
+	 osdmap.primary_temp->count(p->first)))
+      return false;
+  }
+
+  // should we ignore all the pgs?
+  if (ignore_cnt == m->pg_temp.size())
+    goto ignore;
+
+  dout(7) << "preprocess_pgtemp e" << m->map_epoch << " no changes from " << m->get_orig_source_inst() << dendl;
+  _reply_map(op, m->map_epoch);
+  return true;
+
+ ignore:
+  mon.no_reply(op);
+  return true;
+}
+
+void OSDMonitor::update_up_thru(int from, epoch_t up_thru)
+{
+  epoch_t old_up_thru = osdmap.get_up_thru(from);
+  auto ut = pending_inc.new_up_thru.find(from);
+  if (ut != pending_inc.new_up_thru.end()) {
+    old_up_thru = ut->second;
+  }
+  if (up_thru > old_up_thru) {
+    // set up_thru too, so the osd doesn't have to ask again
+    pending_inc.new_up_thru[from] = up_thru;
+  }
+}
+
+bool OSDMonitor::prepare_pgtemp(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MOSDPGTemp>();
+  int from = m->get_orig_source().num();
+  dout(7) << "prepare_pgtemp e" << m->map_epoch << " from " << m->get_orig_source_inst() << dendl;
+  for (map<pg_t,vector<int32_t> >::iterator p = m->pg_temp.begin(); p != m->pg_temp.end(); ++p) {
+    uint64_t pool = p->first.pool();
+    if (pending_inc.old_pools.count(pool)) {
+      dout(10) << __func__ << " ignore " << p->first << " -> " << p->second
+               << ": pool pending removal" << dendl;
+      continue;
+    }
+    if (!osdmap.have_pg_pool(pool)) {
+      dout(10) << __func__ << " ignore " << p->first << " -> " << p->second
+               << ": pool has been removed" << dendl;
+      continue;
+    }
+    pending_inc.new_pg_temp[p->first] =
+      mempool::osdmap::vector<int>(p->second.begin(), p->second.end());
+
+    // unconditionally clear pg_primary (until this message can encode
+    // a change for that, too.. at which point we need to also fix
+    // preprocess_pg_temp)
+    if (osdmap.primary_temp->count(p->first) ||
+	pending_inc.new_primary_temp.count(p->first))
+      pending_inc.new_primary_temp[p->first] = -1;
+  }
+
+  // set up_thru too, so the osd doesn't have to ask again
+  update_up_thru(from, m->map_epoch);
+
+  wait_for_finished_proposal(op, new C_ReplyMap(this, op, m->map_epoch));
+  return true;
+}
+
+
+// ---
+
+bool OSDMonitor::preprocess_remove_snaps(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MRemoveSnaps>();
+  dout(7) << "preprocess_remove_snaps " << *m << dendl;
+
+  // check privilege, ignore if failed
+  MonSession *session = op->get_session();
+  mon.no_reply(op);
+  if (!session)
+    goto ignore;
+  if (!session->caps.is_capable(
+	cct,
+	session->entity_name,
+        "osd", "osd pool rmsnap", {}, true, true, false,
+	session->get_peer_socket_addr())) {
+    dout(0) << "got preprocess_remove_snaps from entity with insufficient caps "
+	    << session->caps << dendl;
+    goto ignore;
+  }
+
+  for (map<int, vector<snapid_t> >::iterator q = m->snaps.begin();
+       q != m->snaps.end();
+       ++q) {
+    if (!osdmap.have_pg_pool(q->first)) {
+      dout(10) << " ignoring removed_snaps " << q->second
+	       << " on non-existent pool " << q->first << dendl;
+      continue;
+    }
+    const pg_pool_t *pi = osdmap.get_pg_pool(q->first);
+    for (vector<snapid_t>::iterator p = q->second.begin();
+	 p != q->second.end();
+	 ++p) {
+      if (*p > pi->get_snap_seq() ||
+	  !_is_removed_snap(q->first, *p)) {
+	return false;
+      }
+    }
+  }
+
+  if (HAVE_FEATURE(m->get_connection()->get_features(), SERVER_OCTOPUS)) {
+    auto reply = make_message<MRemoveSnaps>();
+    reply->snaps = m->snaps;
+    mon.send_reply(op, reply.detach());
+  }
+
+ ignore:
+  return true;
+}
+
+bool OSDMonitor::prepare_remove_snaps(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MRemoveSnaps>();
+  dout(7) << "prepare_remove_snaps " << *m << dendl;
+
+  for (auto& [pool, snaps] : m->snaps) {
+    if (!osdmap.have_pg_pool(pool)) {
+      dout(10) << " ignoring removed_snaps " << snaps
+	       << " on non-existent pool " << pool << dendl;
+      continue;
+    }
+
+    pg_pool_t& pi = osdmap.pools[pool];
+    for (auto s : snaps) {
+      if (!_is_removed_snap(pool, s) &&
+	  (!pending_inc.new_pools.count(pool) ||
+	   !pending_inc.new_pools[pool].removed_snaps.contains(s)) &&
+	  (!pending_inc.new_removed_snaps.count(pool) ||
+	   !pending_inc.new_removed_snaps[pool].contains(s))) {
+	pg_pool_t *newpi = pending_inc.get_new_pool(pool, &pi);
+	if (osdmap.require_osd_release < ceph_release_t::octopus) {
+	  newpi->removed_snaps.insert(s);
+	  dout(10) << " pool " << pool << " removed_snaps added " << s
+		   << " (now " << newpi->removed_snaps << ")" << dendl;
+	}
+	newpi->flags |= pg_pool_t::FLAG_SELFMANAGED_SNAPS;
+	if (s > newpi->get_snap_seq()) {
+	  dout(10) << " pool " << pool << " snap_seq "
+		   << newpi->get_snap_seq() << " -> " << s << dendl;
+	  newpi->set_snap_seq(s);
+	}
+	newpi->set_snap_epoch(pending_inc.epoch);
+	dout(10) << " added pool " << pool << " snap " << s
+		 << " to removed_snaps queue" << dendl;
+	pending_inc.new_removed_snaps[pool].insert(s);
+      }
+    }
+  }
+
+  if (HAVE_FEATURE(m->get_connection()->get_features(), SERVER_OCTOPUS)) {
+    auto reply = make_message<MRemoveSnaps>();
+    reply->snaps = m->snaps;
+    wait_for_finished_proposal(op, new C_ReplyOp(this, op, reply));
+  }
+
+  return true;
+}
+
+bool OSDMonitor::preprocess_get_purged_snaps(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MMonGetPurgedSnaps>();
+  dout(7) << __func__ << " " << *m << dendl;
+
+  map<epoch_t,mempool::osdmap::map<int64_t,snap_interval_set_t>> r;
+
+  string k = make_purged_snap_epoch_key(m->start);
+  auto it = mon.store->get_iterator(OSD_SNAP_PREFIX);
+  it->upper_bound(k);
+  unsigned long epoch = m->last;
+  while (it->valid()) {
+    if (it->key().find("purged_epoch_") != 0) {
+      break;
+    }
+    string k = it->key();
+    int n = sscanf(k.c_str(), "purged_epoch_%lx", &epoch);
+    if (n != 1) {
+      derr << __func__ << " unable to parse key '" << it->key() << "'" << dendl;
+    } else if (epoch > m->last) {
+      break;
+    } else {
+      bufferlist bl = it->value();
+      auto p = bl.cbegin();
+      auto &v = r[epoch];
+      try {
+	ceph::decode(v, p);
+      } catch (ceph::buffer::error& e) {
+	derr << __func__ << " unable to parse value for key '" << it->key()
+	     << "': \n";
+	bl.hexdump(*_dout);
+	*_dout << dendl;
+      }
+      n += 4 + v.size() * 16;
+    }
+    if (n > 1048576) {
+      // impose a semi-arbitrary limit to message size
+      break;
+    }
+    it->next();
+  }
+
+  auto reply = make_message<MMonGetPurgedSnapsReply>(m->start, epoch);
+  reply->purged_snaps.swap(r);
+  mon.send_reply(op, reply.detach());
+
+  return true;
+}
+
+// osd beacon
+bool OSDMonitor::preprocess_beacon(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  // check caps
+  auto session = op->get_session();
+  mon.no_reply(op);
+  if (!session) {
+    dout(10) << __func__ << " no monitor session!" << dendl;
+    return true;
+  }
+  if (!session->is_capable("osd", MON_CAP_X)) {
+    derr << __func__ << " received from entity "
+         << "with insufficient privileges " << session->caps << dendl;
+    return true;
+  }
+  // Always forward the beacon to the leader, even if they are the same as
+  // the old one. The leader will mark as down osds that haven't sent
+  // beacon for a few minutes.
+  return false;
+}
+
+bool OSDMonitor::prepare_beacon(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  const auto beacon = op->get_req<MOSDBeacon>();
+  const auto src = beacon->get_orig_source();
+  dout(10) << __func__ << " " << *beacon
+	   << " from " << src << dendl;
+  int from = src.num();
+
+  if (!src.is_osd() ||
+      !osdmap.is_up(from) ||
+      !osdmap.get_addrs(from).legacy_equals(beacon->get_orig_source_addrs())) {
+    if (src.is_osd() && !osdmap.is_up(from)) {
+      // share some new maps with this guy in case it may not be
+      // aware of its own deadness...
+      send_latest(op, beacon->version+1);
+    }
+    dout(1) << " ignoring beacon from non-active osd." << from << dendl;
+    return false;
+  }
+
+  last_osd_report[from].first = ceph_clock_now();
+  last_osd_report[from].second = beacon->osd_beacon_report_interval;
+  osd_epochs[from] = beacon->version;
+
+  for (const auto& pg : beacon->pgs) {
+    if (auto* pool = osdmap.get_pg_pool(pg.pool()); pool != nullptr) {
+      unsigned pg_num = pool->get_pg_num();
+      last_epoch_clean.report(pg_num, pg, beacon->min_last_epoch_clean);
+    }
+  }
+
+  if (osdmap.osd_xinfo[from].last_purged_snaps_scrub <
+      beacon->last_purged_snaps_scrub) {
+    if (pending_inc.new_xinfo.count(from) == 0) {
+      pending_inc.new_xinfo[from] = osdmap.osd_xinfo[from];
+    }
+    pending_inc.new_xinfo[from].last_purged_snaps_scrub =
+      beacon->last_purged_snaps_scrub;
+    return true;
+  } else {
+    return false;
+  }
+}
+
+// ---------------
+// map helpers
+
+void OSDMonitor::send_latest(MonOpRequestRef op, epoch_t start)
+{
+  op->mark_osdmon_event(__func__);
+  dout(5) << "send_latest to " << op->get_req()->get_orig_source_inst()
+	  << " start " << start << dendl;
+  if (start == 0)
+    send_full(op);
+  else
+    send_incremental(op, start);
+}
+
+
+MOSDMap *OSDMonitor::build_latest_full(uint64_t features)
+{
+  MOSDMap *r = new MOSDMap(mon.monmap->fsid, features);
+  get_version_full(osdmap.get_epoch(), features, r->maps[osdmap.get_epoch()]);
+  r->oldest_map = get_first_committed();
+  r->newest_map = osdmap.get_epoch();
+  return r;
+}
+
+MOSDMap *OSDMonitor::build_incremental(epoch_t from, epoch_t to, uint64_t features)
+{
+  dout(10) << "build_incremental [" << from << ".." << to << "] with features "
+	   << std::hex << features << std::dec << dendl;
+  MOSDMap *m = new MOSDMap(mon.monmap->fsid, features);
+  m->oldest_map = get_first_committed();
+  m->newest_map = osdmap.get_epoch();
+
+  for (epoch_t e = to; e >= from && e > 0; e--) {
+    bufferlist bl;
+    int err = get_version(e, features, bl);
+    if (err == 0) {
+      ceph_assert(bl.length());
+      // if (get_version(e, bl) > 0) {
+      dout(20) << "build_incremental    inc " << e << " "
+	       << bl.length() << " bytes" << dendl;
+      m->incremental_maps[e] = bl;
+    } else {
+      ceph_assert(err == -ENOENT);
+      ceph_assert(!bl.length());
+      get_version_full(e, features, bl);
+      if (bl.length() > 0) {
+      //else if (get_version("full", e, bl) > 0) {
+      dout(20) << "build_incremental   full " << e << " "
+	       << bl.length() << " bytes" << dendl;
+      m->maps[e] = bl;
+      } else {
+	ceph_abort();  // we should have all maps.
+      }
+    }
+  }
+  return m;
+}
+
+void OSDMonitor::send_full(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  dout(5) << "send_full to " << op->get_req()->get_orig_source_inst() << dendl;
+  mon.send_reply(op, build_latest_full(op->get_session()->con_features));
+}
+
+void OSDMonitor::send_incremental(MonOpRequestRef op, epoch_t first)
+{
+  op->mark_osdmon_event(__func__);
+
+  MonSession *s = op->get_session();
+  ceph_assert(s);
+
+  if (s->proxy_con) {
+    // oh, we can tell the other mon to do it
+    dout(10) << __func__ << " asking proxying mon to send_incremental from "
+	     << first << dendl;
+    MRoute *r = new MRoute(s->proxy_tid, NULL);
+    r->send_osdmap_first = first;
+    s->proxy_con->send_message(r);
+    op->mark_event("reply: send routed send_osdmap_first reply");
+  } else {
+    // do it ourselves
+    send_incremental(first, s, false, op);
+  }
+}
+
+void OSDMonitor::send_incremental(epoch_t first,
+				  MonSession *session,
+				  bool onetime,
+				  MonOpRequestRef req)
+{
+  dout(5) << "send_incremental [" << first << ".." << osdmap.get_epoch() << "]"
+	  << " to " << session->name << dendl;
+
+  // get feature of the peer
+  // use quorum_con_features, if it's an anonymous connection.
+  uint64_t features = session->con_features ? session->con_features :
+    mon.get_quorum_con_features();
+
+  if (first <= session->osd_epoch) {
+    dout(10) << __func__ << " " << session->name << " should already have epoch "
+	     << session->osd_epoch << dendl;
+    first = session->osd_epoch + 1;
+  }
+
+  if (first < get_first_committed()) {
+    MOSDMap *m = new MOSDMap(osdmap.get_fsid(), features);
+    m->oldest_map = get_first_committed();
+    m->newest_map = osdmap.get_epoch();
+
+    first = get_first_committed();
+    bufferlist bl;
+    int err = get_version_full(first, features, bl);
+    ceph_assert(err == 0);
+    ceph_assert(bl.length());
+    dout(20) << "send_incremental starting with base full "
+	     << first << " " << bl.length() << " bytes" << dendl;
+    m->maps[first] = bl;
+
+    if (req) {
+      mon.send_reply(req, m);
+      session->osd_epoch = first;
+      return;
+    } else {
+      session->con->send_message(m);
+      session->osd_epoch = first;
+    }
+    first++;
+  }
+
+  while (first <= osdmap.get_epoch()) {
+    epoch_t last = std::min<epoch_t>(first + g_conf()->osd_map_message_max - 1,
+				     osdmap.get_epoch());
+    MOSDMap *m = build_incremental(first, last, features);
+
+    if (req) {
+      // send some maps.  it may not be all of them, but it will get them
+      // started.
+      mon.send_reply(req, m);
+    } else {
+      session->con->send_message(m);
+      first = last + 1;
+    }
+    session->osd_epoch = last;
+    if (onetime || req)
+      break;
+  }
+}
+
+int OSDMonitor::get_version(version_t ver, bufferlist& bl)
+{
+  return get_version(ver, mon.get_quorum_con_features(), bl);
+}
+
+void OSDMonitor::reencode_incremental_map(bufferlist& bl, uint64_t features)
+{
+  OSDMap::Incremental inc;
+  auto q = bl.cbegin();
+  inc.decode(q);
+  // always encode with subset of osdmap's canonical features
+  uint64_t f = features & inc.encode_features;
+  dout(20) << __func__ << " " << inc.epoch << " with features " << f
+	   << dendl;
+  bl.clear();
+  if (inc.fullmap.length()) {
+    // embedded full map?
+    OSDMap m;
+    m.decode(inc.fullmap);
+    inc.fullmap.clear();
+    m.encode(inc.fullmap, f | CEPH_FEATURE_RESERVED);
+  }
+  if (inc.crush.length()) {
+    // embedded crush map
+    CrushWrapper c;
+    auto p = inc.crush.cbegin();
+    c.decode(p);
+    inc.crush.clear();
+    c.encode(inc.crush, f);
+  }
+  inc.encode(bl, f | CEPH_FEATURE_RESERVED);
+}
+
+void OSDMonitor::reencode_full_map(bufferlist& bl, uint64_t features)
+{
+  OSDMap m;
+  auto q = bl.cbegin();
+  m.decode(q);
+  // always encode with subset of osdmap's canonical features
+  uint64_t f = features & m.get_encoding_features();
+  dout(20) << __func__ << " " << m.get_epoch() << " with features " << f
+	   << dendl;
+  bl.clear();
+  m.encode(bl, f | CEPH_FEATURE_RESERVED);
+}
+
+int OSDMonitor::get_version(version_t ver, uint64_t features, bufferlist& bl)
+{
+  uint64_t significant_features = OSDMap::get_significant_features(features);
+  if (inc_osd_cache.lookup({ver, significant_features}, &bl)) {
+    return 0;
+  }
+  int ret = PaxosService::get_version(ver, bl);
+  if (ret < 0) {
+    return ret;
+  }
+  // NOTE: this check is imprecise; the OSDMap encoding features may
+  // be a subset of the latest mon quorum features, but worst case we
+  // reencode once and then cache the (identical) result under both
+  // feature masks.
+  if (significant_features !=
+      OSDMap::get_significant_features(mon.get_quorum_con_features())) {
+    reencode_incremental_map(bl, features);
+  }
+  inc_osd_cache.add_bytes({ver, significant_features}, bl);
+  return 0;
+}
+
+int OSDMonitor::get_inc(version_t ver, OSDMap::Incremental& inc)
+{
+  bufferlist inc_bl;
+  int err = get_version(ver, inc_bl);
+  ceph_assert(err == 0);
+  ceph_assert(inc_bl.length());
+
+  auto p = inc_bl.cbegin();
+  inc.decode(p);
+  dout(10) << __func__ << "     "
+           << " epoch " << inc.epoch
+           << " inc_crc " << inc.inc_crc
+           << " full_crc " << inc.full_crc
+           << " encode_features " << inc.encode_features << dendl;
+  return 0;
+}
+
+int OSDMonitor::get_full_from_pinned_map(version_t ver, bufferlist& bl)
+{
+  dout(10) << __func__ << " ver " << ver << dendl;
+
+  version_t closest_pinned = osdmap_manifest.get_lower_closest_pinned(ver);
+  if (closest_pinned == 0) {
+    return -ENOENT;
+  }
+  if (closest_pinned > ver) {
+    dout(0) << __func__ << " pinned: " << osdmap_manifest.pinned << dendl;
+  }
+  ceph_assert(closest_pinned <= ver);
+
+  dout(10) << __func__ << " closest pinned ver " << closest_pinned << dendl;
+
+  // get osdmap incremental maps and apply on top of this one.
+  bufferlist osdm_bl;
+  bool has_cached_osdmap = false;
+  for (version_t v = ver-1; v >= closest_pinned; --v) {
+    if (full_osd_cache.lookup({v, mon.get_quorum_con_features()},
+                                &osdm_bl)) {
+      dout(10) << __func__ << " found map in cache ver " << v << dendl;
+      closest_pinned = v;
+      has_cached_osdmap = true;
+      break;
+    }
+  }
+
+  if (!has_cached_osdmap) {
+    int err = PaxosService::get_version_full(closest_pinned, osdm_bl);
+    if (err != 0) {
+      derr << __func__ << " closest pinned map ver " << closest_pinned
+           << " not available! error: " << cpp_strerror(err) << dendl;
+    }
+    ceph_assert(err == 0);
+  }
+
+  ceph_assert(osdm_bl.length());
+
+  OSDMap osdm;
+  osdm.decode(osdm_bl);
+
+  dout(10) << __func__ << " loaded osdmap epoch " << closest_pinned
+           << " e" << osdm.epoch
+           << " crc " << osdm.get_crc()
+           << " -- applying incremental maps." << dendl;
+
+  uint64_t encode_features = 0;
+  for (version_t v = closest_pinned + 1; v <= ver; ++v) {
+    dout(20) << __func__ << "    applying inc epoch " << v << dendl;
+
+    OSDMap::Incremental inc;
+    int err = get_inc(v, inc);
+    ceph_assert(err == 0);
+
+    encode_features = inc.encode_features;
+
+    err = osdm.apply_incremental(inc);
+    ceph_assert(err == 0);
+
+    // this block performs paranoid checks on map retrieval
+    if (g_conf().get_val<bool>("mon_debug_extra_checks") &&
+        inc.full_crc != 0) {
+
+      uint64_t f = encode_features;
+      if (!f) {
+        f = (mon.quorum_con_features ? mon.quorum_con_features : -1);
+      }
+
+      // encode osdmap to force calculating crcs
+      bufferlist tbl;
+      osdm.encode(tbl, f | CEPH_FEATURE_RESERVED);
+      // decode osdmap to compare crcs with what's expected by incremental
+      OSDMap tosdm;
+      tosdm.decode(tbl);
+
+      if (tosdm.get_crc() != inc.full_crc) {
+        derr << __func__
+             << "    osdmap crc mismatch! (osdmap crc " << tosdm.get_crc()
+             << ", expected " << inc.full_crc << ")" << dendl;
+        ceph_abort_msg("osdmap crc mismatch");
+      }
+    }
+
+    // note: we cannot add the recently computed map to the cache, as is,
+    // because we have not encoded the map into a bl.
+  }
+
+  if (!encode_features) {
+    dout(10) << __func__
+             << " last incremental map didn't have features;"
+             << " defaulting to quorum's or all" << dendl;
+    encode_features =
+      (mon.quorum_con_features ? mon.quorum_con_features : -1);
+  }
+  osdm.encode(bl, encode_features | CEPH_FEATURE_RESERVED);
+
+  return 0;
+}
+
+int OSDMonitor::get_version_full(version_t ver, bufferlist& bl)
+{
+  return get_version_full(ver, mon.get_quorum_con_features(), bl);
+}
+
+int OSDMonitor::get_version_full(version_t ver, uint64_t features,
+				 bufferlist& bl)
+{
+  uint64_t significant_features = OSDMap::get_significant_features(features);
+  if (full_osd_cache.lookup({ver, significant_features}, &bl)) {
+    return 0;
+  }
+  int ret = PaxosService::get_version_full(ver, bl);
+  if (ret == -ENOENT) {
+    // build map?
+    ret = get_full_from_pinned_map(ver, bl);
+  }
+  if (ret < 0) {
+    return ret;
+  }
+  // NOTE: this check is imprecise; the OSDMap encoding features may
+  // be a subset of the latest mon quorum features, but worst case we
+  // reencode once and then cache the (identical) result under both
+  // feature masks.
+  if (significant_features !=
+      OSDMap::get_significant_features(mon.get_quorum_con_features())) {
+    reencode_full_map(bl, features);
+  }
+  full_osd_cache.add_bytes({ver, significant_features}, bl);
+  return 0;
+}
+
+epoch_t OSDMonitor::blocklist(const entity_addrvec_t& av, utime_t until)
+{
+  dout(10) << "blocklist " << av << " until " << until << dendl;
+  for (auto a : av.v) {
+    if (osdmap.require_osd_release >= ceph_release_t::nautilus) {
+      a.set_type(entity_addr_t::TYPE_ANY);
+    } else {
+      a.set_type(entity_addr_t::TYPE_LEGACY);
+    }
+    pending_inc.new_blocklist[a] = until;
+  }
+  return pending_inc.epoch;
+}
+
+epoch_t OSDMonitor::blocklist(entity_addr_t a, utime_t until)
+{
+  if (osdmap.require_osd_release >= ceph_release_t::nautilus) {
+    a.set_type(entity_addr_t::TYPE_ANY);
+  } else {
+    a.set_type(entity_addr_t::TYPE_LEGACY);
+  }
+  dout(10) << "blocklist " << a << " until " << until << dendl;
+  pending_inc.new_blocklist[a] = until;
+  return pending_inc.epoch;
+}
+
+
+void OSDMonitor::check_osdmap_subs()
+{
+  dout(10) << __func__ << dendl;
+  if (!osdmap.get_epoch()) {
+    return;
+  }
+  auto osdmap_subs = mon.session_map.subs.find("osdmap");
+  if (osdmap_subs == mon.session_map.subs.end()) {
+    return;
+  }
+  auto p = osdmap_subs->second->begin();
+  while (!p.end()) {
+    auto sub = *p;
+    ++p;
+    check_osdmap_sub(sub);
+  }
+}
+
+void OSDMonitor::check_osdmap_sub(Subscription *sub)
+{
+  dout(10) << __func__ << " " << sub << " next " << sub->next
+	   << (sub->onetime ? " (onetime)":" (ongoing)") << dendl;
+  if (sub->next <= osdmap.get_epoch()) {
+    if (sub->next >= 1)
+      send_incremental(sub->next, sub->session, sub->incremental_onetime);
+    else
+      sub->session->con->send_message(build_latest_full(sub->session->con_features));
+    if (sub->onetime)
+      mon.session_map.remove_sub(sub);
+    else
+      sub->next = osdmap.get_epoch() + 1;
+  }
+}
+
+void OSDMonitor::check_pg_creates_subs()
+{
+  if (!osdmap.get_num_up_osds()) {
+    return;
+  }
+  ceph_assert(osdmap.get_up_osd_features() & CEPH_FEATURE_MON_STATEFUL_SUB);
+  mon.with_session_map([this](const MonSessionMap& session_map) {
+      auto pg_creates_subs = session_map.subs.find("osd_pg_creates");
+      if (pg_creates_subs == session_map.subs.end()) {
+	return;
+      }
+      for (auto sub : *pg_creates_subs->second) {
+	check_pg_creates_sub(sub);
+      }
+    });
+}
+
+void OSDMonitor::check_pg_creates_sub(Subscription *sub)
+{
+  dout(20) << __func__ << " .. " << sub->session->name << dendl;
+  ceph_assert(sub->type == "osd_pg_creates");
+  // only send these if the OSD is up.  we will check_subs() when they do
+  // come up so they will get the creates then.
+  if (sub->session->name.is_osd() &&
+      mon.osdmon()->osdmap.is_up(sub->session->name.num())) {
+    sub->next = send_pg_creates(sub->session->name.num(),
+				sub->session->con.get(),
+				sub->next);
+  }
+}
+
+void OSDMonitor::do_application_enable(int64_t pool_id,
+                                       const std::string &app_name,
+				       const std::string &app_key,
+				       const std::string &app_value,
+				       bool force)
+{
+  ceph_assert(paxos.is_plugged() && is_writeable());
+
+  dout(20) << __func__ << ": pool_id=" << pool_id << ", app_name=" << app_name
+           << dendl;
+
+  ceph_assert(osdmap.require_osd_release >= ceph_release_t::luminous);
+
+  auto pp = osdmap.get_pg_pool(pool_id);
+  ceph_assert(pp != nullptr);
+
+  pg_pool_t p = *pp;
+  if (pending_inc.new_pools.count(pool_id)) {
+    p = pending_inc.new_pools[pool_id];
+  }
+
+  if (app_key.empty()) {
+    p.application_metadata.insert({app_name, {}});
+  } else {
+    if (force) {
+      p.application_metadata[app_name][app_key] = app_value;
+    } else {
+      p.application_metadata.insert({app_name, {{app_key, app_value}}});
+    }
+  }
+  p.last_change = pending_inc.epoch;
+  pending_inc.new_pools[pool_id] = p;
+}
+
+void OSDMonitor::do_set_pool_opt(int64_t pool_id,
+				 pool_opts_t::key_t opt,
+				 pool_opts_t::value_t val)
+{
+  dout(10) << __func__ << " pool: " << pool_id << " option: " << opt
+	   << " val: " << val << dendl;
+  auto p = pending_inc.new_pools.try_emplace(
+    pool_id, *osdmap.get_pg_pool(pool_id));
+  p.first->second.opts.set(opt, val);
+}
+
+unsigned OSDMonitor::scan_for_creating_pgs(
+  const mempool::osdmap::map<int64_t,pg_pool_t>& pools,
+  const mempool::osdmap::set<int64_t>& removed_pools,
+  utime_t modified,
+  creating_pgs_t* creating_pgs) const
+{
+  unsigned queued = 0;
+  for (auto& p : pools) {
+    int64_t poolid = p.first;
+    if (creating_pgs->created_pools.count(poolid)) {
+      dout(10) << __func__ << " already created " << poolid << dendl;
+      continue;
+    }
+    const pg_pool_t& pool = p.second;
+    int ruleno = osdmap.crush->find_rule(pool.get_crush_rule(),
+					 pool.get_type(), pool.get_size());
+    if (ruleno < 0 || !osdmap.crush->rule_exists(ruleno))
+      continue;
+
+    const auto last_scan_epoch = creating_pgs->last_scan_epoch;
+    const auto created = pool.get_last_change();
+    if (last_scan_epoch && created <= last_scan_epoch) {
+      dout(10) << __func__ << " no change in pool " << poolid
+	       << " " << pool << dendl;
+      continue;
+    }
+    if (removed_pools.count(poolid)) {
+      dout(10) << __func__ << " pool is being removed: " << poolid
+	       << " " << pool << dendl;
+      continue;
+    }
+    dout(10) << __func__ << " queueing pool create for " << poolid
+	     << " " << pool << dendl;
+    creating_pgs->create_pool(poolid, pool.get_pg_num(),
+			      created, modified);
+    queued++;
+  }
+  return queued;
+}
+
+void OSDMonitor::update_creating_pgs()
+{
+  dout(10) << __func__ << " " << creating_pgs.pgs.size() << " pgs creating, "
+	   << creating_pgs.queue.size() << " pools in queue" << dendl;
+  decltype(creating_pgs_by_osd_epoch) new_pgs_by_osd_epoch;
+  std::lock_guard<std::mutex> l(creating_pgs_lock);
+  for (const auto& pg : creating_pgs.pgs) {
+    int acting_primary = -1;
+    auto pgid = pg.first;
+    if (!osdmap.pg_exists(pgid)) {
+      dout(20) << __func__ << " ignoring " << pgid << " which should not exist"
+	       << dendl;
+      continue;
+    }
+    auto mapped = pg.second.create_epoch;
+    dout(20) << __func__ << " looking up " << pgid << "@" << mapped << dendl;
+    spg_t spgid(pgid);
+    mapping.get_primary_and_shard(pgid, &acting_primary, &spgid);
+    // check the previous creating_pgs, look for the target to whom the pg was
+    // previously mapped
+    for (const auto& pgs_by_epoch : creating_pgs_by_osd_epoch) {
+      const auto last_acting_primary = pgs_by_epoch.first;
+      for (auto& pgs: pgs_by_epoch.second) {
+	if (pgs.second.count(spgid)) {
+	  if (last_acting_primary == acting_primary) {
+	    mapped = pgs.first;
+	  } else {
+	    dout(20) << __func__ << " " << pgid << " "
+		     << " acting_primary:" << last_acting_primary
+		     << " -> " << acting_primary << dendl;
+	    // note epoch if the target of the create message changed.
+	    mapped = mapping.get_epoch();
+          }
+          break;
+        } else {
+	  // newly creating
+	  mapped = mapping.get_epoch();
+	}
+      }
+    }
+    dout(10) << __func__ << " will instruct osd." << acting_primary
+	     << " to create " << pgid << "@" << mapped << dendl;
+    new_pgs_by_osd_epoch[acting_primary][mapped].insert(spgid);
+  }
+  creating_pgs_by_osd_epoch = std::move(new_pgs_by_osd_epoch);
+  creating_pgs_epoch = mapping.get_epoch();
+}
+
+epoch_t OSDMonitor::send_pg_creates(int osd, Connection *con, epoch_t next) const
+{
+  dout(30) << __func__ << " osd." << osd << " next=" << next
+	   << " " << creating_pgs_by_osd_epoch << dendl;
+  std::lock_guard<std::mutex> l(creating_pgs_lock);
+  if (creating_pgs_epoch <= creating_pgs.last_scan_epoch) {
+    dout(20) << __func__
+	     << " not using stale creating_pgs@" << creating_pgs_epoch << dendl;
+    // the subscribers will be updated when the mapping is completed anyway
+    return next;
+  }
+  auto creating_pgs_by_epoch = creating_pgs_by_osd_epoch.find(osd);
+  if (creating_pgs_by_epoch == creating_pgs_by_osd_epoch.end())
+    return next;
+  ceph_assert(!creating_pgs_by_epoch->second.empty());
+
+  MOSDPGCreate *oldm = nullptr; // for pre-mimic OSD compat
+  MOSDPGCreate2 *m = nullptr;
+
+  bool old = osdmap.require_osd_release < ceph_release_t::nautilus;
+
+  epoch_t last = 0;
+  for (auto epoch_pgs = creating_pgs_by_epoch->second.lower_bound(next);
+       epoch_pgs != creating_pgs_by_epoch->second.end(); ++epoch_pgs) {
+    auto epoch = epoch_pgs->first;
+    auto& pgs = epoch_pgs->second;
+    dout(20) << __func__ << " osd." << osd << " from " << next
+             << " : epoch " << epoch << " " << pgs.size() << " pgs" << dendl;
+    last = epoch;
+    for (auto& pg : pgs) {
+      // Need the create time from the monitor using its clock to set
+      // last_scrub_stamp upon pg creation.
+      auto create = creating_pgs.pgs.find(pg.pgid);
+      ceph_assert(create != creating_pgs.pgs.end());
+      if (old) {
+	if (!oldm) {
+	  oldm = new MOSDPGCreate(creating_pgs_epoch);
+	}
+	oldm->mkpg.emplace(pg.pgid,
+			   pg_create_t{create->second.create_epoch, pg.pgid, 0});
+	oldm->ctimes.emplace(pg.pgid, create->second.create_stamp);
+      } else {
+	if (!m) {
+	  m = new MOSDPGCreate2(creating_pgs_epoch);
+	}
+	m->pgs.emplace(pg, make_pair(create->second.create_epoch,
+				     create->second.create_stamp));
+	if (create->second.history.epoch_created) {
+	  dout(20) << __func__ << "   " << pg << " " << create->second.history
+		   << " " << create->second.past_intervals << dendl;
+	  m->pg_extra.emplace(pg, make_pair(create->second.history,
+					    create->second.past_intervals));
+	}
+      }
+      dout(20) << __func__ << " will create " << pg
+	       << " at " << create->second.create_epoch << dendl;
+    }
+  }
+  if (m) {
+    con->send_message(m);
+  } else if (oldm) {
+    con->send_message(oldm);
+  } else {
+    dout(20) << __func__ << " osd." << osd << " from " << next
+             << " has nothing to send" << dendl;
+    return next;
+  }
+
+  // sub is current through last + 1
+  return last + 1;
+}
+
+// TICK
+
+
+void OSDMonitor::tick()
+{
+  if (!is_active()) return;
+
+  dout(10) << osdmap << dendl;
+
+  // always update osdmap manifest, regardless of being the leader.
+  load_osdmap_manifest();
+
+  // always tune priority cache manager memory on leader and peons
+  if (ceph_using_tcmalloc() && mon_memory_autotune) {
+    std::lock_guard l(balancer_lock);
+    if (pcm != nullptr) {
+      pcm->tune_memory();
+      pcm->balance();
+      _set_new_cache_sizes();
+      dout(10) << "tick balancer "
+               << " inc cache_bytes: " << inc_cache->get_cache_bytes()
+               << " inc comtd_bytes: " << inc_cache->get_committed_size()
+               << " inc used_bytes: " << inc_cache->_get_used_bytes()
+               << " inc num_osdmaps: " << inc_cache->_get_num_osdmaps()
+               << dendl;
+      dout(10) << "tick balancer "
+               << " full cache_bytes: " << full_cache->get_cache_bytes()
+               << " full comtd_bytes: " << full_cache->get_committed_size()
+               << " full used_bytes: " << full_cache->_get_used_bytes()
+               << " full num_osdmaps: " << full_cache->_get_num_osdmaps()
+               << dendl;
+    }
+  }
+
+  if (!mon.is_leader()) return;
+
+  bool do_propose = false;
+  utime_t now = ceph_clock_now();
+
+  if (handle_osd_timeouts(now, last_osd_report)) {
+    do_propose = true;
+  }
+
+  // mark osds down?
+  if (check_failures(now)) {
+    do_propose = true;
+  }
+
+  // Force a proposal if we need to prune; pruning is performed on
+  // ``encode_pending()``, hence why we need to regularly trigger a proposal
+  // even if there's nothing going on.
+  if (is_prune_enabled() && should_prune()) {
+    do_propose = true;
+  }
+
+  // mark down osds out?
+
+  /* can_mark_out() checks if we can mark osds as being out. The -1 has no
+   * influence at all. The decision is made based on the ratio of "in" osds,
+   * and the function returns false if this ratio is lower that the minimum
+   * ratio set by g_conf()->mon_osd_min_in_ratio. So it's not really up to us.
+   */
+  if (can_mark_out(-1)) {
+    string down_out_subtree_limit = g_conf().get_val<string>(
+      "mon_osd_down_out_subtree_limit");
+    set<int> down_cache;  // quick cache of down subtrees
+
+    map<int,utime_t>::iterator i = down_pending_out.begin();
+    while (i != down_pending_out.end()) {
+      int o = i->first;
+      utime_t down = now;
+      down -= i->second;
+      ++i;
+
+      if (osdmap.is_down(o) &&
+	  osdmap.is_in(o) &&
+	  can_mark_out(o)) {
+	utime_t orig_grace(g_conf()->mon_osd_down_out_interval, 0);
+	utime_t grace = orig_grace;
+	double my_grace = 0.0;
+
+	if (g_conf()->mon_osd_adjust_down_out_interval) {
+	  // scale grace period the same way we do the heartbeat grace.
+	  const osd_xinfo_t& xi = osdmap.get_xinfo(o);
+	  double halflife = (double)g_conf()->mon_osd_laggy_halflife;
+	  double decay_k = ::log(.5) / halflife;
+	  double decay = exp((double)down * decay_k);
+	  dout(20) << "osd." << o << " laggy halflife " << halflife << " decay_k " << decay_k
+		   << " down for " << down << " decay " << decay << dendl;
+	  my_grace = decay * (double)xi.laggy_interval * xi.laggy_probability;
+	  grace += my_grace;
+	}
+
+	// is this an entire large subtree down?
+	if (down_out_subtree_limit.length()) {
+	  int type = osdmap.crush->get_type_id(down_out_subtree_limit);
+	  if (type > 0) {
+	    if (osdmap.containing_subtree_is_down(cct, o, type, &down_cache)) {
+	      dout(10) << "tick entire containing " << down_out_subtree_limit
+		       << " subtree for osd." << o
+		       << " is down; resetting timer" << dendl;
+	      // reset timer, too.
+	      down_pending_out[o] = now;
+	      continue;
+	    }
+	  }
+	}
+
+        bool down_out = !osdmap.is_destroyed(o) &&
+          g_conf()->mon_osd_down_out_interval > 0 && down.sec() >= grace;
+        bool destroyed_out = osdmap.is_destroyed(o) &&
+          g_conf()->mon_osd_destroyed_out_interval > 0 &&
+        // this is not precise enough as we did not make a note when this osd
+        // was marked as destroyed, but let's not bother with that
+        // complexity for now.
+          down.sec() >= g_conf()->mon_osd_destroyed_out_interval;
+        if (down_out || destroyed_out) {
+	  dout(10) << "tick marking osd." << o << " OUT after " << down
+		   << " sec (target " << grace << " = " << orig_grace << " + " << my_grace << ")" << dendl;
+	  pending_inc.new_weight[o] = CEPH_OSD_OUT;
+
+	  // set the AUTOOUT bit.
+	  if (pending_inc.new_state.count(o) == 0)
+	    pending_inc.new_state[o] = 0;
+	  pending_inc.new_state[o] |= CEPH_OSD_AUTOOUT;
+
+	  // remember previous weight
+	  if (pending_inc.new_xinfo.count(o) == 0)
+	    pending_inc.new_xinfo[o] = osdmap.osd_xinfo[o];
+	  pending_inc.new_xinfo[o].old_weight = osdmap.osd_weight[o];
+
+	  do_propose = true;
+
+	  mon.clog->info() << "Marking osd." << o << " out (has been down for "
+                            << int(down.sec()) << " seconds)";
+	} else
+	  continue;
+      }
+
+      down_pending_out.erase(o);
+    }
+  } else {
+    dout(10) << "tick NOOUT flag set, not checking down osds" << dendl;
+  }
+
+  // expire blocklisted items?
+  for (ceph::unordered_map<entity_addr_t,utime_t>::iterator p = osdmap.blocklist.begin();
+       p != osdmap.blocklist.end();
+       ++p) {
+    if (p->second < now) {
+      dout(10) << "expiring blocklist item " << p->first << " expired " << p->second << " < now " << now << dendl;
+      pending_inc.old_blocklist.push_back(p->first);
+      do_propose = true;
+    }
+  }
+  for (auto p = osdmap.range_blocklist.begin();
+       p != osdmap.range_blocklist.end();
+       ++p) {
+    if (p->second < now) {
+      dout(10) << "expiring range_blocklist item " << p->first
+	       << " expired " << p->second << " < now " << now << dendl;
+      pending_inc.old_range_blocklist.push_back(p->first);
+      do_propose = true;
+    }
+  }
+
+  if (try_prune_purged_snaps()) {
+    do_propose = true;
+  }
+
+  if (update_pools_status())
+    do_propose = true;
+
+  if (do_propose ||
+      !pending_inc.new_pg_temp.empty())  // also propose if we adjusted pg_temp
+    propose_pending();
+}
+
+void OSDMonitor::_set_new_cache_sizes()
+{
+  uint64_t cache_size = 0;
+  int64_t inc_alloc = 0;
+  int64_t full_alloc = 0;
+  int64_t kv_alloc = 0;
+
+  if (pcm != nullptr && rocksdb_binned_kv_cache != nullptr) {
+    cache_size = pcm->get_tuned_mem();
+    inc_alloc = inc_cache->get_committed_size();
+    full_alloc = full_cache->get_committed_size();
+    kv_alloc = rocksdb_binned_kv_cache->get_committed_size();
+  }
+
+  inc_osd_cache.set_bytes(inc_alloc);
+  full_osd_cache.set_bytes(full_alloc);
+
+  dout(1) << __func__ << " cache_size:" << cache_size
+           << " inc_alloc: " << inc_alloc
+           << " full_alloc: " << full_alloc
+           << " kv_alloc: " << kv_alloc
+           << dendl;
+}
+
+bool OSDMonitor::handle_osd_timeouts(const utime_t &now,
+				     std::map<int, std::pair<utime_t, int>> &last_osd_report)
+{
+  utime_t timeo(g_conf()->mon_osd_report_timeout, 0);
+  if (now - mon.get_leader_since() < timeo) {
+    // We haven't been the leader for long enough to consider OSD timeouts
+    return false;
+  }
+
+  int max_osd = osdmap.get_max_osd();
+  bool new_down = false;
+
+  for (int i=0; i < max_osd; ++i) {
+    dout(30) << __func__ << ": checking up on osd " << i << dendl;
+    if (!osdmap.exists(i)) {
+      last_osd_report.erase(i); // if any
+      continue;
+    }
+    if (!osdmap.is_up(i))
+      continue;
+    const std::map<int, std::pair<utime_t, int>>::const_iterator t = last_osd_report.find(i);
+    if (t == last_osd_report.end()) {
+      // it wasn't in the map; start the timer.
+      last_osd_report[i].first = now;
+      last_osd_report[i].second = 0;
+    } else if (can_mark_down(i)) {
+      utime_t diff = now - t->second.first;
+      // we use the max(mon_osd_report_timeout, 2*osd_beacon_report_interval) as timeout
+      // to allow for the osd to miss a beacon.
+      int mon_osd_report_timeout = g_conf()->mon_osd_report_timeout;
+      utime_t max_timeout(std::max(mon_osd_report_timeout,  2 * t->second.second), 0);
+      if (diff > max_timeout) {
+        mon.clog->info() << "osd." << i << " marked down after no beacon for "
+                          << diff << " seconds";
+        derr << "no beacon from osd." << i << " since " << t->second.first
+             << ", " << diff << " seconds ago.  marking down" << dendl;
+        pending_inc.new_state[i] = CEPH_OSD_UP;
+        new_down = true;
+      }
+    }
+  }
+  return new_down;
+}
+
+static void dump_cpu_list(Formatter *f, const char *name,
+			  const string& strlist)
+{
+  cpu_set_t cpu_set;
+  size_t cpu_set_size;
+  if (parse_cpu_set_list(strlist.c_str(), &cpu_set_size, &cpu_set) < 0) {
+    return;
+  }
+  set<int> cpus = cpu_set_to_set(cpu_set_size, &cpu_set);
+  f->open_array_section(name);
+  for (auto cpu : cpus) {
+    f->dump_int("cpu", cpu);
+  }
+  f->close_section();
+}
+
+void OSDMonitor::dump_info(Formatter *f)
+{
+  f->open_object_section("osdmap");
+  osdmap.dump(f);
+  f->close_section();
+
+  f->open_array_section("osd_metadata");
+  for (int i=0; i<osdmap.get_max_osd(); ++i) {
+    if (osdmap.exists(i)) {
+      f->open_object_section("osd");
+      f->dump_unsigned("id", i);
+      dump_osd_metadata(i, f, NULL);
+      f->close_section();
+    }
+  }
+  f->close_section();
+
+  f->open_object_section("osdmap_clean_epochs");
+  f->dump_unsigned("min_last_epoch_clean", get_min_last_epoch_clean());
+
+  f->open_object_section("last_epoch_clean");
+  last_epoch_clean.dump(f);
+  f->close_section();
+
+  f->open_array_section("osd_epochs");
+  for (auto& osd_epoch : osd_epochs) {
+    f->open_object_section("osd");
+    f->dump_unsigned("id", osd_epoch.first);
+    f->dump_unsigned("epoch", osd_epoch.second);
+    f->close_section();
+  }
+  f->close_section(); // osd_epochs
+
+  f->close_section(); // osd_clean_epochs
+
+  f->dump_unsigned("osdmap_first_committed", get_first_committed());
+  f->dump_unsigned("osdmap_last_committed", get_last_committed());
+
+  f->open_object_section("crushmap");
+  osdmap.crush->dump(f);
+  f->close_section();
+
+  if (has_osdmap_manifest) {
+    f->open_object_section("osdmap_manifest");
+    osdmap_manifest.dump(f);
+    f->close_section();
+  }
+}
+
+namespace {
+  enum osd_pool_get_choices {
+    SIZE, MIN_SIZE,
+    PG_NUM, PGP_NUM, CRUSH_RULE, HASHPSPOOL, EC_OVERWRITES,
+    NODELETE, NOPGCHANGE, NOSIZECHANGE,
+    WRITE_FADVISE_DONTNEED, NOSCRUB, NODEEP_SCRUB,
+    HIT_SET_TYPE, HIT_SET_PERIOD, HIT_SET_COUNT, HIT_SET_FPP,
+    USE_GMT_HITSET, TARGET_MAX_OBJECTS, TARGET_MAX_BYTES,
+    CACHE_TARGET_DIRTY_RATIO, CACHE_TARGET_DIRTY_HIGH_RATIO,
+    CACHE_TARGET_FULL_RATIO,
+    CACHE_MIN_FLUSH_AGE, CACHE_MIN_EVICT_AGE,
+    ERASURE_CODE_PROFILE, MIN_READ_RECENCY_FOR_PROMOTE,
+    MIN_WRITE_RECENCY_FOR_PROMOTE, FAST_READ,
+    HIT_SET_GRADE_DECAY_RATE, HIT_SET_SEARCH_LAST_N,
+    SCRUB_MIN_INTERVAL, SCRUB_MAX_INTERVAL, DEEP_SCRUB_INTERVAL,
+    RECOVERY_PRIORITY, RECOVERY_OP_PRIORITY, SCRUB_PRIORITY,
+    COMPRESSION_MODE, COMPRESSION_ALGORITHM, COMPRESSION_REQUIRED_RATIO,
+    COMPRESSION_MAX_BLOB_SIZE, COMPRESSION_MIN_BLOB_SIZE,
+    CSUM_TYPE, CSUM_MAX_BLOCK, CSUM_MIN_BLOCK, FINGERPRINT_ALGORITHM,
+    PG_AUTOSCALE_MODE, PG_NUM_MIN, TARGET_SIZE_BYTES, TARGET_SIZE_RATIO,
+    PG_AUTOSCALE_BIAS, DEDUP_TIER, DEDUP_CHUNK_ALGORITHM,
+    DEDUP_CDC_CHUNK_SIZE, PG_NUM_MAX, BULK };
+
+  std::set<osd_pool_get_choices>
+    subtract_second_from_first(const std::set<osd_pool_get_choices>& first,
+				const std::set<osd_pool_get_choices>& second)
+    {
+      std::set<osd_pool_get_choices> result;
+      std::set_difference(first.begin(), first.end(),
+			  second.begin(), second.end(),
+			  std::inserter(result, result.end()));
+      return result;
+    }
+}
+
+
+bool OSDMonitor::preprocess_command(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MMonCommand>();
+  int r = 0;
+  bufferlist rdata;
+  stringstream ss, ds;
+
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    derr << __func__ << " no session" << dendl;
+    mon.reply_command(op, -EACCES, "access denied", get_last_committed());
+    return true;
+  }
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+
+  string format;
+  cmd_getval(cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+  if (prefix == "osd stat") {
+    if (f) {
+      f->open_object_section("osdmap");
+      osdmap.print_summary(f.get(), ds, "", true);
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      osdmap.print_summary(nullptr, ds, "", true);
+      rdata.append(ds);
+    }
+  }
+  else if (prefix == "osd dump" ||
+	   prefix == "osd tree" ||
+	   prefix == "osd tree-from" ||
+	   prefix == "osd ls" ||
+	   prefix == "osd getmap" ||
+	   prefix == "osd getcrushmap" ||
+	   prefix == "osd ls-tree" ||
+	   prefix == "osd info") {
+
+    epoch_t epoch = 0;
+    int64_t epochnum;
+    cmd_getval(cmdmap, "epoch", epochnum, (int64_t)osdmap.get_epoch());
+    epoch = epochnum;
+    
+    bufferlist osdmap_bl;
+    int err = get_version_full(epoch, osdmap_bl);
+    if (err == -ENOENT) {
+      r = -ENOENT;
+      ss << "there is no map for epoch " << epoch;
+      goto reply;
+    }
+    ceph_assert(err == 0);
+    ceph_assert(osdmap_bl.length());
+
+    OSDMap *p;
+    if (epoch == osdmap.get_epoch()) {
+      p = &osdmap;
+    } else {
+      p = new OSDMap;
+      p->decode(osdmap_bl);
+    }
+
+    auto sg = make_scope_guard([&] {
+      if (p != &osdmap) {
+        delete p;
+      }
+    });
+
+    if (prefix == "osd dump") {
+      stringstream ds;
+      if (f) {
+	f->open_object_section("osdmap");
+	p->dump(f.get());
+	f->close_section();
+	f->flush(ds);
+      } else {
+	p->print(ds);
+      }
+      rdata.append(ds);
+      if (!f)
+	ds << " ";
+    } else if (prefix == "osd ls") {
+      if (f) {
+	f->open_array_section("osds");
+	for (int i = 0; i < osdmap.get_max_osd(); i++) {
+	  if (osdmap.exists(i)) {
+	    f->dump_int("osd", i);
+	  }
+	}
+	f->close_section();
+	f->flush(ds);
+      } else {
+	bool first = true;
+	for (int i = 0; i < osdmap.get_max_osd(); i++) {
+	  if (osdmap.exists(i)) {
+	    if (!first)
+	      ds << "\n";
+	    first = false;
+	    ds << i;
+	  }
+	}
+      }
+      rdata.append(ds);
+    } else if (prefix == "osd info") {
+      int64_t osd_id;
+      bool do_single_osd = true;
+      if (!cmd_getval(cmdmap, "id", osd_id)) {
+	do_single_osd = false;
+      }
+
+      if (do_single_osd && !osdmap.exists(osd_id)) {
+	ss << "osd." << osd_id << " does not exist";
+	r = -EINVAL;
+	goto reply;
+      }
+
+      if (f) {
+	if (do_single_osd) {
+	  osdmap.dump_osd(osd_id, f.get());
+	} else {
+	  osdmap.dump_osds(f.get());
+	}
+	f->flush(ds);
+      } else {
+	if (do_single_osd) {
+	  osdmap.print_osd(osd_id, ds);
+	} else {
+	  osdmap.print_osds(ds);
+	}
+      }
+      rdata.append(ds);
+    } else if (prefix == "osd tree" || prefix == "osd tree-from") {
+      string bucket;
+      if (prefix == "osd tree-from") {
+        cmd_getval(cmdmap, "bucket", bucket);
+        if (!osdmap.crush->name_exists(bucket)) {
+          ss << "bucket '" << bucket << "' does not exist";
+          r = -ENOENT;
+          goto reply;
+        }
+        int id = osdmap.crush->get_item_id(bucket);
+        if (id >= 0) {
+          ss << "\"" << bucket << "\" is not a bucket";
+          r = -EINVAL;
+          goto reply;
+        }
+      }
+
+      vector<string> states;
+      cmd_getval(cmdmap, "states", states);
+      unsigned filter = 0;
+      for (auto& s : states) {
+	if (s == "up") {
+	  filter |= OSDMap::DUMP_UP;
+	} else if (s == "down") {
+	  filter |= OSDMap::DUMP_DOWN;
+	} else if (s == "in") {
+	  filter |= OSDMap::DUMP_IN;
+	} else if (s == "out") {
+	  filter |= OSDMap::DUMP_OUT;
+	} else if (s == "destroyed") {
+	  filter |= OSDMap::DUMP_DESTROYED;
+	} else {
+	  ss << "unrecognized state '" << s << "'";
+	  r = -EINVAL;
+	  goto reply;
+	}
+      }
+      if ((filter & (OSDMap::DUMP_IN|OSDMap::DUMP_OUT)) ==
+	  (OSDMap::DUMP_IN|OSDMap::DUMP_OUT)) {
+        ss << "cannot specify both 'in' and 'out'";
+        r = -EINVAL;
+        goto reply;
+      }
+      if (((filter & (OSDMap::DUMP_UP|OSDMap::DUMP_DOWN)) ==
+	   (OSDMap::DUMP_UP|OSDMap::DUMP_DOWN)) ||
+           ((filter & (OSDMap::DUMP_UP|OSDMap::DUMP_DESTROYED)) ==
+           (OSDMap::DUMP_UP|OSDMap::DUMP_DESTROYED)) ||
+           ((filter & (OSDMap::DUMP_DOWN|OSDMap::DUMP_DESTROYED)) ==
+           (OSDMap::DUMP_DOWN|OSDMap::DUMP_DESTROYED))) {
+	ss << "can specify only one of 'up', 'down' and 'destroyed'";
+	r = -EINVAL;
+	goto reply;
+      }
+      if (f) {
+	f->open_object_section("tree");
+	p->print_tree(f.get(), NULL, filter, bucket);
+	f->close_section();
+	f->flush(ds);
+      } else {
+	p->print_tree(NULL, &ds, filter, bucket);
+      }
+      rdata.append(ds);
+    } else if (prefix == "osd getmap") {
+      rdata.append(osdmap_bl);
+      ss << "got osdmap epoch " << p->get_epoch();
+    } else if (prefix == "osd getcrushmap") {
+      p->crush->encode(rdata, mon.get_quorum_con_features());
+      ss << p->get_crush_version();
+    } else if (prefix == "osd ls-tree") {
+      string bucket_name;
+      cmd_getval(cmdmap, "name", bucket_name);
+      set<int> osds;
+      r = p->get_osds_by_bucket_name(bucket_name, &osds);
+      if (r == -ENOENT) {
+        ss << "\"" << bucket_name << "\" does not exist";
+        goto reply;
+      } else if (r < 0) {
+        ss << "can not parse bucket name:\"" << bucket_name << "\"";
+        goto reply;
+      }
+
+      if (f) {
+        f->open_array_section("osds");
+        for (auto &i : osds) {
+          if (osdmap.exists(i)) {
+            f->dump_int("osd", i);
+          }
+        }
+        f->close_section();
+        f->flush(ds);
+      } else {
+        bool first = true;
+        for (auto &i : osds) {
+          if (osdmap.exists(i)) {
+            if (!first)
+              ds << "\n";
+            first = false;
+            ds << i;
+          }
+        }
+      }
+
+      rdata.append(ds);
+    }
+  } else if (prefix == "osd getmaxosd") {
+    if (f) {
+      f->open_object_section("getmaxosd");
+      f->dump_unsigned("epoch", osdmap.get_epoch());
+      f->dump_int("max_osd", osdmap.get_max_osd());
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      ds << "max_osd = " << osdmap.get_max_osd() << " in epoch " << osdmap.get_epoch();
+      rdata.append(ds);
+    }
+  } else if (prefix == "osd utilization") {
+    string out;
+    osdmap.summarize_mapping_stats(NULL, NULL, &out, f.get());
+    if (f)
+      f->flush(rdata);
+    else
+      rdata.append(out);
+    r = 0;
+    goto reply;
+  } else if (prefix  == "osd find") {
+    int64_t osd;
+    if (!cmd_getval(cmdmap, "id", osd)) {
+      ss << "unable to parse osd id value '"
+         << cmd_vartype_stringify(cmdmap["id"]) << "'";
+      r = -EINVAL;
+      goto reply;
+    }
+    if (!osdmap.exists(osd)) {
+      ss << "osd." << osd << " does not exist";
+      r = -ENOENT;
+      goto reply;
+    }
+    string format;
+    cmd_getval(cmdmap, "format", format);
+    boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
+    f->open_object_section("osd_location");
+    f->dump_int("osd", osd);
+    f->dump_object("addrs", osdmap.get_addrs(osd));
+    f->dump_stream("osd_fsid") << osdmap.get_uuid(osd);
+
+    // try to identify host, pod/container name, etc.
+    map<string,string> m;
+    load_metadata(osd, m, nullptr);
+    if (auto p = m.find("hostname"); p != m.end()) {
+      f->dump_string("host", p->second);
+    }
+    for (auto& k : {
+	"pod_name", "pod_namespace", // set by rook
+	"container_name"             // set by cephadm, ceph-ansible
+	}) {
+      if (auto p = m.find(k); p != m.end()) {
+	f->dump_string(k, p->second);
+      }
+    }
+
+    // crush is helpful too
+    f->open_object_section("crush_location");
+    map<string,string> loc = osdmap.crush->get_full_location(osd);
+    for (map<string,string>::iterator p = loc.begin(); p != loc.end(); ++p)
+      f->dump_string(p->first.c_str(), p->second);
+    f->close_section();
+    f->close_section();
+    f->flush(rdata);
+  } else if (prefix == "osd metadata") {
+    int64_t osd = -1;
+    if (cmd_vartype_stringify(cmdmap["id"]).size() &&
+        !cmd_getval(cmdmap, "id", osd)) {
+      ss << "unable to parse osd id value '"
+         << cmd_vartype_stringify(cmdmap["id"]) << "'";
+      r = -EINVAL;
+      goto reply;
+    }
+    if (osd >= 0 && !osdmap.exists(osd)) {
+      ss << "osd." << osd << " does not exist";
+      r = -ENOENT;
+      goto reply;
+    }
+    string format;
+    cmd_getval(cmdmap, "format", format);
+    boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
+    if (osd >= 0) {
+      f->open_object_section("osd_metadata");
+      f->dump_unsigned("id", osd);
+      r = dump_osd_metadata(osd, f.get(), &ss);
+      if (r < 0)
+        goto reply;
+      f->close_section();
+    } else {
+      r = 0;
+      f->open_array_section("osd_metadata");
+      for (int i=0; i<osdmap.get_max_osd(); ++i) {
+        if (osdmap.exists(i)) {
+          f->open_object_section("osd");
+          f->dump_unsigned("id", i);
+          r = dump_osd_metadata(i, f.get(), NULL);
+          if (r == -EINVAL || r == -ENOENT) {
+            // Drop error, continue to get other daemons' metadata
+            dout(4) << "No metadata for osd." << i << dendl;
+            r = 0;
+          } else if (r < 0) {
+            // Unexpected error
+            goto reply;
+          }
+          f->close_section();
+        }
+      }
+      f->close_section();
+    }
+    f->flush(rdata);
+  } else if (prefix == "osd versions") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    count_metadata("ceph_version", f.get());
+    f->flush(rdata);
+    r = 0;
+  } else if (prefix == "osd count-metadata") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    string field;
+    cmd_getval(cmdmap, "property", field);
+    count_metadata(field, f.get());
+    f->flush(rdata);
+    r = 0;
+  } else if (prefix == "osd numa-status") {
+    TextTable tbl;
+    if (f) {
+      f->open_array_section("osds");
+    } else {
+      tbl.define_column("OSD", TextTable::LEFT, TextTable::RIGHT);
+      tbl.define_column("HOST", TextTable::LEFT, TextTable::LEFT);
+      tbl.define_column("NETWORK", TextTable::RIGHT, TextTable::RIGHT);
+      tbl.define_column("STORAGE", TextTable::RIGHT, TextTable::RIGHT);
+      tbl.define_column("AFFINITY", TextTable::RIGHT, TextTable::RIGHT);
+      tbl.define_column("CPUS", TextTable::LEFT, TextTable::LEFT);
+    }
+    for (int i=0; i<osdmap.get_max_osd(); ++i) {
+      if (osdmap.exists(i)) {
+	map<string,string> m;
+	ostringstream err;
+	if (load_metadata(i, m, &err) < 0) {
+	  continue;
+	}
+	string host;
+	auto p = m.find("hostname");
+	if (p != m.end()) {
+	  host = p->second;
+	}
+	if (f) {
+	  f->open_object_section("osd");
+	  f->dump_int("osd", i);
+	  f->dump_string("host", host);
+	  for (auto n : { "network_numa_node", "objectstore_numa_node",
+		"numa_node" }) {
+	    p = m.find(n);
+	    if (p != m.end()) {
+	      f->dump_int(n, atoi(p->second.c_str()));
+	    }
+	  }
+	  for (auto n : { "network_numa_nodes", "objectstore_numa_nodes" }) {
+	    p = m.find(n);
+	    if (p != m.end()) {
+	      list<string> ls = get_str_list(p->second, ",");
+	      f->open_array_section(n);
+	      for (auto node : ls) {
+		f->dump_int("node", atoi(node.c_str()));
+	      }
+	      f->close_section();
+	    }
+	  }
+	  for (auto n : { "numa_node_cpus" }) {
+	    p = m.find(n);
+	    if (p != m.end()) {
+	      dump_cpu_list(f.get(), n, p->second);
+	    }
+	  }
+	  f->close_section();
+	} else {
+	  tbl << i;
+	  tbl << host;
+	  p = m.find("network_numa_nodes");
+	  if (p != m.end()) {
+	    tbl << p->second;
+	  } else {
+	    tbl << "-";
+	  }
+	  p = m.find("objectstore_numa_nodes");
+	  if (p != m.end()) {
+	    tbl << p->second;
+	  } else {
+	    tbl << "-";
+	  }
+	  p = m.find("numa_node");
+	  auto q = m.find("numa_node_cpus");
+	  if (p != m.end() && q != m.end()) {
+	    tbl << p->second;
+	    tbl << q->second;
+	  } else {
+	    tbl << "-";
+	    tbl << "-";
+	  }
+	  tbl << TextTable::endrow;
+	}
+      }
+    }
+    if (f) {
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      rdata.append(stringify(tbl));
+    }
+  } else if (prefix == "osd map") {
+    string poolstr, objstr, namespacestr;
+    cmd_getval(cmdmap, "pool", poolstr);
+    cmd_getval(cmdmap, "object", objstr);
+    cmd_getval(cmdmap, "nspace", namespacestr);
+
+    int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+    if (pool < 0) {
+      ss << "pool " << poolstr << " does not exist";
+      r = -ENOENT;
+      goto reply;
+    }
+    object_locator_t oloc(pool, namespacestr);
+    object_t oid(objstr);
+    pg_t pgid = osdmap.object_locator_to_pg(oid, oloc);
+    pg_t mpgid = osdmap.raw_pg_to_pg(pgid);
+    vector<int> up, acting;
+    int up_p, acting_p;
+    osdmap.pg_to_up_acting_osds(mpgid, &up, &up_p, &acting, &acting_p);
+
+    string fullobjname;
+    if (!namespacestr.empty())
+      fullobjname = namespacestr + string("/") + oid.name;
+    else
+      fullobjname = oid.name;
+    if (f) {
+      f->open_object_section("osd_map");
+      f->dump_unsigned("epoch", osdmap.get_epoch());
+      f->dump_string("pool", poolstr);
+      f->dump_int("pool_id", pool);
+      f->dump_stream("objname") << fullobjname;
+      f->dump_stream("raw_pgid") << pgid;
+      f->dump_stream("pgid") << mpgid;
+      f->open_array_section("up");
+      for (vector<int>::iterator p = up.begin(); p != up.end(); ++p)
+        f->dump_int("osd", *p);
+      f->close_section();
+      f->dump_int("up_primary", up_p);
+      f->open_array_section("acting");
+      for (vector<int>::iterator p = acting.begin(); p != acting.end(); ++p)
+        f->dump_int("osd", *p);
+      f->close_section();
+      f->dump_int("acting_primary", acting_p);
+      f->close_section(); // osd_map
+      f->flush(rdata);
+    } else {
+      ds << "osdmap e" << osdmap.get_epoch()
+        << " pool '" << poolstr << "' (" << pool << ")"
+        << " object '" << fullobjname << "' ->"
+        << " pg " << pgid << " (" << mpgid << ")"
+        << " -> up (" << pg_vector_string(up) << ", p" << up_p << ") acting ("
+        << pg_vector_string(acting) << ", p" << acting_p << ")";
+      rdata.append(ds);
+    }
+
+  } else if (prefix == "pg map") {
+    pg_t pgid;
+    string pgidstr;
+    cmd_getval(cmdmap, "pgid", pgidstr);
+    if (!pgid.parse(pgidstr.c_str())) {
+      ss << "invalid pgid '" << pgidstr << "'";
+      r = -EINVAL;
+      goto reply;
+    }
+    vector<int> up, acting;
+    if (!osdmap.have_pg_pool(pgid.pool())) {
+      ss << "pg '" << pgidstr << "' does not exist";
+      r = -ENOENT;
+      goto reply;
+    }
+    pg_t mpgid = osdmap.raw_pg_to_pg(pgid);
+    osdmap.pg_to_up_acting_osds(pgid, up, acting);
+    if (f) {
+      f->open_object_section("pg_map");
+      f->dump_unsigned("epoch", osdmap.get_epoch());
+      f->dump_stream("raw_pgid") << pgid;
+      f->dump_stream("pgid") << mpgid;
+      f->open_array_section("up");
+      for (auto osd : up) {
+	f->dump_int("up_osd", osd);
+      }
+      f->close_section();
+      f->open_array_section("acting");
+      for (auto osd : acting) {
+	f->dump_int("acting_osd", osd);
+      }
+      f->close_section();
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      ds << "osdmap e" << osdmap.get_epoch()
+         << " pg " << pgid << " (" << mpgid << ")"
+         << " -> up " << up << " acting " << acting;
+      rdata.append(ds);
+    }
+    goto reply;
+
+  } else if (prefix == "osd lspools") {
+    if (f)
+      f->open_array_section("pools");
+    for (map<int64_t, pg_pool_t>::iterator p = osdmap.pools.begin();
+	 p != osdmap.pools.end();
+	 ++p) {
+      if (f) {
+	f->open_object_section("pool");
+	f->dump_int("poolnum", p->first);
+	f->dump_string("poolname", osdmap.pool_name[p->first]);
+	f->close_section();
+      } else {
+	ds << p->first << ' ' << osdmap.pool_name[p->first];
+	if (next(p) != osdmap.pools.end()) {
+	  ds << '\n';
+	}
+      }
+    }
+    if (f) {
+      f->close_section();
+      f->flush(ds);
+    }
+    rdata.append(ds);
+  } else if (prefix == "osd blocklist ls" ||
+	     prefix == "osd blacklist ls") {
+    if (f)
+      f->open_array_section("blocklist");
+
+    for (ceph::unordered_map<entity_addr_t,utime_t>::iterator p = osdmap.blocklist.begin();
+	 p != osdmap.blocklist.end();
+	 ++p) {
+      if (f) {
+	f->open_object_section("entry");
+	f->dump_string("addr", p->first.get_legacy_str());
+	f->dump_stream("until") << p->second;
+	f->close_section();
+      } else {
+	stringstream ss;
+	string s;
+	ss << p->first << " " << p->second;
+	getline(ss, s);
+	s += "\n";
+	rdata.append(s);
+      }
+    }
+    if (f) {
+      f->close_section();
+      f->flush(rdata);
+    }
+    if (f)
+      f->open_array_section("range_blocklist");
+
+    for (auto p = osdmap.range_blocklist.begin();
+	 p != osdmap.range_blocklist.end();
+	 ++p) {
+      if (f) {
+	f->open_object_section("entry");
+	f->dump_string("range", p->first.get_legacy_str());
+	f->dump_stream("until") << p->second;
+	f->close_section();
+      } else {
+	stringstream ss;
+	string s;
+	ss << p->first << " " << p->second;
+	getline(ss, s);
+	s += "\n";
+	rdata.append(s);
+      }
+    }
+    if (f) {
+      f->close_section();
+      f->flush(rdata);
+    }
+    ss << "listed " << osdmap.blocklist.size() + osdmap.range_blocklist.size() << " entries";
+
+  } else if (prefix == "osd pool ls") {
+    string detail;
+    cmd_getval(cmdmap, "detail", detail);
+    if (!f && detail == "detail") {
+      ostringstream ss;
+      osdmap.print_pools(ss);
+      rdata.append(ss.str());
+    } else {
+      if (f)
+	f->open_array_section("pools");
+      for (map<int64_t,pg_pool_t>::const_iterator it = osdmap.get_pools().begin();
+	   it != osdmap.get_pools().end();
+	   ++it) {
+	if (f) {
+	  if (detail == "detail") {
+	    f->open_object_section("pool");
+	    f->dump_int("pool_id", it->first);
+	    f->dump_string("pool_name", osdmap.get_pool_name(it->first));
+	    it->second.dump(f.get());
+	    f->close_section();
+	  } else {
+	    f->dump_string("pool_name", osdmap.get_pool_name(it->first));
+	  }
+	} else {
+	  rdata.append(osdmap.get_pool_name(it->first) + "\n");
+	}
+      }
+      if (f) {
+	f->close_section();
+	f->flush(rdata);
+      }
+    }
+
+  } else if (prefix == "osd crush get-tunable") {
+    string tunable;
+    cmd_getval(cmdmap, "tunable", tunable);
+    ostringstream rss;
+    if (f)
+      f->open_object_section("tunable");
+    if (tunable == "straw_calc_version") {
+      if (f)
+	f->dump_int(tunable.c_str(), osdmap.crush->get_straw_calc_version());
+      else
+	rss << osdmap.crush->get_straw_calc_version() << "\n";
+    } else {
+      r = -EINVAL;
+      goto reply;
+    }
+    if (f) {
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      rdata.append(rss.str());
+    }
+    r = 0;
+
+  } else if (prefix == "osd pool get") {
+    string poolstr;
+    cmd_getval(cmdmap, "pool", poolstr);
+    int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+    if (pool < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      r = -ENOENT;
+      goto reply;
+    }
+
+    const pg_pool_t *p = osdmap.get_pg_pool(pool);
+    string var;
+    cmd_getval(cmdmap, "var", var);
+
+    typedef std::map<std::string, osd_pool_get_choices> choices_map_t;
+    const choices_map_t ALL_CHOICES = {
+      {"size", SIZE},
+      {"min_size", MIN_SIZE},
+      {"pg_num", PG_NUM}, {"pgp_num", PGP_NUM},
+      {"crush_rule", CRUSH_RULE}, {"hashpspool", HASHPSPOOL},
+      {"allow_ec_overwrites", EC_OVERWRITES}, {"nodelete", NODELETE},
+      {"nopgchange", NOPGCHANGE}, {"nosizechange", NOSIZECHANGE},
+      {"noscrub", NOSCRUB}, {"nodeep-scrub", NODEEP_SCRUB},
+      {"write_fadvise_dontneed", WRITE_FADVISE_DONTNEED},
+      {"hit_set_type", HIT_SET_TYPE}, {"hit_set_period", HIT_SET_PERIOD},
+      {"hit_set_count", HIT_SET_COUNT}, {"hit_set_fpp", HIT_SET_FPP},
+      {"use_gmt_hitset", USE_GMT_HITSET},
+      {"target_max_objects", TARGET_MAX_OBJECTS},
+      {"target_max_bytes", TARGET_MAX_BYTES},
+      {"cache_target_dirty_ratio", CACHE_TARGET_DIRTY_RATIO},
+      {"cache_target_dirty_high_ratio", CACHE_TARGET_DIRTY_HIGH_RATIO},
+      {"cache_target_full_ratio", CACHE_TARGET_FULL_RATIO},
+      {"cache_min_flush_age", CACHE_MIN_FLUSH_AGE},
+      {"cache_min_evict_age", CACHE_MIN_EVICT_AGE},
+      {"erasure_code_profile", ERASURE_CODE_PROFILE},
+      {"min_read_recency_for_promote", MIN_READ_RECENCY_FOR_PROMOTE},
+      {"min_write_recency_for_promote", MIN_WRITE_RECENCY_FOR_PROMOTE},
+      {"fast_read", FAST_READ},
+      {"hit_set_grade_decay_rate", HIT_SET_GRADE_DECAY_RATE},
+      {"hit_set_search_last_n", HIT_SET_SEARCH_LAST_N},
+      {"scrub_min_interval", SCRUB_MIN_INTERVAL},
+      {"scrub_max_interval", SCRUB_MAX_INTERVAL},
+      {"deep_scrub_interval", DEEP_SCRUB_INTERVAL},
+      {"recovery_priority", RECOVERY_PRIORITY},
+      {"recovery_op_priority", RECOVERY_OP_PRIORITY},
+      {"scrub_priority", SCRUB_PRIORITY},
+      {"compression_mode", COMPRESSION_MODE},
+      {"compression_algorithm", COMPRESSION_ALGORITHM},
+      {"compression_required_ratio", COMPRESSION_REQUIRED_RATIO},
+      {"compression_max_blob_size", COMPRESSION_MAX_BLOB_SIZE},
+      {"compression_min_blob_size", COMPRESSION_MIN_BLOB_SIZE},
+      {"csum_type", CSUM_TYPE},
+      {"csum_max_block", CSUM_MAX_BLOCK},
+      {"csum_min_block", CSUM_MIN_BLOCK},
+      {"fingerprint_algorithm", FINGERPRINT_ALGORITHM},
+      {"pg_autoscale_mode", PG_AUTOSCALE_MODE},
+      {"pg_num_min", PG_NUM_MIN},
+      {"pg_num_max", PG_NUM_MAX},
+      {"target_size_bytes", TARGET_SIZE_BYTES},
+      {"target_size_ratio", TARGET_SIZE_RATIO},
+      {"pg_autoscale_bias", PG_AUTOSCALE_BIAS},
+      {"dedup_tier", DEDUP_TIER},
+      {"dedup_chunk_algorithm", DEDUP_CHUNK_ALGORITHM},
+      {"dedup_cdc_chunk_size", DEDUP_CDC_CHUNK_SIZE},
+      {"bulk", BULK}
+    };
+
+    typedef std::set<osd_pool_get_choices> choices_set_t;
+
+    const choices_set_t ONLY_TIER_CHOICES = {
+      HIT_SET_TYPE, HIT_SET_PERIOD, HIT_SET_COUNT, HIT_SET_FPP,
+      TARGET_MAX_OBJECTS, TARGET_MAX_BYTES, CACHE_TARGET_FULL_RATIO,
+      CACHE_TARGET_DIRTY_RATIO, CACHE_TARGET_DIRTY_HIGH_RATIO,
+      CACHE_MIN_FLUSH_AGE, CACHE_MIN_EVICT_AGE,
+      MIN_READ_RECENCY_FOR_PROMOTE,
+      MIN_WRITE_RECENCY_FOR_PROMOTE,
+      HIT_SET_GRADE_DECAY_RATE, HIT_SET_SEARCH_LAST_N
+    };
+    const choices_set_t ONLY_ERASURE_CHOICES = {
+      EC_OVERWRITES, ERASURE_CODE_PROFILE
+    };
+
+    choices_set_t selected_choices;
+    if (var == "all") {
+      for(choices_map_t::const_iterator it = ALL_CHOICES.begin();
+	  it != ALL_CHOICES.end(); ++it) {
+	selected_choices.insert(it->second);
+      }
+
+      if(!p->is_tier()) {
+	selected_choices = subtract_second_from_first(selected_choices,
+						      ONLY_TIER_CHOICES);
+      }
+
+      if(!p->is_erasure()) {
+	selected_choices = subtract_second_from_first(selected_choices,
+						      ONLY_ERASURE_CHOICES);
+      }
+    } else /* var != "all" */  {
+      choices_map_t::const_iterator found = ALL_CHOICES.find(var);
+      if (found == ALL_CHOICES.end()) {
+        ss << "pool '" << poolstr
+	       << "': invalid variable: '" << var << "'";
+        r = -EINVAL;
+        goto reply;
+      }
+
+      osd_pool_get_choices selected = found->second;
+
+      if (!p->is_tier() &&
+	  ONLY_TIER_CHOICES.find(selected) != ONLY_TIER_CHOICES.end()) {
+	ss << "pool '" << poolstr
+	   << "' is not a tier pool: variable not applicable";
+	r = -EACCES;
+	goto reply;
+      }
+
+      if (!p->is_erasure() &&
+	  ONLY_ERASURE_CHOICES.find(selected)
+	  != ONLY_ERASURE_CHOICES.end()) {
+	ss << "pool '" << poolstr
+	   << "' is not a erasure pool: variable not applicable";
+	r = -EACCES;
+	goto reply;
+      }
+
+      if (pool_opts_t::is_opt_name(var) &&
+	  !p->opts.is_set(pool_opts_t::get_opt_desc(var).key)) {
+	ss << "option '" << var << "' is not set on pool '" << poolstr << "'";
+	r = -ENOENT;
+	goto reply;
+      }
+
+      selected_choices.insert(selected);
+    }
+
+    if (f) {
+      f->open_object_section("pool");
+      f->dump_string("pool", poolstr);
+      f->dump_int("pool_id", pool);
+      for(choices_set_t::const_iterator it = selected_choices.begin();
+	  it != selected_choices.end(); ++it) {
+	choices_map_t::const_iterator i;
+        for (i = ALL_CHOICES.begin(); i != ALL_CHOICES.end(); ++i) {
+          if (i->second == *it) {
+            break;
+          }
+        }
+        ceph_assert(i != ALL_CHOICES.end());
+	switch(*it) {
+	  case PG_NUM:
+	    f->dump_int("pg_num", p->get_pg_num());
+	    break;
+	  case PGP_NUM:
+	    f->dump_int("pgp_num", p->get_pgp_num());
+	    break;
+	  case SIZE:
+	    f->dump_int("size", p->get_size());
+	    break;
+	  case MIN_SIZE:
+	    f->dump_int("min_size", p->get_min_size());
+	    break;
+	  case CRUSH_RULE:
+	    if (osdmap.crush->rule_exists(p->get_crush_rule())) {
+	      f->dump_string("crush_rule", osdmap.crush->get_rule_name(
+			       p->get_crush_rule()));
+	    } else {
+	      f->dump_string("crush_rule", stringify(p->get_crush_rule()));
+	    }
+	    break;
+	  case EC_OVERWRITES:
+	    f->dump_bool("allow_ec_overwrites",
+                         p->has_flag(pg_pool_t::FLAG_EC_OVERWRITES));
+	    break;
+	  case PG_AUTOSCALE_MODE:
+	    f->dump_string("pg_autoscale_mode",
+			   pg_pool_t::get_pg_autoscale_mode_name(
+			     p->pg_autoscale_mode));
+	    break;
+	  case HASHPSPOOL:
+	  case NODELETE:
+	  case BULK:
+	  case NOPGCHANGE:
+	  case NOSIZECHANGE:
+	  case WRITE_FADVISE_DONTNEED:
+	  case NOSCRUB:
+	  case NODEEP_SCRUB:
+	    f->dump_bool(i->first.c_str(),
+			   p->has_flag(pg_pool_t::get_flag_by_name(i->first)));
+	    break;
+	  case HIT_SET_PERIOD:
+	    f->dump_int("hit_set_period", p->hit_set_period);
+	    break;
+	  case HIT_SET_COUNT:
+	    f->dump_int("hit_set_count", p->hit_set_count);
+	    break;
+	  case HIT_SET_TYPE:
+	    f->dump_string("hit_set_type",
+			   HitSet::get_type_name(p->hit_set_params.get_type()));
+	    break;
+	  case HIT_SET_FPP:
+	    {
+	      if (p->hit_set_params.get_type() == HitSet::TYPE_BLOOM) {
+		BloomHitSet::Params *bloomp =
+		  static_cast<BloomHitSet::Params*>(p->hit_set_params.impl.get());
+		f->dump_float("hit_set_fpp", bloomp->get_fpp());
+	      } else if(var != "all") {
+		f->close_section();
+		ss << "hit set is not of type Bloom; " <<
+		  "invalid to get a false positive rate!";
+		r = -EINVAL;
+		goto reply;
+	      }
+	    }
+	    break;
+	  case USE_GMT_HITSET:
+	    f->dump_bool("use_gmt_hitset", p->use_gmt_hitset);
+	    break;
+	  case TARGET_MAX_OBJECTS:
+	    f->dump_unsigned("target_max_objects", p->target_max_objects);
+	    break;
+	  case TARGET_MAX_BYTES:
+	    f->dump_unsigned("target_max_bytes", p->target_max_bytes);
+	    break;
+	  case CACHE_TARGET_DIRTY_RATIO:
+	    f->dump_unsigned("cache_target_dirty_ratio_micro",
+			     p->cache_target_dirty_ratio_micro);
+	    f->dump_float("cache_target_dirty_ratio",
+			  ((float)p->cache_target_dirty_ratio_micro/1000000));
+	    break;
+	  case CACHE_TARGET_DIRTY_HIGH_RATIO:
+	    f->dump_unsigned("cache_target_dirty_high_ratio_micro",
+			     p->cache_target_dirty_high_ratio_micro);
+	    f->dump_float("cache_target_dirty_high_ratio",
+			  ((float)p->cache_target_dirty_high_ratio_micro/1000000));
+	    break;
+	  case CACHE_TARGET_FULL_RATIO:
+	    f->dump_unsigned("cache_target_full_ratio_micro",
+			     p->cache_target_full_ratio_micro);
+	    f->dump_float("cache_target_full_ratio",
+			  ((float)p->cache_target_full_ratio_micro/1000000));
+	    break;
+	  case CACHE_MIN_FLUSH_AGE:
+	    f->dump_unsigned("cache_min_flush_age", p->cache_min_flush_age);
+	    break;
+	  case CACHE_MIN_EVICT_AGE:
+	    f->dump_unsigned("cache_min_evict_age", p->cache_min_evict_age);
+	    break;
+	  case ERASURE_CODE_PROFILE:
+	    f->dump_string("erasure_code_profile", p->erasure_code_profile);
+	    break;
+	  case MIN_READ_RECENCY_FOR_PROMOTE:
+	    f->dump_int("min_read_recency_for_promote",
+			p->min_read_recency_for_promote);
+	    break;
+	  case MIN_WRITE_RECENCY_FOR_PROMOTE:
+	    f->dump_int("min_write_recency_for_promote",
+			p->min_write_recency_for_promote);
+	    break;
+          case FAST_READ:
+            f->dump_int("fast_read", p->fast_read);
+            break;
+	  case HIT_SET_GRADE_DECAY_RATE:
+	    f->dump_int("hit_set_grade_decay_rate",
+			p->hit_set_grade_decay_rate);
+	    break;
+	  case HIT_SET_SEARCH_LAST_N:
+	    f->dump_int("hit_set_search_last_n",
+			p->hit_set_search_last_n);
+	    break;
+	  case SCRUB_MIN_INTERVAL:
+	  case SCRUB_MAX_INTERVAL:
+	  case DEEP_SCRUB_INTERVAL:
+          case RECOVERY_PRIORITY:
+          case RECOVERY_OP_PRIORITY:
+          case SCRUB_PRIORITY:
+	  case COMPRESSION_MODE:
+	  case COMPRESSION_ALGORITHM:
+	  case COMPRESSION_REQUIRED_RATIO:
+	  case COMPRESSION_MAX_BLOB_SIZE:
+	  case COMPRESSION_MIN_BLOB_SIZE:
+	  case CSUM_TYPE:
+	  case CSUM_MAX_BLOCK:
+	  case CSUM_MIN_BLOCK:
+	  case FINGERPRINT_ALGORITHM:
+	  case PG_NUM_MIN:
+	  case PG_NUM_MAX:
+	  case TARGET_SIZE_BYTES:
+	  case TARGET_SIZE_RATIO:
+	  case PG_AUTOSCALE_BIAS:
+	  case DEDUP_TIER:
+	  case DEDUP_CHUNK_ALGORITHM:
+	  case DEDUP_CDC_CHUNK_SIZE:
+            pool_opts_t::key_t key = pool_opts_t::get_opt_desc(i->first).key;
+            if (p->opts.is_set(key)) {
+              if(*it == CSUM_TYPE) {
+                int64_t val;
+                p->opts.get(pool_opts_t::CSUM_TYPE, &val);
+                f->dump_string(i->first.c_str(), Checksummer::get_csum_type_string(val));
+              } else {
+                p->opts.dump(i->first, f.get());
+              }
+	    }
+            break;
+	}
+      }
+      f->close_section();
+      f->flush(rdata);
+    } else /* !f */ {
+      for(choices_set_t::const_iterator it = selected_choices.begin();
+	  it != selected_choices.end(); ++it) {
+	choices_map_t::const_iterator i;
+	switch(*it) {
+	  case PG_NUM:
+	    ss << "pg_num: " << p->get_pg_num() << "\n";
+	    break;
+	  case PGP_NUM:
+	    ss << "pgp_num: " << p->get_pgp_num() << "\n";
+	    break;
+	  case SIZE:
+	    ss << "size: " << p->get_size() << "\n";
+	    break;
+	  case MIN_SIZE:
+	    ss << "min_size: " << p->get_min_size() << "\n";
+	    break;
+	  case CRUSH_RULE:
+	    if (osdmap.crush->rule_exists(p->get_crush_rule())) {
+	      ss << "crush_rule: " << osdmap.crush->get_rule_name(
+		p->get_crush_rule()) << "\n";
+	    } else {
+	      ss << "crush_rule: " << p->get_crush_rule() << "\n";
+	    }
+	    break;
+	  case PG_AUTOSCALE_MODE:
+	    ss << "pg_autoscale_mode: " << pg_pool_t::get_pg_autoscale_mode_name(
+	      p->pg_autoscale_mode) <<"\n";
+	    break;
+	  case HIT_SET_PERIOD:
+	    ss << "hit_set_period: " << p->hit_set_period << "\n";
+	    break;
+	  case HIT_SET_COUNT:
+	    ss << "hit_set_count: " << p->hit_set_count << "\n";
+	    break;
+	  case HIT_SET_TYPE:
+	    ss << "hit_set_type: " <<
+	      HitSet::get_type_name(p->hit_set_params.get_type()) << "\n";
+	    break;
+	  case HIT_SET_FPP:
+	    {
+	      if (p->hit_set_params.get_type() == HitSet::TYPE_BLOOM) {
+		BloomHitSet::Params *bloomp =
+		  static_cast<BloomHitSet::Params*>(p->hit_set_params.impl.get());
+		ss << "hit_set_fpp: " << bloomp->get_fpp() << "\n";
+	      } else if(var != "all") {
+		ss << "hit set is not of type Bloom; " <<
+		  "invalid to get a false positive rate!";
+		r = -EINVAL;
+		goto reply;
+	      }
+	    }
+	    break;
+	  case USE_GMT_HITSET:
+	    ss << "use_gmt_hitset: " << p->use_gmt_hitset << "\n";
+	    break;
+	  case TARGET_MAX_OBJECTS:
+	    ss << "target_max_objects: " << p->target_max_objects << "\n";
+	    break;
+	  case TARGET_MAX_BYTES:
+	    ss << "target_max_bytes: " << p->target_max_bytes << "\n";
+	    break;
+	  case CACHE_TARGET_DIRTY_RATIO:
+	    ss << "cache_target_dirty_ratio: "
+	       << ((float)p->cache_target_dirty_ratio_micro/1000000) << "\n";
+	    break;
+	  case CACHE_TARGET_DIRTY_HIGH_RATIO:
+	    ss << "cache_target_dirty_high_ratio: "
+	       << ((float)p->cache_target_dirty_high_ratio_micro/1000000) << "\n";
+	    break;
+	  case CACHE_TARGET_FULL_RATIO:
+	    ss << "cache_target_full_ratio: "
+	       << ((float)p->cache_target_full_ratio_micro/1000000) << "\n";
+	    break;
+	  case CACHE_MIN_FLUSH_AGE:
+	    ss << "cache_min_flush_age: " << p->cache_min_flush_age << "\n";
+	    break;
+	  case CACHE_MIN_EVICT_AGE:
+	    ss << "cache_min_evict_age: " << p->cache_min_evict_age << "\n";
+	    break;
+	  case ERASURE_CODE_PROFILE:
+	    ss << "erasure_code_profile: " << p->erasure_code_profile << "\n";
+	    break;
+	  case MIN_READ_RECENCY_FOR_PROMOTE:
+	    ss << "min_read_recency_for_promote: " <<
+	      p->min_read_recency_for_promote << "\n";
+	    break;
+	  case HIT_SET_GRADE_DECAY_RATE:
+	    ss << "hit_set_grade_decay_rate: " <<
+	      p->hit_set_grade_decay_rate << "\n";
+	    break;
+	  case HIT_SET_SEARCH_LAST_N:
+	    ss << "hit_set_search_last_n: " <<
+	      p->hit_set_search_last_n << "\n";
+	    break;
+	  case EC_OVERWRITES:
+	    ss << "allow_ec_overwrites: " <<
+	      (p->has_flag(pg_pool_t::FLAG_EC_OVERWRITES) ? "true" : "false") <<
+	      "\n";
+	    break;
+	  case HASHPSPOOL:
+	  case NODELETE:
+	  case BULK:
+	  case NOPGCHANGE:
+	  case NOSIZECHANGE:
+	  case WRITE_FADVISE_DONTNEED:
+	  case NOSCRUB:
+	  case NODEEP_SCRUB:
+	    for (i = ALL_CHOICES.begin(); i != ALL_CHOICES.end(); ++i) {
+	      if (i->second == *it)
+		break;
+	    }
+	    ceph_assert(i != ALL_CHOICES.end());
+	    ss << i->first << ": " <<
+	      (p->has_flag(pg_pool_t::get_flag_by_name(i->first)) ?
+	       "true" : "false") << "\n";
+	    break;
+	  case MIN_WRITE_RECENCY_FOR_PROMOTE:
+	    ss << "min_write_recency_for_promote: " <<
+	      p->min_write_recency_for_promote << "\n";
+	    break;
+          case FAST_READ:
+            ss << "fast_read: " << p->fast_read << "\n";
+            break;
+	  case SCRUB_MIN_INTERVAL:
+	  case SCRUB_MAX_INTERVAL:
+	  case DEEP_SCRUB_INTERVAL:
+          case RECOVERY_PRIORITY:
+          case RECOVERY_OP_PRIORITY:
+          case SCRUB_PRIORITY:
+	  case COMPRESSION_MODE:
+	  case COMPRESSION_ALGORITHM:
+	  case COMPRESSION_REQUIRED_RATIO:
+	  case COMPRESSION_MAX_BLOB_SIZE:
+	  case COMPRESSION_MIN_BLOB_SIZE:
+	  case CSUM_TYPE:
+	  case CSUM_MAX_BLOCK:
+	  case CSUM_MIN_BLOCK:
+	  case FINGERPRINT_ALGORITHM:
+	  case PG_NUM_MIN:
+	  case PG_NUM_MAX:
+	  case TARGET_SIZE_BYTES:
+	  case TARGET_SIZE_RATIO:
+	  case PG_AUTOSCALE_BIAS:
+	  case DEDUP_TIER:
+	  case DEDUP_CHUNK_ALGORITHM:
+	  case DEDUP_CDC_CHUNK_SIZE:
+	    for (i = ALL_CHOICES.begin(); i != ALL_CHOICES.end(); ++i) {
+	      if (i->second == *it)
+		break;
+	    }
+	    ceph_assert(i != ALL_CHOICES.end());
+	    {
+	      pool_opts_t::key_t key = pool_opts_t::get_opt_desc(i->first).key;
+	      if (p->opts.is_set(key)) {
+                if(key == pool_opts_t::CSUM_TYPE) {
+                  int64_t val;
+                  p->opts.get(key, &val);
+  		  ss << i->first << ": " << Checksummer::get_csum_type_string(val) << "\n";
+                } else {
+  		  ss << i->first << ": " << p->opts.get(key) << "\n";
+                }
+	      }
+	    }
+	    break;
+	}
+	rdata.append(ss.str());
+	ss.str("");
+      }
+    }
+    r = 0;
+  } else if (prefix == "osd pool get-quota") {
+    string pool_name;
+    cmd_getval(cmdmap, "pool", pool_name);
+
+    int64_t poolid = osdmap.lookup_pg_pool_name(pool_name);
+    if (poolid < 0) {
+      ceph_assert(poolid == -ENOENT);
+      ss << "unrecognized pool '" << pool_name << "'";
+      r = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(poolid);
+    const pool_stat_t* pstat = mon.mgrstatmon()->get_pool_stat(poolid);
+    if (!pstat) {
+      ss << "no stats for pool '" << pool_name << "'";
+      r = -ENOENT;
+      goto reply;
+    }
+    const object_stat_sum_t& sum = pstat->stats.sum;
+    if (f) {
+      f->open_object_section("pool_quotas");
+      f->dump_string("pool_name", pool_name);
+      f->dump_unsigned("pool_id", poolid);
+      f->dump_unsigned("quota_max_objects", p->quota_max_objects);
+      f->dump_int("current_num_objects", sum.num_objects);
+      f->dump_unsigned("quota_max_bytes", p->quota_max_bytes);
+      f->dump_int("current_num_bytes", sum.num_bytes);
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      stringstream rs;
+      rs << "quotas for pool '" << pool_name << "':\n"
+         << "  max objects: ";
+      if (p->quota_max_objects == 0)
+        rs << "N/A";
+      else {
+        rs << si_u_t(p->quota_max_objects) << " objects";
+        rs << "  (current num objects: " << sum.num_objects << " objects)";
+      }
+      rs << "\n"
+         << "  max bytes  : ";
+      if (p->quota_max_bytes == 0)
+        rs << "N/A";
+      else {
+        rs << byte_u_t(p->quota_max_bytes);
+        rs << "  (current num bytes: " << sum.num_bytes << " bytes)";
+      }
+      rdata.append(rs.str());
+    }
+    rdata.append("\n");
+    r = 0;
+  } else if (prefix == "osd crush rule list" ||
+	     prefix == "osd crush rule ls") {
+    if (f) {
+      f->open_array_section("rules");
+      osdmap.crush->list_rules(f.get());
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      ostringstream ss;
+      osdmap.crush->list_rules(&ss);
+      rdata.append(ss.str());
+    }
+  } else if (prefix == "osd crush rule ls-by-class") {
+    string class_name;
+    cmd_getval(cmdmap, "class", class_name);
+    if (class_name.empty()) {
+      ss << "no class specified";
+      r = -EINVAL;
+      goto reply;
+    }
+    set<int> rules;
+    r = osdmap.crush->get_rules_by_class(class_name, &rules);
+    if (r < 0) {
+      ss << "failed to get rules by class '" << class_name << "'";
+      goto reply;
+    }
+    if (f) {
+      f->open_array_section("rules");
+      for (auto &rule: rules) {
+        f->dump_string("name", osdmap.crush->get_rule_name(rule));
+      }
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      ostringstream rs;
+      for (auto &rule: rules) {
+        rs << osdmap.crush->get_rule_name(rule) << "\n";
+      }
+      rdata.append(rs.str());
+    }
+  } else if (prefix == "osd crush rule dump") {
+    string name;
+    cmd_getval(cmdmap, "name", name);
+    string format;
+    cmd_getval(cmdmap, "format", format);
+    boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
+    if (name == "") {
+      f->open_array_section("rules");
+      osdmap.crush->dump_rules(f.get());
+      f->close_section();
+    } else {
+      int ruleno = osdmap.crush->get_rule_id(name);
+      if (ruleno < 0) {
+	ss << "unknown crush rule '" << name << "'";
+	r = ruleno;
+	goto reply;
+      }
+      osdmap.crush->dump_rule(ruleno, f.get());
+    }
+    ostringstream rs;
+    f->flush(rs);
+    rs << "\n";
+    rdata.append(rs.str());
+  } else if (prefix == "osd crush dump") {
+    string format;
+    cmd_getval(cmdmap, "format", format);
+    boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
+    f->open_object_section("crush_map");
+    osdmap.crush->dump(f.get());
+    f->close_section();
+    ostringstream rs;
+    f->flush(rs);
+    rs << "\n";
+    rdata.append(rs.str());
+  } else if (prefix == "osd crush show-tunables") {
+    string format;
+    cmd_getval(cmdmap, "format", format);
+    boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
+    f->open_object_section("crush_map_tunables");
+    osdmap.crush->dump_tunables(f.get());
+    f->close_section();
+    ostringstream rs;
+    f->flush(rs);
+    rs << "\n";
+    rdata.append(rs.str());
+  } else if (prefix == "osd crush tree") {
+    string shadow;
+    cmd_getval(cmdmap, "shadow", shadow);
+    bool show_shadow = shadow == "--show-shadow";
+    boost::scoped_ptr<Formatter> f(Formatter::create(format));
+    if (f) {
+      f->open_object_section("crush_tree");
+      osdmap.crush->dump_tree(nullptr,
+                              f.get(),
+                              osdmap.get_pool_names(),
+                              show_shadow);
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      ostringstream ss;
+      osdmap.crush->dump_tree(&ss,
+                              nullptr,
+                              osdmap.get_pool_names(),
+                              show_shadow);
+      rdata.append(ss.str());
+    }
+  } else if (prefix == "osd crush ls") {
+    string name;
+    if (!cmd_getval(cmdmap, "node", name)) {
+      ss << "no node specified";
+      r = -EINVAL;
+      goto reply;
+    }
+    if (!osdmap.crush->name_exists(name)) {
+      ss << "node '" << name << "' does not exist";
+      r = -ENOENT;
+      goto reply;
+    }
+    int id = osdmap.crush->get_item_id(name);
+    list<int> result;
+    if (id >= 0) {
+      result.push_back(id);
+    } else {
+      int num = osdmap.crush->get_bucket_size(id);
+      for (int i = 0; i < num; ++i) {
+	result.push_back(osdmap.crush->get_bucket_item(id, i));
+      }
+    }
+    if (f) {
+      f->open_array_section("items");
+      for (auto i : result) {
+	f->dump_string("item", osdmap.crush->get_item_name(i));
+      }
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      ostringstream ss;
+      for (auto i : result) {
+	ss << osdmap.crush->get_item_name(i) << "\n";
+      }
+      rdata.append(ss.str());
+    }
+    r = 0;
+  } else if (prefix == "osd crush class ls") {
+    boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
+    f->open_array_section("crush_classes");
+    for (auto i : osdmap.crush->class_name)
+      f->dump_string("class", i.second);
+    f->close_section();
+    f->flush(rdata);
+  } else if (prefix == "osd crush class ls-osd") {
+    string name;
+    cmd_getval(cmdmap, "class", name);
+    set<int> osds;
+    osdmap.crush->get_devices_by_class(name, &osds);
+    if (f) {
+      f->open_array_section("osds");
+      for (auto &osd: osds)
+        f->dump_int("osd", osd);
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      bool first = true;
+      for (auto &osd : osds) {
+        if (!first)
+          ds << "\n";
+        first = false;
+        ds << osd;
+      }
+      rdata.append(ds);
+    }
+  } else if (prefix == "osd crush get-device-class") {
+    vector<string> idvec;
+    cmd_getval(cmdmap, "ids", idvec);
+    map<int, string> class_by_osd;
+    for (auto& id : idvec) {
+      ostringstream ts;
+      long osd = parse_osd_id(id.c_str(), &ts);
+      if (osd < 0) {
+        ss << "unable to parse osd id:'" << id << "'";
+        r = -EINVAL;
+        goto reply;
+      }
+      auto device_class = osdmap.crush->get_item_class(osd);
+      if (device_class)
+        class_by_osd[osd] = device_class;
+      else
+        class_by_osd[osd] = ""; // no class
+    }
+    if (f) {
+      f->open_array_section("osd_device_classes");
+      for (auto& i : class_by_osd) {
+        f->open_object_section("osd_device_class");
+        f->dump_int("osd", i.first);
+        f->dump_string("device_class", i.second);
+        f->close_section();
+      }
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      if (class_by_osd.size() == 1) {
+        // for single input, make a clean output
+        ds << class_by_osd.begin()->second;
+      } else {
+        // note that we do not group osds by class here
+        for (auto it = class_by_osd.begin();
+             it != class_by_osd.end();
+             it++) {
+          ds << "osd." << it->first << ' ' << it->second;
+          if (next(it) != class_by_osd.end())
+            ds << '\n';
+        }
+      }
+      rdata.append(ds);
+    }
+  } else if (prefix == "osd erasure-code-profile ls") {
+    const auto &profiles = osdmap.get_erasure_code_profiles();
+    if (f)
+      f->open_array_section("erasure-code-profiles");
+    for (auto i = profiles.begin(); i != profiles.end(); ++i) {
+      if (f)
+        f->dump_string("profile", i->first.c_str());
+      else
+	rdata.append(i->first + "\n");
+    }
+    if (f) {
+      f->close_section();
+      ostringstream rs;
+      f->flush(rs);
+      rs << "\n";
+      rdata.append(rs.str());
+    }
+  } else if (prefix == "osd crush weight-set ls") {
+    boost::scoped_ptr<Formatter> f(Formatter::create(format));
+    if (f) {
+      f->open_array_section("weight_sets");
+      if (osdmap.crush->have_choose_args(CrushWrapper::DEFAULT_CHOOSE_ARGS)) {
+	f->dump_string("pool", "(compat)");
+      }
+      for (auto& i : osdmap.crush->choose_args) {
+	if (i.first >= 0) {
+	  f->dump_string("pool", osdmap.get_pool_name(i.first));
+	}
+      }
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      ostringstream rs;
+      if (osdmap.crush->have_choose_args(CrushWrapper::DEFAULT_CHOOSE_ARGS)) {
+	rs << "(compat)\n";
+      }
+      for (auto& i : osdmap.crush->choose_args) {
+	if (i.first >= 0) {
+	  rs << osdmap.get_pool_name(i.first) << "\n";
+	}
+      }
+      rdata.append(rs.str());
+    }
+  } else if (prefix == "osd crush weight-set dump") {
+    boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty",
+						     "json-pretty"));
+    osdmap.crush->dump_choose_args(f.get());
+    f->flush(rdata);
+  } else if (prefix == "osd erasure-code-profile get") {
+    string name;
+    cmd_getval(cmdmap, "name", name);
+    if (!osdmap.has_erasure_code_profile(name)) {
+      ss << "unknown erasure code profile '" << name << "'";
+      r = -ENOENT;
+      goto reply;
+    }
+    const map<string,string> &profile = osdmap.get_erasure_code_profile(name);
+    if (f)
+      f->open_object_section("profile");
+    for (map<string,string>::const_iterator i = profile.begin();
+	 i != profile.end();
+	 ++i) {
+      if (f)
+        f->dump_string(i->first.c_str(), i->second.c_str());
+      else
+	rdata.append(i->first + "=" + i->second + "\n");
+    }
+    if (f) {
+      f->close_section();
+      ostringstream rs;
+      f->flush(rs);
+      rs << "\n";
+      rdata.append(rs.str());
+    }
+  } else if (prefix == "osd pool application get") {
+    boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty",
+                                                     "json-pretty"));
+    string pool_name;
+    cmd_getval(cmdmap, "pool", pool_name);
+    string app;
+    cmd_getval(cmdmap, "app", app);
+    string key;
+    cmd_getval(cmdmap, "key", key);
+
+    if (pool_name.empty()) {
+      // all
+      f->open_object_section("pools");
+      for (const auto &pool : osdmap.pools) {
+        std::string name("<unknown>");
+        const auto &pni = osdmap.pool_name.find(pool.first);
+        if (pni != osdmap.pool_name.end())
+          name = pni->second;
+        f->open_object_section(name.c_str());
+        for (auto &app_pair : pool.second.application_metadata) {
+          f->open_object_section(app_pair.first.c_str());
+          for (auto &kv_pair : app_pair.second) {
+            f->dump_string(kv_pair.first.c_str(), kv_pair.second);
+          }
+          f->close_section();
+        }
+        f->close_section(); // name
+      }
+      f->close_section(); // pools
+      f->flush(rdata);
+    } else {
+      int64_t pool = osdmap.lookup_pg_pool_name(pool_name.c_str());
+      if (pool < 0) {
+        ss << "unrecognized pool '" << pool_name << "'";
+        r = -ENOENT;
+        goto reply;
+      }
+      auto p = osdmap.get_pg_pool(pool);
+      // filter by pool
+      if (app.empty()) {
+        f->open_object_section(pool_name.c_str());
+        for (auto &app_pair : p->application_metadata) {
+          f->open_object_section(app_pair.first.c_str());
+          for (auto &kv_pair : app_pair.second) {
+            f->dump_string(kv_pair.first.c_str(), kv_pair.second);
+          }
+          f->close_section(); // application
+        }
+        f->close_section(); // pool_name
+        f->flush(rdata);
+        goto reply;
+      }
+
+      auto app_it = p->application_metadata.find(app);
+      if (app_it == p->application_metadata.end()) {
+        ss << "pool '" << pool_name << "' has no application '" << app << "'";
+        r = -ENOENT;
+        goto reply;
+      }
+      // filter by pool + app
+      if (key.empty()) {
+        f->open_object_section(app_it->first.c_str());
+        for (auto &kv_pair : app_it->second) {
+          f->dump_string(kv_pair.first.c_str(), kv_pair.second);
+        }
+        f->close_section(); // application
+        f->flush(rdata);
+        goto reply;
+      }
+      // filter by pool + app + key
+      auto key_it = app_it->second.find(key);
+      if (key_it == app_it->second.end()) {
+        ss << "application '" << app << "' on pool '" << pool_name
+           << "' does not have key '" << key << "'";
+        r = -ENOENT;
+        goto reply;
+      }
+      ss << key_it->second << "\n";
+      rdata.append(ss.str());
+      ss.str("");
+    }
+  } else if (prefix == "osd get-require-min-compat-client") {
+    ss << osdmap.require_min_compat_client << std::endl;
+    rdata.append(ss.str());
+    ss.str("");
+    goto reply;
+  } else if (prefix == "osd pool application enable" ||
+             prefix == "osd pool application disable" ||
+             prefix == "osd pool application set" ||
+             prefix == "osd pool application rm") {
+    bool changed = false;
+    r = preprocess_command_pool_application(prefix, cmdmap, ss, &changed);
+    if (r != 0) {
+      // Error, reply.
+      goto reply;
+    } else if (changed) {
+      // Valid mutation, proceed to prepare phase
+      return false;
+    } else {
+      // Idempotent case, reply
+      goto reply;
+    }
+  } else {
+    // try prepare update
+    return false;
+  }
+
+ reply:
+  string rs;
+  getline(ss, rs);
+  mon.reply_command(op, r, rs, rdata, get_last_committed());
+  return true;
+}
+
+void OSDMonitor::set_pool_flags(int64_t pool_id, uint64_t flags)
+{
+  pg_pool_t *pool = pending_inc.get_new_pool(pool_id,
+    osdmap.get_pg_pool(pool_id));
+  ceph_assert(pool);
+  pool->set_flag(flags);
+}
+
+void OSDMonitor::clear_pool_flags(int64_t pool_id, uint64_t flags)
+{
+  pg_pool_t *pool = pending_inc.get_new_pool(pool_id,
+    osdmap.get_pg_pool(pool_id));
+  ceph_assert(pool);
+  pool->unset_flag(flags);
+}
+
+string OSDMonitor::make_purged_snap_epoch_key(epoch_t epoch)
+{
+  char k[80];
+  snprintf(k, sizeof(k), "purged_epoch_%08lx", (unsigned long)epoch);
+  return k;
+}
+
+string OSDMonitor::make_purged_snap_key(int64_t pool, snapid_t snap)
+{
+  char k[80];
+  snprintf(k, sizeof(k), "purged_snap_%llu_%016llx",
+	   (unsigned long long)pool, (unsigned long long)snap);
+  return k;
+}
+
+string OSDMonitor::make_purged_snap_key_value(
+  int64_t pool, snapid_t snap, snapid_t num,
+  epoch_t epoch, bufferlist *v)
+{
+  // encode the *last* epoch in the key so that we can use forward
+  // iteration only to search for an epoch in an interval.
+  encode(snap, *v);
+  encode(snap + num, *v);
+  encode(epoch, *v);
+  return make_purged_snap_key(pool, snap + num - 1);
+}
+
+
+int OSDMonitor::lookup_purged_snap(
+  int64_t pool, snapid_t snap,
+  snapid_t *begin, snapid_t *end)
+{
+  string k = make_purged_snap_key(pool, snap);
+  auto it = mon.store->get_iterator(OSD_SNAP_PREFIX);
+  it->lower_bound(k);
+  if (!it->valid()) {
+    dout(20) << __func__
+	     << " pool " << pool << " snap " << snap
+	     << " - key '" << k << "' not found" << dendl;
+    return -ENOENT;
+  }
+  if (it->key().find("purged_snap_") != 0) {
+    dout(20) << __func__
+	     << " pool " << pool << " snap " << snap
+	     << " - key '" << k << "' got '" << it->key()
+	     << "', wrong prefix" << dendl;
+    return -ENOENT;
+  }
+  string gotk = it->key();
+  const char *format = "purged_snap_%llu_";
+  long long int keypool;
+  int n = sscanf(gotk.c_str(), format, &keypool);
+  if (n != 1) {
+    derr << __func__ << " invalid k '" << gotk << "'" << dendl;
+    return -ENOENT;
+  }
+  if (pool != keypool) {
+    dout(20) << __func__
+	     << " pool " << pool << " snap " << snap
+	     << " - key '" << k << "' got '" << gotk
+	     << "', wrong pool " << keypool
+	     << dendl;
+    return -ENOENT;
+  }
+  bufferlist v = it->value();
+  auto p = v.cbegin();
+  decode(*begin, p);
+  decode(*end, p);
+  if (snap < *begin || snap >= *end) {
+    dout(20) << __func__
+	     << " pool " << pool << " snap " << snap
+	     << " - found [" << *begin << "," << *end << "), no overlap"
+	     << dendl;
+    return -ENOENT;
+  }
+  return 0;
+}
+
+void OSDMonitor::insert_purged_snap_update(
+  int64_t pool,
+  snapid_t start, snapid_t end,
+  epoch_t epoch,
+  MonitorDBStore::TransactionRef t)
+{
+  snapid_t before_begin, before_end;
+  snapid_t after_begin, after_end;
+  int b = lookup_purged_snap(pool, start - 1,
+			     &before_begin, &before_end);
+  int a = lookup_purged_snap(pool, end,
+			     &after_begin, &after_end);
+  if (!b && !a) {
+    dout(10) << __func__
+	     << " [" << start << "," << end << ") - joins ["
+	     << before_begin << "," << before_end << ") and ["
+	     << after_begin << "," << after_end << ")" << dendl;
+    // erase only the begin record; we'll overwrite the end one.
+    t->erase(OSD_SNAP_PREFIX, make_purged_snap_key(pool, before_end - 1));
+    bufferlist v;
+    string k = make_purged_snap_key_value(pool,
+					  before_begin, after_end - before_begin,
+					  pending_inc.epoch, &v);
+    t->put(OSD_SNAP_PREFIX, k, v);
+  } else if (!b) {
+    dout(10) << __func__
+	     << " [" << start << "," << end << ") - join with earlier ["
+	     << before_begin << "," << before_end << ")" << dendl;
+    t->erase(OSD_SNAP_PREFIX, make_purged_snap_key(pool, before_end - 1));
+    bufferlist v;
+    string k = make_purged_snap_key_value(pool,
+					  before_begin, end - before_begin,
+					  pending_inc.epoch, &v);
+    t->put(OSD_SNAP_PREFIX, k, v);
+  } else if (!a) {
+    dout(10) << __func__
+	     << " [" << start << "," << end << ") - join with later ["
+	     << after_begin << "," << after_end << ")" << dendl;
+    // overwrite after record
+    bufferlist v;
+    string k = make_purged_snap_key_value(pool,
+					  start, after_end - start,
+					  pending_inc.epoch, &v);
+    t->put(OSD_SNAP_PREFIX, k, v);
+  } else {
+    dout(10) << __func__
+	     << " [" << start << "," << end << ") - new"
+	     << dendl;
+    bufferlist v;
+    string k = make_purged_snap_key_value(pool,
+					  start, end - start,
+					  pending_inc.epoch, &v);
+    t->put(OSD_SNAP_PREFIX, k, v);
+  }
+}
+
+bool OSDMonitor::try_prune_purged_snaps()
+{
+  if (!mon.mgrstatmon()->is_readable()) {
+    return false;
+  }
+  if (!pending_inc.new_purged_snaps.empty()) {
+    return false;  // we already pruned for this epoch
+  }
+
+  unsigned max_prune = cct->_conf.get_val<uint64_t>(
+    "mon_max_snap_prune_per_epoch");
+  if (!max_prune) {
+    max_prune = 100000;
+  }
+  dout(10) << __func__ << " max_prune " << max_prune << dendl;
+
+  unsigned actually_pruned = 0;
+  auto& purged_snaps = mon.mgrstatmon()->get_digest().purged_snaps;
+  for (auto& p : osdmap.get_pools()) {
+    auto q = purged_snaps.find(p.first);
+    if (q == purged_snaps.end()) {
+      continue;
+    }
+    auto& purged = q->second;
+    if (purged.empty()) {
+      dout(20) << __func__ << " " << p.first << " nothing purged" << dendl;
+      continue;
+    }
+    dout(20) << __func__ << " pool " << p.first << " purged " << purged << dendl;
+    snap_interval_set_t to_prune;
+    unsigned maybe_pruned = actually_pruned;
+    for (auto i = purged.begin(); i != purged.end(); ++i) {
+      snapid_t begin = i.get_start();
+      auto end = i.get_start() + i.get_len();
+      snapid_t pbegin = 0, pend = 0;
+      int r = lookup_purged_snap(p.first, begin, &pbegin, &pend);
+      if (r == 0) {
+	// already purged.
+	// be a bit aggressive about backing off here, because the mon may
+	// do a lot of work going through this set, and if we know the
+	// purged set from the OSDs is at least *partly* stale we may as
+	// well wait for it to be fresh.
+	dout(20) << __func__ << "  we've already purged " << pbegin
+		 << "~" << (pend - pbegin) << dendl;
+	break;  // next pool
+      }
+      if (pbegin && pbegin > begin && pbegin < end) {
+	// the tail of [begin,end) is purged; shorten the range
+	end = pbegin;
+      }
+      to_prune.insert(begin, end - begin);
+      maybe_pruned += end - begin;
+      if (maybe_pruned >= max_prune) {
+	break;
+      }
+    }
+    if (!to_prune.empty()) {
+      // PGs may still be reporting things as purged that we have already
+      // pruned from removed_snaps_queue.
+      snap_interval_set_t actual;
+      auto r = osdmap.removed_snaps_queue.find(p.first);
+      if (r != osdmap.removed_snaps_queue.end()) {
+	actual.intersection_of(to_prune, r->second);
+      }
+      actually_pruned += actual.size();
+      dout(10) << __func__ << " pool " << p.first << " reports pruned " << to_prune
+	       << ", actual pruned " << actual << dendl;
+      if (!actual.empty()) {
+	pending_inc.new_purged_snaps[p.first].swap(actual);
+      }
+    }
+    if (actually_pruned >= max_prune) {
+      break;
+    }
+  }
+  dout(10) << __func__ << " actually pruned " << actually_pruned << dendl;
+  return !!actually_pruned;
+}
+
+bool OSDMonitor::update_pools_status()
+{
+  if (!mon.mgrstatmon()->is_readable())
+    return false;
+
+  bool ret = false;
+
+  auto& pools = osdmap.get_pools();
+  for (auto it = pools.begin(); it != pools.end(); ++it) {
+    const pool_stat_t *pstat = mon.mgrstatmon()->get_pool_stat(it->first);
+    if (!pstat)
+      continue;
+    const object_stat_sum_t& sum = pstat->stats.sum;
+    const pg_pool_t &pool = it->second;
+    const string& pool_name = osdmap.get_pool_name(it->first);
+
+    bool pool_is_full =
+      (pool.quota_max_bytes > 0 && (uint64_t)sum.num_bytes >= pool.quota_max_bytes) ||
+      (pool.quota_max_objects > 0 && (uint64_t)sum.num_objects >= pool.quota_max_objects);
+
+    if (pool.has_flag(pg_pool_t::FLAG_FULL_QUOTA)) {
+      if (pool_is_full)
+        continue;
+
+      mon.clog->info() << "pool '" << pool_name
+                       << "' no longer out of quota; removing NO_QUOTA flag";
+      // below we cancel FLAG_FULL too, we'll set it again in
+      // OSDMonitor::encode_pending if it still fails the osd-full checking.
+      clear_pool_flags(it->first,
+                       pg_pool_t::FLAG_FULL_QUOTA | pg_pool_t::FLAG_FULL);
+      ret = true;
+    } else {
+      if (!pool_is_full)
+	continue;
+
+      if (pool.quota_max_bytes > 0 &&
+          (uint64_t)sum.num_bytes >= pool.quota_max_bytes) {
+        mon.clog->warn() << "pool '" << pool_name << "' is full"
+                         << " (reached quota's max_bytes: "
+                         << byte_u_t(pool.quota_max_bytes) << ")";
+      }
+      if (pool.quota_max_objects > 0 &&
+		 (uint64_t)sum.num_objects >= pool.quota_max_objects) {
+        mon.clog->warn() << "pool '" << pool_name << "' is full"
+                         << " (reached quota's max_objects: "
+                         << pool.quota_max_objects << ")";
+      }
+      // set both FLAG_FULL_QUOTA and FLAG_FULL
+      // note that below we try to cancel FLAG_BACKFILLFULL/NEARFULL too
+      // since FLAG_FULL should always take precedence
+      set_pool_flags(it->first,
+                     pg_pool_t::FLAG_FULL_QUOTA | pg_pool_t::FLAG_FULL);
+      clear_pool_flags(it->first,
+                       pg_pool_t::FLAG_NEARFULL |
+                       pg_pool_t::FLAG_BACKFILLFULL);
+      ret = true;
+    }
+  }
+  return ret;
+}
+
+int OSDMonitor::prepare_new_pool(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MPoolOp>();
+  dout(10) << "prepare_new_pool from " << m->get_connection() << dendl;
+  MonSession *session = op->get_session();
+  if (!session)
+    return -EPERM;
+  string erasure_code_profile;
+  stringstream ss;
+  string rule_name;
+  bool bulk = false;
+  int ret = 0;
+  ret = prepare_new_pool(m->name, m->crush_rule, rule_name,
+			 0, 0, 0, 0, 0, 0, 0.0,
+			 erasure_code_profile,
+			 pg_pool_t::TYPE_REPLICATED, 0, FAST_READ_OFF, {}, bulk,
+			 &ss);
+
+  if (ret < 0) {
+    dout(10) << __func__ << " got " << ret << " " << ss.str() << dendl;
+  }
+  return ret;
+}
+
+int OSDMonitor::crush_rename_bucket(const string& srcname,
+				    const string& dstname,
+				    ostream *ss)
+{
+  int ret;
+  //
+  // Avoid creating a pending crush if it does not already exists and
+  // the rename would fail.
+  //
+  if (!_have_pending_crush()) {
+    ret = _get_stable_crush().can_rename_bucket(srcname,
+						dstname,
+						ss);
+    if (ret)
+      return ret;
+  }
+
+  CrushWrapper newcrush;
+  _get_pending_crush(newcrush);
+
+  ret = newcrush.rename_bucket(srcname,
+			       dstname,
+			       ss);
+  if (ret)
+    return ret;
+
+  pending_inc.crush.clear();
+  newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+  *ss << "renamed bucket " << srcname << " into " << dstname;	
+  return 0;
+}
+
+void OSDMonitor::check_legacy_ec_plugin(const string& plugin, const string& profile) const
+{
+  string replacement = "";
+
+  if (plugin == "jerasure_generic" || 
+      plugin == "jerasure_sse3" ||
+      plugin == "jerasure_sse4" ||
+      plugin == "jerasure_neon") {
+    replacement = "jerasure";
+  } else if (plugin == "shec_generic" ||
+	     plugin == "shec_sse3" ||
+	     plugin == "shec_sse4" ||
+             plugin == "shec_neon") {
+    replacement = "shec";
+  }
+
+  if (replacement != "") {
+    dout(0) << "WARNING: erasure coding profile " << profile << " uses plugin "
+	    << plugin << " that has been deprecated. Please use " 
+	    << replacement << " instead." << dendl;
+  }
+}
+
+int OSDMonitor::normalize_profile(const string& profilename,
+				  ErasureCodeProfile &profile,
+				  bool force,
+				  ostream *ss)
+{
+  ErasureCodeInterfaceRef erasure_code;
+  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+  ErasureCodeProfile::const_iterator plugin = profile.find("plugin");
+  check_legacy_ec_plugin(plugin->second, profilename);
+  int err = instance.factory(plugin->second,
+			     g_conf().get_val<std::string>("erasure_code_dir"),
+			     profile, &erasure_code, ss);
+  if (err) {
+    return err;
+  }
+
+  err = erasure_code->init(profile, ss);
+  if (err) {
+    return err;
+  }
+
+  auto it = profile.find("stripe_unit");
+  if (it != profile.end()) {
+    string err_str;
+    uint32_t stripe_unit = strict_iecstrtoll(it->second.c_str(), &err_str);
+    if (!err_str.empty()) {
+      *ss << "could not parse stripe_unit '" << it->second
+	  << "': " << err_str << std::endl;
+      return -EINVAL;
+    }
+    uint32_t data_chunks = erasure_code->get_data_chunk_count();
+    uint32_t chunk_size = erasure_code->get_chunk_size(stripe_unit * data_chunks);
+    if (chunk_size != stripe_unit) {
+      *ss << "stripe_unit " << stripe_unit << " does not match ec profile "
+	  << "alignment. Would be padded to " << chunk_size
+	  << std::endl;
+      return -EINVAL;
+    }
+    if ((stripe_unit % 4096) != 0 && !force) {
+      *ss << "stripe_unit should be a multiple of 4096 bytes for best performance."
+	  << "use --force to override this check" << std::endl;
+      return -EINVAL;
+    }
+  }
+  return 0;
+}
+
+int OSDMonitor::crush_rule_create_erasure(const string &name,
+					     const string &profile,
+					     int *rule,
+					     ostream *ss)
+{
+  int ruleid = osdmap.crush->get_rule_id(name);
+  if (ruleid != -ENOENT) {
+    *rule = osdmap.crush->get_rule_mask_ruleset(ruleid);
+    return -EEXIST;
+  }
+
+  CrushWrapper newcrush;
+  _get_pending_crush(newcrush);
+
+  ruleid = newcrush.get_rule_id(name);
+  if (ruleid != -ENOENT) {
+    *rule = newcrush.get_rule_mask_ruleset(ruleid);
+    return -EALREADY;
+  } else {
+    ErasureCodeInterfaceRef erasure_code;
+    int err = get_erasure_code(profile, &erasure_code, ss);
+    if (err) {
+      *ss << "failed to load plugin using profile " << profile << std::endl;
+      return err;
+    }
+
+    err = erasure_code->create_rule(name, newcrush, ss);
+    erasure_code.reset();
+    if (err < 0)
+      return err;
+    *rule = err;
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    return 0;
+  }
+}
+
+int OSDMonitor::get_erasure_code(const string &erasure_code_profile,
+				 ErasureCodeInterfaceRef *erasure_code,
+				 ostream *ss) const
+{
+  if (pending_inc.has_erasure_code_profile(erasure_code_profile))
+    return -EAGAIN;
+  ErasureCodeProfile profile =
+    osdmap.get_erasure_code_profile(erasure_code_profile);
+  ErasureCodeProfile::const_iterator plugin =
+    profile.find("plugin");
+  if (plugin == profile.end()) {
+    *ss << "cannot determine the erasure code plugin"
+	<< " because there is no 'plugin' entry in the erasure_code_profile "
+	<< profile << std::endl;
+    return -EINVAL;
+  }
+  check_legacy_ec_plugin(plugin->second, erasure_code_profile);
+  auto& instance = ErasureCodePluginRegistry::instance();
+  return instance.factory(plugin->second,
+			  g_conf().get_val<std::string>("erasure_code_dir"),
+			  profile, erasure_code, ss);
+}
+
+int OSDMonitor::check_cluster_features(uint64_t features,
+				       stringstream &ss)
+{
+  stringstream unsupported_ss;
+  int unsupported_count = 0;
+  if ((mon.get_quorum_con_features() & features) != features) {
+    unsupported_ss << "the monitor cluster";
+    ++unsupported_count;
+  }
+
+  set<int32_t> up_osds;
+  osdmap.get_up_osds(up_osds);
+  for (set<int32_t>::iterator it = up_osds.begin();
+       it != up_osds.end(); ++it) {
+    const osd_xinfo_t &xi = osdmap.get_xinfo(*it);
+    if ((xi.features & features) != features) {
+      if (unsupported_count > 0)
+	unsupported_ss << ", ";
+      unsupported_ss << "osd." << *it;
+      unsupported_count ++;
+    }
+  }
+
+  if (unsupported_count > 0) {
+    ss << "features " << features << " unsupported by: "
+       << unsupported_ss.str();
+    return -ENOTSUP;
+  }
+
+  // check pending osd state, too!
+  for (map<int32_t,osd_xinfo_t>::const_iterator p =
+	 pending_inc.new_xinfo.begin();
+       p != pending_inc.new_xinfo.end(); ++p) {
+    const osd_xinfo_t &xi = p->second;
+    if ((xi.features & features) != features) {
+      dout(10) << __func__ << " pending osd." << p->first
+	       << " features are insufficient; retry" << dendl;
+      return -EAGAIN;
+    }
+  }
+
+  return 0;
+}
+
+bool OSDMonitor::validate_crush_against_features(const CrushWrapper *newcrush,
+                                                 stringstream& ss)
+{
+  OSDMap::Incremental new_pending = pending_inc;
+  encode(*newcrush, new_pending.crush, mon.get_quorum_con_features());
+  OSDMap newmap;
+  newmap.deepish_copy_from(osdmap);
+  newmap.apply_incremental(new_pending);
+
+  // client compat
+  if (newmap.require_min_compat_client != ceph_release_t::unknown) {
+    auto mv = newmap.get_min_compat_client();
+    if (mv > newmap.require_min_compat_client) {
+      ss << "new crush map requires client version " << mv
+	 << " but require_min_compat_client is "
+	 << newmap.require_min_compat_client;
+      return false;
+    }
+  }
+
+  // osd compat
+  uint64_t features =
+    newmap.get_features(CEPH_ENTITY_TYPE_MON, NULL) |
+    newmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL);
+  stringstream features_ss;
+  int r = check_cluster_features(features, features_ss);
+  if (r) {
+    ss << "Could not change CRUSH: " << features_ss.str();
+    return false;
+  }
+
+  return true;
+}
+
+bool OSDMonitor::erasure_code_profile_in_use(
+  const mempool::osdmap::map<int64_t, pg_pool_t> &pools,
+  const string &profile,
+  ostream *ss)
+{
+  bool found = false;
+  for (map<int64_t, pg_pool_t>::const_iterator p = pools.begin();
+       p != pools.end();
+       ++p) {
+    if (p->second.erasure_code_profile == profile && p->second.is_erasure()) {
+      *ss << osdmap.pool_name[p->first] << " ";
+      found = true;
+    }
+  }
+  if (found) {
+    *ss << "pool(s) are using the erasure code profile '" << profile << "'";
+  }
+  return found;
+}
+
+int OSDMonitor::parse_erasure_code_profile(const vector<string> &erasure_code_profile,
+					   map<string,string> *erasure_code_profile_map,
+					   ostream *ss)
+{
+  int r = g_conf().with_val<string>("osd_pool_default_erasure_code_profile",
+				   get_json_str_map,
+				   *ss,
+				   erasure_code_profile_map,
+				   true);
+  if (r)
+    return r;
+  ceph_assert((*erasure_code_profile_map).count("plugin"));
+  string default_plugin = (*erasure_code_profile_map)["plugin"];
+  map<string,string> user_map;
+  for (vector<string>::const_iterator i = erasure_code_profile.begin();
+       i != erasure_code_profile.end();
+       ++i) {
+    size_t equal = i->find('=');
+    if (equal == string::npos) {
+      user_map[*i] = string();
+      (*erasure_code_profile_map)[*i] = string();
+    } else {
+      const string key = i->substr(0, equal);
+      equal++;
+      const string value = i->substr(equal);
+      if (key.find("ruleset-") == 0) {
+	*ss << "property '" << key << "' is no longer supported; try "
+	    << "'crush-" << key.substr(8) << "' instead";
+	return -EINVAL;
+      }
+      user_map[key] = value;
+      (*erasure_code_profile_map)[key] = value;
+    }
+  }
+
+  if (user_map.count("plugin") && user_map["plugin"] != default_plugin)
+    (*erasure_code_profile_map) = user_map;
+
+  return 0;
+}
+
+int OSDMonitor::prepare_pool_size(const unsigned pool_type,
+				  const string &erasure_code_profile,
+                                  uint8_t repl_size,
+				  unsigned *size, unsigned *min_size,
+				  ostream *ss)
+{
+  int err = 0;
+  bool set_min_size = false;
+  switch (pool_type) {
+  case pg_pool_t::TYPE_REPLICATED:
+    if (osdmap.stretch_mode_enabled) {
+      if (repl_size == 0)
+	repl_size = g_conf().get_val<uint64_t>("mon_stretch_pool_size");
+      if (repl_size != g_conf().get_val<uint64_t>("mon_stretch_pool_size")) {
+	*ss << "prepare_pool_size: we are in stretch mode but size "
+	   << repl_size << " does not match!";
+	return -EINVAL;
+      }
+      *min_size = g_conf().get_val<uint64_t>("mon_stretch_pool_min_size");
+      set_min_size = true;
+    }
+    if (repl_size == 0) {
+      repl_size = g_conf().get_val<uint64_t>("osd_pool_default_size");
+    }
+    *size = repl_size;
+    if (!set_min_size)
+      *min_size = g_conf().get_osd_pool_default_min_size(repl_size);
+    break;
+  case pg_pool_t::TYPE_ERASURE:
+    {
+      if (osdmap.stretch_mode_enabled) {
+	*ss << "prepare_pool_size: we are in stretch mode; cannot create EC pools!";
+	return -EINVAL;
+      }
+      ErasureCodeInterfaceRef erasure_code;
+      err = get_erasure_code(erasure_code_profile, &erasure_code, ss);
+      if (err == 0) {
+	*size = erasure_code->get_chunk_count();
+	*min_size =
+	  erasure_code->get_data_chunk_count() +
+	  std::min<int>(1, erasure_code->get_coding_chunk_count() - 1);
+	assert(*min_size <= *size);
+	assert(*min_size >= erasure_code->get_data_chunk_count());
+      }
+    }
+    break;
+  default:
+    *ss << "prepare_pool_size: " << pool_type << " is not a known pool type";
+    err = -EINVAL;
+    break;
+  }
+  return err;
+}
+
+int OSDMonitor::prepare_pool_stripe_width(const unsigned pool_type,
+					  const string &erasure_code_profile,
+					  uint32_t *stripe_width,
+					  ostream *ss)
+{
+  int err = 0;
+  switch (pool_type) {
+  case pg_pool_t::TYPE_REPLICATED:
+    // ignored
+    break;
+  case pg_pool_t::TYPE_ERASURE:
+    {
+      ErasureCodeProfile profile =
+	osdmap.get_erasure_code_profile(erasure_code_profile);
+      ErasureCodeInterfaceRef erasure_code;
+      err = get_erasure_code(erasure_code_profile, &erasure_code, ss);
+      if (err)
+	break;
+      uint32_t data_chunks = erasure_code->get_data_chunk_count();
+      uint32_t stripe_unit = g_conf().get_val<Option::size_t>("osd_pool_erasure_code_stripe_unit");
+      auto it = profile.find("stripe_unit");
+      if (it != profile.end()) {
+	string err_str;
+	stripe_unit = strict_iecstrtoll(it->second.c_str(), &err_str);
+	ceph_assert(err_str.empty());
+      }
+      *stripe_width = data_chunks *
+	erasure_code->get_chunk_size(stripe_unit * data_chunks);
+    }
+    break;
+  default:
+    *ss << "prepare_pool_stripe_width: "
+       << pool_type << " is not a known pool type";
+    err = -EINVAL;
+    break;
+  }
+  return err;
+}
+
+int OSDMonitor::get_replicated_stretch_crush_rule()
+{
+  /* we don't write down the stretch rule anywhere, so
+   * we have to guess it. How? Look at all the pools
+   * and count up how many times a given rule is used
+   * on stretch pools and then return the one with
+   * the most users!
+   */
+  map<int,int> rule_counts;
+  for (const auto& pooli : osdmap.pools) {
+    const pg_pool_t& p = pooli.second;
+    if (p.is_replicated() && p.is_stretch_pool()) {
+      if (!rule_counts.count(p.crush_rule)) {
+	rule_counts[p.crush_rule] = 1;
+      } else {
+	++rule_counts[p.crush_rule];
+      }
+    }
+  }
+
+  if (rule_counts.empty()) {
+    return -ENOENT;
+  }
+
+  int most_used_count = 0;
+  int most_used_rule = -1;
+  for (auto i : rule_counts) {
+    if (i.second > most_used_count) {
+      most_used_rule = i.first;
+      most_used_count = i.second;
+    }
+  }
+  ceph_assert(most_used_count > 0);
+  ceph_assert(most_used_rule >= 0);
+  return most_used_rule;
+}
+
+int OSDMonitor::prepare_pool_crush_rule(const unsigned pool_type,
+					const string &erasure_code_profile,
+					const string &rule_name,
+					int *crush_rule,
+					ostream *ss)
+{
+
+  if (*crush_rule < 0) {
+    switch (pool_type) {
+    case pg_pool_t::TYPE_REPLICATED:
+      {
+	if (rule_name == "") {
+	  if (osdmap.stretch_mode_enabled) {
+	    *crush_rule = get_replicated_stretch_crush_rule();
+	  } else {
+	    // Use default rule
+	    *crush_rule = osdmap.crush->get_osd_pool_default_crush_replicated_ruleset(cct);
+	  }
+	  if (*crush_rule < 0) {
+	    // Errors may happen e.g. if no valid rule is available
+	    *ss << "No suitable CRUSH rule exists, check "
+                << "'osd pool default crush *' config options";
+	    return -ENOENT;
+	  }
+	} else {
+	  return get_crush_rule(rule_name, crush_rule, ss);
+	}
+      }
+      break;
+    case pg_pool_t::TYPE_ERASURE:
+      {
+	int err = crush_rule_create_erasure(rule_name,
+					       erasure_code_profile,
+					       crush_rule, ss);
+	switch (err) {
+	case -EALREADY:
+	  dout(20) << "prepare_pool_crush_rule: rule "
+		   << rule_name << " try again" << dendl;
+	  // fall through
+	case 0:
+	  // need to wait for the crush rule to be proposed before proceeding
+	  err = -EAGAIN;
+	  break;
+	case -EEXIST:
+	  err = 0;
+	  break;
+ 	}
+	return err;
+      }
+      break;
+    default:
+      *ss << "prepare_pool_crush_rule: " << pool_type
+	 << " is not a known pool type";
+      return -EINVAL;
+    }
+  } else {
+    if (!osdmap.crush->ruleset_exists(*crush_rule)) {
+      *ss << "CRUSH rule " << *crush_rule << " not found";
+      return -ENOENT;
+    }
+  }
+
+  return 0;
+}
+
+int OSDMonitor::get_crush_rule(const string &rule_name,
+			       int *crush_rule,
+			       ostream *ss)
+{
+  int ret;
+  ret = osdmap.crush->get_rule_id(rule_name);
+  if (ret != -ENOENT) {
+    // found it, use it
+    *crush_rule = ret;
+  } else {
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    ret = newcrush.get_rule_id(rule_name);
+    if (ret != -ENOENT) {
+      // found it, wait for it to be proposed
+      dout(20) << __func__ << ": rule " << rule_name
+	       << " try again" << dendl;
+      return -EAGAIN;
+    } else {
+      // Cannot find it , return error
+      *ss << "specified rule " << rule_name << " doesn't exist";
+      return ret;
+    }
+  }
+  return 0;
+}
+
+int OSDMonitor::check_pg_num(int64_t pool, int pg_num, int size, ostream *ss)
+{
+  auto max_pgs_per_osd = g_conf().get_val<uint64_t>("mon_max_pg_per_osd");
+  auto num_osds = std::max(osdmap.get_num_in_osds(), 3u);   // assume min cluster size 3
+  auto max_pgs = max_pgs_per_osd * num_osds;
+  uint64_t projected = 0;
+  if (pool < 0) {
+    projected += pg_num * size;
+  }
+  for (const auto& i : osdmap.get_pools()) {
+    if (i.first == pool) {
+      projected += pg_num * size;
+    } else {
+      projected += i.second.get_pg_num_target() * i.second.get_size();
+    }
+  }
+  if (projected > max_pgs) {
+    if (pool >= 0) {
+      *ss << "pool id " << pool;
+    }
+    *ss << " pg_num " << pg_num << " size " << size
+	<< " would mean " << projected
+	<< " total pgs, which exceeds max " << max_pgs
+	<< " (mon_max_pg_per_osd " << max_pgs_per_osd
+	<< " * num_in_osds " << num_osds << ")";
+    return -ERANGE;
+  }
+  return 0;
+}
+
+/**
+ * @param name The name of the new pool
+ * @param crush_rule The crush rule to use. If <0, will use the system default
+ * @param crush_rule_name The crush rule to use, if crush_rulset <0
+ * @param pg_num The pg_num to use. If set to 0, will use the system default
+ * @param pgp_num The pgp_num to use. If set to 0, will use the system default
+ * @param pg_num_min min pg_num
+ * @param pg_num_max max pg_num
+ * @param repl_size Replication factor, or 0 for default
+ * @param erasure_code_profile The profile name in OSDMap to be used for erasure code
+ * @param pool_type TYPE_ERASURE, or TYPE_REP
+ * @param expected_num_objects expected number of objects on the pool
+ * @param fast_read fast read type. 
+ * @param ss human readable error message, if any.
+ *
+ * @return 0 on success, negative errno on failure.
+ */
+int OSDMonitor::prepare_new_pool(string& name,
+				 int crush_rule,
+				 const string &crush_rule_name,
+                                 unsigned pg_num, unsigned pgp_num,
+				 unsigned pg_num_min,
+				 unsigned pg_num_max,
+                                 const uint64_t repl_size,
+				 const uint64_t target_size_bytes,
+				 const float target_size_ratio,
+				 const string &erasure_code_profile,
+                                 const unsigned pool_type,
+                                 const uint64_t expected_num_objects,
+                                 FastReadType fast_read,
+				 const string& pg_autoscale_mode,
+				 bool bulk,
+				 ostream *ss)
+{
+  if (name.length() == 0)
+    return -EINVAL;
+  if (pg_num == 0)
+    pg_num = g_conf().get_val<uint64_t>("osd_pool_default_pg_num");
+  if (pgp_num == 0)
+    pgp_num = g_conf().get_val<uint64_t>("osd_pool_default_pgp_num");
+  if (!pgp_num)
+    pgp_num = pg_num;
+  if (pg_num > g_conf().get_val<uint64_t>("mon_max_pool_pg_num")) {
+    *ss << "'pg_num' must be greater than 0 and less than or equal to "
+        << g_conf().get_val<uint64_t>("mon_max_pool_pg_num")
+        << " (you may adjust 'mon max pool pg num' for higher values)";
+    return -ERANGE;
+  }
+  if (pgp_num > pg_num) {
+    *ss << "'pgp_num' must be greater than 0 and lower or equal than 'pg_num'"
+        << ", which in this case is " << pg_num;
+    return -ERANGE;
+  }
+  if (pool_type == pg_pool_t::TYPE_REPLICATED && fast_read == FAST_READ_ON) {
+    *ss << "'fast_read' can only apply to erasure coding pool";
+    return -EINVAL;
+  }
+  int r;
+  r = prepare_pool_crush_rule(pool_type, erasure_code_profile,
+				 crush_rule_name, &crush_rule, ss);
+  if (r) {
+    dout(10) << "prepare_pool_crush_rule returns " << r << dendl;
+    return r;
+  }
+  if (g_conf()->mon_osd_crush_smoke_test) {
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+    ostringstream err;
+    CrushTester tester(newcrush, err);
+    tester.set_min_x(0);
+    tester.set_max_x(50);
+    tester.set_rule(crush_rule);
+    auto start = ceph::coarse_mono_clock::now();
+    r = tester.test_with_fork(g_conf()->mon_lease);
+    auto duration = ceph::coarse_mono_clock::now() - start;
+    if (r < 0) {
+      dout(10) << "tester.test_with_fork returns " << r
+	       << ": " << err.str() << dendl;
+      *ss << "crush test failed with " << r << ": " << err.str();
+      return r;
+    }
+    dout(10) << __func__ << " crush smoke test duration: "
+             << duration << dendl;
+  }
+  unsigned size, min_size;
+  r = prepare_pool_size(pool_type, erasure_code_profile, repl_size,
+                        &size, &min_size, ss);
+  if (r) {
+    dout(10) << "prepare_pool_size returns " << r << dendl;
+    return r;
+  }
+  r = check_pg_num(-1, pg_num, size, ss);
+  if (r) {
+    dout(10) << "check_pg_num returns " << r << dendl;
+    return r;
+  }
+
+  if (!osdmap.crush->check_crush_rule(crush_rule, pool_type, size, *ss)) {
+    return -EINVAL;
+  }
+
+  uint32_t stripe_width = 0;
+  r = prepare_pool_stripe_width(pool_type, erasure_code_profile, &stripe_width, ss);
+  if (r) {
+    dout(10) << "prepare_pool_stripe_width returns " << r << dendl;
+    return r;
+  }
+  
+  bool fread = false;
+  if (pool_type == pg_pool_t::TYPE_ERASURE) {
+    switch (fast_read) {
+      case FAST_READ_OFF:
+        fread = false;
+        break;
+      case FAST_READ_ON:
+        fread = true;
+        break;
+      case FAST_READ_DEFAULT:
+        fread = g_conf()->osd_pool_default_ec_fast_read;
+        break;
+      default:
+        *ss << "invalid fast_read setting: " << fast_read;
+        return -EINVAL;
+    }
+  }
+
+  for (map<int64_t,string>::iterator p = pending_inc.new_pool_names.begin();
+       p != pending_inc.new_pool_names.end();
+       ++p) {
+    if (p->second == name)
+      return 0;
+  }
+
+  if (-1 == pending_inc.new_pool_max)
+    pending_inc.new_pool_max = osdmap.pool_max;
+  int64_t pool = ++pending_inc.new_pool_max;
+  pg_pool_t empty;
+  pg_pool_t *pi = pending_inc.get_new_pool(pool, &empty);
+  pi->create_time = ceph_clock_now();
+  pi->type = pool_type;
+  pi->fast_read = fread; 
+  pi->flags = g_conf()->osd_pool_default_flags;
+  if (bulk) {
+    pi->set_flag(pg_pool_t::FLAG_BULK);
+  } else if (g_conf()->osd_pool_default_flag_bulk) {
+      pi->set_flag(pg_pool_t::FLAG_BULK);
+  }
+  if (g_conf()->osd_pool_default_flag_hashpspool)
+    pi->set_flag(pg_pool_t::FLAG_HASHPSPOOL);
+  if (g_conf()->osd_pool_default_flag_nodelete)
+    pi->set_flag(pg_pool_t::FLAG_NODELETE);
+  if (g_conf()->osd_pool_default_flag_nopgchange)
+    pi->set_flag(pg_pool_t::FLAG_NOPGCHANGE);
+  if (g_conf()->osd_pool_default_flag_nosizechange)
+    pi->set_flag(pg_pool_t::FLAG_NOSIZECHANGE);
+  pi->set_flag(pg_pool_t::FLAG_CREATING);
+  if (g_conf()->osd_pool_use_gmt_hitset)
+    pi->use_gmt_hitset = true;
+  else
+    pi->use_gmt_hitset = false;
+
+  pi->size = size;
+  pi->min_size = min_size;
+  pi->crush_rule = crush_rule;
+  pi->expected_num_objects = expected_num_objects;
+  pi->object_hash = CEPH_STR_HASH_RJENKINS;
+  if (osdmap.stretch_mode_enabled) {
+    pi->peering_crush_bucket_count = osdmap.stretch_bucket_count;
+    pi->peering_crush_bucket_target = osdmap.stretch_bucket_count;
+    pi->peering_crush_bucket_barrier = osdmap.stretch_mode_bucket;
+    pi->peering_crush_mandatory_member = CRUSH_ITEM_NONE;
+    if (osdmap.degraded_stretch_mode) {
+      pi->peering_crush_bucket_count = osdmap.degraded_stretch_mode;
+      pi->peering_crush_bucket_target = osdmap.degraded_stretch_mode;
+      // pi->peering_crush_bucket_mandatory_member = CRUSH_ITEM_NONE;
+      // TODO: drat, we don't record this ^ anywhere, though given that it
+      // necessarily won't exist elsewhere it likely doesn't matter
+      pi->min_size = pi->min_size / 2;
+      pi->size = pi->size / 2; // only support 2 zones now
+    }
+  }
+
+  if (auto m = pg_pool_t::get_pg_autoscale_mode_by_name(
+        g_conf().get_val<string>("osd_pool_default_pg_autoscale_mode"));
+      m != pg_pool_t::pg_autoscale_mode_t::UNKNOWN) {
+    pi->pg_autoscale_mode = m;
+  } else {
+    pi->pg_autoscale_mode = pg_pool_t::pg_autoscale_mode_t::OFF;
+  }
+  auto max = g_conf().get_val<int64_t>("mon_osd_max_initial_pgs");
+  pi->set_pg_num(
+    max > 0 ? std::min<uint64_t>(pg_num, std::max<int64_t>(1, max))
+    : pg_num);
+  pi->set_pg_num_pending(pi->get_pg_num());
+  pi->set_pg_num_target(pg_num);
+  pi->set_pgp_num(pi->get_pg_num());
+  pi->set_pgp_num_target(pgp_num);
+  if (osdmap.require_osd_release >= ceph_release_t::nautilus &&
+      pg_num_min) {
+    pi->opts.set(pool_opts_t::PG_NUM_MIN, static_cast<int64_t>(pg_num_min));
+  }
+  if (osdmap.require_osd_release >= ceph_release_t::pacific &&
+      pg_num_max) {
+    pi->opts.set(pool_opts_t::PG_NUM_MAX, static_cast<int64_t>(pg_num_max));
+  }
+  if (auto m = pg_pool_t::get_pg_autoscale_mode_by_name(
+	pg_autoscale_mode); m != pg_pool_t::pg_autoscale_mode_t::UNKNOWN) {
+    pi->pg_autoscale_mode = m;
+  }
+
+  pi->last_change = pending_inc.epoch;
+  pi->auid = 0;
+
+  if (pool_type == pg_pool_t::TYPE_ERASURE) {
+      pi->erasure_code_profile = erasure_code_profile;
+  } else {
+      pi->erasure_code_profile = "";
+  }
+  pi->stripe_width = stripe_width;
+
+  if (osdmap.require_osd_release >= ceph_release_t::nautilus &&
+      target_size_bytes) {
+    // only store for nautilus+ because TARGET_SIZE_BYTES may be
+    // larger than int32_t max.
+    pi->opts.set(pool_opts_t::TARGET_SIZE_BYTES, static_cast<int64_t>(target_size_bytes));
+  }
+  if (target_size_ratio > 0.0 &&
+      osdmap.require_osd_release >= ceph_release_t::nautilus) {
+    // only store for nautilus+, just to be consistent and tidy.
+    pi->opts.set(pool_opts_t::TARGET_SIZE_RATIO, target_size_ratio);
+  }
+
+  pi->cache_target_dirty_ratio_micro =
+    g_conf()->osd_pool_default_cache_target_dirty_ratio * 1000000;
+  pi->cache_target_dirty_high_ratio_micro =
+    g_conf()->osd_pool_default_cache_target_dirty_high_ratio * 1000000;
+  pi->cache_target_full_ratio_micro =
+    g_conf()->osd_pool_default_cache_target_full_ratio * 1000000;
+  pi->cache_min_flush_age = g_conf()->osd_pool_default_cache_min_flush_age;
+  pi->cache_min_evict_age = g_conf()->osd_pool_default_cache_min_evict_age;
+
+  pending_inc.new_pool_names[pool] = name;
+  return 0;
+}
+
+bool OSDMonitor::prepare_set_flag(MonOpRequestRef op, int flag)
+{
+  op->mark_osdmon_event(__func__);
+  ostringstream ss;
+  if (pending_inc.new_flags < 0)
+    pending_inc.new_flags = osdmap.get_flags();
+  pending_inc.new_flags |= flag;
+  ss << OSDMap::get_flag_string(flag) << " is set";
+  wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, ss.str(),
+						    get_last_committed() + 1));
+  return true;
+}
+
+bool OSDMonitor::prepare_unset_flag(MonOpRequestRef op, int flag)
+{
+  op->mark_osdmon_event(__func__);
+  ostringstream ss;
+  if (pending_inc.new_flags < 0)
+    pending_inc.new_flags = osdmap.get_flags();
+  pending_inc.new_flags &= ~flag;
+  ss << OSDMap::get_flag_string(flag) << " is unset";
+  wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, ss.str(),
+						    get_last_committed() + 1));
+  return true;
+}
+
+int OSDMonitor::prepare_command_pool_set(const cmdmap_t& cmdmap,
+                                         stringstream& ss)
+{
+  string poolstr;
+  cmd_getval(cmdmap, "pool", poolstr);
+  int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+  if (pool < 0) {
+    ss << "unrecognized pool '" << poolstr << "'";
+    return -ENOENT;
+  }
+  string var;
+  cmd_getval(cmdmap, "var", var);
+
+  pg_pool_t p = *osdmap.get_pg_pool(pool);
+  if (pending_inc.new_pools.count(pool))
+    p = pending_inc.new_pools[pool];
+
+  // accept val as a json string in the normal case (current
+  // generation monitor).  parse out int or float values from the
+  // string as needed.  however, if it is not a string, try to pull
+  // out an int, in case an older monitor with an older json schema is
+  // forwarding a request.
+  string val;
+  string interr, floaterr;
+  int64_t n = 0;
+  double f = 0;
+  int64_t uf = 0;  // micro-f
+  cmd_getval(cmdmap, "val", val);
+
+  auto si_options = {
+    "target_max_objects"
+  };
+  auto iec_options = {
+    "target_max_bytes",
+    "target_size_bytes",
+    "compression_max_blob_size",
+    "compression_min_blob_size",
+    "csum_max_block",
+    "csum_min_block",
+  };
+  if (count(begin(si_options), end(si_options), var)) {
+    n = strict_si_cast<int64_t>(val.c_str(), &interr);
+  } else if (count(begin(iec_options), end(iec_options), var)) {
+    n = strict_iec_cast<int64_t>(val.c_str(), &interr);
+  } else {
+    // parse string as both int and float; different fields use different types.
+    n = strict_strtoll(val.c_str(), 10, &interr);
+    f = strict_strtod(val.c_str(), &floaterr);
+    uf = llrintl(f * (double)1000000.0);
+  }
+
+  if (!p.is_tier() &&
+      (var == "hit_set_type" || var == "hit_set_period" ||
+       var == "hit_set_count" || var == "hit_set_fpp" ||
+       var == "target_max_objects" || var == "target_max_bytes" ||
+       var == "cache_target_full_ratio" || var == "cache_target_dirty_ratio" ||
+       var == "cache_target_dirty_high_ratio" || var == "use_gmt_hitset" ||
+       var == "cache_min_flush_age" || var == "cache_min_evict_age" ||
+       var == "hit_set_grade_decay_rate" || var == "hit_set_search_last_n" ||
+       var == "min_read_recency_for_promote" || var == "min_write_recency_for_promote")) {
+    return -EACCES;
+  }
+
+  if (var == "size") {
+    if (p.has_flag(pg_pool_t::FLAG_NOSIZECHANGE)) {
+      ss << "pool size change is disabled; you must unset nosizechange flag for the pool first";
+      return -EPERM;
+    }
+    if (p.type == pg_pool_t::TYPE_ERASURE) {
+      ss << "can not change the size of an erasure-coded pool";
+      return -ENOTSUP;
+    }
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    if (n <= 0 || n > 10) {
+      ss << "pool size must be between 1 and 10";
+      return -EINVAL;
+    }
+    if (n == 1) {
+      if (!g_conf().get_val<bool>("mon_allow_pool_size_one")) {
+	ss << "configuring pool size as 1 is disabled by default.";
+	return -EPERM;
+      }
+      bool sure = false;
+      cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+      if (!sure) { ss << "WARNING: setting pool size 1 could lead to data loss "
+	"without recovery. If you are *ABSOLUTELY CERTAIN* that is what you want, "
+	  "pass the flag --yes-i-really-mean-it.";
+	return -EPERM;
+      }
+    }
+    if (!osdmap.crush->check_crush_rule(p.get_crush_rule(), p.type, n, ss)) {
+      return -EINVAL;
+    }
+    int r = check_pg_num(pool, p.get_pg_num(), n, &ss);
+    if (r < 0) {
+      return r;
+    }
+    p.size = n;
+    p.min_size = g_conf().get_osd_pool_default_min_size(p.size);
+  } else if (var == "min_size") {
+    if (p.has_flag(pg_pool_t::FLAG_NOSIZECHANGE)) {
+      ss << "pool min size change is disabled; you must unset nosizechange flag for the pool first";
+      return -EPERM;
+    }
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+
+    if (p.type != pg_pool_t::TYPE_ERASURE) {
+      if (n < 1 || n > p.size) {
+	ss << "pool min_size must be between 1 and size, which is set to " << (int)p.size;
+	return -EINVAL;
+      }
+    } else {
+       ErasureCodeInterfaceRef erasure_code;
+       int k;
+       stringstream tmp;
+       int err = get_erasure_code(p.erasure_code_profile, &erasure_code, &tmp);
+       if (err == 0) {
+	 k = erasure_code->get_data_chunk_count();
+       } else {
+	 ss << __func__ << " get_erasure_code failed: " << tmp.str();
+	 return err;
+       }
+
+       if (n < k || n > p.size) {
+	 ss << "pool min_size must be between " << k << " and size, which is set to " << (int)p.size;
+	 return -EINVAL;
+       }
+    }
+    p.min_size = n;
+  } else if (var == "pg_num_actual") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    if (n == (int)p.get_pg_num()) {
+      return 0;
+    }
+    if (static_cast<uint64_t>(n) > g_conf().get_val<uint64_t>("mon_max_pool_pg_num")) {
+      ss << "'pg_num' must be greater than 0 and less than or equal to "
+         << g_conf().get_val<uint64_t>("mon_max_pool_pg_num")
+         << " (you may adjust 'mon max pool pg num' for higher values)";
+      return -ERANGE;
+    }
+    if (p.has_flag(pg_pool_t::FLAG_CREATING)) {
+      ss << "cannot adjust pg_num while initial PGs are being created";
+      return -EBUSY;
+    }
+    if (n > (int)p.get_pg_num()) {
+      if (p.get_pg_num() != p.get_pg_num_pending()) {
+	// force pre-nautilus clients to resend their ops, since they
+	// don't understand pg_num_pending changes form a new interval
+	p.last_force_op_resend_prenautilus = pending_inc.epoch;
+      }
+      p.set_pg_num(n);
+    } else {
+      if (osdmap.require_osd_release < ceph_release_t::nautilus) {
+	ss << "nautilus OSDs are required to adjust pg_num_pending";
+	return -EPERM;
+      }
+      if (n < (int)p.get_pgp_num()) {
+	ss << "specified pg_num " << n << " < pgp_num " << p.get_pgp_num();
+	return -EINVAL;
+      }
+      if (n < (int)p.get_pg_num() - 1) {
+	ss << "specified pg_num " << n << " < pg_num (" << p.get_pg_num()
+	   << ") - 1; only single pg decrease is currently supported";
+	return -EINVAL;
+      }
+      p.set_pg_num_pending(n);
+      // force pre-nautilus clients to resend their ops, since they
+      // don't understand pg_num_pending changes form a new interval
+      p.last_force_op_resend_prenautilus = pending_inc.epoch;
+    }
+    // force pre-luminous clients to resend their ops, since they
+    // don't understand that split PGs now form a new interval.
+    p.last_force_op_resend_preluminous = pending_inc.epoch;
+  } else if (var == "pg_num") {
+    if (p.has_flag(pg_pool_t::FLAG_NOPGCHANGE)) {
+      ss << "pool pg_num change is disabled; you must unset nopgchange flag for the pool first";
+      return -EPERM;
+    }
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    if (n == (int)p.get_pg_num_target()) {
+      return 0;
+    }
+    if (n <= 0 || static_cast<uint64_t>(n) >
+                  g_conf().get_val<uint64_t>("mon_max_pool_pg_num")) {
+      ss << "'pg_num' must be greater than 0 and less than or equal to "
+         << g_conf().get_val<uint64_t>("mon_max_pool_pg_num")
+         << " (you may adjust 'mon max pool pg num' for higher values)";
+      return -ERANGE;
+    }
+    if (n > (int)p.get_pg_num_target()) {
+      int r = check_pg_num(pool, n, p.get_size(), &ss);
+      if (r) {
+	return r;
+      }
+      bool force = false;
+      cmd_getval(cmdmap, "yes_i_really_mean_it", force);
+      if (p.cache_mode != pg_pool_t::CACHEMODE_NONE && !force) {
+	ss << "splits in cache pools must be followed by scrubs and leave sufficient free space to avoid overfilling.  use --yes-i-really-mean-it to force.";
+	return -EPERM;
+      }
+    } else {
+      if (osdmap.require_osd_release < ceph_release_t::nautilus) {
+	ss << "nautilus OSDs are required to decrease pg_num";
+	return -EPERM;
+      }
+    }
+    int64_t pg_min = 0, pg_max = 0;
+    p.opts.get(pool_opts_t::PG_NUM_MIN, &pg_min);
+    p.opts.get(pool_opts_t::PG_NUM_MAX, &pg_max);
+    if (pg_min && n < pg_min) {
+      ss << "specified pg_num " << n
+	 << " < pg_num_min " << pg_min;
+      return -EINVAL;
+    }
+    if (pg_max && n > pg_max) {
+      ss << "specified pg_num " << n
+	 << " < pg_num_max " << pg_max;
+      return -EINVAL;
+    }
+    if (osdmap.require_osd_release < ceph_release_t::nautilus) {
+      // pre-nautilus osdmap format; increase pg_num directly
+      assert(n > (int)p.get_pg_num());
+      // force pre-nautilus clients to resend their ops, since they
+      // don't understand pg_num_target changes form a new interval
+      p.last_force_op_resend_prenautilus = pending_inc.epoch;
+      // force pre-luminous clients to resend their ops, since they
+      // don't understand that split PGs now form a new interval.
+      p.last_force_op_resend_preluminous = pending_inc.epoch;
+      p.set_pg_num(n);
+    } else {
+      // set targets; mgr will adjust pg_num_actual and pgp_num later.
+      // make pgp_num track pg_num if it already matches.  if it is set
+      // differently, leave it different and let the user control it
+      // manually.
+      if (p.get_pg_num_target() == p.get_pgp_num_target()) {
+	p.set_pgp_num_target(n);
+      }
+      p.set_pg_num_target(n);
+    }
+  } else if (var == "pgp_num_actual") {
+    if (p.has_flag(pg_pool_t::FLAG_NOPGCHANGE)) {
+      ss << "pool pgp_num change is disabled; you must unset nopgchange flag for the pool first";
+      return -EPERM;
+    }
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    if (n <= 0) {
+      ss << "specified pgp_num must > 0, but you set to " << n;
+      return -EINVAL;
+    }
+    if (n > (int)p.get_pg_num()) {
+      ss << "specified pgp_num " << n << " > pg_num " << p.get_pg_num();
+      return -EINVAL;
+    }
+    if (n > (int)p.get_pg_num_pending()) {
+      ss << "specified pgp_num " << n
+	 << " > pg_num_pending " << p.get_pg_num_pending();
+      return -EINVAL;
+    }
+    p.set_pgp_num(n);
+  } else if (var == "pgp_num") {
+    if (p.has_flag(pg_pool_t::FLAG_NOPGCHANGE)) {
+      ss << "pool pgp_num change is disabled; you must unset nopgchange flag for the pool first";
+      return -EPERM;
+    }
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    if (n <= 0) {
+      ss << "specified pgp_num must > 0, but you set to " << n;
+      return -EINVAL;
+    }
+    if (n > (int)p.get_pg_num_target()) {
+      ss << "specified pgp_num " << n << " > pg_num " << p.get_pg_num_target();
+      return -EINVAL;
+    }
+    if (osdmap.require_osd_release < ceph_release_t::nautilus) {
+      // pre-nautilus osdmap format; increase pgp_num directly
+      p.set_pgp_num(n);
+    } else {
+      p.set_pgp_num_target(n);
+    }
+  } else if (var == "pg_autoscale_mode") {
+    auto m = pg_pool_t::get_pg_autoscale_mode_by_name(val);
+    if (m == pg_pool_t::pg_autoscale_mode_t::UNKNOWN) {
+      ss << "specified invalid mode " << val;
+      return -EINVAL;
+    }
+    if (osdmap.require_osd_release < ceph_release_t::nautilus) {
+      ss << "must set require_osd_release to nautilus or later before setting pg_autoscale_mode";
+      return -EINVAL;
+    }
+    p.pg_autoscale_mode = m;
+  } else if (var == "crush_rule") {
+    int id = osdmap.crush->get_rule_id(val);
+    if (id == -ENOENT) {
+      ss << "crush rule " << val << " does not exist";
+      return -ENOENT;
+    }
+    if (id < 0) {
+      ss << cpp_strerror(id);
+      return -ENOENT;
+    }
+    if (!osdmap.crush->check_crush_rule(id, p.get_type(), p.get_size(), ss)) {
+      return -EINVAL;
+    }
+    p.crush_rule = id;
+  } else if (var == "nodelete" || var == "nopgchange" ||
+	     var == "nosizechange" || var == "write_fadvise_dontneed" ||
+	     var == "noscrub" || var == "nodeep-scrub" || var == "bulk") {
+    uint64_t flag = pg_pool_t::get_flag_by_name(var);
+    // make sure we only compare against 'n' if we didn't receive a string
+    if (val == "true" || (interr.empty() && n == 1)) {
+      p.set_flag(flag);
+    } else if (val == "false" || (interr.empty() && n == 0)) {
+      p.unset_flag(flag);
+    } else {
+      ss << "expecting value 'true', 'false', '0', or '1'";
+      return -EINVAL;
+    }
+  } else if (var == "hashpspool") {
+    uint64_t flag = pg_pool_t::get_flag_by_name(var);
+    bool force = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", force);
+
+    if (!force) {
+      ss << "are you SURE?  this will remap all placement groups in this pool,"
+	    " this triggers large data movement,"
+	    " pass --yes-i-really-mean-it if you really do.";
+      return -EPERM;
+    }
+    // make sure we only compare against 'n' if we didn't receive a string
+    if (val == "true" || (interr.empty() && n == 1)) {
+      p.set_flag(flag);
+    } else if (val == "false" || (interr.empty() && n == 0)) {
+      p.unset_flag(flag);
+    } else {
+      ss << "expecting value 'true', 'false', '0', or '1'";
+      return -EINVAL;
+    }
+  } else if (var == "hit_set_type") {
+    if (val == "none")
+      p.hit_set_params = HitSet::Params();
+    else {
+      int err = check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL, ss);
+      if (err)
+	return err;
+      if (val == "bloom") {
+	BloomHitSet::Params *bsp = new BloomHitSet::Params;
+	bsp->set_fpp(g_conf().get_val<double>("osd_pool_default_hit_set_bloom_fpp"));
+	p.hit_set_params = HitSet::Params(bsp);
+      } else if (val == "explicit_hash")
+	p.hit_set_params = HitSet::Params(new ExplicitHashHitSet::Params);
+      else if (val == "explicit_object")
+	p.hit_set_params = HitSet::Params(new ExplicitObjectHitSet::Params);
+      else {
+	ss << "unrecognized hit_set type '" << val << "'";
+	return -EINVAL;
+      }
+    }
+  } else if (var == "hit_set_period") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    } else if (n < 0) {
+      ss << "hit_set_period should be non-negative";
+      return -EINVAL;
+    }
+    p.hit_set_period = n;
+  } else if (var == "hit_set_count") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    } else if (n < 0) {
+      ss << "hit_set_count should be non-negative";
+      return -EINVAL;
+    }
+    p.hit_set_count = n;
+  } else if (var == "hit_set_fpp") {
+    if (floaterr.length()) {
+      ss << "error parsing floating point value '" << val << "': " << floaterr;
+      return -EINVAL;
+    } else if (f < 0 || f > 1.0) {
+      ss << "hit_set_fpp should be in the range 0..1";
+      return -EINVAL;
+    }
+    if (p.hit_set_params.get_type() != HitSet::TYPE_BLOOM) {
+      ss << "hit set is not of type Bloom; invalid to set a false positive rate!";
+      return -EINVAL;
+    }
+    BloomHitSet::Params *bloomp = static_cast<BloomHitSet::Params*>(p.hit_set_params.impl.get());
+    bloomp->set_fpp(f);
+  } else if (var == "use_gmt_hitset") {
+    if (val == "true" || (interr.empty() && n == 1)) {
+      p.use_gmt_hitset = true;
+    } else {
+      ss << "expecting value 'true' or '1'";
+      return -EINVAL;
+    }
+  } else if (var == "allow_ec_overwrites") {
+    if (!p.is_erasure()) {
+      ss << "ec overwrites can only be enabled for an erasure coded pool";
+      return -EINVAL;
+    }
+    stringstream err;
+    if (!g_conf()->mon_debug_no_require_bluestore_for_ec_overwrites &&
+	!is_pool_currently_all_bluestore(pool, p, &err)) {
+      ss << "pool must only be stored on bluestore for scrubbing to work: " << err.str();
+      return -EINVAL;
+    }
+    if (val == "true" || (interr.empty() && n == 1)) {
+	p.flags |= pg_pool_t::FLAG_EC_OVERWRITES;
+    } else if (val == "false" || (interr.empty() && n == 0)) {
+      ss << "ec overwrites cannot be disabled once enabled";
+      return -EINVAL;
+    } else {
+      ss << "expecting value 'true', 'false', '0', or '1'";
+      return -EINVAL;
+    }
+  } else if (var == "target_max_objects") {
+    if (interr.length()) {
+      ss << "error parsing int '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.target_max_objects = n;
+  } else if (var == "target_max_bytes") {
+    if (interr.length()) {
+      ss << "error parsing int '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.target_max_bytes = n;
+  } else if (var == "cache_target_dirty_ratio") {
+    if (floaterr.length()) {
+      ss << "error parsing float '" << val << "': " << floaterr;
+      return -EINVAL;
+    }
+    if (f < 0 || f > 1.0) {
+      ss << "value must be in the range 0..1";
+      return -ERANGE;
+    }
+    p.cache_target_dirty_ratio_micro = uf;
+  } else if (var == "cache_target_dirty_high_ratio") {
+    if (floaterr.length()) {
+      ss << "error parsing float '" << val << "': " << floaterr;
+      return -EINVAL;
+    }
+    if (f < 0 || f > 1.0) {
+      ss << "value must be in the range 0..1";
+      return -ERANGE;
+    }
+    p.cache_target_dirty_high_ratio_micro = uf;
+  } else if (var == "cache_target_full_ratio") {
+    if (floaterr.length()) {
+      ss << "error parsing float '" << val << "': " << floaterr;
+      return -EINVAL;
+    }
+    if (f < 0 || f > 1.0) {
+      ss << "value must be in the range 0..1";
+      return -ERANGE;
+    }
+    p.cache_target_full_ratio_micro = uf;
+  } else if (var == "cache_min_flush_age") {
+    if (interr.length()) {
+      ss << "error parsing int '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.cache_min_flush_age = n;
+  } else if (var == "cache_min_evict_age") {
+    if (interr.length()) {
+      ss << "error parsing int '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.cache_min_evict_age = n;
+  } else if (var == "min_read_recency_for_promote") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.min_read_recency_for_promote = n;
+  } else if (var == "hit_set_grade_decay_rate") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    if (n > 100 || n < 0) {
+      ss << "value out of range,valid range is 0 - 100";
+      return -EINVAL;
+    }
+    p.hit_set_grade_decay_rate = n;
+  } else if (var == "hit_set_search_last_n") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    if (n > p.hit_set_count || n < 0) {
+      ss << "value out of range,valid range is 0 - hit_set_count";
+      return -EINVAL;
+    }
+    p.hit_set_search_last_n = n;
+  } else if (var == "min_write_recency_for_promote") {
+    if (interr.length()) {
+      ss << "error parsing integer value '" << val << "': " << interr;
+      return -EINVAL;
+    }
+    p.min_write_recency_for_promote = n;
+  } else if (var == "fast_read") {
+    if (p.is_replicated()) {
+        ss << "fast read is not supported in replication pool";
+        return -EINVAL;
+    }
+    if (val == "true" || (interr.empty() && n == 1)) {
+      p.fast_read = true;
+    } else if (val == "false" || (interr.empty() && n == 0)) {
+      p.fast_read = false;
+    } else {
+      ss << "expecting value 'true', 'false', '0', or '1'";
+      return -EINVAL;
+    }
+  } else if (pool_opts_t::is_opt_name(var)) {
+    bool unset = val == "unset";
+    if (var == "compression_mode") {
+      if (!unset) {
+        auto cmode = Compressor::get_comp_mode_type(val);
+        if (!cmode) {
+	  ss << "unrecognized compression mode '" << val << "'";
+	  return -EINVAL;
+        }
+      }
+    } else if (var == "compression_algorithm") {
+      if (!unset) {
+        auto alg = Compressor::get_comp_alg_type(val);
+        if (!alg) {
+          ss << "unrecognized compression_algorithm '" << val << "'";
+	  return -EINVAL;
+        }
+      }
+    } else if (var == "compression_required_ratio") {
+      if (floaterr.length()) {
+        ss << "error parsing float value '" << val << "': " << floaterr;
+        return -EINVAL;
+      }
+      if (f < 0 || f > 1) {
+        ss << "compression_required_ratio is out of range (0-1): '" << val << "'";
+	return -EINVAL;
+      }
+    } else if (var == "csum_type") {
+      auto t = unset ? 0 : Checksummer::get_csum_string_type(val);
+      if (t < 0 ) {
+        ss << "unrecognized csum_type '" << val << "'";
+	return -EINVAL;
+      }
+      //preserve csum_type numeric value
+      n = t;
+      interr.clear(); 
+    } else if (var == "compression_max_blob_size" ||
+               var == "compression_min_blob_size" ||
+               var == "csum_max_block" ||
+               var == "csum_min_block") {
+      if (interr.length()) {
+        ss << "error parsing int value '" << val << "': " << interr;
+        return -EINVAL;
+      }
+    } else if (var == "fingerprint_algorithm") {
+      if (!unset) {
+        auto alg = pg_pool_t::get_fingerprint_from_str(val);
+        if (!alg) {
+          ss << "unrecognized fingerprint_algorithm '" << val << "'";
+	  return -EINVAL;
+        }
+      }
+    } else if (var == "target_size_bytes") {
+      if (interr.length()) {
+        ss << "error parsing unit value '" << val << "': " << interr;
+        return -EINVAL;
+      }
+      if (osdmap.require_osd_release < ceph_release_t::nautilus) {
+        ss << "must set require_osd_release to nautilus or "
+           << "later before setting target_size_bytes";
+        return -EINVAL;
+      }
+    } else if (var == "target_size_ratio") {
+      if (f < 0.0) {
+	ss << "target_size_ratio cannot be negative";
+	return -EINVAL;
+      }
+    } else if (var == "pg_num_min") {
+      if (interr.length()) {
+        ss << "error parsing int value '" << val << "': " << interr;
+        return -EINVAL;
+      }
+      if (n > (int)p.get_pg_num_target()) {
+	ss << "specified pg_num_min " << n
+	   << " > pg_num " << p.get_pg_num_target();
+	return -EINVAL;
+      }
+    } else if (var == "pg_num_max") {
+      if (interr.length()) {
+        ss << "error parsing int value '" << val << "': " << interr;
+        return -EINVAL;
+      }
+      if (n && n < (int)p.get_pg_num_target()) {
+	ss << "specified pg_num_max " << n
+	   << " < pg_num " << p.get_pg_num_target();
+	return -EINVAL;
+      }
+    } else if (var == "recovery_priority") {
+      if (interr.length()) {
+        ss << "error parsing int value '" << val << "': " << interr;
+        return -EINVAL;
+      }
+      if (!g_conf()->debug_allow_any_pool_priority) {
+        if (n > OSD_POOL_PRIORITY_MAX || n < OSD_POOL_PRIORITY_MIN) {
+          ss << "pool recovery_priority must be between " << OSD_POOL_PRIORITY_MIN
+	     << " and " << OSD_POOL_PRIORITY_MAX;
+          return -EINVAL;
+        }
+      }
+    } else if (var == "pg_autoscale_bias") {
+      if (f < 0.0 || f > 1000.0) {
+	ss << "pg_autoscale_bias must be between 0 and 1000";
+	return -EINVAL;
+      }
+    } else if (var == "dedup_tier") {
+      if (interr.empty()) {
+	ss << "expecting value 'pool name'";
+	return -EINVAL;
+      }
+      // Current base tier in dedup does not support ec pool 
+      if (p.is_erasure()) {
+	ss << "pool '" << poolstr
+	   << "' is an ec pool, which cannot be a base tier";
+	return -ENOTSUP;
+      }
+      int64_t lowtierpool_id = osdmap.lookup_pg_pool_name(val);
+      if (lowtierpool_id < 0) {
+	ss << "unrecognized pool '" << val << "'";
+	return -ENOENT;
+      }
+      const pg_pool_t *tp = osdmap.get_pg_pool(lowtierpool_id);
+      ceph_assert(tp);
+      n = lowtierpool_id;
+      // The original input is string (pool name), but we convert it to int64_t.
+      // So, clear interr
+      interr.clear();
+    } else if (var == "dedup_chunk_algorithm") {
+      if (!unset) {
+        auto alg = pg_pool_t::get_dedup_chunk_algorithm_from_str(val);
+        if (!alg) {
+          ss << "unrecognized fingerprint_algorithm '" << val << "'";
+	  return -EINVAL;
+        }
+      }
+    } else if (var == "dedup_cdc_chunk_size") {
+      if (interr.length()) {
+        ss << "error parsing int value '" << val << "': " << interr;
+        return -EINVAL;
+      }
+    }
+
+    pool_opts_t::opt_desc_t desc = pool_opts_t::get_opt_desc(var);
+    switch (desc.type) {
+    case pool_opts_t::STR:
+      if (unset) {
+	p.opts.unset(desc.key);
+      } else {
+	p.opts.set(desc.key, static_cast<std::string>(val));
+      }
+      break;
+    case pool_opts_t::INT:
+      if (interr.length()) {
+	ss << "error parsing integer value '" << val << "': " << interr;
+	return -EINVAL;
+      }
+      if (n == 0) {
+	p.opts.unset(desc.key);
+      } else {
+	p.opts.set(desc.key, static_cast<int64_t>(n));
+      }
+      break;
+    case pool_opts_t::DOUBLE:
+      if (floaterr.length()) {
+	ss << "error parsing floating point value '" << val << "': " << floaterr;
+	return -EINVAL;
+      }
+      if (f == 0) {
+	p.opts.unset(desc.key);
+      } else {
+	p.opts.set(desc.key, static_cast<double>(f));
+      }
+      break;
+    default:
+      ceph_assert(!"unknown type");
+    }
+  } else {
+    ss << "unrecognized variable '" << var << "'";
+    return -EINVAL;
+  }
+  if (val != "unset") {
+    ss << "set pool " << pool << " " << var << " to " << val;
+  } else {
+    ss << "unset pool " << pool << " " << var;
+  }
+  p.last_change = pending_inc.epoch;
+  pending_inc.new_pools[pool] = p;
+  return 0;
+}
+
+int OSDMonitor::prepare_command_pool_application(const string &prefix,
+                                                 const cmdmap_t& cmdmap,
+                                                 stringstream& ss)
+{
+  return _command_pool_application(prefix, cmdmap, ss, nullptr, true);
+}
+
+int OSDMonitor::preprocess_command_pool_application(const string &prefix,
+                                                    const cmdmap_t& cmdmap,
+                                                    stringstream& ss,
+                                                    bool *modified)
+{
+  return _command_pool_application(prefix, cmdmap, ss, modified, false);
+}
+
+
+/**
+ * Common logic for preprocess and prepare phases of pool application
+ * tag commands.  In preprocess mode we're only detecting invalid
+ * commands, and determining whether it was a modification or a no-op.
+ * In prepare mode we're actually updating the pending state.
+ */
+int OSDMonitor::_command_pool_application(const string &prefix,
+                                          const cmdmap_t& cmdmap,
+                                          stringstream& ss,
+                                          bool *modified,
+                                          bool preparing)
+{
+  string pool_name;
+  cmd_getval(cmdmap, "pool", pool_name);
+  int64_t pool = osdmap.lookup_pg_pool_name(pool_name.c_str());
+  if (pool < 0) {
+    ss << "unrecognized pool '" << pool_name << "'";
+    return -ENOENT;
+  }
+
+  pg_pool_t p = *osdmap.get_pg_pool(pool);
+  if (preparing) {
+    if (pending_inc.new_pools.count(pool)) {
+      p = pending_inc.new_pools[pool];
+    }
+  }
+
+  string app;
+  cmd_getval(cmdmap, "app", app);
+  bool app_exists = (p.application_metadata.count(app) > 0);
+
+  string key;
+  cmd_getval(cmdmap, "key", key);
+  if (key == "all") {
+    ss << "key cannot be 'all'";
+    return -EINVAL;
+  }
+
+  string value;
+  cmd_getval(cmdmap, "value", value);
+  if (value == "all") {
+    ss << "value cannot be 'all'";
+    return -EINVAL;
+  }
+
+  if (boost::algorithm::ends_with(prefix, "enable")) {
+    if (app.empty()) {
+      ss << "application name must be provided";
+      return -EINVAL;
+    }
+
+    if (p.is_tier()) {
+      ss << "application must be enabled on base tier";
+      return -EINVAL;
+    }
+
+    bool force = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", force);
+
+    if (!app_exists && !p.application_metadata.empty() && !force) {
+      ss << "Are you SURE? Pool '" << pool_name << "' already has an enabled "
+         << "application; pass --yes-i-really-mean-it to proceed anyway";
+      return -EPERM;
+    }
+
+    if (!app_exists && p.application_metadata.size() >= MAX_POOL_APPLICATIONS) {
+      ss << "too many enabled applications on pool '" << pool_name << "'; "
+         << "max " << MAX_POOL_APPLICATIONS;
+      return -EINVAL;
+    }
+
+    if (app.length() > MAX_POOL_APPLICATION_LENGTH) {
+      ss << "application name '" << app << "' too long; max length "
+         << MAX_POOL_APPLICATION_LENGTH;
+      return -EINVAL;
+    }
+
+    if (!app_exists) {
+      p.application_metadata[app] = {};
+    }
+    ss << "enabled application '" << app << "' on pool '" << pool_name << "'";
+
+  } else if (boost::algorithm::ends_with(prefix, "disable")) {
+    bool force = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", force);
+
+    if (!force) {
+      ss << "Are you SURE? Disabling an application within a pool might result "
+         << "in loss of application functionality; pass "
+         << "--yes-i-really-mean-it to proceed anyway";
+      return -EPERM;
+    }
+
+    if (!app_exists) {
+      ss << "application '" << app << "' is not enabled on pool '" << pool_name
+         << "'";
+      return 0; // idempotent
+    }
+
+    p.application_metadata.erase(app);
+    ss << "disable application '" << app << "' on pool '" << pool_name << "'";
+
+  } else if (boost::algorithm::ends_with(prefix, "set")) {
+    if (p.is_tier()) {
+      ss << "application metadata must be set on base tier";
+      return -EINVAL;
+    }
+
+    if (!app_exists) {
+      ss << "application '" << app << "' is not enabled on pool '" << pool_name
+         << "'";
+      return -ENOENT;
+    }
+
+    string key;
+    cmd_getval(cmdmap, "key", key);
+
+    if (key.empty()) {
+      ss << "key must be provided";
+      return -EINVAL;
+    }
+
+    auto &app_keys = p.application_metadata[app];
+    if (app_keys.count(key) == 0 &&
+        app_keys.size() >= MAX_POOL_APPLICATION_KEYS) {
+      ss << "too many keys set for application '" << app << "' on pool '"
+         << pool_name << "'; max " << MAX_POOL_APPLICATION_KEYS;
+      return -EINVAL;
+    }
+
+    if (key.length() > MAX_POOL_APPLICATION_LENGTH) {
+      ss << "key '" << app << "' too long; max length "
+         << MAX_POOL_APPLICATION_LENGTH;
+      return -EINVAL;
+    }
+
+    string value;
+    cmd_getval(cmdmap, "value", value);
+    if (value.length() > MAX_POOL_APPLICATION_LENGTH) {
+      ss << "value '" << value << "' too long; max length "
+         << MAX_POOL_APPLICATION_LENGTH;
+      return -EINVAL;
+    }
+
+    p.application_metadata[app][key] = value;
+    ss << "set application '" << app << "' key '" << key << "' to '"
+       << value << "' on pool '" << pool_name << "'";
+  } else if (boost::algorithm::ends_with(prefix, "rm")) {
+    if (!app_exists) {
+      ss << "application '" << app << "' is not enabled on pool '" << pool_name
+         << "'";
+      return -ENOENT;
+    }
+
+    string key;
+    cmd_getval(cmdmap, "key", key);
+    auto it = p.application_metadata[app].find(key);
+    if (it == p.application_metadata[app].end()) {
+      ss << "application '" << app << "' on pool '" << pool_name
+         << "' does not have key '" << key << "'";
+      return 0; // idempotent
+    }
+
+    p.application_metadata[app].erase(it);
+    ss << "removed application '" << app << "' key '" << key << "' on pool '"
+       << pool_name << "'";
+  } else {
+    ceph_abort();
+  }
+
+  if (preparing) {
+    p.last_change = pending_inc.epoch;
+    pending_inc.new_pools[pool] = p;
+  }
+
+  // Because we fell through this far, we didn't hit no-op cases,
+  // so pool was definitely modified
+  if (modified != nullptr) {
+    *modified = true;
+  }
+
+  return 0;
+}
+
+int OSDMonitor::_prepare_command_osd_crush_remove(
+    CrushWrapper &newcrush,
+    int32_t id,
+    int32_t ancestor,
+    bool has_ancestor,
+    bool unlink_only)
+{
+  int err = 0;
+
+  if (has_ancestor) {
+    err = newcrush.remove_item_under(cct, id, ancestor,
+        unlink_only);
+  } else {
+    err = newcrush.remove_item(cct, id, unlink_only);
+  }
+  return err;
+}
+
+void OSDMonitor::do_osd_crush_remove(CrushWrapper& newcrush)
+{
+  pending_inc.crush.clear();
+  newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+}
+
+int OSDMonitor::prepare_command_osd_crush_remove(
+    CrushWrapper &newcrush,
+    int32_t id,
+    int32_t ancestor,
+    bool has_ancestor,
+    bool unlink_only)
+{
+  int err = _prepare_command_osd_crush_remove(
+      newcrush, id, ancestor,
+      has_ancestor, unlink_only);
+
+  if (err < 0)
+    return err;
+
+  ceph_assert(err == 0);
+  do_osd_crush_remove(newcrush);
+
+  return 0;
+}
+
+int OSDMonitor::prepare_command_osd_remove(int32_t id)
+{
+  if (osdmap.is_up(id)) {
+    return -EBUSY;
+  }
+
+  pending_inc.new_state[id] = osdmap.get_state(id);
+  pending_inc.new_uuid[id] = uuid_d();
+  pending_metadata_rm.insert(id);
+  pending_metadata.erase(id);
+
+  return 0;
+}
+
+int32_t OSDMonitor::_allocate_osd_id(int32_t* existing_id)
+{
+  ceph_assert(existing_id);
+  *existing_id = -1;
+
+  for (int32_t i = 0; i < osdmap.get_max_osd(); ++i) {
+    if (!osdmap.exists(i) &&
+        pending_inc.new_up_client.count(i) == 0 &&
+        (pending_inc.new_state.count(i) == 0 ||
+         (pending_inc.new_state[i] & CEPH_OSD_EXISTS) == 0)) {
+      *existing_id = i;
+      return -1;
+    }
+  }
+
+  if (pending_inc.new_max_osd < 0) {
+    return osdmap.get_max_osd();
+  }
+  return pending_inc.new_max_osd;
+}
+
+void OSDMonitor::do_osd_create(
+    const int32_t id,
+    const uuid_d& uuid,
+    const string& device_class,
+    int32_t* new_id)
+{
+  dout(10) << __func__ << " uuid " << uuid << dendl;
+  ceph_assert(new_id);
+
+  // We presume validation has been performed prior to calling this
+  // function. We assert with prejudice.
+
+  int32_t allocated_id = -1; // declare here so we can jump
+  int32_t existing_id = -1;
+  if (!uuid.is_zero()) {
+    existing_id = osdmap.identify_osd(uuid);
+    if (existing_id >= 0) {
+      ceph_assert(id < 0 || id == existing_id);
+      *new_id = existing_id;
+      goto out;
+    } else if (id >= 0) {
+      // uuid does not exist, and id has been provided, so just create
+      // the new osd.id
+      *new_id = id;
+      goto out;
+    }
+  }
+
+  // allocate a new id
+  allocated_id = _allocate_osd_id(&existing_id);
+  dout(10) << __func__ << " allocated id " << allocated_id
+           << " existing id " << existing_id << dendl;
+  if (existing_id >= 0) {
+    ceph_assert(existing_id < osdmap.get_max_osd());
+    ceph_assert(allocated_id < 0);
+    *new_id = existing_id;
+  } else if (allocated_id >= 0) {
+    ceph_assert(existing_id < 0);
+    // raise max_osd
+    if (pending_inc.new_max_osd < 0) {
+      pending_inc.new_max_osd = osdmap.get_max_osd() + 1;
+    } else {
+      ++pending_inc.new_max_osd;
+    }
+    *new_id = pending_inc.new_max_osd - 1;
+    ceph_assert(*new_id == allocated_id);
+  } else {
+    ceph_abort_msg("unexpected condition");
+  }
+
+out:
+  if (device_class.size()) {
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+    if (newcrush.get_max_devices() < *new_id + 1) {
+      newcrush.set_max_devices(*new_id + 1);
+    }
+    string name = string("osd.") + stringify(*new_id);
+    if (!newcrush.item_exists(*new_id)) {
+      newcrush.set_item_name(*new_id, name);
+    }
+    ostringstream ss;
+    int r = newcrush.update_device_class(*new_id, device_class, name, &ss);
+    if (r < 0) {
+      derr << __func__ << " failed to set " << name << " device_class "
+	   << device_class << ": " << cpp_strerror(r) << " - " << ss.str()
+	   << dendl;
+      // non-fatal... this might be a replay and we want to be idempotent.
+    } else {
+      dout(20) << __func__ << " set " << name << " device_class " << device_class
+	       << dendl;
+      pending_inc.crush.clear();
+      newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    }
+  } else {
+    dout(20) << __func__ << " no device_class" << dendl;
+  }
+
+  dout(10) << __func__ << " using id " << *new_id << dendl;
+  if (osdmap.get_max_osd() <= *new_id && pending_inc.new_max_osd <= *new_id) {
+    pending_inc.new_max_osd = *new_id + 1;
+  }
+
+  pending_inc.new_weight[*new_id] = CEPH_OSD_IN;
+  // do not set EXISTS; OSDMap::set_weight, called by apply_incremental, will
+  // set it for us.  (ugh.)
+  pending_inc.new_state[*new_id] |= CEPH_OSD_NEW;
+  if (!uuid.is_zero())
+    pending_inc.new_uuid[*new_id] = uuid;
+}
+
+int OSDMonitor::validate_osd_create(
+    const int32_t id,
+    const uuid_d& uuid,
+    const bool check_osd_exists,
+    int32_t* existing_id,
+    stringstream& ss)
+{
+
+  dout(10) << __func__ << " id " << id << " uuid " << uuid
+           << " check_osd_exists " << check_osd_exists << dendl;
+
+  ceph_assert(existing_id);
+
+  if (id < 0 && uuid.is_zero()) {
+    // we have nothing to validate
+    *existing_id = -1;
+    return 0;
+  } else if (uuid.is_zero()) {
+    // we have an id but we will ignore it - because that's what
+    // `osd create` does.
+    return 0;
+  }
+
+  /*
+   * This function will be used to validate whether we are able to
+   * create a new osd when the `uuid` is specified.
+   *
+   * It will be used by both `osd create` and `osd new`, as the checks
+   * are basically the same when it pertains to osd id and uuid validation.
+   * However, `osd create` presumes an `uuid` is optional, for legacy
+   * reasons, while `osd new` requires the `uuid` to be provided. This
+   * means that `osd create` will not be idempotent if an `uuid` is not
+   * provided, but we will always guarantee the idempotency of `osd new`.
+   */
+
+  ceph_assert(!uuid.is_zero());
+  if (pending_inc.identify_osd(uuid) >= 0) {
+    // osd is about to exist
+    return -EAGAIN;
+  }
+
+  int32_t i = osdmap.identify_osd(uuid);
+  if (i >= 0) {
+    // osd already exists
+    if (id >= 0 && i != id) {
+      ss << "uuid " << uuid << " already in use for different id " << i;
+      return -EEXIST;
+    }
+    // return a positive errno to distinguish between a blocking error
+    // and an error we consider to not be a problem (i.e., this would be
+    // an idempotent operation).
+    *existing_id = i;
+    return EEXIST;
+  }
+  // i < 0
+  if (id >= 0) {
+    if (pending_inc.new_state.count(id)) {
+      // osd is about to exist
+      return -EAGAIN;
+    }
+    // we may not care if an osd exists if we are recreating a previously
+    // destroyed osd.
+    if (check_osd_exists && osdmap.exists(id)) {
+      ss << "id " << id << " already in use and does not match uuid "
+         << uuid;
+      return -EINVAL;
+    }
+  }
+  return 0;
+}
+
+int OSDMonitor::prepare_command_osd_create(
+    const int32_t id,
+    const uuid_d& uuid,
+    int32_t* existing_id,
+    stringstream& ss)
+{
+  dout(10) << __func__ << " id " << id << " uuid " << uuid << dendl;
+  ceph_assert(existing_id);
+  if (osdmap.is_destroyed(id)) {
+    ss << "ceph osd create has been deprecated. Please use ceph osd new "
+          "instead.";
+    return -EINVAL;
+  }
+
+  if (uuid.is_zero()) {
+    dout(10) << __func__ << " no uuid; assuming legacy `osd create`" << dendl;
+  }
+
+  return validate_osd_create(id, uuid, true, existing_id, ss);
+}
+
+int OSDMonitor::prepare_command_osd_new(
+    MonOpRequestRef op,
+    const cmdmap_t& cmdmap,
+    const map<string,string>& params,
+    stringstream &ss,
+    Formatter *f)
+{
+  uuid_d uuid;
+  string uuidstr;
+  int64_t id = -1;
+
+  ceph_assert(paxos.is_plugged());
+
+  dout(10) << __func__ << " " << op << dendl;
+
+  /* validate command. abort now if something's wrong. */
+
+  /* `osd new` will expect a `uuid` to be supplied; `id` is optional.
+   *
+   * If `id` is not specified, we will identify any existing osd based
+   * on `uuid`. Operation will be idempotent iff secrets match.
+   *
+   * If `id` is specified, we will identify any existing osd based on
+   * `uuid` and match against `id`. If they match, operation will be
+   * idempotent iff secrets match.
+   *
+   * `-i secrets.json` will be optional. If supplied, will be used
+   * to check for idempotency when `id` and `uuid` match.
+   *
+   * If `id` is not specified, and `uuid` does not exist, an id will
+   * be found or allocated for the osd.
+   *
+   * If `id` is specified, and the osd has been previously marked
+   * as destroyed, then the `id` will be reused.
+   */
+  if (!cmd_getval(cmdmap, "uuid", uuidstr)) {
+    ss << "requires the OSD's UUID to be specified.";
+    return -EINVAL;
+  } else if (!uuid.parse(uuidstr.c_str())) {
+    ss << "invalid UUID value '" << uuidstr << "'.";
+    return -EINVAL;
+  }
+
+  if (cmd_getval(cmdmap, "id", id) &&
+      (id < 0)) {
+    ss << "invalid OSD id; must be greater or equal than zero.";
+    return -EINVAL;
+  }
+
+  // are we running an `osd create`-like command, or recreating
+  // a previously destroyed osd?
+
+  bool is_recreate_destroyed = (id >= 0 && osdmap.is_destroyed(id));
+
+  // we will care about `id` to assess whether osd is `destroyed`, or
+  // to create a new osd.
+  // we will need an `id` by the time we reach auth.
+
+  int32_t existing_id = -1;
+  int err = validate_osd_create(id, uuid, !is_recreate_destroyed,
+                                &existing_id, ss);
+
+  bool may_be_idempotent = false;
+  if (err == EEXIST) {
+    // this is idempotent from the osdmon's point-of-view
+    may_be_idempotent = true;
+    ceph_assert(existing_id >= 0);
+    id = existing_id;
+  } else if (err < 0) {
+    return err;
+  }
+
+  if (!may_be_idempotent) {
+    // idempotency is out of the window. We are either creating a new
+    // osd or recreating a destroyed osd.
+    //
+    // We now need to figure out if we have an `id` (and if it's valid),
+    // of find an `id` if we don't have one.
+
+    // NOTE: we need to consider the case where the `id` is specified for
+    // `osd create`, and we must honor it. So this means checking if
+    // the `id` is destroyed, and if so assume the destroy; otherwise,
+    // check if it `exists` - in which case we complain about not being
+    // `destroyed`. In the end, if nothing fails, we must allow the
+    // creation, so that we are compatible with `create`.
+    if (id >= 0 && osdmap.exists(id) && !osdmap.is_destroyed(id)) {
+      dout(10) << __func__ << " osd." << id << " isn't destroyed" << dendl;
+      ss << "OSD " << id << " has not yet been destroyed";
+      return -EINVAL;
+    } else if (id < 0) {
+      // find an `id`
+      id = _allocate_osd_id(&existing_id);
+      if (id < 0) {
+        ceph_assert(existing_id >= 0);
+        id = existing_id;
+      }
+      dout(10) << __func__ << " found id " << id << " to use" << dendl;
+    } else if (id >= 0 && osdmap.is_destroyed(id)) {
+      dout(10) << __func__ << " recreating osd." << id << dendl;
+    } else {
+      dout(10) << __func__ << " creating new osd." << id << dendl;
+    }
+  } else {
+    ceph_assert(id >= 0);
+    ceph_assert(osdmap.exists(id));
+  }
+
+  // we are now able to either create a brand new osd or reuse an existing
+  // osd that has been previously destroyed.
+
+  dout(10) << __func__ << " id " << id << " uuid " << uuid << dendl;
+
+  if (may_be_idempotent && params.empty()) {
+    // nothing to do, really.
+    dout(10) << __func__ << " idempotent and no params -- no op." << dendl;
+    ceph_assert(id >= 0);
+    if (f) {
+      f->open_object_section("created_osd");
+      f->dump_int("osdid", id);
+      f->close_section();
+    } else {
+      ss << id;
+    }
+    return EEXIST;
+  }
+
+  string device_class;
+  auto p = params.find("crush_device_class");
+  if (p != params.end()) {
+    device_class = p->second;
+    dout(20) << __func__ << " device_class will be " << device_class << dendl;
+  }
+  string cephx_secret, lockbox_secret, dmcrypt_key;
+  bool has_lockbox = false;
+  bool has_secrets = params.count("cephx_secret")
+    || params.count("cephx_lockbox_secret")
+    || params.count("dmcrypt_key");
+
+  KVMonitor *svc = nullptr;
+  AuthMonitor::auth_entity_t cephx_entity, lockbox_entity;
+
+  if (has_secrets) {
+    if (params.count("cephx_secret") == 0) {
+      ss << "requires a cephx secret.";
+      return -EINVAL;
+    }
+    cephx_secret = params.at("cephx_secret");
+
+    bool has_lockbox_secret = (params.count("cephx_lockbox_secret") > 0);
+    bool has_dmcrypt_key = (params.count("dmcrypt_key") > 0);
+
+    dout(10) << __func__ << " has lockbox " << has_lockbox_secret
+             << " dmcrypt " << has_dmcrypt_key << dendl;
+
+    if (has_lockbox_secret && has_dmcrypt_key) {
+      has_lockbox = true;
+      lockbox_secret = params.at("cephx_lockbox_secret");
+      dmcrypt_key = params.at("dmcrypt_key");
+    } else if (!has_lockbox_secret != !has_dmcrypt_key) {
+      ss << "requires both a cephx lockbox secret and a dm-crypt key.";
+      return -EINVAL;
+    }
+
+    dout(10) << __func__ << " validate secrets using osd id " << id << dendl;
+
+    err = mon.authmon()->validate_osd_new(id, uuid,
+        cephx_secret,
+        lockbox_secret,
+        cephx_entity,
+        lockbox_entity,
+        ss);
+    if (err < 0) {
+      return err;
+    } else if (may_be_idempotent && err != EEXIST) {
+      // for this to be idempotent, `id` should already be >= 0; no need
+      // to use validate_id.
+      ceph_assert(id >= 0);
+      ss << "osd." << id << " exists but secrets do not match";
+      return -EEXIST;
+    }
+
+    if (has_lockbox) {
+      svc = mon.kvmon();
+      err = svc->validate_osd_new(uuid, dmcrypt_key, ss);
+      if (err < 0) {
+        return err;
+      } else if (may_be_idempotent && err != EEXIST) {
+        ceph_assert(id >= 0);
+        ss << "osd." << id << " exists but dm-crypt key does not match.";
+        return -EEXIST;
+      }
+    }
+  }
+  ceph_assert(!has_secrets || !cephx_secret.empty());
+  ceph_assert(!has_lockbox || !lockbox_secret.empty());
+
+  if (may_be_idempotent) {
+    // we have nothing to do for either the osdmon or the authmon,
+    // and we have no lockbox - so the config key service will not be
+    // touched. This is therefore an idempotent operation, and we can
+    // just return right away.
+    dout(10) << __func__ << " idempotent -- no op." << dendl;
+    ceph_assert(id >= 0);
+    if (f) {
+      f->open_object_section("created_osd");
+      f->dump_int("osdid", id);
+      f->close_section();
+    } else {
+      ss << id;
+    }
+    return EEXIST;
+  }
+  ceph_assert(!may_be_idempotent);
+
+  // perform updates.
+  if (has_secrets) {
+    ceph_assert(!cephx_secret.empty());
+    ceph_assert((lockbox_secret.empty() && dmcrypt_key.empty()) ||
+           (!lockbox_secret.empty() && !dmcrypt_key.empty()));
+
+    err = mon.authmon()->do_osd_new(cephx_entity,
+        lockbox_entity,
+        has_lockbox);
+    ceph_assert(0 == err);
+
+    if (has_lockbox) {
+      ceph_assert(nullptr != svc);
+      svc->do_osd_new(uuid, dmcrypt_key);
+    }
+  }
+
+  if (is_recreate_destroyed) {
+    ceph_assert(id >= 0);
+    ceph_assert(osdmap.is_destroyed(id));
+    pending_inc.new_state[id] |= CEPH_OSD_DESTROYED;
+    if ((osdmap.get_state(id) & CEPH_OSD_NEW) == 0) {
+      pending_inc.new_state[id] |= CEPH_OSD_NEW;
+    }
+    if (osdmap.get_state(id) & CEPH_OSD_UP) {
+      // due to http://tracker.ceph.com/issues/20751 some clusters may
+      // have UP set for non-existent OSDs; make sure it is cleared
+      // for a newly created osd.
+      pending_inc.new_state[id] |= CEPH_OSD_UP;
+    }
+    pending_inc.new_uuid[id] = uuid;
+  } else {
+    ceph_assert(id >= 0);
+    int32_t new_id = -1;
+    do_osd_create(id, uuid, device_class, &new_id);
+    ceph_assert(new_id >= 0);
+    ceph_assert(id == new_id);
+  }
+
+  if (f) {
+    f->open_object_section("created_osd");
+    f->dump_int("osdid", id);
+    f->close_section();
+  } else {
+    ss << id;
+  }
+
+  return 0;
+}
+
+bool OSDMonitor::prepare_command(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MMonCommand>();
+  stringstream ss;
+  cmdmap_t cmdmap;
+  if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
+    string rs = ss.str();
+    mon.reply_command(op, -EINVAL, rs, get_last_committed());
+    return true;
+  }
+
+  MonSession *session = op->get_session();
+  if (!session) {
+    derr << __func__ << " no session" << dendl;
+    mon.reply_command(op, -EACCES, "access denied", get_last_committed());
+    return true;
+  }
+
+  return prepare_command_impl(op, cmdmap);
+}
+
+static int parse_reweights(CephContext *cct,
+			   const cmdmap_t& cmdmap,
+			   const OSDMap& osdmap,
+			   map<int32_t, uint32_t>* weights)
+{
+  string weights_str;
+  if (!cmd_getval(cmdmap, "weights", weights_str)) {
+    return -EINVAL;
+  }
+  std::replace(begin(weights_str), end(weights_str), '\'', '"');
+  json_spirit::mValue json_value;
+  if (!json_spirit::read(weights_str, json_value)) {
+    return -EINVAL;
+  }
+  if (json_value.type() != json_spirit::obj_type) {
+    return -EINVAL;
+  }
+  const auto obj = json_value.get_obj();
+  try {
+    for (auto& osd_weight : obj) {
+      auto osd_id = std::stoi(osd_weight.first);
+      if (!osdmap.exists(osd_id)) {
+	return -ENOENT;
+      }
+      if (osd_weight.second.type() != json_spirit::str_type) {
+	return -EINVAL;
+      }
+      auto weight = std::stoul(osd_weight.second.get_str());
+      weights->insert({osd_id, weight});
+    }
+  } catch (const std::logic_error& e) {
+    return -EINVAL;
+  }
+  return 0;
+}
+
+int OSDMonitor::prepare_command_osd_destroy(
+    int32_t id,
+    stringstream& ss)
+{
+  ceph_assert(paxos.is_plugged());
+
+  // we check if the osd exists for the benefit of `osd purge`, which may
+  // have previously removed the osd. If the osd does not exist, return
+  // -ENOENT to convey this, and let the caller deal with it.
+  //
+  // we presume that all auth secrets and config keys were removed prior
+  // to this command being called. if they exist by now, we also assume
+  // they must have been created by some other command and do not pertain
+  // to this non-existent osd.
+  if (!osdmap.exists(id)) {
+    dout(10) << __func__ << " osd." << id << " does not exist." << dendl;
+    return -ENOENT;
+  }
+
+  uuid_d uuid = osdmap.get_uuid(id);
+  dout(10) << __func__ << " destroying osd." << id
+           << " uuid " << uuid << dendl;
+
+  // if it has been destroyed, we assume our work here is done.
+  if (osdmap.is_destroyed(id)) {
+    ss << "destroyed osd." << id;
+    return 0;
+  }
+
+  EntityName cephx_entity, lockbox_entity;
+  bool idempotent_auth = false, idempotent_cks = false;
+
+  int err = mon.authmon()->validate_osd_destroy(id, uuid,
+                                                 cephx_entity,
+                                                 lockbox_entity,
+                                                 ss);
+  if (err < 0) {
+    if (err == -ENOENT) {
+      idempotent_auth = true;
+    } else {
+      return err;
+    }
+  }
+
+  auto svc = mon.kvmon();
+  err = svc->validate_osd_destroy(id, uuid);
+  if (err < 0) {
+    ceph_assert(err == -ENOENT);
+    err = 0;
+    idempotent_cks = true;
+  }
+
+  if (!idempotent_auth) {
+    err = mon.authmon()->do_osd_destroy(cephx_entity, lockbox_entity);
+    ceph_assert(0 == err);
+  }
+
+  if (!idempotent_cks) {
+    svc->do_osd_destroy(id, uuid);
+  }
+
+  pending_inc.new_state[id] = CEPH_OSD_DESTROYED;
+  pending_inc.new_uuid[id] = uuid_d();
+
+  // we can only propose_pending() once per service, otherwise we'll be
+  // defying PaxosService and all laws of nature. Therefore, as we may
+  // be used during 'osd purge', let's keep the caller responsible for
+  // proposing.
+  ceph_assert(err == 0);
+  return 0;
+}
+
+int OSDMonitor::prepare_command_osd_purge(
+    int32_t id,
+    stringstream& ss)
+{
+  ceph_assert(paxos.is_plugged());
+  dout(10) << __func__ << " purging osd." << id << dendl;
+
+  ceph_assert(!osdmap.is_up(id));
+
+  /*
+   * This may look a bit weird, but this is what's going to happen:
+   *
+   *  1. we make sure that removing from crush works
+   *  2. we call `prepare_command_osd_destroy()`. If it returns an
+   *     error, then we abort the whole operation, as no updates
+   *     have been made. However, we this function will have
+   *     side-effects, thus we need to make sure that all operations
+   *     performed henceforth will *always* succeed.
+   *  3. we call `prepare_command_osd_remove()`. Although this
+   *     function can return an error, it currently only checks if the
+   *     osd is up - and we have made sure that it is not so, so there
+   *     is no conflict, and it is effectively an update.
+   *  4. finally, we call `do_osd_crush_remove()`, which will perform
+   *     the crush update we delayed from before.
+   */
+
+  CrushWrapper newcrush;
+  _get_pending_crush(newcrush);
+
+  bool may_be_idempotent = false;
+
+  int err = _prepare_command_osd_crush_remove(newcrush, id, 0, false, false);
+  if (err == -ENOENT) {
+    err = 0;
+    may_be_idempotent = true;
+  } else if (err < 0) {
+    ss << "error removing osd." << id << " from crush";
+    return err;
+  }
+
+  // no point destroying the osd again if it has already been marked destroyed
+  if (!osdmap.is_destroyed(id)) {
+    err = prepare_command_osd_destroy(id, ss);
+    if (err < 0) {
+      if (err == -ENOENT) {
+        err = 0;
+      } else {
+        return err;
+      }
+    } else {
+      may_be_idempotent = false;
+    }
+  }
+  ceph_assert(0 == err);
+
+  if (may_be_idempotent && !osdmap.exists(id)) {
+    dout(10) << __func__ << " osd." << id << " does not exist and "
+             << "we are idempotent." << dendl;
+    return -ENOENT;
+  }
+
+  err = prepare_command_osd_remove(id);
+  // we should not be busy, as we should have made sure this id is not up.
+  ceph_assert(0 == err);
+
+  do_osd_crush_remove(newcrush);
+  return 0;
+}
+
+bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
+				      const cmdmap_t& cmdmap)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MMonCommand>();
+  bool ret = false;
+  stringstream ss;
+  string rs;
+  bufferlist rdata;
+  int err = 0;
+
+  string format;
+  cmd_getval(cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+  string prefix;
+  cmd_getval(cmdmap, "prefix", prefix);
+
+  int64_t osdid;
+  string osd_name;
+  bool osdid_present = false;
+  if (prefix != "osd pg-temp" &&
+      prefix != "osd pg-upmap" &&
+      prefix != "osd pg-upmap-items") {  // avoid commands with non-int id arg
+    osdid_present = cmd_getval(cmdmap, "id", osdid);
+  }
+  if (osdid_present) {
+    ostringstream oss;
+    oss << "osd." << osdid;
+    osd_name = oss.str();
+  }
+
+  // Even if there's a pending state with changes that could affect
+  // a command, considering that said state isn't yet committed, we
+  // just don't care about those changes if the command currently being
+  // handled acts as a no-op against the current committed state.
+  // In a nutshell, we assume this command  happens *before*.
+  //
+  // Let me make this clearer:
+  //
+  //   - If we have only one client, and that client issues some
+  //     operation that would conflict with this operation  but is
+  //     still on the pending state, then we would be sure that said
+  //     operation wouldn't have returned yet, so the client wouldn't
+  //     issue this operation (unless the client didn't wait for the
+  //     operation to finish, and that would be the client's own fault).
+  //
+  //   - If we have more than one client, each client will observe
+  //     whatever is the state at the moment of the commit.  So, if we
+  //     have two clients, one issuing an unlink and another issuing a
+  //     link, and if the link happens while the unlink is still on the
+  //     pending state, from the link's point-of-view this is a no-op.
+  //     If different clients are issuing conflicting operations and
+  //     they care about that, then the clients should make sure they
+  //     enforce some kind of concurrency mechanism -- from our
+  //     perspective that's what Douglas Adams would call an SEP.
+  //
+  // This should be used as a general guideline for most commands handled
+  // in this function.  Adapt as you see fit, but please bear in mind that
+  // this is the expected behavior.
+   
+ 
+  if (prefix == "osd setcrushmap" ||
+      (prefix == "osd crush set" && !osdid_present)) {
+    if (pending_inc.crush.length()) {
+      dout(10) << __func__ << " waiting for pending crush update " << dendl;
+      wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+      return true;
+    }
+    dout(10) << "prepare_command setting new crush map" << dendl;
+    bufferlist data(m->get_data());
+    CrushWrapper crush;
+    try {
+      auto bl = data.cbegin();
+      crush.decode(bl);
+    }
+    catch (const std::exception &e) {
+      err = -EINVAL;
+      ss << "Failed to parse crushmap: " << e.what();
+      goto reply;
+    }
+  
+    int64_t prior_version = 0;
+    if (cmd_getval(cmdmap, "prior_version", prior_version)) {
+      if (prior_version == osdmap.get_crush_version() - 1) {
+	// see if we are a resend of the last update.  this is imperfect
+	// (multiple racing updaters may not both get reliable success)
+	// but we expect crush updaters (via this interface) to be rare-ish.
+	bufferlist current, proposed;
+	osdmap.crush->encode(current, mon.get_quorum_con_features());
+	crush.encode(proposed, mon.get_quorum_con_features());
+	if (current.contents_equal(proposed)) {
+	  dout(10) << __func__
+		   << " proposed matches current and version equals previous"
+		   << dendl;
+	  err = 0;
+	  ss << osdmap.get_crush_version();
+	  goto reply;
+	}
+      }
+      if (prior_version != osdmap.get_crush_version()) {
+	err = -EPERM;
+	ss << "prior_version " << prior_version << " != crush version "
+	   << osdmap.get_crush_version();
+	goto reply;
+      }
+    }
+
+    if (crush.has_legacy_rule_ids()) {
+      err = -EINVAL;
+      ss << "crush maps with ruleset != ruleid are no longer allowed";
+      goto reply;
+    }
+    if (!validate_crush_against_features(&crush, ss)) {
+      err = -EINVAL;
+      goto reply;
+    }
+
+    err = osdmap.validate_crush_rules(&crush, &ss);
+    if (err < 0) {
+      goto reply;
+    }
+
+    if (g_conf()->mon_osd_crush_smoke_test) {
+      // sanity check: test some inputs to make sure this map isn't
+      // totally broken
+      dout(10) << " testing map" << dendl;
+      stringstream ess;
+      CrushTester tester(crush, ess);
+      tester.set_min_x(0);
+      tester.set_max_x(50);
+      auto start = ceph::coarse_mono_clock::now();
+      int r = tester.test_with_fork(g_conf()->mon_lease);
+      auto duration = ceph::coarse_mono_clock::now() - start;
+      if (r < 0) {
+	dout(10) << " tester.test_with_fork returns " << r
+		 << ": " << ess.str() << dendl;
+	ss << "crush smoke test failed with " << r << ": " << ess.str();
+	err = r;
+	goto reply;
+      }
+      dout(10) << __func__ << " crush somke test duration: "
+               << duration << ", result: " << ess.str() << dendl;
+    }
+
+    pending_inc.crush = data;
+    ss << osdmap.get_crush_version() + 1;
+    goto update;
+
+  } else if (prefix == "osd crush set-all-straw-buckets-to-straw2") {
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+    for (int b = 0; b < newcrush.get_max_buckets(); ++b) {
+      int bid = -1 - b;
+      if (newcrush.bucket_exists(bid) &&
+	  newcrush.get_bucket_alg(bid) == CRUSH_BUCKET_STRAW) {
+	dout(20) << " bucket " << bid << " is straw, can convert" << dendl;
+	newcrush.bucket_set_alg(bid, CRUSH_BUCKET_STRAW2);
+      }
+    }
+    if (!validate_crush_against_features(&newcrush, ss)) {
+      err = -EINVAL;
+      goto reply;
+    }
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd crush set-device-class") {
+    string device_class;
+    if (!cmd_getval(cmdmap, "class", device_class)) {
+      err = -EINVAL; // no value!
+      goto reply;
+    }
+
+    bool stop = false;
+    vector<string> idvec;
+    cmd_getval(cmdmap, "ids", idvec);
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+    set<int> updated;
+    for (unsigned j = 0; j < idvec.size() && !stop; j++) {
+      set<int> osds;
+      // wildcard?
+      if (j == 0 &&
+          (idvec[0] == "any" || idvec[0] == "all" || idvec[0] == "*")) {
+        osdmap.get_all_osds(osds);
+        stop = true;
+      } else {
+        // try traditional single osd way
+        long osd = parse_osd_id(idvec[j].c_str(), &ss);
+        if (osd < 0) {
+          // ss has reason for failure
+          ss << ", unable to parse osd id:\"" << idvec[j] << "\". ";
+          err = -EINVAL;
+          continue;
+        }
+        osds.insert(osd);
+      }
+
+      for (auto &osd : osds) {
+        if (!osdmap.exists(osd)) {
+          ss << "osd." << osd << " does not exist. ";
+          continue;
+        }
+
+        ostringstream oss;
+        oss << "osd." << osd;
+        string name = oss.str();
+
+	if (newcrush.get_max_devices() < osd + 1) {
+	  newcrush.set_max_devices(osd + 1);
+	}
+        string action;
+        if (newcrush.item_exists(osd)) {
+          action = "updating";
+        } else {
+          action = "creating";
+          newcrush.set_item_name(osd, name);
+        }
+
+        dout(5) << action << " crush item id " << osd << " name '" << name
+                << "' device_class '" << device_class << "'"
+                << dendl;
+        err = newcrush.update_device_class(osd, device_class, name, &ss);
+        if (err < 0) {
+          goto reply;
+        }
+        if (err == 0 && !_have_pending_crush()) {
+          if (!stop) {
+            // for single osd only, wildcard makes too much noise
+            ss << "set-device-class item id " << osd << " name '" << name
+               << "' device_class '" << device_class << "': no change. ";
+          }
+        } else {
+          updated.insert(osd);
+        }
+      }
+    }
+
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    ss << "set osd(s) " << updated << " to class '" << device_class << "'";
+    getline(ss, rs);
+    wait_for_finished_proposal(
+      op,
+      new Monitor::C_Command(mon,op, 0, rs, get_last_committed() + 1));
+    return true;
+ } else if (prefix == "osd crush rm-device-class") {
+    bool stop = false;
+    vector<string> idvec;
+    cmd_getval(cmdmap, "ids", idvec);
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+    set<int> updated;
+
+    for (unsigned j = 0; j < idvec.size() && !stop; j++) {
+      set<int> osds;
+
+      // wildcard?
+      if (j == 0 &&
+          (idvec[0] == "any" || idvec[0] == "all" || idvec[0] == "*")) {
+        osdmap.get_all_osds(osds);
+        stop = true;
+      } else {
+        // try traditional single osd way
+        long osd = parse_osd_id(idvec[j].c_str(), &ss);
+        if (osd < 0) {
+          // ss has reason for failure
+          ss << ", unable to parse osd id:\"" << idvec[j] << "\". ";
+          err = -EINVAL;
+          goto reply;
+        }
+        osds.insert(osd);
+      }
+
+      for (auto &osd : osds) {
+        if (!osdmap.exists(osd)) {
+          ss << "osd." << osd << " does not exist. ";
+          continue;
+        }
+
+        auto class_name = newcrush.get_item_class(osd);
+        if (!class_name) {
+          ss << "osd." << osd << " belongs to no class, ";
+          continue;
+        }
+        // note that we do not verify if class_is_in_use here
+        // in case the device is misclassified and user wants
+        // to overridely reset...
+
+        err = newcrush.remove_device_class(cct, osd, &ss);
+        if (err < 0) {
+          // ss has reason for failure
+          goto reply;
+        }
+        updated.insert(osd);
+      }
+    }
+
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    ss << "done removing class of osd(s): " << updated;
+    getline(ss, rs);
+    wait_for_finished_proposal(
+      op,
+      new Monitor::C_Command(mon,op, 0, rs, get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd crush class create") {
+    string device_class;
+    if (!cmd_getval(cmdmap, "class", device_class)) {
+      err = -EINVAL; // no value!
+      goto reply;
+    }
+    if (osdmap.require_osd_release < ceph_release_t::luminous) {
+      ss << "you must complete the upgrade and 'ceph osd require-osd-release "
+         << "luminous' before using crush device classes";
+      err = -EPERM;
+      goto reply;
+    }
+    if (!_have_pending_crush() &&
+        _get_stable_crush().class_exists(device_class)) {
+      ss << "class '" << device_class << "' already exists";
+      goto reply;
+    }
+     CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+     if (newcrush.class_exists(device_class)) {
+      ss << "class '" << device_class << "' already exists";
+      goto update;
+    }
+    int class_id = newcrush.get_or_create_class_id(device_class);
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    ss << "created class " << device_class << " with id " << class_id
+       << " to crush map";
+    goto update;
+  } else if (prefix == "osd crush class rm") {
+    string device_class;
+    if (!cmd_getval(cmdmap, "class", device_class)) {
+       err = -EINVAL; // no value!
+       goto reply;
+     }
+    if (osdmap.require_osd_release < ceph_release_t::luminous) {
+       ss << "you must complete the upgrade and 'ceph osd require-osd-release "
+         << "luminous' before using crush device classes";
+       err = -EPERM;
+       goto reply;
+     }
+
+     if (!osdmap.crush->class_exists(device_class)) {
+       err = 0;
+       goto reply;
+     }
+
+     CrushWrapper newcrush;
+     _get_pending_crush(newcrush);
+     if (!newcrush.class_exists(device_class)) {
+       err = 0; // make command idempotent
+       goto wait;
+     }
+     int class_id = newcrush.get_class_id(device_class);
+     stringstream ts;
+     if (newcrush.class_is_in_use(class_id, &ts)) {
+       err = -EBUSY;
+       ss << "class '" << device_class << "' " << ts.str();
+       goto reply;
+     }
+
+     // check if class is used by any erasure-code-profiles
+     mempool::osdmap::map<string,map<string,string>> old_ec_profiles =
+       osdmap.get_erasure_code_profiles();
+     auto ec_profiles = pending_inc.get_erasure_code_profiles();
+#ifdef HAVE_STDLIB_MAP_SPLICING
+     ec_profiles.merge(old_ec_profiles);
+#else
+     ec_profiles.insert(make_move_iterator(begin(old_ec_profiles)),
+                        make_move_iterator(end(old_ec_profiles)));
+#endif
+     list<string> referenced_by;
+     for (auto &i: ec_profiles) {
+       for (auto &j: i.second) {
+         if ("crush-device-class" == j.first && device_class == j.second) {
+           referenced_by.push_back(i.first);
+         }
+       }
+     }
+     if (!referenced_by.empty()) {
+       err = -EBUSY;
+       ss << "class '" << device_class
+          << "' is still referenced by erasure-code-profile(s): " << referenced_by;
+       goto reply;
+     }
+
+     set<int> osds;
+     newcrush.get_devices_by_class(device_class, &osds);
+     for (auto& p: osds) {
+       err = newcrush.remove_device_class(g_ceph_context, p, &ss);
+       if (err < 0) {
+         // ss has reason for failure
+         goto reply;
+       }
+     }
+
+     if (osds.empty()) {
+       // empty class, remove directly
+       err = newcrush.remove_class_name(device_class);
+       if (err < 0) {
+         ss << "class '" << device_class << "' cannot be removed '"
+            << cpp_strerror(err) << "'";
+         goto reply;
+       }
+     }
+
+     pending_inc.crush.clear();
+     newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+     ss << "removed class " << device_class << " with id " << class_id
+        << " from crush map";
+     goto update;
+  } else if (prefix == "osd crush class rename") {
+    string srcname, dstname;
+    if (!cmd_getval(cmdmap, "srcname", srcname)) {
+      err = -EINVAL;
+      goto reply;
+    }
+    if (!cmd_getval(cmdmap, "dstname", dstname)) {
+      err = -EINVAL;
+      goto reply;
+    }
+
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+    if (!newcrush.class_exists(srcname) && newcrush.class_exists(dstname)) {
+      // suppose this is a replay and return success
+      // so command is idempotent
+      ss << "already renamed to '" << dstname << "'";
+      err = 0;
+      goto reply;
+    }
+
+    err = newcrush.rename_class(srcname, dstname);
+    if (err < 0) {
+      ss << "fail to rename '" << srcname << "' to '" << dstname << "' : "
+         << cpp_strerror(err);
+      goto reply;
+    }
+
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    ss << "rename class '" << srcname << "' to '" << dstname << "'";
+    goto update;
+  } else if (prefix == "osd crush add-bucket") {
+    // os crush add-bucket <name> <type>
+    string name, typestr;
+    vector<string> argvec;
+    cmd_getval(cmdmap, "name", name);
+    cmd_getval(cmdmap, "type", typestr);
+    cmd_getval(cmdmap, "args", argvec);
+    map<string,string> loc;
+    if (!argvec.empty()) {
+      CrushWrapper::parse_loc_map(argvec, &loc);
+      dout(0) << "will create and move bucket '" << name
+              << "' to location " << loc << dendl;
+    }
+
+    if (!_have_pending_crush() &&
+	_get_stable_crush().name_exists(name)) {
+      ss << "bucket '" << name << "' already exists";
+      goto reply;
+    }
+
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    if (newcrush.name_exists(name)) {
+      ss << "bucket '" << name << "' already exists";
+      goto update;
+    }
+    int type = newcrush.get_type_id(typestr);
+    if (type < 0) {
+      ss << "type '" << typestr << "' does not exist";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (type == 0) {
+      ss << "type '" << typestr << "' is for devices, not buckets";
+      err = -EINVAL;
+      goto reply;
+    }
+    int bucketno;
+    err = newcrush.add_bucket(0, 0,
+			      CRUSH_HASH_DEFAULT, type, 0, NULL,
+			      NULL, &bucketno);
+    if (err < 0) {
+      ss << "add_bucket error: '" << cpp_strerror(err) << "'";
+      goto reply;
+    }
+    err = newcrush.set_item_name(bucketno, name);
+    if (err < 0) {
+      ss << "error setting bucket name to '" << name << "'";
+      goto reply;
+    }
+
+    if (!loc.empty()) {
+      if (!newcrush.check_item_loc(cct, bucketno, loc,
+          (int *)NULL)) {
+        err = newcrush.move_bucket(cct, bucketno, loc);
+        if (err < 0) {
+          ss << "error moving bucket '" << name << "' to location " << loc;
+          goto reply;
+        }
+      } else {
+        ss << "no need to move item id " << bucketno << " name '" << name
+           << "' to location " << loc << " in crush map";
+      }
+    }
+
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    if (loc.empty()) {
+      ss << "added bucket " << name << " type " << typestr
+         << " to crush map";
+    } else {
+      ss << "added bucket " << name << " type " << typestr
+         << " to location " << loc;
+    }
+    goto update;
+  } else if (prefix == "osd crush rename-bucket") {
+    string srcname, dstname;
+    cmd_getval(cmdmap, "srcname", srcname);
+    cmd_getval(cmdmap, "dstname", dstname);
+
+    err = crush_rename_bucket(srcname, dstname, &ss);
+    if (err == -EALREADY) // equivalent to success for idempotency
+      err = 0;
+    if (err)
+      goto reply;
+    else
+      goto update;
+  } else if (prefix == "osd crush weight-set create" ||
+	     prefix == "osd crush weight-set create-compat") {
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+    int64_t pool;
+    int positions;
+    if (newcrush.has_non_straw2_buckets()) {
+      ss << "crush map contains one or more bucket(s) that are not straw2";
+      err = -EPERM;
+      goto reply;
+    }
+    if (prefix == "osd crush weight-set create") {
+      if (osdmap.require_min_compat_client != ceph_release_t::unknown &&
+	  osdmap.require_min_compat_client < ceph_release_t::luminous) {
+	ss << "require_min_compat_client "
+	   << osdmap.require_min_compat_client
+	   << " < luminous, which is required for per-pool weight-sets. "
+           << "Try 'ceph osd set-require-min-compat-client luminous' "
+           << "before using the new interface";
+	err = -EPERM;
+	goto reply;
+      }
+      string poolname, mode;
+      cmd_getval(cmdmap, "pool", poolname);
+      pool = osdmap.lookup_pg_pool_name(poolname.c_str());
+      if (pool < 0) {
+	ss << "pool '" << poolname << "' not found";
+	err = -ENOENT;
+	goto reply;
+      }
+      cmd_getval(cmdmap, "mode", mode);
+      if (mode != "flat" && mode != "positional") {
+	ss << "unrecognized weight-set mode '" << mode << "'";
+	err = -EINVAL;
+	goto reply;
+      }
+      positions = mode == "flat" ? 1 : osdmap.get_pg_pool(pool)->get_size();
+    } else {
+      pool = CrushWrapper::DEFAULT_CHOOSE_ARGS;
+      positions = 1;
+    }
+    if (!newcrush.create_choose_args(pool, positions)) {
+      if (pool == CrushWrapper::DEFAULT_CHOOSE_ARGS) {
+        ss << "compat weight-set already created";
+      } else {
+        ss << "weight-set for pool '" << osdmap.get_pool_name(pool)
+           << "' already created";
+      }
+      goto reply;
+    }
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    goto update;
+
+  } else if (prefix == "osd crush weight-set rm" ||
+	     prefix == "osd crush weight-set rm-compat") {
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+    int64_t pool;
+    if (prefix == "osd crush weight-set rm") {
+      string poolname;
+      cmd_getval(cmdmap, "pool", poolname);
+      pool = osdmap.lookup_pg_pool_name(poolname.c_str());
+      if (pool < 0) {
+	ss << "pool '" << poolname << "' not found";
+	err = -ENOENT;
+	goto reply;
+      }
+    } else {
+      pool = CrushWrapper::DEFAULT_CHOOSE_ARGS;
+    }
+    newcrush.rm_choose_args(pool);
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    goto update;
+
+  } else if (prefix == "osd crush weight-set reweight" ||
+	     prefix == "osd crush weight-set reweight-compat") {
+    string poolname, item;
+    vector<double> weight;
+    cmd_getval(cmdmap, "pool", poolname);
+    cmd_getval(cmdmap, "item", item);
+    cmd_getval(cmdmap, "weight", weight);
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+    int64_t pool;
+    if (prefix == "osd crush weight-set reweight") {
+      pool = osdmap.lookup_pg_pool_name(poolname.c_str());
+      if (pool < 0) {
+	ss << "pool '" << poolname << "' not found";
+	err = -ENOENT;
+	goto reply;
+      }
+      if (!newcrush.have_choose_args(pool)) {
+	ss << "no weight-set for pool '" << poolname << "'";
+	err = -ENOENT;
+	goto reply;
+      }
+      auto arg_map = newcrush.choose_args_get(pool);
+      int positions = newcrush.get_choose_args_positions(arg_map);
+      if (weight.size() != (size_t)positions) {
+         ss << "must specify exact " << positions << " weight values";
+         err = -EINVAL;
+         goto reply;
+      }
+    } else {
+      pool = CrushWrapper::DEFAULT_CHOOSE_ARGS;
+      if (!newcrush.have_choose_args(pool)) {
+	ss << "no backward-compatible weight-set";
+	err = -ENOENT;
+	goto reply;
+      }
+    }
+    if (!newcrush.name_exists(item)) {
+      ss << "item '" << item << "' does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+    err = newcrush.choose_args_adjust_item_weightf(
+      cct,
+      newcrush.choose_args_get(pool),
+      newcrush.get_item_id(item),
+      weight,
+      &ss);
+    if (err < 0) {
+      goto reply;
+    }
+    err = 0;
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    goto update;
+  } else if (osdid_present &&
+	     (prefix == "osd crush set" || prefix == "osd crush add")) {
+    // <OsdName> is 'osd.<id>' or '<id>', passed as int64_t id
+    // osd crush set <OsdName> <weight> <loc1> [<loc2> ...]
+    // osd crush add <OsdName> <weight> <loc1> [<loc2> ...]
+
+    if (!osdmap.exists(osdid)) {
+      err = -ENOENT;
+      ss << osd_name
+	 << " does not exist. Create it before updating the crush map";
+      goto reply;
+    }
+
+    double weight;
+    if (!cmd_getval(cmdmap, "weight", weight)) {
+      ss << "unable to parse weight value '"
+         << cmd_vartype_stringify(cmdmap.at("weight")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    string args;
+    vector<string> argvec;
+    cmd_getval(cmdmap, "args", argvec);
+    map<string,string> loc;
+    CrushWrapper::parse_loc_map(argvec, &loc);
+
+    if (prefix == "osd crush set"
+        && !_get_stable_crush().item_exists(osdid)) {
+      err = -ENOENT;
+      ss << "unable to set item id " << osdid << " name '" << osd_name
+         << "' weight " << weight << " at location " << loc
+         << ": does not exist";
+      goto reply;
+    }
+
+    dout(5) << "adding/updating crush item id " << osdid << " name '"
+      << osd_name << "' weight " << weight << " at location "
+      << loc << dendl;
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    string action;
+    if (prefix == "osd crush set" ||
+        newcrush.check_item_loc(cct, osdid, loc, (int *)NULL)) {
+      action = "set";
+      err = newcrush.update_item(cct, osdid, weight, osd_name, loc);
+    } else {
+      action = "add";
+      err = newcrush.insert_item(cct, osdid, weight, osd_name, loc);
+      if (err == 0)
+        err = 1;
+    }
+
+    if (err < 0)
+      goto reply;
+
+    if (err == 0 && !_have_pending_crush()) {
+      ss << action << " item id " << osdid << " name '" << osd_name
+	 << "' weight " << weight << " at location " << loc << ": no change";
+      goto reply;
+    }
+
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    ss << action << " item id " << osdid << " name '" << osd_name << "' weight "
+       << weight << " at location " << loc << " to crush map";
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd crush create-or-move") {
+    do {
+      // osd crush create-or-move <OsdName> <initial_weight> <loc1> [<loc2> ...]
+      if (!osdmap.exists(osdid)) {
+	err = -ENOENT;
+	ss << osd_name
+	   << " does not exist.  create it before updating the crush map";
+	goto reply;
+      }
+
+      double weight;
+      if (!cmd_getval(cmdmap, "weight", weight)) {
+        ss << "unable to parse weight value '"
+           << cmd_vartype_stringify(cmdmap.at("weight")) << "'";
+        err = -EINVAL;
+        goto reply;
+      }
+
+      string args;
+      vector<string> argvec;
+      cmd_getval(cmdmap, "args", argvec);
+      map<string,string> loc;
+      CrushWrapper::parse_loc_map(argvec, &loc);
+
+      dout(0) << "create-or-move crush item name '" << osd_name
+	      << "' initial_weight " << weight << " at location " << loc
+	      << dendl;
+
+      CrushWrapper newcrush;
+      _get_pending_crush(newcrush);
+
+      err = newcrush.create_or_move_item(cct, osdid, weight, osd_name, loc,
+					 g_conf()->osd_crush_update_weight_set);
+      if (err == 0) {
+	ss << "create-or-move updated item name '" << osd_name
+	   << "' weight " << weight
+	   << " at location " << loc << " to crush map";
+	break;
+      }
+      if (err > 0) {
+	pending_inc.crush.clear();
+	newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+	ss << "create-or-move updating item name '" << osd_name
+	   << "' weight " << weight
+	   << " at location " << loc << " to crush map";
+	getline(ss, rs);
+	wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						  get_last_committed() + 1));
+	return true;
+      }
+    } while (false);
+
+  } else if (prefix == "osd crush move") {
+    do {
+      // osd crush move <name> <loc1> [<loc2> ...]
+      string name;
+      vector<string> argvec;
+      cmd_getval(cmdmap, "name", name);
+      cmd_getval(cmdmap, "args", argvec);
+      map<string,string> loc;
+      CrushWrapper::parse_loc_map(argvec, &loc);
+
+      dout(0) << "moving crush item name '" << name << "' to location " << loc << dendl;
+      CrushWrapper newcrush;
+      _get_pending_crush(newcrush);
+
+      if (!newcrush.name_exists(name)) {
+	err = -ENOENT;
+	ss << "item " << name << " does not exist";
+	break;
+      }
+      int id = newcrush.get_item_id(name);
+
+      if (!newcrush.check_item_loc(cct, id, loc, (int *)NULL)) {
+	if (id >= 0) {
+	  err = newcrush.create_or_move_item(
+	    cct, id, 0, name, loc,
+	    g_conf()->osd_crush_update_weight_set);
+	} else {
+	  err = newcrush.move_bucket(cct, id, loc);
+	}
+	if (err >= 0) {
+	  ss << "moved item id " << id << " name '" << name << "' to location " << loc << " in crush map";
+	  pending_inc.crush.clear();
+	  newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+	  getline(ss, rs);
+	  wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						   get_last_committed() + 1));
+	  return true;
+	}
+      } else {
+	ss << "no need to move item id " << id << " name '" << name << "' to location " << loc << " in crush map";
+	err = 0;
+      }
+    } while (false);
+  } else if (prefix == "osd crush swap-bucket") {
+    string source, dest;
+    cmd_getval(cmdmap, "source", source);
+    cmd_getval(cmdmap, "dest", dest);
+
+    bool force = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", force);
+
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+    if (!newcrush.name_exists(source)) {
+      ss << "source item " << source << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+    if (!newcrush.name_exists(dest)) {
+      ss << "dest item " << dest << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+    int sid = newcrush.get_item_id(source);
+    int did = newcrush.get_item_id(dest);
+    int sparent;
+    if (newcrush.get_immediate_parent_id(sid, &sparent) == 0 && !force) {
+      ss << "source item " << source << " is not an orphan bucket; pass --yes-i-really-mean-it to proceed anyway";
+      err = -EPERM;
+      goto reply;
+    }
+    if (newcrush.get_bucket_alg(sid) != newcrush.get_bucket_alg(did) &&
+	!force) {
+      ss << "source bucket alg " << crush_alg_name(newcrush.get_bucket_alg(sid)) << " != "
+	 << "dest bucket alg " << crush_alg_name(newcrush.get_bucket_alg(did))
+	 << "; pass --yes-i-really-mean-it to proceed anyway";
+      err = -EPERM;
+      goto reply;
+    }
+    int r = newcrush.swap_bucket(cct, sid, did);
+    if (r < 0) {
+      ss << "failed to swap bucket contents: " << cpp_strerror(r);
+      err = r;
+      goto reply;
+    }
+    ss << "swapped bucket of " << source << " to " << dest;
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    wait_for_finished_proposal(op,
+			       new Monitor::C_Command(mon, op, err, ss.str(),
+						      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd crush link") {
+    // osd crush link <name> <loc1> [<loc2> ...]
+    string name;
+    cmd_getval(cmdmap, "name", name);
+    vector<string> argvec;
+    cmd_getval(cmdmap, "args", argvec);
+    map<string,string> loc;
+    CrushWrapper::parse_loc_map(argvec, &loc);
+
+    // Need an explicit check for name_exists because get_item_id returns
+    // 0 on unfound.
+    int id = osdmap.crush->get_item_id(name);
+    if (!osdmap.crush->name_exists(name)) {
+      err = -ENOENT;
+      ss << "item " << name << " does not exist";
+      goto reply;
+    } else {
+      dout(5) << "resolved crush name '" << name << "' to id " << id << dendl;
+    }
+    if (osdmap.crush->check_item_loc(cct, id, loc, (int*) NULL)) {
+      ss << "no need to move item id " << id << " name '" << name
+	 << "' to location " << loc << " in crush map";
+      err = 0;
+      goto reply;
+    }
+
+    dout(5) << "linking crush item name '" << name << "' at location " << loc << dendl;
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    if (!newcrush.name_exists(name)) {
+      err = -ENOENT;
+      ss << "item " << name << " does not exist";
+      goto reply;
+    } else {
+      int id = newcrush.get_item_id(name);
+      if (!newcrush.check_item_loc(cct, id, loc, (int *)NULL)) {
+	err = newcrush.link_bucket(cct, id, loc);
+	if (err >= 0) {
+	  ss << "linked item id " << id << " name '" << name
+             << "' to location " << loc << " in crush map";
+	  pending_inc.crush.clear();
+	  newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+	} else {
+	  ss << "cannot link item id " << id << " name '" << name
+             << "' to location " << loc;
+          goto reply;
+	}
+      } else {
+	ss << "no need to move item id " << id << " name '" << name
+           << "' to location " << loc << " in crush map";
+	err = 0;
+      }
+    }
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, err, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd crush rm" ||
+	     prefix == "osd crush remove" ||
+	     prefix == "osd crush unlink") {
+    do {
+      // osd crush rm <id> [ancestor]
+      CrushWrapper newcrush;
+      _get_pending_crush(newcrush);
+
+      string name;
+      cmd_getval(cmdmap, "name", name);
+
+      if (!osdmap.crush->name_exists(name)) {
+	err = 0;
+	ss << "device '" << name << "' does not appear in the crush map";
+	break;
+      }
+      if (!newcrush.name_exists(name)) {
+	err = 0;
+	ss << "device '" << name << "' does not appear in the crush map";
+	getline(ss, rs);
+	wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						  get_last_committed() + 1));
+	return true;
+      }
+      int id = newcrush.get_item_id(name);
+      int ancestor = 0;
+
+      bool unlink_only = prefix == "osd crush unlink";
+      string ancestor_str;
+      if (cmd_getval(cmdmap, "ancestor", ancestor_str)) {
+	if (!newcrush.name_exists(ancestor_str)) {
+	  err = -ENOENT;
+	  ss << "ancestor item '" << ancestor_str
+	     << "' does not appear in the crush map";
+	  break;
+	}
+        ancestor = newcrush.get_item_id(ancestor_str);
+      }
+
+      err = prepare_command_osd_crush_remove(
+          newcrush,
+          id, ancestor,
+          (ancestor < 0), unlink_only);
+
+      if (err == -ENOENT) {
+	ss << "item " << id << " does not appear in that position";
+	err = 0;
+	break;
+      }
+      if (err == 0) {
+        if (!unlink_only)
+          pending_inc.new_crush_node_flags[id] = 0;
+	ss << "removed item id " << id << " name '" << name << "' from crush map";
+	getline(ss, rs);
+	wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						  get_last_committed() + 1));
+	return true;
+      }
+    } while (false);
+
+  } else if (prefix == "osd crush reweight-all") {
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    newcrush.reweight(cct);
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    ss << "reweighted crush hierarchy";
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						  get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd crush reweight") {
+    // osd crush reweight <name> <weight>
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    string name;
+    cmd_getval(cmdmap, "name", name);
+    if (!newcrush.name_exists(name)) {
+      err = -ENOENT;
+      ss << "device '" << name << "' does not appear in the crush map";
+      goto reply;
+    }
+
+    int id = newcrush.get_item_id(name);
+    if (id < 0) {
+      ss << "device '" << name << "' is not a leaf in the crush map";
+      err = -EINVAL;
+      goto reply;
+    }
+    double w;
+    if (!cmd_getval(cmdmap, "weight", w)) {
+      ss << "unable to parse weight value '"
+	 << cmd_vartype_stringify(cmdmap.at("weight")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    err = newcrush.adjust_item_weightf(cct, id, w,
+				       g_conf()->osd_crush_update_weight_set);
+    if (err < 0)
+      goto reply;
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    ss << "reweighted item id " << id << " name '" << name << "' to " << w
+       << " in crush map";
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						  get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd crush reweight-subtree") {
+    // osd crush reweight <name> <weight>
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    string name;
+    cmd_getval(cmdmap, "name", name);
+    if (!newcrush.name_exists(name)) {
+      err = -ENOENT;
+      ss << "device '" << name << "' does not appear in the crush map";
+      goto reply;
+    }
+
+    int id = newcrush.get_item_id(name);
+    if (id >= 0) {
+      ss << "device '" << name << "' is not a subtree in the crush map";
+      err = -EINVAL;
+      goto reply;
+    }
+    double w;
+    if (!cmd_getval(cmdmap, "weight", w)) {
+      ss << "unable to parse weight value '"
+	 << cmd_vartype_stringify(cmdmap.at("weight")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    err = newcrush.adjust_subtree_weightf(cct, id, w,
+					  g_conf()->osd_crush_update_weight_set);
+    if (err < 0)
+      goto reply;
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    ss << "reweighted subtree id " << id << " name '" << name << "' to " << w
+       << " in crush map";
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd crush tunables") {
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    err = 0;
+    string profile;
+    cmd_getval(cmdmap, "profile", profile);
+    if (profile == "legacy" || profile == "argonaut") {
+      newcrush.set_tunables_legacy();
+    } else if (profile == "bobtail") {
+      newcrush.set_tunables_bobtail();
+    } else if (profile == "firefly") {
+      newcrush.set_tunables_firefly();
+    } else if (profile == "hammer") {
+      newcrush.set_tunables_hammer();
+    } else if (profile == "jewel") {
+      newcrush.set_tunables_jewel();
+    } else if (profile == "optimal") {
+      newcrush.set_tunables_optimal();
+    } else if (profile == "default") {
+      newcrush.set_tunables_default();
+    } else {
+      ss << "unrecognized profile '" << profile << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    if (!validate_crush_against_features(&newcrush, ss)) {
+      err = -EINVAL;
+      goto reply;
+    }
+
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    ss << "adjusted tunables profile to " << profile;
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd crush set-tunable") {
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    err = 0;
+    string tunable;
+    cmd_getval(cmdmap, "tunable", tunable);
+
+    int64_t value = -1;
+    if (!cmd_getval(cmdmap, "value", value)) {
+      err = -EINVAL;
+      ss << "failed to parse integer value "
+	 << cmd_vartype_stringify(cmdmap.at("value"));
+      goto reply;
+    }
+
+    if (tunable == "straw_calc_version") {
+      if (value != 0 && value != 1) {
+	ss << "value must be 0 or 1; got " << value;
+	err = -EINVAL;
+	goto reply;
+      }
+      newcrush.set_straw_calc_version(value);
+    } else {
+      ss << "unrecognized tunable '" << tunable << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    if (!validate_crush_against_features(&newcrush, ss)) {
+      err = -EINVAL;
+      goto reply;
+    }
+
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    ss << "adjusted tunable " << tunable << " to " << value;
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd crush rule create-simple") {
+    string name, root, type, mode;
+    cmd_getval(cmdmap, "name", name);
+    cmd_getval(cmdmap, "root", root);
+    cmd_getval(cmdmap, "type", type);
+    cmd_getval(cmdmap, "mode", mode);
+    if (mode == "")
+      mode = "firstn";
+
+    if (osdmap.crush->rule_exists(name)) {
+      // The name is uniquely associated to a ruleid and the rule it contains
+      // From the user point of view, the rule is more meaningfull.
+      ss << "rule " << name << " already exists";
+      err = 0;
+      goto reply;
+    }
+
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    if (newcrush.rule_exists(name)) {
+      // The name is uniquely associated to a ruleid and the rule it contains
+      // From the user point of view, the rule is more meaningfull.
+      ss << "rule " << name << " already exists";
+      err = 0;
+    } else {
+      int ruleno = newcrush.add_simple_rule(name, root, type, "", mode,
+					       pg_pool_t::TYPE_REPLICATED, &ss);
+      if (ruleno < 0) {
+	err = ruleno;
+	goto reply;
+      }
+
+      pending_inc.crush.clear();
+      newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd crush rule create-replicated") {
+    string name, root, type, device_class;
+    cmd_getval(cmdmap, "name", name);
+    cmd_getval(cmdmap, "root", root);
+    cmd_getval(cmdmap, "type", type);
+    cmd_getval(cmdmap, "class", device_class);
+
+    if (osdmap.crush->rule_exists(name)) {
+      // The name is uniquely associated to a ruleid and the rule it contains
+      // From the user point of view, the rule is more meaningfull.
+      ss << "rule " << name << " already exists";
+      err = 0;
+      goto reply;
+    }
+
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    if (newcrush.rule_exists(name)) {
+      // The name is uniquely associated to a ruleid and the rule it contains
+      // From the user point of view, the rule is more meaningfull.
+      ss << "rule " << name << " already exists";
+      err = 0;
+    } else {
+      int ruleno = newcrush.add_simple_rule(
+	name, root, type, device_class,
+	"firstn", pg_pool_t::TYPE_REPLICATED, &ss);
+      if (ruleno < 0) {
+	err = ruleno;
+	goto reply;
+      }
+
+      pending_inc.crush.clear();
+      newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd erasure-code-profile rm") {
+    string name;
+    cmd_getval(cmdmap, "name", name);
+
+    if (erasure_code_profile_in_use(pending_inc.new_pools, name, &ss))
+      goto wait;
+
+    if (erasure_code_profile_in_use(osdmap.pools, name, &ss)) {
+      err = -EBUSY;
+      goto reply;
+    }
+
+    if (osdmap.has_erasure_code_profile(name) ||
+	pending_inc.new_erasure_code_profiles.count(name)) {
+      if (osdmap.has_erasure_code_profile(name)) {
+	pending_inc.old_erasure_code_profiles.push_back(name);
+      } else {
+	dout(20) << "erasure code profile rm " << name << ": creation canceled" << dendl;
+	pending_inc.new_erasure_code_profiles.erase(name);
+      }
+
+      getline(ss, rs);
+      wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+							get_last_committed() + 1));
+      return true;
+    } else {
+      ss << "erasure-code-profile " << name << " does not exist";
+      err = 0;
+      goto reply;
+    }
+
+  } else if (prefix == "osd erasure-code-profile set") {
+    string name;
+    cmd_getval(cmdmap, "name", name);
+    vector<string> profile;
+    cmd_getval(cmdmap, "profile", profile);
+
+    bool force = false;
+    cmd_getval(cmdmap, "force", force);
+
+    map<string,string> profile_map;
+    err = parse_erasure_code_profile(profile, &profile_map, &ss);
+    if (err)
+      goto reply;
+    if (auto found = profile_map.find("crush-failure-domain");
+	found != profile_map.end()) {
+      const auto& failure_domain = found->second;
+      int failure_domain_type = osdmap.crush->get_type_id(failure_domain);
+      if (failure_domain_type < 0) {
+	ss << "erasure-code-profile " << profile_map
+	  << " contains an invalid failure-domain " << std::quoted(failure_domain);
+	err = -EINVAL;
+	goto reply;
+      }
+    }
+
+    if (profile_map.find("plugin") == profile_map.end()) {
+      ss << "erasure-code-profile " << profile_map
+	 << " must contain a plugin entry" << std::endl;
+      err = -EINVAL;
+      goto reply;
+    }
+    string plugin = profile_map["plugin"];
+
+    if (pending_inc.has_erasure_code_profile(name)) {
+      dout(20) << "erasure code profile " << name << " try again" << dendl;
+      goto wait;
+    } else {
+      err = normalize_profile(name, profile_map, force, &ss);
+      if (err)
+	goto reply;
+
+      if (osdmap.has_erasure_code_profile(name)) {
+	ErasureCodeProfile existing_profile_map =
+	  osdmap.get_erasure_code_profile(name);
+	err = normalize_profile(name, existing_profile_map, force, &ss);
+	if (err)
+	  goto reply;
+
+	if (existing_profile_map == profile_map) {
+	  err = 0;
+	  goto reply;
+	}
+	if (!force) {
+	  err = -EPERM;
+	  ss << "will not override erasure code profile " << name
+	     << " because the existing profile "
+	     << existing_profile_map
+	     << " is different from the proposed profile "
+	     << profile_map;
+	  goto reply;
+	}
+      }
+
+      dout(20) << "erasure code profile set " << name << "="
+	       << profile_map << dendl;
+      pending_inc.set_erasure_code_profile(name, profile_map);
+    }
+
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+                                                      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd crush rule create-erasure") {
+    err = check_cluster_features(CEPH_FEATURE_CRUSH_V2, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err)
+      goto reply;
+    string name, poolstr;
+    cmd_getval(cmdmap, "name", name);
+    string profile;
+    cmd_getval(cmdmap, "profile", profile);
+    if (profile == "")
+      profile = "default";
+    if (profile == "default") {
+      if (!osdmap.has_erasure_code_profile(profile)) {
+	if (pending_inc.has_erasure_code_profile(profile)) {
+	  dout(20) << "erasure code profile " << profile << " already pending" << dendl;
+	  goto wait;
+	}
+
+	map<string,string> profile_map;
+	err = osdmap.get_erasure_code_profile_default(cct,
+						      profile_map,
+						      &ss);
+	if (err)
+	  goto reply;
+	err = normalize_profile(name, profile_map, true, &ss);
+	if (err)
+	  goto reply;
+	dout(20) << "erasure code profile set " << profile << "="
+		 << profile_map << dendl;
+	pending_inc.set_erasure_code_profile(profile, profile_map);
+	goto wait;
+      }
+    }
+
+    int rule;
+    err = crush_rule_create_erasure(name, profile, &rule, &ss);
+    if (err < 0) {
+      switch(err) {
+      case -EEXIST: // return immediately
+	ss << "rule " << name << " already exists";
+	err = 0;
+	goto reply;
+	break;
+      case -EALREADY: // wait for pending to be proposed
+	ss << "rule " << name << " already exists";
+	err = 0;
+	break;
+      default: // non recoverable error
+ 	goto reply;
+	break;
+      }
+    } else {
+      ss << "created rule " << name << " at " << rule;
+    }
+
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+                                                      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd crush rule rm") {
+    string name;
+    cmd_getval(cmdmap, "name", name);
+
+    if (!osdmap.crush->rule_exists(name)) {
+      ss << "rule " << name << " does not exist";
+      err = 0;
+      goto reply;
+    }
+
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+
+    if (!newcrush.rule_exists(name)) {
+      ss << "rule " << name << " does not exist";
+      err = 0;
+    } else {
+      int ruleno = newcrush.get_rule_id(name);
+      ceph_assert(ruleno >= 0);
+
+      // make sure it is not in use.
+      // FIXME: this is ok in some situations, but let's not bother with that
+      // complexity now.
+      int ruleset = newcrush.get_rule_mask_ruleset(ruleno);
+      if (osdmap.crush_rule_in_use(ruleset)) {
+	ss << "crush ruleset " << name << " " << ruleset << " is in use";
+	err = -EBUSY;
+	goto reply;
+      }
+
+      err = newcrush.remove_rule(ruleno);
+      if (err < 0) {
+	goto reply;
+      }
+
+      pending_inc.crush.clear();
+      newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd crush rule rename") {
+    string srcname;
+    string dstname;
+    cmd_getval(cmdmap, "srcname", srcname);
+    cmd_getval(cmdmap, "dstname", dstname);
+    if (srcname.empty() || dstname.empty()) {
+      ss << "must specify both source rule name and destination rule name";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (srcname == dstname) {
+      ss << "destination rule name is equal to source rule name";
+      err = 0;
+      goto reply;
+    }
+
+    CrushWrapper newcrush;
+    _get_pending_crush(newcrush);
+    if (!newcrush.rule_exists(srcname) && newcrush.rule_exists(dstname)) {
+      // srcname does not exist and dstname already exists
+      // suppose this is a replay and return success
+      // (so this command is idempotent)
+      ss << "already renamed to '" << dstname << "'";
+      err = 0;
+      goto reply;
+    }
+
+    err = newcrush.rename_rule(srcname, dstname, &ss);
+    if (err < 0) {
+      // ss has reason for failure
+      goto reply;
+    }
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+                               get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd setmaxosd") {
+    int64_t newmax;
+    if (!cmd_getval(cmdmap, "newmax", newmax)) {
+      ss << "unable to parse 'newmax' value '"
+         << cmd_vartype_stringify(cmdmap.at("newmax")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    if (newmax > g_conf()->mon_max_osd) {
+      err = -ERANGE;
+      ss << "cannot set max_osd to " << newmax << " which is > conf.mon_max_osd ("
+	 << g_conf()->mon_max_osd << ")";
+      goto reply;
+    }
+
+    // Don't allow shrinking OSD number as this will cause data loss
+    // and may cause kernel crashes.
+    // Note: setmaxosd sets the maximum OSD number and not the number of OSDs
+    if (newmax < osdmap.get_max_osd()) {
+      // Check if the OSDs exist between current max and new value.
+      // If there are any OSDs exist, then don't allow shrinking number
+      // of OSDs.
+      for (int i = newmax; i < osdmap.get_max_osd(); i++) {
+        if (osdmap.exists(i)) {
+          err = -EBUSY;
+          ss << "cannot shrink max_osd to " << newmax
+             << " because osd." << i << " (and possibly others) still in use";
+          goto reply;
+        }
+      }
+    }
+
+    pending_inc.new_max_osd = newmax;
+    ss << "set new max_osd = " << pending_inc.new_max_osd;
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd set-full-ratio" ||
+	     prefix == "osd set-backfillfull-ratio" ||
+             prefix == "osd set-nearfull-ratio") {
+    double n;
+    if (!cmd_getval(cmdmap, "ratio", n)) {
+      ss << "unable to parse 'ratio' value '"
+         << cmd_vartype_stringify(cmdmap.at("ratio")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (prefix == "osd set-full-ratio")
+      pending_inc.new_full_ratio = n;
+    else if (prefix == "osd set-backfillfull-ratio")
+      pending_inc.new_backfillfull_ratio = n;
+    else if (prefix == "osd set-nearfull-ratio")
+      pending_inc.new_nearfull_ratio = n;
+    ss << prefix << " " << n;
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd set-require-min-compat-client") {
+    string v;
+    cmd_getval(cmdmap, "version", v);
+    ceph_release_t vno = ceph_release_from_name(v);
+    if (!vno) {
+      ss << "version " << v << " is not recognized";
+      err = -EINVAL;
+      goto reply;
+    }
+    OSDMap newmap;
+    newmap.deepish_copy_from(osdmap);
+    newmap.apply_incremental(pending_inc);
+    newmap.require_min_compat_client = vno;
+    auto mvno = newmap.get_min_compat_client();
+    if (vno < mvno) {
+      ss << "osdmap current utilizes features that require " << mvno
+	 << "; cannot set require_min_compat_client below that to " << vno;
+      err = -EPERM;
+      goto reply;
+    }
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+    if (!sure) {
+      FeatureMap m;
+      mon.get_combined_feature_map(&m);
+      uint64_t features = ceph_release_features(to_integer<int>(vno));
+      bool first = true;
+      bool ok = true;
+      for (int type : {
+	    CEPH_ENTITY_TYPE_CLIENT,
+	    CEPH_ENTITY_TYPE_MDS,
+	    CEPH_ENTITY_TYPE_MGR }) {
+	auto p = m.m.find(type);
+	if (p == m.m.end()) {
+	  continue;
+	}
+	for (auto& q : p->second) {
+	  uint64_t missing = ~q.first & features;
+	  if (missing) {
+	    if (first) {
+	      ss << "cannot set require_min_compat_client to " << v << ": ";
+	    } else {
+	      ss << "; ";
+	    }
+	    first = false;
+	    ss << q.second << " connected " << ceph_entity_type_name(type)
+	       << "(s) look like " << ceph_release_name(
+		 ceph_release_from_features(q.first))
+	       << " (missing 0x" << std::hex << missing << std::dec << ")";
+	    ok = false;
+	  }
+	}
+      }
+      if (!ok) {
+	ss << "; add --yes-i-really-mean-it to do it anyway";
+	err = -EPERM;
+	goto reply;
+      }
+    }
+    ss << "set require_min_compat_client to " << vno;
+    pending_inc.new_require_min_compat_client = vno;
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+							  get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd pause") {
+    return prepare_set_flag(op, CEPH_OSDMAP_PAUSERD | CEPH_OSDMAP_PAUSEWR);
+
+  } else if (prefix == "osd unpause") {
+    return prepare_unset_flag(op, CEPH_OSDMAP_PAUSERD | CEPH_OSDMAP_PAUSEWR);
+
+  } else if (prefix == "osd set") {
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+
+    string key;
+    cmd_getval(cmdmap, "key", key);
+    if (key == "pause")
+      return prepare_set_flag(op, CEPH_OSDMAP_PAUSERD | CEPH_OSDMAP_PAUSEWR);
+    else if (key == "noup")
+      return prepare_set_flag(op, CEPH_OSDMAP_NOUP);
+    else if (key == "nodown")
+      return prepare_set_flag(op, CEPH_OSDMAP_NODOWN);
+    else if (key == "noout")
+      return prepare_set_flag(op, CEPH_OSDMAP_NOOUT);
+    else if (key == "noin")
+      return prepare_set_flag(op, CEPH_OSDMAP_NOIN);
+    else if (key == "nobackfill")
+      return prepare_set_flag(op, CEPH_OSDMAP_NOBACKFILL);
+    else if (key == "norebalance")
+      return prepare_set_flag(op, CEPH_OSDMAP_NOREBALANCE);
+    else if (key == "norecover")
+      return prepare_set_flag(op, CEPH_OSDMAP_NORECOVER);
+    else if (key == "noscrub")
+      return prepare_set_flag(op, CEPH_OSDMAP_NOSCRUB);
+    else if (key == "nodeep-scrub")
+      return prepare_set_flag(op, CEPH_OSDMAP_NODEEP_SCRUB);
+    else if (key == "notieragent")
+      return prepare_set_flag(op, CEPH_OSDMAP_NOTIERAGENT);
+    else if (key == "nosnaptrim")
+      return prepare_set_flag(op, CEPH_OSDMAP_NOSNAPTRIM);
+    else if (key == "pglog_hardlimit") {
+      if (!osdmap.get_num_up_osds() && !sure) {
+        ss << "Not advisable to continue since no OSDs are up. Pass "
+           << "--yes-i-really-mean-it if you really wish to continue.";
+        err = -EPERM;
+        goto reply;
+      }
+      // The release check here is required because for OSD_PGLOG_HARDLIMIT,
+      // we are reusing a jewel feature bit that was retired in luminous.
+      if (osdmap.require_osd_release >= ceph_release_t::luminous &&
+         (HAVE_FEATURE(osdmap.get_up_osd_features(), OSD_PGLOG_HARDLIMIT)
+          || sure)) {
+	return prepare_set_flag(op, CEPH_OSDMAP_PGLOG_HARDLIMIT);
+      } else {
+	ss << "not all up OSDs have OSD_PGLOG_HARDLIMIT feature";
+	err = -EPERM;
+	goto reply;
+      }
+    } else {
+      ss << "unrecognized flag '" << key << "'";
+      err = -EINVAL;
+    }
+
+  } else if (prefix == "osd unset") {
+    string key;
+    cmd_getval(cmdmap, "key", key);
+    if (key == "pause")
+      return prepare_unset_flag(op, CEPH_OSDMAP_PAUSERD | CEPH_OSDMAP_PAUSEWR);
+    else if (key == "noup")
+      return prepare_unset_flag(op, CEPH_OSDMAP_NOUP);
+    else if (key == "nodown")
+      return prepare_unset_flag(op, CEPH_OSDMAP_NODOWN);
+    else if (key == "noout")
+      return prepare_unset_flag(op, CEPH_OSDMAP_NOOUT);
+    else if (key == "noin")
+      return prepare_unset_flag(op, CEPH_OSDMAP_NOIN);
+    else if (key == "nobackfill")
+      return prepare_unset_flag(op, CEPH_OSDMAP_NOBACKFILL);
+    else if (key == "norebalance")
+      return prepare_unset_flag(op, CEPH_OSDMAP_NOREBALANCE);
+    else if (key == "norecover")
+      return prepare_unset_flag(op, CEPH_OSDMAP_NORECOVER);
+    else if (key == "noscrub")
+      return prepare_unset_flag(op, CEPH_OSDMAP_NOSCRUB);
+    else if (key == "nodeep-scrub")
+      return prepare_unset_flag(op, CEPH_OSDMAP_NODEEP_SCRUB);
+    else if (key == "notieragent")
+      return prepare_unset_flag(op, CEPH_OSDMAP_NOTIERAGENT);
+    else if (key == "nosnaptrim")
+      return prepare_unset_flag(op, CEPH_OSDMAP_NOSNAPTRIM);
+    else {
+      ss << "unrecognized flag '" << key << "'";
+      err = -EINVAL;
+    }
+
+  } else if (prefix == "osd require-osd-release") {
+    string release;
+    cmd_getval(cmdmap, "release", release);
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+    ceph_release_t rel = ceph_release_from_name(release.c_str());
+    if (!rel) {
+      ss << "unrecognized release " << release;
+      err = -EINVAL;
+      goto reply;
+    }
+    if (rel == osdmap.require_osd_release) {
+      // idempotent
+      err = 0;
+      goto reply;
+    }
+    ceph_assert(osdmap.require_osd_release >= ceph_release_t::luminous);
+    if (!osdmap.get_num_up_osds() && !sure) {
+      ss << "Not advisable to continue since no OSDs are up. Pass "
+	 << "--yes-i-really-mean-it if you really wish to continue.";
+      err = -EPERM;
+      goto reply;
+    }
+    if (rel == ceph_release_t::mimic) {
+      if (!mon.monmap->get_required_features().contains_all(
+	    ceph::features::mon::FEATURE_MIMIC)) {
+	ss << "not all mons are mimic";
+	err = -EPERM;
+	goto reply;
+      }
+      if ((!HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_MIMIC))
+           && !sure) {
+	ss << "not all up OSDs have CEPH_FEATURE_SERVER_MIMIC feature";
+	err = -EPERM;
+	goto reply;
+      }
+    } else if (rel == ceph_release_t::nautilus) {
+      if (!mon.monmap->get_required_features().contains_all(
+	    ceph::features::mon::FEATURE_NAUTILUS)) {
+	ss << "not all mons are nautilus";
+	err = -EPERM;
+	goto reply;
+      }
+      if ((!HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_NAUTILUS))
+           && !sure) {
+	ss << "not all up OSDs have CEPH_FEATURE_SERVER_NAUTILUS feature";
+	err = -EPERM;
+	goto reply;
+      }
+    } else if (rel == ceph_release_t::octopus) {
+      if (!mon.monmap->get_required_features().contains_all(
+	    ceph::features::mon::FEATURE_OCTOPUS)) {
+	ss << "not all mons are octopus";
+	err = -EPERM;
+	goto reply;
+      }
+      if ((!HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_OCTOPUS))
+           && !sure) {
+	ss << "not all up OSDs have CEPH_FEATURE_SERVER_OCTOPUS feature";
+	err = -EPERM;
+	goto reply;
+      }
+    } else if (rel == ceph_release_t::pacific) {
+      if (!mon.monmap->get_required_features().contains_all(
+	    ceph::features::mon::FEATURE_PACIFIC)) {
+	ss << "not all mons are pacific";
+	err = -EPERM;
+	goto reply;
+      }
+      if ((!HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_PACIFIC))
+           && !sure) {
+	ss << "not all up OSDs have CEPH_FEATURE_SERVER_PACIFIC feature";
+	err = -EPERM;
+	goto reply;
+      }
+    } else {
+      ss << "not supported for this release yet";
+      err = -EPERM;
+      goto reply;
+    }
+    if (rel < osdmap.require_osd_release) {
+      ss << "require_osd_release cannot be lowered once it has been set";
+      err = -EPERM;
+      goto reply;
+    }
+    pending_inc.new_require_osd_release = rel;
+    goto update;
+  } else if (prefix == "osd down" ||
+             prefix == "osd out" ||
+             prefix == "osd in" ||
+             prefix == "osd rm" ||
+             prefix == "osd stop") {
+
+    bool any = false;
+    bool stop = false;
+    bool verbose = true;
+    bool definitely_dead = false;
+
+    vector<string> idvec;
+    cmd_getval(cmdmap, "ids", idvec);
+    cmd_getval(cmdmap, "definitely_dead", definitely_dead);
+    derr << "definitely_dead " << (int)definitely_dead << dendl;
+    for (unsigned j = 0; j < idvec.size() && !stop; j++) {
+      set<int> osds;
+
+      // wildcard?
+      if (j == 0 &&
+          (idvec[0] == "any" || idvec[0] == "all" || idvec[0] == "*")) {
+        if (prefix == "osd in") {
+          // touch out osds only
+          osdmap.get_out_existing_osds(osds);
+        } else {
+          osdmap.get_all_osds(osds);
+        }
+        stop = true;
+        verbose = false; // so the output is less noisy.
+      } else {
+        long osd = parse_osd_id(idvec[j].c_str(), &ss);
+        if (osd < 0) {
+          ss << "invalid osd id" << osd;
+          err = -EINVAL;
+          continue;
+        } else if (!osdmap.exists(osd)) {
+          ss << "osd." << osd << " does not exist. ";
+          continue;
+        }
+
+        osds.insert(osd);
+      }
+
+      for (auto &osd : osds) {
+        if (prefix == "osd down") {
+	  if (osdmap.is_down(osd)) {
+            if (verbose)
+	      ss << "osd." << osd << " is already down. ";
+	  } else {
+            pending_inc.pending_osd_state_set(osd, CEPH_OSD_UP);
+	    ss << "marked down osd." << osd << ". ";
+	    any = true;
+	  }
+	  if (definitely_dead) {
+	    if (!pending_inc.new_xinfo.count(osd)) {
+	      pending_inc.new_xinfo[osd] = osdmap.osd_xinfo[osd];
+	    }
+	    if (pending_inc.new_xinfo[osd].dead_epoch < pending_inc.epoch) {
+	      any = true;
+	    }
+	    pending_inc.new_xinfo[osd].dead_epoch = pending_inc.epoch;
+	  }
+        } else if (prefix == "osd out") {
+	  if (osdmap.is_out(osd)) {
+            if (verbose)
+	      ss << "osd." << osd << " is already out. ";
+	  } else {
+	    pending_inc.new_weight[osd] = CEPH_OSD_OUT;
+	    if (osdmap.osd_weight[osd]) {
+	      if (pending_inc.new_xinfo.count(osd) == 0) {
+	        pending_inc.new_xinfo[osd] = osdmap.osd_xinfo[osd];
+	      }
+	      pending_inc.new_xinfo[osd].old_weight = osdmap.osd_weight[osd];
+	    }
+	    ss << "marked out osd." << osd << ". ";
+            std::ostringstream msg;
+            msg << "Client " << op->get_session()->entity_name
+                << " marked osd." << osd << " out";
+            if (osdmap.is_up(osd)) {
+              msg << ", while it was still marked up";
+            } else {
+              auto period = ceph_clock_now() - down_pending_out[osd];
+              msg << ", after it was down for " << int(period.sec())
+                  << " seconds";
+            }
+
+            mon.clog->info() << msg.str();
+	    any = true;
+	  }
+        } else if (prefix == "osd in") {
+	  if (osdmap.is_in(osd)) {
+            if (verbose)
+	      ss << "osd." << osd << " is already in. ";
+	  } else {
+	    if (osdmap.osd_xinfo[osd].old_weight > 0) {
+	      pending_inc.new_weight[osd] = osdmap.osd_xinfo[osd].old_weight;
+	      if (pending_inc.new_xinfo.count(osd) == 0) {
+	        pending_inc.new_xinfo[osd] = osdmap.osd_xinfo[osd];
+	      }
+	      pending_inc.new_xinfo[osd].old_weight = 0;
+	    } else {
+	      pending_inc.new_weight[osd] = CEPH_OSD_IN;
+	    }
+	    ss << "marked in osd." << osd << ". ";
+	    any = true;
+	  }
+        } else if (prefix == "osd rm") {
+          err = prepare_command_osd_remove(osd);
+
+          if (err == -EBUSY) {
+	    if (any)
+	      ss << ", ";
+            ss << "osd." << osd << " is still up; must be down before removal. ";
+	  } else {
+            ceph_assert(err == 0);
+	    if (any) {
+	      ss << ", osd." << osd;
+            } else {
+	      ss << "removed osd." << osd;
+            }
+	    any = true;
+	  }
+        } else if (prefix == "osd stop") {
+          if (osdmap.is_stop(osd)) {
+            if (verbose)
+              ss << "osd." << osd << " is already stopped. ";
+          } else if (osdmap.is_down(osd)) {
+            pending_inc.pending_osd_state_set(osd, CEPH_OSD_STOP);
+            ss << "stop down osd." << osd << ". ";
+            any = true;
+          } else {
+            pending_inc.pending_osd_state_set(osd, CEPH_OSD_UP | CEPH_OSD_STOP);
+            ss << "stop osd." << osd << ". ";
+            any = true;
+          }
+        }
+      }
+    }
+    if (any) {
+      getline(ss, rs);
+      wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, err, rs,
+						get_last_committed() + 1));
+      return true;
+    }
+  } else if (prefix == "osd set-group" ||
+             prefix == "osd unset-group" ||
+             prefix == "osd add-noup" ||
+             prefix == "osd add-nodown" ||
+             prefix == "osd add-noin" ||
+             prefix == "osd add-noout" ||
+             prefix == "osd rm-noup" ||
+             prefix == "osd rm-nodown" ||
+             prefix == "osd rm-noin" ||
+             prefix == "osd rm-noout") {
+    bool do_set = prefix == "osd set-group" ||
+                  prefix.find("add") != string::npos;
+    string flag_str;
+    unsigned flags = 0;
+    vector<string> who;
+    if (prefix == "osd set-group" || prefix == "osd unset-group") {
+      cmd_getval(cmdmap, "flags", flag_str);
+      cmd_getval(cmdmap, "who", who);
+      vector<string> raw_flags;
+      boost::split(raw_flags, flag_str, boost::is_any_of(","));
+      for (auto& f : raw_flags) {
+        if (f == "noup")
+          flags |= CEPH_OSD_NOUP;
+        else if (f == "nodown")
+          flags |= CEPH_OSD_NODOWN;
+        else if (f == "noin")
+          flags |= CEPH_OSD_NOIN;
+        else if (f == "noout")
+          flags |= CEPH_OSD_NOOUT;
+        else {
+          ss << "unrecognized flag '" << f << "', must be one of "
+             << "{noup,nodown,noin,noout}";
+          err = -EINVAL;
+          goto reply;
+        }
+      }
+    } else {
+      cmd_getval(cmdmap, "ids", who);
+      if (prefix.find("noup") != string::npos)
+        flags = CEPH_OSD_NOUP;
+      else if (prefix.find("nodown") != string::npos)
+        flags = CEPH_OSD_NODOWN;
+      else if (prefix.find("noin") != string::npos)
+        flags = CEPH_OSD_NOIN;
+      else if (prefix.find("noout") != string::npos)
+        flags = CEPH_OSD_NOOUT;
+      else
+        ceph_assert(0 == "Unreachable!");
+    }
+    if (flags == 0) {
+      ss << "must specify flag(s) {noup,nodwon,noin,noout} to set/unset";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (who.empty()) {
+      ss << "must specify at least one or more targets to set/unset";
+      err = -EINVAL;
+      goto reply;
+    }
+    set<int> osds;
+    set<int> crush_nodes;
+    set<int> device_classes;
+    for (auto& w : who) {
+      if (w == "any" || w == "all" || w == "*") {
+        osdmap.get_all_osds(osds);
+        break;
+      }
+      std::stringstream ts;
+      if (auto osd = parse_osd_id(w.c_str(), &ts); osd >= 0) {
+        osds.insert(osd);
+      } else if (osdmap.crush->name_exists(w)) {
+        crush_nodes.insert(osdmap.crush->get_item_id(w));
+      } else if (osdmap.crush->class_exists(w)) {
+        device_classes.insert(osdmap.crush->get_class_id(w));
+      } else {
+        ss << "unable to parse osd id or crush node or device class: "
+           << "\"" << w << "\". ";
+      }
+    }
+    if (osds.empty() && crush_nodes.empty() && device_classes.empty()) {
+      // ss has reason for failure
+      err = -EINVAL;
+      goto reply;
+    }
+    bool any = false;
+    for (auto osd : osds) {
+      if (!osdmap.exists(osd)) {
+        ss << "osd." << osd << " does not exist. ";
+        continue;
+      }
+      if (do_set) {
+        if (flags & CEPH_OSD_NOUP) {
+          any |= osdmap.is_noup_by_osd(osd) ?
+            pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOUP) :
+            pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOUP);
+        }
+        if (flags & CEPH_OSD_NODOWN) {
+          any |= osdmap.is_nodown_by_osd(osd) ?
+            pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NODOWN) :
+            pending_inc.pending_osd_state_set(osd, CEPH_OSD_NODOWN);
+        }
+        if (flags & CEPH_OSD_NOIN) {
+          any |= osdmap.is_noin_by_osd(osd) ?
+            pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOIN) :
+            pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOIN);
+        }
+        if (flags & CEPH_OSD_NOOUT) {
+          any |= osdmap.is_noout_by_osd(osd) ?
+            pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOOUT) :
+            pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOOUT);
+        }
+      } else {
+        if (flags & CEPH_OSD_NOUP) {
+          any |= osdmap.is_noup_by_osd(osd) ?
+            pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOUP) :
+            pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOUP);
+        }
+        if (flags & CEPH_OSD_NODOWN) {
+          any |= osdmap.is_nodown_by_osd(osd) ?
+            pending_inc.pending_osd_state_set(osd, CEPH_OSD_NODOWN) :
+            pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NODOWN);
+        }
+        if (flags & CEPH_OSD_NOIN) {
+          any |= osdmap.is_noin_by_osd(osd) ?
+            pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOIN) :
+            pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOIN);
+        }
+        if (flags & CEPH_OSD_NOOUT) {
+          any |= osdmap.is_noout_by_osd(osd) ?
+            pending_inc.pending_osd_state_set(osd, CEPH_OSD_NOOUT) :
+            pending_inc.pending_osd_state_clear(osd, CEPH_OSD_NOOUT);
+        }
+      }
+    }
+    for (auto& id : crush_nodes) {
+      auto old_flags = osdmap.get_crush_node_flags(id);
+      auto& pending_flags = pending_inc.new_crush_node_flags[id];
+      pending_flags |= old_flags; // adopt existing flags first!
+      if (do_set) {
+        pending_flags |= flags;
+      } else {
+        pending_flags &= ~flags;
+      }
+      any = true;
+    }
+    for (auto& id : device_classes) {
+      auto old_flags = osdmap.get_device_class_flags(id);
+      auto& pending_flags = pending_inc.new_device_class_flags[id];
+      pending_flags |= old_flags;
+      if (do_set) {
+        pending_flags |= flags;
+      } else {
+        pending_flags &= ~flags;
+      }
+      any = true;
+    }
+    if (any) {
+      getline(ss, rs);
+      wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, err, rs,
+                                 get_last_committed() + 1));
+      return true;
+    }
+  } else if (prefix == "osd pg-temp") {
+    string pgidstr;
+    if (!cmd_getval(cmdmap, "pgid", pgidstr)) {
+      ss << "unable to parse 'pgid' value '"
+         << cmd_vartype_stringify(cmdmap.at("pgid")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    pg_t pgid;
+    if (!pgid.parse(pgidstr.c_str())) {
+      ss << "invalid pgid '" << pgidstr << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (!osdmap.pg_exists(pgid)) {
+      ss << "pg " << pgid << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+    if (pending_inc.new_pg_temp.count(pgid)) {
+      dout(10) << __func__ << " waiting for pending update on " << pgid << dendl;
+      wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+      return true;
+    }
+
+    vector<int64_t> id_vec;
+    vector<int32_t> new_pg_temp;
+    cmd_getval(cmdmap, "id", id_vec);
+    if (id_vec.empty())  {
+      pending_inc.new_pg_temp[pgid] = mempool::osdmap::vector<int>();
+      ss << "done cleaning up pg_temp of " << pgid;
+      goto update;
+    }
+    for (auto osd : id_vec) {
+      if (!osdmap.exists(osd)) {
+        ss << "osd." << osd << " does not exist";
+        err = -ENOENT;
+        goto reply;
+      }
+      new_pg_temp.push_back(osd);
+    }
+
+    int pool_min_size = osdmap.get_pg_pool_min_size(pgid);
+    if ((int)new_pg_temp.size() < pool_min_size) {
+      ss << "num of osds (" << new_pg_temp.size() <<") < pool min size ("
+         << pool_min_size << ")";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    int pool_size = osdmap.get_pg_pool_size(pgid);
+    if ((int)new_pg_temp.size() > pool_size) {
+      ss << "num of osds (" << new_pg_temp.size() <<") > pool size ("
+         << pool_size << ")";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    pending_inc.new_pg_temp[pgid] = mempool::osdmap::vector<int>(
+      new_pg_temp.begin(), new_pg_temp.end());
+    ss << "set " << pgid << " pg_temp mapping to " << new_pg_temp;
+    goto update;
+  } else if (prefix == "osd primary-temp") {
+    string pgidstr;
+    if (!cmd_getval(cmdmap, "pgid", pgidstr)) {
+      ss << "unable to parse 'pgid' value '"
+         << cmd_vartype_stringify(cmdmap.at("pgid")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    pg_t pgid;
+    if (!pgid.parse(pgidstr.c_str())) {
+      ss << "invalid pgid '" << pgidstr << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (!osdmap.pg_exists(pgid)) {
+      ss << "pg " << pgid << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+
+    int64_t osd;
+    if (!cmd_getval(cmdmap, "id", osd)) {
+      ss << "unable to parse 'id' value '"
+         << cmd_vartype_stringify(cmdmap.at("id")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (osd != -1 && !osdmap.exists(osd)) {
+      ss << "osd." << osd << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+
+    if (osdmap.require_min_compat_client != ceph_release_t::unknown &&
+	osdmap.require_min_compat_client < ceph_release_t::firefly) {
+      ss << "require_min_compat_client "
+	 << osdmap.require_min_compat_client
+	 << " < firefly, which is required for primary-temp";
+      err = -EPERM;
+      goto reply;
+    }
+
+    pending_inc.new_primary_temp[pgid] = osd;
+    ss << "set " << pgid << " primary_temp mapping to " << osd;
+    goto update;
+  } else if (prefix == "pg repeer") {
+    pg_t pgid;
+    string pgidstr;
+    cmd_getval(cmdmap, "pgid", pgidstr);
+    if (!pgid.parse(pgidstr.c_str())) {
+      ss << "invalid pgid '" << pgidstr << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (!osdmap.pg_exists(pgid)) {
+      ss << "pg '" << pgidstr << "' does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+    vector<int> acting;
+    int primary;
+    osdmap.pg_to_acting_osds(pgid, &acting, &primary);
+    if (primary < 0) {
+      err = -EAGAIN;
+      ss << "pg currently has no primary";
+      goto reply;
+    }
+    if (acting.size() > 1) {
+      // map to just primary; it will map back to what it wants
+      pending_inc.new_pg_temp[pgid] = { primary };
+    } else {
+      // hmm, pick another arbitrary osd to induce a change.  Note
+      // that this won't work if there is only one suitable OSD in the cluster.
+      int i;
+      bool done = false;
+      for (i = 0; i < osdmap.get_max_osd(); ++i) {
+	if (i == primary || !osdmap.is_up(i) || !osdmap.exists(i)) {
+	  continue;
+	}
+	pending_inc.new_pg_temp[pgid] = { primary, i };
+	done = true;
+	break;
+      }
+      if (!done) {
+	err = -EAGAIN;
+	ss << "not enough up OSDs in the cluster to force repeer";
+	goto reply;
+      }
+    }
+    goto update;
+  } else if (prefix == "osd pg-upmap" ||
+             prefix == "osd rm-pg-upmap" ||
+             prefix == "osd pg-upmap-items" ||
+             prefix == "osd rm-pg-upmap-items") {
+    if (osdmap.require_min_compat_client < ceph_release_t::luminous) {
+      ss << "min_compat_client "
+	 << osdmap.require_min_compat_client
+	 << " < luminous, which is required for pg-upmap. "
+         << "Try 'ceph osd set-require-min-compat-client luminous' "
+         << "before using the new interface";
+      err = -EPERM;
+      goto reply;
+    }
+    err = check_cluster_features(CEPH_FEATUREMASK_OSDMAP_PG_UPMAP, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err < 0)
+      goto reply;
+    string pgidstr;
+    if (!cmd_getval(cmdmap, "pgid", pgidstr)) {
+      ss << "unable to parse 'pgid' value '"
+         << cmd_vartype_stringify(cmdmap.at("pgid")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    pg_t pgid;
+    if (!pgid.parse(pgidstr.c_str())) {
+      ss << "invalid pgid '" << pgidstr << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (!osdmap.pg_exists(pgid)) {
+      ss << "pg " << pgid << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+    if (pending_inc.old_pools.count(pgid.pool())) {
+      ss << "pool of " << pgid << " is pending removal";
+      err = -ENOENT;
+      getline(ss, rs);
+      wait_for_finished_proposal(op,
+        new Monitor::C_Command(mon, op, err, rs, get_last_committed() + 1));
+      return true;
+    }
+
+    enum {
+      OP_PG_UPMAP,
+      OP_RM_PG_UPMAP,
+      OP_PG_UPMAP_ITEMS,
+      OP_RM_PG_UPMAP_ITEMS,
+    } option;
+
+    if (prefix == "osd pg-upmap") {
+      option = OP_PG_UPMAP;
+    } else if (prefix == "osd rm-pg-upmap") {
+      option = OP_RM_PG_UPMAP;
+    } else if (prefix == "osd pg-upmap-items") {
+      option = OP_PG_UPMAP_ITEMS;
+    } else {
+      option = OP_RM_PG_UPMAP_ITEMS;
+    }
+
+    // check pending upmap changes
+    switch (option) {
+    case OP_PG_UPMAP: // fall through
+    case OP_RM_PG_UPMAP:
+      if (pending_inc.new_pg_upmap.count(pgid) ||
+          pending_inc.old_pg_upmap.count(pgid)) {
+        dout(10) << __func__ << " waiting for pending update on "
+                 << pgid << dendl;
+        wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+        return true;
+      }
+      break;
+
+    case OP_PG_UPMAP_ITEMS: // fall through
+    case OP_RM_PG_UPMAP_ITEMS:
+      if (pending_inc.new_pg_upmap_items.count(pgid) ||
+          pending_inc.old_pg_upmap_items.count(pgid)) {
+        dout(10) << __func__ << " waiting for pending update on "
+                 << pgid << dendl;
+        wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+        return true;
+      }
+      break;
+
+    default:
+      ceph_abort_msg("invalid option");
+    }
+
+    switch (option) {
+    case OP_PG_UPMAP:
+      {
+        vector<int64_t> id_vec;
+        if (!cmd_getval(cmdmap, "id", id_vec)) {
+          ss << "unable to parse 'id' value(s) '"
+             << cmd_vartype_stringify(cmdmap.at("id")) << "'";
+          err = -EINVAL;
+          goto reply;
+        }
+
+        int pool_min_size = osdmap.get_pg_pool_min_size(pgid);
+        if ((int)id_vec.size() < pool_min_size) {
+          ss << "num of osds (" << id_vec.size() <<") < pool min size ("
+             << pool_min_size << ")";
+          err = -EINVAL;
+          goto reply;
+        }
+
+        int pool_size = osdmap.get_pg_pool_size(pgid);
+        if ((int)id_vec.size() > pool_size) {
+          ss << "num of osds (" << id_vec.size() <<") > pool size ("
+             << pool_size << ")";
+          err = -EINVAL;
+          goto reply;
+        }
+
+        vector<int32_t> new_pg_upmap;
+        for (auto osd : id_vec) {
+          if (osd != CRUSH_ITEM_NONE && !osdmap.exists(osd)) {
+            ss << "osd." << osd << " does not exist";
+            err = -ENOENT;
+            goto reply;
+          }
+          auto it = std::find(new_pg_upmap.begin(), new_pg_upmap.end(), osd);
+          if (it != new_pg_upmap.end()) {
+            ss << "osd." << osd << " already exists, ";
+            continue;
+          }
+          new_pg_upmap.push_back(osd);
+        }
+
+        if (new_pg_upmap.empty()) {
+          ss << "no valid upmap items(pairs) is specified";
+          err = -EINVAL;
+          goto reply;
+        }
+
+        pending_inc.new_pg_upmap[pgid] = mempool::osdmap::vector<int32_t>(
+          new_pg_upmap.begin(), new_pg_upmap.end());
+        ss << "set " << pgid << " pg_upmap mapping to " << new_pg_upmap;
+      }
+      break;
+
+    case OP_RM_PG_UPMAP:
+      {
+        pending_inc.old_pg_upmap.insert(pgid);
+        ss << "clear " << pgid << " pg_upmap mapping";
+      }
+      break;
+
+    case OP_PG_UPMAP_ITEMS:
+      {
+        vector<int64_t> id_vec;
+        if (!cmd_getval(cmdmap, "id", id_vec)) {
+          ss << "unable to parse 'id' value(s) '"
+             << cmd_vartype_stringify(cmdmap.at("id")) << "'";
+          err = -EINVAL;
+          goto reply;
+        }
+
+        if (id_vec.size() % 2) {
+          ss << "you must specify pairs of osd ids to be remapped";
+          err = -EINVAL;
+          goto reply;
+        }
+
+        int pool_size = osdmap.get_pg_pool_size(pgid);
+        if ((int)(id_vec.size() / 2) > pool_size) {
+          ss << "num of osd pairs (" << id_vec.size() / 2 <<") > pool size ("
+             << pool_size << ")";
+          err = -EINVAL;
+          goto reply;
+        }
+
+        vector<pair<int32_t,int32_t>> new_pg_upmap_items;
+        ostringstream items;
+        items << "[";
+        for (auto p = id_vec.begin(); p != id_vec.end(); ++p) {
+          int from = *p++;
+          int to = *p;
+          if (from == to) {
+            ss << "from osd." << from << " == to osd." << to << ", ";
+            continue;
+          }
+          if (!osdmap.exists(from)) {
+            ss << "osd." << from << " does not exist";
+            err = -ENOENT;
+            goto reply;
+          }
+          if (to != CRUSH_ITEM_NONE && !osdmap.exists(to)) {
+            ss << "osd." << to << " does not exist";
+            err = -ENOENT;
+            goto reply;
+          }
+          pair<int32_t,int32_t> entry = make_pair(from, to);
+          auto it = std::find(new_pg_upmap_items.begin(),
+            new_pg_upmap_items.end(), entry);
+          if (it != new_pg_upmap_items.end()) {
+            ss << "osd." << from << " -> osd." << to << " already exists, ";
+            continue;
+          }
+          new_pg_upmap_items.push_back(entry);
+          items << from << "->" << to << ",";
+        }
+        string out(items.str());
+        out.resize(out.size() - 1); // drop last ','
+        out += "]";
+
+        if (new_pg_upmap_items.empty()) {
+          ss << "no valid upmap items(pairs) is specified";
+          err = -EINVAL;
+          goto reply;
+        }
+
+        pending_inc.new_pg_upmap_items[pgid] =
+          mempool::osdmap::vector<pair<int32_t,int32_t>>(
+          new_pg_upmap_items.begin(), new_pg_upmap_items.end());
+        ss << "set " << pgid << " pg_upmap_items mapping to " << out;
+      }
+      break;
+
+    case OP_RM_PG_UPMAP_ITEMS:
+      {
+        pending_inc.old_pg_upmap_items.insert(pgid);
+        ss << "clear " << pgid << " pg_upmap_items mapping";
+      }
+      break;
+
+    default:
+      ceph_abort_msg("invalid option");
+    }
+
+    goto update;
+  } else if (prefix == "osd primary-affinity") {
+    int64_t id;
+    if (!cmd_getval(cmdmap, "id", id)) {
+      ss << "invalid osd id value '"
+         << cmd_vartype_stringify(cmdmap.at("id")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    double w;
+    if (!cmd_getval(cmdmap, "weight", w)) {
+      ss << "unable to parse 'weight' value '"
+	 << cmd_vartype_stringify(cmdmap.at("weight")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    long ww = (int)((double)CEPH_OSD_MAX_PRIMARY_AFFINITY*w);
+    if (ww < 0L) {
+      ss << "weight must be >= 0";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (osdmap.require_min_compat_client != ceph_release_t::unknown &&
+	osdmap.require_min_compat_client < ceph_release_t::firefly) {
+      ss << "require_min_compat_client "
+	 << osdmap.require_min_compat_client
+	 << " < firefly, which is required for primary-affinity";
+      err = -EPERM;
+      goto reply;
+    }
+    if (osdmap.exists(id)) {
+      pending_inc.new_primary_affinity[id] = ww;
+      ss << "set osd." << id << " primary-affinity to " << w << " (" << std::ios::hex << ww << std::ios::dec << ")";
+      getline(ss, rs);
+      wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+                                                get_last_committed() + 1));
+      return true;
+    } else {
+      ss << "osd." << id << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+  } else if (prefix == "osd reweight") {
+    int64_t id;
+    if (!cmd_getval(cmdmap, "id", id)) {
+      ss << "unable to parse osd id value '"
+         << cmd_vartype_stringify(cmdmap.at("id")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    double w;
+    if (!cmd_getval(cmdmap, "weight", w)) {
+      ss << "unable to parse weight value '"
+         << cmd_vartype_stringify(cmdmap.at("weight")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    long ww = (int)((double)CEPH_OSD_IN*w);
+    if (ww < 0L) {
+      ss << "weight must be >= 0";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (osdmap.exists(id)) {
+      pending_inc.new_weight[id] = ww;
+      ss << "reweighted osd." << id << " to " << w << " (" << std::hex << ww << std::dec << ")";
+      getline(ss, rs);
+      wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						get_last_committed() + 1));
+      return true;
+    } else {
+      ss << "osd." << id << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+  } else if (prefix == "osd reweightn") {
+    map<int32_t, uint32_t> weights;
+    err = parse_reweights(cct, cmdmap, osdmap, &weights);
+    if (err) {
+      ss << "unable to parse 'weights' value '"
+         << cmd_vartype_stringify(cmdmap.at("weights")) << "'";
+      goto reply;
+    }
+    pending_inc.new_weight.insert(weights.begin(), weights.end());
+    wait_for_finished_proposal(
+	op,
+	new Monitor::C_Command(mon, op, 0, rs, rdata, get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd lost") {
+    int64_t id;
+    if (!cmd_getval(cmdmap, "id", id)) {
+      ss << "unable to parse osd id value '"
+         << cmd_vartype_stringify(cmdmap.at("id")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+    if (!sure) {
+      ss << "are you SURE?  this might mean real, permanent data loss.  pass "
+	    "--yes-i-really-mean-it if you really do.";
+      err = -EPERM;
+      goto reply;
+    } else if (!osdmap.exists(id)) {
+      ss << "osd." << id << " does not exist";
+      err = -ENOENT;
+      goto reply;
+    } else if (!osdmap.is_down(id)) {
+      ss << "osd." << id << " is not down";
+      err = -EBUSY;
+      goto reply;
+    } else {
+      epoch_t e = osdmap.get_info(id).down_at;
+      pending_inc.new_lost[id] = e;
+      ss << "marked osd lost in epoch " << e;
+      getline(ss, rs);
+      wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						get_last_committed() + 1));
+      return true;
+    }
+
+  } else if (prefix == "osd destroy-actual" ||
+	     prefix == "osd purge-actual" ||
+	     prefix == "osd purge-new") {
+    /* Destroying an OSD means that we don't expect to further make use of
+     * the OSDs data (which may even become unreadable after this operation),
+     * and that we are okay with scrubbing all its cephx keys and config-key
+     * data (which may include lockbox keys, thus rendering the osd's data
+     * unreadable).
+     *
+     * The OSD will not be removed. Instead, we will mark it as destroyed,
+     * such that a subsequent call to `create` will not reuse the osd id.
+     * This will play into being able to recreate the OSD, at the same
+     * crush location, with minimal data movement.
+     */
+
+    // make sure authmon is writeable.
+    if (!mon.authmon()->is_writeable()) {
+      dout(10) << __func__ << " waiting for auth mon to be writeable for "
+               << "osd destroy" << dendl;
+      mon.authmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+      return false;
+    }
+
+    int64_t id;
+    if (!cmd_getval(cmdmap, "id", id)) {
+      auto p = cmdmap.find("id");
+      if (p == cmdmap.end()) {
+	ss << "no osd id specified";
+      } else {
+	ss << "unable to parse osd id value '"
+	   << cmd_vartype_stringify(cmdmap.at("id")) << "";
+      }
+      err = -EINVAL;
+      goto reply;
+    }
+
+    bool is_destroy = (prefix == "osd destroy-actual");
+    if (!is_destroy) {
+      ceph_assert("osd purge-actual" == prefix ||
+	     "osd purge-new" == prefix);
+    }
+
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+    if (!sure) {
+      ss << "Are you SURE?  Did you verify with 'ceph osd safe-to-destroy'?  "
+	 << "This will mean real, permanent data loss, as well "
+         << "as deletion of cephx and lockbox keys. "
+	 << "Pass --yes-i-really-mean-it if you really do.";
+      err = -EPERM;
+      goto reply;
+    } else if (!osdmap.exists(id)) {
+      ss << "osd." << id << " does not exist";
+      err = 0; // idempotent
+      goto reply;
+    } else if (osdmap.is_up(id)) {
+      ss << "osd." << id << " is not `down`.";
+      err = -EBUSY;
+      goto reply;
+    } else if (is_destroy && osdmap.is_destroyed(id)) {
+      ss << "destroyed osd." << id;
+      err = 0;
+      goto reply;
+    }
+
+    if (prefix == "osd purge-new" &&
+	(osdmap.get_state(id) & CEPH_OSD_NEW) == 0) {
+      ss << "osd." << id << " is not new";
+      err = -EPERM;
+      goto reply;
+    }
+
+    bool goto_reply = false;
+
+    paxos.plug();
+    if (is_destroy) {
+      err = prepare_command_osd_destroy(id, ss);
+      // we checked above that it should exist.
+      ceph_assert(err != -ENOENT);
+    } else {
+      err = prepare_command_osd_purge(id, ss);
+      if (err == -ENOENT) {
+        err = 0;
+        ss << "osd." << id << " does not exist.";
+        goto_reply = true;
+      }
+    }
+    paxos.unplug();
+
+    if (err < 0 || goto_reply) {
+      goto reply;
+    }
+
+    if (is_destroy) {
+      ss << "destroyed osd." << id;
+    } else {
+      ss << "purged osd." << id;
+    }
+
+    getline(ss, rs);
+    wait_for_finished_proposal(op,
+        new Monitor::C_Command(mon, op, 0, rs, get_last_committed() + 1));
+    force_immediate_propose();
+    return true;
+
+  } else if (prefix == "osd new") {
+
+    // make sure authmon is writeable.
+    if (!mon.authmon()->is_writeable()) {
+      dout(10) << __func__ << " waiting for auth mon to be writeable for "
+               << "osd new" << dendl;
+      mon.authmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+      return false;
+    }
+
+    // make sure kvmon is writeable.
+    if (!mon.kvmon()->is_writeable()) {
+      dout(10) << __func__ << " waiting for kv mon to be writeable for "
+               << "osd new" << dendl;
+      mon.kvmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+      return false;
+    }
+
+    map<string,string> param_map;
+
+    bufferlist bl = m->get_data();
+    string param_json = bl.to_str();
+    dout(20) << __func__ << " osd new json = " << param_json << dendl;
+
+    err = get_json_str_map(param_json, ss, &param_map);
+    if (err < 0)
+      goto reply;
+
+    dout(20) << __func__ << " osd new params " << param_map << dendl;
+
+    paxos.plug();
+    err = prepare_command_osd_new(op, cmdmap, param_map, ss, f.get());
+    paxos.unplug();
+
+    if (err < 0) {
+      goto reply;
+    }
+
+    if (f) {
+      f->flush(rdata);
+    } else {
+      rdata.append(ss);
+    }
+
+    if (err == EEXIST) {
+      // idempotent operation
+      err = 0;
+      goto reply;
+    }
+
+    wait_for_finished_proposal(op,
+        new Monitor::C_Command(mon, op, 0, rs, rdata,
+                               get_last_committed() + 1));
+    force_immediate_propose();
+    return true;
+
+  } else if (prefix == "osd create") {
+
+    // optional id provided?
+    int64_t id = -1, cmd_id = -1;
+    if (cmd_getval(cmdmap, "id", cmd_id)) {
+      if (cmd_id < 0) {
+	ss << "invalid osd id value '" << cmd_id << "'";
+	err = -EINVAL;
+	goto reply;
+      }
+      dout(10) << " osd create got id " << cmd_id << dendl;
+    }
+
+    uuid_d uuid;
+    string uuidstr;
+    if (cmd_getval(cmdmap, "uuid", uuidstr)) {
+      if (!uuid.parse(uuidstr.c_str())) {
+        ss << "invalid uuid value '" << uuidstr << "'";
+        err = -EINVAL;
+        goto reply;
+      }
+      // we only care about the id if we also have the uuid, to
+      // ensure the operation's idempotency.
+      id = cmd_id;
+    }
+
+    int32_t new_id = -1;
+    err = prepare_command_osd_create(id, uuid, &new_id, ss);
+    if (err < 0) {
+      if (err == -EAGAIN) {
+        wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+        return true;
+      }
+      // a check has failed; reply to the user.
+      goto reply;
+
+    } else if (err == EEXIST) {
+      // this is an idempotent operation; we can go ahead and reply.
+      if (f) {
+        f->open_object_section("created_osd");
+        f->dump_int("osdid", new_id);
+        f->close_section();
+        f->flush(rdata);
+      } else {
+        ss << new_id;
+        rdata.append(ss);
+      }
+      err = 0;
+      goto reply;
+    }
+
+    string empty_device_class;
+    do_osd_create(id, uuid, empty_device_class, &new_id);
+
+    if (f) {
+      f->open_object_section("created_osd");
+      f->dump_int("osdid", new_id);
+      f->close_section();
+      f->flush(rdata);
+    } else {
+      ss << new_id;
+      rdata.append(ss);
+    }
+    wait_for_finished_proposal(op,
+        new Monitor::C_Command(mon, op, 0, rs, rdata,
+                               get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd blocklist clear" ||
+	     prefix == "osd blacklist clear") {
+    pending_inc.new_blocklist.clear();
+    std::list<std::pair<entity_addr_t,utime_t > > blocklist;
+    std::list<std::pair<entity_addr_t,utime_t > > range_b;
+    osdmap.get_blocklist(&blocklist, &range_b);
+    for (const auto &entry : blocklist) {
+      pending_inc.old_blocklist.push_back(entry.first);
+    }
+    for (const auto &entry : range_b) {
+      pending_inc.old_range_blocklist.push_back(entry.first);
+    }
+    ss << " removed all blocklist entries";
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+                                              get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd blocklist" ||
+	     prefix == "osd blacklist") {
+    string addrstr, rangestr;
+    bool range = false;
+    cmd_getval(cmdmap, "addr", addrstr);
+    if (cmd_getval(cmdmap, "range", rangestr)) {
+      if (rangestr == "range") {
+	range = true;
+      } else {
+	ss << "Did you mean to specify \"osd blocklist range\"?";
+	err = -EINVAL;
+	goto reply;
+      }
+    }
+    entity_addr_t addr;
+    if (!addr.parse(addrstr.c_str(), 0)) {
+      ss << "unable to parse address " << addrstr;
+      err = -EINVAL;
+      goto reply;
+    }
+    else {
+      if (range) {
+	if (!addr.maybe_cidr()) {
+	  ss << "You specified a range command, but " << addr
+	     << " does not parse as a CIDR range";
+	  err = -EINVAL;
+	  goto reply;
+	}
+	addr.type = entity_addr_t::TYPE_CIDR;
+	err = check_cluster_features(CEPH_FEATUREMASK_RANGE_BLOCKLIST, ss);
+	if (err) {
+	  goto reply;
+	}
+	if ((addr.is_ipv4() && addr.get_nonce() > 32) ||
+	    (addr.is_ipv6() && addr.get_nonce() > 128)) {
+	  ss << "Too many bits in range for that protocol!";
+	  err = -EINVAL;
+	  goto reply;
+	}
+      } else {
+	if (osdmap.require_osd_release >= ceph_release_t::nautilus) {
+	  // always blocklist type ANY
+	  addr.set_type(entity_addr_t::TYPE_ANY);
+	} else {
+	  addr.set_type(entity_addr_t::TYPE_LEGACY);
+	}
+      }
+
+      string blocklistop;
+      if (!cmd_getval(cmdmap, "blocklistop", blocklistop)) {
+	cmd_getval(cmdmap, "blacklistop", blocklistop);
+      }
+      if (blocklistop == "add") {
+	utime_t expires = ceph_clock_now();
+	double d;
+	// default one hour
+	cmd_getval(cmdmap, "expire", d,
+          g_conf()->mon_osd_blocklist_default_expire);
+	expires += d;
+
+	auto add_to_pending_blocklists = [](auto& nb, auto& ob,
+					    const auto& addr,
+					    const auto& expires) {
+	  nb[addr] = expires;
+	  // cancel any pending un-blocklisting request too
+	  auto it = std::find(ob.begin(),
+			      ob.end(), addr);
+	  if (it != ob.end()) {
+	    ob.erase(it);
+	  }
+	};
+	if (range) {
+	  add_to_pending_blocklists(pending_inc.new_range_blocklist,
+				    pending_inc.old_range_blocklist,
+				    addr, expires);
+
+	} else {
+	  add_to_pending_blocklists(pending_inc.new_blocklist,
+				    pending_inc.old_blocklist,
+				    addr, expires);
+	}
+
+	ss << "blocklisting " << addr << " until " << expires << " (" << d << " sec)";
+	getline(ss, rs);
+	wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						  get_last_committed() + 1));
+	return true;
+      } else if (blocklistop == "rm") {
+	auto rm_from_pending_blocklists = [](const auto& addr,
+					     auto& blocklist,
+					     auto& ob, auto& pb) {
+	  if (blocklist.count(addr)) {
+	    ob.push_back(addr);
+	    return true;
+	  } else if (pb.count(addr)) {
+	    pb.erase(addr);
+	    return true;
+	  }
+	  return false;
+	};
+	if ((!range && rm_from_pending_blocklists(addr, osdmap.blocklist,
+						  pending_inc.old_blocklist,
+						  pending_inc.new_blocklist)) ||
+	    (range && rm_from_pending_blocklists(addr, osdmap.range_blocklist,
+						 pending_inc.old_range_blocklist,
+						 pending_inc.new_range_blocklist))) {
+	  ss << "un-blocklisting " << addr;
+	  getline(ss, rs);
+	  wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						    get_last_committed() + 1));
+	  return true;
+	}
+	ss << addr << " isn't blocklisted";
+	err = 0;
+	goto reply;
+      }
+    }
+  } else if (prefix == "osd pool mksnap") {
+    string poolstr;
+    cmd_getval(cmdmap, "pool", poolstr);
+    int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+    if (pool < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    string snapname;
+    cmd_getval(cmdmap, "snap", snapname);
+    const pg_pool_t *p = osdmap.get_pg_pool(pool);
+    if (p->is_unmanaged_snaps_mode()) {
+      ss << "pool " << poolstr << " is in unmanaged snaps mode";
+      err = -EINVAL;
+      goto reply;
+    } else if (p->snap_exists(snapname.c_str())) {
+      ss << "pool " << poolstr << " snap " << snapname << " already exists";
+      err = 0;
+      goto reply;
+    } else if (p->is_tier()) {
+      ss << "pool " << poolstr << " is a cache tier";
+      err = -EINVAL;
+      goto reply;
+    }
+    pg_pool_t *pp = 0;
+    if (pending_inc.new_pools.count(pool))
+      pp = &pending_inc.new_pools[pool];
+    if (!pp) {
+      pp = &pending_inc.new_pools[pool];
+      *pp = *p;
+    }
+    if (pp->snap_exists(snapname.c_str())) {
+      ss << "pool " << poolstr << " snap " << snapname << " already exists";
+    } else {
+      pp->add_snap(snapname.c_str(), ceph_clock_now());
+      pp->set_snap_epoch(pending_inc.epoch);
+      ss << "created pool " << poolstr << " snap " << snapname;
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd pool rmsnap") {
+    string poolstr;
+    cmd_getval(cmdmap, "pool", poolstr);
+    int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+    if (pool < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    string snapname;
+    cmd_getval(cmdmap, "snap", snapname);
+    const pg_pool_t *p = osdmap.get_pg_pool(pool);
+    if (p->is_unmanaged_snaps_mode()) {
+      ss << "pool " << poolstr << " is in unmanaged snaps mode";
+      err = -EINVAL;
+      goto reply;
+    } else if (!p->snap_exists(snapname.c_str())) {
+      ss << "pool " << poolstr << " snap " << snapname << " does not exist";
+      err = 0;
+      goto reply;
+    }
+    pg_pool_t *pp = 0;
+    if (pending_inc.new_pools.count(pool))
+      pp = &pending_inc.new_pools[pool];
+    if (!pp) {
+      pp = &pending_inc.new_pools[pool];
+      *pp = *p;
+    }
+    snapid_t sn = pp->snap_exists(snapname.c_str());
+    if (sn) {
+      pp->remove_snap(sn);
+      pp->set_snap_epoch(pending_inc.epoch);
+      ss << "removed pool " << poolstr << " snap " << snapname;
+    } else {
+      ss << "already removed pool " << poolstr << " snap " << snapname;
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd pool create") {
+    int64_t pg_num, pgp_num, pg_num_min, pg_num_max;
+    cmd_getval(cmdmap, "pg_num", pg_num, int64_t(0));
+    cmd_getval(cmdmap, "pg_num_min", pg_num_min, int64_t(0));
+    cmd_getval(cmdmap, "pg_num_max", pg_num_max, int64_t(0));
+    cmd_getval(cmdmap, "pgp_num", pgp_num, int64_t(pg_num));
+    string pool_type_str;
+    cmd_getval(cmdmap, "pool_type", pool_type_str);
+    if (pool_type_str.empty())
+      pool_type_str = g_conf().get_val<string>("osd_pool_default_type");
+
+    string poolstr;
+    cmd_getval(cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id >= 0) {
+      const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+      if (pool_type_str != p->get_type_name()) {
+	ss << "pool '" << poolstr << "' cannot change to type " << pool_type_str;
+ 	err = -EINVAL;
+      } else {
+	ss << "pool '" << poolstr << "' already exists";
+	err = 0;
+      }
+      goto reply;
+    }
+
+    int pool_type;
+    if (pool_type_str == "replicated") {
+      pool_type = pg_pool_t::TYPE_REPLICATED;
+    } else if (pool_type_str == "erasure") {
+      pool_type = pg_pool_t::TYPE_ERASURE;
+    } else {
+      ss << "unknown pool type '" << pool_type_str << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    bool implicit_rule_creation = false;
+    int64_t expected_num_objects = 0;
+    string rule_name;
+    cmd_getval(cmdmap, "rule", rule_name);
+    string erasure_code_profile;
+    cmd_getval(cmdmap, "erasure_code_profile", erasure_code_profile);
+
+    if (pool_type == pg_pool_t::TYPE_ERASURE) {
+      if (erasure_code_profile == "")
+	erasure_code_profile = "default";
+      //handle the erasure code profile
+      if (erasure_code_profile == "default") {
+	if (!osdmap.has_erasure_code_profile(erasure_code_profile)) {
+	  if (pending_inc.has_erasure_code_profile(erasure_code_profile)) {
+	    dout(20) << "erasure code profile " << erasure_code_profile << " already pending" << dendl;
+	    goto wait;
+	  }
+
+	  map<string,string> profile_map;
+	  err = osdmap.get_erasure_code_profile_default(cct,
+						      profile_map,
+						      &ss);
+	  if (err)
+	    goto reply;
+	  dout(20) << "erasure code profile " << erasure_code_profile << " set" << dendl;
+	  pending_inc.set_erasure_code_profile(erasure_code_profile, profile_map);
+	  goto wait;
+	}
+      }
+      if (rule_name == "") {
+	implicit_rule_creation = true;
+	if (erasure_code_profile == "default") {
+	  rule_name = "erasure-code";
+	} else {
+	  dout(1) << "implicitly use rule named after the pool: "
+		<< poolstr << dendl;
+	  rule_name = poolstr;
+	}
+      }
+      cmd_getval(cmdmap, "expected_num_objects",
+                 expected_num_objects, int64_t(0));
+    } else {
+      //NOTE:for replicated pool,cmd_map will put rule_name to erasure_code_profile field
+      //     and put expected_num_objects to rule field
+      if (erasure_code_profile != "") { // cmd is from CLI
+        if (rule_name != "") {
+          string interr;
+          expected_num_objects = strict_strtoll(rule_name.c_str(), 10, &interr);
+          if (interr.length()) {
+            ss << "error parsing integer value '" << rule_name << "': " << interr;
+            err = -EINVAL;
+            goto reply;
+          }
+        }
+        rule_name = erasure_code_profile;
+      } else { // cmd is well-formed
+        cmd_getval(cmdmap, "expected_num_objects",
+                   expected_num_objects, int64_t(0));
+      }
+    }
+
+    if (!implicit_rule_creation && rule_name != "") {
+      int rule;
+      err = get_crush_rule(rule_name, &rule, &ss);
+      if (err == -EAGAIN) {
+	wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+	return true;
+      }
+      if (err)
+	goto reply;
+    }
+
+    if (expected_num_objects < 0) {
+      ss << "'expected_num_objects' must be non-negative";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    set<int32_t> osds;
+    osdmap.get_all_osds(osds);
+    bool has_filestore_osd = std::any_of(osds.begin(), osds.end(), [this](int osd) {
+      string type;
+      if (!get_osd_objectstore_type(osd, &type)) {
+        return type == "filestore";
+      } else {
+        return false;
+      }
+    });
+
+    if (has_filestore_osd &&
+        expected_num_objects > 0 &&
+        cct->_conf->filestore_merge_threshold > 0) {
+      ss << "'expected_num_objects' requires 'filestore_merge_threshold < 0'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    if (has_filestore_osd &&
+        expected_num_objects == 0 &&
+        cct->_conf->filestore_merge_threshold < 0) {
+      int osds = osdmap.get_num_osds();
+      bool sure = false;
+      cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+      if (!sure && osds && (pg_num >= 1024 || pg_num / osds >= 100)) {
+        ss << "For better initial performance on pools expected to store a "
+           << "large number of objects, consider supplying the "
+           << "expected_num_objects parameter when creating the pool."
+           << " Pass --yes-i-really-mean-it to ignore it";
+        err = -EPERM;
+        goto reply;
+      }
+    }
+
+    int64_t fast_read_param;
+    cmd_getval(cmdmap, "fast_read", fast_read_param, int64_t(-1));
+    FastReadType fast_read = FAST_READ_DEFAULT;
+    if (fast_read_param == 0)
+      fast_read = FAST_READ_OFF;
+    else if (fast_read_param > 0)
+      fast_read = FAST_READ_ON;
+
+    int64_t repl_size = 0;
+    cmd_getval(cmdmap, "size", repl_size);
+    int64_t target_size_bytes = 0;
+    double target_size_ratio = 0.0;
+    cmd_getval(cmdmap, "target_size_bytes", target_size_bytes);
+    cmd_getval(cmdmap, "target_size_ratio", target_size_ratio);
+
+    string pg_autoscale_mode;
+    cmd_getval(cmdmap, "autoscale_mode", pg_autoscale_mode);
+
+    bool bulk = 0;
+    cmd_getval(cmdmap, "bulk", bulk);
+    err = prepare_new_pool(poolstr,
+			   -1, // default crush rule
+			   rule_name,
+			   pg_num, pgp_num, pg_num_min, pg_num_max,
+                           repl_size, target_size_bytes, target_size_ratio,
+			   erasure_code_profile, pool_type,
+                           (uint64_t)expected_num_objects,
+                           fast_read,
+			   pg_autoscale_mode,
+			   bulk,
+			   &ss);
+    if (err < 0) {
+      switch(err) {
+      case -EEXIST:
+	ss << "pool '" << poolstr << "' already exists";
+	break;
+      case -EAGAIN:
+	wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+	return true;
+      case -ERANGE:
+        goto reply;
+      default:
+	goto reply;
+	break;
+      }
+    } else {
+      ss << "pool '" << poolstr << "' created";
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd pool delete" ||
+             prefix == "osd pool rm") {
+    // osd pool delete/rm <poolname> <poolname again> --yes-i-really-really-mean-it
+    string poolstr, poolstr2, sure;
+    cmd_getval(cmdmap, "pool", poolstr);
+    cmd_getval(cmdmap, "pool2", poolstr2);
+    int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+    if (pool < 0) {
+      ss << "pool '" << poolstr << "' does not exist";
+      err = 0;
+      goto reply;
+    }
+
+    bool force_no_fake = false;
+    cmd_getval(cmdmap, "yes_i_really_really_mean_it", force_no_fake);
+    bool force = false;
+    cmd_getval(cmdmap, "yes_i_really_really_mean_it_not_faking", force);
+    if (poolstr2 != poolstr ||
+	(!force && !force_no_fake)) {
+      ss << "WARNING: this will *PERMANENTLY DESTROY* all data stored in pool " << poolstr
+	 << ".  If you are *ABSOLUTELY CERTAIN* that is what you want, pass the pool name *twice*, "
+	 << "followed by --yes-i-really-really-mean-it.";
+      err = -EPERM;
+      goto reply;
+    }
+    err = _prepare_remove_pool(pool, &ss, force_no_fake);
+    if (err == -EAGAIN) {
+      wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+      return true;
+    }
+    if (err < 0)
+      goto reply;
+    goto update;
+  } else if (prefix == "osd pool rename") {
+    string srcpoolstr, destpoolstr;
+    cmd_getval(cmdmap, "srcpool", srcpoolstr);
+    cmd_getval(cmdmap, "destpool", destpoolstr);
+    int64_t pool_src = osdmap.lookup_pg_pool_name(srcpoolstr.c_str());
+    int64_t pool_dst = osdmap.lookup_pg_pool_name(destpoolstr.c_str());
+
+    if (pool_src < 0) {
+      if (pool_dst >= 0) {
+        // src pool doesn't exist, dst pool does exist: to ensure idempotency
+        // of operations, assume this rename succeeded, as it is not changing
+        // the current state.  Make sure we output something understandable
+        // for whoever is issuing the command, if they are paying attention,
+        // in case it was not intentional; or to avoid a "wtf?" and a bug
+        // report in case it was intentional, while expecting a failure.
+        ss << "pool '" << srcpoolstr << "' does not exist; pool '"
+          << destpoolstr << "' does -- assuming successful rename";
+        err = 0;
+      } else {
+        ss << "unrecognized pool '" << srcpoolstr << "'";
+        err = -ENOENT;
+      }
+      goto reply;
+    } else if (pool_dst >= 0) {
+      // source pool exists and so does the destination pool
+      ss << "pool '" << destpoolstr << "' already exists";
+      err = -EEXIST;
+      goto reply;
+    }
+
+    int ret = _prepare_rename_pool(pool_src, destpoolstr);
+    if (ret == 0) {
+      ss << "pool '" << srcpoolstr << "' renamed to '" << destpoolstr << "'";
+    } else {
+      ss << "failed to rename pool '" << srcpoolstr << "' to '" << destpoolstr << "': "
+        << cpp_strerror(ret);
+    }
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, ret, rs,
+					      get_last_committed() + 1));
+    return true;
+
+  } else if (prefix == "osd pool set") {
+    err = prepare_command_pool_set(cmdmap, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err < 0)
+      goto reply;
+
+    getline(ss, rs);
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+						   get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd tier add") {
+    err = check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err)
+      goto reply;
+    string poolstr;
+    cmd_getval(cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    string tierpoolstr;
+    cmd_getval(cmdmap, "tierpool", tierpoolstr);
+    int64_t tierpool_id = osdmap.lookup_pg_pool_name(tierpoolstr);
+    if (tierpool_id < 0) {
+      ss << "unrecognized pool '" << tierpoolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+    ceph_assert(p);
+    const pg_pool_t *tp = osdmap.get_pg_pool(tierpool_id);
+    ceph_assert(tp);
+
+    if (!_check_become_tier(tierpool_id, tp, pool_id, p, &err, &ss)) {
+      goto reply;
+    }
+
+    // make sure new tier is empty
+    string force_nonempty;
+    cmd_getval(cmdmap, "force_nonempty", force_nonempty);
+    const pool_stat_t *pstats = mon.mgrstatmon()->get_pool_stat(tierpool_id);
+    if (pstats && pstats->stats.sum.num_objects != 0 &&
+	force_nonempty != "--force-nonempty") {
+      ss << "tier pool '" << tierpoolstr << "' is not empty; --force-nonempty to force";
+      err = -ENOTEMPTY;
+      goto reply;
+    }
+    if (tp->is_erasure()) {
+      ss << "tier pool '" << tierpoolstr
+	 << "' is an ec pool, which cannot be a tier";
+      err = -ENOTSUP;
+      goto reply;
+    }
+    if ((!tp->removed_snaps.empty() || !tp->snaps.empty()) &&
+	((force_nonempty != "--force-nonempty") ||
+	 (!g_conf()->mon_debug_unsafe_allow_tier_with_nonempty_snaps))) {
+      ss << "tier pool '" << tierpoolstr << "' has snapshot state; it cannot be added as a tier without breaking the pool";
+      err = -ENOTEMPTY;
+      goto reply;
+    }
+    // go
+    pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+    pg_pool_t *ntp = pending_inc.get_new_pool(tierpool_id, tp);
+    if (np->tiers.count(tierpool_id) || ntp->is_tier()) {
+      wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+      return true;
+    }
+    np->tiers.insert(tierpool_id);
+    np->set_snap_epoch(pending_inc.epoch); // tier will update to our snap info
+    ntp->tier_of = pool_id;
+    ss << "pool '" << tierpoolstr << "' is now (or already was) a tier of '" << poolstr << "'";
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd tier remove" ||
+             prefix == "osd tier rm") {
+    string poolstr;
+    cmd_getval(cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    string tierpoolstr;
+    cmd_getval(cmdmap, "tierpool", tierpoolstr);
+    int64_t tierpool_id = osdmap.lookup_pg_pool_name(tierpoolstr);
+    if (tierpool_id < 0) {
+      ss << "unrecognized pool '" << tierpoolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+    ceph_assert(p);
+    const pg_pool_t *tp = osdmap.get_pg_pool(tierpool_id);
+    ceph_assert(tp);
+
+    if (!_check_remove_tier(pool_id, p, tp, &err, &ss)) {
+      goto reply;
+    }
+
+    if (p->tiers.count(tierpool_id) == 0) {
+      ss << "pool '" << tierpoolstr << "' is now (or already was) not a tier of '" << poolstr << "'";
+      err = 0;
+      goto reply;
+    }
+    if (tp->tier_of != pool_id) {
+      ss << "tier pool '" << tierpoolstr << "' is a tier of '"
+         << osdmap.get_pool_name(tp->tier_of) << "': "
+         // be scary about it; this is an inconsistency and bells must go off
+         << "THIS SHOULD NOT HAVE HAPPENED AT ALL";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (p->read_tier == tierpool_id) {
+      ss << "tier pool '" << tierpoolstr << "' is the overlay for '" << poolstr << "'; please remove-overlay first";
+      err = -EBUSY;
+      goto reply;
+    }
+    // go
+    pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+    pg_pool_t *ntp = pending_inc.get_new_pool(tierpool_id, tp);
+    if (np->tiers.count(tierpool_id) == 0 ||
+	ntp->tier_of != pool_id ||
+	np->read_tier == tierpool_id) {
+      wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+      return true;
+    }
+    np->tiers.erase(tierpool_id);
+    ntp->clear_tier();
+    ss << "pool '" << tierpoolstr << "' is now (or already was) not a tier of '" << poolstr << "'";
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd tier set-overlay") {
+    err = check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err)
+      goto reply;
+    string poolstr;
+    cmd_getval(cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    string overlaypoolstr;
+    cmd_getval(cmdmap, "overlaypool", overlaypoolstr);
+    int64_t overlaypool_id = osdmap.lookup_pg_pool_name(overlaypoolstr);
+    if (overlaypool_id < 0) {
+      ss << "unrecognized pool '" << overlaypoolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+    ceph_assert(p);
+    const pg_pool_t *overlay_p = osdmap.get_pg_pool(overlaypool_id);
+    ceph_assert(overlay_p);
+    if (p->tiers.count(overlaypool_id) == 0) {
+      ss << "tier pool '" << overlaypoolstr << "' is not a tier of '" << poolstr << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (p->read_tier == overlaypool_id) {
+      err = 0;
+      ss << "overlay for '" << poolstr << "' is now (or already was) '" << overlaypoolstr << "'";
+      goto reply;
+    }
+    if (p->has_read_tier()) {
+      ss << "pool '" << poolstr << "' has overlay '"
+	 << osdmap.get_pool_name(p->read_tier)
+	 << "'; please remove-overlay first";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    // go
+    pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+    np->read_tier = overlaypool_id;
+    np->write_tier = overlaypool_id;
+    np->set_last_force_op_resend(pending_inc.epoch);
+    pg_pool_t *noverlay_p = pending_inc.get_new_pool(overlaypool_id, overlay_p);
+    noverlay_p->set_last_force_op_resend(pending_inc.epoch);
+    ss << "overlay for '" << poolstr << "' is now (or already was) '" << overlaypoolstr << "'";
+    if (overlay_p->cache_mode == pg_pool_t::CACHEMODE_NONE)
+      ss <<" (WARNING: overlay pool cache_mode is still NONE)";
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd tier remove-overlay" ||
+             prefix == "osd tier rm-overlay") {
+    string poolstr;
+    cmd_getval(cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+    ceph_assert(p);
+    if (!p->has_read_tier()) {
+      err = 0;
+      ss << "there is now (or already was) no overlay for '" << poolstr << "'";
+      goto reply;
+    }
+
+    if (!_check_remove_tier(pool_id, p, NULL, &err, &ss)) {
+      goto reply;
+    }
+
+    // go
+    pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+    if (np->has_read_tier()) {
+      const pg_pool_t *op = osdmap.get_pg_pool(np->read_tier);
+      pg_pool_t *nop = pending_inc.get_new_pool(np->read_tier,op);
+      nop->set_last_force_op_resend(pending_inc.epoch);
+    }
+    if (np->has_write_tier()) {
+      const pg_pool_t *op = osdmap.get_pg_pool(np->write_tier);
+      pg_pool_t *nop = pending_inc.get_new_pool(np->write_tier, op);
+      nop->set_last_force_op_resend(pending_inc.epoch);
+    }
+    np->clear_read_tier();
+    np->clear_write_tier();
+    np->set_last_force_op_resend(pending_inc.epoch);
+    ss << "there is now (or already was) no overlay for '" << poolstr << "'";
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd tier cache-mode") {
+    err = check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err)
+      goto reply;
+    string poolstr;
+    cmd_getval(cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+    ceph_assert(p);
+    if (!p->is_tier()) {
+      ss << "pool '" << poolstr << "' is not a tier";
+      err = -EINVAL;
+      goto reply;
+    }
+    string modestr;
+    cmd_getval(cmdmap, "mode", modestr);
+    pg_pool_t::cache_mode_t mode = pg_pool_t::get_cache_mode_from_str(modestr);
+    if (int(mode) < 0) {
+      ss << "'" << modestr << "' is not a valid cache mode";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+
+    if (mode == pg_pool_t::CACHEMODE_FORWARD ||
+	mode == pg_pool_t::CACHEMODE_READFORWARD) {
+      ss << "'" << modestr << "' is no longer a supported cache mode";
+      err = -EPERM;
+      goto reply;
+    }
+    if ((mode != pg_pool_t::CACHEMODE_WRITEBACK &&
+	 mode != pg_pool_t::CACHEMODE_NONE &&
+	 mode != pg_pool_t::CACHEMODE_PROXY &&
+	 mode != pg_pool_t::CACHEMODE_READPROXY) &&
+	 !sure) {
+      ss << "'" << modestr << "' is not a well-supported cache mode and may "
+	 << "corrupt your data.  pass --yes-i-really-mean-it to force.";
+      err = -EPERM;
+      goto reply;
+    }
+
+    // pool already has this cache-mode set and there are no pending changes
+    if (p->cache_mode == mode &&
+	(pending_inc.new_pools.count(pool_id) == 0 ||
+	 pending_inc.new_pools[pool_id].cache_mode == p->cache_mode)) {
+      ss << "set cache-mode for pool '" << poolstr << "'"
+         << " to " << pg_pool_t::get_cache_mode_name(mode);
+      err = 0;
+      goto reply;
+    }
+
+    /* Mode description:
+     *
+     *  none:       No cache-mode defined
+     *  forward:    Forward all reads and writes to base pool [removed]
+     *  writeback:  Cache writes, promote reads from base pool
+     *  readonly:   Forward writes to base pool
+     *  readforward: Writes are in writeback mode, Reads are in forward mode [removed]
+     *  proxy:       Proxy all reads and writes to base pool
+     *  readproxy:   Writes are in writeback mode, Reads are in proxy mode
+     *
+     * Hence, these are the allowed transitions:
+     *
+     *  none -> any
+     *  forward -> proxy || readforward || readproxy || writeback || any IF num_objects_dirty == 0
+     *  proxy -> readproxy || writeback || any IF num_objects_dirty == 0
+     *  readforward -> forward || proxy || readproxy || writeback || any IF num_objects_dirty == 0
+     *  readproxy -> proxy || writeback || any IF num_objects_dirty == 0
+     *  writeback -> readproxy || proxy
+     *  readonly -> any
+     */
+
+    // We check if the transition is valid against the current pool mode, as
+    // it is the only committed state thus far.  We will blantly squash
+    // whatever mode is on the pending state.
+
+    if (p->cache_mode == pg_pool_t::CACHEMODE_WRITEBACK &&
+        (mode != pg_pool_t::CACHEMODE_PROXY &&
+	  mode != pg_pool_t::CACHEMODE_READPROXY)) {
+      ss << "unable to set cache-mode '" << pg_pool_t::get_cache_mode_name(mode)
+         << "' on a '" << pg_pool_t::get_cache_mode_name(p->cache_mode)
+         << "' pool; only '"
+         << pg_pool_t::get_cache_mode_name(pg_pool_t::CACHEMODE_READPROXY)
+        << "' allowed.";
+      err = -EINVAL;
+      goto reply;
+    }
+    if ((p->cache_mode == pg_pool_t::CACHEMODE_READFORWARD &&
+        (mode != pg_pool_t::CACHEMODE_WRITEBACK &&
+	  mode != pg_pool_t::CACHEMODE_PROXY &&
+	  mode != pg_pool_t::CACHEMODE_READPROXY)) ||
+
+        (p->cache_mode == pg_pool_t::CACHEMODE_READPROXY &&
+        (mode != pg_pool_t::CACHEMODE_WRITEBACK &&
+	  mode != pg_pool_t::CACHEMODE_PROXY)) ||
+
+        (p->cache_mode == pg_pool_t::CACHEMODE_PROXY &&
+        (mode != pg_pool_t::CACHEMODE_WRITEBACK &&
+	  mode != pg_pool_t::CACHEMODE_READPROXY)) ||
+
+        (p->cache_mode == pg_pool_t::CACHEMODE_FORWARD &&
+        (mode != pg_pool_t::CACHEMODE_WRITEBACK &&
+	  mode != pg_pool_t::CACHEMODE_PROXY &&
+	  mode != pg_pool_t::CACHEMODE_READPROXY))) {
+
+      const pool_stat_t* pstats =
+        mon.mgrstatmon()->get_pool_stat(pool_id);
+
+      if (pstats && pstats->stats.sum.num_objects_dirty > 0) {
+        ss << "unable to set cache-mode '"
+           << pg_pool_t::get_cache_mode_name(mode) << "' on pool '" << poolstr
+           << "': dirty objects found";
+        err = -EBUSY;
+        goto reply;
+      }
+    }
+    // go
+    pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+    np->cache_mode = mode;
+    // set this both when moving to and from cache_mode NONE.  this is to
+    // capture legacy pools that were set up before this flag existed.
+    np->flags |= pg_pool_t::FLAG_INCOMPLETE_CLONES;
+    ss << "set cache-mode for pool '" << poolstr
+	<< "' to " << pg_pool_t::get_cache_mode_name(mode);
+    if (mode == pg_pool_t::CACHEMODE_NONE) {
+      const pg_pool_t *base_pool = osdmap.get_pg_pool(np->tier_of);
+      ceph_assert(base_pool);
+      if (base_pool->read_tier == pool_id ||
+	  base_pool->write_tier == pool_id)
+	ss <<" (WARNING: pool is still configured as read or write tier)";
+    }
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd tier add-cache") {
+    err = check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL, ss);
+    if (err == -EAGAIN)
+      goto wait;
+    if (err)
+      goto reply;
+    string poolstr;
+    cmd_getval(cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    string tierpoolstr;
+    cmd_getval(cmdmap, "tierpool", tierpoolstr);
+    int64_t tierpool_id = osdmap.lookup_pg_pool_name(tierpoolstr);
+    if (tierpool_id < 0) {
+      ss << "unrecognized pool '" << tierpoolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+    const pg_pool_t *p = osdmap.get_pg_pool(pool_id);
+    ceph_assert(p);
+    const pg_pool_t *tp = osdmap.get_pg_pool(tierpool_id);
+    ceph_assert(tp);
+
+    if (!_check_become_tier(tierpool_id, tp, pool_id, p, &err, &ss)) {
+      goto reply;
+    }
+
+    int64_t size = 0;
+    if (!cmd_getval(cmdmap, "size", size)) {
+      ss << "unable to parse 'size' value '"
+         << cmd_vartype_stringify(cmdmap.at("size")) << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    // make sure new tier is empty
+    const pool_stat_t *pstats =
+      mon.mgrstatmon()->get_pool_stat(tierpool_id);
+    if (pstats && pstats->stats.sum.num_objects != 0) {
+      ss << "tier pool '" << tierpoolstr << "' is not empty";
+      err = -ENOTEMPTY;
+      goto reply;
+    }
+    auto& modestr = g_conf().get_val<string>("osd_tier_default_cache_mode");
+    pg_pool_t::cache_mode_t mode = pg_pool_t::get_cache_mode_from_str(modestr);
+    if (int(mode) < 0) {
+      ss << "osd tier cache default mode '" << modestr << "' is not a valid cache mode";
+      err = -EINVAL;
+      goto reply;
+    }
+    HitSet::Params hsp;
+    auto& cache_hit_set_type =
+      g_conf().get_val<string>("osd_tier_default_cache_hit_set_type");
+    if (cache_hit_set_type == "bloom") {
+      BloomHitSet::Params *bsp = new BloomHitSet::Params;
+      bsp->set_fpp(g_conf().get_val<double>("osd_pool_default_hit_set_bloom_fpp"));
+      hsp = HitSet::Params(bsp);
+    } else if (cache_hit_set_type == "explicit_hash") {
+      hsp = HitSet::Params(new ExplicitHashHitSet::Params);
+    } else if (cache_hit_set_type == "explicit_object") {
+      hsp = HitSet::Params(new ExplicitObjectHitSet::Params);
+    } else {
+      ss << "osd tier cache default hit set type '"
+	 << cache_hit_set_type << "' is not a known type";
+      err = -EINVAL;
+      goto reply;
+    }
+    // go
+    pg_pool_t *np = pending_inc.get_new_pool(pool_id, p);
+    pg_pool_t *ntp = pending_inc.get_new_pool(tierpool_id, tp);
+    if (np->tiers.count(tierpool_id) || ntp->is_tier()) {
+      wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+      return true;
+    }
+    np->tiers.insert(tierpool_id);
+    np->read_tier = np->write_tier = tierpool_id;
+    np->set_snap_epoch(pending_inc.epoch); // tier will update to our snap info
+    np->set_last_force_op_resend(pending_inc.epoch);
+    ntp->set_last_force_op_resend(pending_inc.epoch);
+    ntp->tier_of = pool_id;
+    ntp->cache_mode = mode;
+    ntp->hit_set_count = g_conf().get_val<uint64_t>("osd_tier_default_cache_hit_set_count");
+    ntp->hit_set_period = g_conf().get_val<uint64_t>("osd_tier_default_cache_hit_set_period");
+    ntp->min_read_recency_for_promote = g_conf().get_val<uint64_t>("osd_tier_default_cache_min_read_recency_for_promote");
+    ntp->min_write_recency_for_promote = g_conf().get_val<uint64_t>("osd_tier_default_cache_min_write_recency_for_promote");
+    ntp->hit_set_grade_decay_rate = g_conf().get_val<uint64_t>("osd_tier_default_cache_hit_set_grade_decay_rate");
+    ntp->hit_set_search_last_n = g_conf().get_val<uint64_t>("osd_tier_default_cache_hit_set_search_last_n");
+    ntp->hit_set_params = hsp;
+    ntp->target_max_bytes = size;
+    ss << "pool '" << tierpoolstr << "' is now (or already was) a cache tier of '" << poolstr << "'";
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, ss.str(),
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd pool set-quota") {
+    string poolstr;
+    cmd_getval(cmdmap, "pool", poolstr);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(poolstr);
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << poolstr << "'";
+      err = -ENOENT;
+      goto reply;
+    }
+
+    string field;
+    cmd_getval(cmdmap, "field", field);
+    if (field != "max_objects" && field != "max_bytes") {
+      ss << "unrecognized field '" << field << "'; should be 'max_bytes' or 'max_objects'";
+      err = -EINVAL;
+      goto reply;
+    }
+
+    // val could contain unit designations, so we treat as a string
+    string val;
+    cmd_getval(cmdmap, "val", val);
+    string tss;
+    int64_t value;
+    if (field == "max_objects") {
+      value = strict_sistrtoll(val.c_str(), &tss);
+    } else if (field == "max_bytes") {
+      value = strict_iecstrtoll(val.c_str(), &tss);
+    } else {
+      ceph_abort_msg("unrecognized option");
+    }
+    if (!tss.empty()) {
+      ss << "error parsing value '" << val << "': " << tss;
+      err = -EINVAL;
+      goto reply;
+    }
+
+    pg_pool_t *pi = pending_inc.get_new_pool(pool_id, osdmap.get_pg_pool(pool_id));
+    if (field == "max_objects") {
+      pi->quota_max_objects = value;
+    } else if (field == "max_bytes") {
+      pi->quota_max_bytes = value;
+    } else {
+      ceph_abort_msg("unrecognized option");
+    }
+    ss << "set-quota " << field << " = " << value << " for pool " << poolstr;
+    rs = ss.str();
+    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					      get_last_committed() + 1));
+    return true;
+  } else if (prefix == "osd pool application enable" ||
+             prefix == "osd pool application disable" ||
+             prefix == "osd pool application set" ||
+             prefix == "osd pool application rm") {
+    err = prepare_command_pool_application(prefix, cmdmap, ss);
+    if (err == -EAGAIN) {
+      goto wait;
+    } else if (err < 0) {
+      goto reply;
+    } else {
+      goto update;
+    }
+  } else if (prefix == "osd force-create-pg") {
+    pg_t pgid;
+    string pgidstr;
+    cmd_getval(cmdmap, "pgid", pgidstr);
+    if (!pgid.parse(pgidstr.c_str())) {
+      ss << "invalid pgid '" << pgidstr << "'";
+      err = -EINVAL;
+      goto reply;
+    }
+    if (!osdmap.pg_exists(pgid)) {
+      ss << "pg " << pgid << " should not exist";
+      err = -ENOENT;
+      goto reply;
+    }
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+    if (!sure) {
+      ss << "This command will recreate a lost (as in data lost) PG with data in it, such "
+	 << "that the cluster will give up ever trying to recover the lost data.  Do this "
+	 << "only if you are certain that all copies of the PG are in fact lost and you are "
+	 << "willing to accept that the data is permanently destroyed.  Pass "
+	 << "--yes-i-really-mean-it to proceed.";
+      err = -EPERM;
+      goto reply;
+    }
+    bool creating_now;
+    {
+      std::lock_guard<std::mutex> l(creating_pgs_lock);
+      auto emplaced = creating_pgs.pgs.emplace(
+	pgid,
+	creating_pgs_t::pg_create_info(osdmap.get_epoch(),
+				       ceph_clock_now()));
+      creating_now = emplaced.second;
+    }
+    if (creating_now) {
+      ss << "pg " << pgidstr << " now creating, ok";
+      // set the pool's CREATING flag so that (1) the osd won't ignore our
+      // create message and (2) we won't propose any future pg_num changes
+      // until after the PG has been instantiated.
+      if (pending_inc.new_pools.count(pgid.pool()) == 0) {
+	pending_inc.new_pools[pgid.pool()] = *osdmap.get_pg_pool(pgid.pool());
+      }
+      pending_inc.new_pools[pgid.pool()].flags |= pg_pool_t::FLAG_CREATING;
+      err = 0;
+      goto update;
+    } else {
+      ss << "pg " << pgid << " already creating";
+      err = 0;
+      goto reply;
+    }
+  } else if (prefix == "osd force_healthy_stretch_mode") {
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+    if (!sure) {
+      ss << "This command will require peering across multiple CRUSH buckets "
+	"(probably two data centers or availability zones?) and may result in PGs "
+	"going inactive until backfilling is complete. Pass --yes-i-really-mean-it to proceed.";
+      err = -EPERM;
+      goto reply;
+    }
+    try_end_recovery_stretch_mode(true);
+    ss << "Triggering healthy stretch mode";
+    err = 0;
+    goto reply;
+  } else if (prefix == "osd force_recovery_stretch_mode") {
+    bool sure = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", sure);
+    if (!sure) {
+      ss << "This command will increase pool sizes to try and spread them "
+	"across multiple CRUSH buckets (probably two data centers or "
+	"availability zones?) and should have happened automatically"
+	"Pass --yes-i-really-mean-it to proceed.";
+      err = -EPERM;
+      goto reply;
+    }
+    mon.go_recovery_stretch_mode();
+    ss << "Triggering recovery stretch mode";
+    err = 0;
+    goto reply;
+  } else {
+    err = -EINVAL;
+  }
+
+ reply:
+  getline(ss, rs);
+  if (err < 0 && rs.length() == 0)
+    rs = cpp_strerror(err);
+  mon.reply_command(op, err, rs, rdata, get_last_committed());
+  return ret;
+
+ update:
+  getline(ss, rs);
+  wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
+					    get_last_committed() + 1));
+  return true;
+
+ wait:
+  wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+  return true;
+}
+
+bool OSDMonitor::enforce_pool_op_caps(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+
+  auto m = op->get_req<MPoolOp>();
+  MonSession *session = op->get_session();
+  if (!session) {
+    _pool_op_reply(op, -EPERM, osdmap.get_epoch());
+    return true;
+  }
+
+  switch (m->op) {
+  case POOL_OP_CREATE_UNMANAGED_SNAP:
+  case POOL_OP_DELETE_UNMANAGED_SNAP:
+    {
+      const std::string* pool_name = nullptr;
+      const pg_pool_t *pg_pool = osdmap.get_pg_pool(m->pool);
+      if (pg_pool != nullptr) {
+        pool_name = &osdmap.get_pool_name(m->pool);
+      }
+
+      if (!is_unmanaged_snap_op_permitted(cct, mon.key_server,
+                                          session->entity_name, session->caps,
+					  session->get_peer_socket_addr(),
+                                          pool_name)) {
+        dout(0) << "got unmanaged-snap pool op from entity with insufficient "
+                << "privileges. message: " << *m  << std::endl
+                << "caps: " << session->caps << dendl;
+        _pool_op_reply(op, -EPERM, osdmap.get_epoch());
+        return true;
+      }
+    }
+    break;
+  default:
+    if (!session->is_capable("osd", MON_CAP_W)) {
+      dout(0) << "got pool op from entity with insufficient privileges. "
+              << "message: " << *m  << std::endl
+              << "caps: " << session->caps << dendl;
+      _pool_op_reply(op, -EPERM, osdmap.get_epoch());
+      return true;
+    }
+    break;
+  }
+
+  return false;
+}
+
+bool OSDMonitor::preprocess_pool_op(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MPoolOp>();
+
+  if (enforce_pool_op_caps(op)) {
+    return true;
+  }
+
+  if (m->fsid != mon.monmap->fsid) {
+    dout(0) << __func__ << " drop message on fsid " << m->fsid
+            << " != " << mon.monmap->fsid << " for " << *m << dendl;
+    _pool_op_reply(op, -EINVAL, osdmap.get_epoch());
+    return true;
+  }
+
+  if (m->op == POOL_OP_CREATE)
+    return preprocess_pool_op_create(op);
+
+  const pg_pool_t *p = osdmap.get_pg_pool(m->pool);
+  if (p == nullptr) {
+    dout(10) << "attempt to operate on non-existent pool id " << m->pool << dendl;
+    if (m->op == POOL_OP_DELETE) {
+      _pool_op_reply(op, 0, osdmap.get_epoch());
+    } else {
+      _pool_op_reply(op, -ENOENT, osdmap.get_epoch());
+    }
+    return true;
+  }
+
+  // check if the snap and snapname exist
+  bool snap_exists = false;
+  if (p->snap_exists(m->name.c_str()))
+    snap_exists = true;
+
+  switch (m->op) {
+  case POOL_OP_CREATE_SNAP:
+    if (p->is_unmanaged_snaps_mode() || p->is_tier()) {
+      _pool_op_reply(op, -EINVAL, osdmap.get_epoch());
+      return true;
+    }
+    if (snap_exists) {
+      _pool_op_reply(op, 0, osdmap.get_epoch());
+      return true;
+    }
+    return false;
+  case POOL_OP_CREATE_UNMANAGED_SNAP:
+    if (p->is_pool_snaps_mode()) {
+      _pool_op_reply(op, -EINVAL, osdmap.get_epoch());
+      return true;
+    }
+    return false;
+  case POOL_OP_DELETE_SNAP:
+    if (p->is_unmanaged_snaps_mode()) {
+      _pool_op_reply(op, -EINVAL, osdmap.get_epoch());
+      return true;
+    }
+    if (!snap_exists) {
+      _pool_op_reply(op, 0, osdmap.get_epoch());
+      return true;
+    }
+    return false;
+  case POOL_OP_DELETE_UNMANAGED_SNAP:
+    if (p->is_pool_snaps_mode()) {
+      _pool_op_reply(op, -EINVAL, osdmap.get_epoch());
+      return true;
+    }
+    if (_is_removed_snap(m->pool, m->snapid)) {
+      _pool_op_reply(op, 0, osdmap.get_epoch());
+      return true;
+    }
+    return false;
+  case POOL_OP_DELETE:
+    if (osdmap.lookup_pg_pool_name(m->name.c_str()) >= 0) {
+      _pool_op_reply(op, 0, osdmap.get_epoch());
+      return true;
+    }
+    return false;
+  case POOL_OP_AUID_CHANGE:
+    return false;
+  default:
+    ceph_abort();
+    break;
+  }
+
+  return false;
+}
+
+bool OSDMonitor::_is_removed_snap(int64_t pool, snapid_t snap)
+{
+  if (!osdmap.have_pg_pool(pool)) {
+    dout(10) << __func__ << " pool " << pool << " snap " << snap
+	     << " - pool dne" << dendl;
+    return true;
+  }
+  if (osdmap.in_removed_snaps_queue(pool, snap)) {
+    dout(10) << __func__ << " pool " << pool << " snap " << snap
+	     << " - in osdmap removed_snaps_queue" << dendl;
+    return true;
+  }
+  snapid_t begin, end;
+  int r = lookup_purged_snap(pool, snap, &begin, &end);
+  if (r == 0) {
+    dout(10) << __func__ << " pool " << pool << " snap " << snap
+	     << " - purged, [" << begin << "," << end << ")" << dendl;
+    return true;
+  }
+  return false;
+}
+
+bool OSDMonitor::_is_pending_removed_snap(int64_t pool, snapid_t snap)
+{
+  if (pending_inc.old_pools.count(pool)) {
+    dout(10) << __func__ << " pool " << pool << " snap " << snap
+	     << " - pool pending deletion" << dendl;
+    return true;
+  }
+  if (pending_inc.in_new_removed_snaps(pool, snap)) {
+    dout(10) << __func__ << " pool " << pool << " snap " << snap
+	     << " - in pending new_removed_snaps" << dendl;
+    return true;
+  }
+  return false;
+}
+
+bool OSDMonitor::preprocess_pool_op_create(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MPoolOp>();
+  int64_t pool = osdmap.lookup_pg_pool_name(m->name.c_str());
+  if (pool >= 0) {
+    _pool_op_reply(op, 0, osdmap.get_epoch());
+    return true;
+  }
+
+  return false;
+}
+
+bool OSDMonitor::prepare_pool_op(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MPoolOp>();
+  dout(10) << "prepare_pool_op " << *m << dendl;
+  if (m->op == POOL_OP_CREATE) {
+    return prepare_pool_op_create(op);
+  } else if (m->op == POOL_OP_DELETE) {
+    return prepare_pool_op_delete(op);
+  }
+
+  int ret = 0;
+  bool changed = false;
+
+  if (!osdmap.have_pg_pool(m->pool)) {
+    _pool_op_reply(op, -ENOENT, osdmap.get_epoch());
+    return false;
+  }
+
+  const pg_pool_t *pool = osdmap.get_pg_pool(m->pool);
+
+  switch (m->op) {
+    case POOL_OP_CREATE_SNAP:
+      if (pool->is_tier()) {
+        ret = -EINVAL;
+        _pool_op_reply(op, ret, osdmap.get_epoch());
+        return false;
+      }  // else, fall through
+    case POOL_OP_DELETE_SNAP:
+      if (!pool->is_unmanaged_snaps_mode()) {
+        bool snap_exists = pool->snap_exists(m->name.c_str());
+        if ((m->op == POOL_OP_CREATE_SNAP && snap_exists)
+          || (m->op == POOL_OP_DELETE_SNAP && !snap_exists)) {
+          ret = 0;
+        } else {
+          break;
+        }
+      } else {
+        ret = -EINVAL;
+      }
+      _pool_op_reply(op, ret, osdmap.get_epoch());
+      return false;
+
+    case POOL_OP_DELETE_UNMANAGED_SNAP:
+      // we won't allow removal of an unmanaged snapshot from a pool
+      // not in unmanaged snaps mode.
+      if (!pool->is_unmanaged_snaps_mode()) {
+        _pool_op_reply(op, -ENOTSUP, osdmap.get_epoch());
+        return false;
+      }
+      /* fall-thru */
+    case POOL_OP_CREATE_UNMANAGED_SNAP:
+      // but we will allow creating an unmanaged snapshot on any pool
+      // as long as it is not in 'pool' snaps mode.
+      if (pool->is_pool_snaps_mode()) {
+        _pool_op_reply(op, -EINVAL, osdmap.get_epoch());
+        return false;
+      }
+  }
+
+  // projected pool info
+  pg_pool_t pp;
+  if (pending_inc.new_pools.count(m->pool))
+    pp = pending_inc.new_pools[m->pool];
+  else
+    pp = *osdmap.get_pg_pool(m->pool);
+
+  bufferlist reply_data;
+
+  // pool snaps vs unmanaged snaps are mutually exclusive
+  switch (m->op) {
+  case POOL_OP_CREATE_SNAP:
+  case POOL_OP_DELETE_SNAP:
+    if (pp.is_unmanaged_snaps_mode()) {
+      ret = -EINVAL;
+      goto out;
+    }
+    break;
+
+  case POOL_OP_CREATE_UNMANAGED_SNAP:
+  case POOL_OP_DELETE_UNMANAGED_SNAP:
+    if (pp.is_pool_snaps_mode()) {
+      ret = -EINVAL;
+      goto out;
+    }
+  }
+
+  switch (m->op) {
+  case POOL_OP_CREATE_SNAP:
+    if (!pp.snap_exists(m->name.c_str())) {
+      pp.add_snap(m->name.c_str(), ceph_clock_now());
+      dout(10) << "create snap in pool " << m->pool << " " << m->name
+	       << " seq " << pp.get_snap_epoch() << dendl;
+      changed = true;
+    }
+    break;
+
+  case POOL_OP_DELETE_SNAP:
+    {
+      snapid_t s = pp.snap_exists(m->name.c_str());
+      if (s) {
+	pp.remove_snap(s);
+	pending_inc.new_removed_snaps[m->pool].insert(s);
+	changed = true;
+      }
+    }
+    break;
+
+  case POOL_OP_CREATE_UNMANAGED_SNAP:
+    {
+      uint64_t snapid = pp.add_unmanaged_snap(
+	osdmap.require_osd_release < ceph_release_t::octopus);
+      encode(snapid, reply_data);
+      changed = true;
+    }
+    break;
+
+  case POOL_OP_DELETE_UNMANAGED_SNAP:
+    if (!_is_removed_snap(m->pool, m->snapid) &&
+	!_is_pending_removed_snap(m->pool, m->snapid)) {
+      if (m->snapid > pp.get_snap_seq()) {
+        _pool_op_reply(op, -ENOENT, osdmap.get_epoch());
+        return false;
+      }
+      pp.remove_unmanaged_snap(
+	m->snapid,
+	osdmap.require_osd_release < ceph_release_t::octopus);
+      pending_inc.new_removed_snaps[m->pool].insert(m->snapid);
+      // also record the new seq as purged: this avoids a discontinuity
+      // after all of the snaps have been purged, since the seq assigned
+      // during removal lives in the same namespace as the actual snaps.
+      pending_pseudo_purged_snaps[m->pool].insert(pp.get_snap_seq());
+      changed = true;
+    }
+    break;
+
+  case POOL_OP_AUID_CHANGE:
+    _pool_op_reply(op, -EOPNOTSUPP, osdmap.get_epoch());
+    return false;
+
+  default:
+    ceph_abort();
+    break;
+  }
+
+  if (changed) {
+    pp.set_snap_epoch(pending_inc.epoch);
+    pending_inc.new_pools[m->pool] = pp;
+  }
+
+ out:
+  wait_for_finished_proposal(op, new OSDMonitor::C_PoolOp(this, op, ret, pending_inc.epoch, &reply_data));
+  return true;
+}
+
+bool OSDMonitor::prepare_pool_op_create(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  int err = prepare_new_pool(op);
+  wait_for_finished_proposal(op, new OSDMonitor::C_PoolOp(this, op, err, pending_inc.epoch));
+  return true;
+}
+
+int OSDMonitor::_check_remove_pool(int64_t pool_id, const pg_pool_t& pool,
+				   ostream *ss)
+{
+  const string& poolstr = osdmap.get_pool_name(pool_id);
+
+  // If the Pool is in use by CephFS, refuse to delete it
+  FSMap const &pending_fsmap = mon.mdsmon()->get_pending_fsmap();
+  if (pending_fsmap.pool_in_use(pool_id)) {
+    *ss << "pool '" << poolstr << "' is in use by CephFS";
+    return -EBUSY;
+  }
+
+  if (pool.tier_of >= 0) {
+    *ss << "pool '" << poolstr << "' is a tier of '"
+	<< osdmap.get_pool_name(pool.tier_of) << "'";
+    return -EBUSY;
+  }
+  if (!pool.tiers.empty()) {
+    *ss << "pool '" << poolstr << "' has tiers";
+    for(auto tier : pool.tiers) {
+      *ss << " " << osdmap.get_pool_name(tier);
+    }
+    return -EBUSY;
+  }
+
+  if (!g_conf()->mon_allow_pool_delete) {
+    *ss << "pool deletion is disabled; you must first set the mon_allow_pool_delete config option to true before you can destroy a pool";
+    return -EPERM;
+  }
+
+  if (pool.has_flag(pg_pool_t::FLAG_NODELETE)) {
+    *ss << "pool deletion is disabled; you must unset nodelete flag for the pool first";
+    return -EPERM;
+  }
+
+  *ss << "pool '" << poolstr << "' removed";
+  return 0;
+}
+
+/**
+ * Check if it is safe to add a tier to a base pool
+ *
+ * @return
+ * True if the operation should proceed, false if we should abort here
+ * (abort doesn't necessarily mean error, could be idempotency)
+ */
+bool OSDMonitor::_check_become_tier(
+    const int64_t tier_pool_id, const pg_pool_t *tier_pool,
+    const int64_t base_pool_id, const pg_pool_t *base_pool,
+    int *err,
+    ostream *ss) const
+{
+  const std::string &tier_pool_name = osdmap.get_pool_name(tier_pool_id);
+  const std::string &base_pool_name = osdmap.get_pool_name(base_pool_id);
+
+  const FSMap &pending_fsmap = mon.mdsmon()->get_pending_fsmap();
+  if (pending_fsmap.pool_in_use(tier_pool_id)) {
+    *ss << "pool '" << tier_pool_name << "' is in use by CephFS";
+    *err = -EBUSY;
+    return false;
+  }
+
+  if (base_pool->tiers.count(tier_pool_id)) {
+    ceph_assert(tier_pool->tier_of == base_pool_id);
+    *err = 0;
+    *ss << "pool '" << tier_pool_name << "' is now (or already was) a tier of '"
+      << base_pool_name << "'";
+    return false;
+  }
+
+  if (base_pool->is_tier()) {
+    *ss << "pool '" << base_pool_name << "' is already a tier of '"
+      << osdmap.get_pool_name(base_pool->tier_of) << "', "
+      << "multiple tiers are not yet supported.";
+    *err = -EINVAL;
+    return false;
+  }
+
+  if (tier_pool->has_tiers()) {
+    *ss << "pool '" << tier_pool_name << "' has following tier(s) already:";
+    for (set<uint64_t>::iterator it = tier_pool->tiers.begin();
+         it != tier_pool->tiers.end(); ++it)
+      *ss << "'" << osdmap.get_pool_name(*it) << "',";
+    *ss << " multiple tiers are not yet supported.";
+    *err = -EINVAL;
+    return false;
+  }
+
+  if (tier_pool->is_tier()) {
+    *ss << "tier pool '" << tier_pool_name << "' is already a tier of '"
+       << osdmap.get_pool_name(tier_pool->tier_of) << "'";
+    *err = -EINVAL;
+    return false;
+  }
+
+  *err = 0;
+  return true;
+}
+
+
+/**
+ * Check if it is safe to remove a tier from this base pool
+ *
+ * @return
+ * True if the operation should proceed, false if we should abort here
+ * (abort doesn't necessarily mean error, could be idempotency)
+ */
+bool OSDMonitor::_check_remove_tier(
+    const int64_t base_pool_id, const pg_pool_t *base_pool,
+    const pg_pool_t *tier_pool,
+    int *err, ostream *ss) const
+{
+  const std::string &base_pool_name = osdmap.get_pool_name(base_pool_id);
+
+  // Apply CephFS-specific checks
+  const FSMap &pending_fsmap = mon.mdsmon()->get_pending_fsmap();
+  if (pending_fsmap.pool_in_use(base_pool_id)) {
+    if (base_pool->is_erasure() && !base_pool->allows_ecoverwrites()) {
+      // If the underlying pool is erasure coded and does not allow EC
+      // overwrites, we can't permit the removal of the replicated tier that
+      // CephFS relies on to access it
+      *ss << "pool '" << base_pool_name <<
+          "' does not allow EC overwrites and is in use by CephFS"
+          " via its tier";
+      *err = -EBUSY;
+      return false;
+    }
+
+    if (tier_pool && tier_pool->cache_mode == pg_pool_t::CACHEMODE_WRITEBACK) {
+      *ss << "pool '" << base_pool_name << "' is in use by CephFS, and this "
+             "tier is still in use as a writeback cache.  Change the cache "
+             "mode and flush the cache before removing it";
+      *err = -EBUSY;
+      return false;
+    }
+  }
+
+  *err = 0;
+  return true;
+}
+
+int OSDMonitor::_prepare_remove_pool(
+  int64_t pool, ostream *ss, bool no_fake)
+{
+  dout(10) << __func__ << " " << pool << dendl;
+  const pg_pool_t *p = osdmap.get_pg_pool(pool);
+  int r = _check_remove_pool(pool, *p, ss);
+  if (r < 0)
+    return r;
+
+  auto new_pool = pending_inc.new_pools.find(pool);
+  if (new_pool != pending_inc.new_pools.end()) {
+    // if there is a problem with the pending info, wait and retry
+    // this op.
+    const auto& p = new_pool->second;
+    int r = _check_remove_pool(pool, p, ss);
+    if (r < 0)
+      return -EAGAIN;
+  }
+
+  if (pending_inc.old_pools.count(pool)) {
+    dout(10) << __func__ << " " << pool << " already pending removal"
+	     << dendl;
+    return 0;
+  }
+
+  if (g_conf()->mon_fake_pool_delete && !no_fake) {
+    string old_name = osdmap.get_pool_name(pool);
+    string new_name = old_name + "." + stringify(pool) + ".DELETED";
+    dout(1) << __func__ << " faking pool deletion: renaming " << pool << " "
+	    << old_name << " -> " << new_name << dendl;
+    pending_inc.new_pool_names[pool] = new_name;
+    return 0;
+  }
+
+  // remove
+  pending_inc.old_pools.insert(pool);
+
+  // remove any pg_temp mappings for this pool
+  for (auto p = osdmap.pg_temp->begin();
+       p != osdmap.pg_temp->end();
+       ++p) {
+    if (p->first.pool() == pool) {
+      dout(10) << __func__ << " " << pool << " removing obsolete pg_temp "
+	       << p->first << dendl;
+      pending_inc.new_pg_temp[p->first].clear();
+    }
+  }
+  // remove any primary_temp mappings for this pool
+  for (auto p = osdmap.primary_temp->begin();
+      p != osdmap.primary_temp->end();
+      ++p) {
+    if (p->first.pool() == pool) {
+      dout(10) << __func__ << " " << pool
+               << " removing obsolete primary_temp" << p->first << dendl;
+      pending_inc.new_primary_temp[p->first] = -1;
+    }
+  }
+  // remove any pg_upmap mappings for this pool
+  for (auto& p : osdmap.pg_upmap) {
+    if (p.first.pool() == pool) {
+      dout(10) << __func__ << " " << pool
+               << " removing obsolete pg_upmap "
+               << p.first << dendl;
+      pending_inc.old_pg_upmap.insert(p.first);
+    }
+  }
+  // remove any pending pg_upmap mappings for this pool
+  {
+    auto it = pending_inc.new_pg_upmap.begin();
+    while (it != pending_inc.new_pg_upmap.end()) {
+      if (it->first.pool() == pool) {
+        dout(10) << __func__ << " " << pool
+                 << " removing pending pg_upmap "
+                 << it->first << dendl;
+        it = pending_inc.new_pg_upmap.erase(it);
+      } else {
+        it++;
+      }
+    }
+  }
+  // remove any pg_upmap_items mappings for this pool
+  for (auto& p : osdmap.pg_upmap_items) {
+    if (p.first.pool() == pool) {
+      dout(10) << __func__ << " " << pool
+               << " removing obsolete pg_upmap_items " << p.first
+               << dendl;
+      pending_inc.old_pg_upmap_items.insert(p.first);
+    }
+  }
+  // remove any pending pg_upmap mappings for this pool
+  {
+    auto it = pending_inc.new_pg_upmap_items.begin();
+    while (it != pending_inc.new_pg_upmap_items.end()) {
+      if (it->first.pool() == pool) {
+        dout(10) << __func__ << " " << pool
+                 << " removing pending pg_upmap_items "
+                 << it->first << dendl;
+        it = pending_inc.new_pg_upmap_items.erase(it);
+      } else {
+        it++;
+      }
+    }
+  }
+
+  // remove any choose_args for this pool
+  CrushWrapper newcrush;
+  _get_pending_crush(newcrush);
+  if (newcrush.have_choose_args(pool)) {
+    dout(10) << __func__ << " removing choose_args for pool " << pool << dendl;
+    newcrush.rm_choose_args(pool);
+    pending_inc.crush.clear();
+    newcrush.encode(pending_inc.crush, mon.get_quorum_con_features());
+  }
+  return 0;
+}
+
+int OSDMonitor::_prepare_rename_pool(int64_t pool, string newname)
+{
+  dout(10) << "_prepare_rename_pool " << pool << dendl;
+  if (pending_inc.old_pools.count(pool)) {
+    dout(10) << "_prepare_rename_pool " << pool << " pending removal" << dendl;
+    return -ENOENT;
+  }
+  for (map<int64_t,string>::iterator p = pending_inc.new_pool_names.begin();
+       p != pending_inc.new_pool_names.end();
+       ++p) {
+    if (p->second == newname && p->first != pool) {
+      return -EEXIST;
+    }
+  }
+
+  pending_inc.new_pool_names[pool] = newname;
+  return 0;
+}
+
+bool OSDMonitor::prepare_pool_op_delete(MonOpRequestRef op)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MPoolOp>();
+  ostringstream ss;
+  int ret = _prepare_remove_pool(m->pool, &ss, false);
+  if (ret == -EAGAIN) {
+    wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+    return true;
+  }
+  if (ret < 0)
+    dout(10) << __func__ << " got " << ret << " " << ss.str() << dendl;
+  wait_for_finished_proposal(op, new OSDMonitor::C_PoolOp(this, op, ret,
+						      pending_inc.epoch));
+  return true;
+}
+
+void OSDMonitor::_pool_op_reply(MonOpRequestRef op,
+                                int ret, epoch_t epoch, bufferlist *blp)
+{
+  op->mark_osdmon_event(__func__);
+  auto m = op->get_req<MPoolOp>();
+  dout(20) << "_pool_op_reply " << ret << dendl;
+  MPoolOpReply *reply = new MPoolOpReply(m->fsid, m->get_tid(),
+					 ret, epoch, get_last_committed(), blp);
+  mon.send_reply(op, reply);
+}
+
+void OSDMonitor::convert_pool_priorities(void)
+{
+  pool_opts_t::key_t key = pool_opts_t::get_opt_desc("recovery_priority").key;
+  int64_t max_prio = 0;
+  int64_t min_prio = 0;
+  for (const auto &i : osdmap.get_pools()) {
+    const auto &pool = i.second;
+
+    if (pool.opts.is_set(key)) {
+      int64_t prio = 0;
+      pool.opts.get(key, &prio);
+      if (prio > max_prio)
+	max_prio = prio;
+      if (prio < min_prio)
+	min_prio = prio;
+    }
+  }
+  if (max_prio <= OSD_POOL_PRIORITY_MAX && min_prio >= OSD_POOL_PRIORITY_MIN) {
+    dout(20) << __func__ << " nothing to fix" << dendl;
+    return;
+  }
+  // Current pool priorities exceeds new maximum
+  for (const auto &i : osdmap.get_pools()) {
+    const auto pool_id = i.first;
+    pg_pool_t pool = i.second;
+
+    int64_t prio = 0;
+    pool.opts.get(key, &prio);
+    int64_t n;
+
+    if (prio > 0 && max_prio > OSD_POOL_PRIORITY_MAX) { // Likely scenario
+      // Scaled priority range 0 to OSD_POOL_PRIORITY_MAX
+      n = (float)prio / max_prio * OSD_POOL_PRIORITY_MAX;
+    } else if (prio < 0 && min_prio < OSD_POOL_PRIORITY_MIN) {
+      // Scaled  priority range OSD_POOL_PRIORITY_MIN to 0
+      n = (float)prio / min_prio * OSD_POOL_PRIORITY_MIN;
+    } else {
+      continue;
+    }
+    if (n == 0) {
+      pool.opts.unset(key);
+    } else {
+      pool.opts.set(key, static_cast<int64_t>(n));
+    }
+    dout(10) << __func__ << " pool " << pool_id
+	     << " recovery_priority adjusted "
+	     << prio << " to " << n << dendl;
+    pool.last_change = pending_inc.epoch;
+    pending_inc.new_pools[pool_id] = pool;
+  }
+}
+
+void OSDMonitor::try_enable_stretch_mode_pools(stringstream& ss, bool *okay,
+					       int *errcode,
+					       set<pg_pool_t*>* pools,
+					       const string& new_crush_rule)
+{
+  dout(20) << __func__ << dendl;
+  *okay = false;
+  int new_crush_rule_result = osdmap.crush->get_rule_id(new_crush_rule);
+  if (new_crush_rule_result < 0) {
+    ss << "unrecognized crush rule " << new_crush_rule_result;
+    *errcode = new_crush_rule_result;
+    return;
+  }
+  __u8 new_rule = static_cast<__u8>(new_crush_rule_result);
+  for (const auto& pooli : osdmap.pools) {
+    int64_t poolid = pooli.first;
+    const pg_pool_t *p = &pooli.second;
+    if (!p->is_replicated()) {
+      ss << "stretched pools must be replicated; '" << osdmap.pool_name[poolid] << "' is erasure-coded";
+      *errcode = -EINVAL;
+      return;
+    }
+    uint8_t default_size = g_conf().get_val<uint64_t>("osd_pool_default_size");
+    if ((p->get_size() != default_size ||
+	 (p->get_min_size() != g_conf().get_osd_pool_default_min_size(default_size))) &&
+	(p->get_crush_rule() != new_rule)) {
+      ss << "we currently require stretch mode pools start out with the"
+	" default size/min_size, which '" << osdmap.pool_name[poolid] << "' does not";
+      *errcode = -EINVAL;
+      return;
+    }
+    pg_pool_t *pp = pending_inc.get_new_pool(poolid, p);
+    // TODO: The part where we unconditionally copy the pools into pending_inc is bad
+    // the attempt may fail and then we have these pool updates...but they won't do anything
+    // if there is a failure, so if it's hard to change the interface, no need to bother
+    pools->insert(pp);
+  }
+  *okay = true;
+  return;
+}
+
+void OSDMonitor::try_enable_stretch_mode(stringstream& ss, bool *okay,
+					 int *errcode, bool commit,
+					 const string& dividing_bucket,
+					 uint32_t bucket_count,
+					 const set<pg_pool_t*>& pools,
+					 const string& new_crush_rule)
+{
+  dout(20) << __func__ << dendl;
+  *okay = false;
+  CrushWrapper crush;
+  _get_pending_crush(crush);
+  int dividing_id;
+  int retval = crush.get_validated_type_id(dividing_bucket, &dividing_id);
+  if (retval == -1) {
+    ss << dividing_bucket << " is not a valid crush bucket type";
+    *errcode = -ENOENT;
+    ceph_assert(!commit || retval != -1);
+    return;
+  }
+  vector<int> subtrees;
+  crush.get_subtree_of_type(dividing_id, &subtrees);
+  if (subtrees.size() != 2) {
+    ss << "there are " << subtrees.size() << dividing_bucket
+       << "'s in the cluster but stretch mode currently only works with 2!";
+    *errcode = -EINVAL;
+    ceph_assert(!commit || subtrees.size() == 2);
+    return;
+  }
+
+  int new_crush_rule_result = crush.get_rule_id(new_crush_rule);
+  if (new_crush_rule_result < 0) {
+    ss << "unrecognized crush rule " << new_crush_rule;
+    *errcode = new_crush_rule_result;
+    ceph_assert(!commit || (new_crush_rule_result > 0));
+    return;
+  }
+  __u8 new_rule = static_cast<__u8>(new_crush_rule_result);
+
+  int weight1 = crush.get_item_weight(subtrees[0]);
+  int weight2 = crush.get_item_weight(subtrees[1]);
+  if (weight1 != weight2) {
+    // TODO: I'm really not sure this is a good idea?
+    ss << "the 2 " << dividing_bucket
+       << "instances in the cluster have differing weights "
+       << weight1 << " and " << weight2
+       <<" but stretch mode currently requires they be the same!";
+    *errcode = -EINVAL;
+    ceph_assert(!commit || (weight1 == weight2));
+    return;
+  }
+  if (bucket_count != 2) {
+    ss << "currently we only support 2-site stretch clusters!";
+    *errcode = -EINVAL;
+    ceph_assert(!commit || bucket_count == 2);
+    return;
+  }
+  // TODO: check CRUSH rules for pools so that we are appropriately divided
+  if (commit) {
+    for (auto pool : pools) {
+      pool->crush_rule = new_rule;
+      pool->peering_crush_bucket_count = bucket_count;
+      pool->peering_crush_bucket_target = bucket_count;
+      pool->peering_crush_bucket_barrier = dividing_id;
+      pool->peering_crush_mandatory_member = CRUSH_ITEM_NONE;
+      pool->size = g_conf().get_val<uint64_t>("mon_stretch_pool_size");
+      pool->min_size = g_conf().get_val<uint64_t>("mon_stretch_pool_min_size");
+    }
+    pending_inc.change_stretch_mode = true;
+    pending_inc.stretch_mode_enabled = true;
+    pending_inc.new_stretch_bucket_count = bucket_count;
+    pending_inc.new_degraded_stretch_mode = 0;
+    pending_inc.new_stretch_mode_bucket = dividing_id;
+  }
+  *okay = true;
+  return;
+}
+
+bool OSDMonitor::check_for_dead_crush_zones(const map<string,set<string>>& dead_buckets,
+					    set<int> *really_down_buckets,
+					    set<string> *really_down_mons)
+{
+  dout(20) << __func__ << " with dead mon zones " << dead_buckets << dendl;
+  ceph_assert(is_readable());
+  if (dead_buckets.empty()) return false;
+  set<int> down_cache;
+  bool really_down = false;
+  for (auto dbi : dead_buckets) {
+    const string& bucket_name = dbi.first;
+    ceph_assert(osdmap.crush->name_exists(bucket_name));
+    int bucket_id = osdmap.crush->get_item_id(bucket_name);
+    dout(20) << "Checking " << bucket_name << " id " << bucket_id
+	     << " to see if OSDs are also down" << dendl;
+    bool subtree_down = osdmap.subtree_is_down(bucket_id, &down_cache);
+    if (subtree_down) {
+      dout(20) << "subtree is down!" << dendl;
+      really_down = true;
+      really_down_buckets->insert(bucket_id);
+      really_down_mons->insert(dbi.second.begin(), dbi.second.end());
+    }
+  }
+  dout(10) << "We determined CRUSH buckets " << *really_down_buckets
+	   << " and mons " << *really_down_mons << " are really down" << dendl;
+  return really_down;
+}
+
+void OSDMonitor::trigger_degraded_stretch_mode(const set<int>& dead_buckets,
+					       const set<string>& live_zones)
+{
+  dout(20) << __func__ << dendl;
+  stretch_recovery_triggered.set_from_double(0); // reset this; we can't go clean now!
+  // update the general OSDMap changes
+  pending_inc.change_stretch_mode = true;
+  pending_inc.stretch_mode_enabled = osdmap.stretch_mode_enabled;
+  pending_inc.new_stretch_bucket_count = osdmap.stretch_bucket_count;
+  int new_site_count = osdmap.stretch_bucket_count - dead_buckets.size();
+  ceph_assert(new_site_count == 1); // stretch count 2!
+  pending_inc.new_degraded_stretch_mode = new_site_count;
+  pending_inc.new_recovering_stretch_mode = 0;
+  pending_inc.new_stretch_mode_bucket = osdmap.stretch_mode_bucket;
+
+  // and then apply them to all the pg_pool_ts
+  ceph_assert(live_zones.size() == 1); // only support 2 zones now
+  const string& remaining_site_name = *(live_zones.begin());
+  ceph_assert(osdmap.crush->name_exists(remaining_site_name));
+  int remaining_site = osdmap.crush->get_item_id(remaining_site_name);
+  for (auto pgi : osdmap.pools) {
+    if (pgi.second.peering_crush_bucket_count) {
+      pg_pool_t& newp = *pending_inc.get_new_pool(pgi.first, &pgi.second);
+      newp.peering_crush_bucket_count = new_site_count;
+      newp.peering_crush_mandatory_member = remaining_site;
+      newp.min_size = pgi.second.min_size / 2; // only support 2 zones now
+      newp.set_last_force_op_resend(pending_inc.epoch);
+    }
+  }
+  propose_pending();
+}
+
+void OSDMonitor::trigger_recovery_stretch_mode()
+{
+  dout(20) << __func__ << dendl;
+  stretch_recovery_triggered.set_from_double(0); // reset this so we don't go full-active prematurely
+  pending_inc.change_stretch_mode = true;
+  pending_inc.stretch_mode_enabled = osdmap.stretch_mode_enabled;
+  pending_inc.new_stretch_bucket_count = osdmap.stretch_bucket_count;
+  pending_inc.new_degraded_stretch_mode = osdmap.degraded_stretch_mode;
+  pending_inc.new_recovering_stretch_mode = 1;
+  pending_inc.new_stretch_mode_bucket = osdmap.stretch_mode_bucket;
+
+  for (auto pgi : osdmap.pools) {
+    if (pgi.second.peering_crush_bucket_count) {
+      pg_pool_t& newp = *pending_inc.get_new_pool(pgi.first, &pgi.second);
+      newp.set_last_force_op_resend(pending_inc.epoch);
+    }
+  }
+  propose_pending();
+}
+
+void OSDMonitor::set_degraded_stretch_mode()
+{
+  stretch_recovery_triggered.set_from_double(0);
+}
+
+void OSDMonitor::set_recovery_stretch_mode()
+{
+  if (stretch_recovery_triggered.is_zero()) {
+    stretch_recovery_triggered = ceph_clock_now();
+  }
+}
+
+void OSDMonitor::set_healthy_stretch_mode()
+{
+  stretch_recovery_triggered.set_from_double(0);
+}
+
+void OSDMonitor::notify_new_pg_digest()
+{
+  dout(20) << __func__ << dendl;
+  if (!stretch_recovery_triggered.is_zero()) {
+    try_end_recovery_stretch_mode(false);
+  }
+}
+
+struct CMonExitRecovery : public Context {
+  OSDMonitor *m;
+  bool force;
+  CMonExitRecovery(OSDMonitor *mon, bool f) : m(mon), force(f) {}
+  void finish(int r) {
+    m->try_end_recovery_stretch_mode(force);
+  }
+};
+
+void OSDMonitor::try_end_recovery_stretch_mode(bool force)
+{
+  dout(20) << __func__ << dendl;
+  if (!mon.is_leader()) return;
+  if (!mon.is_degraded_stretch_mode()) return;
+  if (!mon.is_recovering_stretch_mode()) return;
+  if (!is_readable()) {
+    wait_for_readable_ctx(new CMonExitRecovery(this, force));
+    return;
+  }
+
+  if (osdmap.recovering_stretch_mode &&
+      ((!stretch_recovery_triggered.is_zero() &&
+	ceph_clock_now() - g_conf().get_val<double>("mon_stretch_recovery_min_wait") >
+	stretch_recovery_triggered) ||
+       force)) {
+    if (!mon.mgrstatmon()->is_readable()) {
+      mon.mgrstatmon()->wait_for_readable_ctx(new CMonExitRecovery(this, force));
+      return;
+    }
+    const PGMapDigest& pgd = mon.mgrstatmon()->get_digest();
+    double misplaced, degraded, inactive, unknown;
+    pgd.get_recovery_stats(&misplaced, &degraded, &inactive, &unknown);
+    if (force || (degraded == 0.0 && inactive == 0.0 && unknown == 0.0)) {
+      // we can exit degraded stretch mode!
+      mon.trigger_healthy_stretch_mode();
+    }
+  }
+}
+
+void OSDMonitor::trigger_healthy_stretch_mode()
+{
+  ceph_assert(is_writeable());
+  stretch_recovery_triggered.set_from_double(0);
+  pending_inc.change_stretch_mode = true;
+  pending_inc.stretch_mode_enabled = osdmap.stretch_mode_enabled;
+  pending_inc.new_stretch_bucket_count = osdmap.stretch_bucket_count;
+  pending_inc.new_degraded_stretch_mode = 0; // turn off degraded mode...
+  pending_inc.new_recovering_stretch_mode = 0; //...and recovering mode!
+  pending_inc.new_stretch_mode_bucket = osdmap.stretch_mode_bucket;
+  for (auto pgi : osdmap.pools) {
+    if (pgi.second.peering_crush_bucket_count) {
+      pg_pool_t& newp = *pending_inc.get_new_pool(pgi.first, &pgi.second);
+      newp.peering_crush_bucket_count = osdmap.stretch_bucket_count;
+      newp.peering_crush_mandatory_member = CRUSH_ITEM_NONE;
+      newp.min_size = g_conf().get_val<uint64_t>("mon_stretch_pool_min_size");
+      newp.set_last_force_op_resend(pending_inc.epoch);
+    }
+  }
+  propose_pending();
+}
diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
new file mode 100644
index 000000000..e7701a639
--- /dev/null
+++ b/src/mon/OSDMonitor.h
@@ -0,0 +1,874 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+/* Object Store Device (OSD) Monitor
+ */
+
+#ifndef CEPH_OSDMONITOR_H
+#define CEPH_OSDMONITOR_H
+
+#include <map>
+#include <set>
+#include <utility>
+
+#include "include/types.h"
+#include "include/encoding.h"
+#include "common/simple_cache.hpp"
+#include "common/PriorityCache.h"
+#include "msg/Messenger.h"
+
+#include "osd/OSDMap.h"
+#include "osd/OSDMapMapping.h"
+
+#include "CreatingPGs.h"
+#include "PaxosService.h"
+
+#include "erasure-code/ErasureCodeInterface.h"
+#include "mon/MonOpRequest.h"
+#include <boost/functional/hash.hpp>
+
+class Monitor;
+class PGMap;
+struct MonSession;
+class MOSDMap;
+
+
+/// information about a particular peer's failure reports for one osd
+struct failure_reporter_t {
+  utime_t failed_since;     ///< when they think it failed
+  MonOpRequestRef op;       ///< failure op request
+
+  failure_reporter_t() {}
+  failure_reporter_t(utime_t s, MonOpRequestRef op)
+    : failed_since(s), op(op) {}
+  ~failure_reporter_t() { }
+};
+
+/// information about all failure reports for one osd
+struct failure_info_t {
+  std::map<int, failure_reporter_t> reporters;  ///< reporter -> failed_since etc
+  utime_t max_failed_since;                ///< most recent failed_since
+
+  failure_info_t() {}
+
+  utime_t get_failed_since() {
+    if (max_failed_since == utime_t() && !reporters.empty()) {
+      // the old max must have canceled; recalculate.
+      for (auto p = reporters.begin(); p != reporters.end(); ++p)
+	if (p->second.failed_since > max_failed_since)
+	  max_failed_since = p->second.failed_since;
+    }
+    return max_failed_since;
+  }
+
+  // set the message for the latest report.
+  void add_report(int who, utime_t failed_since, MonOpRequestRef op) {
+    [[maybe_unused]] auto [it, new_reporter] =
+      reporters.insert_or_assign(who, failure_reporter_t{failed_since, op});
+    if (new_reporter) {
+      if (max_failed_since != utime_t() && max_failed_since < failed_since) {
+	max_failed_since = failed_since;
+      }
+    }
+  }
+
+  void take_report_messages(std::list<MonOpRequestRef>& ls) {
+    for (auto p = reporters.begin(); p != reporters.end(); ++p) {
+      if (p->second.op) {
+	ls.push_back(p->second.op);
+        p->second.op.reset();
+      }
+    }
+  }
+
+  void cancel_report(int who) {
+    reporters.erase(who);
+    max_failed_since = utime_t();
+  }
+};
+
+
+class LastEpochClean {
+  struct Lec {
+    std::vector<epoch_t> epoch_by_pg;
+    ps_t next_missing = 0;
+    epoch_t floor = std::numeric_limits<epoch_t>::max();
+    void report(unsigned pg_num, ps_t pg, epoch_t last_epoch_clean);
+  };
+  std::map<uint64_t, Lec> report_by_pool;
+public:
+  void report(unsigned pg_num, const pg_t& pg, epoch_t last_epoch_clean);
+  void remove_pool(uint64_t pool);
+  epoch_t get_lower_bound(const OSDMap& latest) const;
+
+  void dump(Formatter *f) const;
+};
+
+
+struct osdmap_manifest_t {
+  // all the maps we have pinned -- i.e., won't be removed unless
+  // they are inside a trim interval.
+  std::set<version_t> pinned;
+
+  osdmap_manifest_t() {}
+
+  version_t get_last_pinned() const
+  {
+    auto it = pinned.crbegin();
+    if (it == pinned.crend()) {
+      return 0;
+    }
+    return *it;
+  }
+
+  version_t get_first_pinned() const
+  {
+    auto it = pinned.cbegin();
+    if (it == pinned.cend()) {
+      return 0;
+    }
+    return *it;
+  }
+
+  bool is_pinned(version_t v) const
+  {
+    return pinned.find(v) != pinned.end();
+  }
+
+  void pin(version_t v)
+  {
+    pinned.insert(v);
+  }
+
+  version_t get_lower_closest_pinned(version_t v) const {
+    auto p = pinned.lower_bound(v);
+    if (p == pinned.cend()) {
+      return 0;
+    } else if (*p > v) {
+      if (p == pinned.cbegin()) {
+        return 0;
+      }
+      --p;
+    }
+    return *p;
+  }
+
+  void encode(ceph::buffer::list& bl) const
+  {
+    ENCODE_START(1, 1, bl);
+    encode(pinned, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(ceph::buffer::list::const_iterator& bl)
+  {
+    DECODE_START(1, bl);
+    decode(pinned, bl);
+    DECODE_FINISH(bl);
+  }
+
+  void decode(ceph::buffer::list& bl) {
+    auto p = bl.cbegin();
+    decode(p);
+  }
+
+  void dump(ceph::Formatter *f) {
+    f->dump_unsigned("first_pinned", get_first_pinned());
+    f->dump_unsigned("last_pinned", get_last_pinned());
+    f->open_array_section("pinned_maps");
+    for (auto& i : pinned) {
+      f->dump_unsigned("epoch", i);
+    }
+    f->close_section();
+ }
+};
+WRITE_CLASS_ENCODER(osdmap_manifest_t);
+
+class OSDMonitor : public PaxosService,
+                   public md_config_obs_t {
+  CephContext *cct;
+
+public:
+  OSDMap osdmap;
+
+  // config observer
+  const char** get_tracked_conf_keys() const override;
+  void handle_conf_change(const ConfigProxy& conf,
+    const std::set<std::string> &changed) override;
+  // [leader]
+  OSDMap::Incremental pending_inc;
+  std::map<int, ceph::buffer::list> pending_metadata;
+  std::set<int>             pending_metadata_rm;
+  std::map<int, failure_info_t> failure_info;
+  std::map<int,utime_t>    down_pending_out;  // osd down -> out
+  bool priority_convert = false;
+  std::map<int64_t,std::set<snapid_t>> pending_pseudo_purged_snaps;
+  std::shared_ptr<PriorityCache::PriCache> rocksdb_binned_kv_cache = nullptr;
+  std::shared_ptr<PriorityCache::Manager> pcm = nullptr;
+  ceph::mutex balancer_lock = ceph::make_mutex("OSDMonitor::balancer_lock");
+
+  std::map<int,double> osd_weight;
+
+  using osdmap_key_t = std::pair<version_t, uint64_t>;
+  using osdmap_cache_t = SimpleLRU<osdmap_key_t,
+                                   ceph::buffer::list,
+                                   std::less<osdmap_key_t>,
+                                   boost::hash<osdmap_key_t>>;
+  osdmap_cache_t inc_osd_cache;
+  osdmap_cache_t full_osd_cache;
+
+  bool has_osdmap_manifest;
+  osdmap_manifest_t osdmap_manifest;
+
+  bool check_failures(utime_t now);
+  bool check_failure(utime_t now, int target_osd, failure_info_t& fi);
+  utime_t get_grace_time(utime_t now, int target_osd, failure_info_t& fi) const;
+  bool is_failure_stale(utime_t now, failure_info_t& fi) const;
+  void force_failure(int target_osd, int by);
+
+  bool _have_pending_crush();
+  CrushWrapper &_get_stable_crush();
+  void _get_pending_crush(CrushWrapper& newcrush);
+
+  enum FastReadType {
+    FAST_READ_OFF,
+    FAST_READ_ON,
+    FAST_READ_DEFAULT
+  };
+
+  struct CleanUpmapJob : public ParallelPGMapper::Job {
+    CephContext *cct;
+    const OSDMap& osdmap;
+    OSDMap::Incremental& pending_inc;
+    // lock to protect pending_inc form changing
+    // when checking is done
+    ceph::mutex pending_inc_lock =
+      ceph::make_mutex("CleanUpmapJob::pending_inc_lock");
+
+    CleanUpmapJob(CephContext *cct, const OSDMap& om, OSDMap::Incremental& pi)
+      : ParallelPGMapper::Job(&om),
+        cct(cct),
+        osdmap(om),
+        pending_inc(pi) {}
+
+    void process(const std::vector<pg_t>& to_check) override {
+      std::vector<pg_t> to_cancel;
+      std::map<pg_t, mempool::osdmap::vector<std::pair<int,int>>> to_remap;
+      osdmap.check_pg_upmaps(cct, to_check, &to_cancel, &to_remap);
+      // don't bother taking lock if nothing changes
+      if (!to_cancel.empty() || !to_remap.empty()) {
+        std::lock_guard l(pending_inc_lock);
+        osdmap.clean_pg_upmaps(cct, &pending_inc, to_cancel, to_remap);
+      }
+    }
+
+    void process(int64_t poolid, unsigned ps_begin, unsigned ps_end) override {}
+    void complete() override {}
+  }; // public as this will need to be accessible from TestTestOSDMap.cc
+
+  // svc
+public:
+  void create_initial() override;
+  void get_store_prefixes(std::set<std::string>& s) const override;
+
+private:
+  void update_from_paxos(bool *need_bootstrap) override;
+  void create_pending() override;  // prepare a new pending
+  void encode_pending(MonitorDBStore::TransactionRef t) override;
+  void on_active() override;
+  void on_restart() override;
+  void on_shutdown() override;
+
+  /* osdmap full map prune */
+  void load_osdmap_manifest();
+  bool should_prune() const;
+  void _prune_update_trimmed(
+      MonitorDBStore::TransactionRef tx,
+      version_t first);
+  void prune_init(osdmap_manifest_t& manifest);
+  bool _prune_sanitize_options() const;
+  bool is_prune_enabled() const;
+  bool is_prune_supported() const;
+  bool do_prune(MonitorDBStore::TransactionRef tx);
+
+  // Priority cache control
+  uint32_t mon_osd_cache_size = 0;  ///< Number of cached OSDMaps
+  uint64_t rocksdb_cache_size = 0;  ///< Cache for kv Db
+  double cache_kv_ratio = 0;        ///< Cache ratio dedicated to kv
+  double cache_inc_ratio = 0;       ///< Cache ratio dedicated to inc
+  double cache_full_ratio = 0;      ///< Cache ratio dedicated to full
+  uint64_t mon_memory_base = 0;     ///< Mon base memory for cache autotuning
+  double mon_memory_fragmentation = 0; ///< Expected memory fragmentation
+  uint64_t mon_memory_target = 0;   ///< Mon target memory for cache autotuning
+  uint64_t mon_memory_min = 0;      ///< Min memory to cache osdmaps
+  bool mon_memory_autotune = false; ///< Cache auto tune setting
+  int register_cache_with_pcm();
+  int _set_cache_sizes();
+  int _set_cache_ratios();
+  void _set_new_cache_sizes();
+  void _set_cache_autotuning();
+  int _update_mon_cache_settings();
+
+  friend struct OSDMemCache;
+  friend struct IncCache;
+  friend struct FullCache;
+
+  /**
+   * we haven't delegated full version stashing to paxosservice for some time
+   * now, making this function useless in current context.
+   */
+  void encode_full(MonitorDBStore::TransactionRef t) override { }
+  /**
+   * do not let paxosservice periodically stash full osdmaps, or we will break our
+   * locally-managed full maps.  (update_from_paxos loads the latest and writes them
+   * out going forward from there, but if we just synced that may mean we skip some.)
+   */
+  bool should_stash_full() override {
+    return false;
+  }
+
+  /**
+   * hook into trim to include the oldest full map in the trim transaction
+   *
+   * This ensures that anyone post-sync will have enough to rebuild their
+   * full osdmaps.
+   */
+  void encode_trim_extra(MonitorDBStore::TransactionRef tx, version_t first) override;
+
+  void update_msgr_features();
+  /**
+   * check if the cluster supports the features required by the
+   * given crush map. Outputs the daemons which don't support it
+   * to the stringstream.
+   *
+   * @returns true if the map is passable, false otherwise
+   */
+  bool validate_crush_against_features(const CrushWrapper *newcrush,
+				       std::stringstream &ss);
+  void check_osdmap_subs();
+  void share_map_with_random_osd();
+
+  ceph::mutex prime_pg_temp_lock =
+    ceph::make_mutex("OSDMonitor::prime_pg_temp_lock");
+  struct PrimeTempJob : public ParallelPGMapper::Job {
+    OSDMonitor *osdmon;
+    PrimeTempJob(const OSDMap& om, OSDMonitor *m)
+      : ParallelPGMapper::Job(&om), osdmon(m) {}
+    void process(int64_t pool, unsigned ps_begin, unsigned ps_end) override {
+      for (unsigned ps = ps_begin; ps < ps_end; ++ps) {
+	pg_t pgid(ps, pool);
+	osdmon->prime_pg_temp(*osdmap, pgid);
+      }
+    }
+    void process(const std::vector<pg_t>& pgs) override {}
+    void complete() override {}
+  };
+  void maybe_prime_pg_temp();
+  void prime_pg_temp(const OSDMap& next, pg_t pgid);
+
+  ParallelPGMapper mapper;                        ///< for background pg work
+  OSDMapMapping mapping;                          ///< pg <-> osd mappings
+  std::unique_ptr<ParallelPGMapper::Job> mapping_job;  ///< background mapping job
+  void start_mapping();
+
+  void update_logger();
+
+  void handle_query(PaxosServiceMessage *m);
+  bool preprocess_query(MonOpRequestRef op) override;  // true if processed.
+  bool prepare_update(MonOpRequestRef op) override;
+  bool should_propose(double &delay) override;
+
+  version_t get_trim_to() const override;
+
+  bool can_mark_down(int o);
+  bool can_mark_up(int o);
+  bool can_mark_out(int o);
+  bool can_mark_in(int o);
+
+  // ...
+  MOSDMap *build_latest_full(uint64_t features);
+  MOSDMap *build_incremental(epoch_t first, epoch_t last, uint64_t features);
+  void send_full(MonOpRequestRef op);
+  void send_incremental(MonOpRequestRef op, epoch_t first);
+public:
+  /**
+   * Make sure the existing (up) OSDs support the given features
+   * @return 0 on success, or an error code if any OSDs re missing features.
+   * @param ss Filled in with ane explanation of failure, if any
+   */
+  int check_cluster_features(uint64_t features, std::stringstream &ss);
+  // @param req an optional op request, if the osdmaps are replies to it. so
+  //            @c Monitor::send_reply() can mark_event with it.
+  void send_incremental(epoch_t first, MonSession *session, bool onetime,
+			MonOpRequestRef req = MonOpRequestRef());
+
+private:
+  void print_utilization(std::ostream &out, ceph::Formatter *f, bool tree) const;
+
+  bool check_source(MonOpRequestRef op, uuid_d fsid);
+ 
+  bool preprocess_get_osdmap(MonOpRequestRef op);
+
+  bool preprocess_mark_me_down(MonOpRequestRef op);
+
+  friend class C_AckMarkedDown;
+  bool preprocess_failure(MonOpRequestRef op);
+  bool prepare_failure(MonOpRequestRef op);
+  bool prepare_mark_me_down(MonOpRequestRef op);
+  void process_failures();
+  void take_all_failures(std::list<MonOpRequestRef>& ls);
+
+  bool preprocess_mark_me_dead(MonOpRequestRef op);
+  bool prepare_mark_me_dead(MonOpRequestRef op);
+
+  bool preprocess_full(MonOpRequestRef op);
+  bool prepare_full(MonOpRequestRef op);
+
+  bool preprocess_boot(MonOpRequestRef op);
+  bool prepare_boot(MonOpRequestRef op);
+  void _booted(MonOpRequestRef op, bool logit);
+
+  void update_up_thru(int from, epoch_t up_thru);
+  bool preprocess_alive(MonOpRequestRef op);
+  bool prepare_alive(MonOpRequestRef op);
+  void _reply_map(MonOpRequestRef op, epoch_t e);
+
+  bool preprocess_pgtemp(MonOpRequestRef op);
+  bool prepare_pgtemp(MonOpRequestRef op);
+
+  bool preprocess_pg_created(MonOpRequestRef op);
+  bool prepare_pg_created(MonOpRequestRef op);
+
+  bool preprocess_pg_ready_to_merge(MonOpRequestRef op);
+  bool prepare_pg_ready_to_merge(MonOpRequestRef op);
+
+  int _check_remove_pool(int64_t pool_id, const pg_pool_t &pool, std::ostream *ss);
+  bool _check_become_tier(
+      int64_t tier_pool_id, const pg_pool_t *tier_pool,
+      int64_t base_pool_id, const pg_pool_t *base_pool,
+      int *err, std::ostream *ss) const;
+  bool _check_remove_tier(
+      int64_t base_pool_id, const pg_pool_t *base_pool, const pg_pool_t *tier_pool,
+      int *err, std::ostream *ss) const;
+
+  int _prepare_remove_pool(int64_t pool, std::ostream *ss, bool no_fake);
+  int _prepare_rename_pool(int64_t pool, std::string newname);
+
+  bool enforce_pool_op_caps(MonOpRequestRef op);
+  bool preprocess_pool_op (MonOpRequestRef op);
+  bool preprocess_pool_op_create (MonOpRequestRef op);
+  bool prepare_pool_op (MonOpRequestRef op);
+  bool prepare_pool_op_create (MonOpRequestRef op);
+  bool prepare_pool_op_delete(MonOpRequestRef op);
+  int crush_rename_bucket(const std::string& srcname,
+			  const std::string& dstname,
+			  std::ostream *ss);
+  void check_legacy_ec_plugin(const std::string& plugin, 
+			      const std::string& profile) const;
+  int normalize_profile(const std::string& profilename, 
+			ceph::ErasureCodeProfile &profile,
+			bool force,
+			std::ostream *ss);
+  int crush_rule_create_erasure(const std::string &name,
+				const std::string &profile,
+				int *rule,
+				std::ostream *ss);
+  int get_crush_rule(const std::string &rule_name,
+		     int *crush_rule,
+		     std::ostream *ss);
+  int get_erasure_code(const std::string &erasure_code_profile,
+		       ceph::ErasureCodeInterfaceRef *erasure_code,
+		       std::ostream *ss) const;
+  int prepare_pool_crush_rule(const unsigned pool_type,
+			      const std::string &erasure_code_profile,
+			      const std::string &rule_name,
+			      int *crush_rule,
+			      std::ostream *ss);
+  bool erasure_code_profile_in_use(
+    const mempool::osdmap::map<int64_t, pg_pool_t> &pools,
+    const std::string &profile,
+    std::ostream *ss);
+  int parse_erasure_code_profile(const std::vector<std::string> &erasure_code_profile,
+				 std::map<std::string,std::string> *erasure_code_profile_map,
+				 std::ostream *ss);
+  int prepare_pool_size(const unsigned pool_type,
+			const std::string &erasure_code_profile,
+                        uint8_t repl_size,
+			unsigned *size, unsigned *min_size,
+			std::ostream *ss);
+  int prepare_pool_stripe_width(const unsigned pool_type,
+				const std::string &erasure_code_profile,
+				unsigned *stripe_width,
+				std::ostream *ss);
+  int check_pg_num(int64_t pool, int pg_num, int size, std::ostream* ss);
+  int prepare_new_pool(std::string& name,
+		       int crush_rule,
+		       const std::string &crush_rule_name,
+                       unsigned pg_num, unsigned pgp_num,
+		       unsigned pg_num_min,
+		       unsigned pg_num_max,
+                       uint64_t repl_size,
+		       const uint64_t target_size_bytes,
+		       const float target_size_ratio,
+		       const std::string &erasure_code_profile,
+                       const unsigned pool_type,
+                       const uint64_t expected_num_objects,
+                       FastReadType fast_read,
+		       const std::string& pg_autoscale_mode,
+		       bool bulk,
+		       std::ostream *ss);
+  int prepare_new_pool(MonOpRequestRef op);
+
+  void set_pool_flags(int64_t pool_id, uint64_t flags);
+  void clear_pool_flags(int64_t pool_id, uint64_t flags);
+  bool update_pools_status();
+
+  bool _is_removed_snap(int64_t pool_id, snapid_t snapid);
+  bool _is_pending_removed_snap(int64_t pool_id, snapid_t snapid);
+
+  std::string make_purged_snap_epoch_key(epoch_t epoch);
+  std::string make_purged_snap_key(int64_t pool, snapid_t snap);
+  std::string make_purged_snap_key_value(int64_t pool, snapid_t snap, snapid_t num,
+				    epoch_t epoch, ceph::buffer::list *v);
+
+  bool try_prune_purged_snaps();
+  int lookup_purged_snap(int64_t pool, snapid_t snap,
+			 snapid_t *begin, snapid_t *end);
+
+  void insert_purged_snap_update(
+    int64_t pool,
+    snapid_t start, snapid_t end,
+    epoch_t epoch,
+    MonitorDBStore::TransactionRef t);
+
+  bool prepare_set_flag(MonOpRequestRef op, int flag);
+  bool prepare_unset_flag(MonOpRequestRef op, int flag);
+
+  void _pool_op_reply(MonOpRequestRef op,
+                      int ret, epoch_t epoch, ceph::buffer::list *blp=NULL);
+
+  struct C_Booted : public C_MonOp {
+    OSDMonitor *cmon;
+    bool logit;
+    C_Booted(OSDMonitor *cm, MonOpRequestRef op_, bool l=true) :
+      C_MonOp(op_), cmon(cm), logit(l) {}
+    void _finish(int r) override {
+      if (r >= 0)
+	cmon->_booted(op, logit);
+      else if (r == -ECANCELED)
+        return;
+      else if (r == -EAGAIN)
+        cmon->dispatch(op);
+      else
+	ceph_abort_msg("bad C_Booted return value");
+    }
+  };
+
+  struct C_ReplyMap : public C_MonOp {
+    OSDMonitor *osdmon;
+    epoch_t e;
+    C_ReplyMap(OSDMonitor *o, MonOpRequestRef op_, epoch_t ee)
+      : C_MonOp(op_), osdmon(o), e(ee) {}
+    void _finish(int r) override {
+      if (r >= 0)
+	osdmon->_reply_map(op, e);
+      else if (r == -ECANCELED)
+        return;
+      else if (r == -EAGAIN)
+	osdmon->dispatch(op);
+      else
+	ceph_abort_msg("bad C_ReplyMap return value");
+    }    
+  };
+  struct C_PoolOp : public C_MonOp {
+    OSDMonitor *osdmon;
+    int replyCode;
+    int epoch;
+    ceph::buffer::list reply_data;
+    C_PoolOp(OSDMonitor * osd, MonOpRequestRef op_, int rc, int e, ceph::buffer::list *rd=NULL) :
+      C_MonOp(op_), osdmon(osd), replyCode(rc), epoch(e) {
+      if (rd)
+	reply_data = *rd;
+    }
+    void _finish(int r) override {
+      if (r >= 0)
+	osdmon->_pool_op_reply(op, replyCode, epoch, &reply_data);
+      else if (r == -ECANCELED)
+        return;
+      else if (r == -EAGAIN)
+	osdmon->dispatch(op);
+      else
+	ceph_abort_msg("bad C_PoolOp return value");
+    }
+  };
+
+  bool preprocess_remove_snaps(MonOpRequestRef op);
+  bool prepare_remove_snaps(MonOpRequestRef op);
+
+  bool preprocess_get_purged_snaps(MonOpRequestRef op);
+
+  int load_metadata(int osd, std::map<std::string, std::string>& m,
+		    std::ostream *err);
+  void count_metadata(const std::string& field, ceph::Formatter *f);
+
+  void reencode_incremental_map(ceph::buffer::list& bl, uint64_t features);
+  void reencode_full_map(ceph::buffer::list& bl, uint64_t features);
+public:
+  void count_metadata(const std::string& field, std::map<std::string,int> *out);
+  void get_versions(std::map<std::string, std::list<std::string>> &versions);
+protected:
+  int get_osd_objectstore_type(int osd, std::string *type);
+  bool is_pool_currently_all_bluestore(int64_t pool_id, const pg_pool_t &pool,
+				       std::ostream *err);
+
+  // when we last received PG stats from each osd and the osd's osd_beacon_report_interval
+  std::map<int, std::pair<utime_t, int>> last_osd_report;
+  // TODO: use last_osd_report to store the osd report epochs, once we don't
+  //       need to upgrade from pre-luminous releases.
+  std::map<int,epoch_t> osd_epochs;
+  LastEpochClean last_epoch_clean;
+  bool preprocess_beacon(MonOpRequestRef op);
+  bool prepare_beacon(MonOpRequestRef op);
+  epoch_t get_min_last_epoch_clean() const;
+
+  friend class C_UpdateCreatingPGs;
+  std::map<int, std::map<epoch_t, std::set<spg_t>>> creating_pgs_by_osd_epoch;
+  std::vector<pg_t> pending_created_pgs;
+  // the epoch when the pg mapping was calculated
+  epoch_t creating_pgs_epoch = 0;
+  creating_pgs_t creating_pgs;
+  mutable std::mutex creating_pgs_lock;
+
+  creating_pgs_t update_pending_pgs(const OSDMap::Incremental& inc,
+				    const OSDMap& nextmap);
+  unsigned scan_for_creating_pgs(
+    const mempool::osdmap::map<int64_t,pg_pool_t>& pools,
+    const mempool::osdmap::set<int64_t>& removed_pools,
+    utime_t modified,
+    creating_pgs_t* creating_pgs) const;
+  std::pair<int32_t, pg_t> get_parent_pg(pg_t pgid) const;
+  void update_creating_pgs();
+  void check_pg_creates_subs();
+  epoch_t send_pg_creates(int osd, Connection *con, epoch_t next) const;
+
+  int32_t _allocate_osd_id(int32_t* existing_id);
+
+  int get_grace_interval_threshold();
+  bool grace_interval_threshold_exceeded(int last_failed);
+  void set_default_laggy_params(int target_osd);
+
+public:
+  OSDMonitor(CephContext *cct, Monitor &mn, Paxos &p, const std::string& service_name);
+
+  void tick() override;  // check state, take actions
+
+  bool preprocess_command(MonOpRequestRef op);
+  bool prepare_command(MonOpRequestRef op);
+  bool prepare_command_impl(MonOpRequestRef op, const cmdmap_t& cmdmap);
+
+  int validate_osd_create(
+      const int32_t id,
+      const uuid_d& uuid,
+      const bool check_osd_exists,
+      int32_t* existing_id,
+      std::stringstream& ss);
+  int prepare_command_osd_create(
+      const int32_t id,
+      const uuid_d& uuid,
+      int32_t* existing_id,
+      std::stringstream& ss);
+  void do_osd_create(const int32_t id, const uuid_d& uuid,
+		     const std::string& device_class,
+		     int32_t* new_id);
+  int prepare_command_osd_purge(int32_t id, std::stringstream& ss);
+  int prepare_command_osd_destroy(int32_t id, std::stringstream& ss);
+  int _prepare_command_osd_crush_remove(
+      CrushWrapper &newcrush,
+      int32_t id,
+      int32_t ancestor,
+      bool has_ancestor,
+      bool unlink_only);
+  void do_osd_crush_remove(CrushWrapper& newcrush);
+  int prepare_command_osd_crush_remove(
+      CrushWrapper &newcrush,
+      int32_t id,
+      int32_t ancestor,
+      bool has_ancestor,
+      bool unlink_only);
+  int prepare_command_osd_remove(int32_t id);
+  int prepare_command_osd_new(
+      MonOpRequestRef op,
+      const cmdmap_t& cmdmap,
+      const std::map<std::string,std::string>& secrets,
+      std::stringstream &ss,
+      ceph::Formatter *f);
+
+  int prepare_command_pool_set(const cmdmap_t& cmdmap,
+                               std::stringstream& ss);
+
+  int prepare_command_pool_application(const std::string &prefix,
+                                       const cmdmap_t& cmdmap,
+                                       std::stringstream& ss);
+  int preprocess_command_pool_application(const std::string &prefix,
+                                          const cmdmap_t& cmdmap,
+                                          std::stringstream& ss,
+                                          bool *modified);
+  int _command_pool_application(const std::string &prefix,
+				const cmdmap_t& cmdmap,
+				std::stringstream& ss,
+				bool *modified,
+				bool preparing);
+
+  bool handle_osd_timeouts(const utime_t &now,
+			   std::map<int, std::pair<utime_t, int>> &last_osd_report);
+
+  void send_latest(MonOpRequestRef op, epoch_t start=0);
+  void send_latest_now_nodelete(MonOpRequestRef op, epoch_t start=0) {
+    op->mark_osdmon_event(__func__);
+    send_incremental(op, start);
+  }
+
+  int get_version(version_t ver, ceph::buffer::list& bl) override;
+  int get_version(version_t ver, uint64_t feature, ceph::buffer::list& bl);
+
+  int get_version_full(version_t ver, uint64_t feature, ceph::buffer::list& bl);
+  int get_version_full(version_t ver, ceph::buffer::list& bl) override;
+  int get_inc(version_t ver, OSDMap::Incremental& inc);
+  int get_full_from_pinned_map(version_t ver, ceph::buffer::list& bl);
+
+  epoch_t blocklist(const entity_addrvec_t& av, utime_t until);
+  epoch_t blocklist(entity_addr_t a, utime_t until);
+
+  void dump_info(ceph::Formatter *f);
+  int dump_osd_metadata(int osd, ceph::Formatter *f, std::ostream *err);
+  void print_nodes(ceph::Formatter *f);
+
+  void check_osdmap_sub(Subscription *sub);
+  void check_pg_creates_sub(Subscription *sub);
+
+  void do_application_enable(int64_t pool_id, const std::string &app_name,
+			     const std::string &app_key="",
+			     const std::string &app_value="",
+			     bool force=false);
+  void do_set_pool_opt(int64_t pool_id, pool_opts_t::key_t opt,
+		       pool_opts_t::value_t);
+
+  void add_flag(int flag) {
+    if (!(osdmap.flags & flag)) {
+      if (pending_inc.new_flags < 0)
+	pending_inc.new_flags = osdmap.flags;
+      pending_inc.new_flags |= flag;
+    }
+  }
+
+  void remove_flag(int flag) {
+    if(osdmap.flags & flag) {
+      if (pending_inc.new_flags < 0)
+	pending_inc.new_flags = osdmap.flags;
+      pending_inc.new_flags &= ~flag;
+    }
+  }
+  void convert_pool_priorities(void);
+  /**
+   * Find the pools which are requested to be put into stretch mode,
+   * validate that they are allowed to be in stretch mode (eg, are replicated)
+   * and place copies of them in the pools set.
+   * This does not make any changes to the pools or state; it's just
+   * a safety-check-and-collect function.
+   */
+  void try_enable_stretch_mode_pools(stringstream& ss, bool *okay,
+				     int *errcode,
+				     set<pg_pool_t*>* pools, const string& new_crush_rule);
+  /**
+   * Check validity of inputs and OSD/CRUSH state to
+   * engage stretch mode. Designed to be used with
+   * MonmapMonitor::try_enable_stretch_mode() where we call both twice,
+   * first with commit=false to validate.
+   * @param ss: a stringstream to write errors into
+   * @param okay: Filled to true if okay, false if validation fails
+   * @param errcode: filled with -errno if there's a problem
+   * @param commit: true if we should commit the change, false if just testing
+   * @param dividing_bucket: the bucket type (eg 'dc') that divides the cluster
+   * @param bucket_count: The number of buckets required in peering.
+   *  Currently must be 2.
+   * @param pools: The pg_pool_ts which are being set to stretch mode (obtained
+   *   from try_enable_stretch_mode_pools()).
+   * @param new_crush_rule: The crush rule to set the pools to.
+   */
+  void try_enable_stretch_mode(stringstream& ss, bool *okay,
+			       int *errcode, bool commit,
+			       const string& dividing_bucket,
+			       uint32_t bucket_count,
+			       const set<pg_pool_t*>& pools,
+			       const string& new_crush_rule);
+  /**
+   * Check the input dead_buckets mapping (buckets->dead monitors) to see
+   * if the OSDs are also down. If so, fill in really_down_buckets and
+   * really_down_mons and return true; else return false.
+   */
+  bool check_for_dead_crush_zones(const map<string,set<string>>& dead_buckets,
+				  set<int> *really_down_buckets,
+				  set<string> *really_down_mons);
+  /**
+   * Set degraded mode in the OSDMap, adding the given dead buckets to the dead set
+   * and using the live_zones (should presently be size 1)
+   */
+  void trigger_degraded_stretch_mode(const set<int>& dead_buckets,
+				     const set<string>& live_zones);
+  /**
+   * This is just to maintain stretch_recovery_triggered; below
+   */
+  void set_degraded_stretch_mode();
+  /**
+   * Set recovery stretch mode in the OSDMap, resetting pool size back to normal
+   */
+  void trigger_recovery_stretch_mode();
+  /**
+   * This is just to maintain stretch_recovery_triggered; below
+   */
+  void set_recovery_stretch_mode();
+  /**
+   * This is just to maintain stretch_recovery_triggered; below
+   */
+  void set_healthy_stretch_mode();
+  /**
+   * Tells the OSD there's a new pg digest, in case it's interested.
+   * (It's interested when in recovering stretch mode.)
+   */
+  void notify_new_pg_digest();
+  /**
+   * Check if we can exit recovery stretch mode and go back to normal.
+   * @param force If true, we will force the exit through once it is legal,
+   * without regard to the reported PG status.
+   */
+  void try_end_recovery_stretch_mode(bool force);
+  /**
+   * Sets the osdmap and pg_pool_t values back to healthy stretch mode status.
+   */
+  void trigger_healthy_stretch_mode();
+  /**
+   * Obtain the crush rule being used for stretch pools.
+   * Note that right now this is heuristic and simply selects the
+   * most-used rule on replicated stretch pools.
+   * @return the crush rule ID, or a negative errno
+   */
+  int get_replicated_stretch_crush_rule();
+private:
+  utime_t stretch_recovery_triggered; // what time we committed a switch to recovery mode
+};
+
+#endif
diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc
new file mode 100644
index 000000000..220317603
--- /dev/null
+++ b/src/mon/PGMap.cc
@@ -0,0 +1,4171 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <boost/algorithm/string.hpp>
+
+#include "PGMap.h"
+
+#define dout_subsys ceph_subsys_mon
+#include "common/debug.h"
+#include "common/Clock.h"
+#include "common/Formatter.h"
+#include "global/global_context.h"
+#include "include/ceph_features.h"
+#include "include/stringify.h"
+
+#include "osd/osd_types.h"
+#include "osd/OSDMap.h"
+#include <boost/range/adaptor/reversed.hpp>
+
+#define dout_context g_ceph_context
+
+using std::list;
+using std::make_pair;
+using std::map;
+using std::pair;
+using std::ostream;
+using std::ostringstream;
+using std::set;
+using std::string;
+using std::stringstream;
+using std::vector;
+
+using ceph::bufferlist;
+using ceph::fixed_u_to_string;
+
+using TOPNSPC::common::cmd_getval;
+
+MEMPOOL_DEFINE_OBJECT_FACTORY(PGMapDigest, pgmap_digest, pgmap);
+MEMPOOL_DEFINE_OBJECT_FACTORY(PGMap, pgmap, pgmap);
+MEMPOOL_DEFINE_OBJECT_FACTORY(PGMap::Incremental, pgmap_inc, pgmap);
+
+
+// ---------------------
+// PGMapDigest
+
+void PGMapDigest::encode(bufferlist& bl, uint64_t features) const
+{
+  // NOTE: see PGMap::encode_digest
+  uint8_t v = 4;
+  if (!HAVE_FEATURE(features, SERVER_MIMIC)) {
+    v = 1;
+  } else if (!HAVE_FEATURE(features, SERVER_NAUTILUS)) {
+    v = 3;
+  }
+  ENCODE_START(v, 1, bl);
+  encode(num_pg, bl);
+  encode(num_pg_active, bl);
+  encode(num_pg_unknown, bl);
+  encode(num_osd, bl);
+  encode(pg_pool_sum, bl, features);
+  encode(pg_sum, bl, features);
+  encode(osd_sum, bl, features);
+  if (v >= 2) {
+    encode(num_pg_by_state, bl);
+  } else {
+    uint32_t n = num_pg_by_state.size();
+    encode(n, bl);
+    for (auto p : num_pg_by_state) {
+      encode((int32_t)p.first, bl);
+      encode(p.second, bl);
+    }
+  }
+  encode(num_pg_by_osd, bl);
+  encode(num_pg_by_pool, bl);
+  encode(osd_last_seq, bl);
+  encode(per_pool_sum_delta, bl, features);
+  encode(per_pool_sum_deltas_stamps, bl);
+  encode(pg_sum_delta, bl, features);
+  encode(stamp_delta, bl);
+  encode(avail_space_by_rule, bl);
+  if (struct_v >= 3) {
+    encode(purged_snaps, bl);
+  }
+  if (struct_v >= 4) {
+    encode(osd_sum_by_class, bl, features);
+  }
+  ENCODE_FINISH(bl);
+}
+
+void PGMapDigest::decode(bufferlist::const_iterator& p)
+{
+  DECODE_START(4, p);
+  decode(num_pg, p);
+  decode(num_pg_active, p);
+  decode(num_pg_unknown, p);
+  decode(num_osd, p);
+  decode(pg_pool_sum, p);
+  decode(pg_sum, p);
+  decode(osd_sum, p);
+  if (struct_v >= 2) {
+    decode(num_pg_by_state, p);
+  } else {
+    map<int32_t, int32_t> nps;
+    decode(nps, p);
+    num_pg_by_state.clear();
+    for (auto i : nps) {
+      num_pg_by_state[i.first] = i.second;
+    }
+  }
+  decode(num_pg_by_osd, p);
+  decode(num_pg_by_pool, p);
+  decode(osd_last_seq, p);
+  decode(per_pool_sum_delta, p);
+  decode(per_pool_sum_deltas_stamps, p);
+  decode(pg_sum_delta, p);
+  decode(stamp_delta, p);
+  decode(avail_space_by_rule, p);
+  if (struct_v >= 3) {
+    decode(purged_snaps, p);
+  }
+  if (struct_v >= 4) {
+    decode(osd_sum_by_class, p);
+  }
+  DECODE_FINISH(p);
+}
+
+void PGMapDigest::dump(ceph::Formatter *f) const
+{
+  f->dump_unsigned("num_pg", num_pg);
+  f->dump_unsigned("num_pg_active", num_pg_active);
+  f->dump_unsigned("num_pg_unknown", num_pg_unknown);
+  f->dump_unsigned("num_osd", num_osd);
+  f->dump_object("pool_sum", pg_sum);
+  f->dump_object("osd_sum", osd_sum);
+
+  f->open_object_section("osd_sum_by_class");
+  for (auto& i : osd_sum_by_class) {
+    f->dump_object(i.first.c_str(), i.second);
+  }
+  f->close_section();
+
+  f->open_array_section("pool_stats");
+  for (auto& p : pg_pool_sum) {
+    f->open_object_section("pool_stat");
+    f->dump_int("poolid", p.first);
+    auto q = num_pg_by_pool.find(p.first);
+    if (q != num_pg_by_pool.end())
+      f->dump_unsigned("num_pg", q->second);
+    p.second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+  f->open_array_section("osd_stats");
+  int i = 0;
+  // TODO: this isn't really correct since we can dump non-existent OSDs
+  // I dunno what osd_last_seq is set to in that case...
+  for (auto& p : osd_last_seq) {
+    f->open_object_section("osd_stat");
+    f->dump_int("osd", i);
+    f->dump_unsigned("seq", p);
+    f->close_section();
+    ++i;
+  }
+  f->close_section();
+  f->open_array_section("num_pg_by_state");
+  for (auto& p : num_pg_by_state) {
+    f->open_object_section("count");
+    f->dump_string("state", pg_state_string(p.first));
+    f->dump_unsigned("num", p.second);
+    f->close_section();
+  }
+  f->close_section();
+  f->open_array_section("num_pg_by_osd");
+  for (auto& p : num_pg_by_osd) {
+    f->open_object_section("count");
+    f->dump_unsigned("osd", p.first);
+    f->dump_unsigned("num_primary_pg", p.second.primary);
+    f->dump_unsigned("num_acting_pg", p.second.acting);
+    f->dump_unsigned("num_up_not_acting_pg", p.second.up_not_acting);
+    f->close_section();
+  }
+  f->close_section();
+  f->open_array_section("purged_snaps");
+  for (auto& j : purged_snaps) {
+    f->open_object_section("pool");
+    f->dump_int("pool", j.first);
+    f->open_object_section("purged_snaps");
+    for (auto i = j.second.begin(); i != j.second.end(); ++i) {
+      f->open_object_section("interval");
+      f->dump_stream("start") << i.get_start();
+      f->dump_stream("length") << i.get_len();
+      f->close_section();
+    }
+    f->close_section();
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void PGMapDigest::generate_test_instances(list<PGMapDigest*>& ls)
+{
+  ls.push_back(new PGMapDigest);
+}
+
+inline std::string percentify(const float& a) {
+  std::stringstream ss;
+  if (a < 0.01)
+    ss << "0";
+  else
+    ss << std::fixed << std::setprecision(2) << a;
+  return ss.str();
+}
+
+void PGMapDigest::print_summary(ceph::Formatter *f, ostream *out) const
+{
+  if (f)
+    f->open_array_section("pgs_by_state");
+
+  // list is descending numeric order (by count)
+  std::multimap<int,uint64_t> state_by_count;  // count -> state
+  for (auto p = num_pg_by_state.begin();
+       p != num_pg_by_state.end();
+       ++p) {
+    state_by_count.insert(make_pair(p->second, p->first));
+  }
+  if (f) {
+    for (auto p = state_by_count.rbegin();
+         p != state_by_count.rend();
+         ++p)
+    {
+      f->open_object_section("pgs_by_state_element");
+      f->dump_string("state_name", pg_state_string(p->second));
+      f->dump_unsigned("count", p->first);
+      f->close_section();
+    }
+  }
+  if (f)
+    f->close_section();
+
+  if (f) {
+    f->dump_unsigned("num_pgs", num_pg);
+    f->dump_unsigned("num_pools", pg_pool_sum.size());
+    f->dump_unsigned("num_objects", pg_sum.stats.sum.num_objects);
+    f->dump_unsigned("data_bytes", pg_sum.stats.sum.num_bytes);
+    f->dump_unsigned("bytes_used", osd_sum.statfs.get_used_raw());
+    f->dump_unsigned("bytes_avail", osd_sum.statfs.available);
+    f->dump_unsigned("bytes_total", osd_sum.statfs.total);
+  } else {
+    *out << "    pools:   " << pg_pool_sum.size() << " pools, "
+         << num_pg << " pgs\n";
+    *out << "    objects: " << si_u_t(pg_sum.stats.sum.num_objects) << " objects, "
+         << byte_u_t(pg_sum.stats.sum.num_bytes) << "\n";
+    *out << "    usage:   "
+         << byte_u_t(osd_sum.statfs.get_used_raw()) << " used, "
+         << byte_u_t(osd_sum.statfs.available) << " / "
+         << byte_u_t(osd_sum.statfs.total) << " avail\n";
+    *out << "    pgs:     ";
+  }
+
+  bool pad = false;
+
+  if (num_pg_unknown > 0) {
+    float p = (float)num_pg_unknown / (float)num_pg;
+    if (f) {
+      f->dump_float("unknown_pgs_ratio", p);
+    } else {
+      char b[20];
+      snprintf(b, sizeof(b), "%.3lf", p * 100.0);
+      *out << b << "% pgs unknown\n";
+      pad = true;
+    }
+  }
+
+  int num_pg_inactive = num_pg - num_pg_active - num_pg_unknown;
+  if (num_pg_inactive > 0) {
+    float p = (float)num_pg_inactive / (float)num_pg;
+    if (f) {
+      f->dump_float("inactive_pgs_ratio", p);
+    } else {
+      if (pad) {
+        *out << "             ";
+      }
+      char b[20];
+      snprintf(b, sizeof(b), "%.3f", p * 100.0);
+      *out << b << "% pgs not active\n";
+      pad = true;
+    }
+  }
+
+  list<string> sl;
+  overall_recovery_summary(f, &sl);
+  if (!f && !sl.empty()) {
+    for (auto p = sl.begin(); p != sl.end(); ++p) {
+      if (pad) {
+        *out << "             ";
+      }
+      *out << *p << "\n";
+      pad = true;
+    }
+  }
+  sl.clear();
+
+  if (!f) {
+    unsigned max_width = 1;
+    for (auto p = state_by_count.rbegin(); p != state_by_count.rend(); ++p)
+    {
+      std::stringstream ss;
+      ss << p->first;
+      max_width = std::max<size_t>(ss.str().size(), max_width);
+    }
+
+    for (auto p = state_by_count.rbegin(); p != state_by_count.rend(); ++p)
+    {
+      if (pad) {
+        *out << "             ";
+      }
+      pad = true;
+      out->setf(std::ios::left);
+      *out << std::setw(max_width) << p->first
+           << " " << pg_state_string(p->second) << "\n";
+      out->unsetf(std::ios::left);
+    }
+  }
+
+  ostringstream ss_rec_io;
+  overall_recovery_rate_summary(f, &ss_rec_io);
+  ostringstream ss_client_io;
+  overall_client_io_rate_summary(f, &ss_client_io);
+  ostringstream ss_cache_io;
+  overall_cache_io_rate_summary(f, &ss_cache_io);
+
+  if (!f && (ss_client_io.str().length() || ss_rec_io.str().length()
+             || ss_cache_io.str().length())) {
+    *out << "\n \n";
+    *out << "  io:\n";
+  }
+
+  if (!f && ss_client_io.str().length())
+    *out << "    client:   " << ss_client_io.str() << "\n";
+  if (!f && ss_rec_io.str().length())
+    *out << "    recovery: " << ss_rec_io.str() << "\n";
+  if (!f && ss_cache_io.str().length())
+    *out << "    cache:    " << ss_cache_io.str() << "\n";
+}
+
+void PGMapDigest::print_oneline_summary(ceph::Formatter *f, ostream *out) const
+{
+  std::stringstream ss;
+
+  if (f)
+    f->open_array_section("num_pg_by_state");
+  for (auto p = num_pg_by_state.begin();
+       p != num_pg_by_state.end();
+       ++p) {
+    if (f) {
+      f->open_object_section("state");
+      f->dump_string("name", pg_state_string(p->first));
+      f->dump_unsigned("num", p->second);
+      f->close_section();
+    }
+    if (p != num_pg_by_state.begin())
+      ss << ", ";
+    ss << p->second << " " << pg_state_string(p->first);
+  }
+  if (f)
+    f->close_section();
+
+  string states = ss.str();
+  if (out)
+    *out << num_pg << " pgs: "
+         << states << "; "
+         << byte_u_t(pg_sum.stats.sum.num_bytes) << " data, "
+         << byte_u_t(osd_sum.statfs.get_used()) << " used, "
+         << byte_u_t(osd_sum.statfs.available) << " / "
+         << byte_u_t(osd_sum.statfs.total) << " avail";
+  if (f) {
+    f->dump_unsigned("num_pgs", num_pg);
+    f->dump_unsigned("num_bytes", pg_sum.stats.sum.num_bytes);
+    f->dump_int("total_bytes", osd_sum.statfs.total);
+    f->dump_int("total_avail_bytes", osd_sum.statfs.available);
+    f->dump_int("total_used_bytes", osd_sum.statfs.get_used());
+    f->dump_int("total_used_raw_bytes", osd_sum.statfs.get_used_raw());
+  }
+
+  // make non-negative; we can get negative values if osds send
+  // uncommitted stats and then "go backward" or if they are just
+  // buggy/wrong.
+  pool_stat_t pos_delta = pg_sum_delta;
+  pos_delta.floor(0);
+  if (pos_delta.stats.sum.num_rd ||
+      pos_delta.stats.sum.num_wr) {
+    if (out)
+      *out << "; ";
+    if (pos_delta.stats.sum.num_rd) {
+      int64_t rd = (pos_delta.stats.sum.num_rd_kb << 10) / (double)stamp_delta;
+      if (out)
+	*out << byte_u_t(rd) << "/s rd, ";
+      if (f)
+	f->dump_unsigned("read_bytes_sec", rd);
+    }
+    if (pos_delta.stats.sum.num_wr) {
+      int64_t wr = (pos_delta.stats.sum.num_wr_kb << 10) / (double)stamp_delta;
+      if (out)
+	*out << byte_u_t(wr) << "/s wr, ";
+      if (f)
+	f->dump_unsigned("write_bytes_sec", wr);
+    }
+    int64_t iops = (pos_delta.stats.sum.num_rd + pos_delta.stats.sum.num_wr) / (double)stamp_delta;
+    if (out)
+      *out << si_u_t(iops) << " op/s";
+    if (f)
+      f->dump_unsigned("io_sec", iops);
+  }
+
+  list<string> sl;
+  overall_recovery_summary(f, &sl);
+  if (out)
+    for (auto p = sl.begin(); p != sl.end(); ++p)
+      *out << "; " << *p;
+  std::stringstream ssr;
+  overall_recovery_rate_summary(f, &ssr);
+  if (out && ssr.str().length())
+    *out << "; " << ssr.str() << " recovering";
+}
+
+void PGMapDigest::get_recovery_stats(
+    double *misplaced_ratio,
+    double *degraded_ratio,
+    double *inactive_pgs_ratio,
+    double *unknown_pgs_ratio) const
+{
+  if (pg_sum.stats.sum.num_objects_degraded &&
+      pg_sum.stats.sum.num_object_copies > 0) {
+    *degraded_ratio = (double)pg_sum.stats.sum.num_objects_degraded /
+      (double)pg_sum.stats.sum.num_object_copies;
+  } else {
+    *degraded_ratio = 0;
+  }
+  if (pg_sum.stats.sum.num_objects_misplaced &&
+      pg_sum.stats.sum.num_object_copies > 0) {
+    *misplaced_ratio = (double)pg_sum.stats.sum.num_objects_misplaced /
+      (double)pg_sum.stats.sum.num_object_copies;
+  } else {
+    *misplaced_ratio = 0;
+  }
+  if (num_pg > 0) {
+    int num_pg_inactive = num_pg - num_pg_active - num_pg_unknown;
+    *inactive_pgs_ratio = (double)num_pg_inactive / (double)num_pg;
+    *unknown_pgs_ratio = (double)num_pg_unknown / (double)num_pg;
+ } else {
+    *inactive_pgs_ratio = 0;
+    *unknown_pgs_ratio = 0;
+  }
+}
+
+void PGMapDigest::recovery_summary(ceph::Formatter *f, list<string> *psl,
+                             const pool_stat_t& pool_sum) const
+{
+  if (pool_sum.stats.sum.num_objects_degraded && pool_sum.stats.sum.num_object_copies > 0) {
+    double pc = (double)pool_sum.stats.sum.num_objects_degraded /
+                (double)pool_sum.stats.sum.num_object_copies * (double)100.0;
+    char b[20];
+    snprintf(b, sizeof(b), "%.3lf", pc);
+    if (f) {
+      f->dump_unsigned("degraded_objects", pool_sum.stats.sum.num_objects_degraded);
+      f->dump_unsigned("degraded_total", pool_sum.stats.sum.num_object_copies);
+      f->dump_float("degraded_ratio", pc / 100.0);
+    } else {
+      ostringstream ss;
+      ss << pool_sum.stats.sum.num_objects_degraded
+         << "/" << pool_sum.stats.sum.num_object_copies << " objects degraded (" << b << "%)";
+      psl->push_back(ss.str());
+    }
+  }
+  if (pool_sum.stats.sum.num_objects_misplaced && pool_sum.stats.sum.num_object_copies > 0) {
+    double pc = (double)pool_sum.stats.sum.num_objects_misplaced /
+                (double)pool_sum.stats.sum.num_object_copies * (double)100.0;
+    char b[20];
+    snprintf(b, sizeof(b), "%.3lf", pc);
+    if (f) {
+      f->dump_unsigned("misplaced_objects", pool_sum.stats.sum.num_objects_misplaced);
+      f->dump_unsigned("misplaced_total", pool_sum.stats.sum.num_object_copies);
+      f->dump_float("misplaced_ratio", pc / 100.0);
+    } else {
+      ostringstream ss;
+      ss << pool_sum.stats.sum.num_objects_misplaced
+         << "/" << pool_sum.stats.sum.num_object_copies << " objects misplaced (" << b << "%)";
+      psl->push_back(ss.str());
+    }
+  }
+  if (pool_sum.stats.sum.num_objects_unfound && pool_sum.stats.sum.num_objects) {
+    double pc = (double)pool_sum.stats.sum.num_objects_unfound /
+                (double)pool_sum.stats.sum.num_objects * (double)100.0;
+    char b[20];
+    snprintf(b, sizeof(b), "%.3lf", pc);
+    if (f) {
+      f->dump_unsigned("unfound_objects", pool_sum.stats.sum.num_objects_unfound);
+      f->dump_unsigned("unfound_total", pool_sum.stats.sum.num_objects);
+      f->dump_float("unfound_ratio", pc / 100.0);
+    } else {
+      ostringstream ss;
+      ss << pool_sum.stats.sum.num_objects_unfound
+         << "/" << pool_sum.stats.sum.num_objects << " objects unfound (" << b << "%)";
+      psl->push_back(ss.str());
+    }
+  }
+}
+
+void PGMapDigest::recovery_rate_summary(ceph::Formatter *f, ostream *out,
+                                  const pool_stat_t& delta_sum,
+                                  utime_t delta_stamp) const
+{
+  // make non-negative; we can get negative values if osds send
+  // uncommitted stats and then "go backward" or if they are just
+  // buggy/wrong.
+  pool_stat_t pos_delta = delta_sum;
+  pos_delta.floor(0);
+  if (pos_delta.stats.sum.num_objects_recovered ||
+      pos_delta.stats.sum.num_bytes_recovered ||
+      pos_delta.stats.sum.num_keys_recovered) {
+    int64_t objps = pos_delta.stats.sum.num_objects_recovered / (double)delta_stamp;
+    int64_t bps = pos_delta.stats.sum.num_bytes_recovered / (double)delta_stamp;
+    int64_t kps = pos_delta.stats.sum.num_keys_recovered / (double)delta_stamp;
+    if (f) {
+      f->dump_int("recovering_objects_per_sec", objps);
+      f->dump_int("recovering_bytes_per_sec", bps);
+      f->dump_int("recovering_keys_per_sec", kps);
+      f->dump_int("num_objects_recovered", pos_delta.stats.sum.num_objects_recovered);
+      f->dump_int("num_bytes_recovered", pos_delta.stats.sum.num_bytes_recovered);
+      f->dump_int("num_keys_recovered", pos_delta.stats.sum.num_keys_recovered);
+    } else {
+      *out << byte_u_t(bps) << "/s";
+      if (pos_delta.stats.sum.num_keys_recovered)
+	*out << ", " << si_u_t(kps) << " keys/s";
+      *out << ", " << si_u_t(objps) << " objects/s";
+    }
+  }
+}
+
+void PGMapDigest::overall_recovery_rate_summary(ceph::Formatter *f, ostream *out) const
+{
+  recovery_rate_summary(f, out, pg_sum_delta, stamp_delta);
+}
+
+void PGMapDigest::overall_recovery_summary(ceph::Formatter *f, list<string> *psl) const
+{
+  recovery_summary(f, psl, pg_sum);
+}
+
+void PGMapDigest::pool_recovery_rate_summary(ceph::Formatter *f, ostream *out,
+                                       uint64_t poolid) const
+{
+  auto p = per_pool_sum_delta.find(poolid);
+  if (p == per_pool_sum_delta.end())
+    return;
+
+  auto ts = per_pool_sum_deltas_stamps.find(p->first);
+  ceph_assert(ts != per_pool_sum_deltas_stamps.end());
+  recovery_rate_summary(f, out, p->second.first, ts->second);
+}
+
+void PGMapDigest::pool_recovery_summary(ceph::Formatter *f, list<string> *psl,
+                                  uint64_t poolid) const
+{
+  auto p = pg_pool_sum.find(poolid);
+  if (p == pg_pool_sum.end())
+    return;
+
+  recovery_summary(f, psl, p->second);
+}
+
+void PGMapDigest::client_io_rate_summary(ceph::Formatter *f, ostream *out,
+                                   const pool_stat_t& delta_sum,
+                                   utime_t delta_stamp) const
+{
+  pool_stat_t pos_delta = delta_sum;
+  pos_delta.floor(0);
+  if (pos_delta.stats.sum.num_rd ||
+      pos_delta.stats.sum.num_wr) {
+    if (pos_delta.stats.sum.num_rd) {
+      int64_t rd = (pos_delta.stats.sum.num_rd_kb << 10) / (double)delta_stamp;
+      if (f) {
+	f->dump_int("read_bytes_sec", rd);
+      } else {
+	*out << byte_u_t(rd) << "/s rd, ";
+      }
+    }
+    if (pos_delta.stats.sum.num_wr) {
+      int64_t wr = (pos_delta.stats.sum.num_wr_kb << 10) / (double)delta_stamp;
+      if (f) {
+	f->dump_int("write_bytes_sec", wr);
+      } else {
+	*out << byte_u_t(wr) << "/s wr, ";
+      }
+    }
+    int64_t iops_rd = pos_delta.stats.sum.num_rd / (double)delta_stamp;
+    int64_t iops_wr = pos_delta.stats.sum.num_wr / (double)delta_stamp;
+    if (f) {
+      f->dump_int("read_op_per_sec", iops_rd);
+      f->dump_int("write_op_per_sec", iops_wr);
+    } else {
+      *out << si_u_t(iops_rd) << " op/s rd, " << si_u_t(iops_wr) << " op/s wr";
+    }
+  }
+}
+
+void PGMapDigest::overall_client_io_rate_summary(ceph::Formatter *f, ostream *out) const
+{
+  client_io_rate_summary(f, out, pg_sum_delta, stamp_delta);
+}
+
+void PGMapDigest::pool_client_io_rate_summary(ceph::Formatter *f, ostream *out,
+                                        uint64_t poolid) const
+{
+  auto p = per_pool_sum_delta.find(poolid);
+  if (p == per_pool_sum_delta.end())
+    return;
+
+  auto ts = per_pool_sum_deltas_stamps.find(p->first);
+  ceph_assert(ts != per_pool_sum_deltas_stamps.end());
+  client_io_rate_summary(f, out, p->second.first, ts->second);
+}
+
+void PGMapDigest::cache_io_rate_summary(ceph::Formatter *f, ostream *out,
+                                  const pool_stat_t& delta_sum,
+                                  utime_t delta_stamp) const
+{
+  pool_stat_t pos_delta = delta_sum;
+  pos_delta.floor(0);
+  bool have_output = false;
+
+  if (pos_delta.stats.sum.num_flush) {
+    int64_t flush = (pos_delta.stats.sum.num_flush_kb << 10) / (double)delta_stamp;
+    if (f) {
+      f->dump_int("flush_bytes_sec", flush);
+    } else {
+      *out << byte_u_t(flush) << "/s flush";
+      have_output = true;
+    }
+  }
+  if (pos_delta.stats.sum.num_evict) {
+    int64_t evict = (pos_delta.stats.sum.num_evict_kb << 10) / (double)delta_stamp;
+    if (f) {
+      f->dump_int("evict_bytes_sec", evict);
+    } else {
+      if (have_output)
+	*out << ", ";
+      *out << byte_u_t(evict) << "/s evict";
+      have_output = true;
+    }
+  }
+  if (pos_delta.stats.sum.num_promote) {
+    int64_t promote = pos_delta.stats.sum.num_promote / (double)delta_stamp;
+    if (f) {
+      f->dump_int("promote_op_per_sec", promote);
+    } else {
+      if (have_output)
+	*out << ", ";
+      *out << si_u_t(promote) << " op/s promote";
+      have_output = true;
+    }
+  }
+  if (pos_delta.stats.sum.num_flush_mode_low) {
+    if (f) {
+      f->dump_int("num_flush_mode_low", pos_delta.stats.sum.num_flush_mode_low);
+    } else {
+      if (have_output)
+	*out << ", ";
+      *out << si_u_t(pos_delta.stats.sum.num_flush_mode_low) << " PGs flushing";
+      have_output = true;
+    }
+  }
+  if (pos_delta.stats.sum.num_flush_mode_high) {
+    if (f) {
+      f->dump_int("num_flush_mode_high", pos_delta.stats.sum.num_flush_mode_high);
+    } else {
+      if (have_output)
+	*out << ", ";
+      *out << si_u_t(pos_delta.stats.sum.num_flush_mode_high) << " PGs flushing (high)";
+      have_output = true;
+    }
+  }
+  if (pos_delta.stats.sum.num_evict_mode_some) {
+    if (f) {
+      f->dump_int("num_evict_mode_some", pos_delta.stats.sum.num_evict_mode_some);
+    } else {
+      if (have_output)
+	*out << ", ";
+      *out << si_u_t(pos_delta.stats.sum.num_evict_mode_some) << " PGs evicting";
+      have_output = true;
+    }
+  }
+  if (pos_delta.stats.sum.num_evict_mode_full) {
+    if (f) {
+      f->dump_int("num_evict_mode_full", pos_delta.stats.sum.num_evict_mode_full);
+    } else {
+      if (have_output)
+	*out << ", ";
+      *out << si_u_t(pos_delta.stats.sum.num_evict_mode_full) << " PGs evicting (full)";
+    }
+  }
+}
+
+void PGMapDigest::overall_cache_io_rate_summary(ceph::Formatter *f, ostream *out) const
+{
+  cache_io_rate_summary(f, out, pg_sum_delta, stamp_delta);
+}
+
+void PGMapDigest::pool_cache_io_rate_summary(ceph::Formatter *f, ostream *out,
+                                       uint64_t poolid) const
+{
+  auto p = per_pool_sum_delta.find(poolid);
+  if (p == per_pool_sum_delta.end())
+    return;
+
+  auto ts = per_pool_sum_deltas_stamps.find(p->first);
+  ceph_assert(ts != per_pool_sum_deltas_stamps.end());
+  cache_io_rate_summary(f, out, p->second.first, ts->second);
+}
+
+ceph_statfs PGMapDigest::get_statfs(OSDMap &osdmap,
+				    boost::optional<int64_t> data_pool) const
+{
+  ceph_statfs statfs;
+  bool filter = false;
+  object_stat_sum_t sum;
+
+  if (data_pool) {
+    auto i = pg_pool_sum.find(*data_pool);
+    if (i != pg_pool_sum.end()) {
+      sum = i->second.stats.sum;
+      filter = true;
+    }
+  }
+
+  if (filter) {
+    statfs.kb_used = (sum.num_bytes >> 10);
+    statfs.kb_avail = get_pool_free_space(osdmap, *data_pool) >> 10;
+    statfs.num_objects = sum.num_objects;
+    statfs.kb = statfs.kb_used + statfs.kb_avail;
+  } else {
+    // these are in KB.
+    statfs.kb = osd_sum.statfs.kb();
+    statfs.kb_used = osd_sum.statfs.kb_used_raw();
+    statfs.kb_avail = osd_sum.statfs.kb_avail();
+    statfs.num_objects = pg_sum.stats.sum.num_objects;
+  }
+
+  return statfs;
+}
+
+void PGMapDigest::dump_pool_stats_full(
+  const OSDMap &osd_map,
+  stringstream *ss,
+  ceph::Formatter *f,
+  bool verbose) const
+{
+  TextTable tbl;
+
+  if (f) {
+    f->open_array_section("pools");
+  } else {
+    tbl.define_column("POOL", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("ID", TextTable::RIGHT, TextTable::RIGHT);
+    tbl.define_column("PGS", TextTable::RIGHT, TextTable::RIGHT);
+    tbl.define_column("STORED", TextTable::RIGHT, TextTable::RIGHT);
+    if (verbose) {
+      tbl.define_column("(DATA)", TextTable::RIGHT, TextTable::RIGHT);
+      tbl.define_column("(OMAP)", TextTable::RIGHT, TextTable::RIGHT);
+    }
+    tbl.define_column("OBJECTS", TextTable::RIGHT, TextTable::RIGHT);
+    tbl.define_column("USED", TextTable::RIGHT, TextTable::RIGHT);
+    if (verbose) {
+      tbl.define_column("(DATA)", TextTable::RIGHT, TextTable::RIGHT);
+      tbl.define_column("(OMAP)", TextTable::RIGHT, TextTable::RIGHT);
+    }
+    tbl.define_column("%USED", TextTable::RIGHT, TextTable::RIGHT);
+    tbl.define_column("MAX AVAIL", TextTable::RIGHT, TextTable::RIGHT);
+
+    if (verbose) {
+      tbl.define_column("QUOTA OBJECTS", TextTable::RIGHT, TextTable::RIGHT);
+      tbl.define_column("QUOTA BYTES", TextTable::RIGHT, TextTable::RIGHT);
+      tbl.define_column("DIRTY", TextTable::RIGHT, TextTable::RIGHT);
+      tbl.define_column("USED COMPR", TextTable::RIGHT, TextTable::RIGHT);
+      tbl.define_column("UNDER COMPR", TextTable::RIGHT, TextTable::RIGHT);
+    }
+  }
+
+  map<int,uint64_t> avail_by_rule;
+  for (auto p = osd_map.get_pools().begin();
+       p != osd_map.get_pools().end(); ++p) {
+    int64_t pool_id = p->first;
+    if ((pool_id < 0) || (pg_pool_sum.count(pool_id) == 0))
+      continue;
+
+    const string& pool_name = osd_map.get_pool_name(pool_id);
+    auto pool_pg_num = osd_map.get_pg_num(pool_id);
+    const pool_stat_t &stat = pg_pool_sum.at(pool_id);
+
+    const pg_pool_t *pool = osd_map.get_pg_pool(pool_id);
+    int ruleno = osd_map.crush->find_rule(pool->get_crush_rule(),
+                                         pool->get_type(),
+                                         pool->get_size());
+    int64_t avail;
+    if (avail_by_rule.count(ruleno) == 0) {
+      // FIXME: we don't guarantee avail_space_by_rule is up-to-date before this function is invoked
+      avail = get_rule_avail(ruleno);
+      if (avail < 0)
+	avail = 0;
+      avail_by_rule[ruleno] = avail;
+    } else {
+      avail = avail_by_rule[ruleno];
+    }
+    if (f) {
+      f->open_object_section("pool");
+      f->dump_string("name", pool_name);
+      f->dump_int("id", pool_id);
+      f->open_object_section("stats");
+    } else {
+      tbl << pool_name
+          << pool_id
+          << pool_pg_num;
+    }
+    float raw_used_rate = osd_map.pool_raw_used_rate(pool_id);
+    bool per_pool = use_per_pool_stats();
+    bool per_pool_omap = use_per_pool_omap_stats();
+    dump_object_stat_sum(tbl, f, stat, avail, raw_used_rate, verbose, per_pool,
+			 per_pool_omap, pool);
+    if (f) {
+      f->close_section();  // stats
+      f->close_section();  // pool
+    } else {
+      tbl << TextTable::endrow;
+    }
+  }
+  if (f)
+    f->close_section();
+  else {
+    ceph_assert(ss != nullptr);
+    *ss << "--- POOLS ---\n";
+    *ss << tbl;
+  }
+}
+
+void PGMapDigest::dump_cluster_stats(stringstream *ss,
+				     ceph::Formatter *f,
+				     bool verbose) const
+{
+  if (f) {
+    f->open_object_section("stats");
+    f->dump_int("total_bytes", osd_sum.statfs.total);
+    f->dump_int("total_avail_bytes", osd_sum.statfs.available);
+    f->dump_int("total_used_bytes", osd_sum.statfs.get_used());
+    f->dump_int("total_used_raw_bytes", osd_sum.statfs.get_used_raw());
+    f->dump_float("total_used_raw_ratio", osd_sum.statfs.get_used_raw_ratio());
+    f->dump_unsigned("num_osds", osd_sum.num_osds);
+    f->dump_unsigned("num_per_pool_osds", osd_sum.num_per_pool_osds);
+    f->dump_unsigned("num_per_pool_omap_osds", osd_sum.num_per_pool_omap_osds);
+    f->close_section();
+    f->open_object_section("stats_by_class");
+    for (auto& i : osd_sum_by_class) {
+      f->open_object_section(i.first.c_str());
+      f->dump_int("total_bytes", i.second.statfs.total);
+      f->dump_int("total_avail_bytes", i.second.statfs.available);
+      f->dump_int("total_used_bytes", i.second.statfs.get_used());
+      f->dump_int("total_used_raw_bytes", i.second.statfs.get_used_raw());
+      f->dump_float("total_used_raw_ratio",
+		    i.second.statfs.get_used_raw_ratio());
+      f->close_section();
+    }
+    f->close_section();
+  } else {
+    ceph_assert(ss != nullptr);
+    TextTable tbl;
+    tbl.define_column("CLASS", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("SIZE", TextTable::RIGHT, TextTable::RIGHT);
+    tbl.define_column("AVAIL", TextTable::RIGHT, TextTable::RIGHT);
+    tbl.define_column("USED", TextTable::RIGHT, TextTable::RIGHT);
+    tbl.define_column("RAW USED", TextTable::RIGHT, TextTable::RIGHT);
+    tbl.define_column("%RAW USED", TextTable::RIGHT, TextTable::RIGHT);
+
+
+    for (auto& i : osd_sum_by_class) {
+      tbl << i.first;
+      tbl << stringify(byte_u_t(i.second.statfs.total))
+	  << stringify(byte_u_t(i.second.statfs.available))
+	  << stringify(byte_u_t(i.second.statfs.get_used()))
+	  << stringify(byte_u_t(i.second.statfs.get_used_raw()))
+	  << percentify(i.second.statfs.get_used_raw_ratio()*100.0)
+	  << TextTable::endrow;
+    }
+    tbl << "TOTAL";
+    tbl << stringify(byte_u_t(osd_sum.statfs.total))
+        << stringify(byte_u_t(osd_sum.statfs.available))
+        << stringify(byte_u_t(osd_sum.statfs.get_used()))
+        << stringify(byte_u_t(osd_sum.statfs.get_used_raw()))
+	<< percentify(osd_sum.statfs.get_used_raw_ratio()*100.0)
+	<< TextTable::endrow;
+
+    *ss << "--- RAW STORAGE ---\n";
+    *ss << tbl;
+  }
+}
+
+void PGMapDigest::dump_object_stat_sum(
+  TextTable &tbl, ceph::Formatter *f,
+  const pool_stat_t &pool_stat, uint64_t avail,
+  float raw_used_rate, bool verbose, bool per_pool, bool per_pool_omap,
+  const pg_pool_t *pool)
+{
+  const object_stat_sum_t &sum = pool_stat.stats.sum;
+  const store_statfs_t statfs = pool_stat.store_stats;
+
+  if (sum.num_object_copies > 0) {
+    raw_used_rate *= (float)(sum.num_object_copies - sum.num_objects_degraded) / sum.num_object_copies;
+  }
+
+  uint64_t used_data_bytes = pool_stat.get_allocated_data_bytes(per_pool);
+  uint64_t used_omap_bytes = pool_stat.get_allocated_omap_bytes(per_pool_omap);
+  uint64_t used_bytes = used_data_bytes + used_omap_bytes;
+
+  float used = 0.0;
+  // note avail passed in is raw_avail, calc raw_used here.
+  if (avail) {
+    used = used_bytes;
+    used /= used + avail;
+  } else if (used_bytes) {
+    used = 1.0;
+  }
+  auto avail_res = raw_used_rate ? avail / raw_used_rate : 0;
+  // an approximation for actually stored user data
+  auto stored_data_normalized = pool_stat.get_user_data_bytes(
+    raw_used_rate, per_pool);
+  auto stored_omap_normalized = pool_stat.get_user_omap_bytes(
+    raw_used_rate, per_pool_omap);
+  auto stored_normalized = stored_data_normalized + stored_omap_normalized;
+  // same, amplied by replication or EC
+  auto stored_raw = stored_normalized * raw_used_rate;
+  if (f) {
+    f->dump_int("stored", stored_normalized);
+    if (verbose) {
+      f->dump_int("stored_data", stored_data_normalized);
+      f->dump_int("stored_omap", stored_omap_normalized);
+    }
+    f->dump_int("objects", sum.num_objects);
+    f->dump_int("kb_used", shift_round_up(used_bytes, 10));
+    f->dump_int("bytes_used", used_bytes);
+    if (verbose) {
+      f->dump_int("data_bytes_used", used_data_bytes);
+      f->dump_int("omap_bytes_used", used_omap_bytes);
+    }
+    f->dump_float("percent_used", used);
+    f->dump_unsigned("max_avail", avail_res);
+    if (verbose) {
+      f->dump_int("quota_objects", pool->quota_max_objects);
+      f->dump_int("quota_bytes", pool->quota_max_bytes);
+      if (pool->is_tier()) {
+        f->dump_int("dirty", sum.num_objects_dirty);
+      } else {
+        f->dump_int("dirty", 0);
+      }
+      f->dump_int("rd", sum.num_rd);
+      f->dump_int("rd_bytes", sum.num_rd_kb * 1024ull);
+      f->dump_int("wr", sum.num_wr);
+      f->dump_int("wr_bytes", sum.num_wr_kb * 1024ull);
+      f->dump_int("compress_bytes_used", statfs.data_compressed_allocated);
+      f->dump_int("compress_under_bytes", statfs.data_compressed_original);
+      // Stored by user amplified by replication
+      f->dump_int("stored_raw", stored_raw);
+      f->dump_unsigned("avail_raw", avail);
+    }
+  } else {
+    tbl << stringify(byte_u_t(stored_normalized));
+    if (verbose) {
+      tbl << stringify(byte_u_t(stored_data_normalized));
+      tbl << stringify(byte_u_t(stored_omap_normalized));
+    }
+    tbl << stringify(si_u_t(sum.num_objects));
+    tbl << stringify(byte_u_t(used_bytes));
+    if (verbose) {
+      tbl << stringify(byte_u_t(used_data_bytes));
+      tbl << stringify(byte_u_t(used_omap_bytes));
+    }
+    tbl << percentify(used*100);
+    tbl << stringify(byte_u_t(avail_res));
+    if (verbose) {
+      if (pool->quota_max_objects == 0)
+        tbl << "N/A";
+      else
+        tbl << stringify(si_u_t(pool->quota_max_objects));
+      if (pool->quota_max_bytes == 0)
+        tbl << "N/A";
+      else
+        tbl << stringify(byte_u_t(pool->quota_max_bytes));
+      if (pool->is_tier()) {
+        tbl << stringify(si_u_t(sum.num_objects_dirty));
+      } else {
+        tbl << "N/A";
+      }
+      tbl << stringify(byte_u_t(statfs.data_compressed_allocated));
+      tbl << stringify(byte_u_t(statfs.data_compressed_original));
+    }
+  }
+}
+
+int64_t PGMapDigest::get_pool_free_space(const OSDMap &osd_map,
+					 int64_t poolid) const
+{
+  const pg_pool_t *pool = osd_map.get_pg_pool(poolid);
+  int ruleno = osd_map.crush->find_rule(pool->get_crush_rule(),
+					pool->get_type(),
+					pool->get_size());
+  int64_t avail;
+  avail = get_rule_avail(ruleno);
+  if (avail < 0)
+    avail = 0;
+
+  return avail / osd_map.pool_raw_used_rate(poolid);
+}
+
+int64_t PGMap::get_rule_avail(const OSDMap& osdmap, int ruleno) const
+{
+  map<int,float> wm;
+  int r = osdmap.crush->get_rule_weight_osd_map(ruleno, &wm);
+  if (r < 0) {
+    return r;
+  }
+  if (wm.empty()) {
+    return 0;
+  }
+
+  float fratio = osdmap.get_full_ratio();
+
+  int64_t min = -1;
+  for (auto p = wm.begin(); p != wm.end(); ++p) {
+    auto osd_info = osd_stat.find(p->first);
+    if (osd_info != osd_stat.end()) {
+      if (osd_info->second.statfs.total == 0 || p->second == 0) {
+	// osd must be out, hence its stats have been zeroed
+	// (unless we somehow managed to have a disk with size 0...)
+	//
+	// (p->second == 0), if osd weight is 0, no need to
+	// calculate proj below.
+	continue;
+      }
+      double unusable = (double)osd_info->second.statfs.kb() *
+	(1.0 - fratio);
+      double avail = std::max(0.0, (double)osd_info->second.statfs.kb_avail() - unusable);
+      avail *= 1024.0;
+      int64_t proj = (int64_t)(avail / (double)p->second);
+      if (min < 0 || proj < min) {
+	min = proj;
+      }
+    } else {
+      if (osdmap.is_up(p->first)) {
+        // This is a level 4 rather than an error, because we might have
+        // only just started, and not received the first stats message yet.
+        dout(4) << "OSD " << p->first << " is up, but has no stats" << dendl;
+      }
+    }
+  }
+  return min;
+}
+
+void PGMap::get_rules_avail(const OSDMap& osdmap,
+			    std::map<int,int64_t> *avail_map) const
+{
+  avail_map->clear();
+  for (auto p : osdmap.get_pools()) {
+    int64_t pool_id = p.first;
+    if ((pool_id < 0) || (pg_pool_sum.count(pool_id) == 0))
+      continue;
+    const pg_pool_t *pool = osdmap.get_pg_pool(pool_id);
+    int ruleno = osdmap.crush->find_rule(pool->get_crush_rule(),
+					 pool->get_type(),
+					 pool->get_size());
+    if (avail_map->count(ruleno) == 0)
+      (*avail_map)[ruleno] = get_rule_avail(osdmap, ruleno);
+  }
+}
+
+// ---------------------
+// PGMap
+
+void PGMap::Incremental::dump(ceph::Formatter *f) const
+{
+  f->dump_unsigned("version", version);
+  f->dump_stream("stamp") << stamp;
+  f->dump_unsigned("osdmap_epoch", osdmap_epoch);
+  f->dump_unsigned("pg_scan_epoch", pg_scan);
+
+  f->open_array_section("pg_stat_updates");
+  for (auto p = pg_stat_updates.begin(); p != pg_stat_updates.end(); ++p) {
+    f->open_object_section("pg_stat");
+    f->dump_stream("pgid") << p->first;
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("osd_stat_updates");
+  for (auto p = osd_stat_updates.begin(); p != osd_stat_updates.end(); ++p) {
+    f->open_object_section("osd_stat");
+    f->dump_int("osd", p->first);
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+  f->open_array_section("pool_statfs_updates");
+  for (auto p = pool_statfs_updates.begin(); p != pool_statfs_updates.end(); ++p) {
+    f->open_object_section("pool_statfs");
+    f->dump_stream("poolid/osd") << p->first;
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("osd_stat_removals");
+  for (auto p = osd_stat_rm.begin(); p != osd_stat_rm.end(); ++p)
+    f->dump_int("osd", *p);
+  f->close_section();
+
+  f->open_array_section("pg_removals");
+  for (auto p = pg_remove.begin(); p != pg_remove.end(); ++p)
+    f->dump_stream("pgid") << *p;
+  f->close_section();
+}
+
+void PGMap::Incremental::generate_test_instances(list<PGMap::Incremental*>& o)
+{
+  o.push_back(new Incremental);
+  o.push_back(new Incremental);
+  o.back()->version = 1;
+  o.back()->stamp = utime_t(123,345);
+  o.push_back(new Incremental);
+  o.back()->version = 2;
+  o.back()->pg_stat_updates[pg_t(1,2)] = pg_stat_t();
+  o.back()->osd_stat_updates[5] = osd_stat_t();
+  o.push_back(new Incremental);
+  o.back()->version = 3;
+  o.back()->osdmap_epoch = 1;
+  o.back()->pg_scan = 2;
+  o.back()->pg_stat_updates[pg_t(4,5)] = pg_stat_t();
+  o.back()->osd_stat_updates[6] = osd_stat_t();
+  o.back()->pg_remove.insert(pg_t(1,2));
+  o.back()->osd_stat_rm.insert(5);
+  o.back()->pool_statfs_updates[std::make_pair(1234,4)] = store_statfs_t();
+}
+
+// --
+
+void PGMap::apply_incremental(CephContext *cct, const Incremental& inc)
+{
+  ceph_assert(inc.version == version+1);
+  version++;
+
+  pool_stat_t pg_sum_old = pg_sum;
+  mempool::pgmap::unordered_map<int32_t, pool_stat_t> pg_pool_sum_old;
+  pg_pool_sum_old = pg_pool_sum;
+
+  for (auto p = inc.pg_stat_updates.begin();
+       p != inc.pg_stat_updates.end();
+       ++p) {
+    const pg_t &update_pg(p->first);
+    auto update_pool = update_pg.pool();
+    const pg_stat_t &update_stat(p->second);
+
+    auto pg_stat_iter = pg_stat.find(update_pg);
+    pool_stat_t &pool_sum_ref = pg_pool_sum[update_pool];
+    if (pg_stat_iter == pg_stat.end()) {
+      pg_stat.insert(make_pair(update_pg, update_stat));
+    } else {
+      stat_pg_sub(update_pg, pg_stat_iter->second);
+      pool_sum_ref.sub(pg_stat_iter->second);
+      pg_stat_iter->second = update_stat;
+    }
+    stat_pg_add(update_pg, update_stat);
+    pool_sum_ref.add(update_stat);
+  }
+
+  for (auto p = inc.pool_statfs_updates.begin();
+       p != inc.pool_statfs_updates.end();
+       ++p) {
+    auto update_pool = p->first.first;
+    auto update_osd =  p->first.second;
+    auto& statfs_inc = p->second;
+
+    auto pool_statfs_iter =
+      pool_statfs.find(std::make_pair(update_pool, update_osd));
+    if (pg_pool_sum.count(update_pool)) {
+      pool_stat_t &pool_sum_ref = pg_pool_sum[update_pool];
+      if (pool_statfs_iter == pool_statfs.end()) {
+        pool_statfs.emplace(std::make_pair(update_pool, update_osd), statfs_inc);
+      } else {
+        pool_sum_ref.sub(pool_statfs_iter->second);
+        pool_statfs_iter->second = statfs_inc;
+      }
+      pool_sum_ref.add(statfs_inc);
+    }
+  }
+
+  for (auto p = inc.get_osd_stat_updates().begin();
+       p != inc.get_osd_stat_updates().end();
+       ++p) {
+    int osd = p->first;
+    const osd_stat_t &new_stats(p->second);
+
+    auto t = osd_stat.find(osd);
+    if (t == osd_stat.end()) {
+      osd_stat.insert(make_pair(osd, new_stats));
+    } else {
+      stat_osd_sub(t->first, t->second);
+      t->second = new_stats;
+    }
+    stat_osd_add(osd, new_stats);
+  }
+  set<int64_t> deleted_pools;
+  for (auto p = inc.pg_remove.begin();
+       p != inc.pg_remove.end();
+       ++p) {
+    const pg_t &removed_pg(*p);
+    auto s = pg_stat.find(removed_pg);
+    bool pool_erased = false;
+    if (s != pg_stat.end()) {
+      pool_erased = stat_pg_sub(removed_pg, s->second);
+
+      // decrease pool stats if pg was removed
+      auto pool_stats_it = pg_pool_sum.find(removed_pg.pool());
+      if (pool_stats_it != pg_pool_sum.end()) {
+        pool_stats_it->second.sub(s->second);
+      }
+
+      pg_stat.erase(s);
+      if (pool_erased) {
+        deleted_pools.insert(removed_pg.pool());
+      }
+    }
+  }
+
+  for (auto p = inc.get_osd_stat_rm().begin();
+       p != inc.get_osd_stat_rm().end();
+       ++p) {
+    auto t = osd_stat.find(*p);
+    if (t != osd_stat.end()) {
+      stat_osd_sub(t->first, t->second);
+      osd_stat.erase(t);
+    }
+    for (auto i = pool_statfs.begin();  i != pool_statfs.end(); ++i) {
+      if (i->first.second == *p) {
+	pg_pool_sum[i->first.first].sub(i->second);
+	pool_statfs.erase(i);
+      }
+    }
+  }
+
+  // skip calculating delta while sum was not synchronized
+  if (!stamp.is_zero() && !pg_sum_old.stats.sum.is_zero()) {
+    utime_t delta_t;
+    delta_t = inc.stamp;
+    delta_t -= stamp;
+    // calculate a delta, and average over the last 2 deltas.
+    pool_stat_t d = pg_sum;
+    d.stats.sub(pg_sum_old.stats);
+    pg_sum_deltas.push_back(make_pair(d, delta_t));
+    stamp_delta += delta_t;
+    pg_sum_delta.stats.add(d.stats);
+    auto smooth_intervals =
+      cct ? cct->_conf.get_val<uint64_t>("mon_stat_smooth_intervals") : 1;
+    while (pg_sum_deltas.size() > smooth_intervals) {
+      pg_sum_delta.stats.sub(pg_sum_deltas.front().first.stats);
+      stamp_delta -= pg_sum_deltas.front().second;
+      pg_sum_deltas.pop_front();
+    }
+  }
+  stamp = inc.stamp;
+
+  update_pool_deltas(cct, inc.stamp, pg_pool_sum_old);
+
+  for (auto p : deleted_pools) {
+    if (cct)
+      dout(20) << " deleted pool " << p << dendl;
+    deleted_pool(p);
+  }
+
+  if (inc.osdmap_epoch)
+    last_osdmap_epoch = inc.osdmap_epoch;
+  if (inc.pg_scan)
+    last_pg_scan = inc.pg_scan;
+}
+
+void PGMap::calc_stats()
+{
+  num_pg = 0;
+  num_pg_active = 0;
+  num_pg_unknown = 0;
+  num_osd = 0;
+  pg_pool_sum.clear();
+  num_pg_by_pool.clear();
+  pg_by_osd.clear();
+  pg_sum = pool_stat_t();
+  osd_sum = osd_stat_t();
+  osd_sum_by_class.clear();
+  num_pg_by_state.clear();
+  num_pg_by_pool_state.clear();
+  num_pg_by_osd.clear();
+
+  for (auto p = pg_stat.begin();
+       p != pg_stat.end();
+       ++p) {
+    auto pg = p->first;
+    stat_pg_add(pg, p->second);
+    pg_pool_sum[pg.pool()].add(p->second);
+  }
+  for (auto p = pool_statfs.begin();
+       p != pool_statfs.end();
+       ++p) {
+    auto pool = p->first.first;
+    pg_pool_sum[pool].add(p->second);
+  }
+  for (auto p = osd_stat.begin();
+       p != osd_stat.end();
+       ++p)
+    stat_osd_add(p->first, p->second);
+}
+
+void PGMap::stat_pg_add(const pg_t &pgid, const pg_stat_t &s,
+                        bool sameosds)
+{
+  auto pool = pgid.pool();
+  pg_sum.add(s);
+
+  num_pg++;
+  num_pg_by_state[s.state]++;
+  num_pg_by_pool_state[pgid.pool()][s.state]++;
+  num_pg_by_pool[pool]++;
+
+  if ((s.state & PG_STATE_CREATING) &&
+      s.parent_split_bits == 0) {
+    creating_pgs.insert(pgid);
+    if (s.acting_primary >= 0) {
+      creating_pgs_by_osd_epoch[s.acting_primary][s.mapping_epoch].insert(pgid);
+    }
+  }
+
+  if (s.state & PG_STATE_ACTIVE) {
+    ++num_pg_active;
+  }
+  if (s.state == 0) {
+    ++num_pg_unknown;
+  }
+
+  if (sameosds)
+    return;
+
+  for (auto p = s.blocked_by.begin();
+       p != s.blocked_by.end();
+       ++p) {
+    ++blocked_by_sum[*p];
+  }
+
+  for (auto p = s.acting.begin(); p != s.acting.end(); ++p) {
+    pg_by_osd[*p].insert(pgid);
+    num_pg_by_osd[*p].acting++;
+  }
+  for (auto p = s.up.begin(); p != s.up.end(); ++p) {
+    auto& t = pg_by_osd[*p];
+    if (t.find(pgid) == t.end()) {
+      t.insert(pgid);
+      num_pg_by_osd[*p].up_not_acting++;
+    }
+  }
+
+  if (s.up_primary >= 0) {
+    num_pg_by_osd[s.up_primary].primary++;
+  }
+}
+
+bool PGMap::stat_pg_sub(const pg_t &pgid, const pg_stat_t &s,
+                        bool sameosds)
+{
+  bool pool_erased = false;
+  pg_sum.sub(s);
+
+  num_pg--;
+  int end = --num_pg_by_state[s.state];
+  ceph_assert(end >= 0);
+  if (end == 0)
+    num_pg_by_state.erase(s.state);
+  if (--num_pg_by_pool_state[pgid.pool()][s.state] == 0) {
+    num_pg_by_pool_state[pgid.pool()].erase(s.state);
+  }
+  end = --num_pg_by_pool[pgid.pool()];
+  if (end == 0) {
+    pool_erased = true;
+  }
+
+  if ((s.state & PG_STATE_CREATING) &&
+      s.parent_split_bits == 0) {
+    creating_pgs.erase(pgid);
+    if (s.acting_primary >= 0) {
+      map<epoch_t,set<pg_t> >& r = creating_pgs_by_osd_epoch[s.acting_primary];
+      r[s.mapping_epoch].erase(pgid);
+      if (r[s.mapping_epoch].empty())
+	r.erase(s.mapping_epoch);
+      if (r.empty())
+	creating_pgs_by_osd_epoch.erase(s.acting_primary);
+    }
+  }
+
+  if (s.state & PG_STATE_ACTIVE) {
+    --num_pg_active;
+  }
+  if (s.state == 0) {
+    --num_pg_unknown;
+  }
+
+  if (sameosds)
+    return pool_erased;
+
+  for (auto p = s.blocked_by.begin();
+       p != s.blocked_by.end();
+       ++p) {
+    auto q = blocked_by_sum.find(*p);
+    ceph_assert(q != blocked_by_sum.end());
+    --q->second;
+    if (q->second == 0)
+      blocked_by_sum.erase(q);
+  }
+
+  set<int32_t> actingset;
+  for (auto p = s.acting.begin(); p != s.acting.end(); ++p) {
+    actingset.insert(*p);
+    auto& oset = pg_by_osd[*p];
+    oset.erase(pgid);
+    if (oset.empty())
+      pg_by_osd.erase(*p);
+    auto it = num_pg_by_osd.find(*p);
+    if (it != num_pg_by_osd.end() && it->second.acting > 0)
+      it->second.acting--;
+  }
+  for (auto p = s.up.begin(); p != s.up.end(); ++p) {
+    auto& oset = pg_by_osd[*p];
+    oset.erase(pgid);
+    if (oset.empty())
+      pg_by_osd.erase(*p);
+    if (actingset.count(*p))
+      continue;
+    auto it = num_pg_by_osd.find(*p);
+    if (it != num_pg_by_osd.end() && it->second.up_not_acting > 0)
+      it->second.up_not_acting--;
+  }
+
+  if (s.up_primary >= 0) {
+    auto it = num_pg_by_osd.find(s.up_primary);
+    if (it != num_pg_by_osd.end() && it->second.primary > 0)
+      it->second.primary--;
+  }
+  return pool_erased;
+}
+
+void PGMap::calc_purged_snaps()
+{
+  purged_snaps.clear();
+  set<int64_t> unknown;
+  for (auto& i : pg_stat) {
+    if (i.second.state == 0) {
+      unknown.insert(i.first.pool());
+      purged_snaps.erase(i.first.pool());
+      continue;
+    } else if (unknown.count(i.first.pool())) {
+      continue;
+    }
+    auto j = purged_snaps.find(i.first.pool());
+    if (j == purged_snaps.end()) {
+      // base case
+      purged_snaps[i.first.pool()] = i.second.purged_snaps;
+    } else {
+      j->second.intersection_of(i.second.purged_snaps);
+    }
+  }
+}
+
+void PGMap::calc_osd_sum_by_class(const OSDMap& osdmap)
+{
+  osd_sum_by_class.clear();
+  for (auto& i : osd_stat) {
+    const char *class_name = osdmap.crush->get_item_class(i.first);
+    if (class_name) {
+      osd_sum_by_class[class_name].add(i.second);
+    }
+  }
+}
+
+void PGMap::stat_osd_add(int osd, const osd_stat_t &s)
+{
+  num_osd++;
+  osd_sum.add(s);
+  if (osd >= (int)osd_last_seq.size()) {
+    osd_last_seq.resize(osd + 1);
+  }
+  osd_last_seq[osd] = s.seq;
+}
+
+void PGMap::stat_osd_sub(int osd, const osd_stat_t &s)
+{
+  num_osd--;
+  osd_sum.sub(s);
+  ceph_assert(osd < (int)osd_last_seq.size());
+  osd_last_seq[osd] = 0;
+}
+
+void PGMap::encode_digest(const OSDMap& osdmap,
+			  bufferlist& bl, uint64_t features)
+{
+  get_rules_avail(osdmap, &avail_space_by_rule);
+  calc_osd_sum_by_class(osdmap);
+  calc_purged_snaps();
+  PGMapDigest::encode(bl, features);
+}
+
+void PGMap::encode(bufferlist &bl, uint64_t features) const
+{
+  ENCODE_START(8, 8, bl);
+  encode(version, bl);
+  encode(pg_stat, bl);
+  encode(osd_stat, bl, features);
+  encode(last_osdmap_epoch, bl);
+  encode(last_pg_scan, bl);
+  encode(stamp, bl);
+  encode(pool_statfs, bl, features);
+  ENCODE_FINISH(bl);
+}
+
+void PGMap::decode(bufferlist::const_iterator &bl)
+{
+  DECODE_START(8, bl);
+  decode(version, bl);
+  decode(pg_stat, bl);
+  decode(osd_stat, bl);
+  decode(last_osdmap_epoch, bl);
+  decode(last_pg_scan, bl);
+  decode(stamp, bl);
+  decode(pool_statfs, bl);
+  DECODE_FINISH(bl);
+
+  calc_stats();
+}
+
+void PGMap::dump(ceph::Formatter *f, bool with_net) const
+{
+  dump_basic(f);
+  dump_pg_stats(f, false);
+  dump_pool_stats(f);
+  dump_osd_stats(f, with_net);
+}
+
+void PGMap::dump_basic(ceph::Formatter *f) const
+{
+  f->dump_unsigned("version", version);
+  f->dump_stream("stamp") << stamp;
+  f->dump_unsigned("last_osdmap_epoch", last_osdmap_epoch);
+  f->dump_unsigned("last_pg_scan", last_pg_scan);
+
+  f->open_object_section("pg_stats_sum");
+  pg_sum.dump(f);
+  f->close_section();
+
+  f->open_object_section("osd_stats_sum");
+  osd_sum.dump(f);
+  f->close_section();
+
+  dump_delta(f);
+}
+
+void PGMap::dump_delta(ceph::Formatter *f) const
+{
+  f->open_object_section("pg_stats_delta");
+  pg_sum_delta.dump(f);
+  f->dump_stream("stamp_delta") << stamp_delta;
+  f->close_section();
+}
+
+void PGMap::dump_pg_stats(ceph::Formatter *f, bool brief) const
+{
+  f->open_array_section("pg_stats");
+  for (auto i = pg_stat.begin();
+       i != pg_stat.end();
+       ++i) {
+    f->open_object_section("pg_stat");
+    f->dump_stream("pgid") << i->first;
+    if (brief)
+      i->second.dump_brief(f);
+    else
+      i->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void PGMap::dump_pg_progress(ceph::Formatter *f) const
+{
+  f->open_object_section("pgs");
+  for (auto& i : pg_stat) {
+    std::string n = stringify(i.first);
+    f->open_object_section(n.c_str());
+    f->dump_int("num_bytes_recovered", i.second.stats.sum.num_bytes_recovered);
+    f->dump_int("num_bytes", i.second.stats.sum.num_bytes);
+    f->dump_unsigned("reported_epoch", i.second.reported_epoch);
+    f->dump_string("state", pg_state_string(i.second.state));
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void PGMap::dump_pool_stats(ceph::Formatter *f) const
+{
+  f->open_array_section("pool_stats");
+  for (auto p = pg_pool_sum.begin();
+       p != pg_pool_sum.end();
+       ++p) {
+    f->open_object_section("pool_stat");
+    f->dump_int("poolid", p->first);
+    auto q = num_pg_by_pool.find(p->first);
+    if (q != num_pg_by_pool.end())
+      f->dump_unsigned("num_pg", q->second);
+    p->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void PGMap::dump_osd_stats(ceph::Formatter *f, bool with_net) const
+{
+  f->open_array_section("osd_stats");
+  for (auto q = osd_stat.begin();
+       q != osd_stat.end();
+       ++q) {
+    f->open_object_section("osd_stat");
+    f->dump_int("osd", q->first);
+    q->second.dump(f, with_net);
+    f->close_section();
+  }
+  f->close_section();
+
+  f->open_array_section("pool_statfs");
+  for (auto& p : pool_statfs) {
+    f->open_object_section("item");
+    f->dump_int("poolid", p.first.first);
+    f->dump_int("osd", p.first.second);
+    p.second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void PGMap::dump_osd_ping_times(ceph::Formatter *f) const
+{
+  f->open_array_section("osd_ping_times");
+  for (auto& [osd, stat] : osd_stat) {
+    f->open_object_section("osd_ping_time");
+    f->dump_int("osd", osd);
+    stat.dump_ping_time(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void PGMap::dump_pg_stats_plain(
+  ostream& ss,
+  const mempool::pgmap::unordered_map<pg_t, pg_stat_t>& pg_stats,
+  bool brief) const
+{
+  TextTable tab;
+
+  if (brief){
+    tab.define_column("PG_STAT", TextTable::LEFT, TextTable::LEFT);
+    tab.define_column("STATE", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("UP", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("UP_PRIMARY", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("ACTING", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("ACTING_PRIMARY", TextTable::LEFT, TextTable::RIGHT);
+  }
+  else {
+    tab.define_column("PG_STAT", TextTable::LEFT, TextTable::LEFT);
+    tab.define_column("OBJECTS", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("MISSING_ON_PRIMARY", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("DEGRADED", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("MISPLACED", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("UNFOUND", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("BYTES", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("OMAP_BYTES*", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("OMAP_KEYS*", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("LOG", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("DISK_LOG", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("STATE", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("STATE_STAMP", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("VERSION", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("REPORTED", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("UP", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("UP_PRIMARY", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("ACTING", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("ACTING_PRIMARY", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("LAST_SCRUB", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("SCRUB_STAMP", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("LAST_DEEP_SCRUB", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("DEEP_SCRUB_STAMP", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("SNAPTRIMQ_LEN", TextTable::LEFT, TextTable::RIGHT);
+  }
+
+  for (auto i = pg_stats.begin();
+       i != pg_stats.end(); ++i) {
+    const pg_stat_t &st(i->second);
+    if (brief) {
+      tab << i->first
+          << pg_state_string(st.state)
+          << st.up
+          << st.up_primary
+          << st.acting
+          << st.acting_primary
+          << TextTable::endrow;
+    } else {
+      ostringstream reported;
+      reported << st.reported_epoch << ":" << st.reported_seq;
+
+      tab << i->first
+          << st.stats.sum.num_objects
+          << st.stats.sum.num_objects_missing_on_primary
+          << st.stats.sum.num_objects_degraded
+          << st.stats.sum.num_objects_misplaced
+          << st.stats.sum.num_objects_unfound
+          << st.stats.sum.num_bytes
+          << st.stats.sum.num_omap_bytes
+          << st.stats.sum.num_omap_keys
+          << st.log_size
+          << st.ondisk_log_size
+          << pg_state_string(st.state)
+          << st.last_change
+          << st.version
+          << reported.str()
+          << pg_vector_string(st.up)
+          << st.up_primary
+          << pg_vector_string(st.acting)
+          << st.acting_primary
+          << st.last_scrub
+          << st.last_scrub_stamp
+          << st.last_deep_scrub
+          << st.last_deep_scrub_stamp
+          << st.snaptrimq_len
+          << TextTable::endrow;
+    }
+  }
+
+  ss << tab;
+}
+
+void PGMap::dump(ostream& ss) const
+{
+  dump_basic(ss);
+  dump_pg_stats(ss, false);
+  dump_pool_stats(ss, false);
+  dump_pg_sum_stats(ss, false);
+  dump_osd_stats(ss);
+}
+
+void PGMap::dump_basic(ostream& ss) const
+{
+  ss << "version " << version << std::endl;
+  ss << "stamp " << stamp << std::endl;
+  ss << "last_osdmap_epoch " << last_osdmap_epoch << std::endl;
+  ss << "last_pg_scan " << last_pg_scan << std::endl;
+}
+
+void PGMap::dump_pg_stats(ostream& ss, bool brief) const
+{
+  dump_pg_stats_plain(ss, pg_stat, brief);
+}
+
+void PGMap::dump_pool_stats(ostream& ss, bool header) const
+{
+  TextTable tab;
+
+  if (header) {
+    tab.define_column("POOLID", TextTable::LEFT, TextTable::LEFT);
+    tab.define_column("OBJECTS", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("MISSING_ON_PRIMARY", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("DEGRADED", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("MISPLACED", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("UNFOUND", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("BYTES", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("OMAP_BYTES*", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("OMAP_KEYS*", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("LOG", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("DISK_LOG", TextTable::LEFT, TextTable::RIGHT);
+  } else {
+    tab.define_column("", TextTable::LEFT, TextTable::LEFT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+  }
+
+  for (auto p = pg_pool_sum.begin();
+       p != pg_pool_sum.end();
+       ++p) {
+    tab << p->first
+        << p->second.stats.sum.num_objects
+        << p->second.stats.sum.num_objects_missing_on_primary
+        << p->second.stats.sum.num_objects_degraded
+        << p->second.stats.sum.num_objects_misplaced
+        << p->second.stats.sum.num_objects_unfound
+        << p->second.stats.sum.num_bytes
+        << p->second.stats.sum.num_omap_bytes
+        << p->second.stats.sum.num_omap_keys
+        << p->second.log_size
+        << p->second.ondisk_log_size
+        << TextTable::endrow;
+  }
+
+  ss << tab;
+}
+
+void PGMap::dump_pg_sum_stats(ostream& ss, bool header) const
+{
+  TextTable tab;
+
+  if (header) {
+    tab.define_column("PG_STAT", TextTable::LEFT, TextTable::LEFT);
+    tab.define_column("OBJECTS", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("MISSING_ON_PRIMARY", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("DEGRADED", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("MISPLACED", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("UNFOUND", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("BYTES", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("OMAP_BYTES*", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("OMAP_KEYS*", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("LOG", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("DISK_LOG", TextTable::LEFT, TextTable::RIGHT);
+  } else {
+    tab.define_column("", TextTable::LEFT, TextTable::LEFT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+    tab.define_column("", TextTable::LEFT, TextTable::RIGHT);
+  };
+
+  tab << "sum"
+      << pg_sum.stats.sum.num_objects
+      << pg_sum.stats.sum.num_objects_missing_on_primary
+      << pg_sum.stats.sum.num_objects_degraded
+      << pg_sum.stats.sum.num_objects_misplaced
+      << pg_sum.stats.sum.num_objects_unfound
+      << pg_sum.stats.sum.num_bytes
+      << pg_sum.stats.sum.num_omap_bytes
+      << pg_sum.stats.sum.num_omap_keys
+      << pg_sum.log_size
+      << pg_sum.ondisk_log_size
+      << TextTable::endrow;
+
+  ss << tab;
+}
+
+void PGMap::dump_osd_stats(ostream& ss) const
+{
+  TextTable tab;
+
+  tab.define_column("OSD_STAT", TextTable::LEFT, TextTable::LEFT);
+  tab.define_column("USED", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("AVAIL", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("USED_RAW", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("TOTAL", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("HB_PEERS", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("PG_SUM", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("PRIMARY_PG_SUM", TextTable::LEFT, TextTable::RIGHT);
+
+  for (auto p = osd_stat.begin();
+       p != osd_stat.end();
+       ++p) {
+    tab << p->first
+        << byte_u_t(p->second.statfs.get_used())
+        << byte_u_t(p->second.statfs.available)
+        << byte_u_t(p->second.statfs.get_used_raw())
+        << byte_u_t(p->second.statfs.total)
+        << p->second.hb_peers
+        << get_num_pg_by_osd(p->first)
+        << get_num_primary_pg_by_osd(p->first)
+        << TextTable::endrow;
+  }
+
+  tab << "sum"
+      << byte_u_t(osd_sum.statfs.get_used())
+      << byte_u_t(osd_sum.statfs.available)
+      << byte_u_t(osd_sum.statfs.get_used_raw())
+      << byte_u_t(osd_sum.statfs.total)
+      << TextTable::endrow;
+
+  ss << tab;
+}
+
+void PGMap::dump_osd_sum_stats(ostream& ss) const
+{
+  TextTable tab;
+
+  tab.define_column("OSD_STAT", TextTable::LEFT, TextTable::LEFT);
+  tab.define_column("USED", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("AVAIL", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("USED_RAW", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("TOTAL", TextTable::LEFT, TextTable::RIGHT);
+
+  tab << "sum"
+      << byte_u_t(osd_sum.statfs.get_used())
+      << byte_u_t(osd_sum.statfs.available)
+      << byte_u_t(osd_sum.statfs.get_used_raw())
+      << byte_u_t(osd_sum.statfs.total)
+      << TextTable::endrow;
+
+  ss << tab;
+}
+
+void PGMap::get_stuck_stats(
+  int types, const utime_t cutoff,
+  mempool::pgmap::unordered_map<pg_t, pg_stat_t>& stuck_pgs) const
+{
+  ceph_assert(types != 0);
+  for (auto i = pg_stat.begin();
+       i != pg_stat.end();
+       ++i) {
+    utime_t val = cutoff; // don't care about >= cutoff so that is infinity
+
+    if ((types & STUCK_INACTIVE) && !(i->second.state & PG_STATE_ACTIVE)) {
+      if (i->second.last_active < val)
+	val = i->second.last_active;
+    }
+
+    if ((types & STUCK_UNCLEAN) && !(i->second.state & PG_STATE_CLEAN)) {
+      if (i->second.last_clean < val)
+	val = i->second.last_clean;
+    }
+
+    if ((types & STUCK_DEGRADED) && (i->second.state & PG_STATE_DEGRADED)) {
+      if (i->second.last_undegraded < val)
+	val = i->second.last_undegraded;
+    }
+
+    if ((types & STUCK_UNDERSIZED) && (i->second.state & PG_STATE_UNDERSIZED)) {
+      if (i->second.last_fullsized < val)
+	val = i->second.last_fullsized;
+    }
+
+    if ((types & STUCK_STALE) && (i->second.state & PG_STATE_STALE)) {
+      if (i->second.last_unstale < val)
+	val = i->second.last_unstale;
+    }
+
+    // val is now the earliest any of the requested stuck states began
+    if (val < cutoff) {
+      stuck_pgs[i->first] = i->second;
+    }
+  }
+}
+
+bool PGMap::get_stuck_counts(const utime_t cutoff, map<string, int>& note) const
+{
+  int inactive = 0;
+  int unclean = 0;
+  int degraded = 0;
+  int undersized = 0;
+  int stale = 0;
+
+  for (auto i = pg_stat.begin();
+       i != pg_stat.end();
+       ++i) {
+    if (! (i->second.state & PG_STATE_ACTIVE)) {
+      if (i->second.last_active < cutoff)
+        ++inactive;
+    }
+    if (! (i->second.state & PG_STATE_CLEAN)) {
+      if (i->second.last_clean < cutoff)
+        ++unclean;
+    }
+    if (i->second.state & PG_STATE_DEGRADED) {
+      if (i->second.last_undegraded < cutoff)
+        ++degraded;
+    }
+    if (i->second.state & PG_STATE_UNDERSIZED) {
+      if (i->second.last_fullsized < cutoff)
+        ++undersized;
+    }
+    if (i->second.state & PG_STATE_STALE) {
+      if (i->second.last_unstale < cutoff)
+        ++stale;
+    }
+  }
+
+  if (inactive)
+    note["stuck inactive"] = inactive;
+
+  if (unclean)
+    note["stuck unclean"] = unclean;
+
+  if (undersized)
+    note["stuck undersized"] = undersized;
+
+  if (degraded)
+    note["stuck degraded"] = degraded;
+
+  if (stale)
+    note["stuck stale"] = stale;
+
+  return inactive || unclean || undersized || degraded || stale;
+}
+
+void PGMap::dump_stuck(ceph::Formatter *f, int types, utime_t cutoff) const
+{
+  mempool::pgmap::unordered_map<pg_t, pg_stat_t> stuck_pg_stats;
+  get_stuck_stats(types, cutoff, stuck_pg_stats);
+  f->open_array_section("stuck_pg_stats");
+  for (auto i = stuck_pg_stats.begin();
+       i != stuck_pg_stats.end();
+       ++i) {
+    f->open_object_section("pg_stat");
+    f->dump_stream("pgid") << i->first;
+    i->second.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void PGMap::dump_stuck_plain(ostream& ss, int types, utime_t cutoff) const
+{
+  mempool::pgmap::unordered_map<pg_t, pg_stat_t> stuck_pg_stats;
+  get_stuck_stats(types, cutoff, stuck_pg_stats);
+  if (!stuck_pg_stats.empty())
+    dump_pg_stats_plain(ss, stuck_pg_stats, true);
+}
+
+int PGMap::dump_stuck_pg_stats(
+  stringstream &ds,
+  ceph::Formatter *f,
+  int threshold,
+  vector<string>& args) const
+{
+  int stuck_types = 0;
+
+  for (auto i = args.begin(); i != args.end(); ++i) {
+    if (*i == "inactive")
+      stuck_types |= PGMap::STUCK_INACTIVE;
+    else if (*i == "unclean")
+      stuck_types |= PGMap::STUCK_UNCLEAN;
+    else if (*i == "undersized")
+      stuck_types |= PGMap::STUCK_UNDERSIZED;
+    else if (*i == "degraded")
+      stuck_types |= PGMap::STUCK_DEGRADED;
+    else if (*i == "stale")
+      stuck_types |= PGMap::STUCK_STALE;
+    else {
+      ds << "Unknown type: " << *i << std::endl;
+      return -EINVAL;
+    }
+  }
+
+  utime_t now(ceph_clock_now());
+  utime_t cutoff = now - utime_t(threshold, 0);
+
+  if (!f) {
+    dump_stuck_plain(ds, stuck_types, cutoff);
+  } else {
+    dump_stuck(f, stuck_types, cutoff);
+    f->flush(ds);
+  }
+
+  return 0;
+}
+
+void PGMap::dump_osd_perf_stats(ceph::Formatter *f) const
+{
+  f->open_array_section("osd_perf_infos");
+  for (auto i = osd_stat.begin();
+       i != osd_stat.end();
+       ++i) {
+    f->open_object_section("osd");
+    f->dump_int("id", i->first);
+    {
+      f->open_object_section("perf_stats");
+      i->second.os_perf_stat.dump(f);
+      f->close_section();
+    }
+    f->close_section();
+  }
+  f->close_section();
+}
+void PGMap::print_osd_perf_stats(std::ostream *ss) const
+{
+  TextTable tab;
+  tab.define_column("osd", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("commit_latency(ms)", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("apply_latency(ms)", TextTable::LEFT, TextTable::RIGHT);
+  for (auto i = osd_stat.begin();
+       i != osd_stat.end();
+       ++i) {
+    tab << i->first;
+    tab << i->second.os_perf_stat.os_commit_latency_ns / 1000000ull;
+    tab << i->second.os_perf_stat.os_apply_latency_ns / 1000000ull;
+    tab << TextTable::endrow;
+  }
+  (*ss) << tab;
+}
+
+void PGMap::dump_osd_blocked_by_stats(ceph::Formatter *f) const
+{
+  f->open_array_section("osd_blocked_by_infos");
+  for (auto i = blocked_by_sum.begin();
+       i != blocked_by_sum.end();
+       ++i) {
+    f->open_object_section("osd");
+    f->dump_int("id", i->first);
+    f->dump_int("num_blocked", i->second);
+    f->close_section();
+  }
+  f->close_section();
+}
+void PGMap::print_osd_blocked_by_stats(std::ostream *ss) const
+{
+  TextTable tab;
+  tab.define_column("osd", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("num_blocked", TextTable::LEFT, TextTable::RIGHT);
+  for (auto i = blocked_by_sum.begin();
+       i != blocked_by_sum.end();
+       ++i) {
+    tab << i->first;
+    tab << i->second;
+    tab << TextTable::endrow;
+  }
+  (*ss) << tab;
+}
+
+
+/**
+ * update aggregated delta
+ *
+ * @param cct               ceph context
+ * @param ts                Timestamp for the stats being delta'ed
+ * @param old_pool_sum      Previous stats sum
+ * @param last_ts           Last timestamp for pool
+ * @param result_pool_sum   Resulting stats
+ * @param result_pool_delta Resulting pool delta
+ * @param result_ts_delta   Resulting timestamp delta
+ * @param delta_avg_list    List of last N computed deltas, used to average
+ */
+void PGMap::update_delta(
+  CephContext *cct,
+  const utime_t ts,
+  const pool_stat_t& old_pool_sum,
+  utime_t *last_ts,
+  const pool_stat_t& current_pool_sum,
+  pool_stat_t *result_pool_delta,
+  utime_t *result_ts_delta,
+  mempool::pgmap::list<pair<pool_stat_t,utime_t> > *delta_avg_list)
+{
+  /* @p ts is the timestamp we want to associate with the data
+   * in @p old_pool_sum, and on which we will base ourselves to
+   * calculate the delta, stored in 'delta_t'.
+   */
+  utime_t delta_t;
+  delta_t = ts;         // start with the provided timestamp
+  delta_t -= *last_ts;  // take the last timestamp we saw
+  *last_ts = ts;        // @p ts becomes the last timestamp we saw
+
+  // adjust delta_t, quick start if there is no update in a long period
+  delta_t = std::min(delta_t,
+                    utime_t(2 * (cct ? cct->_conf->mon_delta_reset_interval : 10), 0));
+
+  // calculate a delta, and average over the last 6 deltas by default.
+  /* start by taking a copy of our current @p result_pool_sum, and by
+   * taking out the stats from @p old_pool_sum.  This generates a stats
+   * delta.  Stash this stats delta in @p delta_avg_list, along with the
+   * timestamp delta for these results.
+   */
+  pool_stat_t d = current_pool_sum;
+  d.stats.sub(old_pool_sum.stats);
+
+  /* Aggregate current delta, and take out the last seen delta (if any) to
+   * average it out.
+   * Skip calculating delta while sum was not synchronized.
+   */
+  if(!old_pool_sum.stats.sum.is_zero()) {
+    delta_avg_list->push_back(make_pair(d,delta_t));
+    *result_ts_delta += delta_t;
+    result_pool_delta->stats.add(d.stats);
+  }
+  size_t s = cct ? cct->_conf.get_val<uint64_t>("mon_stat_smooth_intervals") : 1;
+  while (delta_avg_list->size() > s) {
+    result_pool_delta->stats.sub(delta_avg_list->front().first.stats);
+    *result_ts_delta -= delta_avg_list->front().second;
+    delta_avg_list->pop_front();
+  }
+}
+
+/**
+ * Update a given pool's deltas
+ *
+ * @param cct           Ceph Context
+ * @param ts            Timestamp for the stats being delta'ed
+ * @param pool          Pool's id
+ * @param old_pool_sum  Previous stats sum
+ */
+void PGMap::update_one_pool_delta(
+  CephContext *cct,
+  const utime_t ts,
+  const int64_t pool,
+  const pool_stat_t& old_pool_sum)
+{
+  if (per_pool_sum_deltas.count(pool) == 0) {
+    ceph_assert(per_pool_sum_deltas_stamps.count(pool) == 0);
+    ceph_assert(per_pool_sum_delta.count(pool) == 0);
+  }
+
+  auto& sum_delta = per_pool_sum_delta[pool];
+
+  update_delta(cct, ts, old_pool_sum, &sum_delta.second, pg_pool_sum[pool],
+               &sum_delta.first, &per_pool_sum_deltas_stamps[pool],
+               &per_pool_sum_deltas[pool]);
+}
+
+/**
+ * Update pools' deltas
+ *
+ * @param cct               CephContext
+ * @param ts                Timestamp for the stats being delta'ed
+ * @param pg_pool_sum_old   Map of pool stats for delta calcs.
+ */
+void PGMap::update_pool_deltas(
+  CephContext *cct, const utime_t ts,
+  const mempool::pgmap::unordered_map<int32_t,pool_stat_t>& pg_pool_sum_old)
+{
+  for (auto it = pg_pool_sum_old.begin();
+       it != pg_pool_sum_old.end(); ++it) {
+    update_one_pool_delta(cct, ts, it->first, it->second);
+  }
+}
+
+void PGMap::clear_delta()
+{
+  pg_sum_delta = pool_stat_t();
+  pg_sum_deltas.clear();
+  stamp_delta = utime_t();
+}
+
+void PGMap::generate_test_instances(list<PGMap*>& o)
+{
+  o.push_back(new PGMap);
+  list<Incremental*> inc;
+  Incremental::generate_test_instances(inc);
+  delete inc.front();
+  inc.pop_front();
+  while (!inc.empty()) {
+    PGMap *pmp = new PGMap();
+    *pmp = *o.back();
+    o.push_back(pmp);
+    o.back()->apply_incremental(NULL, *inc.front());
+    delete inc.front();
+    inc.pop_front();
+  }
+}
+
+void PGMap::get_filtered_pg_stats(uint64_t state, int64_t poolid, int64_t osdid,
+                                  bool primary, set<pg_t>& pgs) const
+{
+  for (auto i = pg_stat.begin();
+       i != pg_stat.end();
+       ++i) {
+    if ((poolid >= 0) && (poolid != i->first.pool()))
+      continue;
+    if ((osdid >= 0) && !(i->second.is_acting_osd(osdid,primary)))
+      continue;
+    if (state == (uint64_t)-1 ||                 // "all"
+	(i->second.state & state) ||             // matches a state bit
+	(state == 0 && i->second.state == 0)) {  // matches "unknown" (== 0)
+      pgs.insert(i->first);
+    }
+  }
+}
+
+void PGMap::dump_filtered_pg_stats(ceph::Formatter *f, set<pg_t>& pgs) const
+{
+  f->open_array_section("pg_stats");
+  for (auto i = pgs.begin(); i != pgs.end(); ++i) {
+    const pg_stat_t& st = pg_stat.at(*i);
+    f->open_object_section("pg_stat");
+    f->dump_stream("pgid") << *i;
+    st.dump(f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+void PGMap::dump_filtered_pg_stats(ostream& ss, set<pg_t>& pgs) const
+{
+  TextTable tab;
+  utime_t now = ceph_clock_now();
+
+  tab.define_column("PG", TextTable::LEFT, TextTable::LEFT);
+  tab.define_column("OBJECTS", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("DEGRADED", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("MISPLACED", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("UNFOUND", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("BYTES", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("OMAP_BYTES*", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("OMAP_KEYS*", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("LOG", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("STATE", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("SINCE", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("VERSION", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("REPORTED", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("UP", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("ACTING", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("SCRUB_STAMP", TextTable::LEFT, TextTable::RIGHT);
+  tab.define_column("DEEP_SCRUB_STAMP", TextTable::LEFT, TextTable::RIGHT);
+
+  for (auto i = pgs.begin(); i != pgs.end(); ++i) {
+    const pg_stat_t& st = pg_stat.at(*i);
+
+    ostringstream reported;
+    reported << st.reported_epoch << ":" << st.reported_seq;
+
+    ostringstream upstr, actingstr;
+    upstr << pg_vector_string(st.up) << 'p' << st.up_primary;
+    actingstr << pg_vector_string(st.acting) << 'p' << st.acting_primary;
+    tab << *i
+        << st.stats.sum.num_objects
+        << st.stats.sum.num_objects_degraded
+        << st.stats.sum.num_objects_misplaced
+        << st.stats.sum.num_objects_unfound
+        << st.stats.sum.num_bytes
+        << st.stats.sum.num_omap_bytes
+        << st.stats.sum.num_omap_keys
+        << st.log_size
+        << pg_state_string(st.state)
+        << utimespan_str(now - st.last_change)
+        << st.version
+        << reported.str()
+        << upstr.str()
+        << actingstr.str()
+        << st.last_scrub_stamp
+        << st.last_deep_scrub_stamp
+        << TextTable::endrow;
+  }
+
+  ss << tab;
+}
+
+void PGMap::dump_pool_stats_and_io_rate(int64_t poolid, const OSDMap &osd_map,
+                                        ceph::Formatter *f,
+                                        stringstream *rs) const {
+  string pool_name = osd_map.get_pool_name(poolid);
+  if (f) {
+    f->open_object_section("pool");
+    f->dump_string("pool_name", pool_name.c_str());
+    f->dump_int("pool_id", poolid);
+    f->open_object_section("recovery");
+  }
+  list<string> sl;
+  stringstream tss;
+  pool_recovery_summary(f, &sl, poolid);
+  if (!f && !sl.empty()) {
+    for (auto &p : sl)
+      tss << "  " << p << "\n";
+  }
+  if (f) {
+    f->close_section(); // object section recovery
+    f->open_object_section("recovery_rate");
+  }
+  ostringstream rss;
+  pool_recovery_rate_summary(f, &rss, poolid);
+  if (!f && !rss.str().empty())
+    tss << "  recovery io " << rss.str() << "\n";
+  if (f) {
+    f->close_section(); // object section recovery_rate
+    f->open_object_section("client_io_rate");
+  }
+  rss.clear();
+  rss.str("");
+  pool_client_io_rate_summary(f, &rss, poolid);
+  if (!f && !rss.str().empty())
+    tss << "  client io " << rss.str() << "\n";
+  // dump cache tier IO rate for cache pool
+  const pg_pool_t *pool = osd_map.get_pg_pool(poolid);
+  if (pool->is_tier()) {
+    if (f) {
+      f->close_section(); // object section client_io_rate
+      f->open_object_section("cache_io_rate");
+    }
+    rss.clear();
+    rss.str("");
+    pool_cache_io_rate_summary(f, &rss, poolid);
+    if (!f && !rss.str().empty())
+      tss << "  cache tier io " << rss.str() << "\n";
+  }
+  if (f) {
+    f->close_section(); // object section cache_io_rate
+    f->close_section(); // object section pool
+  } else {
+    *rs << "pool " << pool_name << " id " << poolid << "\n";
+    if (!tss.str().empty())
+      *rs << tss.str() << "\n";
+    else
+      *rs << "  nothing is going on\n\n";
+  }
+}
+
+// Get crush parentage for an osd (skip root)
+set<std::string> PGMap::osd_parentage(const OSDMap& osdmap, int id) const
+{
+  set<std::string> reporters_by_subtree;
+  auto reporter_subtree_level = g_conf().get_val<string>("mon_osd_reporter_subtree_level");
+
+  auto loc = osdmap.crush->get_full_location(id);
+  for (auto& [parent_bucket_type, parent_id] : loc) {
+    // Should we show the root?  Might not be too informative like "default"
+    if (parent_bucket_type != "root" &&
+        parent_bucket_type != reporter_subtree_level) {
+      reporters_by_subtree.insert(parent_id);
+    }
+  }
+  return reporters_by_subtree;
+}
+
+void PGMap::get_health_checks(
+  CephContext *cct,
+  const OSDMap& osdmap,
+  health_check_map_t *checks) const
+{
+  utime_t now = ceph_clock_now();
+  const auto max = cct->_conf.get_val<uint64_t>("mon_health_max_detail");
+  const auto& pools = osdmap.get_pools();
+
+  typedef enum pg_consequence_t {
+    UNAVAILABLE = 1,   // Client IO to the pool may block
+    DEGRADED = 2,      // Fewer than the requested number of replicas are present
+    BACKFILL_FULL = 3, // Backfill is blocked for space considerations
+                       // This may or may not be a deadlock condition.
+    DAMAGED = 4,        // The data may be missing or inconsistent on disk and
+                       //  requires repair
+    RECOVERY_FULL = 5  // Recovery is blocked because OSDs are full
+  } pg_consequence_t;
+
+  // For a given PG state, how should it be reported at the pool level?
+  class PgStateResponse {
+    public:
+    pg_consequence_t consequence;
+    typedef std::function< utime_t(const pg_stat_t&) > stuck_cb;
+    stuck_cb stuck_since;
+    bool invert;
+
+    PgStateResponse(const pg_consequence_t& c, stuck_cb&& s)
+      : consequence(c), stuck_since(std::move(s)), invert(false)
+    {
+    }
+
+    PgStateResponse(const pg_consequence_t& c, stuck_cb&& s, bool i)
+      : consequence(c), stuck_since(std::move(s)), invert(i)
+    {
+    }
+  };
+
+  // Record the PG state counts that contributed to a reported pool state
+  class PgCauses {
+    public:
+    // Map of PG_STATE_* to number of pgs in that state.
+    std::map<unsigned, unsigned> states;
+
+    // List of all PG IDs that had a state contributing
+    // to this health condition.
+    std::set<pg_t> pgs;
+
+    std::map<pg_t, std::string> pg_messages;
+  };
+
+  // Map of PG state to how to respond to it
+  std::map<unsigned, PgStateResponse> state_to_response = {
+    // Immediate reports
+    { PG_STATE_INCONSISTENT,     {DAMAGED,     {}} },
+    { PG_STATE_INCOMPLETE,       {UNAVAILABLE, {}} },
+    { PG_STATE_SNAPTRIM_ERROR,   {DAMAGED,     {}} },
+    { PG_STATE_RECOVERY_UNFOUND, {DAMAGED,     {}} },
+    { PG_STATE_BACKFILL_UNFOUND, {DAMAGED,     {}} },
+    { PG_STATE_BACKFILL_TOOFULL, {BACKFILL_FULL, {}} },
+    { PG_STATE_RECOVERY_TOOFULL, {RECOVERY_FULL, {}} },
+    { PG_STATE_DEGRADED,         {DEGRADED,    {}} },
+    { PG_STATE_DOWN,             {UNAVAILABLE, {}} },
+    // Delayed (wait until stuck) reports
+    { PG_STATE_PEERING,          {UNAVAILABLE, [](const pg_stat_t &p){return p.last_peered;}    } },
+    { PG_STATE_UNDERSIZED,       {DEGRADED,    [](const pg_stat_t &p){return p.last_fullsized;} } },
+    { PG_STATE_STALE,            {UNAVAILABLE, [](const pg_stat_t &p){return p.last_unstale;}   } },
+    // Delayed and inverted reports
+    { PG_STATE_ACTIVE,           {UNAVAILABLE, [](const pg_stat_t &p){return p.last_active;}, true} }
+  };
+
+  // Specialized state printer that takes account of inversion of
+  // ACTIVE, CLEAN checks.
+  auto state_name = [](const uint64_t &state) {
+    // Special cases for the states that are inverted checks
+    if (state == PG_STATE_CLEAN) {
+      return std::string("unclean");
+    } else if (state == PG_STATE_ACTIVE) {
+      return std::string("inactive");
+    } else {
+      return pg_state_string(state);
+    }
+  };
+
+  // Map of what is wrong to information about why, implicitly also stores
+  // the list of what is wrong.
+  std::map<pg_consequence_t, PgCauses> detected;
+
+  // Optimisation: trim down the number of checks to apply based on
+  // the summary counters
+  std::map<unsigned, PgStateResponse> possible_responses;
+  for (const auto &i : num_pg_by_state) {
+    for (const auto &j : state_to_response) {
+      if (!j.second.invert) {
+        // Check for normal tests by seeing if any pgs have the flag
+        if (i.first & j.first) {
+          possible_responses.insert(j);
+        }
+      }
+    }
+  }
+
+  for (const auto &j : state_to_response) {
+    if (j.second.invert) {
+      // Check for inverted tests by seeing if not-all pgs have the flag
+      const auto &found = num_pg_by_state.find(j.first);
+      if (found == num_pg_by_state.end() || found->second != num_pg) {
+        possible_responses.insert(j);
+      }
+    }
+  }
+
+  utime_t cutoff = now - utime_t(cct->_conf.get_val<int64_t>("mon_pg_stuck_threshold"), 0);
+  // Loop over all PGs, if there are any possibly-unhealthy states in there
+  if (!possible_responses.empty()) {
+    for (const auto& i : pg_stat) {
+      const auto &pg_id = i.first;
+      const auto &pg_info = i.second;
+
+      for (const auto &j : state_to_response) {
+        const auto &pg_response_state = j.first;
+        const auto &pg_response = j.second;
+
+        // Apply the state test
+        if (!(bool(pg_info.state & pg_response_state) != pg_response.invert)) {
+          continue;
+        }
+
+        // Apply stuckness test if needed
+        if (pg_response.stuck_since) {
+          // Delayed response, check for stuckness
+          utime_t last_whatever = pg_response.stuck_since(pg_info);
+          if (last_whatever.is_zero() &&
+            pg_info.last_change >= cutoff) {
+            // still moving, ignore
+            continue;
+          } else if (last_whatever >= cutoff) {
+            // Not stuck enough, ignore.
+            continue;
+          } else {
+
+          }
+        }
+
+        auto &causes = detected[pg_response.consequence];
+        causes.states[pg_response_state]++;
+        causes.pgs.insert(pg_id);
+
+        // Don't bother composing detail string if we have already recorded
+        // too many
+        if (causes.pg_messages.size() > max) {
+          continue;
+        }
+
+        std::ostringstream ss;
+        if (pg_response.stuck_since) {
+          utime_t since = pg_response.stuck_since(pg_info);
+          ss << "pg " << pg_id << " is stuck " << state_name(pg_response_state);
+          if (since == utime_t()) {
+            ss << " since forever";
+          } else {
+            utime_t dur = now - since;
+            ss << " for " << utimespan_str(dur);
+          }
+          ss << ", current state " << pg_state_string(pg_info.state)
+             << ", last acting " << pg_info.acting;
+        } else {
+          ss << "pg " << pg_id << " is "
+             << pg_state_string(pg_info.state);
+          ss << ", acting " << pg_info.acting;
+          if (pg_info.stats.sum.num_objects_unfound) {
+            ss << ", " << pg_info.stats.sum.num_objects_unfound
+               << " unfound";
+          }
+        }
+
+        if (pg_info.state & PG_STATE_INCOMPLETE) {
+          const pg_pool_t *pi = osdmap.get_pg_pool(pg_id.pool());
+          if (pi && pi->min_size > 1) {
+            ss << " (reducing pool "
+               << osdmap.get_pool_name(pg_id.pool())
+               << " min_size from " << (int)pi->min_size
+               << " may help; search ceph.com/docs for 'incomplete')";
+          }
+        }
+
+        causes.pg_messages[pg_id] = ss.str();
+      }
+    }
+  } else {
+    dout(10) << __func__ << " skipping loop over PGs: counters look OK" << dendl;
+  }
+
+  for (const auto &i : detected) {
+    std::string health_code;
+    health_status_t sev;
+    std::string summary;
+    switch(i.first) {
+      case UNAVAILABLE:
+        health_code = "PG_AVAILABILITY";
+        sev = HEALTH_WARN;
+        summary = "Reduced data availability: ";
+        break;
+      case DEGRADED:
+        health_code = "PG_DEGRADED";
+        summary = "Degraded data redundancy: ";
+        sev = HEALTH_WARN;
+        break;
+      case BACKFILL_FULL:
+        health_code = "PG_BACKFILL_FULL";
+        summary = "Low space hindering backfill (add storage if this doesn't resolve itself): ";
+        sev = HEALTH_WARN;
+        break;
+      case DAMAGED:
+        health_code = "PG_DAMAGED";
+        summary = "Possible data damage: ";
+        sev = HEALTH_ERR;
+        break;
+      case RECOVERY_FULL:
+        health_code = "PG_RECOVERY_FULL";
+        summary = "Full OSDs blocking recovery: ";
+        sev = HEALTH_ERR;
+        break;
+      default:
+        ceph_abort();
+    }
+
+    if (i.first == DEGRADED) {
+      if (pg_sum.stats.sum.num_objects_degraded &&
+          pg_sum.stats.sum.num_object_copies > 0) {
+        double pc = (double)pg_sum.stats.sum.num_objects_degraded /
+          (double)pg_sum.stats.sum.num_object_copies * (double)100.0;
+        char b[20];
+        snprintf(b, sizeof(b), "%.3lf", pc);
+        ostringstream ss;
+        ss << pg_sum.stats.sum.num_objects_degraded
+           << "/" << pg_sum.stats.sum.num_object_copies << " objects degraded ("
+           << b << "%)";
+
+        // Throw in a comma for the benefit of the following PG counts
+        summary += ss.str() + ", ";
+      }
+    }
+
+    // Compose summary message saying how many PGs in what states led
+    // to this health check failing
+    std::vector<std::string> pg_msgs;
+    int64_t count = 0;
+    for (const auto &j : i.second.states) {
+      std::ostringstream msg;
+      msg << j.second << (j.second > 1 ? " pgs " : " pg ") << state_name(j.first);
+      pg_msgs.push_back(msg.str());
+      count += j.second;
+    }
+    summary += joinify(pg_msgs.begin(), pg_msgs.end(), std::string(", "));
+
+    health_check_t *check = &checks->add(
+        health_code,
+        sev,
+        summary,
+	count);
+
+    // Compose list of PGs contributing to this health check failing
+    for (const auto &j : i.second.pg_messages) {
+      check->detail.push_back(j.second);
+    }
+  }
+
+  // OSD_SCRUB_ERRORS
+  if (pg_sum.stats.sum.num_scrub_errors) {
+    ostringstream ss;
+    ss << pg_sum.stats.sum.num_scrub_errors << " scrub errors";
+    checks->add("OSD_SCRUB_ERRORS", HEALTH_ERR, ss.str(),
+		pg_sum.stats.sum.num_scrub_errors);
+  }
+
+  // LARGE_OMAP_OBJECTS
+  if (pg_sum.stats.sum.num_large_omap_objects) {
+    list<string> detail;
+    for (auto &pool : pools) {
+      const string& pool_name = osdmap.get_pool_name(pool.first);
+      auto it2 = pg_pool_sum.find(pool.first);
+      if (it2 == pg_pool_sum.end()) {
+        continue;
+      }
+      const pool_stat_t *pstat = &it2->second;
+      if (pstat == nullptr) {
+        continue;
+      }
+      const object_stat_sum_t& sum = pstat->stats.sum;
+      if (sum.num_large_omap_objects) {
+        stringstream ss;
+        ss << sum.num_large_omap_objects << " large objects found in pool "
+           << "'" << pool_name << "'";
+        detail.push_back(ss.str());
+      }
+    }
+    if (!detail.empty()) {
+      ostringstream ss;
+      ss << pg_sum.stats.sum.num_large_omap_objects << " large omap objects";
+      auto& d = checks->add("LARGE_OMAP_OBJECTS", HEALTH_WARN, ss.str(),
+			    pg_sum.stats.sum.num_large_omap_objects);
+      stringstream tip;
+      tip << "Search the cluster log for 'Large omap object found' for more "
+          << "details.";
+      detail.push_back(tip.str());
+      d.detail.swap(detail);
+    }
+  }
+
+  // CACHE_POOL_NEAR_FULL
+  {
+    list<string> detail;
+    unsigned num_pools = 0;
+    for (auto& p : pools) {
+      if ((!p.second.target_max_objects && !p.second.target_max_bytes) ||
+	  !pg_pool_sum.count(p.first)) {
+	continue;
+      }
+      bool nearfull = false;
+      const string& name = osdmap.get_pool_name(p.first);
+      const pool_stat_t& st = get_pg_pool_sum_stat(p.first);
+      uint64_t ratio = p.second.cache_target_full_ratio_micro +
+	((1000000 - p.second.cache_target_full_ratio_micro) *
+	 cct->_conf->mon_cache_target_full_warn_ratio);
+      if (p.second.target_max_objects &&
+	  (uint64_t)(st.stats.sum.num_objects -
+		     st.stats.sum.num_objects_hit_set_archive) >
+	  p.second.target_max_objects * (ratio / 1000000.0)) {
+	ostringstream ss;
+	ss << "cache pool '" << name << "' with "
+	   << si_u_t(st.stats.sum.num_objects)
+	   << " objects at/near target max "
+	   << si_u_t(p.second.target_max_objects) << " objects";
+	detail.push_back(ss.str());
+	nearfull = true;
+      }
+      if (p.second.target_max_bytes &&
+	  (uint64_t)(st.stats.sum.num_bytes -
+		     st.stats.sum.num_bytes_hit_set_archive) >
+	  p.second.target_max_bytes * (ratio / 1000000.0)) {
+	ostringstream ss;
+	ss << "cache pool '" << name
+	   << "' with " << byte_u_t(st.stats.sum.num_bytes)
+	   << " at/near target max "
+	   << byte_u_t(p.second.target_max_bytes);
+	detail.push_back(ss.str());
+	nearfull = true;
+      }
+      if (nearfull) {
+	++num_pools;
+      }
+    }
+    if (!detail.empty()) {
+      ostringstream ss;
+      ss << num_pools << " cache pools at or near target size";
+      auto& d = checks->add("CACHE_POOL_NEAR_FULL", HEALTH_WARN, ss.str(),
+			    num_pools);
+      d.detail.swap(detail);
+    }
+  }
+
+  // TOO_FEW_PGS
+  unsigned num_in = osdmap.get_num_in_osds();
+  auto sum_pg_up = std::max(static_cast<size_t>(pg_sum.up), pg_stat.size());
+  const auto min_pg_per_osd =
+    cct->_conf.get_val<uint64_t>("mon_pg_warn_min_per_osd");
+  if (num_in && min_pg_per_osd > 0 && osdmap.get_pools().size() > 0) {
+    auto per = sum_pg_up / num_in;
+    if (per < min_pg_per_osd && per) {
+      ostringstream ss;
+      ss << "too few PGs per OSD (" << per
+	 << " < min " << min_pg_per_osd << ")";
+      checks->add("TOO_FEW_PGS", HEALTH_WARN, ss.str(),
+		  min_pg_per_osd - per);
+    }
+  }
+
+  // TOO_MANY_PGS
+  auto max_pg_per_osd = cct->_conf.get_val<uint64_t>("mon_max_pg_per_osd");
+  if (num_in && max_pg_per_osd > 0) {
+    auto per = sum_pg_up / num_in;
+    if (per > max_pg_per_osd) {
+      ostringstream ss;
+      ss << "too many PGs per OSD (" << per
+	 << " > max " << max_pg_per_osd << ")";
+      checks->add("TOO_MANY_PGS", HEALTH_WARN, ss.str(),
+		  per - max_pg_per_osd);
+    }
+  }
+
+  // TOO_FEW_OSDS
+  auto warn_too_few_osds = cct->_conf.get_val<bool>("mon_warn_on_too_few_osds");
+  auto osd_pool_default_size = cct->_conf.get_val<uint64_t>("osd_pool_default_size");
+  if (warn_too_few_osds && osdmap.get_num_osds() < osd_pool_default_size) {
+    ostringstream ss;
+    ss << "OSD count " << osdmap.get_num_osds()
+	 << " < osd_pool_default_size " << osd_pool_default_size;
+    checks->add("TOO_FEW_OSDS", HEALTH_WARN, ss.str(),
+		osd_pool_default_size - osdmap.get_num_osds());
+  }
+
+  // SLOW_PING_TIME
+  // Convert milliseconds to microseconds
+  auto warn_slow_ping_time = cct->_conf.get_val<double>("mon_warn_on_slow_ping_time") * 1000;
+  auto grace = cct->_conf.get_val<int64_t>("osd_heartbeat_grace");
+  if (warn_slow_ping_time == 0) {
+    double ratio = cct->_conf.get_val<double>("mon_warn_on_slow_ping_ratio");
+    warn_slow_ping_time = grace;
+    warn_slow_ping_time *= 1000000 * ratio; // Seconds of grace to microseconds at ratio
+  }
+  if (warn_slow_ping_time > 0) {
+
+    struct mon_ping_item_t {
+      uint32_t pingtime;
+      int from;
+      int to;
+      bool improving;
+
+      bool operator<(const mon_ping_item_t& rhs) const {
+        if (pingtime < rhs.pingtime)
+          return true;
+        if (pingtime > rhs.pingtime)
+          return false;
+        if (from < rhs.from)
+          return true;
+        if (from > rhs.from)
+          return false;
+        return to < rhs.to;
+      }
+    };
+
+    list<string> detail_back;
+    list<string> detail_front;
+    list<string> detail;
+    set<mon_ping_item_t> back_sorted, front_sorted;
+    for (auto i : osd_stat) {
+      for (auto j : i.second.hb_pingtime) {
+
+	// Maybe source info is old
+	if (now.sec() - j.second.last_update > grace * 60)
+	  continue;
+
+	mon_ping_item_t back;
+	back.pingtime = std::max(j.second.back_pingtime[0], j.second.back_pingtime[1]);
+	back.pingtime = std::max(back.pingtime, j.second.back_pingtime[2]);
+	back.from = i.first;
+	back.to = j.first;
+	if (back.pingtime > warn_slow_ping_time) {
+	  back.improving = (j.second.back_pingtime[0] < j.second.back_pingtime[1]
+			    && j.second.back_pingtime[1] < j.second.back_pingtime[2]);
+	  back_sorted.emplace(back);
+	}
+
+	mon_ping_item_t front;
+	front.pingtime = std::max(j.second.front_pingtime[0], j.second.front_pingtime[1]);
+	front.pingtime = std::max(front.pingtime, j.second.front_pingtime[2]);
+	front.from = i.first;
+	front.to = j.first;
+	if (front.pingtime > warn_slow_ping_time) {
+	  front.improving = (j.second.front_pingtime[0] < j.second.front_pingtime[1]
+			     && j.second.front_pingtime[1] < j.second.back_pingtime[2]);
+	  front_sorted.emplace(front);
+	}
+      }
+      if (i.second.num_shards_repaired >
+		      cct->_conf.get_val<uint64_t>("mon_osd_warn_num_repaired")) {
+        ostringstream ss;
+	ss << "osd." << i.first << " had " << i.second.num_shards_repaired << " reads repaired";
+        detail.push_back(ss.str());
+      }
+    }
+    if (!detail.empty()) {
+      ostringstream ss;
+      ss << "Too many repaired reads on " << detail.size() << " OSDs";
+      auto& d = checks->add("OSD_TOO_MANY_REPAIRS", HEALTH_WARN, ss.str(),
+		      detail.size());
+      d.detail.swap(detail);
+    }
+    int max_detail = 10;
+    for (auto &sback : boost::adaptors::reverse(back_sorted)) {
+      ostringstream ss;
+      if (max_detail == 0) {
+	ss << "Truncated long network list.  Use ceph daemon mgr.# dump_osd_network for more information";
+        detail_back.push_back(ss.str());
+        break;
+      }
+      max_detail--;
+      ss << "Slow OSD heartbeats on back from osd." << sback.from
+	 << " [" << osd_parentage(osdmap, sback.from) << "]"
+         << (osdmap.is_down(sback.from) ? " (down)" : "")
+	 << " to osd." << sback.to
+	 << " [" << osd_parentage(osdmap, sback.to) << "]"
+         << (osdmap.is_down(sback.to) ? " (down)" : "")
+	 << " " << fixed_u_to_string(sback.pingtime, 3) << " msec"
+	 << (sback.improving ? " possibly improving" : "");
+      detail_back.push_back(ss.str());
+    }
+    max_detail = 10;
+    for (auto &sfront : boost::adaptors::reverse(front_sorted)) {
+      ostringstream ss;
+      if (max_detail == 0) {
+	ss << "Truncated long network list.  Use ceph daemon mgr.# dump_osd_network for more information";
+        detail_front.push_back(ss.str());
+        break;
+      }
+      max_detail--;
+      // Get crush parentage for each osd
+      ss << "Slow OSD heartbeats on front from osd." << sfront.from
+	 << " [" << osd_parentage(osdmap, sfront.from) << "]"
+         << (osdmap.is_down(sfront.from) ? " (down)" : "")
+         << " to osd." << sfront.to
+	 << " [" << osd_parentage(osdmap, sfront.to) << "]"
+         << (osdmap.is_down(sfront.to) ? " (down)" : "")
+	 << " " << fixed_u_to_string(sfront.pingtime, 3) << " msec"
+	 << (sfront.improving ? " possibly improving" : "");
+      detail_front.push_back(ss.str());
+    }
+    if (detail_back.size() != 0) {
+      ostringstream ss;
+      ss << "Slow OSD heartbeats on back (longest "
+	 << fixed_u_to_string(back_sorted.rbegin()->pingtime, 3) << "ms)";
+      auto& d = checks->add("OSD_SLOW_PING_TIME_BACK", HEALTH_WARN, ss.str(),
+		      back_sorted.size());
+      d.detail.swap(detail_back);
+    }
+    if (detail_front.size() != 0) {
+      ostringstream ss;
+      ss << "Slow OSD heartbeats on front (longest "
+	 << fixed_u_to_string(front_sorted.rbegin()->pingtime, 3) << "ms)";
+      auto& d = checks->add("OSD_SLOW_PING_TIME_FRONT", HEALTH_WARN, ss.str(),
+		      front_sorted.size());
+      d.detail.swap(detail_front);
+    }
+  }
+
+  // SMALLER_PGP_NUM
+  // MANY_OBJECTS_PER_PG
+  if (!pg_stat.empty()) {
+    list<string> pgp_detail, many_detail;
+    const auto mon_pg_warn_min_objects =
+      cct->_conf.get_val<int64_t>("mon_pg_warn_min_objects");
+    const auto mon_pg_warn_min_pool_objects =
+      cct->_conf.get_val<int64_t>("mon_pg_warn_min_pool_objects");
+    const auto mon_pg_warn_max_object_skew =
+      cct->_conf.get_val<double>("mon_pg_warn_max_object_skew");
+    for (auto p = pg_pool_sum.begin();
+         p != pg_pool_sum.end();
+         ++p) {
+      const pg_pool_t *pi = osdmap.get_pg_pool(p->first);
+      if (!pi)
+	continue;   // in case osdmap changes haven't propagated to PGMap yet
+      const string& name = osdmap.get_pool_name(p->first);
+      // NOTE: we use pg_num_target and pgp_num_target for the purposes of
+      // the warnings.  If the cluster is failing to converge on the target
+      // values that is a separate issue!
+      if (pi->get_pg_num_target() > pi->get_pgp_num_target() &&
+	  !(name.find(".DELETED") != string::npos &&
+	    cct->_conf->mon_fake_pool_delete)) {
+	ostringstream ss;
+	ss << "pool " << name << " pg_num "
+	   << pi->get_pg_num_target()
+	   << " > pgp_num " << pi->get_pgp_num_target();
+	pgp_detail.push_back(ss.str());
+      }
+      int average_objects_per_pg = pg_sum.stats.sum.num_objects / pg_stat.size();
+      if (average_objects_per_pg > 0 &&
+          pg_sum.stats.sum.num_objects >= mon_pg_warn_min_objects &&
+          p->second.stats.sum.num_objects >= mon_pg_warn_min_pool_objects) {
+	int objects_per_pg = p->second.stats.sum.num_objects /
+	  pi->get_pg_num_target();
+	float ratio = (float)objects_per_pg / (float)average_objects_per_pg;
+	if (mon_pg_warn_max_object_skew > 0 &&
+	    ratio > mon_pg_warn_max_object_skew) {
+	  ostringstream ss;
+	  if (pi->pg_autoscale_mode != pg_pool_t::pg_autoscale_mode_t::ON) {
+	      ss << "pool " << name << " objects per pg ("
+		 << objects_per_pg << ") is more than " << ratio
+		 << " times cluster average ("
+		 << average_objects_per_pg << ")";
+	      many_detail.push_back(ss.str());
+	  }
+	}
+      }
+    }
+    if (!pgp_detail.empty()) {
+      ostringstream ss;
+      ss << pgp_detail.size() << " pools have pg_num > pgp_num";
+      auto& d = checks->add("SMALLER_PGP_NUM", HEALTH_WARN, ss.str(),
+			    pgp_detail.size());
+      d.detail.swap(pgp_detail);
+    }
+    if (!many_detail.empty()) {
+      ostringstream ss;
+      ss << many_detail.size() << " pools have many more objects per pg than"
+	 << " average";
+      auto& d = checks->add("MANY_OBJECTS_PER_PG", HEALTH_WARN, ss.str(),
+			    many_detail.size());
+      d.detail.swap(many_detail);
+    }
+  }
+
+  // POOL_FULL
+  // POOL_NEAR_FULL
+  {
+    float warn_threshold = (float)g_conf().get_val<int64_t>("mon_pool_quota_warn_threshold")/100;
+    float crit_threshold = (float)g_conf().get_val<int64_t>("mon_pool_quota_crit_threshold")/100;
+    list<string> full_detail, nearfull_detail;
+    unsigned full_pools = 0, nearfull_pools = 0;
+    for (auto it : pools) {
+      auto it2 = pg_pool_sum.find(it.first);
+      if (it2 == pg_pool_sum.end()) {
+	continue;
+      }
+      const pool_stat_t *pstat = &it2->second;
+      const object_stat_sum_t& sum = pstat->stats.sum;
+      const string& pool_name = osdmap.get_pool_name(it.first);
+      const pg_pool_t &pool = it.second;
+      bool full = false, nearfull = false;
+      if (pool.quota_max_objects > 0) {
+	stringstream ss;
+	if ((uint64_t)sum.num_objects >= pool.quota_max_objects) {
+	} else if (crit_threshold > 0 &&
+		   sum.num_objects >= pool.quota_max_objects*crit_threshold) {
+	  ss << "pool '" << pool_name
+	     << "' has " << sum.num_objects << " objects"
+	     << " (max " << pool.quota_max_objects << ")";
+	  full_detail.push_back(ss.str());
+	  full = true;
+	} else if (warn_threshold > 0 &&
+		   sum.num_objects >= pool.quota_max_objects*warn_threshold) {
+	  ss << "pool '" << pool_name
+	     << "' has " << sum.num_objects << " objects"
+	     << " (max " << pool.quota_max_objects << ")";
+	  nearfull_detail.push_back(ss.str());
+	  nearfull = true;
+	}
+      }
+      if (pool.quota_max_bytes > 0) {
+	stringstream ss;
+	if ((uint64_t)sum.num_bytes >= pool.quota_max_bytes) {
+	} else if (crit_threshold > 0 &&
+		   sum.num_bytes >= pool.quota_max_bytes*crit_threshold) {
+	  ss << "pool '" << pool_name
+	     << "' has " << byte_u_t(sum.num_bytes)
+	     << " (max " << byte_u_t(pool.quota_max_bytes) << ")";
+	  full_detail.push_back(ss.str());
+	  full = true;
+	} else if (warn_threshold > 0 &&
+		   sum.num_bytes >= pool.quota_max_bytes*warn_threshold) {
+	  ss << "pool '" << pool_name
+	     << "' has " << byte_u_t(sum.num_bytes)
+	     << " (max " << byte_u_t(pool.quota_max_bytes) << ")";
+	  nearfull_detail.push_back(ss.str());
+	  nearfull = true;
+	}
+      }
+      if (full) {
+	++full_pools;
+      }
+      if (nearfull) {
+	++nearfull_pools;
+      }
+    }
+    if (full_pools) {
+      ostringstream ss;
+      ss << full_pools << " pools full";
+      auto& d = checks->add("POOL_FULL", HEALTH_ERR, ss.str(), full_pools);
+      d.detail.swap(full_detail);
+    }
+    if (nearfull_pools) {
+      ostringstream ss;
+      ss << nearfull_pools << " pools nearfull";
+      auto& d = checks->add("POOL_NEAR_FULL", HEALTH_WARN, ss.str(), nearfull_pools);
+      d.detail.swap(nearfull_detail);
+    }
+  }
+
+  // OBJECT_MISPLACED
+  if (pg_sum.stats.sum.num_objects_misplaced &&
+      pg_sum.stats.sum.num_object_copies > 0 &&
+      cct->_conf->mon_warn_on_misplaced) {
+    double pc = (double)pg_sum.stats.sum.num_objects_misplaced /
+      (double)pg_sum.stats.sum.num_object_copies * (double)100.0;
+    char b[20];
+    snprintf(b, sizeof(b), "%.3lf", pc);
+    ostringstream ss;
+    ss << pg_sum.stats.sum.num_objects_misplaced
+       << "/" << pg_sum.stats.sum.num_object_copies << " objects misplaced ("
+       << b << "%)";
+    checks->add("OBJECT_MISPLACED", HEALTH_WARN, ss.str(),
+		pg_sum.stats.sum.num_objects_misplaced);
+  }
+
+  // OBJECT_UNFOUND
+  if (pg_sum.stats.sum.num_objects_unfound &&
+      pg_sum.stats.sum.num_objects) {
+    double pc = (double)pg_sum.stats.sum.num_objects_unfound /
+      (double)pg_sum.stats.sum.num_objects * (double)100.0;
+    char b[20];
+    snprintf(b, sizeof(b), "%.3lf", pc);
+    ostringstream ss;
+    ss << pg_sum.stats.sum.num_objects_unfound
+       << "/" << pg_sum.stats.sum.num_objects << " objects unfound (" << b << "%)";
+    auto& d = checks->add("OBJECT_UNFOUND", HEALTH_WARN, ss.str(),
+			  pg_sum.stats.sum.num_objects_unfound);
+
+    for (auto& p : pg_stat) {
+      if (p.second.stats.sum.num_objects_unfound) {
+	ostringstream ss;
+	ss << "pg " << p.first
+	   << " has " << p.second.stats.sum.num_objects_unfound
+	   << " unfound objects";
+	d.detail.push_back(ss.str());
+	if (d.detail.size() > max) {
+	  d.detail.push_back("(additional pgs left out for brevity)");
+	  break;
+	}
+      }
+    }
+  }
+
+  // REQUEST_SLOW
+  // REQUEST_STUCK
+  // SLOW_OPS unifies them in mimic.
+  if (osdmap.require_osd_release < ceph_release_t::mimic &&
+      cct->_conf->mon_osd_warn_op_age > 0 &&
+      !osd_sum.op_queue_age_hist.h.empty() &&
+      osd_sum.op_queue_age_hist.upper_bound() / 1000.0 >
+      cct->_conf->mon_osd_warn_op_age) {
+    list<string> warn_detail, error_detail;
+    unsigned warn = 0, error = 0;
+    float err_age =
+      cct->_conf->mon_osd_warn_op_age * cct->_conf->mon_osd_err_op_age_ratio;
+    const pow2_hist_t& h = osd_sum.op_queue_age_hist;
+    for (unsigned i = h.h.size() - 1; i > 0; --i) {
+      float ub = (float)(1 << i) / 1000.0;
+      if (ub < cct->_conf->mon_osd_warn_op_age)
+	break;
+      if (h.h[i]) {
+	ostringstream ss;
+	ss << h.h[i] << " ops are blocked > " << ub << " sec";
+	if (ub > err_age) {
+	  error += h.h[i];
+	  error_detail.push_back(ss.str());
+	} else {
+	  warn += h.h[i];
+	  warn_detail.push_back(ss.str());
+	}
+      }
+    }
+
+    map<float,set<int>> warn_osd_by_max; // max -> osds
+    map<float,set<int>> error_osd_by_max; // max -> osds
+    if (!warn_detail.empty() || !error_detail.empty()) {
+      for (auto& p : osd_stat) {
+	const pow2_hist_t& h = p.second.op_queue_age_hist;
+	for (unsigned i = h.h.size() - 1; i > 0; --i) {
+	  float ub = (float)(1 << i) / 1000.0;
+	  if (ub < cct->_conf->mon_osd_warn_op_age)
+	    break;
+	  if (h.h[i]) {
+	    if (ub > err_age) {
+	      error_osd_by_max[ub].insert(p.first);
+	    } else {
+	      warn_osd_by_max[ub].insert(p.first);
+	    }
+	    break;
+	  }
+	}
+      }
+    }
+
+    if (!warn_detail.empty()) {
+      ostringstream ss;
+      ss << warn << " slow requests are blocked > "
+	 << cct->_conf->mon_osd_warn_op_age << " sec";
+      auto& d = checks->add("REQUEST_SLOW", HEALTH_WARN, ss.str(), warn);
+      d.detail.swap(warn_detail);
+      int left = max;
+      for (auto& p : warn_osd_by_max) {
+	ostringstream ss;
+	if (p.second.size() > 1) {
+	  ss << "osds " << p.second
+             << " have blocked requests > " << p.first << " sec";
+	} else {
+	  ss << "osd." << *p.second.begin()
+             << " has blocked requests > " << p.first << " sec";
+	}
+	d.detail.push_back(ss.str());
+	if (--left == 0) {
+	  break;
+	}
+      }
+    }
+    if (!error_detail.empty()) {
+      ostringstream ss;
+      ss << error << " stuck requests are blocked > "
+	 << err_age << " sec";
+      auto& d = checks->add("REQUEST_STUCK", HEALTH_ERR, ss.str(), error);
+      d.detail.swap(error_detail);
+      int left = max;
+      for (auto& p : error_osd_by_max) {
+	ostringstream ss;
+	if (p.second.size() > 1) {
+	  ss << "osds " << p.second
+             << " have stuck requests > " << p.first << " sec";
+	} else {
+	  ss << "osd." << *p.second.begin()
+             << " has stuck requests > " << p.first << " sec";
+	}
+	d.detail.push_back(ss.str());
+	if (--left == 0) {
+	  break;
+	}
+      }
+    }
+  }
+
+  // OBJECT_STORE_WARN
+  if (osd_sum.os_alerts.size()) {
+    map<string, pair<size_t, list<string>>> os_alerts_sum;
+
+    for (auto& a : osd_sum.os_alerts) {
+      int left = max;
+      string s0 = " osd.";
+      s0 += stringify(a.first);
+      for (auto& aa : a.second) {
+        string s(s0);
+        s += " ";
+        s += aa.second;
+        auto it = os_alerts_sum.find(aa.first);
+        if (it == os_alerts_sum.end()) {
+          list<string> d;
+          d.emplace_back(s);
+          os_alerts_sum.emplace(aa.first, std::make_pair(1, d));
+        } else {
+          auto& p = it->second;
+          ++p.first;
+          p.second.emplace_back(s);
+        }
+	if (--left == 0) {
+	  break;
+	}
+      }
+    }
+
+    for (auto& asum : os_alerts_sum) {
+      string summary = stringify(asum.second.first) + " OSD(s)";
+      if (asum.first == "BLUEFS_SPILLOVER") {
+	summary += " experiencing BlueFS spillover";
+      } else if (asum.first == "BLUESTORE_NO_COMPRESSION") {
+	summary += " have broken BlueStore compression";
+      } else if (asum.first == "BLUESTORE_LEGACY_STATFS") {
+	summary += " reporting legacy (not per-pool) BlueStore stats";
+      } else if (asum.first == "BLUESTORE_DISK_SIZE_MISMATCH") {
+	summary += " have dangerous mismatch between BlueStore block device and free list sizes";
+      } else if (asum.first == "BLUESTORE_NO_PER_PG_OMAP") {
+	summary += " reporting legacy (not per-pg) BlueStore omap";
+      } else if (asum.first == "BLUESTORE_NO_PER_POOL_OMAP") {
+	summary += " reporting legacy (not per-pool) BlueStore omap usage stats";
+      } else if (asum.first == "BLUESTORE_SPURIOUS_READ_ERRORS") {
+        summary += " have spurious read errors";
+      }
+
+      auto& d = checks->add(asum.first, HEALTH_WARN, summary, asum.second.first);
+      for (auto& s : asum.second.second) {
+        d.detail.push_back(s);
+      }
+    }
+  }
+  // PG_NOT_SCRUBBED
+  // PG_NOT_DEEP_SCRUBBED
+  if (cct->_conf->mon_warn_pg_not_scrubbed_ratio ||
+        cct->_conf->mon_warn_pg_not_deep_scrubbed_ratio) {
+    list<string> detail, deep_detail;
+    int detail_max = max, deep_detail_max = max;
+    int detail_more = 0, deep_detail_more = 0;
+    int detail_total = 0, deep_detail_total = 0;
+    for (auto& p : pg_stat) {
+      int64_t pnum =  p.first.pool();
+      auto pool = osdmap.get_pg_pool(pnum);
+      if (!pool)
+        continue;
+      if (cct->_conf->mon_warn_pg_not_scrubbed_ratio) {
+        double scrub_max_interval = 0;
+        pool->opts.get(pool_opts_t::SCRUB_MAX_INTERVAL, &scrub_max_interval);
+        if (scrub_max_interval <= 0) {
+          scrub_max_interval = cct->_conf->osd_scrub_max_interval;
+        }
+        const double age = (cct->_conf->mon_warn_pg_not_scrubbed_ratio * scrub_max_interval) +
+          scrub_max_interval;
+        utime_t cutoff = now;
+        cutoff -= age;
+        if (p.second.last_scrub_stamp < cutoff) {
+          if (detail_max > 0) {
+            ostringstream ss;
+            ss << "pg " << p.first << " not scrubbed since "
+               << p.second.last_scrub_stamp;
+            detail.push_back(ss.str());
+            --detail_max;
+          } else {
+            ++detail_more;
+          }
+          ++detail_total;
+        }
+      }
+      if (cct->_conf->mon_warn_pg_not_deep_scrubbed_ratio) {
+        double deep_scrub_interval = 0;
+        pool->opts.get(pool_opts_t::DEEP_SCRUB_INTERVAL, &deep_scrub_interval);
+        if (deep_scrub_interval <= 0) {
+          deep_scrub_interval = cct->_conf->osd_deep_scrub_interval;
+        }
+        double deep_age = (cct->_conf->mon_warn_pg_not_deep_scrubbed_ratio * deep_scrub_interval) +
+          deep_scrub_interval;
+        utime_t deep_cutoff = now;
+        deep_cutoff -= deep_age;
+        if (p.second.last_deep_scrub_stamp < deep_cutoff) {
+          if (deep_detail_max > 0) {
+            ostringstream ss;
+            ss << "pg " << p.first << " not deep-scrubbed since "
+               << p.second.last_deep_scrub_stamp;
+            deep_detail.push_back(ss.str());
+            --deep_detail_max;
+          } else {
+            ++deep_detail_more;
+          }
+          ++deep_detail_total;
+        }
+      }
+    }
+    if (detail_total) {
+      ostringstream ss;
+      ss << detail_total << " pgs not scrubbed in time";
+      auto& d = checks->add("PG_NOT_SCRUBBED", HEALTH_WARN, ss.str(), detail_total);
+
+      if (!detail.empty()) {
+        d.detail.swap(detail);
+
+        if (detail_more) {
+          ostringstream ss;
+          ss << detail_more << " more pgs... ";
+          d.detail.push_back(ss.str());
+        }
+      }
+    }
+    if (deep_detail_total) {
+      ostringstream ss;
+      ss << deep_detail_total << " pgs not deep-scrubbed in time";
+      auto& d = checks->add("PG_NOT_DEEP_SCRUBBED", HEALTH_WARN, ss.str(),
+			    deep_detail_total);
+
+      if (!deep_detail.empty()) {
+        d.detail.swap(deep_detail);
+
+        if (deep_detail_more) {
+          ostringstream ss;
+          ss << deep_detail_more << " more pgs... ";
+          d.detail.push_back(ss.str());
+        }
+      }
+    }
+  }
+
+  // POOL_APP
+  if (g_conf().get_val<bool>("mon_warn_on_pool_no_app")) {
+    list<string> detail;
+    for (auto &it : pools) {
+      const pg_pool_t &pool = it.second;
+      const string& pool_name = osdmap.get_pool_name(it.first);
+      auto it2 = pg_pool_sum.find(it.first);
+      if (it2 == pg_pool_sum.end()) {
+        continue;
+      }
+      const pool_stat_t *pstat = &it2->second;
+      if (pstat == nullptr) {
+        continue;
+      }
+      const object_stat_sum_t& sum = pstat->stats.sum;
+      // application metadata is not encoded until luminous is minimum
+      // required release
+      if (sum.num_objects > 0 && pool.application_metadata.empty() &&
+          !pool.is_tier()) {
+        stringstream ss;
+        ss << "application not enabled on pool '" << pool_name << "'";
+        detail.push_back(ss.str());
+      }
+    }
+    if (!detail.empty()) {
+      ostringstream ss;
+      ss << detail.size() << " pool(s) do not have an application enabled";
+      auto& d = checks->add("POOL_APP_NOT_ENABLED", HEALTH_WARN, ss.str(),
+			    detail.size());
+      stringstream tip;
+      tip << "use 'ceph osd pool application enable <pool-name> "
+          << "<app-name>', where <app-name> is 'cephfs', 'rbd', 'rgw', "
+          << "or freeform for custom applications.";
+      detail.push_back(tip.str());
+      d.detail.swap(detail);
+    }
+  }
+
+  // PG_SLOW_SNAP_TRIMMING
+  if (!pg_stat.empty() && cct->_conf->mon_osd_snap_trim_queue_warn_on > 0) {
+    uint32_t snapthreshold = cct->_conf->mon_osd_snap_trim_queue_warn_on;
+    uint64_t snaptrimq_exceeded = 0;
+    uint32_t longest_queue = 0;
+    const pg_t* longest_q_pg = nullptr;
+    list<string> detail;
+
+    for (auto& i: pg_stat) {
+      uint32_t current_len = i.second.snaptrimq_len;
+      if (current_len >= snapthreshold) {
+        snaptrimq_exceeded++;
+        if (longest_queue <= current_len) {
+          longest_q_pg = &i.first;
+          longest_queue = current_len;
+        }
+        if (detail.size() < max - 1) {
+          stringstream ss;
+          ss << "snap trim queue for pg " << i.first << " at " << current_len;
+          detail.push_back(ss.str());
+          continue;
+        }
+        if (detail.size() < max) {
+          detail.push_back("...more pgs affected");
+          continue;
+        }
+      }
+    }
+
+    if (snaptrimq_exceeded) {
+      {
+         ostringstream ss;
+         ss << "longest queue on pg " << *longest_q_pg << " at " << longest_queue;
+         detail.push_back(ss.str());
+      }
+
+      stringstream ss;
+      ss << "snap trim queue for " << snaptrimq_exceeded << " pg(s) >= " << snapthreshold << " (mon_osd_snap_trim_queue_warn_on)";
+      auto& d = checks->add("PG_SLOW_SNAP_TRIMMING", HEALTH_WARN, ss.str(),
+			    snaptrimq_exceeded);
+      detail.push_back("try decreasing \"osd snap trim sleep\" and/or increasing \"osd pg max concurrent snap trims\".");
+      d.detail.swap(detail);
+    }
+  }
+}
+
+void PGMap::print_summary(ceph::Formatter *f, ostream *out) const
+{
+  if (f) {
+    f->open_array_section("pgs_by_pool_state");
+    for (auto& i: num_pg_by_pool_state) {
+      f->open_object_section("per_pool_pgs_by_state");
+      f->dump_int("pool_id", i.first);
+      f->open_array_section("pg_state_counts");
+      for (auto& j : i.second) {
+        f->open_object_section("pg_state_count");
+        f->dump_string("state_name", pg_state_string(j.first));
+        f->dump_int("count", j.second);
+        f->close_section();
+      }
+      f->close_section();
+      f->close_section();
+    }
+    f->close_section();
+  }
+  PGMapDigest::print_summary(f, out);
+}
+
+int process_pg_map_command(
+  const string& orig_prefix,
+  const cmdmap_t& orig_cmdmap,
+  const PGMap& pg_map,
+  const OSDMap& osdmap,
+  ceph::Formatter *f,
+  stringstream *ss,
+  bufferlist *odata)
+{
+  string prefix = orig_prefix;
+  auto cmdmap = orig_cmdmap;
+
+  string omap_stats_note =
+      "\n* NOTE: Omap statistics are gathered during deep scrub and "
+      "may be inaccurate soon afterwards depending on utilization. See "
+      "http://docs.ceph.com/en/latest/dev/placement-group/#omap-statistics "
+      "for further details.\n";
+  bool omap_stats_note_required = false;
+
+  // perhaps these would be better in the parsing, but it's weird
+  bool primary = false;
+  if (prefix == "pg dump_json") {
+    vector<string> v;
+    v.push_back(string("all"));
+    cmd_putval(g_ceph_context, cmdmap, "dumpcontents", v);
+    prefix = "pg dump";
+  } else if (prefix == "pg dump_pools_json") {
+    vector<string> v;
+    v.push_back(string("pools"));
+    cmd_putval(g_ceph_context, cmdmap, "dumpcontents", v);
+    prefix = "pg dump";
+  } else if (prefix == "pg ls-by-primary") {
+    primary = true;
+    prefix = "pg ls";
+  } else if (prefix == "pg ls-by-osd") {
+    prefix = "pg ls";
+  } else if (prefix == "pg ls-by-pool") {
+    prefix = "pg ls";
+    string poolstr;
+    cmd_getval(cmdmap, "poolstr", poolstr);
+    int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str());
+    if (pool < 0) {
+      *ss << "pool " << poolstr << " does not exist";
+      return -ENOENT;
+    }
+    cmd_putval(g_ceph_context, cmdmap, "pool", pool);
+  }
+
+  stringstream ds;
+  if (prefix == "pg stat") {
+    if (f) {
+      f->open_object_section("pg_summary");
+      pg_map.print_oneline_summary(f, NULL);
+      f->close_section();
+      f->flush(ds);
+    } else {
+      ds << pg_map;
+    }
+    odata->append(ds);
+    return 0;
+  }
+
+  if (prefix == "pg getmap") {
+    pg_map.encode(*odata);
+    *ss << "got pgmap version " << pg_map.version;
+    return 0;
+  }
+
+  if (prefix == "pg dump") {
+    string val;
+    vector<string> dumpcontents;
+    set<string> what;
+    if (cmd_getval(cmdmap, "dumpcontents", dumpcontents)) {
+      copy(dumpcontents.begin(), dumpcontents.end(),
+           inserter(what, what.end()));
+    }
+    if (what.empty())
+      what.insert("all");
+    if (f) {
+      if (what.count("all")) {
+	f->open_object_section("pg_map");
+	pg_map.dump(f);
+	f->close_section();
+      } else if (what.count("summary") || what.count("sum")) {
+	f->open_object_section("pg_map");
+	pg_map.dump_basic(f);
+	f->close_section();
+      } else {
+	if (what.count("pools")) {
+	  pg_map.dump_pool_stats(f);
+	}
+	if (what.count("osds")) {
+	  pg_map.dump_osd_stats(f);
+	}
+	if (what.count("pgs")) {
+	  pg_map.dump_pg_stats(f, false);
+	}
+	if (what.count("pgs_brief")) {
+	  pg_map.dump_pg_stats(f, true);
+	}
+	if (what.count("delta")) {
+	  f->open_object_section("delta");
+	  pg_map.dump_delta(f);
+	  f->close_section();
+	}
+      }
+      f->flush(*odata);
+    } else {
+      if (what.count("all")) {
+	pg_map.dump(ds);
+        omap_stats_note_required = true;
+      } else if (what.count("summary") || what.count("sum")) {
+	pg_map.dump_basic(ds);
+	pg_map.dump_pg_sum_stats(ds, true);
+	pg_map.dump_osd_sum_stats(ds);
+        omap_stats_note_required = true;
+      } else {
+	if (what.count("pgs_brief")) {
+	  pg_map.dump_pg_stats(ds, true);
+	}
+	bool header = true;
+	if (what.count("pgs")) {
+	  pg_map.dump_pg_stats(ds, false);
+	  header = false;
+          omap_stats_note_required = true;
+	}
+	if (what.count("pools")) {
+	  pg_map.dump_pool_stats(ds, header);
+          omap_stats_note_required = true;
+	}
+	if (what.count("osds")) {
+	  pg_map.dump_osd_stats(ds);
+	}
+      }
+      odata->append(ds);
+      if (omap_stats_note_required) {
+        odata->append(omap_stats_note);
+      }
+    }
+    *ss << "dumped " << what;
+    return 0;
+  }
+
+  if (prefix == "pg ls") {
+    int64_t osd = -1;
+    int64_t pool = -1;
+    vector<string>states;
+    set<pg_t> pgs;
+    cmd_getval(cmdmap, "pool", pool);
+    cmd_getval(cmdmap, "osd", osd);
+    cmd_getval(cmdmap, "states", states);
+    if (pool >= 0 && !osdmap.have_pg_pool(pool)) {
+      *ss << "pool " << pool << " does not exist";
+      return -ENOENT;
+    }
+    if (osd >= 0 && !osdmap.is_up(osd)) {
+      *ss << "osd " << osd << " is not up";
+      return -EAGAIN;
+    }
+    if (states.empty())
+      states.push_back("all");
+
+    uint64_t state = 0;
+
+    while (!states.empty()) {
+      string state_str = states.back();
+
+      if (state_str == "all") {
+        state = -1;
+        break;
+      } else {
+        auto filter = pg_string_state(state_str);
+        if (!filter) {
+          *ss << "'" << state_str << "' is not a valid pg state,"
+              << " available choices: " << pg_state_string(0xFFFFFFFF);
+          return -EINVAL;
+        }
+        state |= *filter;
+      }
+
+      states.pop_back();
+    }
+
+    pg_map.get_filtered_pg_stats(state, pool, osd, primary, pgs);
+
+    if (f && !pgs.empty()) {
+      pg_map.dump_filtered_pg_stats(f, pgs);
+      f->flush(*odata);
+    } else if (!pgs.empty()) {
+      pg_map.dump_filtered_pg_stats(ds, pgs);
+      odata->append(ds);
+      odata->append(omap_stats_note);
+    }
+    return 0;
+  }
+
+  if (prefix == "pg dump_stuck") {
+    vector<string> stuckop_vec;
+    cmd_getval(cmdmap, "stuckops", stuckop_vec);
+    if (stuckop_vec.empty())
+      stuckop_vec.push_back("unclean");
+    int64_t threshold;
+    cmd_getval(cmdmap, "threshold", threshold,
+               g_conf().get_val<int64_t>("mon_pg_stuck_threshold"));
+
+    if (pg_map.dump_stuck_pg_stats(ds, f, (int)threshold, stuckop_vec) < 0) {
+      *ss << "failed";
+    } else {
+      *ss << "ok";
+    }
+    odata->append(ds);
+    return 0;
+  }
+
+  if (prefix == "pg debug") {
+    string debugop;
+    cmd_getval(cmdmap, "debugop", debugop,
+	       string("unfound_objects_exist"));
+    if (debugop == "unfound_objects_exist") {
+      bool unfound_objects_exist = false;
+      for (const auto& p : pg_map.pg_stat) {
+	if (p.second.stats.sum.num_objects_unfound > 0) {
+	  unfound_objects_exist = true;
+	  break;
+	}
+      }
+      if (unfound_objects_exist)
+	ds << "TRUE";
+      else
+	ds << "FALSE";
+      odata->append(ds);
+      return 0;
+    }
+    if (debugop == "degraded_pgs_exist") {
+      bool degraded_pgs_exist = false;
+      for (const auto& p : pg_map.pg_stat) {
+	if (p.second.stats.sum.num_objects_degraded > 0) {
+	  degraded_pgs_exist = true;
+	  break;
+	}
+      }
+      if (degraded_pgs_exist)
+	ds << "TRUE";
+      else
+	ds << "FALSE";
+      odata->append(ds);
+      return 0;
+    }
+  }
+
+  if (prefix == "osd perf") {
+    if (f) {
+      f->open_object_section("osdstats");
+      pg_map.dump_osd_perf_stats(f);
+      f->close_section();
+      f->flush(ds);
+    } else {
+      pg_map.print_osd_perf_stats(&ds);
+    }
+    odata->append(ds);
+    return 0;
+  }
+
+  if (prefix == "osd blocked-by") {
+    if (f) {
+      f->open_object_section("osd_blocked_by");
+      pg_map.dump_osd_blocked_by_stats(f);
+      f->close_section();
+      f->flush(ds);
+    } else {
+      pg_map.print_osd_blocked_by_stats(&ds);
+    }
+    odata->append(ds);
+    return 0;
+  }
+
+  return -EOPNOTSUPP;
+}
+
+void PGMapUpdater::check_osd_map(
+  CephContext *cct,
+  const OSDMap& osdmap,
+  const PGMap& pgmap,
+  PGMap::Incremental *pending_inc)
+{
+  for (auto& p : pgmap.osd_stat) {
+    if (!osdmap.exists(p.first)) {
+      // remove osd_stat
+      pending_inc->rm_stat(p.first);
+    } else if (osdmap.is_out(p.first)) {
+      // zero osd_stat
+      if (p.second.statfs.total != 0) {
+	pending_inc->stat_osd_out(p.first);
+      }
+    } else if (!osdmap.is_up(p.first)) {
+      // zero the op_queue_age_hist
+      if (!p.second.op_queue_age_hist.empty()) {
+	pending_inc->stat_osd_down_up(p.first, pgmap);
+      }
+    }
+  }
+
+  // deleted pgs (pools)?
+  for (auto& p : pgmap.pg_pool_sum) {
+    if (!osdmap.have_pg_pool(p.first)) {
+      ldout(cct, 10) << __func__ << " pool " << p.first << " gone, removing pgs"
+		     << dendl;
+      for (auto& q : pgmap.pg_stat) {
+	if (q.first.pool() == p.first) {
+	  pending_inc->pg_remove.insert(q.first);
+	}
+      }
+      auto q = pending_inc->pg_stat_updates.begin();
+      while (q != pending_inc->pg_stat_updates.end()) {
+	if (q->first.pool() == p.first) {
+	  q = pending_inc->pg_stat_updates.erase(q);
+	} else {
+	  ++q;
+	}
+      }
+    }
+  }
+
+  // new (split or new pool) or merged pgs?
+  map<int64_t,unsigned> new_pg_num;
+  for (auto& p : osdmap.get_pools()) {
+    int64_t poolid = p.first;
+    const pg_pool_t& pi = p.second;
+    auto q = pgmap.num_pg_by_pool.find(poolid);
+    unsigned my_pg_num = 0;
+    if (q != pgmap.num_pg_by_pool.end())
+      my_pg_num = q->second;
+    unsigned pg_num = pi.get_pg_num();
+    new_pg_num[poolid] = pg_num;
+    if (my_pg_num < pg_num) {
+      ldout(cct,10) << __func__ << " pool " << poolid << " pg_num " << pg_num
+		    << " > my pg_num " << my_pg_num << dendl;
+      for (unsigned ps = my_pg_num; ps < pg_num; ++ps) {
+	pg_t pgid(ps, poolid);
+	if (pending_inc->pg_stat_updates.count(pgid) == 0) {
+	  ldout(cct,20) << __func__ << " adding " << pgid << dendl;
+	  pg_stat_t &stats = pending_inc->pg_stat_updates[pgid];
+	  stats.last_fresh = osdmap.get_modified();
+	  stats.last_active = osdmap.get_modified();
+	  stats.last_change = osdmap.get_modified();
+	  stats.last_peered = osdmap.get_modified();
+	  stats.last_clean = osdmap.get_modified();
+	  stats.last_unstale = osdmap.get_modified();
+	  stats.last_undegraded = osdmap.get_modified();
+	  stats.last_fullsized = osdmap.get_modified();
+	  stats.last_scrub_stamp = osdmap.get_modified();
+	  stats.last_deep_scrub_stamp = osdmap.get_modified();
+	  stats.last_clean_scrub_stamp = osdmap.get_modified();
+	}
+      }
+    } else if (my_pg_num > pg_num) {
+      ldout(cct,10) << __func__ << " pool " << poolid << " pg_num " << pg_num
+		    << " < my pg_num " << my_pg_num << dendl;
+      for (unsigned i = pg_num; i < my_pg_num; ++i) {
+	pg_t pgid(i, poolid);
+	ldout(cct,20) << __func__ << " removing merged " << pgid << dendl;
+	if (pgmap.pg_stat.count(pgid)) {
+	  pending_inc->pg_remove.insert(pgid);
+	}
+	pending_inc->pg_stat_updates.erase(pgid);
+      }
+    }
+  }
+  auto i = pending_inc->pg_stat_updates.begin();
+  while (i != pending_inc->pg_stat_updates.end()) {
+    auto j = new_pg_num.find(i->first.pool());
+    if (j == new_pg_num.end() ||
+	i->first.ps() >= j->second) {
+      ldout(cct,20) << __func__ << " removing pending update to old "
+		    << i->first << dendl;
+      i = pending_inc->pg_stat_updates.erase(i);
+    } else {
+      ++i;
+    }
+  }
+}
+
+static void _try_mark_pg_stale(
+  const OSDMap& osdmap,
+  pg_t pgid,
+  const pg_stat_t& cur,
+  PGMap::Incremental *pending_inc)
+{
+  if ((cur.state & PG_STATE_STALE) == 0 &&
+      cur.acting_primary != -1 &&
+      osdmap.is_down(cur.acting_primary)) {
+    pg_stat_t *newstat;
+    auto q = pending_inc->pg_stat_updates.find(pgid);
+    if (q != pending_inc->pg_stat_updates.end()) {
+      if ((q->second.acting_primary == cur.acting_primary) ||
+	  ((q->second.state & PG_STATE_STALE) == 0 &&
+	   q->second.acting_primary != -1 &&
+	   osdmap.is_down(q->second.acting_primary))) {
+	newstat = &q->second;
+      } else {
+	// pending update is no longer down or already stale
+	return;
+      }
+    } else {
+      newstat = &pending_inc->pg_stat_updates[pgid];
+      *newstat = cur;
+    }
+    dout(10) << __func__ << " marking pg " << pgid
+	     << " stale (acting_primary " << newstat->acting_primary
+	     << ")" << dendl;
+    newstat->state |= PG_STATE_STALE;
+    newstat->last_unstale = ceph_clock_now();
+  }
+}
+
+void PGMapUpdater::check_down_pgs(
+    const OSDMap &osdmap,
+    const PGMap &pg_map,
+    bool check_all,
+    const set<int>& need_check_down_pg_osds,
+    PGMap::Incremental *pending_inc)
+{
+  // if a large number of osds changed state, just iterate over the whole
+  // pg map.
+  if (need_check_down_pg_osds.size() > (unsigned)osdmap.get_num_osds() *
+      g_conf().get_val<double>("mon_pg_check_down_all_threshold")) {
+    check_all = true;
+  }
+
+  if (check_all) {
+    for (const auto& p : pg_map.pg_stat) {
+      _try_mark_pg_stale(osdmap, p.first, p.second, pending_inc);
+    }
+  } else {
+    for (auto osd : need_check_down_pg_osds) {
+      if (osdmap.is_down(osd)) {
+	auto p = pg_map.pg_by_osd.find(osd);
+	if (p == pg_map.pg_by_osd.end()) {
+	  continue;
+	}
+	for (auto pgid : p->second) {
+	  const pg_stat_t &stat = pg_map.pg_stat.at(pgid);
+	  ceph_assert(stat.acting_primary == osd);
+	  _try_mark_pg_stale(osdmap, pgid, stat, pending_inc);
+	}
+      }
+    }
+  }
+}
+
+int reweight::by_utilization(
+    const OSDMap &osdmap,
+    const PGMap &pgm,
+    int oload,
+    double max_changef,
+    int max_osds,
+    bool by_pg, const set<int64_t> *pools,
+    bool no_increasing,
+    mempool::osdmap::map<int32_t, uint32_t>* new_weights,
+    std::stringstream *ss,
+    std::string *out_str,
+    ceph::Formatter *f)
+{
+  if (oload <= 100) {
+    *ss << "You must give a percentage higher than 100. "
+      "The reweighting threshold will be calculated as <average-utilization> "
+      "times <input-percentage>. For example, an argument of 200 would "
+      "reweight OSDs which are twice as utilized as the average OSD.\n";
+    return -EINVAL;
+  }
+
+  vector<int> pgs_by_osd(osdmap.get_max_osd());
+
+  // Avoid putting a small number (or 0) in the denominator when calculating
+  // average_util
+  double average_util;
+  if (by_pg) {
+    // by pg mapping
+    double weight_sum = 0.0;      // sum up the crush weights
+    unsigned num_pg_copies = 0;
+    int num_osds = 0;
+    for (const auto& pg : pgm.pg_stat) {
+      if (pools && pools->count(pg.first.pool()) == 0)
+	continue;
+      for (const auto acting : pg.second.acting) {
+        if (!osdmap.exists(acting)) {
+          continue;
+        }
+	if (acting >= (int)pgs_by_osd.size())
+	  pgs_by_osd.resize(acting);
+	if (pgs_by_osd[acting] == 0) {
+          if (osdmap.crush->get_item_weightf(acting) <= 0) {
+            //skip if we currently can not identify item
+            continue;
+          }
+	  weight_sum += osdmap.crush->get_item_weightf(acting);
+	  ++num_osds;
+	}
+	++pgs_by_osd[acting];
+	++num_pg_copies;
+      }
+    }
+
+    if (!num_osds || (num_pg_copies / num_osds < g_conf()->mon_reweight_min_pgs_per_osd)) {
+      *ss << "Refusing to reweight: we only have " << num_pg_copies
+	  << " PGs across " << num_osds << " osds!\n";
+      return -EDOM;
+    }
+
+    average_util = (double)num_pg_copies / weight_sum;
+  } else {
+    // by osd utilization
+    int num_osd = std::max<size_t>(1, pgm.osd_stat.size());
+    if ((uint64_t)pgm.osd_sum.statfs.total / num_osd
+	< g_conf()->mon_reweight_min_bytes_per_osd) {
+      *ss << "Refusing to reweight: we only have " << pgm.osd_sum.statfs.kb()
+	  << " kb across all osds!\n";
+      return -EDOM;
+    }
+    if ((uint64_t)pgm.osd_sum.statfs.get_used_raw() / num_osd
+	< g_conf()->mon_reweight_min_bytes_per_osd) {
+      *ss << "Refusing to reweight: we only have "
+	  << pgm.osd_sum.statfs.kb_used_raw()
+	  << " kb used across all osds!\n";
+      return -EDOM;
+    }
+
+    average_util = (double)pgm.osd_sum.statfs.get_used_raw() /
+      (double)pgm.osd_sum.statfs.total;
+  }
+
+  // adjust down only if we are above the threshold
+  const double overload_util = average_util * (double)oload / 100.0;
+
+  // but aggressively adjust weights up whenever possible.
+  const double underload_util = average_util;
+
+  const unsigned max_change = (unsigned)(max_changef * (double)0x10000);
+
+  ostringstream oss;
+  if (f) {
+    f->open_object_section("reweight_by_utilization");
+    f->dump_int("overload_min", oload);
+    f->dump_float("max_change", max_changef);
+    f->dump_int("max_change_osds", max_osds);
+    f->dump_float("average_utilization", average_util);
+    f->dump_float("overload_utilization", overload_util);
+  } else {
+    oss << "oload " << oload << "\n";
+    oss << "max_change " << max_changef << "\n";
+    oss << "max_change_osds " << max_osds << "\n";
+    oss.precision(4);
+    oss << "average_utilization " << std::fixed << average_util << "\n";
+    oss << "overload_utilization " << overload_util << "\n";
+  }
+  int num_changed = 0;
+
+  // precompute util for each OSD
+  std::vector<std::pair<int, float> > util_by_osd;
+  for (const auto& p : pgm.osd_stat) {
+    std::pair<int, float> osd_util;
+    osd_util.first = p.first;
+    if (by_pg) {
+      if (p.first >= (int)pgs_by_osd.size() ||
+        pgs_by_osd[p.first] == 0) {
+        // skip if this OSD does not contain any pg
+        // belonging to the specified pool(s).
+        continue;
+      }
+
+      if (osdmap.crush->get_item_weightf(p.first) <= 0) {
+        // skip if we are unable to locate item.
+        continue;
+      }
+
+      osd_util.second =
+	pgs_by_osd[p.first] / osdmap.crush->get_item_weightf(p.first);
+    } else {
+      osd_util.second =
+	(double)p.second.statfs.get_used_raw() / (double)p.second.statfs.total;
+    }
+    util_by_osd.push_back(osd_util);
+  }
+
+  // sort by absolute deviation from the mean utilization,
+  // in descending order.
+  std::sort(util_by_osd.begin(), util_by_osd.end(),
+    [average_util](std::pair<int, float> l, std::pair<int, float> r) {
+      return abs(l.second - average_util) > abs(r.second - average_util);
+    }
+  );
+
+  if (f)
+    f->open_array_section("reweights");
+
+  for (const auto& p : util_by_osd) {
+    unsigned weight = osdmap.get_weight(p.first);
+    if (weight == 0) {
+      // skip if OSD is currently out
+      continue;
+    }
+    float util = p.second;
+
+    if (util >= overload_util) {
+      // Assign a lower weight to overloaded OSDs. The current weight
+      // is a factor to take into account the original weights,
+      // to represent e.g. differing storage capacities
+      unsigned new_weight = (unsigned)((average_util / util) * (float)weight);
+      if (weight > max_change)
+	new_weight = std::max(new_weight, weight - max_change);
+      new_weights->insert({p.first, new_weight});
+      if (f) {
+	f->open_object_section("osd");
+	f->dump_int("osd", p.first);
+	f->dump_float("weight", (float)weight / (float)0x10000);
+	f->dump_float("new_weight", (float)new_weight / (float)0x10000);
+	f->close_section();
+      } else {
+        oss << "osd." << p.first << " weight "
+            << (float)weight / (float)0x10000 << " -> "
+            << (float)new_weight / (float)0x10000 << "\n";
+      }
+      if (++num_changed >= max_osds)
+	break;
+    }
+    if (!no_increasing && util <= underload_util) {
+      // assign a higher weight.. if we can.
+      unsigned new_weight = (unsigned)((average_util / util) * (float)weight);
+      new_weight = std::min(new_weight, weight + max_change);
+      if (new_weight > 0x10000)
+	new_weight = 0x10000;
+      if (new_weight > weight) {
+	new_weights->insert({p.first, new_weight});
+        oss << "osd." << p.first << " weight "
+            << (float)weight / (float)0x10000 << " -> "
+            << (float)new_weight / (float)0x10000 << "\n";
+	if (++num_changed >= max_osds)
+	  break;
+      }
+    }
+  }
+  if (f) {
+    f->close_section();
+  }
+
+  OSDMap newmap;
+  newmap.deepish_copy_from(osdmap);
+  OSDMap::Incremental newinc;
+  newinc.fsid = newmap.get_fsid();
+  newinc.epoch = newmap.get_epoch() + 1;
+  newinc.new_weight = *new_weights;
+  newmap.apply_incremental(newinc);
+
+  osdmap.summarize_mapping_stats(&newmap, pools, out_str, f);
+
+  if (f) {
+    f->close_section();
+  } else {
+    *out_str += "\n";
+    *out_str += oss.str();
+  }
+  return num_changed;
+}
diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h
new file mode 100644
index 000000000..9bdabb046
--- /dev/null
+++ b/src/mon/PGMap.h
@@ -0,0 +1,558 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+ 
+/*
+ * Placement Group Map. Placement Groups are logical sets of objects
+ * that are replicated by the same set of devices. pgid=(r,hash(o)&m)
+ * where & is a bit-wise AND and m=2^k-1
+ */
+
+#ifndef CEPH_PGMAP_H
+#define CEPH_PGMAP_H
+
+#include "include/health.h"
+#include "common/debug.h"
+#include "common/TextTable.h"
+#include "osd/osd_types.h"
+#include "include/mempool.h"
+#include "mon/health_check.h"
+#include <sstream>
+
+namespace ceph { class Formatter; }
+
+class PGMapDigest {
+public:
+  MEMPOOL_CLASS_HELPERS();
+  virtual ~PGMapDigest() {}
+
+  mempool::pgmap::vector<uint64_t> osd_last_seq;
+
+  mutable std::map<int, int64_t> avail_space_by_rule;
+
+  // aggregate state, populated by PGMap child
+  int64_t num_pg = 0, num_osd = 0;
+  int64_t num_pg_active = 0;
+  int64_t num_pg_unknown = 0;
+  mempool::pgmap::unordered_map<int32_t,pool_stat_t> pg_pool_sum;
+  mempool::pgmap::map<int64_t,int64_t> num_pg_by_pool;
+  pool_stat_t pg_sum;
+  osd_stat_t osd_sum;
+  mempool::pgmap::map<std::string,osd_stat_t> osd_sum_by_class;
+  mempool::pgmap::unordered_map<uint64_t,int32_t> num_pg_by_state;
+  struct pg_count {
+    int32_t acting = 0;
+    int32_t up_not_acting = 0;
+    int32_t primary = 0;
+    void encode(ceph::buffer::list& bl) const {
+      using ceph::encode;
+      encode(acting, bl);
+      encode(up_not_acting, bl);
+      encode(primary, bl);
+    }
+    void decode(ceph::buffer::list::const_iterator& p) {
+      using ceph::decode;
+      decode(acting, p);
+      decode(up_not_acting, p);
+      decode(primary, p);
+    }
+  };
+  mempool::pgmap::unordered_map<int32_t,pg_count> num_pg_by_osd;
+
+  mempool::pgmap::map<int64_t,interval_set<snapid_t>> purged_snaps;
+
+  bool use_per_pool_stats() const {
+    return osd_sum.num_osds == osd_sum.num_per_pool_osds;
+  }
+  bool use_per_pool_omap_stats() const {
+    return osd_sum.num_osds == osd_sum.num_per_pool_omap_osds;
+  }
+
+  // recent deltas, and summation
+  /**
+   * keep track of last deltas for each pool, calculated using
+   * @p pg_pool_sum as baseline.
+   */
+  mempool::pgmap::unordered_map<int64_t, mempool::pgmap::list<std::pair<pool_stat_t, utime_t> > > per_pool_sum_deltas;
+  /**
+   * keep track of per-pool timestamp deltas, according to last update on
+   * each pool.
+   */
+  mempool::pgmap::unordered_map<int64_t, utime_t> per_pool_sum_deltas_stamps;
+  /**
+   * keep track of sum deltas, per-pool, taking into account any previous
+   * deltas existing in @p per_pool_sum_deltas.  The utime_t as second member
+   * of the pair is the timestamp referring to the last update (i.e., the first
+   * member of the pair) for a given pool.
+   */
+  mempool::pgmap::unordered_map<int64_t, std::pair<pool_stat_t,utime_t> > per_pool_sum_delta;
+
+  pool_stat_t pg_sum_delta;
+  utime_t stamp_delta;
+
+  void get_recovery_stats(
+    double *misplaced_ratio,
+    double *degraded_ratio,
+    double *inactive_ratio,
+    double *unknown_pgs_ratio) const;
+
+  void print_summary(ceph::Formatter *f, std::ostream *out) const;
+  void print_oneline_summary(ceph::Formatter *f, std::ostream *out) const;
+
+  void recovery_summary(ceph::Formatter *f, std::list<std::string> *psl,
+                        const pool_stat_t& pool_sum) const;
+  void overall_recovery_summary(ceph::Formatter *f, std::list<std::string> *psl) const;
+  void pool_recovery_summary(ceph::Formatter *f, std::list<std::string> *psl,
+                             uint64_t poolid) const;
+  void recovery_rate_summary(ceph::Formatter *f, std::ostream *out,
+                             const pool_stat_t& delta_sum,
+                             utime_t delta_stamp) const;
+  void overall_recovery_rate_summary(ceph::Formatter *f, std::ostream *out) const;
+  void pool_recovery_rate_summary(ceph::Formatter *f, std::ostream *out,
+                                  uint64_t poolid) const;
+  /**
+   * Obtain a formatted/plain output for client I/O, source from stats for a
+   * given @p delta_sum pool over a given @p delta_stamp period of time.
+   */
+  void client_io_rate_summary(ceph::Formatter *f, std::ostream *out,
+                              const pool_stat_t& delta_sum,
+                              utime_t delta_stamp) const;
+  /**
+   * Obtain a formatted/plain output for the overall client I/O, which is
+   * calculated resorting to @p pg_sum_delta and @p stamp_delta.
+   */
+  void overall_client_io_rate_summary(ceph::Formatter *f, std::ostream *out) const;
+  /**
+   * Obtain a formatted/plain output for client I/O over a given pool
+   * with id @p pool_id.  We will then obtain pool-specific data
+   * from @p per_pool_sum_delta.
+   */
+  void pool_client_io_rate_summary(ceph::Formatter *f, std::ostream *out,
+                                   uint64_t poolid) const;
+  /**
+   * Obtain a formatted/plain output for cache tier IO, source from stats for a
+   * given @p delta_sum pool over a given @p delta_stamp period of time.
+   */
+  void cache_io_rate_summary(ceph::Formatter *f, std::ostream *out,
+                             const pool_stat_t& delta_sum,
+                             utime_t delta_stamp) const;
+  /**
+   * Obtain a formatted/plain output for the overall cache tier IO, which is
+   * calculated resorting to @p pg_sum_delta and @p stamp_delta.
+   */
+  void overall_cache_io_rate_summary(ceph::Formatter *f, std::ostream *out) const;
+  /**
+   * Obtain a formatted/plain output for cache tier IO over a given pool
+   * with id @p pool_id.  We will then obtain pool-specific data
+   * from @p per_pool_sum_delta.
+   */
+  void pool_cache_io_rate_summary(ceph::Formatter *f, std::ostream *out,
+                                  uint64_t poolid) const;
+
+  /**
+   * Return the number of additional bytes that can be stored in this
+   * pool before the first OSD fills up, accounting for PG overhead.
+   */
+  int64_t get_pool_free_space(const OSDMap &osd_map, int64_t poolid) const;
+
+
+  /**
+   * Dump pool usage and io ops/bytes, used by "ceph df" command
+   */
+  virtual void dump_pool_stats_full(const OSDMap &osd_map, std::stringstream *ss,
+				    ceph::Formatter *f, bool verbose) const;
+  void dump_cluster_stats(std::stringstream *ss, ceph::Formatter *f, bool verbose) const;
+  static void dump_object_stat_sum(TextTable &tbl, ceph::Formatter *f,
+				   const pool_stat_t &pool_stat,
+				   uint64_t avail,
+				   float raw_used_rate,
+				   bool verbose,
+				   bool per_pool,
+				   bool per_pool_omap,
+				   const pg_pool_t *pool);
+
+  size_t get_num_pg_by_osd(int osd) const {
+    auto p = num_pg_by_osd.find(osd);
+    if (p == num_pg_by_osd.end())
+      return 0;
+    else
+      return p->second.acting;
+  }
+  int get_num_primary_pg_by_osd(int osd) const {
+    auto p = num_pg_by_osd.find(osd);
+    if (p == num_pg_by_osd.end())
+      return 0;
+    else
+      return p->second.primary;
+  }
+
+  ceph_statfs get_statfs(OSDMap &osdmap,
+                         boost::optional<int64_t> data_pool) const;
+
+  int64_t get_rule_avail(int ruleno) const {
+    auto i = avail_space_by_rule.find(ruleno);
+    if (i != avail_space_by_rule.end())
+      return avail_space_by_rule[ruleno];
+    else
+      return 0;
+  }
+
+  // kill me post-mimic or -nautilus
+  bool definitely_converted_snapsets() const {
+    // false negative is okay; false positive is not!
+    return
+      num_pg &&
+      num_pg_unknown == 0 &&
+      pg_sum.stats.sum.num_legacy_snapsets == 0;
+  }
+
+  uint64_t get_last_osd_stat_seq(int osd) {
+    if (osd < (int)osd_last_seq.size())
+      return osd_last_seq[osd];
+    return 0;
+  }
+
+  void encode(ceph::buffer::list& bl, uint64_t features) const;
+  void decode(ceph::buffer::list::const_iterator& p);
+  void dump(ceph::Formatter *f) const;
+  static void generate_test_instances(std::list<PGMapDigest*>& ls);
+};
+WRITE_CLASS_ENCODER(PGMapDigest::pg_count);
+WRITE_CLASS_ENCODER_FEATURES(PGMapDigest);
+
+class PGMap : public PGMapDigest {
+public:
+  MEMPOOL_CLASS_HELPERS();
+
+  // the map
+  version_t version;
+  epoch_t last_osdmap_epoch;   // last osdmap epoch i applied to the pgmap
+  epoch_t last_pg_scan;  // osdmap epoch
+  mempool::pgmap::unordered_map<int32_t,osd_stat_t> osd_stat;
+  mempool::pgmap::unordered_map<pg_t,pg_stat_t> pg_stat;
+
+  typedef mempool::pgmap::map<
+    std::pair<int64_t, int>,  // <pool, osd>
+    store_statfs_t>
+      per_osd_pool_statfs_t;
+
+  per_osd_pool_statfs_t pool_statfs;
+
+  class Incremental {
+  public:
+    MEMPOOL_CLASS_HELPERS();
+    version_t version;
+    mempool::pgmap::map<pg_t,pg_stat_t> pg_stat_updates;
+    epoch_t osdmap_epoch;
+    epoch_t pg_scan;  // osdmap epoch
+    mempool::pgmap::set<pg_t> pg_remove;
+    utime_t stamp;
+    per_osd_pool_statfs_t pool_statfs_updates;
+
+  private:
+    mempool::pgmap::map<int32_t,osd_stat_t> osd_stat_updates;
+    mempool::pgmap::set<int32_t> osd_stat_rm;
+  public:
+
+    const mempool::pgmap::map<int32_t, osd_stat_t> &get_osd_stat_updates() const {
+      return osd_stat_updates;
+    }
+    const mempool::pgmap::set<int32_t> &get_osd_stat_rm() const {
+      return osd_stat_rm;
+    }
+    template<typename OsdStat>
+    void update_stat(int32_t osd, OsdStat&& stat) {
+      osd_stat_updates[osd] = std::forward<OsdStat>(stat);
+    }
+    void stat_osd_out(int32_t osd) {
+      osd_stat_updates[osd] = osd_stat_t();
+    }
+    void stat_osd_down_up(int32_t osd, const PGMap& pg_map) {
+      // 0 the op_queue_age_hist for this osd
+      auto p = osd_stat_updates.find(osd);
+      if (p != osd_stat_updates.end()) {
+	p->second.op_queue_age_hist.clear();
+	return;
+      }
+      auto q = pg_map.osd_stat.find(osd);
+      if (q != pg_map.osd_stat.end()) {
+	osd_stat_t& t = osd_stat_updates[osd] = q->second;
+	t.op_queue_age_hist.clear();
+      }
+    }
+    void rm_stat(int32_t osd) {
+      osd_stat_rm.insert(osd);
+      osd_stat_updates.erase(osd);
+    }
+    void dump(ceph::Formatter *f) const;
+    static void generate_test_instances(std::list<Incremental*>& o);
+
+    Incremental() : version(0), osdmap_epoch(0), pg_scan(0) {}
+  };
+
+
+  // aggregate stats (soft state), generated by calc_stats()
+  mempool::pgmap::unordered_map<int,std::set<pg_t> > pg_by_osd;
+  mempool::pgmap::unordered_map<int,int> blocked_by_sum;
+  mempool::pgmap::list<std::pair<pool_stat_t, utime_t> > pg_sum_deltas;
+  mempool::pgmap::unordered_map<int64_t,mempool::pgmap::unordered_map<uint64_t,int32_t>> num_pg_by_pool_state;
+
+  utime_t stamp;
+
+  void update_pool_deltas(
+    CephContext *cct,
+    const utime_t ts,
+    const mempool::pgmap::unordered_map<int32_t, pool_stat_t>& pg_pool_sum_old);
+  void clear_delta();
+
+  void deleted_pool(int64_t pool) {
+    for (auto i = pool_statfs.begin();  i != pool_statfs.end();) {
+      if (i->first.first == pool) {
+	i = pool_statfs.erase(i);
+      } else {
+        ++i;
+      }
+    }
+
+    pg_pool_sum.erase(pool);
+    num_pg_by_pool_state.erase(pool);
+    num_pg_by_pool.erase(pool);
+    per_pool_sum_deltas.erase(pool);
+    per_pool_sum_deltas_stamps.erase(pool);
+    per_pool_sum_delta.erase(pool);
+  }
+
+ private:
+  void update_delta(
+    CephContext *cct,
+    const utime_t ts,
+    const pool_stat_t& old_pool_sum,
+    utime_t *last_ts,
+    const pool_stat_t& current_pool_sum,
+    pool_stat_t *result_pool_delta,
+    utime_t *result_ts_delta,
+    mempool::pgmap::list<std::pair<pool_stat_t,utime_t> > *delta_avg_list);
+
+  void update_one_pool_delta(CephContext *cct,
+                             const utime_t ts,
+                             const int64_t pool,
+                             const pool_stat_t& old_pool_sum);
+
+ public:
+
+  mempool::pgmap::set<pg_t> creating_pgs;
+  mempool::pgmap::map<int,std::map<epoch_t,std::set<pg_t> > > creating_pgs_by_osd_epoch;
+
+  // Bits that use to be enum StuckPG
+  static const int STUCK_INACTIVE = (1<<0);
+  static const int STUCK_UNCLEAN = (1<<1);
+  static const int STUCK_UNDERSIZED = (1<<2);
+  static const int STUCK_DEGRADED = (1<<3);
+  static const int STUCK_STALE = (1<<4);
+  
+  PGMap()
+    : version(0),
+      last_osdmap_epoch(0), last_pg_scan(0)
+  {}
+
+  version_t get_version() const {
+    return version;
+  }
+  void set_version(version_t v) {
+    version = v;
+  }
+  epoch_t get_last_osdmap_epoch() const {
+    return last_osdmap_epoch;
+  }
+  void set_last_osdmap_epoch(epoch_t e) {
+    last_osdmap_epoch = e;
+  }
+  epoch_t get_last_pg_scan() const {
+    return last_pg_scan;
+  }
+  void set_last_pg_scan(epoch_t e) {
+    last_pg_scan = e;
+  }
+  utime_t get_stamp() const {
+    return stamp;
+  }
+  void set_stamp(utime_t s) {
+    stamp = s;
+  }
+
+  pool_stat_t get_pg_pool_sum_stat(int64_t pool) const {
+    auto p = pg_pool_sum.find(pool);
+    if (p != pg_pool_sum.end())
+      return p->second;
+    return pool_stat_t();
+  }
+
+  osd_stat_t get_osd_sum(const std::set<int>& osds) const {
+    if (osds.empty()) // all
+      return osd_sum;
+    osd_stat_t sum;
+    for (auto i : osds) {
+      auto os = get_osd_stat(i);
+      if (os)
+        sum.add(*os);
+    }
+    return sum;
+  }
+
+  const osd_stat_t *get_osd_stat(int osd) const {
+    auto i = osd_stat.find(osd);
+    if (i == osd_stat.end()) {
+      return nullptr;
+    }
+    return &i->second;
+  }
+
+
+  void apply_incremental(CephContext *cct, const Incremental& inc);
+  void calc_stats();
+  void stat_pg_add(const pg_t &pgid, const pg_stat_t &s,
+		   bool sameosds=false);
+  bool stat_pg_sub(const pg_t &pgid, const pg_stat_t &s,
+		   bool sameosds=false);
+  void calc_purged_snaps();
+  void calc_osd_sum_by_class(const OSDMap& osdmap);
+  void stat_osd_add(int osd, const osd_stat_t &s);
+  void stat_osd_sub(int osd, const osd_stat_t &s);
+  
+  void encode(ceph::buffer::list &bl, uint64_t features=-1) const;
+  void decode(ceph::buffer::list::const_iterator &bl);
+
+  /// encode subset of our data to a PGMapDigest
+  void encode_digest(const OSDMap& osdmap,
+		     ceph::buffer::list& bl, uint64_t features);
+
+  int64_t get_rule_avail(const OSDMap& osdmap, int ruleno) const;
+  void get_rules_avail(const OSDMap& osdmap,
+		       std::map<int,int64_t> *avail_map) const;
+  void dump(ceph::Formatter *f, bool with_net = true) const;
+  void dump_basic(ceph::Formatter *f) const;
+  void dump_pg_stats(ceph::Formatter *f, bool brief) const;
+  void dump_pg_progress(ceph::Formatter *f) const;
+  void dump_pool_stats(ceph::Formatter *f) const;
+  void dump_osd_stats(ceph::Formatter *f, bool with_net = true) const;
+  void dump_osd_ping_times(ceph::Formatter *f) const;
+  void dump_delta(ceph::Formatter *f) const;
+  void dump_filtered_pg_stats(ceph::Formatter *f, std::set<pg_t>& pgs) const;
+  void dump_pool_stats_full(const OSDMap &osd_map, std::stringstream *ss,
+			    ceph::Formatter *f, bool verbose) const override {
+    get_rules_avail(osd_map, &avail_space_by_rule);
+    PGMapDigest::dump_pool_stats_full(osd_map, ss, f, verbose);
+  }
+
+  /*
+  * Dump client io rate, recovery io rate, cache io rate and recovery information.
+  * this function is used by "ceph osd pool stats" command
+  */
+  void dump_pool_stats_and_io_rate(int64_t poolid, const OSDMap &osd_map, ceph::Formatter *f,
+				   std::stringstream *ss) const;
+
+  void dump_pg_stats_plain(
+    std::ostream& ss,
+    const mempool::pgmap::unordered_map<pg_t, pg_stat_t>& pg_stats,
+    bool brief) const;
+  void get_stuck_stats(
+    int types, const utime_t cutoff,
+    mempool::pgmap::unordered_map<pg_t, pg_stat_t>& stuck_pgs) const;
+  bool get_stuck_counts(const utime_t cutoff, std::map<std::string, int>& note) const;
+  void dump_stuck(ceph::Formatter *f, int types, utime_t cutoff) const;
+  void dump_stuck_plain(std::ostream& ss, int types, utime_t cutoff) const;
+  int dump_stuck_pg_stats(std::stringstream &ds,
+			  ceph::Formatter *f,
+			  int threshold,
+			  std::vector<std::string>& args) const;
+  void dump(std::ostream& ss) const;
+  void dump_basic(std::ostream& ss) const;
+  void dump_pg_stats(std::ostream& ss, bool brief) const;
+  void dump_pg_sum_stats(std::ostream& ss, bool header) const;
+  void dump_pool_stats(std::ostream& ss, bool header) const;
+  void dump_osd_stats(std::ostream& ss) const;
+  void dump_osd_sum_stats(std::ostream& ss) const;
+  void dump_filtered_pg_stats(std::ostream& ss, std::set<pg_t>& pgs) const;
+
+  void dump_osd_perf_stats(ceph::Formatter *f) const;
+  void print_osd_perf_stats(std::ostream *ss) const;
+
+  void dump_osd_blocked_by_stats(ceph::Formatter *f) const;
+  void print_osd_blocked_by_stats(std::ostream *ss) const;
+
+  void get_filtered_pg_stats(uint64_t state, int64_t poolid, int64_t osdid,
+                             bool primary, std::set<pg_t>& pgs) const;
+
+  std::set<std::string> osd_parentage(const OSDMap& osdmap, int id) const;
+  void get_health_checks(
+    CephContext *cct,
+    const OSDMap& osdmap,
+    health_check_map_t *checks) const;
+  void print_summary(ceph::Formatter *f, std::ostream *out) const;
+
+  static void generate_test_instances(std::list<PGMap*>& o);
+};
+WRITE_CLASS_ENCODER_FEATURES(PGMap)
+
+inline std::ostream& operator<<(std::ostream& out, const PGMapDigest& m) {
+  m.print_oneline_summary(NULL, &out);
+  return out;
+}
+
+int process_pg_map_command(
+  const std::string& prefix,
+  const cmdmap_t& cmdmap,
+  const PGMap& pg_map,
+  const OSDMap& osdmap,
+  ceph::Formatter *f,
+  std::stringstream *ss,
+  ceph::buffer::list *odata);
+
+class PGMapUpdater
+{
+public:
+  static void check_osd_map(
+    CephContext *cct,
+    const OSDMap &osdmap,
+    const PGMap& pg_map,
+    PGMap::Incremental *pending_inc);
+
+  // mark pg's state stale if its acting primary osd is down
+  static void check_down_pgs(
+      const OSDMap &osd_map,
+      const PGMap &pg_map,
+      bool check_all,
+      const std::set<int>& need_check_down_pg_osds,
+      PGMap::Incremental *pending_inc);
+};
+
+namespace reweight {
+/* Assign a lower weight to overloaded OSDs.
+ *
+ * The osds that will get a lower weight are those with with a utilization
+ * percentage 'oload' percent greater than the average utilization.
+ */
+  int by_utilization(const OSDMap &osd_map,
+		     const PGMap &pg_map,
+		     int oload,
+		     double max_changef,
+		     int max_osds,
+		     bool by_pg, const std::set<int64_t> *pools,
+		     bool no_increasing,
+		     mempool::osdmap::map<int32_t, uint32_t>* new_weights,
+		     std::stringstream *ss,
+		     std::string *out_str,
+		     ceph::Formatter *f);
+}
+
+#endif
diff --git a/src/mon/Paxos.cc b/src/mon/Paxos.cc
new file mode 100644
index 000000000..21f244239
--- /dev/null
+++ b/src/mon/Paxos.cc
@@ -0,0 +1,1591 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include <sstream>
+#include "Paxos.h"
+#include "Monitor.h"
+#include "messages/MMonPaxos.h"
+
+#include "mon/mon_types.h"
+#include "common/config.h"
+#include "include/ceph_assert.h"
+#include "include/stringify.h"
+#include "common/Timer.h"
+#include "messages/PaxosServiceMessage.h"
+
+using std::string;
+using std::unique_lock;
+
+using ceph::bufferlist;
+using ceph::Formatter;
+using ceph::JSONFormatter;
+using ceph::to_timespan;
+
+#define dout_subsys ceph_subsys_paxos
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, mon.name, mon.rank, paxos_name, state, first_committed, last_committed)
+static std::ostream& _prefix(std::ostream *_dout, Monitor &mon, const string& name,
+			     int rank, const string& paxos_name, int state,
+			     version_t first_committed, version_t last_committed)
+{
+  return *_dout << "mon." << name << "@" << rank
+		<< "(" << mon.get_state_name() << ")"
+		<< ".paxos(" << paxos_name << " " << Paxos::get_statename(state)
+		<< " c " << first_committed << ".." << last_committed
+		<< ") ";
+}
+
+class Paxos::C_Trimmed : public Context {
+  Paxos *paxos;
+public:
+  explicit C_Trimmed(Paxos *p) : paxos(p) { }
+  void finish(int r) override {
+    paxos->trimming = false;
+  }
+};
+
+MonitorDBStore *Paxos::get_store()
+{
+  return mon.store;
+}
+
+void Paxos::read_and_prepare_transactions(MonitorDBStore::TransactionRef tx,
+					  version_t first, version_t last)
+{
+  dout(10) << __func__ << " first " << first << " last " << last << dendl;
+  for (version_t v = first; v <= last; ++v) {
+    dout(30) << __func__ << " apply version " << v << dendl;
+    bufferlist bl;
+    int err = get_store()->get(get_name(), v, bl);
+    ceph_assert(err == 0);
+    ceph_assert(bl.length());
+    decode_append_transaction(tx, bl);
+  }
+  dout(15) << __func__ << " total versions " << (last-first) << dendl;
+}
+
+void Paxos::init()
+{
+  // load paxos variables from stable storage
+  last_pn = get_store()->get(get_name(), "last_pn");
+  accepted_pn = get_store()->get(get_name(), "accepted_pn");
+  last_committed = get_store()->get(get_name(), "last_committed");
+  first_committed = get_store()->get(get_name(), "first_committed");
+
+  dout(10) << __func__ << " last_pn: " << last_pn << " accepted_pn: "
+	   << accepted_pn << " last_committed: " << last_committed
+	   << " first_committed: " << first_committed << dendl;
+
+  dout(10) << "init" << dendl;
+  ceph_assert(is_consistent());
+}
+
+void Paxos::init_logger()
+{
+  PerfCountersBuilder pcb(g_ceph_context, "paxos", l_paxos_first, l_paxos_last);
+
+  // Because monitors are so few in number, the resource cost of capturing
+  // almost all their perf counters at USEFUL is trivial.
+  pcb.set_prio_default(PerfCountersBuilder::PRIO_USEFUL);
+
+  pcb.add_u64_counter(l_paxos_start_leader, "start_leader", "Starts in leader role");
+  pcb.add_u64_counter(l_paxos_start_peon, "start_peon", "Starts in peon role");
+  pcb.add_u64_counter(l_paxos_restart, "restart", "Restarts");
+  pcb.add_u64_counter(l_paxos_refresh, "refresh", "Refreshes");
+  pcb.add_time_avg(l_paxos_refresh_latency, "refresh_latency", "Refresh latency");
+  pcb.add_u64_counter(l_paxos_begin, "begin", "Started and handled begins");
+  pcb.add_u64_avg(l_paxos_begin_keys, "begin_keys", "Keys in transaction on begin");
+  pcb.add_u64_avg(l_paxos_begin_bytes, "begin_bytes", "Data in transaction on begin", NULL, 0, unit_t(UNIT_BYTES));
+  pcb.add_time_avg(l_paxos_begin_latency, "begin_latency", "Latency of begin operation");
+  pcb.add_u64_counter(l_paxos_commit, "commit",
+      "Commits", "cmt");
+  pcb.add_u64_avg(l_paxos_commit_keys, "commit_keys", "Keys in transaction on commit");
+  pcb.add_u64_avg(l_paxos_commit_bytes, "commit_bytes", "Data in transaction on commit", NULL, 0, unit_t(UNIT_BYTES));
+  pcb.add_time_avg(l_paxos_commit_latency, "commit_latency",
+      "Commit latency", "clat");
+  pcb.add_u64_counter(l_paxos_collect, "collect", "Peon collects");
+  pcb.add_u64_avg(l_paxos_collect_keys, "collect_keys", "Keys in transaction on peon collect");
+  pcb.add_u64_avg(l_paxos_collect_bytes, "collect_bytes", "Data in transaction on peon collect", NULL, 0, unit_t(UNIT_BYTES));
+  pcb.add_time_avg(l_paxos_collect_latency, "collect_latency", "Peon collect latency");
+  pcb.add_u64_counter(l_paxos_collect_uncommitted, "collect_uncommitted", "Uncommitted values in started and handled collects");
+  pcb.add_u64_counter(l_paxos_collect_timeout, "collect_timeout", "Collect timeouts");
+  pcb.add_u64_counter(l_paxos_accept_timeout, "accept_timeout", "Accept timeouts");
+  pcb.add_u64_counter(l_paxos_lease_ack_timeout, "lease_ack_timeout", "Lease acknowledgement timeouts");
+  pcb.add_u64_counter(l_paxos_lease_timeout, "lease_timeout", "Lease timeouts");
+  pcb.add_u64_counter(l_paxos_store_state, "store_state", "Store a shared state on disk");
+  pcb.add_u64_avg(l_paxos_store_state_keys, "store_state_keys", "Keys in transaction in stored state");
+  pcb.add_u64_avg(l_paxos_store_state_bytes, "store_state_bytes", "Data in transaction in stored state", NULL, 0, unit_t(UNIT_BYTES));
+  pcb.add_time_avg(l_paxos_store_state_latency, "store_state_latency", "Storing state latency");
+  pcb.add_u64_counter(l_paxos_share_state, "share_state", "Sharings of state");
+  pcb.add_u64_avg(l_paxos_share_state_keys, "share_state_keys", "Keys in shared state");
+  pcb.add_u64_avg(l_paxos_share_state_bytes, "share_state_bytes", "Data in shared state", NULL, 0, unit_t(UNIT_BYTES));
+  pcb.add_u64_counter(l_paxos_new_pn, "new_pn", "New proposal number queries");
+  pcb.add_time_avg(l_paxos_new_pn_latency, "new_pn_latency", "New proposal number getting latency");
+  logger = pcb.create_perf_counters();
+  g_ceph_context->get_perfcounters_collection()->add(logger);
+}
+
+void Paxos::dump_info(Formatter *f)
+{
+  f->open_object_section("paxos");
+  f->dump_unsigned("first_committed", first_committed);
+  f->dump_unsigned("last_committed", last_committed);
+  f->dump_unsigned("last_pn", last_pn);
+  f->dump_unsigned("accepted_pn", accepted_pn);
+  f->close_section();
+}
+
+// ---------------------------------
+
+// PHASE 1
+
+// leader
+void Paxos::collect(version_t oldpn)
+{
+  // we're recoverying, it seems!
+  state = STATE_RECOVERING;
+  ceph_assert(mon.is_leader());
+
+  // reset the number of lasts received
+  uncommitted_v = 0;
+  uncommitted_pn = 0;
+  uncommitted_value.clear();
+  peer_first_committed.clear();
+  peer_last_committed.clear();
+
+  // look for uncommitted value
+  if (get_store()->exists(get_name(), last_committed+1)) {
+    version_t v = get_store()->get(get_name(), "pending_v");
+    version_t pn = get_store()->get(get_name(), "pending_pn");
+    if (v && pn && v == last_committed + 1) {
+      uncommitted_pn = pn;
+    } else {
+      dout(10) << "WARNING: no pending_pn on disk, using previous accepted_pn " << accepted_pn
+	       << " and crossing our fingers" << dendl;
+      uncommitted_pn = accepted_pn;
+    }
+    uncommitted_v = last_committed+1;
+
+    get_store()->get(get_name(), last_committed+1, uncommitted_value);
+    ceph_assert(uncommitted_value.length());
+    dout(10) << "learned uncommitted " << (last_committed+1)
+	     << " pn " << uncommitted_pn
+	     << " (" << uncommitted_value.length() << " bytes) from myself" 
+	     << dendl;
+
+    logger->inc(l_paxos_collect_uncommitted);
+  }
+
+  // pick new pn
+  accepted_pn = get_new_proposal_number(std::max(accepted_pn, oldpn));
+  accepted_pn_from = last_committed;
+  num_last = 1;
+  dout(10) << "collect with pn " << accepted_pn << dendl;
+
+  // send collect
+  for (auto p = mon.get_quorum().begin();
+       p != mon.get_quorum().end();
+       ++p) {
+    if (*p == mon.rank) continue;
+
+    MMonPaxos *collect = new MMonPaxos(mon.get_epoch(), MMonPaxos::OP_COLLECT,
+				       ceph_clock_now());
+    collect->last_committed = last_committed;
+    collect->first_committed = first_committed;
+    collect->pn = accepted_pn;
+    mon.send_mon_message(collect, *p);
+  }
+
+  // set timeout event
+  collect_timeout_event = mon.timer.add_event_after(
+    g_conf()->mon_accept_timeout_factor *
+    g_conf()->mon_lease,
+    new C_MonContext{&mon, [this](int r) {
+	if (r == -ECANCELED)
+	  return;
+	collect_timeout();
+    }});
+}
+
+
+// peon
+void Paxos::handle_collect(MonOpRequestRef op)
+{
+  
+  op->mark_paxos_event("handle_collect");
+
+  auto collect = op->get_req<MMonPaxos>();
+  dout(10) << "handle_collect " << *collect << dendl;
+
+  ceph_assert(mon.is_peon()); // mon epoch filter should catch strays
+
+  // we're recoverying, it seems!
+  state = STATE_RECOVERING;
+
+  //update the peon recovery timeout 
+  reset_lease_timeout();
+
+  if (collect->first_committed > last_committed+1) {
+    dout(2) << __func__
+            << " leader's lowest version is too high for our last committed"
+            << " (theirs: " << collect->first_committed
+            << "; ours: " << last_committed << ") -- bootstrap!" << dendl;
+    op->mark_paxos_event("need to bootstrap");
+    mon.bootstrap();
+    return;
+  }
+
+  // reply
+  MMonPaxos *last = new MMonPaxos(mon.get_epoch(), MMonPaxos::OP_LAST,
+				  ceph_clock_now());
+  last->last_committed = last_committed;
+  last->first_committed = first_committed;
+  
+  version_t previous_pn = accepted_pn;
+
+  // can we accept this pn?
+  if (collect->pn > accepted_pn) {
+    // ok, accept it
+    accepted_pn = collect->pn;
+    accepted_pn_from = collect->pn_from;
+    dout(10) << "accepting pn " << accepted_pn << " from " 
+	     << accepted_pn_from << dendl;
+  
+    auto t(std::make_shared<MonitorDBStore::Transaction>());
+    t->put(get_name(), "accepted_pn", accepted_pn);
+
+    dout(30) << __func__ << " transaction dump:\n";
+    JSONFormatter f(true);
+    t->dump(&f);
+    f.flush(*_dout);
+    *_dout << dendl;
+
+    logger->inc(l_paxos_collect);
+    logger->inc(l_paxos_collect_keys, t->get_keys());
+    logger->inc(l_paxos_collect_bytes, t->get_bytes());
+
+    auto start = ceph::coarse_mono_clock::now();
+    get_store()->apply_transaction(t);
+    auto end = ceph::coarse_mono_clock::now();
+
+    logger->tinc(l_paxos_collect_latency, to_timespan(end - start));
+  } else {
+    // don't accept!
+    dout(10) << "NOT accepting pn " << collect->pn << " from " << collect->pn_from
+	     << ", we already accepted " << accepted_pn
+	     << " from " << accepted_pn_from << dendl;
+  }
+  last->pn = accepted_pn;
+  last->pn_from = accepted_pn_from;
+
+  // share whatever committed values we have
+  if (collect->last_committed < last_committed)
+    share_state(last, collect->first_committed, collect->last_committed);
+
+  // do we have an accepted but uncommitted value?
+  //  (it'll be at last_committed+1)
+  bufferlist bl;
+  if (collect->last_committed <= last_committed &&
+      get_store()->exists(get_name(), last_committed+1)) {
+    get_store()->get(get_name(), last_committed+1, bl);
+    ceph_assert(bl.length() > 0);
+    dout(10) << " sharing our accepted but uncommitted value for " 
+	     << last_committed+1 << " (" << bl.length() << " bytes)" << dendl;
+    last->values[last_committed+1] = bl;
+
+    version_t v = get_store()->get(get_name(), "pending_v");
+    version_t pn = get_store()->get(get_name(), "pending_pn");
+    if (v && pn && v == last_committed + 1) {
+      last->uncommitted_pn = pn;
+    } else {
+      // previously we didn't record which pn a value was accepted
+      // under!  use the pn value we just had...  :(
+      dout(10) << "WARNING: no pending_pn on disk, using previous accepted_pn " << previous_pn
+	       << " and crossing our fingers" << dendl;
+      last->uncommitted_pn = previous_pn;
+    }
+
+    logger->inc(l_paxos_collect_uncommitted);
+  }
+
+  // send reply
+  collect->get_connection()->send_message(last);
+}
+
+/**
+ * @note This is Okay. We share our versions between peer_last_committed and
+ *	 our last_committed (inclusive), and add their bufferlists to the
+ *	 message. It will be the peer's job to apply them to its store, as
+ *	 these bufferlists will contain raw transactions.
+ *	 This function is called by both the Peon and the Leader. The Peon will
+ *	 share the state with the Leader during handle_collect(), sharing any
+ *	 values the leader may be missing (i.e., the leader's last_committed is
+ *	 lower than the peon's last_committed). The Leader will share the state
+ *	 with the Peon during handle_last(), if the peon's last_committed is
+ *	 lower than the leader's last_committed.
+ */
+void Paxos::share_state(MMonPaxos *m, version_t peer_first_committed,
+			version_t peer_last_committed)
+{
+  ceph_assert(peer_last_committed < last_committed);
+
+  dout(10) << "share_state peer has fc " << peer_first_committed 
+	   << " lc " << peer_last_committed << dendl;
+  version_t v = peer_last_committed + 1;
+
+  // include incrementals
+  uint64_t bytes = 0;
+  for ( ; v <= last_committed; v++) {
+    if (get_store()->exists(get_name(), v)) {
+      get_store()->get(get_name(), v, m->values[v]);
+      ceph_assert(m->values[v].length());
+      dout(10) << " sharing " << v << " ("
+	       << m->values[v].length() << " bytes)" << dendl;
+      bytes += m->values[v].length() + 16;  // paxos_ + 10 digits = 16
+    }
+  }
+  logger->inc(l_paxos_share_state);
+  logger->inc(l_paxos_share_state_keys, m->values.size());
+  logger->inc(l_paxos_share_state_bytes, bytes);
+
+  m->last_committed = last_committed;
+}
+
+/**
+ * Store on disk a state that was shared with us
+ *
+ * Basically, we received a set of version. Or just one. It doesn't matter.
+ * What matters is that we have to stash it in the store. So, we will simply
+ * write every single bufferlist into their own versions on our side (i.e.,
+ * onto paxos-related keys), and then we will decode those same bufferlists
+ * we just wrote and apply the transactions they hold. We will also update
+ * our first and last committed values to point to the new values, if need
+ * be. All all this is done tightly wrapped in a transaction to ensure we
+ * enjoy the atomicity guarantees given by our awesome k/v store.
+ */
+bool Paxos::store_state(MMonPaxos *m)
+{
+  auto t(std::make_shared<MonitorDBStore::Transaction>());
+  auto start = m->values.begin();
+  bool changed = false;
+
+  // build map of values to store
+  // we want to write the range [last_committed, m->last_committed] only.
+  if (start != m->values.end() &&
+      start->first > last_committed + 1) {
+    // ignore everything if values start in the future.
+    dout(10) << "store_state ignoring all values, they start at " << start->first
+	     << " > last_committed+1" << dendl;
+    return false;
+  }
+
+  // push forward the start position on the message's values iterator, up until
+  // we run out of positions or we find a position matching 'last_committed'.
+  while (start != m->values.end() && start->first <= last_committed) {
+    ++start;
+  }
+
+  // make sure we get the right interval of values to apply by pushing forward
+  // the 'end' iterator until it matches the message's 'last_committed'.
+  auto end = start;
+  while (end != m->values.end() && end->first <= m->last_committed) {
+    last_committed = end->first;
+    ++end;
+  }
+
+  if (start == end) {
+    dout(10) << "store_state nothing to commit" << dendl;
+  } else {
+    dout(10) << "store_state [" << start->first << ".." 
+	     << last_committed << "]" << dendl;
+    t->put(get_name(), "last_committed", last_committed);
+
+    // we should apply the state here -- decode every single bufferlist in the
+    // map and append the transactions to 't'.
+    for (auto it = start; it != end; ++it) {
+      // write the bufferlist as the version's value
+      t->put(get_name(), it->first, it->second);
+      // decode the bufferlist and append it to the transaction we will shortly
+      // apply.
+      decode_append_transaction(t, it->second);
+    }
+
+    // discard obsolete uncommitted value?
+    if (uncommitted_v && uncommitted_v <= last_committed) {
+      dout(10) << " forgetting obsolete uncommitted value " << uncommitted_v
+	       << " pn " << uncommitted_pn << dendl;
+      uncommitted_v = 0;
+      uncommitted_pn = 0;
+      uncommitted_value.clear();
+    }
+  }
+  if (!t->empty()) {
+    dout(30) << __func__ << " transaction dump:\n";
+    JSONFormatter f(true);
+    t->dump(&f);
+    f.flush(*_dout);
+    *_dout << dendl;
+
+    logger->inc(l_paxos_store_state);
+    logger->inc(l_paxos_store_state_bytes, t->get_bytes());
+    logger->inc(l_paxos_store_state_keys, t->get_keys());
+
+    auto start = ceph::coarse_mono_clock::now();
+    get_store()->apply_transaction(t);
+    auto end = ceph::coarse_mono_clock::now();
+
+    logger->tinc(l_paxos_store_state_latency, to_timespan(end-start));
+
+    // refresh first_committed; this txn may have trimmed.
+    first_committed = get_store()->get(get_name(), "first_committed");
+
+    _sanity_check_store();
+    changed = true;
+  }
+
+  return changed;
+}
+
+void Paxos::_sanity_check_store()
+{
+  version_t lc = get_store()->get(get_name(), "last_committed");
+  ceph_assert(lc == last_committed);
+}
+
+
+// leader
+void Paxos::handle_last(MonOpRequestRef op)
+{
+  op->mark_paxos_event("handle_last");
+  auto last = op->get_req<MMonPaxos>();
+  bool need_refresh = false;
+  int from = last->get_source().num();
+
+  dout(10) << "handle_last " << *last << dendl;
+
+  if (!mon.is_leader()) {
+    dout(10) << "not leader, dropping" << dendl;
+    return;
+  }
+
+  // note peer's first_ and last_committed, in case we learn a new
+  // commit and need to push it to them.
+  peer_first_committed[from] = last->first_committed;
+  peer_last_committed[from] = last->last_committed;
+
+  if (last->first_committed > last_committed + 1) {
+    dout(5) << __func__
+            << " mon." << from
+	    << " lowest version is too high for our last committed"
+            << " (theirs: " << last->first_committed
+            << "; ours: " << last_committed << ") -- bootstrap!" << dendl;
+    op->mark_paxos_event("need to bootstrap");
+    mon.bootstrap();
+    return;
+  }
+
+  ceph_assert(g_conf()->paxos_kill_at != 1);
+
+  // store any committed values if any are specified in the message
+  need_refresh = store_state(last);
+
+  ceph_assert(g_conf()->paxos_kill_at != 2);
+
+  // is everyone contiguous and up to date?
+  for (auto p = peer_last_committed.begin();
+       p != peer_last_committed.end();
+       ++p) {
+    if (p->second + 1 < first_committed && first_committed > 1) {
+      dout(5) << __func__
+	      << " peon " << p->first
+	      << " last_committed (" << p->second
+	      << ") is too low for our first_committed (" << first_committed
+	      << ") -- bootstrap!" << dendl;
+      op->mark_paxos_event("need to bootstrap");
+      mon.bootstrap();
+      return;
+    }
+    if (p->second < last_committed) {
+      // share committed values
+      dout(10) << " sending commit to mon." << p->first << dendl;
+      MMonPaxos *commit = new MMonPaxos(mon.get_epoch(),
+					MMonPaxos::OP_COMMIT,
+					ceph_clock_now());
+      share_state(commit, peer_first_committed[p->first], p->second);
+      mon.send_mon_message(commit, p->first);
+    }
+  }
+
+  // do they accept your pn?
+  if (last->pn > accepted_pn) {
+    // no, try again.
+    dout(10) << " they had a higher pn than us, picking a new one." << dendl;
+
+    // cancel timeout event
+    mon.timer.cancel_event(collect_timeout_event);
+    collect_timeout_event = 0;
+
+    collect(last->pn);
+  } else if (last->pn == accepted_pn) {
+    // yes, they accepted our pn.  great.
+    num_last++;
+    dout(10) << " they accepted our pn, we now have " 
+	     << num_last << " peons" << dendl;
+
+    // did this person send back an accepted but uncommitted value?
+    if (last->uncommitted_pn) {
+      if (last->uncommitted_pn >= uncommitted_pn &&
+	  last->last_committed >= last_committed &&
+	  last->last_committed + 1 >= uncommitted_v) {
+	uncommitted_v = last->last_committed+1;
+	uncommitted_pn = last->uncommitted_pn;
+	uncommitted_value = last->values[uncommitted_v];
+	dout(10) << "we learned an uncommitted value for " << uncommitted_v
+		 << " pn " << uncommitted_pn
+		 << " " << uncommitted_value.length() << " bytes"
+		 << dendl;
+      } else {
+	dout(10) << "ignoring uncommitted value for " << (last->last_committed+1)
+		 << " pn " << last->uncommitted_pn
+		 << " " << last->values[last->last_committed+1].length() << " bytes"
+		 << dendl;
+      }
+    }
+    
+    // is that everyone?
+    if (num_last == mon.get_quorum().size()) {
+      // cancel timeout event
+      mon.timer.cancel_event(collect_timeout_event);
+      collect_timeout_event = 0;
+      peer_first_committed.clear();
+      peer_last_committed.clear();
+
+      // almost...
+
+      // did we learn an old value?
+      if (uncommitted_v == last_committed+1 &&
+	  uncommitted_value.length()) {
+	dout(10) << "that's everyone.  begin on old learned value" << dendl;
+	state = STATE_UPDATING_PREVIOUS;
+	begin(uncommitted_value);
+      } else {
+	// active!
+	dout(10) << "that's everyone.  active!" << dendl;
+	extend_lease();
+
+	need_refresh = false;
+	if (do_refresh()) {
+	  finish_round();
+	}
+      }
+    }
+  } else {
+    // no, this is an old message, discard
+    dout(10) << "old pn, ignoring" << dendl;
+  }
+
+  if (need_refresh)
+    (void)do_refresh();
+}
+
+void Paxos::collect_timeout()
+{
+  dout(1) << "collect timeout, calling fresh election" << dendl;
+  collect_timeout_event = 0;
+  logger->inc(l_paxos_collect_timeout);
+  ceph_assert(mon.is_leader());
+  mon.bootstrap();
+}
+
+
+// leader
+void Paxos::begin(bufferlist& v)
+{
+  dout(10) << "begin for " << last_committed+1 << " " 
+	   << v.length() << " bytes"
+	   << dendl;
+
+  ceph_assert(mon.is_leader());
+  ceph_assert(is_updating() || is_updating_previous());
+
+  // we must already have a majority for this to work.
+  ceph_assert(mon.get_quorum().size() == 1 ||
+	 num_last > (unsigned)mon.monmap->size()/2);
+  
+  // and no value, yet.
+  ceph_assert(new_value.length() == 0);
+
+  // accept it ourselves
+  accepted.clear();
+  accepted.insert(mon.rank);
+  new_value = v;
+
+  if (last_committed == 0) {
+    auto t(std::make_shared<MonitorDBStore::Transaction>());
+    // initial base case; set first_committed too
+    t->put(get_name(), "first_committed", 1);
+    decode_append_transaction(t, new_value);
+
+    bufferlist tx_bl;
+    t->encode(tx_bl);
+
+    new_value = tx_bl;
+  }
+
+  // store the proposed value in the store. IF it is accepted, we will then
+  // have to decode it into a transaction and apply it.
+  auto t(std::make_shared<MonitorDBStore::Transaction>());
+  t->put(get_name(), last_committed+1, new_value);
+
+  // note which pn this pending value is for.
+  t->put(get_name(), "pending_v", last_committed + 1);
+  t->put(get_name(), "pending_pn", accepted_pn);
+
+  dout(30) << __func__ << " transaction dump:\n";
+  JSONFormatter f(true);
+  t->dump(&f);
+  f.flush(*_dout);
+  auto debug_tx(std::make_shared<MonitorDBStore::Transaction>());
+  auto new_value_it = new_value.cbegin();
+  debug_tx->decode(new_value_it);
+  debug_tx->dump(&f);
+  *_dout << "\nbl dump:\n";
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  logger->inc(l_paxos_begin);
+  logger->inc(l_paxos_begin_keys, t->get_keys());
+  logger->inc(l_paxos_begin_bytes, t->get_bytes());
+
+  auto start = ceph::coarse_mono_clock::now();
+  get_store()->apply_transaction(t);
+  auto end = ceph::coarse_mono_clock::now();
+
+  logger->tinc(l_paxos_begin_latency, to_timespan(end - start));
+
+  ceph_assert(g_conf()->paxos_kill_at != 3);
+
+  if (mon.get_quorum().size() == 1) {
+    // we're alone, take it easy
+    commit_start();
+    return;
+  }
+
+  // ask others to accept it too!
+  for (auto p = mon.get_quorum().begin();
+       p != mon.get_quorum().end();
+       ++p) {
+    if (*p == mon.rank) continue;
+    
+    dout(10) << " sending begin to mon." << *p << dendl;
+    MMonPaxos *begin = new MMonPaxos(mon.get_epoch(), MMonPaxos::OP_BEGIN,
+				     ceph_clock_now());
+    begin->values[last_committed+1] = new_value;
+    begin->last_committed = last_committed;
+    begin->pn = accepted_pn;
+    
+    mon.send_mon_message(begin, *p);
+  }
+
+  // set timeout event
+  accept_timeout_event = mon.timer.add_event_after(
+    g_conf()->mon_accept_timeout_factor * g_conf()->mon_lease,
+    new C_MonContext{&mon, [this](int r) {
+	if (r == -ECANCELED)
+	  return;
+	accept_timeout();
+      }});
+}
+
+// peon
+void Paxos::handle_begin(MonOpRequestRef op)
+{
+  op->mark_paxos_event("handle_begin");
+  auto begin = op->get_req<MMonPaxos>();
+  dout(10) << "handle_begin " << *begin << dendl;
+
+  // can we accept this?
+  if (begin->pn < accepted_pn) {
+    dout(10) << " we accepted a higher pn " << accepted_pn << ", ignoring" << dendl;
+    op->mark_paxos_event("have higher pn, ignore");
+    return;
+  }
+  ceph_assert(begin->pn == accepted_pn);
+  ceph_assert(begin->last_committed == last_committed);
+  
+  ceph_assert(g_conf()->paxos_kill_at != 4);
+
+  logger->inc(l_paxos_begin);
+
+  // set state.
+  state = STATE_UPDATING;
+  lease_expire = {};  // cancel lease
+
+  // yes.
+  version_t v = last_committed+1;
+  dout(10) << "accepting value for " << v << " pn " << accepted_pn << dendl;
+  // store the accepted value onto our store. We will have to decode it and
+  // apply its transaction once we receive permission to commit.
+  auto t(std::make_shared<MonitorDBStore::Transaction>());
+  t->put(get_name(), v, begin->values[v]);
+
+  // note which pn this pending value is for.
+  t->put(get_name(), "pending_v", v);
+  t->put(get_name(), "pending_pn", accepted_pn);
+
+  dout(30) << __func__ << " transaction dump:\n";
+  JSONFormatter f(true);
+  t->dump(&f);
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  logger->inc(l_paxos_begin_bytes, t->get_bytes());
+
+  auto start = ceph::coarse_mono_clock::now();
+  get_store()->apply_transaction(t);
+  auto end = ceph::coarse_mono_clock::now();
+
+  logger->tinc(l_paxos_begin_latency, to_timespan(end - start));
+
+  ceph_assert(g_conf()->paxos_kill_at != 5);
+
+  // reply
+  MMonPaxos *accept = new MMonPaxos(mon.get_epoch(), MMonPaxos::OP_ACCEPT,
+				    ceph_clock_now());
+  accept->pn = accepted_pn;
+  accept->last_committed = last_committed;
+  begin->get_connection()->send_message(accept);
+}
+
+// leader
+void Paxos::handle_accept(MonOpRequestRef op)
+{
+  op->mark_paxos_event("handle_accept");
+  auto accept = op->get_req<MMonPaxos>();
+  dout(10) << "handle_accept " << *accept << dendl;
+  int from = accept->get_source().num();
+
+  if (accept->pn != accepted_pn) {
+    // we accepted a higher pn, from some other leader
+    dout(10) << " we accepted a higher pn " << accepted_pn << ", ignoring" << dendl;
+    op->mark_paxos_event("have higher pn, ignore");
+    return;
+  }
+  if (last_committed > 0 &&
+      accept->last_committed < last_committed-1) {
+    dout(10) << " this is from an old round, ignoring" << dendl;
+    op->mark_paxos_event("old round, ignore");
+    return;
+  }
+  ceph_assert(accept->last_committed == last_committed ||   // not committed
+	 accept->last_committed == last_committed-1);  // committed
+
+  ceph_assert(is_updating() || is_updating_previous());
+  ceph_assert(accepted.count(from) == 0);
+  accepted.insert(from);
+  dout(10) << " now " << accepted << " have accepted" << dendl;
+
+  ceph_assert(g_conf()->paxos_kill_at != 6);
+
+  // only commit (and expose committed state) when we get *all* quorum
+  // members to accept.  otherwise, they may still be sharing the now
+  // stale state.
+  // FIXME: we can improve this with an additional lease revocation message
+  // that doesn't block for the persist.
+  if (accepted == mon.get_quorum()) {
+    // yay, commit!
+    dout(10) << " got majority, committing, done with update" << dendl;
+    op->mark_paxos_event("commit_start");
+    commit_start();
+  }
+}
+
+void Paxos::accept_timeout()
+{
+  dout(1) << "accept timeout, calling fresh election" << dendl;
+  accept_timeout_event = 0;
+  ceph_assert(mon.is_leader());
+  ceph_assert(is_updating() || is_updating_previous() || is_writing() ||
+	 is_writing_previous());
+  logger->inc(l_paxos_accept_timeout);
+  mon.bootstrap();
+}
+
+struct C_Committed : public Context {
+  Paxos *paxos;
+  explicit C_Committed(Paxos *p) : paxos(p) {}
+  void finish(int r) override {
+    ceph_assert(r >= 0);
+    std::lock_guard l(paxos->mon.lock);
+    if (paxos->is_shutdown()) {
+      paxos->abort_commit();
+      return;
+    }
+    paxos->commit_finish();
+  }
+};
+
+void Paxos::abort_commit()
+{
+  ceph_assert(commits_started > 0);
+  --commits_started;
+  if (commits_started == 0)
+    shutdown_cond.notify_all();
+}
+
+void Paxos::commit_start()
+{
+  dout(10) << __func__ << " " << (last_committed+1) << dendl;
+
+  ceph_assert(g_conf()->paxos_kill_at != 7);
+
+  auto t(std::make_shared<MonitorDBStore::Transaction>());
+
+  // commit locally
+  t->put(get_name(), "last_committed", last_committed + 1);
+
+  // decode the value and apply its transaction to the store.
+  // this value can now be read from last_committed.
+  decode_append_transaction(t, new_value);
+
+  dout(30) << __func__ << " transaction dump:\n";
+  JSONFormatter f(true);
+  t->dump(&f);
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  logger->inc(l_paxos_commit);
+  logger->inc(l_paxos_commit_keys, t->get_keys());
+  logger->inc(l_paxos_commit_bytes, t->get_bytes());
+  commit_start_stamp = ceph_clock_now();
+
+  get_store()->queue_transaction(t, new C_Committed(this));
+
+  if (is_updating_previous())
+    state = STATE_WRITING_PREVIOUS;
+  else if (is_updating())
+    state = STATE_WRITING;
+  else
+    ceph_abort();
+  ++commits_started;
+
+  if (mon.get_quorum().size() > 1) {
+    // cancel timeout event
+    mon.timer.cancel_event(accept_timeout_event);
+    accept_timeout_event = 0;
+  }
+}
+
+void Paxos::commit_finish()
+{
+  dout(20) << __func__ << " " << (last_committed+1) << dendl;
+  utime_t end = ceph_clock_now();
+  logger->tinc(l_paxos_commit_latency, end - commit_start_stamp);
+
+  ceph_assert(g_conf()->paxos_kill_at != 8);
+
+  // cancel lease - it was for the old value.
+  //  (this would only happen if message layer lost the 'begin', but
+  //   leader still got a majority and committed with out us.)
+  lease_expire = {};  // cancel lease
+
+  last_committed++;
+  last_commit_time = ceph_clock_now();
+
+  // refresh first_committed; this txn may have trimmed.
+  first_committed = get_store()->get(get_name(), "first_committed");
+
+  _sanity_check_store();
+
+  // tell everyone
+  for (auto p = mon.get_quorum().begin();
+       p != mon.get_quorum().end();
+       ++p) {
+    if (*p == mon.rank) continue;
+
+    dout(10) << " sending commit to mon." << *p << dendl;
+    MMonPaxos *commit = new MMonPaxos(mon.get_epoch(), MMonPaxos::OP_COMMIT,
+				      ceph_clock_now());
+    commit->values[last_committed] = new_value;
+    commit->pn = accepted_pn;
+    commit->last_committed = last_committed;
+
+    mon.send_mon_message(commit, *p);
+  }
+
+  ceph_assert(g_conf()->paxos_kill_at != 9);
+
+  // get ready for a new round.
+  new_value.clear();
+
+  // WRITING -> REFRESH
+  // among other things, this lets do_refresh() -> mon.bootstrap() ->
+  // wait_for_paxos_write() know that it doesn't need to flush the store
+  // queue. and it should not, as we are in the async completion thread now!
+  ceph_assert(is_writing() || is_writing_previous());
+  state = STATE_REFRESH;
+  ceph_assert(commits_started > 0);
+  --commits_started;
+
+  if (do_refresh()) {
+    commit_proposal();
+    if (mon.get_quorum().size() > 1) {
+      extend_lease();
+    }
+
+    ceph_assert(g_conf()->paxos_kill_at != 10);
+
+    finish_round();
+  }
+}
+
+
+void Paxos::handle_commit(MonOpRequestRef op)
+{
+  op->mark_paxos_event("handle_commit");
+  auto commit = op->get_req<MMonPaxos>();
+  dout(10) << "handle_commit on " << commit->last_committed << dendl;
+
+  logger->inc(l_paxos_commit);
+
+  if (!mon.is_peon()) {
+    dout(10) << "not a peon, dropping" << dendl;
+    ceph_abort();
+    return;
+  }
+
+  op->mark_paxos_event("store_state");
+  store_state(commit);
+
+  (void)do_refresh();
+}
+
+void Paxos::extend_lease()
+{
+  ceph_assert(mon.is_leader());
+  //assert(is_active());
+
+  lease_expire = ceph::real_clock::now();
+  lease_expire += ceph::make_timespan(g_conf()->mon_lease);
+  acked_lease.clear();
+  acked_lease.insert(mon.rank);
+
+  dout(7) << "extend_lease now+" << g_conf()->mon_lease
+	  << " (" << lease_expire << ")" << dendl;
+
+  // bcast
+  for (auto p = mon.get_quorum().begin();
+      p != mon.get_quorum().end(); ++p) {
+
+    if (*p == mon.rank) continue;
+    MMonPaxos *lease = new MMonPaxos(mon.get_epoch(), MMonPaxos::OP_LEASE,
+				     ceph_clock_now());
+    lease->last_committed = last_committed;
+    lease->lease_timestamp = utime_t{lease_expire};
+    lease->first_committed = first_committed;
+    mon.send_mon_message(lease, *p);
+  }
+
+  // set timeout event.
+  //  if old timeout is still in place, leave it.
+  if (!lease_ack_timeout_event) {
+    lease_ack_timeout_event = mon.timer.add_event_after(
+      g_conf()->mon_lease_ack_timeout_factor * g_conf()->mon_lease,
+      new C_MonContext{&mon, [this](int r) {
+	  if (r == -ECANCELED)
+	    return;
+	  lease_ack_timeout();
+	}});
+  }
+
+  // set renew event
+  auto at = lease_expire;
+  at -= ceph::make_timespan(g_conf()->mon_lease);
+  at += ceph::make_timespan(g_conf()->mon_lease_renew_interval_factor *
+			    g_conf()->mon_lease);
+  lease_renew_event = mon.timer.add_event_at(
+    at, new C_MonContext{&mon, [this](int r) {
+	if (r == -ECANCELED)
+	  return;
+	lease_renew_timeout();
+    }});
+}
+
+void Paxos::warn_on_future_time(utime_t t, entity_name_t from)
+{
+  utime_t now = ceph_clock_now();
+  if (t > now) {
+    utime_t diff = t - now;
+    if (diff > g_conf()->mon_clock_drift_allowed) {
+      utime_t warn_diff = now - last_clock_drift_warn;
+      if (warn_diff >
+	  pow(g_conf()->mon_clock_drift_warn_backoff, clock_drift_warned)) {
+	mon.clog->warn() << "message from " << from << " was stamped " << diff
+			 << "s in the future, clocks not synchronized";
+	last_clock_drift_warn = ceph_clock_now();
+	++clock_drift_warned;
+      }
+    }
+  }
+
+}
+
+bool Paxos::do_refresh()
+{
+  bool need_bootstrap = false;
+
+  // make sure we have the latest state loaded up
+  auto start = ceph::coarse_mono_clock::now();
+  mon.refresh_from_paxos(&need_bootstrap);
+  auto end = ceph::coarse_mono_clock::now();
+
+  logger->inc(l_paxos_refresh);
+  logger->tinc(l_paxos_refresh_latency, to_timespan(end - start));
+
+  if (need_bootstrap) {
+    dout(10) << " doing requested bootstrap" << dendl;
+    mon.bootstrap();
+    return false;
+  }
+
+  return true;
+}
+
+void Paxos::commit_proposal()
+{
+  dout(10) << __func__ << dendl;
+  ceph_assert(mon.is_leader());
+  ceph_assert(is_refresh());
+
+  finish_contexts(g_ceph_context, committing_finishers);
+}
+
+void Paxos::finish_round()
+{
+  dout(10) << __func__ << dendl;
+  ceph_assert(mon.is_leader());
+
+  // ok, now go active!
+  state = STATE_ACTIVE;
+
+  dout(20) << __func__ << " waiting_for_acting" << dendl;
+  finish_contexts(g_ceph_context, waiting_for_active);
+  dout(20) << __func__ << " waiting_for_readable" << dendl;
+  finish_contexts(g_ceph_context, waiting_for_readable);
+  dout(20) << __func__ << " waiting_for_writeable" << dendl;
+  finish_contexts(g_ceph_context, waiting_for_writeable);
+  
+  dout(10) << __func__ << " done w/ waiters, state " << get_statename(state) << dendl;
+
+  if (should_trim()) {
+    trim();
+  }
+
+  if (is_active() && pending_proposal) {
+    propose_pending();
+  }
+}
+
+
+// peon
+void Paxos::handle_lease(MonOpRequestRef op)
+{
+  op->mark_paxos_event("handle_lease");
+  auto lease = op->get_req<MMonPaxos>();
+  // sanity
+  if (!mon.is_peon() ||
+      last_committed != lease->last_committed) {
+    dout(10) << "handle_lease i'm not a peon, or they're not the leader,"
+	     << " or the last_committed doesn't match, dropping" << dendl;
+    op->mark_paxos_event("invalid lease, ignore");
+    return;
+  }
+
+  warn_on_future_time(lease->sent_timestamp, lease->get_source());
+
+  // extend lease
+  if (auto new_expire = lease->lease_timestamp.to_real_time();
+      lease_expire < new_expire) {
+    lease_expire = new_expire;
+
+    auto now = ceph::real_clock::now();
+    if (lease_expire < now) {
+      auto diff = now - lease_expire;
+      derr << "lease_expire from " << lease->get_source_inst() << " is " << diff << " seconds in the past; mons are probably laggy (or possibly clocks are too skewed)" << dendl;
+    }
+  }
+
+  state = STATE_ACTIVE;
+
+  dout(10) << "handle_lease on " << lease->last_committed
+	   << " now " << lease_expire << dendl;
+
+  // ack
+  MMonPaxos *ack = new MMonPaxos(mon.get_epoch(), MMonPaxos::OP_LEASE_ACK,
+				 ceph_clock_now());
+  ack->last_committed = last_committed;
+  ack->first_committed = first_committed;
+  ack->lease_timestamp = ceph_clock_now();
+  encode(mon.session_map.feature_map, ack->feature_map);
+  lease->get_connection()->send_message(ack);
+
+  // (re)set timeout event.
+  reset_lease_timeout();
+
+  // kick waiters
+  finish_contexts(g_ceph_context, waiting_for_active);
+  if (is_readable())
+    finish_contexts(g_ceph_context, waiting_for_readable);
+}
+
+void Paxos::handle_lease_ack(MonOpRequestRef op)
+{
+  op->mark_paxos_event("handle_lease_ack");
+  auto ack = op->get_req<MMonPaxos>();
+  int from = ack->get_source().num();
+
+  if (!lease_ack_timeout_event) {
+    dout(10) << "handle_lease_ack from " << ack->get_source()
+	     << " -- stray (probably since revoked)" << dendl;
+
+  } else if (acked_lease.count(from) == 0) {
+    acked_lease.insert(from);
+    if (ack->feature_map.length()) {
+      auto p = ack->feature_map.cbegin();
+      FeatureMap& t = mon.quorum_feature_map[from];
+      decode(t, p);
+    }
+    if (acked_lease == mon.get_quorum()) {
+      // yay!
+      dout(10) << "handle_lease_ack from " << ack->get_source()
+	       << " -- got everyone" << dendl;
+      mon.timer.cancel_event(lease_ack_timeout_event);
+      lease_ack_timeout_event = 0;
+
+
+    } else {
+      dout(10) << "handle_lease_ack from " << ack->get_source()
+	       << " -- still need "
+	       << mon.get_quorum().size() - acked_lease.size()
+	       << " more" << dendl;
+    }
+  } else {
+    dout(10) << "handle_lease_ack from " << ack->get_source()
+	     << " dup (lagging!), ignoring" << dendl;
+  }
+
+  warn_on_future_time(ack->sent_timestamp, ack->get_source());
+}
+
+void Paxos::lease_ack_timeout()
+{
+  dout(1) << "lease_ack_timeout -- calling new election" << dendl;
+  ceph_assert(mon.is_leader());
+  ceph_assert(is_active());
+  logger->inc(l_paxos_lease_ack_timeout);
+  lease_ack_timeout_event = 0;
+  mon.bootstrap();
+}
+
+void Paxos::reset_lease_timeout()
+{
+  dout(20) << "reset_lease_timeout - setting timeout event" << dendl;
+  if (lease_timeout_event)
+    mon.timer.cancel_event(lease_timeout_event);
+  lease_timeout_event = mon.timer.add_event_after(
+    g_conf()->mon_lease_ack_timeout_factor * g_conf()->mon_lease,
+    new C_MonContext{&mon, [this](int r) {
+	if (r == -ECANCELED)
+	  return;
+	lease_timeout();
+      }});
+}
+
+void Paxos::lease_timeout()
+{
+  dout(1) << "lease_timeout -- calling new election" << dendl;
+  ceph_assert(mon.is_peon());
+  logger->inc(l_paxos_lease_timeout);
+  lease_timeout_event = 0;
+  mon.bootstrap();
+}
+
+void Paxos::lease_renew_timeout()
+{
+  lease_renew_event = 0;
+  extend_lease();
+}
+
+
+/*
+ * trim old states
+ */
+void Paxos::trim()
+{
+  ceph_assert(should_trim());
+  version_t end = std::min(get_version() - g_conf()->paxos_min,
+		      get_first_committed() + g_conf()->paxos_trim_max);
+
+  if (first_committed >= end)
+    return;
+
+  dout(10) << "trim to " << end << " (was " << first_committed << ")" << dendl;
+
+  MonitorDBStore::TransactionRef t = get_pending_transaction();
+
+  for (version_t v = first_committed; v < end; ++v) {
+    dout(10) << "trim " << v << dendl;
+    t->erase(get_name(), v);
+  }
+  t->put(get_name(), "first_committed", end);
+  if (g_conf()->mon_compact_on_trim) {
+    dout(10) << " compacting trimmed range" << dendl;
+    t->compact_range(get_name(), stringify(first_committed - 1), stringify(end));
+  }
+
+  trimming = true;
+  queue_pending_finisher(new C_Trimmed(this));
+}
+
+/*
+ * return a globally unique, monotonically increasing proposal number
+ */
+version_t Paxos::get_new_proposal_number(version_t gt)
+{
+  if (last_pn < gt) 
+    last_pn = gt;
+  
+  // update. make it unique among all monitors.
+  last_pn /= 100;
+  last_pn++;
+  last_pn *= 100;
+  last_pn += (version_t)mon.rank;
+
+  // write
+  auto t(std::make_shared<MonitorDBStore::Transaction>());
+  t->put(get_name(), "last_pn", last_pn);
+
+  dout(30) << __func__ << " transaction dump:\n";
+  JSONFormatter f(true);
+  t->dump(&f);
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  logger->inc(l_paxos_new_pn);
+
+  auto start = ceph::coarse_mono_clock::now();
+  get_store()->apply_transaction(t);
+  auto end = ceph::coarse_mono_clock::now();
+
+  logger->tinc(l_paxos_new_pn_latency, to_timespan(end - start));
+
+  dout(10) << "get_new_proposal_number = " << last_pn << dendl;
+  return last_pn;
+}
+
+
+void Paxos::cancel_events()
+{
+  if (collect_timeout_event) {
+    mon.timer.cancel_event(collect_timeout_event);
+    collect_timeout_event = 0;
+  }
+  if (accept_timeout_event) {
+    mon.timer.cancel_event(accept_timeout_event);
+    accept_timeout_event = 0;
+  }
+  if (lease_renew_event) {
+    mon.timer.cancel_event(lease_renew_event);
+    lease_renew_event = 0;
+  }
+  if (lease_ack_timeout_event) {
+    mon.timer.cancel_event(lease_ack_timeout_event);
+    lease_ack_timeout_event = 0;
+  }  
+  if (lease_timeout_event) {
+    mon.timer.cancel_event(lease_timeout_event);
+    lease_timeout_event = 0;
+  }
+}
+
+void Paxos::shutdown()
+{
+  dout(10) << __func__ << " cancel all contexts" << dendl;
+
+  state = STATE_SHUTDOWN;
+
+  // discard pending transaction
+  pending_proposal.reset();
+
+  // Let store finish commits in progress
+  // XXX: I assume I can't use finish_contexts() because the store
+  // is going to trigger
+  unique_lock l{mon.lock, std::adopt_lock};
+  shutdown_cond.wait(l, [this] { return commits_started <= 0; });
+  // Monitor::shutdown() will unlock it
+  l.release();
+
+  finish_contexts(g_ceph_context, waiting_for_writeable, -ECANCELED);
+  finish_contexts(g_ceph_context, waiting_for_readable, -ECANCELED);
+  finish_contexts(g_ceph_context, waiting_for_active, -ECANCELED);
+  finish_contexts(g_ceph_context, pending_finishers, -ECANCELED);
+  finish_contexts(g_ceph_context, committing_finishers, -ECANCELED);
+  if (logger)
+    g_ceph_context->get_perfcounters_collection()->remove(logger);
+}
+
+void Paxos::leader_init()
+{
+  cancel_events();
+  new_value.clear();
+
+  // discard pending transaction
+  pending_proposal.reset();
+
+  reset_pending_committing_finishers();
+
+  logger->inc(l_paxos_start_leader);
+
+  if (mon.get_quorum().size() == 1) {
+    state = STATE_ACTIVE;
+    return;
+  }
+
+  state = STATE_RECOVERING;
+  lease_expire = {};
+  dout(10) << "leader_init -- starting paxos recovery" << dendl;
+  collect(0);
+}
+
+void Paxos::peon_init()
+{
+  cancel_events();
+  new_value.clear();
+
+  state = STATE_RECOVERING;
+  lease_expire = {};
+  dout(10) << "peon_init -- i am a peon" << dendl;
+
+  // start a timer, in case the leader never manages to issue a lease
+  reset_lease_timeout();
+
+  // discard pending transaction
+  pending_proposal.reset();
+
+  // no chance to write now!
+  reset_pending_committing_finishers();
+  finish_contexts(g_ceph_context, waiting_for_writeable, -EAGAIN);
+
+  logger->inc(l_paxos_start_peon);
+}
+
+void Paxos::restart()
+{
+  dout(10) << "restart -- canceling timeouts" << dendl;
+  cancel_events();
+  new_value.clear();
+
+  if (is_writing() || is_writing_previous()) {
+    dout(10) << __func__ << " flushing" << dendl;
+    mon.lock.unlock();
+    mon.store->flush();
+    mon.lock.lock();
+    dout(10) << __func__ << " flushed" << dendl;
+  }
+  state = STATE_RECOVERING;
+
+  // discard pending transaction
+  pending_proposal.reset();
+
+  reset_pending_committing_finishers();
+  finish_contexts(g_ceph_context, waiting_for_active, -EAGAIN);
+
+  logger->inc(l_paxos_restart);
+}
+
+void Paxos::reset_pending_committing_finishers()
+{
+  committing_finishers.splice(committing_finishers.end(), pending_finishers);
+  finish_contexts(g_ceph_context, committing_finishers, -EAGAIN);
+}
+
+void Paxos::dispatch(MonOpRequestRef op)
+{
+  ceph_assert(op->is_type_paxos());
+  op->mark_paxos_event("dispatch");
+
+  if (op->get_req()->get_type() != MSG_MON_PAXOS) {
+    dout(0) << "Got unexpected message type " << op->get_req()->get_type()
+	    << " in Paxos::dispatch, aborting!" << dendl;
+    ceph_abort();
+  }
+  
+  auto *req = op->get_req<MMonPaxos>();
+
+  // election in progress?
+  if (!mon.is_leader() && !mon.is_peon()) {
+    dout(5) << "election in progress, dropping " << *req << dendl;
+    return;    
+  }
+
+  // check sanity
+  ceph_assert(mon.is_leader() || 
+	      (mon.is_peon() && req->get_source().num() == mon.get_leader()));  
+
+  // NOTE: these ops are defined in messages/MMonPaxos.h
+  switch (req->op) {
+    // learner
+  case MMonPaxos::OP_COLLECT:
+    handle_collect(op);
+    break;
+  case MMonPaxos::OP_LAST:
+    handle_last(op);
+    break;
+  case MMonPaxos::OP_BEGIN:
+    handle_begin(op);
+    break;
+  case MMonPaxos::OP_ACCEPT:
+    handle_accept(op);
+    break;		
+  case MMonPaxos::OP_COMMIT:
+    handle_commit(op);
+    break;
+  case MMonPaxos::OP_LEASE:
+    handle_lease(op);
+    break;
+  case MMonPaxos::OP_LEASE_ACK:
+    handle_lease_ack(op);
+    break;
+  default:
+    ceph_abort();
+  }
+}
+
+
+// -----------------
+// service interface
+
+// -- READ --
+
+bool Paxos::is_readable(version_t v)
+{
+  bool ret;
+  if (v > last_committed)
+    ret = false;
+  else
+    ret =
+      (mon.is_peon() || mon.is_leader()) &&
+      (is_active() || is_updating() || is_writing()) &&
+      last_committed > 0 && is_lease_valid(); // must have a value alone, or have lease
+  dout(5) << __func__ << " = " << (int)ret
+	  << " - now=" << ceph_clock_now()
+	  << " lease_expire=" << lease_expire
+	  << " has v" << v << " lc " << last_committed
+	  << dendl;
+  return ret;
+}
+
+bool Paxos::read(version_t v, bufferlist &bl)
+{
+  if (!get_store()->get(get_name(), v, bl))
+    return false;
+  return true;
+}
+
+version_t Paxos::read_current(bufferlist &bl)
+{
+  if (read(last_committed, bl))
+    return last_committed;
+  return 0;
+}
+
+
+bool Paxos::is_lease_valid()
+{
+  return ((mon.get_quorum().size() == 1)
+	  || (ceph::real_clock::now() < lease_expire));
+}
+
+// -- WRITE --
+
+bool Paxos::is_writeable()
+{
+  return
+    mon.is_leader() &&
+    is_active() &&
+    is_lease_valid();
+}
+
+void Paxos::propose_pending()
+{
+  ceph_assert(is_active());
+  ceph_assert(pending_proposal);
+
+  cancel_events();
+
+  bufferlist bl;
+  pending_proposal->encode(bl);
+
+  dout(10) << __func__ << " " << (last_committed + 1)
+	   << " " << bl.length() << " bytes" << dendl;
+  dout(30) << __func__ << " transaction dump:\n";
+  JSONFormatter f(true);
+  pending_proposal->dump(&f);
+  f.flush(*_dout);
+  *_dout << dendl;
+
+  pending_proposal.reset();
+
+  committing_finishers.swap(pending_finishers);
+  state = STATE_UPDATING;
+  begin(bl);
+}
+
+void Paxos::queue_pending_finisher(Context *onfinished)
+{
+  dout(5) << __func__ << " " << onfinished << dendl;
+  ceph_assert(onfinished);
+  pending_finishers.push_back(onfinished);
+}
+
+MonitorDBStore::TransactionRef Paxos::get_pending_transaction()
+{
+  ceph_assert(mon.is_leader());
+  if (!pending_proposal) {
+    pending_proposal.reset(new MonitorDBStore::Transaction);
+    ceph_assert(pending_finishers.empty());
+  }
+  return pending_proposal;
+}
+
+bool Paxos::trigger_propose()
+{
+  if (plugged) {
+    dout(10) << __func__ << " plugged, not proposing now" << dendl;
+    return false;
+  } else if (is_active()) {
+    dout(10) << __func__ << " active, proposing now" << dendl;
+    propose_pending();
+    return true;
+  } else {
+    dout(10) << __func__ << " not active, will propose later" << dendl;
+    return false;
+  }
+}
+
+bool Paxos::is_consistent()
+{
+  return (first_committed <= last_committed);
+}
+
diff --git a/src/mon/Paxos.h b/src/mon/Paxos.h
new file mode 100644
index 000000000..c197f26f7
--- /dev/null
+++ b/src/mon/Paxos.h
@@ -0,0 +1,1384 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+/*
+time---->
+
+cccccccccccccccccca????????????????????????????????????????
+cccccccccccccccccca????????????????????????????????????????
+cccccccccccccccccca???????????????????????????????????????? leader
+cccccccccccccccccc????????????????????????????????????????? 
+ccccc?????????????????????????????????????????????????????? 
+
+last_committed
+
+pn_from
+pn
+
+a 12v 
+b 12v
+c 14v
+d
+e 12v
+*/
+
+/**
+ * Paxos storage layout and behavior
+ *
+ * Currently, we use a key/value store to hold all the Paxos-related data, but
+ * it can logically be depicted as this:
+ *
+ *  paxos:
+ *    first_committed -> 1
+ *     last_committed -> 4
+ *		    1 -> value_1
+ *		    2 -> value_2
+ *		    3 -> value_3
+ *		    4 -> value_4
+ *
+ * Since we are relying on a k/v store supporting atomic transactions, we can
+ * guarantee that if 'last_committed' has a value of '4', then we have up to
+ * version 4 on the store, and no more than that; the same applies to
+ * 'first_committed', which holding '1' will strictly meaning that our lowest
+ * version is 1.
+ *
+ * Each version's value (value_1, value_2, ..., value_n) is a blob of data,
+ * incomprehensible to the Paxos. These values are proposed to the Paxos on
+ * propose_new_value() and each one is a transaction encoded in a ceph::buffer::list.
+ *
+ * The Paxos will write the value to disk, associating it with its version,
+ * but will take a step further: the value shall be decoded, and the operations
+ * on that transaction shall be applied during the same transaction that will
+ * write the value's encoded ceph::buffer::list to disk. This behavior ensures that
+ * whatever is being proposed will only be available on the store when it is
+ * applied by Paxos, which will then be aware of such new values, guaranteeing
+ * the store state is always consistent without requiring shady workarounds.
+ *
+ * So, let's say that FooMonitor proposes the following transaction, neatly
+ * encoded on a ceph::buffer::list of course:
+ *
+ *  Tx_Foo
+ *    put(foo, last_committed, 3)
+ *    put(foo, 3, foo_value_3)
+ *    erase(foo, 2)
+ *    erase(foo, 1)
+ *    put(foo, first_committed, 3)
+ *
+ * And knowing that the Paxos is proposed Tx_Foo as a ceph::buffer::list, once it is
+ * ready to commit, and assuming we are now committing version 5 of the Paxos,
+ * we will do something along the lines of:
+ *
+ *  Tx proposed_tx;
+ *  proposed_tx.decode(Tx_foo_ceph::buffer::list);
+ *
+ *  Tx our_tx;
+ *  our_tx.put(paxos, last_committed, 5);
+ *  our_tx.put(paxos, 5, Tx_foo_ceph::buffer::list);
+ *  our_tx.append(proposed_tx);
+ *
+ *  store_apply(our_tx);
+ *
+ * And the store should look like this after we apply 'our_tx':
+ *
+ *  paxos:
+ *    first_committed -> 1
+ *     last_committed -> 5
+ *		    1 -> value_1
+ *		    2 -> value_2
+ *		    3 -> value_3
+ *		    4 -> value_4
+ *		    5 -> Tx_foo_ceph::buffer::list
+ *  foo:
+ *    first_committed -> 3
+ *     last_committed -> 3
+ *		    3 -> foo_value_3
+ *
+ */
+
+#ifndef CEPH_MON_PAXOS_H
+#define CEPH_MON_PAXOS_H
+
+#include "include/types.h"
+#include "mon_types.h"
+#include "include/buffer.h"
+#include "msg/msg_types.h"
+#include "include/Context.h"
+#include "common/perf_counters.h"
+#include <errno.h>
+
+#include "MonitorDBStore.h"
+#include "mon/MonOpRequest.h"
+
+class Monitor;
+class MMonPaxos;
+
+enum {
+  l_paxos_first = 45800,
+  l_paxos_start_leader,
+  l_paxos_start_peon,
+  l_paxos_restart,
+  l_paxos_refresh,
+  l_paxos_refresh_latency,
+  l_paxos_begin,
+  l_paxos_begin_keys,
+  l_paxos_begin_bytes,
+  l_paxos_begin_latency,
+  l_paxos_commit,
+  l_paxos_commit_keys,
+  l_paxos_commit_bytes,
+  l_paxos_commit_latency,
+  l_paxos_collect,
+  l_paxos_collect_keys,
+  l_paxos_collect_bytes,
+  l_paxos_collect_latency,
+  l_paxos_collect_uncommitted,
+  l_paxos_collect_timeout,
+  l_paxos_accept_timeout,
+  l_paxos_lease_ack_timeout,
+  l_paxos_lease_timeout,
+  l_paxos_store_state,
+  l_paxos_store_state_keys,
+  l_paxos_store_state_bytes,
+  l_paxos_store_state_latency,
+  l_paxos_share_state,
+  l_paxos_share_state_keys,
+  l_paxos_share_state_bytes,
+  l_paxos_new_pn,
+  l_paxos_new_pn_latency,
+  l_paxos_last,
+};
+
+
+// i am one state machine.
+/**
+ * This library is based on the Paxos algorithm, but varies in a few key ways:
+ *  1- Only a single new value is generated at a time, simplifying the recovery logic.
+ *  2- Nodes track "committed" values, and share them generously (and trustingly)
+ *  3- A 'leasing' mechanism is built-in, allowing nodes to determine when it is 
+ *     safe to "read" their copy of the last committed value.
+ *
+ * This provides a simple replication substrate that services can be built on top of.
+ * See PaxosService.h
+ */
+class Paxos {
+  /**
+   * @defgroup Paxos_h_class Paxos
+   * @{
+   */
+  /**
+   * The Monitor to which this Paxos class is associated with.
+   */
+  Monitor &mon;
+
+  /// perf counter for internal instrumentations
+  PerfCounters *logger;
+
+  void init_logger();
+
+  // my state machine info
+  const std::string paxos_name;
+
+  friend class Monitor;
+  friend class PaxosService;
+
+  std::list<std::string> extra_state_dirs;
+
+  // LEADER+PEON
+
+  // -- generic state --
+public:
+  /**
+   * @defgroup Paxos_h_states States on which the leader/peon may be.
+   * @{
+   */
+  enum {
+    /**
+     * Leader/Peon is in Paxos' Recovery state
+     */
+    STATE_RECOVERING,
+    /**
+     * Leader/Peon is idle, and the Peon may or may not have a valid lease.
+     */
+    STATE_ACTIVE,
+    /**
+     * Leader/Peon is updating to a new value.
+     */
+    STATE_UPDATING,
+    /*
+     * Leader proposing an old value
+     */
+    STATE_UPDATING_PREVIOUS,
+    /*
+     * Leader/Peon is writing a new commit.  readable, but not
+     * writeable.
+     */
+    STATE_WRITING,
+    /*
+     * Leader/Peon is writing a new commit from a previous round.
+     */
+    STATE_WRITING_PREVIOUS,
+    // leader: refresh following a commit
+    STATE_REFRESH,
+    // Shutdown after WRITING or WRITING_PREVIOUS
+    STATE_SHUTDOWN
+  };
+
+  /**
+   * Obtain state name from constant value.
+   *
+   * @note This function will raise a fatal error if @p s is not
+   *	   a valid state value.
+   *
+   * @param s State value.
+   * @return The state's name.
+   */
+  static const std::string get_statename(int s) {
+    switch (s) {
+    case STATE_RECOVERING:
+      return "recovering";
+    case STATE_ACTIVE:
+      return "active";
+    case STATE_UPDATING:
+      return "updating";
+    case STATE_UPDATING_PREVIOUS:
+      return "updating-previous";
+    case STATE_WRITING:
+      return "writing";
+    case STATE_WRITING_PREVIOUS:
+      return "writing-previous";
+    case STATE_REFRESH:
+      return "refresh";
+    case STATE_SHUTDOWN:
+      return "shutdown";
+    default:
+      return "UNKNOWN";
+    }
+  }
+
+private:
+  /**
+   * The state we are in.
+   */
+  int state;
+  /**
+   * @}
+   */
+  int commits_started = 0;
+
+  ceph::condition_variable shutdown_cond;
+
+public:
+  /**
+   * Check if we are recovering.
+   *
+   * @return 'true' if we are on the Recovering state; 'false' otherwise.
+   */
+  bool is_recovering() const { return (state == STATE_RECOVERING); }
+  /**
+   * Check if we are active.
+   *
+   * @return 'true' if we are on the Active state; 'false' otherwise.
+   */
+  bool is_active() const { return state == STATE_ACTIVE; }
+  /**
+   * Check if we are updating.
+   *
+   * @return 'true' if we are on the Updating state; 'false' otherwise.
+   */
+  bool is_updating() const { return state == STATE_UPDATING; }
+
+  /**
+   * Check if we are updating/proposing a previous value from a
+   * previous quorum
+   */
+  bool is_updating_previous() const { return state == STATE_UPDATING_PREVIOUS; }
+
+  /// @return 'true' if we are writing an update to disk
+  bool is_writing() const { return state == STATE_WRITING; }
+
+  /// @return 'true' if we are writing an update-previous to disk
+  bool is_writing_previous() const { return state == STATE_WRITING_PREVIOUS; }
+
+  /// @return 'true' if we are refreshing an update just committed
+  bool is_refresh() const { return state == STATE_REFRESH; }
+
+  /// @return 'true' if we are in the process of shutting down
+  bool is_shutdown() const { return state == STATE_SHUTDOWN; }
+
+private:
+  /**
+   * @defgroup Paxos_h_recovery_vars Common recovery-related member variables
+   * @note These variables are common to both the Leader and the Peons.
+   * @{
+   */
+  /**
+   *
+   */
+  version_t first_committed;
+  /**
+   * Last Proposal Number
+   *
+   * @todo Expand description
+   */
+  version_t last_pn;
+  /**
+   * Last committed value's version.
+   *
+   * On both the Leader and the Peons, this is the last value's version that 
+   * was accepted by a given quorum and thus committed, that this instance 
+   * knows about.
+   *
+   * @note It may not be the last committed value's version throughout the
+   *	   system. If we are a Peon, we may have not been part of the quorum
+   *	   that accepted the value, and for this very same reason we may still
+   *	   be a (couple of) version(s) behind, until we learn about the most
+   *	   recent version. This should only happen if we are not active (i.e.,
+   *	   part of the quorum), which should not happen if we are up, running
+   *	   and able to communicate with others -- thus able to be part of the
+   *	   monmap and trigger new elections.
+   */
+  version_t last_committed;
+  /**
+   * Last committed value's time.
+   *
+   * When the commit finished.
+   */
+  utime_t last_commit_time;
+  /**
+   * The last Proposal Number we have accepted.
+   *
+   * On the Leader, it will be the Proposal Number picked by the Leader 
+   * itself. On the Peon, however, it will be the proposal sent by the Leader
+   * and it will only be updated if its value is higher than the one
+   * already known by the Peon.
+   */
+  version_t accepted_pn;
+  /**
+   * The last_committed epoch of the leader at the time we accepted the last pn.
+   *
+   * This has NO SEMANTIC MEANING, and is there only for the debug output.
+   */
+  version_t accepted_pn_from;
+  /**
+   * Map holding the first committed version by each quorum member.
+   *
+   * The versions kept in this map are updated during the collect phase.
+   * When the Leader starts the collect phase, each Peon will reply with its
+   * first committed version, which will then be kept in this map.
+   */
+  std::map<int,version_t> peer_first_committed;
+  /**
+   * Map holding the last committed version by each quorum member.
+   *
+   * The versions kept in this map are updated during the collect phase.
+   * When the Leader starts the collect phase, each Peon will reply with its
+   * last committed version, which will then be kept in this map.
+   */
+  std::map<int,version_t> peer_last_committed;
+  /**
+   * @}
+   */
+
+  // active (phase 2)
+  /**
+   * @defgroup Paxos_h_active_vars Common active-related member variables
+   * @{
+   */
+  /**
+   * When does our read lease expires.
+   *
+   * Instead of performing a full commit each time a read is requested, we
+   * keep leases. Each lease will have an expiration date, which may or may
+   * not be extended. 
+   */
+  ceph::real_clock::time_point lease_expire;
+  /**
+   * List of callbacks waiting for our state to change into STATE_ACTIVE.
+   */
+  std::list<Context*> waiting_for_active;
+  /**
+   * List of callbacks waiting for the chance to read a version from us.
+   *
+   * Each entry on the list may result from an attempt to read a version that
+   * wasn't available at the time, or an attempt made during a period during
+   * which we could not satisfy the read request. The first case happens if
+   * the requested version is greater than our last committed version. The
+   * second scenario may happen if we are recovering, or if we don't have a
+   * valid lease.
+   *
+   * The list will be woken up once we change to STATE_ACTIVE with an extended
+   * lease -- which can be achieved if we have everyone on the quorum on board
+   * with the latest proposal, or if we don't really care about the remaining
+   * uncommitted values --, or if we're on a quorum of one.
+   */
+  std::list<Context*> waiting_for_readable;
+  /**
+   * @}
+   */
+
+  // -- leader --
+  // recovery (paxos phase 1)
+  /**
+   * @defgroup Paxos_h_leader_recovery Leader-specific Recovery-related vars
+   * @{
+   */
+  /**
+   * Number of replies to the collect phase we've received so far.
+   *
+   * This variable is reset to 1 each time we start a collect phase; it is
+   * incremented each time we receive a reply to the collect message, and
+   * is used to determine whether or not we have received replies from the
+   * whole quorum.
+   */
+  unsigned   num_last;
+  /**
+   * Uncommitted value's version.
+   *
+   * If we have, or end up knowing about, an uncommitted value, then its
+   * version will be kept in this variable.
+   *
+   * @note If this version equals @p last_committed+1 when we reach the final
+   *	   steps of recovery, then the algorithm will assume this is a value
+   *	   the Leader does not know about, and trustingly the Leader will 
+   *	   propose this version's value.
+   */
+  version_t  uncommitted_v;
+  /**
+   * Uncommitted value's Proposal Number.
+   *
+   * We use this variable to assess if the Leader should take into consideration
+   * an uncommitted value sent by a Peon. Given that the Peon will send back to
+   * the Leader the last Proposal Number it accepted, the Leader will be able
+   * to infer if this value is more recent than the one the Leader has, thus
+   * more relevant.
+   */
+  version_t  uncommitted_pn;
+  /**
+   * Uncommitted Value.
+   *
+   * If the system fails in-between the accept replies from the Peons and the
+   * instruction to commit from the Leader, then we may end up with accepted
+   * but yet-uncommitted values. During the Leader's recovery, it will attempt
+   * to bring the whole system to the latest state, and that means committing
+   * past accepted but uncommitted values.
+   *
+   * This variable will hold an uncommitted value, which may originate either
+   * on the Leader, or learnt by the Leader from a Peon during the collect
+   * phase.
+   */
+  ceph::buffer::list uncommitted_value;
+  /**
+   * Used to specify when an on-going collect phase times out.
+   */
+  Context    *collect_timeout_event;
+  /**
+   * @}
+   */
+
+  // active
+  /**
+   * @defgroup Paxos_h_leader_active Leader-specific Active-related vars
+   * @{
+   */
+  /**
+   * Set of participants (Leader & Peons) that have acked a lease extension.
+   *
+   * Each Peon that acknowledges a lease extension will have its place in this
+   * set, which will be used to account for all the acks from all the quorum
+   * members, guaranteeing that we trigger new elections if some don't ack in
+   * the expected timeframe.
+   */
+  std::set<int>   acked_lease;
+  /**
+   * Callback responsible for extending the lease periodically.
+   */
+  Context    *lease_renew_event;
+  /**
+   * Callback to trigger new elections once the time for acks is out.
+   */
+  Context    *lease_ack_timeout_event;
+  /**
+   * @}
+   */
+  /**
+   * @defgroup Paxos_h_peon_active Peon-specific Active-related vars
+   * @{
+   */
+  /**
+   * Callback to trigger new elections when the Peon's lease times out.
+   *
+   * If the Peon's lease is extended, this callback will be reset (i.e.,
+   * we cancel the event and reschedule a new one with starting from the
+   * beginning).
+   */
+  Context    *lease_timeout_event;
+  /**
+   * @}
+   */
+
+  // updating (paxos phase 2)
+  /**
+   * @defgroup Paxos_h_leader_updating Leader-specific Updating-related vars
+   * @{
+   */
+  /**
+   * New Value being proposed to the Peons.
+   *
+   * This ceph::buffer::list holds the value the Leader is proposing to the Peons, and
+   * that will be committed if the Peons do accept the proposal.
+   */
+  ceph::buffer::list new_value;
+  /**
+   * Set of participants (Leader & Peons) that accepted the new proposed value.
+   *
+   * This set is used to keep track of those who have accepted the proposed
+   * value, so the leader may know when to issue a commit (when a majority of
+   * participants has accepted the proposal), and when to extend the lease
+   * (when all the quorum members have accepted the proposal).
+   */
+  std::set<int>   accepted;
+  /**
+   * Callback to trigger a new election if the proposal is not accepted by the
+   * full quorum within a given timeframe.
+   *
+   * If the full quorum does not accept the proposal, then it means that the
+   * Leader may no longer be recognized as the leader, or that the quorum has
+   * changed, and the value may have not reached all the participants. Thus,
+   * the leader must call new elections, and go through a recovery phase in
+   * order to propagate the new value throughout the system.
+   *
+   * This does not mean that we won't commit. We will commit as soon as we
+   * have a majority of acceptances. But if we do not have full acceptance
+   * from the quorum, then we cannot extend the lease, as some participants
+   * may not have the latest committed value.
+   */
+  Context    *accept_timeout_event;
+
+  /**
+   * List of callbacks waiting for it to be possible to write again.
+   *
+   * @remarks It is not possible to write if we are not the Leader, or we are
+   *	      not on the active state, or if the lease has expired.
+   */
+  std::list<Context*> waiting_for_writeable;
+
+  /**
+   * Pending proposal transaction
+   *
+   * This is the transaction that is under construction and pending
+   * proposal.  We will add operations to it until we decide it is
+   * time to start a paxos round.
+   */
+  MonitorDBStore::TransactionRef pending_proposal;
+
+  /**
+   * Finishers for pending transaction
+   *
+   * These are waiting for updates in the pending proposal/transaction
+   * to be committed.
+   */
+  std::list<Context*> pending_finishers;
+
+  /**
+   * Finishers for committing transaction
+   *
+   * When the pending_proposal is submitted, pending_finishers move to
+   * this list.  When it commits, these finishers are notified.
+   */
+  std::list<Context*> committing_finishers;
+  /**
+   * This function re-triggers pending_ and committing_finishers
+   * safely, so as to maintain existing system invariants. In particular
+   * we maintain ordering by triggering committing before pending, and
+   * we clear out pending_finishers prior to any triggers so that
+   * we don't trigger asserts on them being empty. You should
+   * use it instead of sending -EAGAIN to them with finish_contexts.
+   */
+  void reset_pending_committing_finishers();
+
+  /**
+   * @defgroup Paxos_h_sync_warns Synchronization warnings
+   * @todo Describe these variables
+   * @{
+   */
+  utime_t last_clock_drift_warn;
+  int clock_drift_warned;
+  /**
+   * @}
+   */
+
+  /**
+   * Should be true if we have proposed to trim, or are in the middle of
+   * trimming; false otherwise.
+   */
+  bool trimming;
+
+  /**
+   * true if we want trigger_propose to *not* propose (yet)
+   */
+  bool plugged = false;
+
+  /**
+   * @defgroup Paxos_h_callbacks Callback classes.
+   * @{
+   */
+  /**
+   * Callback class responsible for handling a Collect Timeout.
+   */
+  class C_CollectTimeout;
+  /**
+   * Callback class responsible for handling an Accept Timeout.
+   */
+  class C_AcceptTimeout;
+  /**
+   * Callback class responsible for handling a Lease Ack Timeout.
+   */
+  class C_LeaseAckTimeout;
+
+  /**
+   * Callback class responsible for handling a Lease Timeout.
+   */
+  class C_LeaseTimeout;
+
+  /**
+   * Callback class responsible for handling a Lease Renew Timeout.
+   */
+  class C_LeaseRenew;
+
+  class C_Trimmed;
+  /**
+   *
+   */
+public:
+  class C_Proposal : public Context {
+    Context *proposer_context;
+  public:
+    ceph::buffer::list bl;
+    // for debug purposes. Will go away. Soon.
+    bool proposed;
+    utime_t proposal_time;
+
+    C_Proposal(Context *c, ceph::buffer::list& proposal_bl) :
+	proposer_context(c),
+	bl(proposal_bl),
+	proposed(false),
+	proposal_time(ceph_clock_now())
+      { }
+
+    void finish(int r) override {
+      if (proposer_context) {
+	proposer_context->complete(r);
+	proposer_context = NULL;
+      }
+    }
+  };
+  /**
+   * @}
+   */
+private:
+  /**
+   * @defgroup Paxos_h_election_triggered Steps triggered by an election.
+   *
+   * @note All these functions play a significant role in the Recovery Phase,
+   *	   which is triggered right after an election once someone becomes
+   *	   the Leader.
+   * @{
+   */
+  /**
+   * Create a new Proposal Number and propose it to the Peons.
+   *
+   * This function starts the Recovery Phase, which can be directly mapped
+   * onto the original Paxos' Prepare phase. Basically, we'll generate a
+   * Proposal Number, taking @p oldpn into consideration, and we will send
+   * it to a quorum, along with our first and last committed versions. By
+   * sending these information in a message to the quorum, we expect to
+   * obtain acceptances from a majority, allowing us to commit, or be
+   * informed of a higher Proposal Number known by one or more of the Peons
+   * in the quorum.
+   *
+   * @pre We are the Leader.
+   * @post Recovery Phase initiated by sending messages to the quorum.
+   *
+   * @param oldpn A proposal number taken as the highest known so far, that
+   *		  should be taken into consideration when generating a new 
+   *		  Proposal Number for the Recovery Phase.
+   */
+  void collect(version_t oldpn);
+  /**
+   * Handle the reception of a collect message from the Leader and reply
+   * accordingly.
+   *
+   * Once a Peon receives a collect message from the Leader it will reply
+   * with its first and last committed versions, as well as information so
+   * the Leader may know if its Proposal Number was, or was not, accepted by
+   * the Peon. The Peon will accept the Leader's Proposal Number if it is
+   * higher than the Peon's currently accepted Proposal Number. The Peon may
+   * also inform the Leader of accepted but uncommitted values.
+   *
+   * @invariant The message is an operation of type OP_COLLECT.
+   * @pre We are a Peon.
+   * @post Replied to the Leader, accepting or not accepting its PN.
+   *
+   * @param collect The collect message sent by the Leader to the Peon.
+   */
+  void handle_collect(MonOpRequestRef op);
+  /**
+   * Handle a response from a Peon to the Leader's collect phase.
+   *
+   * The received message will state the Peon's last committed version, as 
+   * well as its last proposal number. This will lead to one of the following
+   * scenarios: if the replied Proposal Number is equal to the one we proposed,
+   * then the Peon has accepted our proposal, and if all the Peons do accept
+   * our Proposal Number, then we are allowed to proceed with the commit;
+   * however, if a Peon replies with a higher Proposal Number, we assume he
+   * knows something we don't and the Leader will have to abort the current
+   * proposal in order to retry with the Proposal Number specified by the Peon.
+   * It may also occur that the Peon replied with a lower Proposal Number, in
+   * which case we assume it is a reply to an older value and we'll simply
+   * drop it.
+   * This function will also check if the Peon replied with an accepted but
+   * yet uncommitted value. In this case, if its version is higher than our
+   * last committed value by one, we assume that the Peon knows a value from a
+   * previous proposal that has never been committed, and we should try to
+   * commit that value by proposing it next. On the other hand, if that is
+   * not the case, we'll assume it is an old, uncommitted value, we do not
+   * care about and we'll consider the system active by extending the leases.
+   *
+   * @invariant The message is an operation of type OP_LAST.
+   * @pre We are the Leader.
+   * @post We initiate a commit, or we retry with a higher Proposal Number, 
+   *	   or we drop the message.
+   * @post We move from STATE_RECOVERING to STATE_ACTIVE.
+   *
+   * @param last The message sent by the Peon to the Leader.
+   */
+  void handle_last(MonOpRequestRef op);
+  /**
+   * The Recovery Phase timed out, meaning that a significant part of the
+   * quorum does not believe we are the Leader, and we thus should trigger new
+   * elections.
+   *
+   * @pre We believe to be the Leader.
+   * @post Trigger new elections.
+   */
+  void collect_timeout();
+  /**
+   * @}
+   */
+
+  /**
+   * @defgroup Paxos_h_updating_funcs Functions used during the Updating State
+   * 
+   * These functions may easily be mapped to the original Paxos Algorithm's 
+   * phases. 
+   *
+   * Taking into account the algorithm can be divided in 4 phases (Prepare,
+   * Promise, Accept Request and Accepted), we can easily map Paxos::begin to
+   * both the Prepare and Accept Request phases; the Paxos::handle_begin to
+   * the Promise phase; and the Paxos::handle_accept to the Accepted phase.
+   * @{
+   */
+  /**
+   * Start a new proposal with the intent of committing @p value.
+   *
+   * If we are alone on the system (i.e., a quorum of one), then we will
+   * simply commit the value, but if we are not alone, then we need to propose
+   * the value to the quorum.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_ACTIVE
+   * @post We commit, if we are alone, or we send a message to each quorum 
+   *	   member
+   * @post We are on STATE_ACTIVE, if we are alone, or on 
+   *	   STATE_UPDATING otherwise
+   *
+   * @param value The value being proposed to the quorum
+   */
+  void begin(ceph::buffer::list& value);
+  /**
+   * Accept or decline (by ignoring) a proposal from the Leader.
+   *
+   * We will decline the proposal (by ignoring it) if we have promised to
+   * accept a higher numbered proposal. If that is not the case, we will
+   * accept it and accordingly reply to the Leader.
+   *
+   * @pre We are a Peon
+   * @pre We are on STATE_ACTIVE
+   * @post We are on STATE_UPDATING if we accept the Leader's proposal
+   * @post We send a reply message to the Leader if we accept its proposal
+   *
+   * @invariant The received message is an operation of type OP_BEGIN
+   *
+   * @param begin The message sent by the Leader to the Peon during the
+   *		  Paxos::begin function
+   *
+   */
+  void handle_begin(MonOpRequestRef op);
+  /**
+   * Handle an Accept message sent by a Peon.
+   *
+   * In order to commit, the Leader has to receive accepts from a majority of
+   * the quorum. If that does happen, then the Leader may proceed with the
+   * commit. However, the Leader needs the accepts from all the quorum members
+   * in order to extend the lease and move on to STATE_ACTIVE.
+   *
+   * This function handles these two situations, accounting for the amount of
+   * received accepts.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_UPDATING
+   * @post We are on STATE_ACTIVE if we received accepts from the full quorum
+   * @post We extended the lease if we moved on to STATE_ACTIVE
+   * @post We are on STATE_UPDATING if we didn't received accepts from the
+   *	   full quorum
+   * @post We have committed if we received accepts from a majority
+   *
+   * @invariant The received message is an operation of type OP_ACCEPT
+   *
+   * @param accept The message sent by the Peons to the Leader during the
+   *		   Paxos::handle_begin function
+   */
+  void handle_accept(MonOpRequestRef op);
+  /**
+   * Trigger a fresh election.
+   *
+   * During Paxos::begin we set a Callback of type Paxos::C_AcceptTimeout in
+   * order to limit the amount of time we spend waiting for Accept replies.
+   * This callback will call Paxos::accept_timeout when it is fired.
+   *
+   * This is essential to the algorithm because there may be the chance that
+   * we are no longer the Leader (i.e., others don't believe in us) and we
+   * are getting ignored, or we dropped out of the quorum and haven't realised
+   * it. So, our only option is to trigger fresh elections.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_UPDATING
+   * @post Triggered fresh elections
+   */
+  void accept_timeout();
+  /**
+   * @}
+   */
+
+
+  utime_t commit_start_stamp;
+  friend struct C_Committed;
+
+  /**
+   * Commit a value throughout the system.
+   *
+   * The Leader will cancel the current lease (as it was for the old value),
+   * and will store the committed value locally. It will then instruct every
+   * quorum member to do so as well.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_UPDATING
+   * @pre A majority of quorum members accepted our proposal
+   * @post Value locally stored
+   * @post Quorum members instructed to commit the new value.
+   */
+  void commit_start();
+  void commit_finish();   ///< finish a commit after txn becomes durable
+  void abort_commit();    ///< Handle commit finish after shutdown started
+  /**
+   * Commit the new value to stable storage as being the latest available
+   * version.
+   *
+   * @pre We are a Peon
+   * @post The new value is locally stored
+   * @post Fire up the callbacks waiting on waiting_for_commit
+   *
+   * @invariant The received message is an operation of type OP_COMMIT
+   *
+   * @param commit The message sent by the Leader to the Peon during
+   *		   Paxos::commit
+   */
+  void handle_commit(MonOpRequestRef op);
+  /**
+   * Extend the system's lease.
+   *
+   * This means that the Leader considers that it should now safe to read from
+   * any node on the system, since every quorum member is now in possession of
+   * the latest version. Therefore, the Leader will send a message stating just
+   * this to each quorum member, and will impose a limited timeframe during
+   * which acks will be accepted. If there aren't as many acks as expected
+   * (i.e, if at least one quorum member does not ack the lease) during this
+   * timeframe, then we will force fresh elections.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_ACTIVE
+   * @post A message extending the lease is sent to each quorum member
+   * @post A timeout callback is set to limit the amount of time we will wait
+   *	   for lease acks.
+   * @post A timer is set in order to renew the lease after a certain amount
+   *	   of time.
+   */
+  void extend_lease();
+  /**
+   * Update the lease on the Peon's side of things.
+   *
+   * Once a Peon receives a Lease message, it will update its lease_expire
+   * variable, reply to the Leader acknowledging the lease update and set a
+   * timeout callback to be fired upon the lease's expiration. Finally, the
+   * Peon will fire up all the callbacks waiting for it to become active,
+   * which it just did, and all those waiting for it to become readable,
+   * which should be true if the Peon's lease didn't expire in the mean time.
+   *
+   * @pre We are a Peon
+   * @post We update the lease accordingly
+   * @post A lease timeout callback is set
+   * @post Move to STATE_ACTIVE
+   * @post Fire up all the callbacks waiting for STATE_ACTIVE
+   * @post Fire up all the callbacks waiting for readable if we are readable
+   * @post Ack the lease to the Leader
+   *
+   * @invariant The received message is an operation of type OP_LEASE
+   *
+   * @param lease The message sent by the Leader to the Peon during the
+   *	    Paxos::extend_lease function
+   */
+  void handle_lease(MonOpRequestRef op);
+  /**
+   * Account for all the Lease Acks the Leader receives from the Peons.
+   *
+   * Once the Leader receives all the Lease Acks from the Peons, it will be
+   * able to cancel the Lease Ack timeout callback, thus avoiding calling
+   * fresh elections.
+   *
+   * @pre We are the Leader
+   * @post Cancel the Lease Ack timeout callback if we receive acks from all
+   *	   the quorum members
+   *
+   * @invariant The received message is an operation of type OP_LEASE_ACK
+   *
+   * @param ack The message sent by a Peon to the Leader during the
+   *		Paxos::handle_lease function
+   */
+  void handle_lease_ack(MonOpRequestRef op);
+  /**
+   * Call fresh elections because at least one Peon didn't acked our lease.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_ACTIVE
+   * @post Trigger fresh elections
+   */
+  void lease_ack_timeout();
+  /**
+   * Extend lease since we haven't had new committed values meanwhile.
+   *
+   * @pre We are the Leader
+   * @pre We are on STATE_ACTIVE
+   * @post Go through with Paxos::extend_lease
+   */
+  void lease_renew_timeout();
+  /**
+   * Call fresh elections because the Peon's lease expired without being
+   * renewed or receiving a fresh lease.
+   *
+   * This means that the Peon is no longer assumed as being in the quorum
+   * (or there is no Leader to speak of), so just trigger fresh elections
+   * to circumvent this issue.
+   *
+   * @pre We are a Peon
+   * @post Trigger fresh elections
+   */
+  void lease_timeout();        // on peon, if lease isn't extended
+
+  /// restart the lease timeout timer
+  void reset_lease_timeout();
+
+  /**
+   * Cancel all of Paxos' timeout/renew events. 
+   */
+  void cancel_events();
+  /**
+   * Shutdown this Paxos machine
+   */
+  void shutdown();
+
+  /**
+   * Generate a new Proposal Number based on @p gt
+   *
+   * @todo Check what @p gt actually means and what its usage entails
+   * @param gt A hint for the geration of the Proposal Number
+   * @return A globally unique, monotonically increasing Proposal Number
+   */
+  version_t get_new_proposal_number(version_t gt=0);
+ 
+  /**
+   * @todo document sync function
+   */
+  void warn_on_future_time(utime_t t, entity_name_t from);
+
+  /**
+   * Begin proposing the pending_proposal.
+   */
+  void propose_pending();
+
+  /**
+   * refresh state from store
+   *
+   * Called when we have new state for the mon to consume.  If we return false,
+   * abort (we triggered a bootstrap).
+   *
+   * @returns true on success, false if we are now bootstrapping
+   */
+  bool do_refresh();
+
+  void commit_proposal();
+  void finish_round();
+
+public:
+  /**
+   * @param m A monitor
+   * @param name A name for the paxos service. It serves as the naming space
+   * of the underlying persistent storage for this service.
+   */
+  Paxos(Monitor &m, const std::string &name) 
+		 : mon(m),
+		   logger(NULL),
+		   paxos_name(name),
+		   state(STATE_RECOVERING),
+		   first_committed(0),
+		   last_pn(0),
+		   last_committed(0),
+		   accepted_pn(0),
+		   accepted_pn_from(0),
+		   num_last(0),
+		   uncommitted_v(0), uncommitted_pn(0),
+		   collect_timeout_event(0),
+		   lease_renew_event(0),
+		   lease_ack_timeout_event(0),
+		   lease_timeout_event(0),
+		   accept_timeout_event(0),
+		   clock_drift_warned(0),
+		   trimming(false) { }
+
+  ~Paxos() {
+    delete logger;
+  }
+
+  const std::string get_name() const {
+    return paxos_name;
+  }
+
+  void dispatch(MonOpRequestRef op);
+
+  void read_and_prepare_transactions(MonitorDBStore::TransactionRef tx,
+				     version_t from, version_t last);
+
+  void init();
+
+  /**
+   * dump state info to a formatter
+   */
+  void dump_info(ceph::Formatter *f);
+
+  /**
+   * This function runs basic consistency checks. Importantly, if
+   * it is inconsistent and shouldn't be, it asserts out.
+   *
+   * @return True if consistent, false if not.
+   */
+  bool is_consistent();
+
+  void restart();
+  /**
+   * Initiate the Leader after it wins an election.
+   *
+   * Once an election is won, the Leader will be initiated and there are two
+   * possible outcomes of this method: the Leader directly jumps to the active
+   * state (STATE_ACTIVE) if it believes to be the only one in the quorum, or
+   * will start recovering (STATE_RECOVERING) by initiating the collect phase. 
+   *
+   * @pre Our monitor is the Leader.
+   * @post We are either on STATE_ACTIVE if we're the only one in the quorum,
+   *	   or on STATE_RECOVERING otherwise.
+   */
+  void leader_init();
+  /**
+   * Initiate a Peon after it loses an election.
+   *
+   * If we are a Peon, then there must be a Leader and we are not alone in the
+   * quorum, thus automatically assume we are on STATE_RECOVERING, which means
+   * we will soon be enrolled into the Leader's collect phase.
+   *
+   * @pre There is a Leader, and it?s about to start the collect phase.
+   * @post We are on STATE_RECOVERING and will soon receive collect phase's 
+   *	   messages.
+   */
+  void peon_init();
+
+  /**
+   * Include an incremental state of values, ranging from peer_first_committed
+   * to the last committed value, on the message m
+   *
+   * @param m A message
+   * @param peer_first_committed Lowest version to take into account
+   * @param peer_last_committed Highest version to take into account
+   */
+  void share_state(MMonPaxos *m, version_t peer_first_committed,
+		   version_t peer_last_committed);
+  /**
+   * Store on disk a state that was shared with us
+   *
+   * Basically, we received a set of version. Or just one. It doesn't matter.
+   * What matters is that we have to stash it in the store. So, we will simply
+   * write every single ceph::buffer::list into their own versions on our side (i.e.,
+   * onto paxos-related keys), and then we will decode those same ceph::buffer::lists
+   * we just wrote and apply the transactions they hold. We will also update
+   * our first and last committed values to point to the new values, if need
+   * be. All this is done tightly wrapped in a transaction to ensure we
+   * enjoy the atomicity guarantees given by our awesome k/v store.
+   *
+   * @param m A message
+   * @returns true if we stored something new; false otherwise
+   */
+  bool store_state(MMonPaxos *m);
+  void _sanity_check_store();
+
+  /**
+   * Helper function to decode a ceph::buffer::list into a transaction and append it
+   * to another transaction.
+   *
+   * This function is used during the Leader's commit and during the
+   * Paxos::store_state in order to apply the ceph::buffer::list's transaction onto
+   * the store.
+   *
+   * @param t The transaction to which we will append the operations
+   * @param bl A ceph::buffer::list containing an encoded transaction
+   */
+  static void decode_append_transaction(MonitorDBStore::TransactionRef t,
+					ceph::buffer::list& bl) {
+    auto vt(std::make_shared<MonitorDBStore::Transaction>());
+    auto it = bl.cbegin();
+    vt->decode(it);
+    t->append(vt);
+  }
+
+  /**
+   * @todo This appears to be used only by the OSDMonitor, and I would say
+   *	   its objective is to allow a third-party to have a "private"
+   *	   state dir. -JL
+   */
+  void add_extra_state_dir(std::string s) {
+    extra_state_dirs.push_back(s);
+  }
+
+  // -- service interface --
+  /**
+   * Add c to the list of callbacks waiting for us to become active.
+   *
+   * @param c A callback
+   */
+  void wait_for_active(MonOpRequestRef op, Context *c) {
+    if (op)
+      op->mark_event("paxos:wait_for_active");
+    waiting_for_active.push_back(c);
+  }
+  void wait_for_active(Context *c) {
+    MonOpRequestRef o;
+    wait_for_active(o, c);
+  }
+
+  /**
+   * Trim the Paxos state as much as we can.
+   */
+  void trim();
+
+  /**
+   * Check if we should trim.
+   *
+   * If trimming is disabled, we must take that into consideration and only
+   * return true if we are positively sure that we should trim soon.
+   *
+   * @returns true if we should trim; false otherwise.
+   */
+  bool should_trim() {
+    int available_versions = get_version() - get_first_committed();
+    int maximum_versions = g_conf()->paxos_min + g_conf()->paxos_trim_min;
+
+    if (trimming || (available_versions <= maximum_versions))
+      return false;
+
+    return true;
+  }
+
+  bool is_plugged() const {
+    return plugged;
+  }
+  void plug() {
+    ceph_assert(plugged == false);
+    plugged = true;
+  }
+  void unplug() {
+    ceph_assert(plugged == true);
+    plugged = false;
+  }
+
+  // read
+  /**
+   * @defgroup Paxos_h_read_funcs Read-related functions
+   * @{
+   */
+  /**
+   * Get latest committed version
+   *
+   * @return latest committed version
+   */
+  version_t get_version() { return last_committed; }
+  /**
+   * Get first committed version
+   *
+   * @return the first committed version
+   */
+  version_t get_first_committed() { return first_committed; }
+  /**
+   * Check if a given version is readable.
+   *
+   * A version may not be readable for a myriad of reasons:
+   *  @li the version @e v is higher that the last committed version
+   *  @li we are not the Leader nor a Peon (election may be on-going)
+   *  @li we do not have a committed value yet
+   *  @li we do not have a valid lease
+   *
+   * @param seen The version we want to check if it is readable.
+   * @return 'true' if the version is readable; 'false' otherwise.
+   */
+  bool is_readable(version_t seen=0);
+  /**
+   * Read version @e v and store its value in @e bl
+   *
+   * @param[in] v The version we want to read
+   * @param[out] bl The version's value
+   * @return 'true' if we successfully read the value; 'false' otherwise
+   */
+  bool read(version_t v, ceph::buffer::list &bl);
+  /**
+   * Read the latest committed version
+   *
+   * @param[out] bl The version's value
+   * @return the latest committed version if we successfully read the value;
+   *	     or 0 (zero) otherwise.
+   */
+  version_t read_current(ceph::buffer::list &bl);
+  /**
+   * Add onreadable to the list of callbacks waiting for us to become readable.
+   *
+   * @param onreadable A callback
+   */
+  void wait_for_readable(MonOpRequestRef op, Context *onreadable) {
+    ceph_assert(!is_readable());
+    if (op)
+      op->mark_event("paxos:wait_for_readable");
+    waiting_for_readable.push_back(onreadable);
+  }
+  void wait_for_readable(Context *onreadable) {
+    MonOpRequestRef o;
+    wait_for_readable(o, onreadable);
+  }
+  /**
+   * @}
+   */
+
+  /**
+   * Check if we have a valid lease.
+   *
+   * @returns true if the lease is still valid; false otherwise.
+   */
+  bool is_lease_valid();
+  // write
+  /**
+   * @defgroup Paxos_h_write_funcs Write-related functions
+   * @{
+   */
+  /**
+   * Check if we are writeable.
+   *
+   * We are writeable if we are alone (i.e., a quorum of one), or if we match
+   * all the following conditions:
+   *  @li We are the Leader
+   *  @li We are on STATE_ACTIVE
+   *  @li We have a valid lease
+   *
+   * @return 'true' if we are writeable; 'false' otherwise.
+   */
+  bool is_writeable();
+  /**
+   * Add c to the list of callbacks waiting for us to become writeable.
+   *
+   * @param c A callback
+   */
+  void wait_for_writeable(MonOpRequestRef op, Context *c) {
+    ceph_assert(!is_writeable());
+    if (op)
+      op->mark_event("paxos:wait_for_writeable");
+    waiting_for_writeable.push_back(c);
+  }
+  void wait_for_writeable(Context *c) {
+    MonOpRequestRef o;
+    wait_for_writeable(o, c);
+  }
+
+  /**
+   * Get a transaction to submit operations to propose against
+   *
+   * Apply operations to this transaction.  It will eventually be proposed
+   * to paxos.
+   */
+  MonitorDBStore::TransactionRef get_pending_transaction();
+
+  /**
+   * Queue a completion for the pending proposal
+   *
+   * This completion will get triggered when the pending proposal
+   * transaction commits.
+   */
+  void queue_pending_finisher(Context *onfinished);
+
+  /**
+   * (try to) trigger a proposal
+   *
+   * Tell paxos that it should submit the pending proposal.  Note that if it
+   * is not active (e.g., because it is already in the midst of committing
+   * something) that will be deferred (e.g., until the current round finishes).
+   */
+  bool trigger_propose();
+  /**
+   * @}
+   */
+
+  /**
+   * @}
+   */
+ protected:
+  MonitorDBStore *get_store();
+};
+
+inline std::ostream& operator<<(std::ostream& out, Paxos::C_Proposal& p)
+{
+  std::string proposed = (p.proposed ? "proposed" : "unproposed");
+  out << " " << proposed
+      << " queued " << (ceph_clock_now() - p.proposal_time)
+      << " tx dump:\n";
+  auto t(std::make_shared<MonitorDBStore::Transaction>());
+  auto p_it = p.bl.cbegin();
+  t->decode(p_it);
+  ceph::JSONFormatter f(true);
+  t->dump(&f);
+  f.flush(out);
+  return out;
+}
+
+#endif
diff --git a/src/mon/PaxosFSMap.h b/src/mon/PaxosFSMap.h
new file mode 100644
index 000000000..e32c44e0b
--- /dev/null
+++ b/src/mon/PaxosFSMap.h
@@ -0,0 +1,54 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_PAXOS_FSMAP_H
+#define CEPH_PAXOS_FSMAP_H
+
+#include "mds/FSMap.h"
+#include "mds/MDSMap.h"
+
+#include "include/ceph_assert.h"
+
+class PaxosFSMap {
+public:
+  virtual ~PaxosFSMap() {}
+
+  const FSMap &get_pending_fsmap() const { ceph_assert(is_leader()); return pending_fsmap; }
+  const FSMap &get_fsmap() const { return fsmap; }
+
+  virtual bool is_leader() const = 0;
+
+protected:
+  FSMap &get_pending_fsmap_writeable() { ceph_assert(is_leader()); return pending_fsmap; }
+
+  FSMap &create_pending() {
+    ceph_assert(is_leader());
+    pending_fsmap = fsmap;
+    pending_fsmap.epoch++;
+    return pending_fsmap;
+  }
+
+  void decode(ceph::buffer::list &bl) {
+    fsmap.decode(bl);
+    pending_fsmap = FSMap(); /* nuke it to catch invalid access */
+  }
+
+private:
+  /* Keep these PRIVATE to prevent unprotected manipulation. */
+  FSMap fsmap; /* the current epoch */
+  FSMap pending_fsmap; /* the next epoch */
+};
+
+
+#endif
diff --git a/src/mon/PaxosService.cc b/src/mon/PaxosService.cc
new file mode 100644
index 000000000..0a6a9a9ea
--- /dev/null
+++ b/src/mon/PaxosService.cc
@@ -0,0 +1,466 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "PaxosService.h"
+#include "common/Clock.h"
+#include "common/config.h"
+#include "include/stringify.h"
+#include "include/ceph_assert.h"
+#include "mon/MonOpRequest.h"
+
+using std::ostream;
+using std::string;
+
+using ceph::bufferlist;
+
+#define dout_subsys ceph_subsys_paxos
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, paxos, service_name, get_first_committed(), get_last_committed())
+static ostream& _prefix(std::ostream *_dout, Monitor &mon, Paxos &paxos, string service_name,
+			version_t fc, version_t lc) {
+  return *_dout << "mon." << mon.name << "@" << mon.rank
+		<< "(" << mon.get_state_name()
+		<< ").paxosservice(" << service_name << " " << fc << ".." << lc << ") ";
+}
+
+bool PaxosService::dispatch(MonOpRequestRef op)
+{
+  ceph_assert(op->is_type_service() || op->is_type_command());
+  auto m = op->get_req<PaxosServiceMessage>();
+  op->mark_event("psvc:dispatch");
+
+  dout(10) << __func__ << " " << m << " " << *m
+	   << " from " << m->get_orig_source_inst()
+	   << " con " << m->get_connection() << dendl;
+
+  if (mon.is_shutdown()) {
+    return true;
+  }
+
+  // make sure this message isn't forwarded from a previous election epoch
+  if (m->rx_election_epoch &&
+      m->rx_election_epoch < mon.get_epoch()) {
+    dout(10) << " discarding forwarded message from previous election epoch "
+	     << m->rx_election_epoch << " < " << mon.get_epoch() << dendl;
+    return true;
+  }
+
+  // make sure the client is still connected.  note that a proxied
+  // connection will be disconnected with a null message; don't drop
+  // those.  also ignore loopback (e.g., log) messages.
+  if (m->get_connection() &&
+      !m->get_connection()->is_connected() &&
+      m->get_connection() != mon.con_self &&
+      m->get_connection()->get_messenger() != NULL) {
+    dout(10) << " discarding message from disconnected client "
+	     << m->get_source_inst() << " " << *m << dendl;
+    return true;
+  }
+
+  // make sure our map is readable and up to date
+  if (!is_readable(m->version)) {
+    dout(10) << " waiting for paxos -> readable (v" << m->version << ")" << dendl;
+    wait_for_readable(op, new C_RetryMessage(this, op), m->version);
+    return true;
+  }
+
+  // preprocess
+  if (preprocess_query(op)) 
+    return true;  // easy!
+
+  // leader?
+  if (!mon.is_leader()) {
+    mon.forward_request_leader(op);
+    return true;
+  }
+  
+  // writeable?
+  if (!is_writeable()) {
+    dout(10) << " waiting for paxos -> writeable" << dendl;
+    wait_for_writeable(op, new C_RetryMessage(this, op));
+    return true;
+  }
+
+  // update
+  if (!prepare_update(op)) {
+    // no changes made.
+    return true;
+  }
+
+  if (need_immediate_propose) {
+    dout(10) << __func__ << " forced immediate propose" << dendl;
+    need_immediate_propose = false;
+    propose_pending();
+    return true;
+  }
+
+  double delay = 0.0;
+  if (!should_propose(delay)) {
+    dout(10) << " not proposing" << dendl;
+    return true;
+  }
+
+  if (delay == 0.0) {
+    propose_pending();
+    return true;
+  }
+
+  // delay a bit
+  if (!proposal_timer) {
+    /**
+       * Callback class used to propose the pending value once the proposal_timer
+       * fires up.
+       */
+    auto do_propose = new C_MonContext{&mon, [this](int r) {
+        proposal_timer = 0;
+        if (r >= 0) {
+          propose_pending();
+        } else if (r == -ECANCELED || r == -EAGAIN) {
+          return;
+        } else {
+          ceph_abort_msg("bad return value for proposal_timer");
+        }
+    }};
+    dout(10) << " setting proposal_timer " << do_propose
+             << " with delay of " << delay << dendl;
+    proposal_timer = mon.timer.add_event_after(delay, do_propose);
+  } else {
+    dout(10) << " proposal_timer already set" << dendl;
+  }
+  return true;
+}
+
+void PaxosService::refresh(bool *need_bootstrap)
+{
+  // update cached versions
+  cached_first_committed = mon.store->get(get_service_name(), first_committed_name);
+  cached_last_committed = mon.store->get(get_service_name(), last_committed_name);
+
+  version_t new_format = get_value("format_version");
+  if (new_format != format_version) {
+    dout(1) << __func__ << " upgraded, format " << format_version << " -> " << new_format << dendl;
+    on_upgrade();
+  }
+  format_version = new_format;
+
+  dout(10) << __func__ << dendl;
+
+  update_from_paxos(need_bootstrap);
+}
+
+void PaxosService::post_refresh()
+{
+  dout(10) << __func__ << dendl;
+
+  post_paxos_update();
+
+  if (mon.is_peon() && !waiting_for_finished_proposal.empty()) {
+    finish_contexts(g_ceph_context, waiting_for_finished_proposal, -EAGAIN);
+  }
+}
+
+bool PaxosService::should_propose(double& delay)
+{
+  // simple default policy: quick startup, then some damping.
+  if (get_last_committed() <= 1) {
+    delay = 0.0;
+  } else {
+    utime_t now = ceph_clock_now();
+    if ((now - paxos.last_commit_time) > g_conf()->paxos_propose_interval)
+      delay = (double)g_conf()->paxos_min_wait;
+    else
+      delay = (double)(g_conf()->paxos_propose_interval + paxos.last_commit_time
+		       - now);
+  }
+  return true;
+}
+
+
+void PaxosService::propose_pending()
+{
+  dout(10) << __func__ << dendl;
+  ceph_assert(have_pending);
+  ceph_assert(!proposing);
+  ceph_assert(mon.is_leader());
+  ceph_assert(is_active());
+
+  if (proposal_timer) {
+    dout(10) << " canceling proposal_timer " << proposal_timer << dendl;
+    mon.timer.cancel_event(proposal_timer);
+    proposal_timer = NULL;
+  }
+
+  /**
+   * @note What we contribute to the pending Paxos transaction is
+   *	   obtained by calling a function that must be implemented by
+   *	   the class implementing us.  I.e., the function
+   *	   encode_pending will be the one responsible to encode
+   *	   whatever is pending on the implementation class into a
+   *	   bufferlist, so we can then propose that as a value through
+   *	   Paxos.
+   */
+  MonitorDBStore::TransactionRef t = paxos.get_pending_transaction();
+
+  if (should_stash_full())
+    encode_full(t);
+
+  encode_pending(t);
+  have_pending = false;
+
+  if (format_version > 0) {
+    t->put(get_service_name(), "format_version", format_version);
+  }
+
+  // apply to paxos
+  proposing = true;
+  /**
+   * Callback class used to mark us as active once a proposal finishes going
+   * through Paxos.
+   *
+   * We should wake people up *only* *after* we inform the service we
+   * just went active. And we should wake people up only once we finish
+   * going active. This is why we first go active, avoiding to wake up the
+   * wrong people at the wrong time, such as waking up a C_RetryMessage
+   * before waking up a C_Active, thus ending up without a pending value.
+   */
+  class C_Committed : public Context {
+    PaxosService *ps;
+  public:
+    explicit C_Committed(PaxosService *p) : ps(p) { }
+    void finish(int r) override {
+      ps->proposing = false;
+      if (r >= 0)
+	ps->_active();
+      else if (r == -ECANCELED || r == -EAGAIN)
+	return;
+      else
+	ceph_abort_msg("bad return value for C_Committed");
+    }
+  };
+  paxos.queue_pending_finisher(new C_Committed(this));
+  paxos.trigger_propose();
+}
+
+bool PaxosService::should_stash_full()
+{
+  version_t latest_full = get_version_latest_full();
+  /* @note The first member of the condition is moot and it is here just for
+   *	   clarity's sake. The second member would end up returing true
+   *	   nonetheless because, in that event,
+   *	      latest_full == get_trim_to() == 0.
+   */
+  return (!latest_full ||
+	  (latest_full <= get_trim_to()) ||
+	  (get_last_committed() - latest_full > (version_t)g_conf()->paxos_stash_full_interval));
+}
+
+void PaxosService::restart()
+{
+  dout(10) << __func__ << dendl;
+  if (proposal_timer) {
+    dout(10) << " canceling proposal_timer " << proposal_timer << dendl;
+    mon.timer.cancel_event(proposal_timer);
+    proposal_timer = 0;
+  }
+
+  finish_contexts(g_ceph_context, waiting_for_finished_proposal, -EAGAIN);
+
+  if (have_pending) {
+    discard_pending();
+    have_pending = false;
+  }
+  proposing = false;
+
+  on_restart();
+}
+
+void PaxosService::election_finished()
+{
+  dout(10) << __func__ << dendl;
+
+  finish_contexts(g_ceph_context, waiting_for_finished_proposal, -EAGAIN);
+
+  // make sure we update our state
+  _active();
+}
+
+void PaxosService::_active()
+{
+  if (is_proposing()) {
+    dout(10) << __func__ << " - proposing" << dendl;
+    return;
+  }
+  if (!is_active()) {
+    dout(10) << __func__ << " - not active" << dendl;
+    /**
+     * Callback used to make sure we call the PaxosService::_active function
+     * whenever a condition is fulfilled.
+     *
+     * This is used in multiple situations, from waiting for the Paxos to commit
+     * our proposed value, to waiting for the Paxos to become active once an
+     * election is finished.
+     */
+    class C_Active : public Context {
+      PaxosService *svc;
+    public:
+      explicit C_Active(PaxosService *s) : svc(s) {}
+      void finish(int r) override {
+	if (r >= 0)
+	  svc->_active();
+      }
+    };
+    wait_for_active_ctx(new C_Active(this));
+    return;
+  }
+  dout(10) << __func__ << dendl;
+
+  // create pending state?
+  if (mon.is_leader()) {
+    dout(7) << __func__ << " creating new pending" << dendl;
+    if (!have_pending) {
+      create_pending();
+      have_pending = true;
+    }
+
+    if (get_last_committed() == 0) {
+      // create initial state
+      create_initial();
+      propose_pending();
+      return;
+    }
+  } else {
+    dout(7) << __func__ << " we are not the leader, hence we propose nothing!" << dendl;
+  }
+
+  // wake up anyone who came in while we were proposing.  note that
+  // anyone waiting for the previous proposal to commit is no longer
+  // on this list; it is on Paxos's.
+  finish_contexts(g_ceph_context, waiting_for_finished_proposal, 0);
+
+  if (mon.is_leader())
+    upgrade_format();
+
+  // NOTE: it's possible that this will get called twice if we commit
+  // an old paxos value.  Implementations should be mindful of that.
+  on_active();
+}
+
+
+void PaxosService::shutdown()
+{
+  cancel_events();
+
+  if (proposal_timer) {
+    dout(10) << " canceling proposal_timer " << proposal_timer << dendl;
+    mon.timer.cancel_event(proposal_timer);
+    proposal_timer = 0;
+  }
+
+  finish_contexts(g_ceph_context, waiting_for_finished_proposal, -EAGAIN);
+
+  on_shutdown();
+}
+
+void PaxosService::maybe_trim()
+{
+  if (!is_writeable())
+    return;
+
+  const version_t first_committed = get_first_committed();
+  version_t trim_to = get_trim_to();
+  dout(20) << __func__ << " " << first_committed << "~" << trim_to << dendl;
+
+  if (trim_to < first_committed) {
+    dout(10) << __func__ << " trim_to " << trim_to << " < first_committed "
+	     << first_committed << dendl;
+    return;
+  }
+
+  version_t to_remove = trim_to - first_committed;
+  const version_t trim_min = g_conf().get_val<version_t>("paxos_service_trim_min");
+  if (trim_min > 0 &&
+      to_remove < trim_min) {
+    dout(10) << __func__ << " trim_to " << trim_to << " would only trim " << to_remove
+	     << " < paxos_service_trim_min " << trim_min << dendl;
+    return;
+  }
+
+  to_remove = [to_remove, trim_to, this] {
+    const version_t trim_max = g_conf().get_val<version_t>("paxos_service_trim_max");
+    if (trim_max == 0 || to_remove < trim_max) {
+      return to_remove;
+    }
+    if (to_remove < trim_max * 1.5) {
+      dout(10) << __func__ << " trim to " << trim_to << " would only trim " << to_remove
+             << " > paxos_service_trim_max, limiting to " << trim_max
+             << dendl;
+      return trim_max;
+    }
+    const version_t new_trim_max = (trim_max + to_remove) / 2;
+    const uint64_t trim_max_multiplier = g_conf().get_val<uint64_t>("paxos_service_trim_max_multiplier");
+    if (trim_max_multiplier) {
+      return std::min(new_trim_max, trim_max * trim_max_multiplier);
+    } else {
+      return new_trim_max;
+    }
+  }();
+  trim_to = first_committed + to_remove;
+
+  dout(10) << __func__ << " trimming to " << trim_to << ", " << to_remove << " states" << dendl;
+  MonitorDBStore::TransactionRef t = paxos.get_pending_transaction();
+  trim(t, first_committed, trim_to);
+  put_first_committed(t, trim_to);
+  cached_first_committed = trim_to;
+
+  // let the service add any extra stuff
+  encode_trim_extra(t, trim_to);
+
+  paxos.trigger_propose();
+}
+
+void PaxosService::trim(MonitorDBStore::TransactionRef t,
+			version_t from, version_t to)
+{
+  dout(10) << __func__ << " from " << from << " to " << to << dendl;
+  ceph_assert(from != to);
+
+  for (version_t v = from; v < to; ++v) {
+    dout(20) << __func__ << " " << v << dendl;
+    t->erase(get_service_name(), v);
+
+    string full_key = mon.store->combine_strings("full", v);
+    if (mon.store->exists(get_service_name(), full_key)) {
+      dout(20) << __func__ << " " << full_key << dendl;
+      t->erase(get_service_name(), full_key);
+    }
+  }
+  if (g_conf()->mon_compact_on_trim) {
+    dout(20) << " compacting prefix " << get_service_name() << dendl;
+    t->compact_range(get_service_name(), stringify(from - 1), stringify(to));
+    t->compact_range(get_service_name(),
+		     mon.store->combine_strings(full_prefix_name, from - 1),
+		     mon.store->combine_strings(full_prefix_name, to));
+  }
+}
+
+void PaxosService::load_health()
+{
+  bufferlist bl;
+  mon.store->get("health", service_name, bl);
+  if (bl.length()) {
+    auto p = bl.cbegin();
+    using ceph::decode;
+    decode(health_checks, p);
+  }
+}
diff --git a/src/mon/PaxosService.h b/src/mon/PaxosService.h
new file mode 100644
index 000000000..93c5e7c81
--- /dev/null
+++ b/src/mon/PaxosService.h
@@ -0,0 +1,901 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_PAXOSSERVICE_H
+#define CEPH_PAXOSSERVICE_H
+
+#include "include/Context.h"
+#include "Paxos.h"
+#include "Monitor.h"
+#include "MonitorDBStore.h"
+
+/**
+ * A Paxos Service is an abstraction that easily allows one to obtain an
+ * association between a Monitor and a Paxos class, in order to implement any
+ * service.
+ */
+class PaxosService {
+  /**
+   * @defgroup PaxosService_h_class Paxos Service
+   * @{
+   */
+ public:
+  /**
+   * The Monitor to which this class is associated with
+   */
+  Monitor &mon;
+  /**
+   * The Paxos instance to which this class is associated with
+   */
+  Paxos &paxos;
+  /**
+   * Our name. This will be associated with the class implementing us, and will
+   * be used mainly for store-related operations.
+   */
+  std::string service_name;
+  /**
+   * If we are or have queued anything for proposal, this variable will be true
+   * until our proposal has been finished.
+   */
+  bool proposing;
+
+  bool need_immediate_propose = false;
+
+protected:
+  /**
+   * Services implementing us used to depend on the Paxos version, back when
+   * each service would have a Paxos instance for itself. However, now we only
+   * have a single Paxos instance, shared by all the services. Each service now
+   * must keep its own version, if so they wish. This variable should be used
+   * for that purpose.
+   */
+  version_t service_version;
+
+ private:
+  /**
+   * Event callback responsible for proposing our pending value once a timer 
+   * runs out and fires.
+   */
+  Context *proposal_timer;
+  /**
+   * If the implementation class has anything pending to be proposed to Paxos,
+   * then have_pending should be true; otherwise, false.
+   */
+  bool have_pending; 
+
+  /**
+   * health checks for this service
+   *
+   * Child must populate this during encode_pending() by calling encode_health().
+   */
+  health_check_map_t health_checks;
+protected:
+  /**
+   * format of our state in leveldb, 0 for default
+   */
+  version_t format_version;
+
+public:
+  const health_check_map_t& get_health_checks() const {
+    return health_checks;
+  }
+
+  /**
+   * @defgroup PaxosService_h_callbacks Callback classes
+   * @{
+   */
+  /**
+   * Retry dispatching a given service message
+   *
+   * This callback class is used when we had to wait for some condition to
+   * become true while we were dispatching it.
+   *
+   * For instance, if the message's version isn't readable, according to Paxos,
+   * then we must wait for it to become readable. So, we just queue an
+   * instance of this class onto the Paxos::wait_for_readable function, and
+   * we will retry the whole dispatch again once the callback is fired.
+   */
+  class C_RetryMessage : public C_MonOp {
+    PaxosService *svc;
+  public:
+    C_RetryMessage(PaxosService *s, MonOpRequestRef op_) :
+      C_MonOp(op_), svc(s) { }
+    void _finish(int r) override {
+      if (r == -EAGAIN || r >= 0)
+	svc->dispatch(op);
+      else if (r == -ECANCELED)
+        return;
+      else
+	ceph_abort_msg("bad C_RetryMessage return value");
+    }
+  };
+
+  class C_ReplyOp : public C_MonOp {
+    Monitor &mon;
+    MonOpRequestRef op;
+    MessageRef reply;
+  public:
+    C_ReplyOp(PaxosService *s, MonOpRequestRef o, MessageRef r) :
+      C_MonOp(o), mon(s->mon), op(o), reply(r) { }
+    void _finish(int r) override {
+      if (r >= 0) {
+	mon.send_reply(op, reply.detach());
+      }
+    }
+  };
+
+  /**
+   * @}
+   */
+
+  /**
+   * @param mn A Monitor instance
+   * @param p A Paxos instance
+   * @param name Our service's name.
+   */
+  PaxosService(Monitor &mn, Paxos &p, std::string name) 
+    : mon(mn), paxos(p), service_name(name),
+      proposing(false),
+      service_version(0), proposal_timer(0), have_pending(false),
+      format_version(0),
+      last_committed_name("last_committed"),
+      first_committed_name("first_committed"),
+      full_prefix_name("full"), full_latest_name("latest"),
+      cached_first_committed(0), cached_last_committed(0)
+  {
+  }
+
+  virtual ~PaxosService() {}
+
+  /**
+   * Get the service's name.
+   *
+   * @returns The service's name.
+   */
+  const std::string& get_service_name() const { return service_name; }
+
+  /**
+   * Get the store prefixes we utilize
+   */
+  virtual void get_store_prefixes(std::set<std::string>& s) const {
+    s.insert(service_name);
+  }
+  
+  // i implement and you ignore
+  /**
+   * Informs this instance that it should consider itself restarted.
+   *
+   * This means that we will cancel our proposal_timer event, if any exists.
+   */
+  void restart();
+  /**
+   * Informs this instance that an election has finished.
+   *
+   * This means that we will invoke a PaxosService::discard_pending while
+   * setting have_pending to false (basically, ignore our pending state) and
+   * we will then make sure we obtain a new state.
+   *
+   * Our state shall be updated by PaxosService::_active if the Paxos is
+   * active; otherwise, we will wait for it to become active by adding a 
+   * PaxosService::C_Active callback to it.
+   */
+  void election_finished();
+  /**
+   * Informs this instance that it is supposed to shutdown.
+   *
+   * Basically, it will instruct Paxos to cancel all events/callbacks and then
+   * will cancel the proposal_timer event if any exists.
+   */
+  void shutdown();
+
+private:
+  /**
+   * Update our state by updating it from Paxos, and then creating a new
+   * pending state if need be.
+   *
+   * @remarks We only create a pending state we our Monitor is the Leader.
+   *
+   * @pre Paxos is active
+   * @post have_pending is true if our Monitor is the Leader and Paxos is
+   *	   active
+   */
+  void _active();
+
+public:
+  /**
+   * Propose a new value through Paxos.
+   *
+   * This function should be called by the classes implementing 
+   * PaxosService, in order to propose a new value through Paxos.
+   *
+   * @pre The implementation class implements the encode_pending function.
+   * @pre have_pending is true
+   * @pre Our monitor is the Leader
+   * @pre Paxos is active
+   * @post Cancel the proposal timer, if any
+   * @post have_pending is false
+   * @post propose pending value through Paxos
+   *
+   * @note This function depends on the implementation of encode_pending on
+   *	   the class that is implementing PaxosService
+   */
+  void propose_pending();
+
+  /**
+   * Let others request us to propose.
+   *
+   * At the moment, this is just a wrapper to propose_pending() with an
+   * extra check for is_writeable(), but it's a good practice to dissociate
+   * requests for proposals from direct usage of propose_pending() for
+   * future use -- we might want to perform additional checks or put a
+   * request on hold, for instance.
+   */
+  void request_proposal() {
+    ceph_assert(is_writeable());
+
+    propose_pending();
+  }
+  /**
+   * Request service @p other to perform a proposal.
+   *
+   * We could simply use the function above, requesting @p other directly,
+   * but we might eventually want to do something to the request -- say,
+   * set a flag stating we're waiting on a cross-proposal to be finished.
+   */
+  void request_proposal(PaxosService *other) {
+    ceph_assert(other != NULL);
+    ceph_assert(other->is_writeable());
+
+    other->request_proposal();
+  }
+
+  /**
+   * Dispatch a message by passing it to several different functions that are
+   * either implemented directly by this service, or that should be implemented
+   * by the class implementing this service.
+   *
+   * @param m A message
+   * @returns 'true' on successful dispatch; 'false' otherwise.
+   */
+  bool dispatch(MonOpRequestRef op);
+
+  void refresh(bool *need_bootstrap);
+  void post_refresh();
+
+  /**
+   * @defgroup PaxosService_h_override_funcs Functions that should be
+   *					     overridden.
+   *
+   * These functions should be overridden at will by the class implementing
+   * this service.
+   * @{
+   */
+  /**
+   * Create the initial state for your system.
+   *
+   * In some of ours the state is actually set up elsewhere so this does
+   * nothing.
+   */
+  virtual void create_initial() = 0;
+
+  /**
+   * Query the Paxos system for the latest state and apply it if it's newer
+   * than the current Monitor state.
+   */
+  virtual void update_from_paxos(bool *need_bootstrap) = 0;
+
+  /**
+   * Hook called after all services have refreshed their state from paxos
+   *
+   * This is useful for doing any update work that depends on other
+   * service's having up-to-date state.
+   */
+  virtual void post_paxos_update() {}
+
+  /**
+   * Init on startup
+   *
+   * This is called on mon startup, after all of the PaxosService instances'
+   * update_from_paxos() methods have been called
+   */
+  virtual void init() {}
+
+  /**
+   * Create the pending state.
+   *
+   * @invariant This function is only called on a Leader.
+   * @remarks This created state is then modified by incoming messages.
+   * @remarks Called at startup and after every Paxos ratification round.
+   */
+  virtual void create_pending() = 0;
+
+  /**
+   * Encode the pending state into a ceph::buffer::list for ratification and
+   * transmission as the next state.
+   *
+   * @invariant This function is only called on a Leader.
+   *
+   * @param t The transaction to hold all changes.
+   */
+  virtual void encode_pending(MonitorDBStore::TransactionRef t) = 0;
+
+  /**
+   * Discard the pending state
+   *
+   * @invariant This function is only called on a Leader.
+   *
+   * @remarks This function is NOT overridden in any of our code, but it is
+   *	      called in PaxosService::election_finished if have_pending is
+   *	      true.
+   */
+  virtual void discard_pending() { }
+
+  /**
+   * Look at the query; if the query can be handled without changing state,
+   * do so.
+   *
+   * @param m A query message
+   * @returns 'true' if the query was handled (e.g., was a read that got
+   *	      answered, was a state change that has no effect); 'false' 
+   *	      otherwise.
+   */
+  virtual bool preprocess_query(MonOpRequestRef op) = 0;
+
+  /**
+   * Apply the message to the pending state.
+   *
+   * @invariant This function is only called on a Leader.
+   *
+   * @param m An update message
+   * @returns 'true' if the update message was handled (e.g., a command that
+   *	      went through); 'false' otherwise.
+   */
+  virtual bool prepare_update(MonOpRequestRef op) = 0;
+  /**
+   * @}
+   */
+
+  /**
+   * Determine if the Paxos system should vote on pending, and if so how long
+   * it should wait to vote.
+   *
+   * @param[out] delay The wait time, used so we can limit the update traffic
+   *		       spamming.
+   * @returns 'true' if the Paxos system should propose; 'false' otherwise.
+   */
+  virtual bool should_propose(double &delay);
+
+  /**
+   * force an immediate propose.
+   *
+   * This is meant to be called from prepare_update(op).
+   */
+  void force_immediate_propose() {
+    need_immediate_propose = true;
+  }
+
+  /**
+   * @defgroup PaxosService_h_courtesy Courtesy functions
+   *
+   * Courtesy functions, in case the class implementing this service has
+   * anything it wants/needs to do at these times.
+   * @{
+   */
+  /**
+   * This is called when the Paxos state goes to active.
+   *
+   * On the peon, this is after each election.
+   * On the leader, this is after each election, *and* after each completed
+   * proposal.
+   *
+   * @note This function may get called twice in certain recovery cases.
+   */
+  virtual void on_active() { }
+
+  /**
+   * This is called when we are shutting down
+   */
+  virtual void on_shutdown() {}
+
+  /**
+   * this is called when activating on the leader
+   *
+   * it should conditionally upgrade the on-disk format by proposing a transaction
+   */
+  virtual void upgrade_format() { }
+
+  /**
+   * this is called when we detect the store has just upgraded underneath us
+   */
+  virtual void on_upgrade() {}
+
+  /**
+   * Called when the Paxos system enters a Leader election.
+   *
+   * @remarks It's a courtesy method, in case the class implementing this
+   *	      service has anything it wants/needs to do at that time.
+   */
+  virtual void on_restart() { }
+  /**
+   * @}
+   */
+
+  /**
+   * Tick.
+   */
+  virtual void tick() {}
+
+  void encode_health(const health_check_map_t& next,
+		     MonitorDBStore::TransactionRef t) {
+    using ceph::encode;
+    ceph::buffer::list bl;
+    encode(next, bl);
+    t->put("health", service_name, bl);
+    mon.log_health(next, health_checks, t);
+  }
+  void load_health();
+
+  /**
+   * @defgroup PaxosService_h_store_keys Set of keys that are usually used on
+   *					 all the services implementing this
+   *					 class, and, being almost the only keys
+   *					 used, should be standardized to avoid
+   *					 mistakes.
+   * @{
+   */
+  const std::string last_committed_name;
+  const std::string first_committed_name;
+  const std::string full_prefix_name;
+  const std::string full_latest_name;
+  /**
+   * @}
+   */
+
+ private:
+  /**
+   * @defgroup PaxosService_h_version_cache Variables holding cached values
+   *                                        for the most used versions (first
+   *                                        and last committed); we only have
+   *                                        to read them when the store is
+   *                                        updated, so in-between updates we
+   *                                        may very well use cached versions
+   *                                        and avoid the overhead.
+   * @{
+   */
+  version_t cached_first_committed;
+  version_t cached_last_committed;
+  /**
+   * @}
+   */
+
+  /**
+   * Callback list to be used whenever we are running a proposal through
+   * Paxos. These callbacks will be awaken whenever the said proposal
+   * finishes.
+   */
+  std::list<Context*> waiting_for_finished_proposal;
+
+ public:
+
+  /**
+   * Check if we are proposing a value through Paxos
+   *
+   * @returns true if we are proposing; false otherwise.
+   */
+  bool is_proposing() const {
+    return proposing;
+  }
+
+  /**
+   * Check if we are in the Paxos ACTIVE state.
+   *
+   * @note This function is a wrapper for Paxos::is_active
+   *
+   * @returns true if in state ACTIVE; false otherwise.
+   */
+  bool is_active() const {
+    return
+      !is_proposing() &&
+      (paxos.is_active() || paxos.is_updating() || paxos.is_writing());
+  }
+
+  /**
+   * Check if we are readable.
+   *
+   * This mirrors on the paxos check, except that we also verify that
+   *
+   *  - the client hasn't seen the future relative to this PaxosService
+   *  - this service isn't proposing.
+   *  - we have committed our initial state (last_committed > 0)
+   *
+   * @param ver The version we want to check if is readable
+   * @returns true if it is readable; false otherwise
+   */
+  bool is_readable(version_t ver = 0) const {
+    if (ver > get_last_committed() ||
+	!paxos.is_readable(0) ||
+	get_last_committed() == 0)
+      return false;
+    return true;
+  }
+
+  /**
+   * Check if we are writeable.
+   *
+   * We consider to be writeable iff:
+   *
+   *  - we are not proposing a new version;
+   *  - we are ready to be written to -- i.e., we have a pending value.
+   *  - paxos is (active or updating or writing or refresh)
+   *
+   * @returns true if writeable; false otherwise
+   */
+  bool is_writeable() const {
+    return is_active() && have_pending;
+  }
+
+  /**
+   * Wait for a proposal to finish.
+   *
+   * Add a callback to be awaken whenever our current proposal finishes being
+   * proposed through Paxos.
+   *
+   * @param c The callback to be awaken once the proposal is finished.
+   */
+  void wait_for_finished_proposal(MonOpRequestRef op, Context *c) {
+    if (op)
+      op->mark_event(service_name + ":wait_for_finished_proposal");
+    waiting_for_finished_proposal.push_back(c);
+  }
+  void wait_for_finished_proposal_ctx(Context *c) {
+    MonOpRequestRef o;
+    wait_for_finished_proposal(o, c);
+  }
+
+  /**
+   * Wait for us to become active
+   *
+   * @param c The callback to be awaken once we become active.
+   */
+  void wait_for_active(MonOpRequestRef op, Context *c) {
+    if (op)
+      op->mark_event(service_name + ":wait_for_active");
+
+    if (!is_proposing()) {
+      paxos.wait_for_active(op, c);
+      return;
+    }
+    wait_for_finished_proposal(op, c);
+  }
+  void wait_for_active_ctx(Context *c) {
+    MonOpRequestRef o;
+    wait_for_active(o, c);
+  }
+
+  /**
+   * Wait for us to become readable
+   *
+   * @param c The callback to be awaken once we become active.
+   * @param ver The version we want to wait on.
+   */
+  void wait_for_readable(MonOpRequestRef op, Context *c, version_t ver = 0) {
+    /* This is somewhat of a hack. We only do check if a version is readable on
+     * PaxosService::dispatch(), but, nonetheless, we must make sure that if that
+     * is why we are not readable, then we must wait on PaxosService and not on
+     * Paxos; otherwise, we may assert on Paxos::wait_for_readable() if it
+     * happens to be readable at that specific point in time.
+     */
+    if (op)
+      op->mark_event(service_name + ":wait_for_readable");
+
+    if (is_proposing() ||
+	ver > get_last_committed() ||
+	get_last_committed() == 0)
+      wait_for_finished_proposal(op, c);
+    else {
+      if (op)
+        op->mark_event(service_name + ":wait_for_readable/paxos");
+
+      paxos.wait_for_readable(op, c);
+    }
+  }
+
+  void wait_for_readable_ctx(Context *c, version_t ver = 0) {
+    MonOpRequestRef o; // will initialize the shared_ptr to NULL
+    wait_for_readable(o, c, ver);
+  }
+
+  /**
+   * Wait for us to become writeable
+   *
+   * @param c The callback to be awaken once we become writeable.
+   */
+  void wait_for_writeable(MonOpRequestRef op, Context *c) {
+    if (op)
+      op->mark_event(service_name + ":wait_for_writeable");
+
+    if (is_proposing())
+      wait_for_finished_proposal(op, c);
+    else if (!is_writeable())
+      wait_for_active(op, c);
+    else
+      paxos.wait_for_writeable(op, c);
+  }
+  void wait_for_writeable_ctx(Context *c) {
+    MonOpRequestRef o;
+    wait_for_writeable(o, c);
+  }
+
+  
+  /**
+   * @defgroup PaxosService_h_Trim Functions for trimming states
+   * @{
+   */
+  /**
+   * trim service states if appropriate
+   *
+   * Called at same interval as tick()
+   */
+  void maybe_trim();
+
+  /**
+   * Auxiliary function to trim our state from version @p from to version
+   * @p to, not including; i.e., the interval [from, to[
+   *
+   * @param t The transaction to which we will add the trim operations.
+   * @param from the lower limit of the interval to be trimmed
+   * @param to the upper limit of the interval to be trimmed (not including)
+   */
+  void trim(MonitorDBStore::TransactionRef t, version_t from, version_t to);
+
+  /**
+   * encode service-specific extra bits into trim transaction
+   *
+   * @param tx transaction
+   * @param first new first_committed value
+   */
+  virtual void encode_trim_extra(MonitorDBStore::TransactionRef tx,
+				 version_t first) {}
+
+  /**
+   * Get the version we should trim to.
+   *
+   * Should be overloaded by service if it wants to trim states.
+   *
+   * @returns the version we should trim to; if we return zero, it should be
+   *	      assumed that there's no version to trim to.
+   */
+  virtual version_t get_trim_to() const {
+    return 0;
+  }
+
+  /**
+   * @}
+   */
+  /**
+   * @defgroup PaxosService_h_Stash_Full
+   * @{
+   */
+  virtual bool should_stash_full();
+  /**
+   * Encode a full version on @p t
+   *
+   * @note We force every service to implement this function, since we strongly
+   *	   desire the encoding of full versions.
+   * @note Services that do not trim their state, will be bound to only create
+   *	   one full version. Full version stashing is determined/controlled by
+   *	   trimming: we stash a version each time a trim is bound to erase the
+   *	   latest full version.
+   *
+   * @param t Transaction on which the full version shall be encoded.
+   */
+  virtual void encode_full(MonitorDBStore::TransactionRef t) = 0;
+
+  /**
+   * @}
+   */
+
+  /**
+   * Cancel events.
+   *
+   * @note This function is a wrapper for Paxos::cancel_events
+   */
+  void cancel_events() {
+    paxos.cancel_events();
+  }
+
+  /**
+   * @defgroup PaxosService_h_store_funcs Back storage interface functions
+   * @{
+   */
+  /**
+   * @defgroup PaxosService_h_store_modify Wrapper function interface to access
+   *					   the back store for modification
+   *					   purposes
+   * @{
+   */
+  void put_first_committed(MonitorDBStore::TransactionRef t, version_t ver) {
+    t->put(get_service_name(), first_committed_name, ver);
+  }
+  /**
+   * Set the last committed version to @p ver
+   *
+   * @param t A transaction to which we add this put operation
+   * @param ver The last committed version number being put
+   */
+  void put_last_committed(MonitorDBStore::TransactionRef t, version_t ver) {
+    t->put(get_service_name(), last_committed_name, ver);
+
+    /* We only need to do this once, and that is when we are about to make our
+     * first proposal. There are some services that rely on first_committed
+     * being set -- and it should! -- so we need to guarantee that it is,
+     * specially because the services itself do not do it themselves. They do
+     * rely on it, but they expect us to deal with it, and so we shall.
+     */
+    if (!get_first_committed())
+      put_first_committed(t, ver);
+  }
+  /**
+   * Put the contents of @p bl into version @p ver
+   *
+   * @param t A transaction to which we will add this put operation
+   * @param ver The version to which we will add the value
+   * @param bl A ceph::buffer::list containing the version's value
+   */
+  void put_version(MonitorDBStore::TransactionRef t, version_t ver,
+		   ceph::buffer::list& bl) {
+    t->put(get_service_name(), ver, bl);
+  }
+  /**
+   * Put the contents of @p bl into a full version key for this service, that
+   * will be created with @p ver in mind.
+   *
+   * @param t The transaction to which we will add this put operation
+   * @param ver A version number
+   * @param bl A ceph::buffer::list containing the version's value
+   */
+  void put_version_full(MonitorDBStore::TransactionRef t,
+			version_t ver, ceph::buffer::list& bl) {
+    std::string key = mon.store->combine_strings(full_prefix_name, ver);
+    t->put(get_service_name(), key, bl);
+  }
+  /**
+   * Put the version number in @p ver into the key pointing to the latest full
+   * version of this service.
+   *
+   * @param t The transaction to which we will add this put operation
+   * @param ver A version number
+   */
+  void put_version_latest_full(MonitorDBStore::TransactionRef t, version_t ver) {
+    std::string key = mon.store->combine_strings(full_prefix_name, full_latest_name);
+    t->put(get_service_name(), key, ver);
+  }
+  /**
+   * Put the contents of @p bl into the key @p key.
+   *
+   * @param t A transaction to which we will add this put operation
+   * @param key The key to which we will add the value
+   * @param bl A ceph::buffer::list containing the value
+   */
+  void put_value(MonitorDBStore::TransactionRef t,
+		 const std::string& key, ceph::buffer::list& bl) {
+    t->put(get_service_name(), key, bl);
+  }
+
+  /**
+   * Put integer value @v into the key @p key.
+   *
+   * @param t A transaction to which we will add this put operation
+   * @param key The key to which we will add the value
+   * @param v An integer
+   */
+  void put_value(MonitorDBStore::TransactionRef t,
+		 const std::string& key, version_t v) {
+    t->put(get_service_name(), key, v);
+  }
+
+  /**
+   * @}
+   */
+
+  /**
+   * @defgroup PaxosService_h_store_get Wrapper function interface to access
+   *					the back store for reading purposes
+   * @{
+   */
+
+  /**
+   * @defgroup PaxosService_h_version_cache Obtain cached versions for this
+   *                                        service.
+   * @{
+   */
+  /**
+   * Get the first committed version
+   *
+   * @returns Our first committed version (that is available)
+   */
+  version_t get_first_committed() const{
+    return cached_first_committed;
+  }
+  /**
+   * Get the last committed version
+   *
+   * @returns Our last committed version
+   */
+  version_t get_last_committed() const{
+    return cached_last_committed;
+  }
+
+  /**
+   * @}
+   */
+
+  /**
+   * Get the contents of a given version @p ver
+   *
+   * @param ver The version being obtained
+   * @param bl The ceph::buffer::list to be populated
+   * @return 0 on success; <0 otherwise
+   */
+  virtual int get_version(version_t ver, ceph::buffer::list& bl) {
+    return mon.store->get(get_service_name(), ver, bl);
+  }
+  /**
+   * Get the contents of a given full version of this service.
+   *
+   * @param ver A version number
+   * @param bl The ceph::buffer::list to be populated
+   * @returns 0 on success; <0 otherwise
+   */
+  virtual int get_version_full(version_t ver, ceph::buffer::list& bl) {
+    std::string key = mon.store->combine_strings(full_prefix_name, ver);
+    return mon.store->get(get_service_name(), key, bl);
+  }
+  /**
+   * Get the latest full version number
+   *
+   * @returns A version number
+   */
+  version_t get_version_latest_full() {
+    std::string key = mon.store->combine_strings(full_prefix_name, full_latest_name);
+    return mon.store->get(get_service_name(), key);
+  }
+
+  /**
+   * Get a value from a given key.
+   *
+   * @param[in] key The key
+   * @param[out] bl The ceph::buffer::list to be populated with the value
+   */
+  int get_value(const std::string& key, ceph::buffer::list& bl) {
+    return mon.store->get(get_service_name(), key, bl);
+  }
+  /**
+   * Get an integer value from a given key.
+   *
+   * @param[in] key The key
+   */
+  version_t get_value(const std::string& key) {
+    return mon.store->get(get_service_name(), key);
+  }
+
+  /**
+   * @}
+   */
+  /**
+   * @}
+   */
+};
+
+#endif
diff --git a/src/mon/Session.h b/src/mon/Session.h
new file mode 100644
index 000000000..3009d0239
--- /dev/null
+++ b/src/mon/Session.h
@@ -0,0 +1,295 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MON_SESSION_H
+#define CEPH_MON_SESSION_H
+
+#include <string>
+#include <string_view>
+
+#include "include/utime.h"
+#include "include/xlist.h"
+
+#include "global/global_context.h"
+#include "msg/msg_types.h"
+#include "mon/mon_types.h"
+
+#include "auth/AuthServiceHandler.h"
+#include "osd/OSDMap.h"
+
+#include "MonCap.h"
+
+struct MonSession;
+
+struct Subscription {
+  MonSession *session;
+  std::string type;
+  xlist<Subscription*>::item type_item;
+  version_t next;
+  bool onetime;
+  bool incremental_onetime;  // has CEPH_FEATURE_INCSUBOSDMAP
+  
+  Subscription(MonSession *s, const std::string& t) : session(s), type(t), type_item(this),
+						 next(0), onetime(false), incremental_onetime(false) {}
+};
+
+struct MonSession : public RefCountedObject {
+  ConnectionRef con;
+  int con_type = 0;
+  uint64_t con_features = 0;  // zero if AnonConnection
+  entity_name_t name;
+  entity_addrvec_t addrs;
+  entity_addr_t socket_addr;
+  utime_t session_timeout;
+  bool closed = false;
+  xlist<MonSession*>::item item;
+  std::set<uint64_t> routed_request_tids;
+  MonCap caps;
+  bool validated_stretch_connection = false;
+
+  bool authenticated = false;  ///< true if auth handshake is complete
+
+  std::map<std::string, Subscription*> sub_map;
+  epoch_t osd_epoch = 0;       ///< the osdmap epoch sent to the mon client
+
+  AuthServiceHandler *auth_handler = nullptr;
+  EntityName entity_name;
+  uint64_t global_id = 0;
+  global_id_status_t global_id_status = global_id_status_t::NONE;
+
+  ConnectionRef proxy_con;
+  uint64_t proxy_tid = 0;
+
+  std::string remote_host;                ///< remote host name
+  std::map<std::string,std::string,std::less<>> last_config;    ///< most recently shared config
+  bool any_config = false;
+
+  MonSession(Connection *c)
+    : RefCountedObject(g_ceph_context),
+      con(c),
+      item(this) { }
+
+  void _ident(const entity_name_t& n, const entity_addrvec_t& av) {
+    con_type = con->get_peer_type();
+    name = n;
+    addrs = av;
+    socket_addr = con->get_peer_socket_addr();
+    if (con->get_messenger()) {
+      // only fill in features if this is a non-anonymous connection
+      con_features = con->get_features();
+    }
+  }
+
+  ~MonSession() override {
+    //generic_dout(0) << "~MonSession " << this << dendl;
+    // we should have been removed before we get destructed; see MonSessionMap::remove_session()
+    ceph_assert(!item.is_on_list());
+    ceph_assert(sub_map.empty());
+    delete auth_handler;
+  }
+
+  bool is_capable(std::string service, int mask) {
+    std::map<std::string,std::string> args;
+    return caps.is_capable(
+      g_ceph_context,
+      entity_name,
+      service, "", args,
+      mask & MON_CAP_R, mask & MON_CAP_W, mask & MON_CAP_X,
+      get_peer_socket_addr());
+  }
+
+  std::vector<string> get_allowed_fs_names() const {
+    return caps.allowed_fs_names();
+  }
+
+  bool fs_name_capable(string_view fsname, __u8 mask) {
+    return caps.fs_name_capable(entity_name, fsname, mask);
+  }
+
+  const entity_addr_t& get_peer_socket_addr() {
+    return socket_addr;
+  }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_stream("name") << name;
+    f->dump_stream("entity_name") << entity_name;
+    f->dump_object("addrs", addrs);
+    f->dump_object("socket_addr", socket_addr);
+    f->dump_string("con_type", ceph_entity_type_name(con_type));
+    f->dump_unsigned("con_features", con_features);
+    f->dump_stream("con_features_hex") << std::hex << con_features << std::dec;
+    f->dump_string("con_features_release",
+		   ceph_release_name(ceph_release_from_features(con_features)));
+    f->dump_bool("open", !closed);
+    f->dump_object("caps", caps);
+    f->dump_bool("authenticated", authenticated);
+    f->dump_unsigned("global_id", global_id);
+    f->dump_stream("global_id_status") << global_id_status;
+    f->dump_unsigned("osd_epoch", osd_epoch);
+    f->dump_string("remote_host", remote_host);
+  }
+};
+
+
+struct MonSessionMap {
+  xlist<MonSession*> sessions;
+  std::map<std::string, xlist<Subscription*>* > subs;
+  std::multimap<int, MonSession*> by_osd;
+  FeatureMap feature_map; // type -> features -> count
+
+  MonSessionMap() {}
+  ~MonSessionMap() {
+    while (!subs.empty()) {
+      ceph_assert(subs.begin()->second->empty());
+      delete subs.begin()->second;
+      subs.erase(subs.begin());
+    }
+  }
+
+  unsigned get_size() const {
+    return sessions.size();
+  }
+
+  void remove_session(MonSession *s) {
+    ceph_assert(!s->closed);
+    for (std::map<std::string,Subscription*>::iterator p = s->sub_map.begin(); p != s->sub_map.end(); ++p) {
+      p->second->type_item.remove_myself();
+      delete p->second;
+    }
+    s->sub_map.clear();
+    s->item.remove_myself();
+    if (s->name.is_osd() &&
+	s->name.num() >= 0) {
+      for (auto p = by_osd.find(s->name.num());
+	   p->first == s->name.num();
+	   ++p)
+	if (p->second == s) {
+	  by_osd.erase(p);
+	  break;
+	}
+    }
+    if (s->con_features) {
+      feature_map.rm(s->con_type, s->con_features);
+    }
+    s->closed = true;
+    s->put();
+  }
+
+  MonSession *new_session(const entity_name_t& n,
+			  const entity_addrvec_t& av,
+			  Connection *c) {
+    MonSession *s = new MonSession(c);
+    ceph_assert(s);
+    s->_ident(n, av);
+    add_session(s);
+    return s;
+  }
+
+  void add_session(MonSession *s) {
+    s->session_timeout = ceph_clock_now();
+    s->session_timeout += g_conf()->mon_session_timeout;
+
+    sessions.push_back(&s->item);
+    s->get();
+    if (s->name.is_osd() &&
+	s->name.num() >= 0) {
+      by_osd.insert(std::pair<int,MonSession*>(s->name.num(), s));
+    }
+    if (s->con_features) {
+      feature_map.add(s->con_type, s->con_features);
+    }
+  }
+
+  MonSession *get_random_osd_session(OSDMap *osdmap) {
+    // ok, this isn't actually random, but close enough.
+    if (by_osd.empty())
+      return 0;
+    int n = by_osd.rbegin()->first + 1;
+    int r = rand() % n;
+
+    auto p = by_osd.lower_bound(r);
+    if (p == by_osd.end())
+      --p;
+
+    if (!osdmap) {
+      return p->second;
+    }
+
+    MonSession *s = NULL;
+
+    auto b = p;
+    auto f = p;
+    bool backward = true, forward = true;
+    while (backward || forward) {
+      if (backward) {
+        if (osdmap->is_up(b->first) &&
+	    osdmap->get_addrs(b->first) == b->second->con->get_peer_addrs()) {
+          s = b->second;
+          break;
+        }
+        if (b != by_osd.begin())
+          --b;
+        else
+          backward = false;
+      }
+
+      forward = (f != by_osd.end());
+      if (forward) {
+        if (osdmap->is_up(f->first)) {
+          s = f->second;
+          break;
+        }
+        ++f;
+      }
+    }
+
+    return s;
+  }
+
+  void add_update_sub(MonSession *s, const std::string& what, version_t start, bool onetime, bool incremental_onetime) {
+    Subscription *sub = 0;
+    if (s->sub_map.count(what)) {
+      sub = s->sub_map[what];
+    } else {
+      sub = new Subscription(s, what);
+      s->sub_map[what] = sub;
+
+      if (!subs.count(what))
+	subs[what] = new xlist<Subscription*>;
+      subs[what]->push_back(&sub->type_item);
+    }
+    sub->next = start;
+    sub->onetime = onetime;
+    sub->incremental_onetime = onetime && incremental_onetime;
+  }
+
+  void remove_sub(Subscription *sub) {
+    sub->session->sub_map.erase(sub->type);
+    sub->type_item.remove_myself();
+    delete sub;
+  }
+};
+
+inline std::ostream& operator<<(std::ostream& out, const MonSession& s)
+{
+  out << "MonSession(" << s.name << " " << s.addrs
+      << " is " << (s.closed ? "closed" : "open")
+      << " " << s.caps
+      << ", features 0x" << std::hex << s.con_features << std::dec
+      <<  " (" << ceph_release_name(ceph_release_from_features(s.con_features))
+      << "))";
+  return out;
+}
+
+#endif
diff --git a/src/mon/error_code.cc b/src/mon/error_code.cc
new file mode 100644
index 000000000..a2cd39299
--- /dev/null
+++ b/src/mon/error_code.cc
@@ -0,0 +1,85 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2019 Red Hat <contact@redhat.com>
+ * Author: Adam C. Emerson <aemerson@redhat.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <string>
+
+#include "common/error_code.h"
+#include "common/errno.h"
+#include "error_code.h"
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wnon-virtual-dtor"
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wnon-virtual-dtor"
+
+namespace bs = boost::system;
+
+class mon_error_category : public ceph::converting_category {
+public:
+  mon_error_category(){}
+  const char* name() const noexcept override;
+  const char* message(int ev, char*, std::size_t) const noexcept override;
+  std::string message(int ev) const override;
+  bs::error_condition default_error_condition(int ev) const noexcept
+    override;
+  bool equivalent(int ev, const bs::error_condition& c) const
+    noexcept override;
+  using ceph::converting_category::equivalent;
+  int from_code(int ev) const noexcept override;
+};
+
+const char* mon_error_category::name() const noexcept {
+  return "mon";
+}
+
+const char* mon_error_category::message(int ev, char* buf,
+					std::size_t len) const noexcept {
+  if (ev == 0)
+    return "No error";
+
+  if (len) {
+    auto s = cpp_strerror(ev);
+    auto n = s.copy(buf, len - 1);
+    *(buf + n) = '\0';
+  }
+  return buf;
+}
+
+std::string mon_error_category::message(int ev) const {
+  if (ev == 0)
+    return "No error";
+
+  return cpp_strerror(ev);
+}
+
+bs::error_condition
+mon_error_category::default_error_condition(int ev) const noexcept {
+  return { ev, bs::generic_category() };
+}
+
+bool mon_error_category::equivalent(int ev,const bs::error_condition& c) const noexcept {
+  return default_error_condition(ev) == c;
+}
+
+int mon_error_category::from_code(int ev) const noexcept {
+  return -ev;
+}
+
+const bs::error_category& mon_category() noexcept {
+  static const mon_error_category c;
+  return c;
+}
+#pragma GCC diagnostic pop
+#pragma clang diagnostic pop
diff --git a/src/mon/error_code.h b/src/mon/error_code.h
new file mode 100644
index 000000000..2a6e88061
--- /dev/null
+++ b/src/mon/error_code.h
@@ -0,0 +1,49 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2019 Red Hat <contact@redhat.com>
+ * Author: Adam C. Emerson <aemerson@redhat.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <boost/system/error_code.hpp>
+
+#include "include/rados.h"
+
+const boost::system::error_category& mon_category() noexcept;
+
+// The Monitor, like the OSD, mostly replies with POSIX error codes.
+
+enum class mon_errc {
+};
+
+namespace boost::system {
+template<>
+struct is_error_code_enum<::mon_errc> {
+  static const bool value = true;
+};
+
+template<>
+struct is_error_condition_enum<::mon_errc> {
+  static const bool value = false;
+};
+}
+
+//  explicit conversion:
+inline boost::system::error_code make_error_code(mon_errc e) noexcept {
+  return { static_cast<int>(e), mon_category() };
+}
+
+// implicit conversion:
+inline boost::system::error_condition make_error_condition(mon_errc e) noexcept {
+  return { static_cast<int>(e), mon_category() };
+}
diff --git a/src/mon/health_check.h b/src/mon/health_check.h
new file mode 100644
index 000000000..4e74637f9
--- /dev/null
+++ b/src/mon/health_check.h
@@ -0,0 +1,198 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <string>
+#include <map>
+
+#include "include/health.h"
+#include "include/utime.h"
+#include "common/Formatter.h"
+
+struct health_check_t {
+  health_status_t severity;
+  std::string summary;
+  std::list<std::string> detail;
+  int64_t count = 0;
+
+  DENC(health_check_t, v, p) {
+    DENC_START(2, 1, p);
+    denc(v.severity, p);
+    denc(v.summary, p);
+    denc(v.detail, p);
+    if (struct_v >= 2) {
+      denc(v.count, p);
+    }
+    DENC_FINISH(p);
+  }
+
+  friend bool operator==(const health_check_t& l,
+			 const health_check_t& r) {
+    return l.severity == r.severity &&
+      l.summary == r.summary &&
+      l.detail == r.detail &&
+      l.count == r.count;
+  }
+  friend bool operator!=(const health_check_t& l,
+			 const health_check_t& r) {
+    return !(l == r);
+  }
+
+  void dump(ceph::Formatter *f, bool want_detail=true) const {
+    f->dump_stream("severity") << severity;
+
+    f->open_object_section("summary");
+    f->dump_string("message", summary);
+    f->dump_int("count", count);
+    f->close_section();
+
+    if (want_detail) {
+      f->open_array_section("detail");
+      for (auto& p : detail) {
+	f->open_object_section("detail_item");
+	f->dump_string("message", p);
+	f->close_section();
+      }
+      f->close_section();
+    }
+  }
+
+  static void generate_test_instances(std::list<health_check_t*>& ls) {
+    ls.push_back(new health_check_t);
+    ls.push_back(new health_check_t);
+    ls.back()->severity = HEALTH_ERR;
+    ls.back()->summary = "summarization";
+    ls.back()->detail = {"one", "two", "three"};
+    ls.back()->count = 42;
+  }
+};
+WRITE_CLASS_DENC(health_check_t)
+
+
+struct health_mute_t {
+  std::string code;
+  utime_t ttl;
+  bool sticky = false;
+  std::string summary;
+  int64_t count;
+
+  DENC(health_mute_t, v, p) {
+    DENC_START(1, 1, p);
+    denc(v.code, p);
+    denc(v.ttl, p);
+    denc(v.sticky, p);
+    denc(v.summary, p);
+    denc(v.count, p);
+    DENC_FINISH(p);
+  }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_string("code", code);
+    if (ttl != utime_t()) {
+      f->dump_stream("ttl") << ttl;
+    }
+    f->dump_bool("sticky", sticky);
+    f->dump_string("summary", summary);
+    f->dump_int("count", count);
+  }
+
+  static void generate_test_instances(std::list<health_mute_t*>& ls) {
+    ls.push_back(new health_mute_t);
+    ls.push_back(new health_mute_t);
+    ls.back()->code = "OSD_DOWN";
+    ls.back()->ttl = utime_t(1, 2);
+    ls.back()->sticky = true;
+    ls.back()->summary = "foo bar";
+    ls.back()->count = 2;
+  }
+};
+WRITE_CLASS_DENC(health_mute_t)
+
+struct health_check_map_t {
+  std::map<std::string,health_check_t> checks;
+
+  DENC(health_check_map_t, v, p) {
+    DENC_START(1, 1, p);
+    denc(v.checks, p);
+    DENC_FINISH(p);
+  }
+
+  void dump(ceph::Formatter *f) const {
+    for (auto& [code, check] : checks) {
+      f->dump_object(code, check);
+    }
+  }
+
+  static void generate_test_instances(std::list<health_check_map_t*>& ls) {
+    ls.push_back(new health_check_map_t);
+    ls.push_back(new health_check_map_t);
+    {
+      auto& d = ls.back()->add("FOO", HEALTH_WARN, "foo", 2);
+      d.detail.push_back("a");
+      d.detail.push_back("b");
+    }
+    {
+      auto& d = ls.back()->add("BAR", HEALTH_ERR, "bar!", 3);
+      d.detail.push_back("c");
+      d.detail.push_back("d");
+      d.detail.push_back("e");
+    }
+  }
+
+  void clear() {
+    checks.clear();
+  }
+  bool empty() const {
+    return checks.empty();
+  }
+  void swap(health_check_map_t& other) {
+    checks.swap(other.checks);
+  }
+
+  health_check_t& add(const std::string& code,
+		      health_status_t severity,
+		      const std::string& summary,
+		      int64_t count) {
+    ceph_assert(checks.count(code) == 0);
+    health_check_t& r = checks[code];
+    r.severity = severity;
+    r.summary = summary;
+    r.count = count;
+    return r;
+  }
+  health_check_t& get_or_add(const std::string& code,
+			     health_status_t severity,
+			     const std::string& summary,
+			     int64_t count) {
+    health_check_t& r = checks[code];
+    r.severity = severity;
+    r.summary = summary;
+    r.count += count;
+    return r;
+  }
+
+  void merge(const health_check_map_t& o) {
+    for (auto& [code, check] : o.checks) {
+      auto [it, new_check] = checks.try_emplace(code, check);
+      if (!new_check) {
+        // merge details, and hope the summary matches!
+        it->second.detail.insert(
+          it->second.detail.end(),
+          check.detail.begin(),
+          check.detail.end());
+        it->second.count += check.count;
+      }
+    }
+  }
+
+  friend bool operator==(const health_check_map_t& l,
+			 const health_check_map_t& r) {
+    return l.checks == r.checks;
+  }
+  friend bool operator!=(const health_check_map_t& l,
+			 const health_check_map_t& r) {
+    return !(l == r);
+  }
+};
+WRITE_CLASS_DENC(health_check_map_t)
diff --git a/src/mon/mon_types.h b/src/mon/mon_types.h
new file mode 100644
index 000000000..ce7184f37
--- /dev/null
+++ b/src/mon/mon_types.h
@@ -0,0 +1,660 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_MON_TYPES_H
+#define CEPH_MON_TYPES_H
+
+#include <map>
+
+#include "include/Context.h"
+#include "include/util.h"
+#include "include/utime.h"
+#include "common/Formatter.h"
+#include "common/bit_str.h"
+#include "common/ceph_releases.h"
+
+// use as paxos_service index
+enum {
+  PAXOS_MDSMAP,
+  PAXOS_OSDMAP,
+  PAXOS_LOG,
+  PAXOS_MONMAP,
+  PAXOS_AUTH,
+  PAXOS_MGR,
+  PAXOS_MGRSTAT,
+  PAXOS_HEALTH,
+  PAXOS_CONFIG,
+  PAXOS_KV,
+  PAXOS_NUM
+};
+
+#define CEPH_MON_ONDISK_MAGIC "ceph mon volume v012"
+
+// map of entity_type -> features -> count
+struct FeatureMap {
+  std::map<uint32_t,std::map<uint64_t,uint64_t>> m;
+
+  void add(uint32_t type, uint64_t features) {
+    if (type == CEPH_ENTITY_TYPE_MON) {
+      return;
+    }
+    m[type][features]++;
+  }
+
+  void add_mon(uint64_t features) {
+    m[CEPH_ENTITY_TYPE_MON][features]++;
+  }
+
+  void rm(uint32_t type, uint64_t features) {
+    if (type == CEPH_ENTITY_TYPE_MON) {
+      return;
+    }
+    auto p = m.find(type);
+    ceph_assert(p != m.end());
+    auto q = p->second.find(features);
+    ceph_assert(q != p->second.end());
+    if (--q->second == 0) {
+      p->second.erase(q);
+      if (p->second.empty()) {
+	m.erase(p);
+      }
+    }
+  }
+
+  FeatureMap& operator+=(const FeatureMap& o) {
+    for (auto& p : o.m) {
+      auto &v = m[p.first];
+      for (auto& q : p.second) {
+	v[q.first] += q.second;
+      }
+    }
+    return *this;
+  }
+
+  void encode(ceph::buffer::list& bl) const {
+    ENCODE_START(1, 1, bl);
+    encode(m, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(ceph::buffer::list::const_iterator& p) {
+    DECODE_START(1, p);
+    decode(m, p);
+    DECODE_FINISH(p);
+  }
+
+  void dump(ceph::Formatter *f) const {
+    for (auto& p : m) {
+      f->open_array_section(ceph_entity_type_name(p.first));
+      for (auto& q : p.second) {
+	f->open_object_section("group");
+        std::stringstream ss;
+        ss << "0x" << std::hex << q.first << std::dec;
+        f->dump_string("features", ss.str());
+	f->dump_string("release", ceph_release_name(
+			 ceph_release_from_features(q.first)));
+	f->dump_unsigned("num", q.second);
+	f->close_section();
+      }
+      f->close_section();
+    }
+  }
+};
+WRITE_CLASS_ENCODER(FeatureMap)
+
+/**
+ * leveldb store stats
+ *
+ * If we ever decide to support multiple backends for the monitor store,
+ * we should then create an abstract class 'MonitorStoreStats' of sorts
+ * and inherit it on LevelDBStoreStats.  I'm sure you'll figure something
+ * out.
+ */
+struct LevelDBStoreStats {
+  uint64_t bytes_total;
+  uint64_t bytes_sst;
+  uint64_t bytes_log;
+  uint64_t bytes_misc;
+  utime_t last_update;
+
+  LevelDBStoreStats() :
+    bytes_total(0),
+    bytes_sst(0),
+    bytes_log(0),
+    bytes_misc(0)
+  {}
+
+  void dump(ceph::Formatter *f) const {
+    ceph_assert(f != NULL);
+    f->dump_int("bytes_total", bytes_total);
+    f->dump_int("bytes_sst", bytes_sst);
+    f->dump_int("bytes_log", bytes_log);
+    f->dump_int("bytes_misc", bytes_misc);
+    f->dump_stream("last_updated") << last_update;
+  }
+
+  void encode(ceph::buffer::list &bl) const {
+    ENCODE_START(1, 1, bl);
+    encode(bytes_total, bl);
+    encode(bytes_sst, bl);
+    encode(bytes_log, bl);
+    encode(bytes_misc, bl);
+    encode(last_update, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(ceph::buffer::list::const_iterator &p) {
+    DECODE_START(1, p);
+    decode(bytes_total, p);
+    decode(bytes_sst, p);
+    decode(bytes_log, p);
+    decode(bytes_misc, p);
+    decode(last_update, p);
+    DECODE_FINISH(p);
+  }
+
+  static void generate_test_instances(std::list<LevelDBStoreStats*>& ls) {
+    ls.push_back(new LevelDBStoreStats);
+    ls.push_back(new LevelDBStoreStats);
+    ls.back()->bytes_total = 1024*1024;
+    ls.back()->bytes_sst = 512*1024;
+    ls.back()->bytes_log = 256*1024;
+    ls.back()->bytes_misc = 256*1024;
+    ls.back()->last_update = utime_t();
+  }
+};
+WRITE_CLASS_ENCODER(LevelDBStoreStats)
+
+// data stats
+
+struct DataStats {
+  ceph_data_stats_t fs_stats;
+  // data dir
+  utime_t last_update;
+  LevelDBStoreStats store_stats;
+
+  void dump(ceph::Formatter *f) const {
+    ceph_assert(f != NULL);
+    f->dump_int("kb_total", (fs_stats.byte_total/1024));
+    f->dump_int("kb_used", (fs_stats.byte_used/1024));
+    f->dump_int("kb_avail", (fs_stats.byte_avail/1024));
+    f->dump_int("avail_percent", fs_stats.avail_percent);
+    f->dump_stream("last_updated") << last_update;
+    f->open_object_section("store_stats");
+    store_stats.dump(f);
+    f->close_section();
+  }
+
+  void encode(ceph::buffer::list &bl) const {
+    ENCODE_START(3, 1, bl);
+    encode(fs_stats.byte_total, bl);
+    encode(fs_stats.byte_used, bl);
+    encode(fs_stats.byte_avail, bl);
+    encode(fs_stats.avail_percent, bl);
+    encode(last_update, bl);
+    encode(store_stats, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(ceph::buffer::list::const_iterator &p) {
+    DECODE_START(1, p);
+    // we moved from having fields in kb to fields in byte
+    if (struct_v > 2) {
+      decode(fs_stats.byte_total, p);
+      decode(fs_stats.byte_used, p);
+      decode(fs_stats.byte_avail, p);
+    } else {
+      uint64_t t;
+      decode(t, p);
+      fs_stats.byte_total = t*1024;
+      decode(t, p);
+      fs_stats.byte_used = t*1024;
+      decode(t, p);
+      fs_stats.byte_avail = t*1024;
+    }
+    decode(fs_stats.avail_percent, p);
+    decode(last_update, p);
+    if (struct_v > 1)
+      decode(store_stats, p);
+
+    DECODE_FINISH(p);
+  }
+};
+WRITE_CLASS_ENCODER(DataStats)
+
+struct ScrubResult {
+  std::map<std::string,uint32_t> prefix_crc;  ///< prefix -> crc
+  std::map<std::string,uint64_t> prefix_keys; ///< prefix -> key count
+
+  bool operator!=(const ScrubResult& other) {
+    return prefix_crc != other.prefix_crc || prefix_keys != other.prefix_keys;
+  }
+
+  void encode(ceph::buffer::list& bl) const {
+    ENCODE_START(1, 1, bl);
+    encode(prefix_crc, bl);
+    encode(prefix_keys, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(ceph::buffer::list::const_iterator& p) {
+    DECODE_START(1, p);
+    decode(prefix_crc, p);
+    decode(prefix_keys, p);
+    DECODE_FINISH(p);
+  }
+  void dump(ceph::Formatter *f) const {
+    f->open_object_section("crc");
+    for (auto p = prefix_crc.begin(); p != prefix_crc.end(); ++p)
+      f->dump_unsigned(p->first.c_str(), p->second);
+    f->close_section();
+    f->open_object_section("keys");
+    for (auto p = prefix_keys.begin(); p != prefix_keys.end(); ++p)
+      f->dump_unsigned(p->first.c_str(), p->second);
+    f->close_section();
+  }
+  static void generate_test_instances(std::list<ScrubResult*>& ls) {
+    ls.push_back(new ScrubResult);
+    ls.push_back(new ScrubResult);
+    ls.back()->prefix_crc["foo"] = 123;
+    ls.back()->prefix_keys["bar"] = 456;
+  }
+};
+WRITE_CLASS_ENCODER(ScrubResult)
+
+inline std::ostream& operator<<(std::ostream& out, const ScrubResult& r) {
+  return out << "ScrubResult(keys " << r.prefix_keys << " crc " << r.prefix_crc << ")";
+}
+
+/// for information like os, kernel, hostname, memory info, cpu model.
+typedef std::map<std::string, std::string> Metadata;
+
+namespace ceph {
+  namespace features {
+    namespace mon {
+      /**
+       * Get a feature's name based on its value.
+       *
+       * @param b raw feature value
+       *
+       * @remarks
+       *    Consumers should not assume this interface will never change.
+       * @remarks
+       *    As the number of features increase, so may the internal representation
+       *    of the raw features. When this happens, this interface will change
+       *    accordingly. So should consumers of this interface.
+       */
+      static inline const char *get_feature_name(uint64_t b);
+    }
+  }
+}
+
+
+inline const char *ceph_mon_feature_name(uint64_t b)
+{
+  return ceph::features::mon::get_feature_name(b);
+};
+
+class mon_feature_t {
+
+  static constexpr int HEAD_VERSION = 1;
+  static constexpr int COMPAT_VERSION = 1;
+
+  // mon-specific features
+  uint64_t features;
+
+public:
+
+  explicit constexpr
+  mon_feature_t(const uint64_t f) : features(f) { }
+
+  mon_feature_t() :
+    features(0) { }
+
+  constexpr
+  mon_feature_t(const mon_feature_t &o) :
+    features(o.features) { }
+
+  mon_feature_t& operator&=(const mon_feature_t other) {
+    features &= other.features;
+    return (*this);
+  }
+
+  /**
+   * Obtain raw features
+   *
+   * @remarks
+   *    Consumers should not assume this interface will never change.
+   * @remarks
+   *    As the number of features increase, so may the internal representation
+   *    of the raw features. When this happens, this interface will change
+   *    accordingly. So should consumers of this interface.
+   */
+  uint64_t get_raw() const {
+    return features;
+  }
+
+  constexpr
+  friend mon_feature_t operator&(const mon_feature_t a,
+                                 const mon_feature_t b) {
+    return mon_feature_t(a.features & b.features);
+  }
+
+  mon_feature_t& operator|=(const mon_feature_t other) {
+    features |= other.features;
+    return (*this);
+  }
+
+  constexpr
+  friend mon_feature_t operator|(const mon_feature_t a,
+                                 const mon_feature_t b) {
+    return mon_feature_t(a.features | b.features);
+  }
+
+  constexpr
+  friend mon_feature_t operator^(const mon_feature_t a,
+                                 const mon_feature_t b) {
+    return mon_feature_t(a.features ^ b.features);
+  }
+
+  mon_feature_t& operator^=(const mon_feature_t other) {
+    features ^= other.features;
+    return (*this);
+  }
+
+  bool operator==(const mon_feature_t other) const {
+    return (features == other.features);
+  }
+
+  bool operator!=(const mon_feature_t other) const {
+    return (features != other.features);
+  }
+
+  bool empty() const {
+    return features == 0;
+  }
+
+  /**
+   * Set difference of our features in respect to @p other
+   *
+   * Returns all the elements in our features that are not in @p other
+   *
+   * @returns all the features not in @p other
+   */
+  mon_feature_t diff(const mon_feature_t other) const {
+    return mon_feature_t((features ^ other.features) & features);
+  }
+
+  /**
+   * Set intersection of our features and @p other
+   *
+   * Returns all the elements common to both our features and the
+   * features of @p other
+   *
+   * @returns the features common to @p other and us
+   */
+  mon_feature_t intersection(const mon_feature_t other) const {
+    return mon_feature_t((features & other.features));
+  }
+
+  /**
+   * Checks whether we have all the features in @p other
+   *
+   * Returns true if we have all the features in @p other
+   *
+   * @returns true if we contain all the features in @p other
+   * @returns false if we do not contain some of the features in @p other
+   */
+  bool contains_all(const mon_feature_t other) const {
+    mon_feature_t d = intersection(other);
+    return d == other;
+  }
+
+  /**
+   * Checks whether we contain any of the features in @p other.
+   *
+   * @returns true if we contain any of the features in @p other
+   * @returns false if we don't contain any of the features in @p other
+   */
+  bool contains_any(const mon_feature_t other) const {
+    mon_feature_t d = intersection(other);
+    return !d.empty();
+  }
+
+  void set_feature(const mon_feature_t f) {
+    features |= f.features;
+  }
+
+  void unset_feature(const mon_feature_t f) {
+    features &= ~(f.features);
+  }
+
+  void print(std::ostream& out) const {
+    out << "[";
+    print_bit_str(features, out, ceph::features::mon::get_feature_name);
+    out << "]";
+  }
+
+  void print_with_value(std::ostream& out) const {
+    out << "[";
+    print_bit_str(features, out, ceph::features::mon::get_feature_name, true);
+    out << "]";
+  }
+
+  void dump(ceph::Formatter *f, const char *sec_name = NULL) const {
+    f->open_array_section((sec_name ? sec_name : "features"));
+    dump_bit_str(features, f, ceph::features::mon::get_feature_name);
+    f->close_section();
+  }
+
+  void dump_with_value(ceph::Formatter *f, const char *sec_name = NULL) const {
+    f->open_array_section((sec_name ? sec_name : "features"));
+    dump_bit_str(features, f, ceph::features::mon::get_feature_name, true);
+    f->close_section();
+  }
+
+  void encode(ceph::buffer::list& bl) const {
+    ENCODE_START(HEAD_VERSION, COMPAT_VERSION, bl);
+    encode(features, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(ceph::buffer::list::const_iterator& p) {
+    DECODE_START(COMPAT_VERSION, p);
+    decode(features, p);
+    DECODE_FINISH(p);
+  }
+};
+WRITE_CLASS_ENCODER(mon_feature_t)
+
+namespace ceph {
+  namespace features {
+    namespace mon {
+      constexpr mon_feature_t FEATURE_KRAKEN(     (1ULL << 0));
+      constexpr mon_feature_t FEATURE_LUMINOUS(   (1ULL << 1));
+      constexpr mon_feature_t FEATURE_MIMIC(      (1ULL << 2));
+      constexpr mon_feature_t FEATURE_OSDMAP_PRUNE (1ULL << 3);
+      constexpr mon_feature_t FEATURE_NAUTILUS(    (1ULL << 4));
+      constexpr mon_feature_t FEATURE_OCTOPUS(    (1ULL << 5));
+      constexpr mon_feature_t FEATURE_PACIFIC(    (1ULL << 6));
+      // elector pinging and CONNECTIVITY mode:
+      constexpr mon_feature_t FEATURE_PINGING(    (1ULL << 7));
+
+      constexpr mon_feature_t FEATURE_RESERVED(   (1ULL << 63));
+      constexpr mon_feature_t FEATURE_NONE(       (0ULL));
+
+      /**
+       * All the features this monitor supports
+       *
+       * If there's a feature above, it should be OR'ed to this list.
+       */
+      constexpr mon_feature_t get_supported() {
+        return (
+	  FEATURE_KRAKEN |
+	  FEATURE_LUMINOUS |
+	  FEATURE_MIMIC |
+          FEATURE_OSDMAP_PRUNE |
+	  FEATURE_NAUTILUS |
+	  FEATURE_OCTOPUS |
+	  FEATURE_PACIFIC |
+	  FEATURE_PINGING |
+	  FEATURE_NONE
+	  );
+      }
+      /**
+       * All the features that, once set, cannot be removed.
+       *
+       * Features should only be added to this list if you want to make
+       * sure downgrades are not possible after a quorum supporting all
+       * these features has been formed.
+       *
+       * Any feature in this list will be automatically set on the monmap's
+       * features once all the monitors in the quorum support it.
+       */
+      constexpr mon_feature_t get_persistent() {
+        return (
+	  FEATURE_KRAKEN |
+	  FEATURE_LUMINOUS |
+	  FEATURE_MIMIC |
+	  FEATURE_NAUTILUS |
+	  FEATURE_OSDMAP_PRUNE |
+	  FEATURE_OCTOPUS |
+	  FEATURE_PACIFIC |
+	  FEATURE_PINGING |
+	  FEATURE_NONE
+	  );
+      }
+
+      constexpr mon_feature_t get_optional() {
+        return (
+          FEATURE_OSDMAP_PRUNE |
+          FEATURE_NONE
+          );
+      }
+
+      static inline mon_feature_t get_feature_by_name(const std::string &n);
+    }
+  }
+}
+
+static inline ceph_release_t infer_ceph_release_from_mon_features(mon_feature_t f)
+{
+  if (f.contains_all(ceph::features::mon::FEATURE_PACIFIC)) {
+    return ceph_release_t::pacific;
+  }
+  if (f.contains_all(ceph::features::mon::FEATURE_OCTOPUS)) {
+    return ceph_release_t::octopus;
+  }
+  if (f.contains_all(ceph::features::mon::FEATURE_NAUTILUS)) {
+    return ceph_release_t::nautilus;
+  }
+  if (f.contains_all(ceph::features::mon::FEATURE_MIMIC)) {
+    return ceph_release_t::mimic;
+  }
+  if (f.contains_all(ceph::features::mon::FEATURE_LUMINOUS)) {
+    return ceph_release_t::luminous;
+  }
+  if (f.contains_all(ceph::features::mon::FEATURE_KRAKEN)) {
+    return ceph_release_t::kraken;
+  }
+  return ceph_release_t::unknown;
+}
+
+static inline const char *ceph::features::mon::get_feature_name(uint64_t b) {
+  mon_feature_t f(b);
+
+  if (f == FEATURE_KRAKEN) {
+    return "kraken";
+  } else if (f == FEATURE_LUMINOUS) {
+    return "luminous";
+  } else if (f == FEATURE_MIMIC) {
+    return "mimic";
+  } else if (f == FEATURE_OSDMAP_PRUNE) {
+    return "osdmap-prune";
+  } else if (f == FEATURE_NAUTILUS) {
+    return "nautilus";
+  } else if (f == FEATURE_PINGING) {
+    return "elector-pinging";
+  } else if (f == FEATURE_OCTOPUS) {
+    return "octopus";
+  } else if (f == FEATURE_PACIFIC) {
+    return "pacific";
+  } else if (f == FEATURE_RESERVED) {
+    return "reserved";
+  }
+  return "unknown";
+}
+
+inline mon_feature_t ceph::features::mon::get_feature_by_name(const std::string &n) {
+
+  if (n == "kraken") {
+    return FEATURE_KRAKEN;
+  } else if (n == "luminous") {
+    return FEATURE_LUMINOUS;
+  } else if (n == "mimic") {
+    return FEATURE_MIMIC;
+  } else if (n == "osdmap-prune") {
+    return FEATURE_OSDMAP_PRUNE;
+  } else if (n == "nautilus") {
+    return FEATURE_NAUTILUS;
+  } else if (n == "feature-pinging") {
+    return FEATURE_PINGING;
+  } else if (n == "octopus") {
+    return FEATURE_OCTOPUS;
+  } else if (n == "pacific") {
+    return FEATURE_PACIFIC;
+  } else if (n == "reserved") {
+    return FEATURE_RESERVED;
+  }
+  return FEATURE_NONE;
+}
+
+inline std::ostream& operator<<(std::ostream& out, const mon_feature_t& f) {
+  out << "mon_feature_t(";
+  f.print(out);
+  out << ")";
+  return out;
+}
+
+
+struct ProgressEvent {
+  std::string message;                  ///< event description
+  float progress;                  ///< [0..1]
+  bool add_to_ceph_s;
+  void encode(ceph::buffer::list& bl) const {
+    ENCODE_START(2, 1, bl);
+    encode(message, bl);
+    encode(progress, bl);
+    encode(add_to_ceph_s, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(ceph::buffer::list::const_iterator& p) {
+    DECODE_START(2, p);
+    decode(message, p);
+    decode(progress, p);
+    if (struct_v >= 2){
+	decode(add_to_ceph_s, p);
+    } else {
+      if (!message.empty()) {
+	add_to_ceph_s = true;
+      }
+    }
+    DECODE_FINISH(p);
+  }
+  void dump(ceph::Formatter *f) const {
+    f->dump_string("message", message);
+    f->dump_float("progress", progress);
+    f->dump_bool("add_to_ceph_s", add_to_ceph_s);
+  }
+};
+WRITE_CLASS_ENCODER(ProgressEvent)
+
+#endif