summaryrefslogtreecommitdiffstats
path: root/src/mgr/MgrStandby.cc
diff options
context:
space:
mode:
Diffstat (limited to 'src/mgr/MgrStandby.cc')
-rw-r--r--src/mgr/MgrStandby.cc503
1 files changed, 503 insertions, 0 deletions
diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc
new file mode 100644
index 000000000..2821bf4cf
--- /dev/null
+++ b/src/mgr/MgrStandby.cc
@@ -0,0 +1,503 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2016 John Spray <john.spray@redhat.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ */
+
+#include <Python.h>
+
+#include "common/errno.h"
+#include "common/signal.h"
+#include "include/compat.h"
+
+#include "include/stringify.h"
+#include "global/global_context.h"
+#include "global/signal_handler.h"
+
+#include "mgr/MgrContext.h"
+#include "mgr/mgr_commands.h"
+#include "mgr/mgr_perf_counters.h"
+
+#include "messages/MMgrBeacon.h"
+#include "messages/MMgrMap.h"
+#include "Mgr.h"
+
+#include "MgrStandby.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_mgr
+#undef dout_prefix
+#define dout_prefix *_dout << "mgr " << __func__ << " "
+
+
+MgrStandby::MgrStandby(int argc, const char **argv) :
+ Dispatcher(g_ceph_context),
+ monc{g_ceph_context, poolctx},
+ client_messenger(Messenger::create(
+ g_ceph_context,
+ cct->_conf.get_val<std::string>("ms_type"),
+ entity_name_t::MGR(),
+ "mgr",
+ Messenger::get_pid_nonce())),
+ objecter{g_ceph_context, client_messenger.get(), &monc, poolctx},
+ client{client_messenger.get(), &monc, &objecter},
+ mgrc(g_ceph_context, client_messenger.get(), &monc.monmap),
+ log_client(g_ceph_context, client_messenger.get(), &monc.monmap, LogClient::NO_FLAGS),
+ clog(log_client.create_channel(CLOG_CHANNEL_CLUSTER)),
+ audit_clog(log_client.create_channel(CLOG_CHANNEL_AUDIT)),
+ finisher(g_ceph_context, "MgrStandby", "mgrsb-fin"),
+ timer(g_ceph_context, lock),
+ py_module_registry(clog),
+ active_mgr(nullptr),
+ orig_argc(argc),
+ orig_argv(argv),
+ available_in_map(false)
+{
+}
+
+MgrStandby::~MgrStandby() = default;
+
+const char** MgrStandby::get_tracked_conf_keys() const
+{
+ static const char* KEYS[] = {
+ // clog & admin clog
+ "clog_to_monitors",
+ "clog_to_syslog",
+ "clog_to_syslog_facility",
+ "clog_to_syslog_level",
+ "clog_to_graylog",
+ "clog_to_graylog_host",
+ "clog_to_graylog_port",
+ "mgr_standby_modules",
+ "host",
+ "fsid",
+ NULL
+ };
+ return KEYS;
+}
+
+void MgrStandby::handle_conf_change(
+ const ConfigProxy& conf,
+ const std::set <std::string> &changed)
+{
+ if (changed.count("clog_to_monitors") ||
+ changed.count("clog_to_syslog") ||
+ changed.count("clog_to_syslog_level") ||
+ changed.count("clog_to_syslog_facility") ||
+ changed.count("clog_to_graylog") ||
+ changed.count("clog_to_graylog_host") ||
+ changed.count("clog_to_graylog_port") ||
+ changed.count("host") ||
+ changed.count("fsid")) {
+ _update_log_config();
+ }
+ if (changed.count("mgr_standby_modules") && !active_mgr) {
+ if (g_conf().get_val<bool>("mgr_standby_modules") != py_module_registry.have_standby_modules()) {
+ dout(1) << "mgr_standby_modules now "
+ << (int)g_conf().get_val<bool>("mgr_standby_modules")
+ << ", standby modules are "
+ << (py_module_registry.have_standby_modules() ? "":"not ")
+ << "active, respawning"
+ << dendl;
+ respawn();
+ }
+ }
+}
+
+int MgrStandby::init()
+{
+ init_async_signal_handler();
+ register_async_signal_handler(SIGHUP, sighup_handler);
+
+ cct->_conf.add_observer(this);
+
+ std::lock_guard l(lock);
+
+ // Start finisher
+ finisher.start();
+
+ // Initialize Messenger
+ client_messenger->add_dispatcher_tail(this);
+ client_messenger->add_dispatcher_head(&objecter);
+ client_messenger->add_dispatcher_tail(&client);
+ client_messenger->start();
+
+ poolctx.start(2);
+
+ // Initialize MonClient
+ if (monc.build_initial_monmap() < 0) {
+ client_messenger->shutdown();
+ client_messenger->wait();
+ return -1;
+ }
+
+ monc.sub_want("mgrmap", 0, 0);
+
+ monc.set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD
+ |CEPH_ENTITY_TYPE_MDS|CEPH_ENTITY_TYPE_MGR);
+ monc.set_messenger(client_messenger.get());
+
+ // We must register our config callback before calling init(), so
+ // that we see the initial configuration message
+ monc.register_config_callback([this](const std::string &k, const std::string &v){
+ // removing value to hide sensitive data going into mgr logs
+ // leaving this for debugging purposes
+ // dout(10) << "config_callback: " << k << " : " << v << dendl;
+ dout(10) << "config_callback: " << k << " : " << dendl;
+ if (k.substr(0, 4) == "mgr/") {
+ py_module_registry.handle_config(k, v);
+ return true;
+ }
+ return false;
+ });
+ monc.register_config_notify_callback([this]() {
+ py_module_registry.handle_config_notify();
+ });
+ dout(4) << "Registered monc callback" << dendl;
+
+ int r = monc.init();
+ if (r < 0) {
+ monc.shutdown();
+ client_messenger->shutdown();
+ client_messenger->wait();
+ return r;
+ }
+ mgrc.init();
+ client_messenger->add_dispatcher_tail(&mgrc);
+
+ r = monc.authenticate();
+ if (r < 0) {
+ derr << "Authentication failed, did you specify a mgr ID with a valid keyring?" << dendl;
+ monc.shutdown();
+ client_messenger->shutdown();
+ client_messenger->wait();
+ return r;
+ }
+ // only forward monmap updates after authentication finishes, otherwise
+ // monc.authenticate() will be waiting for MgrStandy::ms_dispatch()
+ // to acquire the lock forever, as it is already locked in the beginning of
+ // this method.
+ monc.set_passthrough_monmap();
+
+ client_t whoami = monc.get_global_id();
+ client_messenger->set_myname(entity_name_t::MGR(whoami.v));
+ monc.set_log_client(&log_client);
+ _update_log_config();
+ objecter.set_client_incarnation(0);
+ objecter.init();
+ objecter.start();
+ client.init();
+ timer.init();
+
+ py_module_registry.init();
+ mgr_perf_start(g_ceph_context);
+
+
+ tick();
+
+ dout(4) << "Complete." << dendl;
+ return 0;
+}
+
+void MgrStandby::send_beacon()
+{
+ ceph_assert(ceph_mutex_is_locked_by_me(lock));
+ dout(20) << state_str() << dendl;
+
+ auto modules = py_module_registry.get_modules();
+
+ // Construct a list of the info about each loaded module
+ // which we will transmit to the monitor.
+ std::vector<MgrMap::ModuleInfo> module_info;
+ for (const auto &module : modules) {
+ MgrMap::ModuleInfo info;
+ info.name = module->get_name();
+ info.error_string = module->get_error_string();
+ info.can_run = module->get_can_run();
+ info.module_options = module->get_options();
+ module_info.push_back(std::move(info));
+ }
+
+ auto clients = py_module_registry.get_clients();
+ for (const auto& client : clients) {
+ dout(15) << "noting RADOS client for blocklist: " << client << dendl;
+ }
+
+ // Whether I think I am available (request MgrMonitor to set me
+ // as available in the map)
+ bool available = active_mgr != nullptr && active_mgr->is_initialized();
+
+ auto addrs = available ? active_mgr->get_server_addrs() : entity_addrvec_t();
+ dout(10) << "sending beacon as gid " << monc.get_global_id() << dendl;
+
+ map<string,string> metadata;
+ metadata["addr"] = client_messenger->get_myaddr_legacy().ip_only_to_str();
+ metadata["addrs"] = stringify(client_messenger->get_myaddrs());
+ collect_sys_info(&metadata, g_ceph_context);
+
+ auto m = ceph::make_message<MMgrBeacon>(monc.get_fsid(),
+ monc.get_global_id(),
+ g_conf()->name.get_id(),
+ addrs,
+ available,
+ std::move(module_info),
+ std::move(metadata),
+ std::move(clients),
+ CEPH_FEATURES_ALL);
+
+ if (available) {
+ if (!available_in_map) {
+ // We are informing the mon that we are done initializing: inform
+ // it of our command set. This has to happen after init() because
+ // it needs the python modules to have loaded.
+ std::vector<MonCommand> commands = mgr_commands;
+ std::vector<MonCommand> py_commands = py_module_registry.get_commands();
+ commands.insert(commands.end(), py_commands.begin(), py_commands.end());
+ m->set_command_descs(commands);
+ dout(4) << "going active, including " << m->get_command_descs().size()
+ << " commands in beacon" << dendl;
+ }
+
+ m->set_services(active_mgr->get_services());
+ }
+
+ monc.send_mon_message(std::move(m));
+}
+
+void MgrStandby::tick()
+{
+ dout(10) << __func__ << dendl;
+ send_beacon();
+
+ timer.add_event_after(
+ g_conf().get_val<std::chrono::seconds>("mgr_tick_period").count(),
+ new LambdaContext([this](int r){
+ tick();
+ }
+ ));
+}
+
+void MgrStandby::shutdown()
+{
+ finisher.queue(new LambdaContext([&](int) {
+ std::lock_guard l(lock);
+
+ dout(4) << "Shutting down" << dendl;
+
+ py_module_registry.shutdown();
+ // stop sending beacon first, I use monc to talk with monitors
+ timer.shutdown();
+ // client uses monc and objecter
+ client.shutdown();
+ mgrc.shutdown();
+ // Stop asio threads, so leftover events won't call into shut down
+ // monclient/objecter.
+ poolctx.finish();
+ // stop monc, so mon won't be able to instruct me to shutdown/activate after
+ // the active_mgr is stopped
+ monc.shutdown();
+ if (active_mgr) {
+ active_mgr->shutdown();
+ }
+ // objecter is used by monc and active_mgr
+ objecter.shutdown();
+ // client_messenger is used by all of them, so stop it in the end
+ client_messenger->shutdown();
+ }));
+
+ // Then stop the finisher to ensure its enqueued contexts aren't going
+ // to touch references to the things we're about to tear down
+ finisher.wait_for_empty();
+ finisher.stop();
+ mgr_perf_stop(g_ceph_context);
+}
+
+void MgrStandby::respawn()
+{
+ // --- WARNING TO FUTURE COPY/PASTERS ---
+ // You must also add a call like
+ //
+ // ceph_pthread_setname(pthread_self(), "ceph-mgr");
+ //
+ // to main() so that /proc/$pid/stat field 2 contains "(ceph-mgr)"
+ // instead of "(exe)", so that killall (and log rotation) will work.
+
+ char *new_argv[orig_argc+1];
+ dout(1) << " e: '" << orig_argv[0] << "'" << dendl;
+ for (int i=0; i<orig_argc; i++) {
+ new_argv[i] = (char *)orig_argv[i];
+ dout(1) << " " << i << ": '" << orig_argv[i] << "'" << dendl;
+ }
+ new_argv[orig_argc] = NULL;
+
+ /* Determine the path to our executable, test if Linux /proc/self/exe exists.
+ * This allows us to exec the same executable even if it has since been
+ * unlinked.
+ */
+ char exe_path[PATH_MAX] = "";
+ if (readlink(PROCPREFIX "/proc/self/exe", exe_path, PATH_MAX-1) == -1) {
+ /* Print CWD for the user's interest */
+ char buf[PATH_MAX];
+ char *cwd = getcwd(buf, sizeof(buf));
+ ceph_assert(cwd);
+ dout(1) << " cwd " << cwd << dendl;
+
+ /* Fall back to a best-effort: just running in our CWD */
+ strncpy(exe_path, orig_argv[0], PATH_MAX-1);
+ } else {
+ dout(1) << "respawning with exe " << exe_path << dendl;
+ strcpy(exe_path, PROCPREFIX "/proc/self/exe");
+ }
+
+ dout(1) << " exe_path " << exe_path << dendl;
+
+ unblock_all_signals(NULL);
+ execv(exe_path, new_argv);
+
+ derr << "respawn execv " << orig_argv[0]
+ << " failed with " << cpp_strerror(errno) << dendl;
+ ceph_abort();
+}
+
+void MgrStandby::_update_log_config()
+{
+ map<string,string> log_to_monitors;
+ map<string,string> log_to_syslog;
+ map<string,string> log_channel;
+ map<string,string> log_prio;
+ map<string,string> log_to_graylog;
+ map<string,string> log_to_graylog_host;
+ map<string,string> log_to_graylog_port;
+ uuid_d fsid;
+ string host;
+
+ if (parse_log_client_options(cct, log_to_monitors, log_to_syslog,
+ log_channel, log_prio, log_to_graylog,
+ log_to_graylog_host, log_to_graylog_port,
+ fsid, host) == 0) {
+ clog->update_config(log_to_monitors, log_to_syslog,
+ log_channel, log_prio, log_to_graylog,
+ log_to_graylog_host, log_to_graylog_port,
+ fsid, host);
+ audit_clog->update_config(log_to_monitors, log_to_syslog,
+ log_channel, log_prio, log_to_graylog,
+ log_to_graylog_host, log_to_graylog_port,
+ fsid, host);
+ }
+}
+
+void MgrStandby::handle_mgr_map(ref_t<MMgrMap> mmap)
+{
+ auto &map = mmap->get_map();
+ dout(4) << "received map epoch " << map.get_epoch() << dendl;
+ const bool active_in_map = map.active_gid == monc.get_global_id();
+ dout(4) << "active in map: " << active_in_map
+ << " active is " << map.active_gid << dendl;
+
+ // PyModuleRegistry may ask us to respawn if it sees that
+ // this MgrMap is changing its set of enabled modules
+ bool need_respawn = py_module_registry.handle_mgr_map(map);
+ if (need_respawn) {
+ dout(1) << "respawning because set of enabled modules changed!" << dendl;
+ respawn();
+ }
+
+ if (active_in_map) {
+ if (!active_mgr) {
+ dout(1) << "Activating!" << dendl;
+ active_mgr.reset(new Mgr(&monc, map, &py_module_registry,
+ client_messenger.get(), &objecter,
+ &client, clog, audit_clog));
+ active_mgr->background_init(new LambdaContext(
+ [this](int r){
+ // Advertise our active-ness ASAP instead of waiting for
+ // next tick.
+ std::lock_guard l(lock);
+ send_beacon();
+ }));
+ dout(1) << "I am now activating" << dendl;
+ } else {
+ dout(10) << "I was already active" << dendl;
+ bool need_respawn = active_mgr->got_mgr_map(map);
+ if (need_respawn) {
+ respawn();
+ }
+ }
+
+ if (!available_in_map && map.get_available()) {
+ dout(4) << "Map now says I am available" << dendl;
+ available_in_map = true;
+ }
+ } else if (active_mgr != nullptr) {
+ derr << "I was active but no longer am" << dendl;
+ respawn();
+ } else {
+ if (map.active_gid != 0 && map.active_name != g_conf()->name.get_id()) {
+ // I am the standby and someone else is active, start modules
+ // in standby mode to do redirects if needed
+ if (!py_module_registry.is_standby_running() &&
+ g_conf().get_val<bool>("mgr_standby_modules")) {
+ py_module_registry.standby_start(monc, finisher);
+ }
+ }
+ }
+}
+
+bool MgrStandby::ms_dispatch2(const ref_t<Message>& m)
+{
+ std::lock_guard l(lock);
+ dout(10) << state_str() << " " << *m << dendl;
+
+ if (m->get_type() == MSG_MGR_MAP) {
+ handle_mgr_map(ref_cast<MMgrMap>(m));
+ }
+ bool handled = false;
+ if (active_mgr) {
+ auto am = active_mgr;
+ lock.unlock();
+ handled = am->ms_dispatch2(m);
+ lock.lock();
+ }
+ if (m->get_type() == MSG_MGR_MAP) {
+ // let this pass through for mgrc
+ handled = false;
+ }
+ return handled;
+}
+
+
+bool MgrStandby::ms_handle_refused(Connection *con)
+{
+ // do nothing for now
+ return false;
+}
+
+int MgrStandby::main(vector<const char *> args)
+{
+ client_messenger->wait();
+
+ // Disable signal handlers
+ unregister_async_signal_handler(SIGHUP, sighup_handler);
+ shutdown_async_signal_handler();
+
+ return 0;
+}
+
+
+std::string MgrStandby::state_str()
+{
+ if (active_mgr == nullptr) {
+ return "standby";
+ } else if (active_mgr->is_initialized()) {
+ return "active";
+ } else {
+ return "active (starting)";
+ }
+}