diff options
Diffstat (limited to 'daemons/fenced')
| -rw-r--r-- | daemons/fenced/Makefile.am | 52 | ||||
| -rw-r--r-- | daemons/fenced/cts-fence-helper.c | 681 | ||||
| -rw-r--r-- | daemons/fenced/fenced_commands.c | 3674 | ||||
| -rw-r--r-- | daemons/fenced/fenced_history.c | 548 | ||||
| -rw-r--r-- | daemons/fenced/fenced_remote.c | 2509 | ||||
| -rw-r--r-- | daemons/fenced/pacemaker-fenced.c | 1751 | ||||
| -rw-r--r-- | daemons/fenced/pacemaker-fenced.h | 315 |
7 files changed, 9530 insertions, 0 deletions
diff --git a/daemons/fenced/Makefile.am b/daemons/fenced/Makefile.am new file mode 100644 index 0000000..2ca0088 --- /dev/null +++ b/daemons/fenced/Makefile.am @@ -0,0 +1,52 @@ +# +# Original Author: Sun Jiang Dong <sunjd@cn.ibm.com> +# Copyright 2004 International Business Machines +# +# with later changes copyright 2004-2023 the Pacemaker project contributors. +# The version control history for this file may have further details. +# +# This source code is licensed under the GNU General Public License version 2 +# or later (GPLv2+) WITHOUT ANY WARRANTY. +# + +include $(top_srcdir)/mk/common.mk +include $(top_srcdir)/mk/man.mk + +halibdir = $(CRM_DAEMON_DIR) + +halib_PROGRAMS = pacemaker-fenced cts-fence-helper + +noinst_HEADERS = pacemaker-fenced.h + +if BUILD_XML_HELP +man7_MANS = pacemaker-fenced.7 +endif + +cts_fence_helper_SOURCES = cts-fence-helper.c +cts_fence_helper_LDADD = $(top_builddir)/lib/common/libcrmcommon.la \ + $(top_builddir)/lib/fencing/libstonithd.la + +pacemaker_fenced_YFLAGS = -d +pacemaker_fenced_CFLAGS = $(CFLAGS_HARDENED_EXE) +pacemaker_fenced_LDFLAGS = $(LDFLAGS_HARDENED_EXE) +pacemaker_fenced_LDADD = $(top_builddir)/lib/common/libcrmcommon.la \ + $(top_builddir)/lib/cib/libcib.la \ + $(top_builddir)/lib/cluster/libcrmcluster.la \ + $(top_builddir)/lib/fencing/libstonithd.la \ + $(top_builddir)/lib/pengine/libpe_status.la \ + $(top_builddir)/lib/pacemaker/libpacemaker.la \ + $(CLUSTERLIBS) +pacemaker_fenced_SOURCES = pacemaker-fenced.c \ + fenced_commands.c \ + fenced_remote.c \ + fenced_history.c + +CLEANFILES = $(man7_MANS) $(man8_MANS) + +if BUILD_LEGACY_LINKS +install-exec-hook: + cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f stonithd && $(LN_S) pacemaker-fenced stonithd + +uninstall-hook: + cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f stonithd +endif diff --git a/daemons/fenced/cts-fence-helper.c b/daemons/fenced/cts-fence-helper.c new file mode 100644 index 0000000..e18a1f4 --- /dev/null +++ b/daemons/fenced/cts-fence-helper.c @@ -0,0 +1,681 @@ +/* + * Copyright 2009-2023 the Pacemaker project contributors + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. + */ + +#include <crm_internal.h> + +#include <sys/param.h> +#include <stdio.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <sys/utsname.h> + +#include <stdlib.h> +#include <errno.h> +#include <fcntl.h> + +#include <crm/crm.h> +#include <crm/msg_xml.h> +#include <crm/common/ipc.h> +#include <crm/cluster/internal.h> + +#include <crm/stonith-ng.h> +#include <crm/fencing/internal.h> +#include <crm/common/agents.h> +#include <crm/common/cmdline_internal.h> +#include <crm/common/xml.h> + +#include <crm/common/mainloop.h> + +#define SUMMARY "cts-fence-helper - inject commands into the Pacemaker fencer and watch for events" + +static GMainLoop *mainloop = NULL; +static crm_trigger_t *trig = NULL; +static int mainloop_iter = 0; +static pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; + +typedef void (*mainloop_test_iteration_cb) (int check_event); + +#define MAINLOOP_DEFAULT_TIMEOUT 2 + +enum test_modes { + test_standard = 0, // test using a specific developer environment + test_passive, // watch notifications only + test_api_sanity, // sanity-test stonith client API using fence_dummy + test_api_mainloop, // sanity-test mainloop code with async responses +}; + +struct { + enum test_modes mode; +} options = { + .mode = test_standard +}; + +static gboolean +mode_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) { + if (pcmk__str_any_of(option_name, "--mainloop_api_test", "-m", NULL)) { + options.mode = test_api_mainloop; + } else if (pcmk__str_any_of(option_name, "--api_test", "-t", NULL)) { + options.mode = test_api_sanity; + } else if (pcmk__str_any_of(option_name, "--passive", "-p", NULL)) { + options.mode = test_passive; + } + + return TRUE; +} + +static GOptionEntry entries[] = { + { "mainloop_api_test", 'm', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, mode_cb, + NULL, NULL, + }, + + { "api_test", 't', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, mode_cb, + NULL, NULL, + }, + + { "passive", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, mode_cb, + NULL, NULL, + }, + + { NULL } +}; + +static stonith_t *st = NULL; +static struct pollfd pollfd; +static const int st_opts = st_opt_sync_call; +static int expected_notifications = 0; +static int verbose = 0; + +static void +mainloop_test_done(const char *origin, bool pass) +{ + if (pass) { + crm_info("SUCCESS - %s", origin); + mainloop_iter++; + mainloop_set_trigger(trig); + result.execution_status = PCMK_EXEC_DONE; + result.exit_status = CRM_EX_OK; + } else { + crm_err("FAILURE - %s (%d: %s)", origin, result.exit_status, + pcmk_exec_status_str(result.execution_status)); + crm_exit(CRM_EX_ERROR); + } +} + + +static void +dispatch_helper(int timeout) +{ + int rc; + + crm_debug("Looking for notification"); + pollfd.events = POLLIN; + while (true) { + rc = poll(&pollfd, 1, timeout); /* wait 10 minutes, -1 forever */ + if (rc > 0) { + if (!stonith_dispatch(st)) { + break; + } + } else { + break; + } + } +} + +static void +st_callback(stonith_t * st, stonith_event_t * e) +{ + char *desc = NULL; + + if (st->state == stonith_disconnected) { + crm_exit(CRM_EX_DISCONNECT); + } + + desc = stonith__event_description(e); + crm_notice("%s", desc); + free(desc); + + if (expected_notifications) { + expected_notifications--; + } +} + +static void +st_global_callback(stonith_t * stonith, stonith_callback_data_t * data) +{ + crm_notice("Call %d exited %d: %s (%s)", + data->call_id, stonith__exit_status(data), + stonith__execution_status(data), + pcmk__s(stonith__exit_reason(data), "unspecified reason")); +} + +static void +passive_test(void) +{ + int rc = 0; + + rc = st->cmds->connect(st, crm_system_name, &pollfd.fd); + if (rc != pcmk_ok) { + stonith_api_delete(st); + crm_exit(CRM_EX_DISCONNECT); + } + st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT, st_callback); + st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, st_callback); + st->cmds->register_notification(st, STONITH_OP_DEVICE_ADD, st_callback); + st->cmds->register_notification(st, STONITH_OP_DEVICE_DEL, st_callback); + st->cmds->register_callback(st, 0, 120, st_opt_timeout_updates, NULL, "st_global_callback", + st_global_callback); + + dispatch_helper(600 * 1000); +} + +#define single_test(cmd, str, num_notifications, expected_rc) \ +{ \ + int rc = 0; \ + rc = cmd; \ + expected_notifications = 0; \ + if (num_notifications) { \ + expected_notifications = num_notifications; \ + dispatch_helper(500); \ + } \ + if (rc != expected_rc) { \ + crm_err("FAILURE - expected rc %d != %d(%s) for cmd - %s", expected_rc, rc, pcmk_strerror(rc), str); \ + crm_exit(CRM_EX_ERROR); \ + } else if (expected_notifications) { \ + crm_err("FAILURE - expected %d notifications, got only %d for cmd - %s", \ + num_notifications, num_notifications - expected_notifications, str); \ + crm_exit(CRM_EX_ERROR); \ + } else { \ + if (verbose) { \ + crm_info("SUCCESS - %s: %d", str, rc); \ + } else { \ + crm_debug("SUCCESS - %s: %d", str, rc); \ + } \ + } \ +}\ + +static void +run_fence_failure_test(void) +{ + stonith_key_value_t *params = NULL; + + params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP, + "false_1_node1=1,2 false_1_node2=3,4"); + params = stonith_key_value_add(params, "mode", "fail"); + + single_test(st-> + cmds->register_device(st, st_opts, "test-id1", "stonith-ng", "fence_dummy", params), + "Register device1 for failure test", 1, 0); + + single_test(st->cmds->fence(st, st_opts, "false_1_node2", "off", 3, 0), + "Fence failure results off", 1, -ENODATA); + + single_test(st->cmds->fence(st, st_opts, "false_1_node2", "reboot", 3, 0), + "Fence failure results reboot", 1, -ENODATA); + + single_test(st->cmds->remove_device(st, st_opts, "test-id1"), + "Remove device1 for failure test", 1, 0); + + stonith_key_value_freeall(params, 1, 1); +} + +static void +run_fence_failure_rollover_test(void) +{ + stonith_key_value_t *params = NULL; + + params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP, + "false_1_node1=1,2 false_1_node2=3,4"); + params = stonith_key_value_add(params, "mode", "fail"); + + single_test(st-> + cmds->register_device(st, st_opts, "test-id1", "stonith-ng", "fence_dummy", params), + "Register device1 for rollover test", 1, 0); + stonith_key_value_freeall(params, 1, 1); + params = NULL; + params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP, + "false_1_node1=1,2 false_1_node2=3,4"); + params = stonith_key_value_add(params, "mode", "pass"); + + single_test(st-> + cmds->register_device(st, st_opts, "test-id2", "stonith-ng", "fence_dummy", params), + "Register device2 for rollover test", 1, 0); + + single_test(st->cmds->fence(st, st_opts, "false_1_node2", "off", 3, 0), + "Fence rollover results off", 1, 0); + + /* Expect -ENODEV because fence_dummy requires 'on' to be executed on target */ + single_test(st->cmds->fence(st, st_opts, "false_1_node2", "on", 3, 0), + "Fence rollover results on", 1, -ENODEV); + + single_test(st->cmds->remove_device(st, st_opts, "test-id1"), + "Remove device1 for rollover tests", 1, 0); + + single_test(st->cmds->remove_device(st, st_opts, "test-id2"), + "Remove device2 for rollover tests", 1, 0); + + stonith_key_value_freeall(params, 1, 1); +} + +static void +run_standard_test(void) +{ + stonith_key_value_t *params = NULL; + + params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP, + "false_1_node1=1,2 false_1_node2=3,4"); + params = stonith_key_value_add(params, "mode", "pass"); + params = stonith_key_value_add(params, "mock_dynamic_hosts", "false_1_node1 false_1_node2"); + + single_test(st-> + cmds->register_device(st, st_opts, "test-id", "stonith-ng", "fence_dummy", params), + "Register", 1, 0); + stonith_key_value_freeall(params, 1, 1); + params = NULL; + + single_test(st->cmds->list(st, st_opts, "test-id", NULL, 1), "list", 1, 0); + + single_test(st->cmds->monitor(st, st_opts, "test-id", 1), "Monitor", 1, 0); + + single_test(st->cmds->status(st, st_opts, "test-id", "false_1_node2", 1), + "Status false_1_node2", 1, 0); + + single_test(st->cmds->status(st, st_opts, "test-id", "false_1_node1", 1), + "Status false_1_node1", 1, 0); + + single_test(st->cmds->fence(st, st_opts, "unknown-host", "off", 1, 0), + "Fence unknown-host (expected failure)", 0, -ENODEV); + + single_test(st->cmds->fence(st, st_opts, "false_1_node1", "off", 1, 0), + "Fence false_1_node1", 1, 0); + + /* Expect -ENODEV because fence_dummy requires 'on' to be executed on target */ + single_test(st->cmds->fence(st, st_opts, "false_1_node1", "on", 1, 0), + "Unfence false_1_node1", 1, -ENODEV); + + /* Confirm that an invalid level index is rejected */ + single_test(st->cmds->register_level(st, st_opts, "node1", 999, params), + "Attempt to register an invalid level index", 0, -EINVAL); + + single_test(st->cmds->remove_device(st, st_opts, "test-id"), "Remove test-id", 1, 0); + + stonith_key_value_freeall(params, 1, 1); +} + +static void +sanity_tests(void) +{ + int rc = 0; + + rc = st->cmds->connect(st, crm_system_name, &pollfd.fd); + if (rc != pcmk_ok) { + stonith_api_delete(st); + crm_exit(CRM_EX_DISCONNECT); + } + st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT, st_callback); + st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, st_callback); + st->cmds->register_notification(st, STONITH_OP_DEVICE_ADD, st_callback); + st->cmds->register_notification(st, STONITH_OP_DEVICE_DEL, st_callback); + st->cmds->register_callback(st, 0, 120, st_opt_timeout_updates, NULL, "st_global_callback", + st_global_callback); + + crm_info("Starting API Sanity Tests"); + run_standard_test(); + run_fence_failure_test(); + run_fence_failure_rollover_test(); + crm_info("Sanity Tests Passed"); +} + +static void +standard_dev_test(void) +{ + int rc = 0; + char *tmp = NULL; + stonith_key_value_t *params = NULL; + + rc = st->cmds->connect(st, crm_system_name, &pollfd.fd); + if (rc != pcmk_ok) { + stonith_api_delete(st); + crm_exit(CRM_EX_DISCONNECT); + } + + params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP, + "some-host=pcmk-7 true_1_node1=3,4"); + + rc = st->cmds->register_device(st, st_opts, "test-id", "stonith-ng", "fence_xvm", params); + crm_debug("Register: %d", rc); + + rc = st->cmds->list(st, st_opts, "test-id", &tmp, 10); + crm_debug("List: %d output: %s", rc, tmp ? tmp : "<none>"); + + rc = st->cmds->monitor(st, st_opts, "test-id", 10); + crm_debug("Monitor: %d", rc); + + rc = st->cmds->status(st, st_opts, "test-id", "false_1_node2", 10); + crm_debug("Status false_1_node2: %d", rc); + + rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10); + crm_debug("Status false_1_node1: %d", rc); + + rc = st->cmds->fence(st, st_opts, "unknown-host", "off", 60, 0); + crm_debug("Fence unknown-host: %d", rc); + + rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10); + crm_debug("Status false_1_node1: %d", rc); + + rc = st->cmds->fence(st, st_opts, "false_1_node1", "off", 60, 0); + crm_debug("Fence false_1_node1: %d", rc); + + rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10); + crm_debug("Status false_1_node1: %d", rc); + + rc = st->cmds->fence(st, st_opts, "false_1_node1", "on", 10, 0); + crm_debug("Unfence false_1_node1: %d", rc); + + rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10); + crm_debug("Status false_1_node1: %d", rc); + + rc = st->cmds->fence(st, st_opts, "some-host", "off", 10, 0); + crm_debug("Fence alias: %d", rc); + + rc = st->cmds->status(st, st_opts, "test-id", "some-host", 10); + crm_debug("Status alias: %d", rc); + + rc = st->cmds->fence(st, st_opts, "false_1_node1", "on", 10, 0); + crm_debug("Unfence false_1_node1: %d", rc); + + rc = st->cmds->remove_device(st, st_opts, "test-id"); + crm_debug("Remove test-id: %d", rc); + + stonith_key_value_freeall(params, 1, 1); +} + +static void + iterate_mainloop_tests(gboolean event_ready); + +static void +mainloop_callback(stonith_t * stonith, stonith_callback_data_t * data) +{ + pcmk__set_result(&result, stonith__exit_status(data), + stonith__execution_status(data), + stonith__exit_reason(data)); + iterate_mainloop_tests(TRUE); +} + +static int +register_callback_helper(int callid) +{ + return st->cmds->register_callback(st, + callid, + MAINLOOP_DEFAULT_TIMEOUT, + st_opt_timeout_updates, NULL, "callback", mainloop_callback); +} + +static void +test_async_fence_pass(int check_event) +{ + int rc = 0; + + if (check_event) { + mainloop_test_done(__func__, (result.exit_status == CRM_EX_OK)); + return; + } + + rc = st->cmds->fence(st, 0, "true_1_node1", "off", MAINLOOP_DEFAULT_TIMEOUT, 0); + if (rc < 0) { + crm_err("fence failed with rc %d", rc); + mainloop_test_done(__func__, false); + } + register_callback_helper(rc); + /* wait for event */ +} + +#define CUSTOM_TIMEOUT_ADDITION 10 +static void +test_async_fence_custom_timeout(int check_event) +{ + int rc = 0; + static time_t begin = 0; + + if (check_event) { + uint32_t diff = (time(NULL) - begin); + + if (result.execution_status != PCMK_EXEC_TIMEOUT) { + mainloop_test_done(__func__, false); + } else if (diff < CUSTOM_TIMEOUT_ADDITION + MAINLOOP_DEFAULT_TIMEOUT) { + crm_err + ("Custom timeout test failed, callback expiration should be updated to %d, actual timeout was %d", + CUSTOM_TIMEOUT_ADDITION + MAINLOOP_DEFAULT_TIMEOUT, diff); + mainloop_test_done(__func__, false); + } else { + mainloop_test_done(__func__, true); + } + return; + } + begin = time(NULL); + + rc = st->cmds->fence(st, 0, "custom_timeout_node1", "off", MAINLOOP_DEFAULT_TIMEOUT, 0); + if (rc < 0) { + crm_err("fence failed with rc %d", rc); + mainloop_test_done(__func__, false); + } + register_callback_helper(rc); + /* wait for event */ +} + +static void +test_async_fence_timeout(int check_event) +{ + int rc = 0; + + if (check_event) { + mainloop_test_done(__func__, + (result.execution_status == PCMK_EXEC_NO_FENCE_DEVICE)); + return; + } + + rc = st->cmds->fence(st, 0, "false_1_node2", "off", MAINLOOP_DEFAULT_TIMEOUT, 0); + if (rc < 0) { + crm_err("fence failed with rc %d", rc); + mainloop_test_done(__func__, false); + } + register_callback_helper(rc); + /* wait for event */ +} + +static void +test_async_monitor(int check_event) +{ + int rc = 0; + + if (check_event) { + mainloop_test_done(__func__, (result.exit_status == CRM_EX_OK)); + return; + } + + rc = st->cmds->monitor(st, 0, "false_1", MAINLOOP_DEFAULT_TIMEOUT); + if (rc < 0) { + crm_err("monitor failed with rc %d", rc); + mainloop_test_done(__func__, false); + } + + register_callback_helper(rc); + /* wait for event */ +} + +static void +test_register_async_devices(int check_event) +{ + char buf[16] = { 0, }; + stonith_key_value_t *params = NULL; + + params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP, + "false_1_node1=1,2"); + params = stonith_key_value_add(params, "mode", "fail"); + st->cmds->register_device(st, st_opts, "false_1", "stonith-ng", "fence_dummy", params); + stonith_key_value_freeall(params, 1, 1); + + params = NULL; + params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP, + "true_1_node1=1,2"); + params = stonith_key_value_add(params, "mode", "pass"); + st->cmds->register_device(st, st_opts, "true_1", "stonith-ng", "fence_dummy", params); + stonith_key_value_freeall(params, 1, 1); + + params = NULL; + params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP, + "custom_timeout_node1=1,2"); + params = stonith_key_value_add(params, "mode", "fail"); + params = stonith_key_value_add(params, "delay", "1000"); + snprintf(buf, sizeof(buf) - 1, "%d", MAINLOOP_DEFAULT_TIMEOUT + CUSTOM_TIMEOUT_ADDITION); + params = stonith_key_value_add(params, "pcmk_off_timeout", buf); + st->cmds->register_device(st, st_opts, "false_custom_timeout", "stonith-ng", "fence_dummy", + params); + stonith_key_value_freeall(params, 1, 1); + + mainloop_test_done(__func__, true); +} + +static void +try_mainloop_connect(int check_event) +{ + int rc = stonith_api_connect_retry(st, crm_system_name, 10); + + if (rc == pcmk_ok) { + mainloop_test_done(__func__, true); + return; + } + crm_err("API CONNECTION FAILURE"); + mainloop_test_done(__func__, false); +} + +static void +iterate_mainloop_tests(gboolean event_ready) +{ + static mainloop_test_iteration_cb callbacks[] = { + try_mainloop_connect, + test_register_async_devices, + test_async_monitor, + test_async_fence_pass, + test_async_fence_timeout, + test_async_fence_custom_timeout, + }; + + if (mainloop_iter == (sizeof(callbacks) / sizeof(mainloop_test_iteration_cb))) { + /* all tests ran, everything passed */ + crm_info("ALL MAINLOOP TESTS PASSED!"); + crm_exit(CRM_EX_OK); + } + + callbacks[mainloop_iter] (event_ready); +} + +static gboolean +trigger_iterate_mainloop_tests(gpointer user_data) +{ + iterate_mainloop_tests(FALSE); + return TRUE; +} + +static void +test_shutdown(int nsig) +{ + int rc = 0; + + if (st) { + rc = st->cmds->disconnect(st); + crm_info("Disconnect: %d", rc); + + crm_debug("Destroy"); + stonith_api_delete(st); + } + + if (rc) { + crm_exit(CRM_EX_ERROR); + } +} + +static void +mainloop_tests(void) +{ + trig = mainloop_add_trigger(G_PRIORITY_HIGH, trigger_iterate_mainloop_tests, NULL); + mainloop_set_trigger(trig); + mainloop_add_signal(SIGTERM, test_shutdown); + + crm_info("Starting"); + mainloop = g_main_loop_new(NULL, FALSE); + g_main_loop_run(mainloop); +} + +static GOptionContext * +build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) { + GOptionContext *context = NULL; + + context = pcmk__build_arg_context(args, NULL, group, NULL); + pcmk__add_main_args(context, entries); + return context; +} + +int +main(int argc, char **argv) +{ + GError *error = NULL; + crm_exit_t exit_code = CRM_EX_OK; + + pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY); + gchar **processed_args = pcmk__cmdline_preproc(argv, NULL); + GOptionContext *context = build_arg_context(args, NULL); + + if (!g_option_context_parse_strv(context, &processed_args, &error)) { + exit_code = CRM_EX_USAGE; + goto done; + } + + /* We have to use crm_log_init here to set up the logging because there's + * different handling for daemons vs. command line programs, and + * pcmk__cli_init_logging is set up to only handle the latter. + */ + crm_log_init(NULL, LOG_INFO, TRUE, (verbose? TRUE : FALSE), argc, argv, + FALSE); + + for (int i = 0; i < args->verbosity; i++) { + crm_bump_log_level(argc, argv); + } + + st = stonith_api_new(); + if (st == NULL) { + exit_code = CRM_EX_DISCONNECT; + g_set_error(&error, PCMK__EXITC_ERROR, exit_code, + "Could not connect to fencer: API memory allocation failed"); + goto done; + } + + switch (options.mode) { + case test_standard: + standard_dev_test(); + break; + case test_passive: + passive_test(); + break; + case test_api_sanity: + sanity_tests(); + break; + case test_api_mainloop: + mainloop_tests(); + break; + } + + test_shutdown(0); + +done: + g_strfreev(processed_args); + pcmk__free_arg_context(context); + + pcmk__output_and_clear_error(&error, NULL); + crm_exit(exit_code); +} diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c new file mode 100644 index 0000000..ba63cf8 --- /dev/null +++ b/daemons/fenced/fenced_commands.c @@ -0,0 +1,3674 @@ +/* + * Copyright 2009-2023 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. + */ + +#include <crm_internal.h> + +#include <sys/param.h> +#include <stdio.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <sys/stat.h> +#include <unistd.h> +#include <sys/utsname.h> + +#include <stdlib.h> +#include <errno.h> +#include <fcntl.h> +#include <ctype.h> + +#include <crm/crm.h> +#include <crm/msg_xml.h> +#include <crm/common/ipc.h> +#include <crm/common/ipc_internal.h> +#include <crm/cluster/internal.h> +#include <crm/common/mainloop.h> + +#include <crm/stonith-ng.h> +#include <crm/fencing/internal.h> +#include <crm/common/xml.h> + +#include <pacemaker-fenced.h> + +GHashTable *device_list = NULL; +GHashTable *topology = NULL; +static GList *cmd_list = NULL; + +static GHashTable *fenced_handlers = NULL; + +struct device_search_s { + /* target of fence action */ + char *host; + /* requested fence action */ + char *action; + /* timeout to use if a device is queried dynamically for possible targets */ + int per_device_timeout; + /* number of registered fencing devices at time of request */ + int replies_needed; + /* number of device replies received so far */ + int replies_received; + /* whether the target is eligible to perform requested action (or off) */ + bool allow_suicide; + + /* private data to pass to search callback function */ + void *user_data; + /* function to call when all replies have been received */ + void (*callback) (GList * devices, void *user_data); + /* devices capable of performing requested action (or off if remapping) */ + GList *capable; + /* Whether to perform searches that support the action */ + uint32_t support_action_only; +}; + +static gboolean stonith_device_dispatch(gpointer user_data); +static void st_child_done(int pid, const pcmk__action_result_t *result, + void *user_data); +static void stonith_send_reply(xmlNode * reply, int call_options, const char *remote_peer, + pcmk__client_t *client); + +static void search_devices_record_result(struct device_search_s *search, const char *device, + gboolean can_fence); + +static int get_agent_metadata(const char *agent, xmlNode **metadata); +static void read_action_metadata(stonith_device_t *device); +static enum fenced_target_by unpack_level_kind(const xmlNode *level); + +typedef struct async_command_s { + + int id; + int pid; + int fd_stdout; + int options; + int default_timeout; /* seconds */ + int timeout; /* seconds */ + + int start_delay; // seconds (-1 means disable static/random fencing delays) + int delay_id; + + char *op; + char *origin; + char *client; + char *client_name; + char *remote_op_id; + + char *target; + uint32_t target_nodeid; + char *action; + char *device; + + GList *device_list; + GList *next_device_iter; // device_list entry for next device to execute + + void *internal_user_data; + void (*done_cb) (int pid, const pcmk__action_result_t *result, + void *user_data); + guint timer_sigterm; + guint timer_sigkill; + /*! If the operation timed out, this is the last signal + * we sent to the process to get it to terminate */ + int last_timeout_signo; + + stonith_device_t *active_on; + stonith_device_t *activating_on; +} async_command_t; + +static xmlNode *construct_async_reply(const async_command_t *cmd, + const pcmk__action_result_t *result); + +static gboolean +is_action_required(const char *action, const stonith_device_t *device) +{ + return (device != NULL) && device->automatic_unfencing + && pcmk__str_eq(action, "on", pcmk__str_none); +} + +static int +get_action_delay_max(const stonith_device_t *device, const char *action) +{ + const char *value = NULL; + int delay_max = 0; + + if (!pcmk__is_fencing_action(action)) { + return 0; + } + + value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_MAX); + if (value) { + delay_max = crm_parse_interval_spec(value) / 1000; + } + + return delay_max; +} + +static int +get_action_delay_base(const stonith_device_t *device, const char *action, + const char *target) +{ + char *hash_value = NULL; + int delay_base = 0; + + if (!pcmk__is_fencing_action(action)) { + return 0; + } + + hash_value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_BASE); + + if (hash_value) { + char *value = strdup(hash_value); + char *valptr = value; + + CRM_ASSERT(value != NULL); + + if (target != NULL) { + for (char *val = strtok(value, "; \t"); val != NULL; val = strtok(NULL, "; \t")) { + char *mapval = strchr(val, ':'); + + if (mapval == NULL || mapval[1] == 0) { + crm_err("pcmk_delay_base: empty value in mapping", val); + continue; + } + + if (mapval != val && strncasecmp(target, val, (size_t)(mapval - val)) == 0) { + value = mapval + 1; + crm_debug("pcmk_delay_base mapped to %s for %s", + value, target); + break; + } + } + } + + if (strchr(value, ':') == 0) { + delay_base = crm_parse_interval_spec(value) / 1000; + } + + free(valptr); + } + + return delay_base; +} + +/*! + * \internal + * \brief Override STONITH timeout with pcmk_*_timeout if available + * + * \param[in] device STONITH device to use + * \param[in] action STONITH action name + * \param[in] default_timeout Timeout to use if device does not have + * a pcmk_*_timeout parameter for action + * + * \return Value of pcmk_(action)_timeout if available, otherwise default_timeout + * \note For consistency, it would be nice if reboot/off/on timeouts could be + * set the same way as start/stop/monitor timeouts, i.e. with an + * <operation> entry in the fencing resource configuration. However that + * is insufficient because fencing devices may be registered directly via + * the fencer's register_device() API instead of going through the CIB + * (e.g. stonith_admin uses it for its -R option, and the executor uses it + * to ensure a device is registered when a command is issued). As device + * properties, pcmk_*_timeout parameters can be grabbed by the fencer when + * the device is registered, whether by CIB change or API call. + */ +static int +get_action_timeout(const stonith_device_t *device, const char *action, + int default_timeout) +{ + if (action && device && device->params) { + char buffer[64] = { 0, }; + const char *value = NULL; + + /* If "reboot" was requested but the device does not support it, + * we will remap to "off", so check timeout for "off" instead + */ + if (pcmk__str_eq(action, "reboot", pcmk__str_none) + && !pcmk_is_set(device->flags, st_device_supports_reboot)) { + crm_trace("%s doesn't support reboot, using timeout for off instead", + device->id); + action = "off"; + } + + /* If the device config specified an action-specific timeout, use it */ + snprintf(buffer, sizeof(buffer), "pcmk_%s_timeout", action); + value = g_hash_table_lookup(device->params, buffer); + if (value) { + return atoi(value); + } + } + return default_timeout; +} + +/*! + * \internal + * \brief Get the currently executing device for a fencing operation + * + * \param[in] cmd Fencing operation to check + * + * \return Currently executing device for \p cmd if any, otherwise NULL + */ +static stonith_device_t * +cmd_device(const async_command_t *cmd) +{ + if ((cmd == NULL) || (cmd->device == NULL) || (device_list == NULL)) { + return NULL; + } + return g_hash_table_lookup(device_list, cmd->device); +} + +/*! + * \internal + * \brief Return the configured reboot action for a given device + * + * \param[in] device_id Device ID + * + * \return Configured reboot action for \p device_id + */ +const char * +fenced_device_reboot_action(const char *device_id) +{ + const char *action = NULL; + + if ((device_list != NULL) && (device_id != NULL)) { + stonith_device_t *device = g_hash_table_lookup(device_list, device_id); + + if ((device != NULL) && (device->params != NULL)) { + action = g_hash_table_lookup(device->params, "pcmk_reboot_action"); + } + } + return pcmk__s(action, "reboot"); +} + +/*! + * \internal + * \brief Check whether a given device supports the "on" action + * + * \param[in] device_id Device ID + * + * \return true if \p device_id supports "on", otherwise false + */ +bool +fenced_device_supports_on(const char *device_id) +{ + if ((device_list != NULL) && (device_id != NULL)) { + stonith_device_t *device = g_hash_table_lookup(device_list, device_id); + + if (device != NULL) { + return pcmk_is_set(device->flags, st_device_supports_on); + } + } + return false; +} + +static void +free_async_command(async_command_t * cmd) +{ + if (!cmd) { + return; + } + + if (cmd->delay_id) { + g_source_remove(cmd->delay_id); + } + + cmd_list = g_list_remove(cmd_list, cmd); + + g_list_free_full(cmd->device_list, free); + free(cmd->device); + free(cmd->action); + free(cmd->target); + free(cmd->remote_op_id); + free(cmd->client); + free(cmd->client_name); + free(cmd->origin); + free(cmd->op); + free(cmd); +} + +/*! + * \internal + * \brief Create a new asynchronous fencing operation from request XML + * + * \param[in] msg Fencing request XML (from IPC or CPG) + * + * \return Newly allocated fencing operation on success, otherwise NULL + * + * \note This asserts on memory errors, so a NULL return indicates an + * unparseable message. + */ +static async_command_t * +create_async_command(xmlNode *msg) +{ + xmlNode *op = NULL; + async_command_t *cmd = NULL; + + if (msg == NULL) { + return NULL; + } + + op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR); + if (op == NULL) { + return NULL; + } + + cmd = calloc(1, sizeof(async_command_t)); + CRM_ASSERT(cmd != NULL); + + // All messages must include these + cmd->action = crm_element_value_copy(op, F_STONITH_ACTION); + cmd->op = crm_element_value_copy(msg, F_STONITH_OPERATION); + cmd->client = crm_element_value_copy(msg, F_STONITH_CLIENTID); + if ((cmd->action == NULL) || (cmd->op == NULL) || (cmd->client == NULL)) { + free_async_command(cmd); + return NULL; + } + + crm_element_value_int(msg, F_STONITH_CALLID, &(cmd->id)); + crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options)); + crm_element_value_int(msg, F_STONITH_DELAY, &(cmd->start_delay)); + crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->default_timeout)); + cmd->timeout = cmd->default_timeout; + + cmd->origin = crm_element_value_copy(msg, F_ORIG); + cmd->remote_op_id = crm_element_value_copy(msg, F_STONITH_REMOTE_OP_ID); + cmd->client_name = crm_element_value_copy(msg, F_STONITH_CLIENTNAME); + cmd->target = crm_element_value_copy(op, F_STONITH_TARGET); + cmd->device = crm_element_value_copy(op, F_STONITH_DEVICE); + + cmd->done_cb = st_child_done; + + // Track in global command list + cmd_list = g_list_append(cmd_list, cmd); + + return cmd; +} + +static int +get_action_limit(stonith_device_t * device) +{ + const char *value = NULL; + int action_limit = 1; + + value = g_hash_table_lookup(device->params, PCMK_STONITH_ACTION_LIMIT); + if ((value == NULL) + || (pcmk__scan_min_int(value, &action_limit, INT_MIN) != pcmk_rc_ok) + || (action_limit == 0)) { + action_limit = 1; + } + return action_limit; +} + +static int +get_active_cmds(stonith_device_t * device) +{ + int counter = 0; + GList *gIter = NULL; + GList *gIterNext = NULL; + + CRM_CHECK(device != NULL, return 0); + + for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) { + async_command_t *cmd = gIter->data; + + gIterNext = gIter->next; + + if (cmd->active_on == device) { + counter++; + } + } + + return counter; +} + +static void +fork_cb(int pid, void *user_data) +{ + async_command_t *cmd = (async_command_t *) user_data; + stonith_device_t * device = + /* in case of a retry we've done the move from + activating_on to active_on already + */ + cmd->activating_on?cmd->activating_on:cmd->active_on; + + CRM_ASSERT(device); + crm_debug("Operation '%s' [%d]%s%s using %s now running with %ds timeout", + cmd->action, pid, + ((cmd->target == NULL)? "" : " targeting "), + pcmk__s(cmd->target, ""), device->id, cmd->timeout); + cmd->active_on = device; + cmd->activating_on = NULL; +} + +static int +get_agent_metadata_cb(gpointer data) { + stonith_device_t *device = data; + guint period_ms; + + switch (get_agent_metadata(device->agent, &device->agent_metadata)) { + case pcmk_rc_ok: + if (device->agent_metadata) { + read_action_metadata(device); + stonith__device_parameter_flags(&(device->flags), device->id, + device->agent_metadata); + } + return G_SOURCE_REMOVE; + + case EAGAIN: + period_ms = pcmk__mainloop_timer_get_period(device->timer); + if (period_ms < 160 * 1000) { + mainloop_timer_set_period(device->timer, 2 * period_ms); + } + return G_SOURCE_CONTINUE; + + default: + return G_SOURCE_REMOVE; + } +} + +/*! + * \internal + * \brief Call a command's action callback for an internal (not library) result + * + * \param[in,out] cmd Command to report result for + * \param[in] execution_status Execution status to use for result + * \param[in] exit_status Exit status to use for result + * \param[in] exit_reason Exit reason to use for result + */ +static void +report_internal_result(async_command_t *cmd, int exit_status, + int execution_status, const char *exit_reason) +{ + pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; + + pcmk__set_result(&result, exit_status, execution_status, exit_reason); + cmd->done_cb(0, &result, cmd); + pcmk__reset_result(&result); +} + +static gboolean +stonith_device_execute(stonith_device_t * device) +{ + int exec_rc = 0; + const char *action_str = NULL; + const char *host_arg = NULL; + async_command_t *cmd = NULL; + stonith_action_t *action = NULL; + int active_cmds = 0; + int action_limit = 0; + GList *gIter = NULL; + GList *gIterNext = NULL; + + CRM_CHECK(device != NULL, return FALSE); + + active_cmds = get_active_cmds(device); + action_limit = get_action_limit(device); + if (action_limit > -1 && active_cmds >= action_limit) { + crm_trace("%s is over its action limit of %d (%u active action%s)", + device->id, action_limit, active_cmds, + pcmk__plural_s(active_cmds)); + return TRUE; + } + + for (gIter = device->pending_ops; gIter != NULL; gIter = gIterNext) { + async_command_t *pending_op = gIter->data; + + gIterNext = gIter->next; + + if (pending_op && pending_op->delay_id) { + crm_trace("Operation '%s'%s%s using %s was asked to run too early, " + "waiting for start delay of %ds", + pending_op->action, + ((pending_op->target == NULL)? "" : " targeting "), + pcmk__s(pending_op->target, ""), + device->id, pending_op->start_delay); + continue; + } + + device->pending_ops = g_list_remove_link(device->pending_ops, gIter); + g_list_free_1(gIter); + + cmd = pending_op; + break; + } + + if (cmd == NULL) { + crm_trace("No actions using %s are needed", device->id); + return TRUE; + } + + if (pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT, + STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) { + if (pcmk__is_fencing_action(cmd->action)) { + if (node_does_watchdog_fencing(stonith_our_uname)) { + pcmk__panic(__func__); + goto done; + } + } else { + crm_info("Faking success for %s watchdog operation", cmd->action); + report_internal_result(cmd, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + goto done; + } + } + +#if SUPPORT_CIBSECRETS + exec_rc = pcmk__substitute_secrets(device->id, device->params); + if (exec_rc != pcmk_rc_ok) { + if (pcmk__str_eq(cmd->action, "stop", pcmk__str_none)) { + crm_info("Proceeding with stop operation for %s " + "despite being unable to load CIB secrets (%s)", + device->id, pcmk_rc_str(exec_rc)); + } else { + crm_err("Considering %s unconfigured " + "because unable to load CIB secrets: %s", + device->id, pcmk_rc_str(exec_rc)); + report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_SECRETS, + "Failed to get CIB secrets"); + goto done; + } + } +#endif + + action_str = cmd->action; + if (pcmk__str_eq(cmd->action, "reboot", pcmk__str_none) + && !pcmk_is_set(device->flags, st_device_supports_reboot)) { + + crm_notice("Remapping 'reboot' action%s%s using %s to 'off' " + "because agent '%s' does not support reboot", + ((cmd->target == NULL)? "" : " targeting "), + pcmk__s(cmd->target, ""), device->id, device->agent); + action_str = "off"; + } + + if (pcmk_is_set(device->flags, st_device_supports_parameter_port)) { + host_arg = "port"; + + } else if (pcmk_is_set(device->flags, st_device_supports_parameter_plug)) { + host_arg = "plug"; + } + + action = stonith__action_create(device->agent, action_str, cmd->target, + cmd->target_nodeid, cmd->timeout, + device->params, device->aliases, host_arg); + + /* for async exec, exec_rc is negative for early error exit + otherwise handling of success/errors is done via callbacks */ + cmd->activating_on = device; + exec_rc = stonith__execute_async(action, (void *)cmd, cmd->done_cb, + fork_cb); + if (exec_rc < 0) { + cmd->activating_on = NULL; + cmd->done_cb(0, stonith__action_result(action), cmd); + stonith__destroy_action(action); + } + +done: + /* Device might get triggered to work by multiple fencing commands + * simultaneously. Trigger the device again to make sure any + * remaining concurrent commands get executed. */ + if (device->pending_ops) { + mainloop_set_trigger(device->work); + } + return TRUE; +} + +static gboolean +stonith_device_dispatch(gpointer user_data) +{ + return stonith_device_execute(user_data); +} + +static gboolean +start_delay_helper(gpointer data) +{ + async_command_t *cmd = data; + stonith_device_t *device = cmd_device(cmd); + + cmd->delay_id = 0; + if (device) { + mainloop_set_trigger(device->work); + } + + return FALSE; +} + +static void +schedule_stonith_command(async_command_t * cmd, stonith_device_t * device) +{ + int delay_max = 0; + int delay_base = 0; + int requested_delay = cmd->start_delay; + + CRM_CHECK(cmd != NULL, return); + CRM_CHECK(device != NULL, return); + + if (cmd->device) { + free(cmd->device); + } + + if (device->include_nodeid && (cmd->target != NULL)) { + crm_node_t *node = crm_get_peer(0, cmd->target); + + cmd->target_nodeid = node->id; + } + + cmd->device = strdup(device->id); + cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout); + + if (cmd->remote_op_id) { + crm_debug("Scheduling '%s' action%s%s using %s for remote peer %s " + "with op id %.8s and timeout %ds", + cmd->action, + (cmd->target == NULL)? "" : " targeting ", + pcmk__s(cmd->target, ""), + device->id, cmd->origin, cmd->remote_op_id, cmd->timeout); + } else { + crm_debug("Scheduling '%s' action%s%s using %s for %s with timeout %ds", + cmd->action, + (cmd->target == NULL)? "" : " targeting ", + pcmk__s(cmd->target, ""), + device->id, cmd->client, cmd->timeout); + } + + device->pending_ops = g_list_append(device->pending_ops, cmd); + mainloop_set_trigger(device->work); + + // Value -1 means disable any static/random fencing delays + if (requested_delay < 0) { + return; + } + + delay_max = get_action_delay_max(device, cmd->action); + delay_base = get_action_delay_base(device, cmd->action, cmd->target); + if (delay_max == 0) { + delay_max = delay_base; + } + if (delay_max < delay_base) { + crm_warn(PCMK_STONITH_DELAY_BASE " (%ds) is larger than " + PCMK_STONITH_DELAY_MAX " (%ds) for %s using %s " + "(limiting to maximum delay)", + delay_base, delay_max, cmd->action, device->id); + delay_base = delay_max; + } + if (delay_max > 0) { + // coverity[dont_call] We're not using rand() for security + cmd->start_delay += + ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0) + + delay_base; + } + + if (cmd->start_delay > 0) { + crm_notice("Delaying '%s' action%s%s using %s for %ds " CRM_XS + " timeout=%ds requested_delay=%ds base=%ds max=%ds", + cmd->action, + (cmd->target == NULL)? "" : " targeting ", + pcmk__s(cmd->target, ""), + device->id, cmd->start_delay, cmd->timeout, + requested_delay, delay_base, delay_max); + cmd->delay_id = + g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd); + } +} + +static void +free_device(gpointer data) +{ + GList *gIter = NULL; + stonith_device_t *device = data; + + g_hash_table_destroy(device->params); + g_hash_table_destroy(device->aliases); + + for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) { + async_command_t *cmd = gIter->data; + + crm_warn("Removal of device '%s' purged operation '%s'", device->id, cmd->action); + report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE, + "Device was removed before action could be executed"); + } + g_list_free(device->pending_ops); + + g_list_free_full(device->targets, free); + + if (device->timer) { + mainloop_timer_stop(device->timer); + mainloop_timer_del(device->timer); + } + + mainloop_destroy_trigger(device->work); + + free_xml(device->agent_metadata); + free(device->namespace); + if (device->on_target_actions != NULL) { + g_string_free(device->on_target_actions, TRUE); + } + free(device->agent); + free(device->id); + free(device); +} + +void free_device_list(void) +{ + if (device_list != NULL) { + g_hash_table_destroy(device_list); + device_list = NULL; + } +} + +void +init_device_list(void) +{ + if (device_list == NULL) { + device_list = pcmk__strkey_table(NULL, free_device); + } +} + +static GHashTable * +build_port_aliases(const char *hostmap, GList ** targets) +{ + char *name = NULL; + int last = 0, lpc = 0, max = 0, added = 0; + GHashTable *aliases = pcmk__strikey_table(free, free); + + if (hostmap == NULL) { + return aliases; + } + + max = strlen(hostmap); + for (; lpc <= max; lpc++) { + switch (hostmap[lpc]) { + /* Skip escaped chars */ + case '\\': + lpc++; + break; + + /* Assignment chars */ + case '=': + case ':': + if (lpc > last) { + free(name); + name = calloc(1, 1 + lpc - last); + memcpy(name, hostmap + last, lpc - last); + } + last = lpc + 1; + break; + + /* Delimeter chars */ + /* case ',': Potentially used to specify multiple ports */ + case 0: + case ';': + case ' ': + case '\t': + if (name) { + char *value = NULL; + int k = 0; + + value = calloc(1, 1 + lpc - last); + memcpy(value, hostmap + last, lpc - last); + + for (int i = 0; value[i] != '\0'; i++) { + if (value[i] != '\\') { + value[k++] = value[i]; + } + } + value[k] = '\0'; + + crm_debug("Adding alias '%s'='%s'", name, value); + g_hash_table_replace(aliases, name, value); + if (targets) { + *targets = g_list_append(*targets, strdup(value)); + } + value = NULL; + name = NULL; + added++; + + } else if (lpc > last) { + crm_debug("Parse error at offset %d near '%s'", lpc - last, hostmap + last); + } + + last = lpc + 1; + break; + } + + if (hostmap[lpc] == 0) { + break; + } + } + + if (added == 0) { + crm_info("No host mappings detected in '%s'", hostmap); + } + + free(name); + return aliases; +} + +GHashTable *metadata_cache = NULL; + +void +free_metadata_cache(void) { + if (metadata_cache != NULL) { + g_hash_table_destroy(metadata_cache); + metadata_cache = NULL; + } +} + +static void +init_metadata_cache(void) { + if (metadata_cache == NULL) { + metadata_cache = pcmk__strkey_table(free, free); + } +} + +int +get_agent_metadata(const char *agent, xmlNode ** metadata) +{ + char *buffer = NULL; + + if (metadata == NULL) { + return EINVAL; + } + *metadata = NULL; + if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT_INTERNAL, pcmk__str_none)) { + return pcmk_rc_ok; + } + init_metadata_cache(); + buffer = g_hash_table_lookup(metadata_cache, agent); + if (buffer == NULL) { + stonith_t *st = stonith_api_new(); + int rc; + + if (st == NULL) { + crm_warn("Could not get agent meta-data: " + "API memory allocation failed"); + return EAGAIN; + } + rc = st->cmds->metadata(st, st_opt_sync_call, agent, + NULL, &buffer, 10); + stonith_api_delete(st); + if (rc || !buffer) { + crm_err("Could not retrieve metadata for fencing agent %s", agent); + return EAGAIN; + } + g_hash_table_replace(metadata_cache, strdup(agent), buffer); + } + + *metadata = string2xml(buffer); + return pcmk_rc_ok; +} + +static gboolean +is_nodeid_required(xmlNode * xml) +{ + xmlXPathObjectPtr xpath = NULL; + + if (stand_alone) { + return FALSE; + } + + if (!xml) { + return FALSE; + } + + xpath = xpath_search(xml, "//parameter[@name='nodeid']"); + if (numXpathResults(xpath) <= 0) { + freeXpathObject(xpath); + return FALSE; + } + + freeXpathObject(xpath); + return TRUE; +} + +static void +read_action_metadata(stonith_device_t *device) +{ + xmlXPathObjectPtr xpath = NULL; + int max = 0; + int lpc = 0; + + if (device->agent_metadata == NULL) { + return; + } + + xpath = xpath_search(device->agent_metadata, "//action"); + max = numXpathResults(xpath); + + if (max <= 0) { + freeXpathObject(xpath); + return; + } + + for (lpc = 0; lpc < max; lpc++) { + const char *action = NULL; + xmlNode *match = getXpathResult(xpath, lpc); + + CRM_LOG_ASSERT(match != NULL); + if(match == NULL) { continue; }; + + action = crm_element_value(match, "name"); + + if (pcmk__str_eq(action, "list", pcmk__str_none)) { + stonith__set_device_flags(device->flags, device->id, + st_device_supports_list); + } else if (pcmk__str_eq(action, "status", pcmk__str_none)) { + stonith__set_device_flags(device->flags, device->id, + st_device_supports_status); + } else if (pcmk__str_eq(action, "reboot", pcmk__str_none)) { + stonith__set_device_flags(device->flags, device->id, + st_device_supports_reboot); + } else if (pcmk__str_eq(action, "on", pcmk__str_none)) { + /* "automatic" means the cluster will unfence node when it joins */ + /* "required" is a deprecated synonym for "automatic" */ + if (pcmk__xe_attr_is_true(match, "automatic") || pcmk__xe_attr_is_true(match, "required")) { + device->automatic_unfencing = TRUE; + } + stonith__set_device_flags(device->flags, device->id, + st_device_supports_on); + } + + if ((action != NULL) && pcmk__xe_attr_is_true(match, "on_target")) { + pcmk__add_word(&(device->on_target_actions), 64, action); + } + } + + freeXpathObject(xpath); +} + +/*! + * \internal + * \brief Set a pcmk_*_action parameter if not already set + * + * \param[in,out] params Device parameters + * \param[in] action Name of action + * \param[in] value Value to use if action is not already set + */ +static void +map_action(GHashTable *params, const char *action, const char *value) +{ + char *key = crm_strdup_printf("pcmk_%s_action", action); + + if (g_hash_table_lookup(params, key)) { + crm_warn("Ignoring %s='%s', see %s instead", + STONITH_ATTR_ACTION_OP, value, key); + free(key); + } else { + crm_warn("Mapping %s='%s' to %s='%s'", + STONITH_ATTR_ACTION_OP, value, key, value); + g_hash_table_insert(params, key, strdup(value)); + } +} + +/*! + * \internal + * \brief Create device parameter table from XML + * + * \param[in] name Device name (used for logging only) + * \param[in] dev XML containing device parameters + */ +static GHashTable * +xml2device_params(const char *name, const xmlNode *dev) +{ + GHashTable *params = xml2list(dev); + const char *value; + + /* Action should never be specified in the device configuration, + * but we support it for users who are familiar with other software + * that worked that way. + */ + value = g_hash_table_lookup(params, STONITH_ATTR_ACTION_OP); + if (value != NULL) { + crm_warn("%s has '%s' parameter, which should never be specified in configuration", + name, STONITH_ATTR_ACTION_OP); + + if (*value == '\0') { + crm_warn("Ignoring empty '%s' parameter", STONITH_ATTR_ACTION_OP); + + } else if (strcmp(value, "reboot") == 0) { + crm_warn("Ignoring %s='reboot' (see stonith-action cluster property instead)", + STONITH_ATTR_ACTION_OP); + + } else if (strcmp(value, "off") == 0) { + map_action(params, "reboot", value); + + } else { + map_action(params, "off", value); + map_action(params, "reboot", value); + } + + g_hash_table_remove(params, STONITH_ATTR_ACTION_OP); + } + + return params; +} + +static const char * +target_list_type(stonith_device_t * dev) +{ + const char *check_type = NULL; + + check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK); + + if (check_type == NULL) { + + if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) { + check_type = "static-list"; + } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) { + check_type = "static-list"; + } else if (pcmk_is_set(dev->flags, st_device_supports_list)) { + check_type = "dynamic-list"; + } else if (pcmk_is_set(dev->flags, st_device_supports_status)) { + check_type = "status"; + } else { + check_type = PCMK__VALUE_NONE; + } + } + + return check_type; +} + +static stonith_device_t * +build_device_from_xml(xmlNode *dev) +{ + const char *value; + stonith_device_t *device = NULL; + char *agent = crm_element_value_copy(dev, "agent"); + + CRM_CHECK(agent != NULL, return device); + + device = calloc(1, sizeof(stonith_device_t)); + + CRM_CHECK(device != NULL, {free(agent); return device;}); + + device->id = crm_element_value_copy(dev, XML_ATTR_ID); + device->agent = agent; + device->namespace = crm_element_value_copy(dev, "namespace"); + device->params = xml2device_params(device->id, dev); + + value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_LIST); + if (value) { + device->targets = stonith__parse_targets(value); + } + + value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP); + device->aliases = build_port_aliases(value, &(device->targets)); + + value = target_list_type(device); + if (!pcmk__str_eq(value, "static-list", pcmk__str_casei) && device->targets) { + /* Other than "static-list", dev-> targets is unnecessary. */ + g_list_free_full(device->targets, free); + device->targets = NULL; + } + switch (get_agent_metadata(device->agent, &device->agent_metadata)) { + case pcmk_rc_ok: + if (device->agent_metadata) { + read_action_metadata(device); + stonith__device_parameter_flags(&(device->flags), device->id, + device->agent_metadata); + } + break; + + case EAGAIN: + if (device->timer == NULL) { + device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000, + TRUE, get_agent_metadata_cb, device); + } + if (!mainloop_timer_running(device->timer)) { + mainloop_timer_start(device->timer); + } + break; + + default: + break; + } + + value = g_hash_table_lookup(device->params, "nodeid"); + if (!value) { + device->include_nodeid = is_nodeid_required(device->agent_metadata); + } + + value = crm_element_value(dev, "rsc_provides"); + if (pcmk__str_eq(value, PCMK__VALUE_UNFENCING, pcmk__str_casei)) { + device->automatic_unfencing = TRUE; + } + + if (is_action_required("on", device)) { + crm_info("Fencing device '%s' requires unfencing", device->id); + } + + if (device->on_target_actions != NULL) { + crm_info("Fencing device '%s' requires actions (%s) to be executed " + "on target", device->id, + (const char *) device->on_target_actions->str); + } + + device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device); + /* TODO: Hook up priority */ + + return device; +} + +static void +schedule_internal_command(const char *origin, + stonith_device_t * device, + const char *action, + const char *target, + int timeout, + void *internal_user_data, + void (*done_cb) (int pid, + const pcmk__action_result_t *result, + void *user_data)) +{ + async_command_t *cmd = NULL; + + cmd = calloc(1, sizeof(async_command_t)); + + cmd->id = -1; + cmd->default_timeout = timeout ? timeout : 60; + cmd->timeout = cmd->default_timeout; + cmd->action = strdup(action); + pcmk__str_update(&cmd->target, target); + cmd->device = strdup(device->id); + cmd->origin = strdup(origin); + cmd->client = strdup(crm_system_name); + cmd->client_name = strdup(crm_system_name); + + cmd->internal_user_data = internal_user_data; + cmd->done_cb = done_cb; /* cmd, not internal_user_data, is passed to 'done_cb' as the userdata */ + + schedule_stonith_command(cmd, device); +} + +// Fence agent status commands use custom exit status codes +enum fence_status_code { + fence_status_invalid = -1, + fence_status_active = 0, + fence_status_unknown = 1, + fence_status_inactive = 2, +}; + +static void +status_search_cb(int pid, const pcmk__action_result_t *result, void *user_data) +{ + async_command_t *cmd = user_data; + struct device_search_s *search = cmd->internal_user_data; + stonith_device_t *dev = cmd_device(cmd); + gboolean can = FALSE; + + free_async_command(cmd); + + if (!dev) { + search_devices_record_result(search, NULL, FALSE); + return; + } + + mainloop_set_trigger(dev->work); + + if (result->execution_status != PCMK_EXEC_DONE) { + crm_warn("Assuming %s cannot fence %s " + "because status could not be executed: %s%s%s%s", + dev->id, search->host, + pcmk_exec_status_str(result->execution_status), + ((result->exit_reason == NULL)? "" : " ("), + ((result->exit_reason == NULL)? "" : result->exit_reason), + ((result->exit_reason == NULL)? "" : ")")); + search_devices_record_result(search, dev->id, FALSE); + return; + } + + switch (result->exit_status) { + case fence_status_unknown: + crm_trace("%s reported it cannot fence %s", dev->id, search->host); + break; + + case fence_status_active: + case fence_status_inactive: + crm_trace("%s reported it can fence %s", dev->id, search->host); + can = TRUE; + break; + + default: + crm_warn("Assuming %s cannot fence %s " + "(status returned unknown code %d)", + dev->id, search->host, result->exit_status); + break; + } + search_devices_record_result(search, dev->id, can); +} + +static void +dynamic_list_search_cb(int pid, const pcmk__action_result_t *result, + void *user_data) +{ + async_command_t *cmd = user_data; + struct device_search_s *search = cmd->internal_user_data; + stonith_device_t *dev = cmd_device(cmd); + gboolean can_fence = FALSE; + + free_async_command(cmd); + + /* Host/alias must be in the list output to be eligible to be fenced + * + * Will cause problems if down'd nodes aren't listed or (for virtual nodes) + * if the guest is still listed despite being moved to another machine + */ + if (!dev) { + search_devices_record_result(search, NULL, FALSE); + return; + } + + mainloop_set_trigger(dev->work); + + if (pcmk__result_ok(result)) { + crm_info("Refreshing target list for %s", dev->id); + g_list_free_full(dev->targets, free); + dev->targets = stonith__parse_targets(result->action_stdout); + dev->targets_age = time(NULL); + + } else if (dev->targets != NULL) { + if (result->execution_status == PCMK_EXEC_DONE) { + crm_info("Reusing most recent target list for %s " + "because list returned error code %d", + dev->id, result->exit_status); + } else { + crm_info("Reusing most recent target list for %s " + "because list could not be executed: %s%s%s%s", + dev->id, pcmk_exec_status_str(result->execution_status), + ((result->exit_reason == NULL)? "" : " ("), + ((result->exit_reason == NULL)? "" : result->exit_reason), + ((result->exit_reason == NULL)? "" : ")")); + } + + } else { // We have never successfully executed list + if (result->execution_status == PCMK_EXEC_DONE) { + crm_warn("Assuming %s cannot fence %s " + "because list returned error code %d", + dev->id, search->host, result->exit_status); + } else { + crm_warn("Assuming %s cannot fence %s " + "because list could not be executed: %s%s%s%s", + dev->id, search->host, + pcmk_exec_status_str(result->execution_status), + ((result->exit_reason == NULL)? "" : " ("), + ((result->exit_reason == NULL)? "" : result->exit_reason), + ((result->exit_reason == NULL)? "" : ")")); + } + + /* Fall back to pcmk_host_check="status" if the user didn't explicitly + * specify "dynamic-list". + */ + if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK) == NULL) { + crm_notice("Switching to pcmk_host_check='status' for %s", dev->id); + g_hash_table_replace(dev->params, strdup(PCMK_STONITH_HOST_CHECK), + strdup("status")); + } + } + + if (dev->targets) { + const char *alias = g_hash_table_lookup(dev->aliases, search->host); + + if (!alias) { + alias = search->host; + } + if (pcmk__str_in_list(alias, dev->targets, pcmk__str_casei)) { + can_fence = TRUE; + } + } + search_devices_record_result(search, dev->id, can_fence); +} + +/*! + * \internal + * \brief Returns true if any key in first is not in second or second has a different value for key + */ +static int +device_params_diff(GHashTable *first, GHashTable *second) { + char *key = NULL; + char *value = NULL; + GHashTableIter gIter; + + g_hash_table_iter_init(&gIter, first); + while (g_hash_table_iter_next(&gIter, (void **)&key, (void **)&value)) { + + if(strstr(key, "CRM_meta") == key) { + continue; + } else if(strcmp(key, "crm_feature_set") == 0) { + continue; + } else { + char *other_value = g_hash_table_lookup(second, key); + + if (!other_value || !pcmk__str_eq(other_value, value, pcmk__str_casei)) { + crm_trace("Different value for %s: %s != %s", key, other_value, value); + return 1; + } + } + } + + return 0; +} + +/*! + * \internal + * \brief Checks to see if an identical device already exists in the device_list + */ +static stonith_device_t * +device_has_duplicate(const stonith_device_t *device) +{ + stonith_device_t *dup = g_hash_table_lookup(device_list, device->id); + + if (!dup) { + crm_trace("No match for %s", device->id); + return NULL; + + } else if (!pcmk__str_eq(dup->agent, device->agent, pcmk__str_casei)) { + crm_trace("Different agent: %s != %s", dup->agent, device->agent); + return NULL; + } + + /* Use calculate_operation_digest() here? */ + if (device_params_diff(device->params, dup->params) || + device_params_diff(dup->params, device->params)) { + return NULL; + } + + crm_trace("Match"); + return dup; +} + +int +stonith_device_register(xmlNode *dev, gboolean from_cib) +{ + stonith_device_t *dup = NULL; + stonith_device_t *device = build_device_from_xml(dev); + guint ndevices = 0; + int rv = pcmk_ok; + + CRM_CHECK(device != NULL, return -ENOMEM); + + /* do we have a watchdog-device? */ + if (pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none) || + pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT, + STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) do { + if (stonith_watchdog_timeout_ms <= 0) { + crm_err("Ignoring watchdog fence device without " + "stonith-watchdog-timeout set."); + rv = -ENODEV; + /* fall through to cleanup & return */ + } else if (!pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT, + STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) { + crm_err("Ignoring watchdog fence device with unknown " + "agent '%s' unequal '" STONITH_WATCHDOG_AGENT "'.", + device->agent?device->agent:""); + rv = -ENODEV; + /* fall through to cleanup & return */ + } else if (!pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, + pcmk__str_none)) { + crm_err("Ignoring watchdog fence device " + "named %s !='"STONITH_WATCHDOG_ID"'.", + device->id?device->id:""); + rv = -ENODEV; + /* fall through to cleanup & return */ + } else { + if (pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT, + pcmk__str_none)) { + /* this either has an empty list or the targets + configured for watchdog-fencing + */ + g_list_free_full(stonith_watchdog_targets, free); + stonith_watchdog_targets = device->targets; + device->targets = NULL; + } + if (node_does_watchdog_fencing(stonith_our_uname)) { + g_list_free_full(device->targets, free); + device->targets = stonith__parse_targets(stonith_our_uname); + g_hash_table_replace(device->params, + strdup(PCMK_STONITH_HOST_LIST), + strdup(stonith_our_uname)); + /* proceed as with any other stonith-device */ + break; + } + + crm_debug("Skip registration of watchdog fence device on node not in host-list."); + /* cleanup and fall through to more cleanup and return */ + device->targets = NULL; + stonith_device_remove(device->id, from_cib); + } + free_device(device); + return rv; + } while (0); + + dup = device_has_duplicate(device); + if (dup) { + ndevices = g_hash_table_size(device_list); + crm_debug("Device '%s' already in device list (%d active device%s)", + device->id, ndevices, pcmk__plural_s(ndevices)); + free_device(device); + device = dup; + dup = g_hash_table_lookup(device_list, device->id); + dup->dirty = FALSE; + + } else { + stonith_device_t *old = g_hash_table_lookup(device_list, device->id); + + if (from_cib && old && old->api_registered) { + /* If the cib is writing over an entry that is shared with a stonith client, + * copy any pending ops that currently exist on the old entry to the new one. + * Otherwise the pending ops will be reported as failures + */ + crm_info("Overwriting existing entry for %s from CIB", device->id); + device->pending_ops = old->pending_ops; + device->api_registered = TRUE; + old->pending_ops = NULL; + if (device->pending_ops) { + mainloop_set_trigger(device->work); + } + } + g_hash_table_replace(device_list, device->id, device); + + ndevices = g_hash_table_size(device_list); + crm_notice("Added '%s' to device list (%d active device%s)", + device->id, ndevices, pcmk__plural_s(ndevices)); + } + + if (from_cib) { + device->cib_registered = TRUE; + } else { + device->api_registered = TRUE; + } + + return pcmk_ok; +} + +void +stonith_device_remove(const char *id, bool from_cib) +{ + stonith_device_t *device = g_hash_table_lookup(device_list, id); + guint ndevices = 0; + + if (!device) { + ndevices = g_hash_table_size(device_list); + crm_info("Device '%s' not found (%d active device%s)", + id, ndevices, pcmk__plural_s(ndevices)); + return; + } + + if (from_cib) { + device->cib_registered = FALSE; + } else { + device->verified = FALSE; + device->api_registered = FALSE; + } + + if (!device->cib_registered && !device->api_registered) { + g_hash_table_remove(device_list, id); + ndevices = g_hash_table_size(device_list); + crm_info("Removed '%s' from device list (%d active device%s)", + id, ndevices, pcmk__plural_s(ndevices)); + } else { + crm_trace("Not removing '%s' from device list (%d active) because " + "still registered via:%s%s", + id, g_hash_table_size(device_list), + (device->cib_registered? " cib" : ""), + (device->api_registered? " api" : "")); + } +} + +/*! + * \internal + * \brief Return the number of stonith levels registered for a node + * + * \param[in] tp Node's topology table entry + * + * \return Number of non-NULL levels in topology entry + * \note This function is used only for log messages. + */ +static int +count_active_levels(const stonith_topology_t *tp) +{ + int lpc = 0; + int count = 0; + + for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) { + if (tp->levels[lpc] != NULL) { + count++; + } + } + return count; +} + +static void +free_topology_entry(gpointer data) +{ + stonith_topology_t *tp = data; + + int lpc = 0; + + for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) { + if (tp->levels[lpc] != NULL) { + g_list_free_full(tp->levels[lpc], free); + } + } + free(tp->target); + free(tp->target_value); + free(tp->target_pattern); + free(tp->target_attribute); + free(tp); +} + +void +free_topology_list(void) +{ + if (topology != NULL) { + g_hash_table_destroy(topology); + topology = NULL; + } +} + +void +init_topology_list(void) +{ + if (topology == NULL) { + topology = pcmk__strkey_table(NULL, free_topology_entry); + } +} + +char * +stonith_level_key(const xmlNode *level, enum fenced_target_by mode) +{ + if (mode == fenced_target_by_unknown) { + mode = unpack_level_kind(level); + } + switch (mode) { + case fenced_target_by_name: + return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET); + + case fenced_target_by_pattern: + return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN); + + case fenced_target_by_attribute: + return crm_strdup_printf("%s=%s", + crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE), + crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE)); + + default: + return crm_strdup_printf("unknown-%s", ID(level)); + } +} + +/*! + * \internal + * \brief Parse target identification from topology level XML + * + * \param[in] level Topology level XML to parse + * + * \return How to identify target of \p level + */ +static enum fenced_target_by +unpack_level_kind(const xmlNode *level) +{ + if (crm_element_value(level, XML_ATTR_STONITH_TARGET) != NULL) { + return fenced_target_by_name; + } + if (crm_element_value(level, XML_ATTR_STONITH_TARGET_PATTERN) != NULL) { + return fenced_target_by_pattern; + } + if (!stand_alone /* if standalone, there's no attribute manager */ + && (crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE) != NULL) + && (crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE) != NULL)) { + return fenced_target_by_attribute; + } + return fenced_target_by_unknown; +} + +static stonith_key_value_t * +parse_device_list(const char *devices) +{ + int lpc = 0; + int max = 0; + int last = 0; + stonith_key_value_t *output = NULL; + + if (devices == NULL) { + return output; + } + + max = strlen(devices); + for (lpc = 0; lpc <= max; lpc++) { + if (devices[lpc] == ',' || devices[lpc] == 0) { + char *line = strndup(devices + last, lpc - last); + + output = stonith_key_value_add(output, NULL, line); + free(line); + + last = lpc + 1; + } + } + + return output; +} + +/*! + * \internal + * \brief Unpack essential information from topology request XML + * + * \param[in] xml Request XML to search + * \param[out] mode If not NULL, where to store level kind + * \param[out] target If not NULL, where to store representation of target + * \param[out] id If not NULL, where to store level number + * \param[out] desc If not NULL, where to store log-friendly level description + * + * \return Topology level XML from within \p xml, or NULL if not found + * \note The caller is responsible for freeing \p *target and \p *desc if set. + */ +static xmlNode * +unpack_level_request(xmlNode *xml, enum fenced_target_by *mode, char **target, + int *id, char **desc) +{ + enum fenced_target_by local_mode = fenced_target_by_unknown; + char *local_target = NULL; + int local_id = 0; + + /* The level element can be the top element or lower. If top level, don't + * search by xpath, because it might give multiple hits if the XML is the + * CIB. + */ + if ((xml != NULL) + && !pcmk__str_eq(TYPE(xml), XML_TAG_FENCING_LEVEL, pcmk__str_none)) { + xml = get_xpath_object("//" XML_TAG_FENCING_LEVEL, xml, LOG_WARNING); + } + + if (xml == NULL) { + if (desc != NULL) { + *desc = crm_strdup_printf("missing"); + } + } else { + local_mode = unpack_level_kind(xml); + local_target = stonith_level_key(xml, local_mode); + crm_element_value_int(xml, XML_ATTR_STONITH_INDEX, &local_id); + if (desc != NULL) { + *desc = crm_strdup_printf("%s[%d]", local_target, local_id); + } + } + + if (mode != NULL) { + *mode = local_mode; + } + if (id != NULL) { + *id = local_id; + } + + if (target != NULL) { + *target = local_target; + } else { + free(local_target); + } + + return xml; +} + +/*! + * \internal + * \brief Register a fencing topology level for a target + * + * Given an XML request specifying the target name, level index, and device IDs + * for the level, this will create an entry for the target in the global topology + * table if one does not already exist, then append the specified device IDs to + * the entry's device list for the specified level. + * + * \param[in] msg XML request for STONITH level registration + * \param[out] desc If not NULL, set to string representation "TARGET[LEVEL]" + * \param[out] result Where to set result of registration + */ +void +fenced_register_level(xmlNode *msg, char **desc, pcmk__action_result_t *result) +{ + int id = 0; + xmlNode *level; + enum fenced_target_by mode; + char *target; + + stonith_topology_t *tp; + stonith_key_value_t *dIter = NULL; + stonith_key_value_t *devices = NULL; + + CRM_CHECK((msg != NULL) && (result != NULL), return); + + level = unpack_level_request(msg, &mode, &target, &id, desc); + if (level == NULL) { + fenced_set_protocol_error(result); + return; + } + + // Ensure an ID was given (even the client API adds an ID) + if (pcmk__str_empty(ID(level))) { + crm_warn("Ignoring registration for topology level without ID"); + free(target); + crm_log_xml_trace(level, "Bad level"); + pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID, + "Topology level is invalid without ID"); + return; + } + + // Ensure a valid target was specified + if (mode == fenced_target_by_unknown) { + crm_warn("Ignoring registration for topology level '%s' " + "without valid target", ID(level)); + free(target); + crm_log_xml_trace(level, "Bad level"); + pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID, + "Invalid target for topology level '%s'", + ID(level)); + return; + } + + // Ensure level ID is in allowed range + if ((id <= 0) || (id >= ST_LEVEL_MAX)) { + crm_warn("Ignoring topology registration for %s with invalid level %d", + target, id); + free(target); + crm_log_xml_trace(level, "Bad level"); + pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID, + "Invalid level number '%s' for topology level '%s'", + pcmk__s(crm_element_value(level, + XML_ATTR_STONITH_INDEX), + ""), + ID(level)); + return; + } + + /* Find or create topology table entry */ + tp = g_hash_table_lookup(topology, target); + if (tp == NULL) { + tp = calloc(1, sizeof(stonith_topology_t)); + if (tp == NULL) { + pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_ERROR, + strerror(ENOMEM)); + free(target); + return; + } + tp->kind = mode; + tp->target = target; + tp->target_value = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_VALUE); + tp->target_pattern = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN); + tp->target_attribute = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE); + + g_hash_table_replace(topology, tp->target, tp); + crm_trace("Added %s (%d) to the topology (%d active entries)", + target, (int) mode, g_hash_table_size(topology)); + } else { + free(target); + } + + if (tp->levels[id] != NULL) { + crm_info("Adding to the existing %s[%d] topology entry", + tp->target, id); + } + + devices = parse_device_list(crm_element_value(level, XML_ATTR_STONITH_DEVICES)); + for (dIter = devices; dIter; dIter = dIter->next) { + const char *device = dIter->value; + + crm_trace("Adding device '%s' for %s[%d]", device, tp->target, id); + tp->levels[id] = g_list_append(tp->levels[id], strdup(device)); + } + stonith_key_value_freeall(devices, 1, 1); + + { + int nlevels = count_active_levels(tp); + + crm_info("Target %s has %d active fencing level%s", + tp->target, nlevels, pcmk__plural_s(nlevels)); + } + + pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); +} + +/*! + * \internal + * \brief Unregister a fencing topology level for a target + * + * Given an XML request specifying the target name and level index (or 0 for all + * levels), this will remove any corresponding entry for the target from the + * global topology table. + * + * \param[in] msg XML request for STONITH level registration + * \param[out] desc If not NULL, set to string representation "TARGET[LEVEL]" + * \param[out] result Where to set result of unregistration + */ +void +fenced_unregister_level(xmlNode *msg, char **desc, + pcmk__action_result_t *result) +{ + int id = -1; + stonith_topology_t *tp; + char *target; + xmlNode *level = NULL; + + CRM_CHECK(result != NULL, return); + + level = unpack_level_request(msg, NULL, &target, &id, desc); + if (level == NULL) { + fenced_set_protocol_error(result); + return; + } + + // Ensure level ID is in allowed range + if ((id < 0) || (id >= ST_LEVEL_MAX)) { + crm_warn("Ignoring topology unregistration for %s with invalid level %d", + target, id); + free(target); + crm_log_xml_trace(level, "Bad level"); + pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID, + "Invalid level number '%s' for topology level %s", + pcmk__s(crm_element_value(level, + XML_ATTR_STONITH_INDEX), + "<null>"), + + // Client API doesn't add ID to unregistration XML + pcmk__s(ID(level), "")); + return; + } + + tp = g_hash_table_lookup(topology, target); + if (tp == NULL) { + guint nentries = g_hash_table_size(topology); + + crm_info("No fencing topology found for %s (%d active %s)", + target, nentries, + pcmk__plural_alt(nentries, "entry", "entries")); + + } else if (id == 0 && g_hash_table_remove(topology, target)) { + guint nentries = g_hash_table_size(topology); + + crm_info("Removed all fencing topology entries related to %s " + "(%d active %s remaining)", target, nentries, + pcmk__plural_alt(nentries, "entry", "entries")); + + } else if (tp->levels[id] != NULL) { + guint nlevels; + + g_list_free_full(tp->levels[id], free); + tp->levels[id] = NULL; + + nlevels = count_active_levels(tp); + crm_info("Removed level %d from fencing topology for %s " + "(%d active level%s remaining)", + id, target, nlevels, pcmk__plural_s(nlevels)); + } + + free(target); + pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); +} + +static char * +list_to_string(GList *list, const char *delim, gboolean terminate_with_delim) +{ + int max = g_list_length(list); + size_t delim_len = delim?strlen(delim):0; + size_t alloc_size = 1 + (max?((max-1+(terminate_with_delim?1:0))*delim_len):0); + char *rv; + GList *gIter; + + for (gIter = list; gIter != NULL; gIter = gIter->next) { + const char *value = (const char *) gIter->data; + + alloc_size += strlen(value); + } + rv = calloc(alloc_size, sizeof(char)); + if (rv) { + char *pos = rv; + const char *lead_delim = ""; + + for (gIter = list; gIter != NULL; gIter = gIter->next) { + const char *value = (const char *) gIter->data; + + pos = &pos[sprintf(pos, "%s%s", lead_delim, value)]; + lead_delim = delim; + } + if (max && terminate_with_delim) { + sprintf(pos, "%s", delim); + } + } + return rv; +} + +/*! + * \internal + * \brief Execute a fence agent action directly (and asynchronously) + * + * Handle a STONITH_OP_EXEC API message by scheduling a requested agent action + * directly on a specified device. Only list, monitor, and status actions are + * expected to use this call, though it should work with any agent command. + * + * \param[in] msg Request XML specifying action + * \param[out] result Where to store result of action + * + * \note If the action is monitor, the device must be registered via the API + * (CIB registration is not sufficient), because monitor should not be + * possible unless the device is "started" (API registered). + */ +static void +execute_agent_action(xmlNode *msg, pcmk__action_result_t *result) +{ + xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, msg, LOG_ERR); + xmlNode *op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR); + const char *id = crm_element_value(dev, F_STONITH_DEVICE); + const char *action = crm_element_value(op, F_STONITH_ACTION); + async_command_t *cmd = NULL; + stonith_device_t *device = NULL; + + if ((id == NULL) || (action == NULL)) { + crm_info("Malformed API action request: device %s, action %s", + (id? id : "not specified"), + (action? action : "not specified")); + fenced_set_protocol_error(result); + return; + } + + if (pcmk__str_eq(id, STONITH_WATCHDOG_ID, pcmk__str_none)) { + // Watchdog agent actions are implemented internally + if (stonith_watchdog_timeout_ms <= 0) { + pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE, + "Watchdog fence device not configured"); + return; + + } else if (pcmk__str_eq(action, "list", pcmk__str_none)) { + pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + pcmk__set_result_output(result, + list_to_string(stonith_watchdog_targets, + "\n", TRUE), + NULL); + return; + + } else if (pcmk__str_eq(action, "monitor", pcmk__str_none)) { + pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + return; + } + } + + device = g_hash_table_lookup(device_list, id); + if (device == NULL) { + crm_info("Ignoring API '%s' action request because device %s not found", + action, id); + pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE, + "'%s' not found", id); + return; + + } else if (!device->api_registered && !strcmp(action, "monitor")) { + // Monitors may run only on "started" (API-registered) devices + crm_info("Ignoring API '%s' action request because device %s not active", + action, id); + pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE, + "'%s' not active", id); + return; + } + + cmd = create_async_command(msg); + if (cmd == NULL) { + crm_log_xml_warn(msg, "invalid"); + fenced_set_protocol_error(result); + return; + } + + schedule_stonith_command(cmd, device); + pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL); +} + +static void +search_devices_record_result(struct device_search_s *search, const char *device, gboolean can_fence) +{ + search->replies_received++; + if (can_fence && device) { + if (search->support_action_only != st_device_supports_none) { + stonith_device_t *dev = g_hash_table_lookup(device_list, device); + if (dev && !pcmk_is_set(dev->flags, search->support_action_only)) { + return; + } + } + search->capable = g_list_append(search->capable, strdup(device)); + } + + if (search->replies_needed == search->replies_received) { + + guint ndevices = g_list_length(search->capable); + + crm_debug("Search found %d device%s that can perform '%s' targeting %s", + ndevices, pcmk__plural_s(ndevices), + (search->action? search->action : "unknown action"), + (search->host? search->host : "any node")); + + search->callback(search->capable, search->user_data); + free(search->host); + free(search->action); + free(search); + } +} + +/*! + * \internal + * \brief Check whether the local host is allowed to execute a fencing action + * + * \param[in] device Fence device to check + * \param[in] action Fence action to check + * \param[in] target Hostname of fence target + * \param[in] allow_suicide Whether self-fencing is allowed for this operation + * + * \return TRUE if local host is allowed to execute action, FALSE otherwise + */ +static gboolean +localhost_is_eligible(const stonith_device_t *device, const char *action, + const char *target, gboolean allow_suicide) +{ + gboolean localhost_is_target = pcmk__str_eq(target, stonith_our_uname, + pcmk__str_casei); + + if ((device != NULL) && (action != NULL) + && (device->on_target_actions != NULL) + && (strstr((const char*) device->on_target_actions->str, + action) != NULL)) { + + if (!localhost_is_target) { + crm_trace("Operation '%s' using %s can only be executed for local " + "host, not %s", action, device->id, target); + return FALSE; + } + + } else if (localhost_is_target && !allow_suicide) { + crm_trace("'%s' operation does not support self-fencing", action); + return FALSE; + } + return TRUE; +} + +/*! + * \internal + * \brief Check if local node is allowed to execute (possibly remapped) action + * + * \param[in] device Fence device to check + * \param[in] action Fence action to check + * \param[in] target Node name of fence target + * \param[in] allow_self Whether self-fencing is allowed for this operation + * + * \return true if local node is allowed to execute \p action or any actions it + * might be remapped to, otherwise false + */ +static bool +localhost_is_eligible_with_remap(const stonith_device_t *device, + const char *action, const char *target, + gboolean allow_self) +{ + // Check exact action + if (localhost_is_eligible(device, action, target, allow_self)) { + return true; + } + + // Check potential remaps + + if (pcmk__str_eq(action, "reboot", pcmk__str_none)) { + /* "reboot" might get remapped to "off" then "on", so even if reboot is + * disallowed, return true if either of those is allowed. We'll report + * the disallowed actions with the results. We never allow self-fencing + * for remapped "on" actions because the target is off at that point. + */ + if (localhost_is_eligible(device, "off", target, allow_self) + || localhost_is_eligible(device, "on", target, FALSE)) { + return true; + } + } + + return false; +} + +static void +can_fence_host_with_device(stonith_device_t *dev, + struct device_search_s *search) +{ + gboolean can = FALSE; + const char *check_type = "Internal bug"; + const char *target = NULL; + const char *alias = NULL; + const char *dev_id = "Unspecified device"; + const char *action = (search == NULL)? NULL : search->action; + + CRM_CHECK((dev != NULL) && (action != NULL), goto search_report_results); + + if (dev->id != NULL) { + dev_id = dev->id; + } + + target = search->host; + if (target == NULL) { + can = TRUE; + check_type = "No target"; + goto search_report_results; + } + + /* Answer immediately if the device does not support the action + * or the local node is not allowed to perform it + */ + if (pcmk__str_eq(action, "on", pcmk__str_none) + && !pcmk_is_set(dev->flags, st_device_supports_on)) { + check_type = "Agent does not support 'on'"; + goto search_report_results; + + } else if (!localhost_is_eligible_with_remap(dev, action, target, + search->allow_suicide)) { + check_type = "This node is not allowed to execute action"; + goto search_report_results; + } + + // Check eligibility as specified by pcmk_host_check + check_type = target_list_type(dev); + alias = g_hash_table_lookup(dev->aliases, target); + if (pcmk__str_eq(check_type, PCMK__VALUE_NONE, pcmk__str_casei)) { + can = TRUE; + + } else if (pcmk__str_eq(check_type, "static-list", pcmk__str_casei)) { + if (pcmk__str_in_list(target, dev->targets, pcmk__str_casei)) { + can = TRUE; + } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP) + && g_hash_table_lookup(dev->aliases, target)) { + can = TRUE; + } + + } else if (pcmk__str_eq(check_type, "dynamic-list", pcmk__str_casei)) { + time_t now = time(NULL); + + if (dev->targets == NULL || dev->targets_age + 60 < now) { + int device_timeout = get_action_timeout(dev, "list", search->per_device_timeout); + + if (device_timeout > search->per_device_timeout) { + crm_notice("Since the pcmk_list_timeout(%ds) parameter of %s is larger than stonith-timeout(%ds), timeout may occur", + device_timeout, dev_id, search->per_device_timeout); + } + + crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)", + check_type, dev_id, target, action); + + schedule_internal_command(__func__, dev, "list", NULL, + search->per_device_timeout, search, dynamic_list_search_cb); + + /* we'll respond to this search request async in the cb */ + return; + } + + if (pcmk__str_in_list(((alias == NULL)? target : alias), dev->targets, + pcmk__str_casei)) { + can = TRUE; + } + + } else if (pcmk__str_eq(check_type, "status", pcmk__str_casei)) { + int device_timeout = get_action_timeout(dev, check_type, search->per_device_timeout); + + if (device_timeout > search->per_device_timeout) { + crm_notice("Since the pcmk_status_timeout(%ds) parameter of %s is larger than stonith-timeout(%ds), timeout may occur", + device_timeout, dev_id, search->per_device_timeout); + } + + crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)", + check_type, dev_id, target, action); + schedule_internal_command(__func__, dev, "status", target, + search->per_device_timeout, search, status_search_cb); + /* we'll respond to this search request async in the cb */ + return; + } else { + crm_err("Invalid value for " PCMK_STONITH_HOST_CHECK ": %s", check_type); + check_type = "Invalid " PCMK_STONITH_HOST_CHECK; + } + + search_report_results: + crm_info("%s is%s eligible to fence (%s) %s%s%s%s: %s", + dev_id, (can? "" : " not"), pcmk__s(action, "unspecified action"), + pcmk__s(target, "unspecified target"), + (alias == NULL)? "" : " (as '", pcmk__s(alias, ""), + (alias == NULL)? "" : "')", check_type); + search_devices_record_result(search, ((dev == NULL)? NULL : dev_id), can); +} + +static void +search_devices(gpointer key, gpointer value, gpointer user_data) +{ + stonith_device_t *dev = value; + struct device_search_s *search = user_data; + + can_fence_host_with_device(dev, search); +} + +#define DEFAULT_QUERY_TIMEOUT 20 +static void +get_capable_devices(const char *host, const char *action, int timeout, bool suicide, void *user_data, + void (*callback) (GList * devices, void *user_data), uint32_t support_action_only) +{ + struct device_search_s *search; + guint ndevices = g_hash_table_size(device_list); + + if (ndevices == 0) { + callback(NULL, user_data); + return; + } + + search = calloc(1, sizeof(struct device_search_s)); + if (!search) { + crm_crit("Cannot search for capable fence devices: %s", + strerror(ENOMEM)); + callback(NULL, user_data); + return; + } + + pcmk__str_update(&search->host, host); + pcmk__str_update(&search->action, action); + search->per_device_timeout = timeout; + search->allow_suicide = suicide; + search->callback = callback; + search->user_data = user_data; + search->support_action_only = support_action_only; + + /* We are guaranteed this many replies, even if a device is + * unregistered while the search is in progress. + */ + search->replies_needed = ndevices; + + crm_debug("Searching %d device%s to see which can execute '%s' targeting %s", + ndevices, pcmk__plural_s(ndevices), + (search->action? search->action : "unknown action"), + (search->host? search->host : "any node")); + g_hash_table_foreach(device_list, search_devices, search); +} + +struct st_query_data { + xmlNode *reply; + char *remote_peer; + char *client_id; + char *target; + char *action; + int call_options; +}; + +/*! + * \internal + * \brief Add action-specific attributes to query reply XML + * + * \param[in,out] xml XML to add attributes to + * \param[in] action Fence action + * \param[in] device Fence device + * \param[in] target Fence target + */ +static void +add_action_specific_attributes(xmlNode *xml, const char *action, + const stonith_device_t *device, + const char *target) +{ + int action_specific_timeout; + int delay_max; + int delay_base; + + CRM_CHECK(xml && action && device, return); + + if (is_action_required(action, device)) { + crm_trace("Action '%s' is required using %s", action, device->id); + crm_xml_add_int(xml, F_STONITH_DEVICE_REQUIRED, 1); + } + + action_specific_timeout = get_action_timeout(device, action, 0); + if (action_specific_timeout) { + crm_trace("Action '%s' has timeout %dms using %s", + action, action_specific_timeout, device->id); + crm_xml_add_int(xml, F_STONITH_ACTION_TIMEOUT, action_specific_timeout); + } + + delay_max = get_action_delay_max(device, action); + if (delay_max > 0) { + crm_trace("Action '%s' has maximum random delay %ds using %s", + action, delay_max, device->id); + crm_xml_add_int(xml, F_STONITH_DELAY_MAX, delay_max); + } + + delay_base = get_action_delay_base(device, action, target); + if (delay_base > 0) { + crm_xml_add_int(xml, F_STONITH_DELAY_BASE, delay_base); + } + + if ((delay_max > 0) && (delay_base == 0)) { + crm_trace("Action '%s' has maximum random delay %ds using %s", + action, delay_max, device->id); + } else if ((delay_max == 0) && (delay_base > 0)) { + crm_trace("Action '%s' has a static delay of %ds using %s", + action, delay_base, device->id); + } else if ((delay_max > 0) && (delay_base > 0)) { + crm_trace("Action '%s' has a minimum delay of %ds and a randomly chosen " + "maximum delay of %ds using %s", + action, delay_base, delay_max, device->id); + } +} + +/*! + * \internal + * \brief Add "disallowed" attribute to query reply XML if appropriate + * + * \param[in,out] xml XML to add attribute to + * \param[in] action Fence action + * \param[in] device Fence device + * \param[in] target Fence target + * \param[in] allow_suicide Whether self-fencing is allowed + */ +static void +add_disallowed(xmlNode *xml, const char *action, const stonith_device_t *device, + const char *target, gboolean allow_suicide) +{ + if (!localhost_is_eligible(device, action, target, allow_suicide)) { + crm_trace("Action '%s' using %s is disallowed for local host", + action, device->id); + pcmk__xe_set_bool_attr(xml, F_STONITH_ACTION_DISALLOWED, true); + } +} + +/*! + * \internal + * \brief Add child element with action-specific values to query reply XML + * + * \param[in,out] xml XML to add attribute to + * \param[in] action Fence action + * \param[in] device Fence device + * \param[in] target Fence target + * \param[in] allow_suicide Whether self-fencing is allowed + */ +static void +add_action_reply(xmlNode *xml, const char *action, + const stonith_device_t *device, const char *target, + gboolean allow_suicide) +{ + xmlNode *child = create_xml_node(xml, F_STONITH_ACTION); + + crm_xml_add(child, XML_ATTR_ID, action); + add_action_specific_attributes(child, action, device, target); + add_disallowed(child, action, device, target, allow_suicide); +} + +static void +stonith_query_capable_device_cb(GList * devices, void *user_data) +{ + struct st_query_data *query = user_data; + int available_devices = 0; + xmlNode *dev = NULL; + xmlNode *list = NULL; + GList *lpc = NULL; + pcmk__client_t *client = NULL; + + if (query->client_id != NULL) { + client = pcmk__find_client_by_id(query->client_id); + if ((client == NULL) && (query->remote_peer == NULL)) { + crm_trace("Skipping reply to %s: no longer a client", + query->client_id); + goto done; + } + } + + /* Pack the results into XML */ + list = create_xml_node(NULL, __func__); + crm_xml_add(list, F_STONITH_TARGET, query->target); + for (lpc = devices; lpc != NULL; lpc = lpc->next) { + stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data); + const char *action = query->action; + + if (!device) { + /* It is possible the device got unregistered while + * determining who can fence the target */ + continue; + } + + available_devices++; + + dev = create_xml_node(list, F_STONITH_DEVICE); + crm_xml_add(dev, XML_ATTR_ID, device->id); + crm_xml_add(dev, "namespace", device->namespace); + crm_xml_add(dev, "agent", device->agent); + crm_xml_add_int(dev, F_STONITH_DEVICE_VERIFIED, device->verified); + crm_xml_add_int(dev, F_STONITH_DEVICE_SUPPORT_FLAGS, device->flags); + + /* If the originating fencer wants to reboot the node, and we have a + * capable device that doesn't support "reboot", remap to "off" instead. + */ + if (!pcmk_is_set(device->flags, st_device_supports_reboot) + && pcmk__str_eq(query->action, "reboot", pcmk__str_none)) { + crm_trace("%s doesn't support reboot, using values for off instead", + device->id); + action = "off"; + } + + /* Add action-specific values if available */ + add_action_specific_attributes(dev, action, device, query->target); + if (pcmk__str_eq(query->action, "reboot", pcmk__str_none)) { + /* A "reboot" *might* get remapped to "off" then "on", so after + * sending the "reboot"-specific values in the main element, we add + * sub-elements for "off" and "on" values. + * + * We short-circuited earlier if "reboot", "off" and "on" are all + * disallowed for the local host. However if only one or two are + * disallowed, we send back the results and mark which ones are + * disallowed. If "reboot" is disallowed, this might cause problems + * with older fencer versions, which won't check for it. Older + * versions will ignore "off" and "on", so they are not a problem. + */ + add_disallowed(dev, action, device, query->target, + pcmk_is_set(query->call_options, st_opt_allow_suicide)); + add_action_reply(dev, "off", device, query->target, + pcmk_is_set(query->call_options, st_opt_allow_suicide)); + add_action_reply(dev, "on", device, query->target, FALSE); + } + + /* A query without a target wants device parameters */ + if (query->target == NULL) { + xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS); + + g_hash_table_foreach(device->params, hash2field, attrs); + } + } + + crm_xml_add_int(list, F_STONITH_AVAILABLE_DEVICES, available_devices); + if (query->target) { + crm_debug("Found %d matching device%s for target '%s'", + available_devices, pcmk__plural_s(available_devices), + query->target); + } else { + crm_debug("%d device%s installed", + available_devices, pcmk__plural_s(available_devices)); + } + + if (list != NULL) { + crm_log_xml_trace(list, "Add query results"); + add_message_xml(query->reply, F_STONITH_CALLDATA, list); + } + + stonith_send_reply(query->reply, query->call_options, query->remote_peer, + client); + +done: + free_xml(query->reply); + free(query->remote_peer); + free(query->client_id); + free(query->target); + free(query->action); + free(query); + free_xml(list); + g_list_free_full(devices, free); +} + +/*! + * \internal + * \brief Log the result of an asynchronous command + * + * \param[in] cmd Command the result is for + * \param[in] result Result of command + * \param[in] pid Process ID of command, if available + * \param[in] next Alternate device that will be tried if command failed + * \param[in] op_merged Whether this command was merged with an earlier one + */ +static void +log_async_result(const async_command_t *cmd, + const pcmk__action_result_t *result, + int pid, const char *next, bool op_merged) +{ + int log_level = LOG_ERR; + int output_log_level = LOG_NEVER; + guint devices_remaining = g_list_length(cmd->next_device_iter); + + GString *msg = g_string_sized_new(80); // Reasonable starting size + + // Choose log levels appropriately if we have a result + if (pcmk__result_ok(result)) { + log_level = (cmd->target == NULL)? LOG_DEBUG : LOG_NOTICE; + if ((result->action_stdout != NULL) + && !pcmk__str_eq(cmd->action, "metadata", pcmk__str_none)) { + output_log_level = LOG_DEBUG; + } + next = NULL; + } else { + log_level = (cmd->target == NULL)? LOG_NOTICE : LOG_ERR; + if ((result->action_stdout != NULL) + && !pcmk__str_eq(cmd->action, "metadata", pcmk__str_none)) { + output_log_level = LOG_WARNING; + } + } + + // Build the log message piece by piece + pcmk__g_strcat(msg, "Operation '", cmd->action, "' ", NULL); + if (pid != 0) { + g_string_append_printf(msg, "[%d] ", pid); + } + if (cmd->target != NULL) { + pcmk__g_strcat(msg, "targeting ", cmd->target, " ", NULL); + } + if (cmd->device != NULL) { + pcmk__g_strcat(msg, "using ", cmd->device, " ", NULL); + } + + // Add exit status or execution status as appropriate + if (result->execution_status == PCMK_EXEC_DONE) { + g_string_append_printf(msg, "returned %d", result->exit_status); + } else { + pcmk__g_strcat(msg, "could not be executed: ", + pcmk_exec_status_str(result->execution_status), NULL); + } + + // Add exit reason and next device if appropriate + if (result->exit_reason != NULL) { + pcmk__g_strcat(msg, " (", result->exit_reason, ")", NULL); + } + if (next != NULL) { + pcmk__g_strcat(msg, ", retrying with ", next, NULL); + } + if (devices_remaining > 0) { + g_string_append_printf(msg, " (%u device%s remaining)", + (unsigned int) devices_remaining, + pcmk__plural_s(devices_remaining)); + } + g_string_append_printf(msg, " " CRM_XS " %scall %d from %s", + (op_merged? "merged " : ""), cmd->id, + cmd->client_name); + + // Log the result + do_crm_log(log_level, "%s", msg->str); + g_string_free(msg, TRUE); + + // Log the output (which may have multiple lines), if appropriate + if (output_log_level != LOG_NEVER) { + char *prefix = crm_strdup_printf("%s[%d]", cmd->device, pid); + + crm_log_output(output_log_level, prefix, result->action_stdout); + free(prefix); + } +} + +/*! + * \internal + * \brief Reply to requester after asynchronous command completion + * + * \param[in] cmd Command that completed + * \param[in] result Result of command + * \param[in] pid Process ID of command, if available + * \param[in] merged If true, command was merged with another, not executed + */ +static void +send_async_reply(const async_command_t *cmd, const pcmk__action_result_t *result, + int pid, bool merged) +{ + xmlNode *reply = NULL; + pcmk__client_t *client = NULL; + + CRM_CHECK((cmd != NULL) && (result != NULL), return); + + log_async_result(cmd, result, pid, NULL, merged); + + if (cmd->client != NULL) { + client = pcmk__find_client_by_id(cmd->client); + if ((client == NULL) && (cmd->origin == NULL)) { + crm_trace("Skipping reply to %s: no longer a client", cmd->client); + return; + } + } + + reply = construct_async_reply(cmd, result); + if (merged) { + pcmk__xe_set_bool_attr(reply, F_STONITH_MERGED, true); + } + + if (!stand_alone && pcmk__is_fencing_action(cmd->action) + && pcmk__str_eq(cmd->origin, cmd->target, pcmk__str_casei)) { + /* The target was also the originator, so broadcast the result on its + * behalf (since it will be unable to). + */ + crm_trace("Broadcast '%s' result for %s (target was also originator)", + cmd->action, cmd->target); + crm_xml_add(reply, F_SUBTYPE, "broadcast"); + crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY); + send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE); + } else { + // Reply only to the originator + stonith_send_reply(reply, cmd->options, cmd->origin, client); + } + + crm_log_xml_trace(reply, "Reply"); + free_xml(reply); + + if (stand_alone) { + /* Do notification with a clean data object */ + xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE); + + stonith__xe_set_result(notify_data, result); + crm_xml_add(notify_data, F_STONITH_TARGET, cmd->target); + crm_xml_add(notify_data, F_STONITH_OPERATION, cmd->op); + crm_xml_add(notify_data, F_STONITH_DELEGATE, "localhost"); + crm_xml_add(notify_data, F_STONITH_DEVICE, cmd->device); + crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id); + crm_xml_add(notify_data, F_STONITH_ORIGIN, cmd->client); + + fenced_send_notification(T_STONITH_NOTIFY_FENCE, result, notify_data); + fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL); + } +} + +static void +cancel_stonith_command(async_command_t * cmd) +{ + stonith_device_t *device = cmd_device(cmd); + + if (device) { + crm_trace("Cancel scheduled '%s' action using %s", + cmd->action, device->id); + device->pending_ops = g_list_remove(device->pending_ops, cmd); + } +} + +/*! + * \internal + * \brief Cancel and reply to any duplicates of a just-completed operation + * + * Check whether any fencing operations are scheduled to do the same thing as + * one that just succeeded. If so, rather than performing the same operation + * twice, return the result of this operation for all matching pending commands. + * + * \param[in,out] cmd Fencing operation that just succeeded + * \param[in] result Result of \p cmd + * \param[in] pid If nonzero, process ID of agent invocation (for logs) + * + * \note Duplicate merging will do the right thing for either type of remapped + * reboot. If the executing fencer remapped an unsupported reboot to off, + * then cmd->action will be "reboot" and will be merged with any other + * reboot requests. If the originating fencer remapped a topology reboot + * to off then on, we will get here once with cmd->action "off" and once + * with "on", and they will be merged separately with similar requests. + */ +static void +reply_to_duplicates(async_command_t *cmd, const pcmk__action_result_t *result, + int pid) +{ + GList *next = NULL; + + for (GList *iter = cmd_list; iter != NULL; iter = next) { + async_command_t *cmd_other = iter->data; + + next = iter->next; // We might delete this entry, so grab next now + + if (cmd == cmd_other) { + continue; + } + + /* A pending operation matches if: + * 1. The client connections are different. + * 2. The target is the same. + * 3. The fencing action is the same. + * 4. The device scheduled to execute the action is the same. + */ + if (pcmk__str_eq(cmd->client, cmd_other->client, pcmk__str_casei) || + !pcmk__str_eq(cmd->target, cmd_other->target, pcmk__str_casei) || + !pcmk__str_eq(cmd->action, cmd_other->action, pcmk__str_none) || + !pcmk__str_eq(cmd->device, cmd_other->device, pcmk__str_casei)) { + + continue; + } + + crm_notice("Merging fencing action '%s'%s%s originating from " + "client %s with identical fencing request from client %s", + cmd_other->action, + (cmd_other->target == NULL)? "" : " targeting ", + pcmk__s(cmd_other->target, ""), cmd_other->client_name, + cmd->client_name); + + // Stop tracking the duplicate, send its result, and cancel it + cmd_list = g_list_remove_link(cmd_list, iter); + send_async_reply(cmd_other, result, pid, true); + cancel_stonith_command(cmd_other); + + free_async_command(cmd_other); + g_list_free_1(iter); + } +} + +/*! + * \internal + * \brief Return the next required device (if any) for an operation + * + * \param[in,out] cmd Fencing operation that just succeeded + * + * \return Next device required for action if any, otherwise NULL + */ +static stonith_device_t * +next_required_device(async_command_t *cmd) +{ + for (GList *iter = cmd->next_device_iter; iter != NULL; iter = iter->next) { + stonith_device_t *next_device = g_hash_table_lookup(device_list, + iter->data); + + if (is_action_required(cmd->action, next_device)) { + /* This is only called for successful actions, so it's OK to skip + * non-required devices. + */ + cmd->next_device_iter = iter->next; + return next_device; + } + } + return NULL; +} + +static void +st_child_done(int pid, const pcmk__action_result_t *result, void *user_data) +{ + async_command_t *cmd = user_data; + + stonith_device_t *device = NULL; + stonith_device_t *next_device = NULL; + + CRM_CHECK(cmd != NULL, return); + + device = cmd_device(cmd); + cmd->active_on = NULL; + + /* The device is ready to do something else now */ + if (device) { + if (!device->verified && pcmk__result_ok(result) && + (pcmk__strcase_any_of(cmd->action, "list", "monitor", "status", NULL))) { + + device->verified = TRUE; + } + + mainloop_set_trigger(device->work); + } + + if (pcmk__result_ok(result)) { + next_device = next_required_device(cmd); + + } else if ((cmd->next_device_iter != NULL) + && !is_action_required(cmd->action, device)) { + /* if this device didn't work out, see if there are any others we can try. + * if the failed device was 'required', we can't pick another device. */ + next_device = g_hash_table_lookup(device_list, + cmd->next_device_iter->data); + cmd->next_device_iter = cmd->next_device_iter->next; + } + + if (next_device == NULL) { + send_async_reply(cmd, result, pid, false); + if (pcmk__result_ok(result)) { + reply_to_duplicates(cmd, result, pid); + } + free_async_command(cmd); + + } else { // This operation requires more fencing + log_async_result(cmd, result, pid, next_device->id, false); + schedule_stonith_command(cmd, next_device); + } +} + +static gint +sort_device_priority(gconstpointer a, gconstpointer b) +{ + const stonith_device_t *dev_a = a; + const stonith_device_t *dev_b = b; + + if (dev_a->priority > dev_b->priority) { + return -1; + } else if (dev_a->priority < dev_b->priority) { + return 1; + } + return 0; +} + +static void +stonith_fence_get_devices_cb(GList * devices, void *user_data) +{ + async_command_t *cmd = user_data; + stonith_device_t *device = NULL; + guint ndevices = g_list_length(devices); + + crm_info("Found %d matching device%s for target '%s'", + ndevices, pcmk__plural_s(ndevices), cmd->target); + + if (devices != NULL) { + /* Order based on priority */ + devices = g_list_sort(devices, sort_device_priority); + device = g_hash_table_lookup(device_list, devices->data); + } + + if (device == NULL) { // No device found + pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; + + pcmk__format_result(&result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE, + "No device configured for target '%s'", + cmd->target); + send_async_reply(cmd, &result, 0, false); + pcmk__reset_result(&result); + free_async_command(cmd); + g_list_free_full(devices, free); + + } else { // Device found, schedule it for fencing + cmd->device_list = devices; + cmd->next_device_iter = devices->next; + schedule_stonith_command(cmd, device); + } +} + +/*! + * \internal + * \brief Execute a fence action via the local node + * + * \param[in] msg Fencing request + * \param[out] result Where to store result of fence action + */ +static void +fence_locally(xmlNode *msg, pcmk__action_result_t *result) +{ + const char *device_id = NULL; + stonith_device_t *device = NULL; + async_command_t *cmd = NULL; + xmlNode *dev = NULL; + + CRM_CHECK((msg != NULL) && (result != NULL), return); + + dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR); + + cmd = create_async_command(msg); + if (cmd == NULL) { + crm_log_xml_warn(msg, "invalid"); + fenced_set_protocol_error(result); + return; + } + + device_id = crm_element_value(dev, F_STONITH_DEVICE); + if (device_id != NULL) { + device = g_hash_table_lookup(device_list, device_id); + if (device == NULL) { + crm_err("Requested device '%s' is not available", device_id); + pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE, + "Requested device '%s' not found", device_id); + return; + } + schedule_stonith_command(cmd, device); + + } else { + const char *host = crm_element_value(dev, F_STONITH_TARGET); + + if (pcmk_is_set(cmd->options, st_opt_cs_nodeid)) { + int nodeid = 0; + crm_node_t *node = NULL; + + pcmk__scan_min_int(host, &nodeid, 0); + node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY); + if (node != NULL) { + host = node->uname; + } + } + + /* If we get to here, then self-fencing is implicitly allowed */ + get_capable_devices(host, cmd->action, cmd->default_timeout, + TRUE, cmd, stonith_fence_get_devices_cb, + fenced_support_flag(cmd->action)); + } + + pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL); +} + +/*! + * \internal + * \brief Build an XML reply for a fencing operation + * + * \param[in] request Request that reply is for + * \param[in] data If not NULL, add to reply as call data + * \param[in] result Full result of fencing operation + * + * \return Newly created XML reply + * \note The caller is responsible for freeing the result. + * \note This has some overlap with construct_async_reply(), but that copies + * values from an async_command_t, whereas this one copies them from the + * request. + */ +xmlNode * +fenced_construct_reply(const xmlNode *request, xmlNode *data, + const pcmk__action_result_t *result) +{ + xmlNode *reply = NULL; + + reply = create_xml_node(NULL, T_STONITH_REPLY); + + crm_xml_add(reply, "st_origin", __func__); + crm_xml_add(reply, F_TYPE, T_STONITH_NG); + stonith__xe_set_result(reply, result); + + if (request == NULL) { + /* Most likely, this is the result of a stonith operation that was + * initiated before we came up. Unfortunately that means we lack enough + * information to provide clients with a full result. + * + * @TODO Maybe synchronize this information at start-up? + */ + crm_warn("Missing request information for client notifications for " + "operation with result '%s' (initiated before we came up?)", + pcmk_exec_status_str(result->execution_status)); + + } else { + const char *name = NULL; + const char *value = NULL; + + // Attributes to copy from request to reply + const char *names[] = { + F_STONITH_OPERATION, + F_STONITH_CALLID, + F_STONITH_CLIENTID, + F_STONITH_CLIENTNAME, + F_STONITH_REMOTE_OP_ID, + F_STONITH_CALLOPTS + }; + + for (int lpc = 0; lpc < PCMK__NELEM(names); lpc++) { + name = names[lpc]; + value = crm_element_value(request, name); + crm_xml_add(reply, name, value); + } + if (data != NULL) { + add_message_xml(reply, F_STONITH_CALLDATA, data); + } + } + return reply; +} + +/*! + * \internal + * \brief Build an XML reply to an asynchronous fencing command + * + * \param[in] cmd Fencing command that reply is for + * \param[in] result Command result + */ +static xmlNode * +construct_async_reply(const async_command_t *cmd, + const pcmk__action_result_t *result) +{ + xmlNode *reply = create_xml_node(NULL, T_STONITH_REPLY); + + crm_xml_add(reply, "st_origin", __func__); + crm_xml_add(reply, F_TYPE, T_STONITH_NG); + crm_xml_add(reply, F_STONITH_OPERATION, cmd->op); + crm_xml_add(reply, F_STONITH_DEVICE, cmd->device); + crm_xml_add(reply, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id); + crm_xml_add(reply, F_STONITH_CLIENTID, cmd->client); + crm_xml_add(reply, F_STONITH_CLIENTNAME, cmd->client_name); + crm_xml_add(reply, F_STONITH_TARGET, cmd->target); + crm_xml_add(reply, F_STONITH_ACTION, cmd->op); + crm_xml_add(reply, F_STONITH_ORIGIN, cmd->origin); + crm_xml_add_int(reply, F_STONITH_CALLID, cmd->id); + crm_xml_add_int(reply, F_STONITH_CALLOPTS, cmd->options); + + stonith__xe_set_result(reply, result); + return reply; +} + +bool fencing_peer_active(crm_node_t *peer) +{ + if (peer == NULL) { + return FALSE; + } else if (peer->uname == NULL) { + return FALSE; + } else if (pcmk_is_set(peer->processes, crm_get_cluster_proc())) { + return TRUE; + } + return FALSE; +} + +void +set_fencing_completed(remote_fencing_op_t *op) +{ + struct timespec tv; + + qb_util_timespec_from_epoch_get(&tv); + op->completed = tv.tv_sec; + op->completed_nsec = tv.tv_nsec; +} + +/*! + * \internal + * \brief Look for alternate node needed if local node shouldn't fence target + * + * \param[in] target Node that must be fenced + * + * \return Name of an alternate node that should fence \p target if any, + * or NULL otherwise + */ +static const char * +check_alternate_host(const char *target) +{ + if (pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) { + GHashTableIter gIter; + crm_node_t *entry = NULL; + + g_hash_table_iter_init(&gIter, crm_peer_cache); + while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) { + if (fencing_peer_active(entry) + && !pcmk__str_eq(entry->uname, target, pcmk__str_casei)) { + crm_notice("Forwarding self-fencing request to %s", + entry->uname); + return entry->uname; + } + } + crm_warn("Will handle own fencing because no peer can"); + } + return NULL; +} + +/*! + * \internal + * \brief Send a reply to a CPG peer or IPC client + * + * \param[in] reply XML reply to send + * \param[in] call_options Send synchronously if st_opt_sync_call is set + * \param[in] remote_peer If not NULL, name of peer node to send CPG reply + * \param[in,out] client If not NULL, client to send IPC reply + */ +static void +stonith_send_reply(xmlNode *reply, int call_options, const char *remote_peer, + pcmk__client_t *client) +{ + CRM_CHECK((reply != NULL) && ((remote_peer != NULL) || (client != NULL)), + return); + + if (remote_peer == NULL) { + do_local_reply(reply, client, call_options); + } else { + send_cluster_message(crm_get_peer(0, remote_peer), crm_msg_stonith_ng, + reply, FALSE); + } +} + +static void +remove_relay_op(xmlNode * request) +{ + xmlNode *dev = get_xpath_object("//@" F_STONITH_ACTION, request, LOG_TRACE); + const char *relay_op_id = NULL; + const char *op_id = NULL; + const char *client_name = NULL; + const char *target = NULL; + remote_fencing_op_t *relay_op = NULL; + + if (dev) { + target = crm_element_value(dev, F_STONITH_TARGET); + } + + relay_op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID_RELAY); + op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID); + client_name = crm_element_value(request, F_STONITH_CLIENTNAME); + + /* Delete RELAY operation. */ + if (relay_op_id && target && pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) { + relay_op = g_hash_table_lookup(stonith_remote_op_list, relay_op_id); + + if (relay_op) { + GHashTableIter iter; + remote_fencing_op_t *list_op = NULL; + g_hash_table_iter_init(&iter, stonith_remote_op_list); + + /* If the operation to be deleted is registered as a duplicate, delete the registration. */ + while (g_hash_table_iter_next(&iter, NULL, (void **)&list_op)) { + GList *dup_iter = NULL; + if (list_op != relay_op) { + for (dup_iter = list_op->duplicates; dup_iter != NULL; dup_iter = dup_iter->next) { + remote_fencing_op_t *other = dup_iter->data; + if (other == relay_op) { + other->duplicates = g_list_remove(other->duplicates, relay_op); + break; + } + } + } + } + crm_debug("Deleting relay op %s ('%s'%s%s for %s), " + "replaced by op %s ('%s'%s%s for %s)", + relay_op->id, relay_op->action, + (relay_op->target == NULL)? "" : " targeting ", + pcmk__s(relay_op->target, ""), + relay_op->client_name, op_id, relay_op->action, + (target == NULL)? "" : " targeting ", pcmk__s(target, ""), + client_name); + + g_hash_table_remove(stonith_remote_op_list, relay_op_id); + } + } +} + +/*! + * \internal + * \brief Check whether an API request was sent by a privileged user + * + * API commands related to fencing configuration may be done only by privileged + * IPC users (i.e. root or hacluster), because all other users should go through + * the CIB to have ACLs applied. If no client was given, this is a peer request, + * which is always allowed. + * + * \param[in] c IPC client that sent request (or NULL if sent by CPG peer) + * \param[in] op Requested API operation (for logging only) + * + * \return true if sender is peer or privileged client, otherwise false + */ +static inline bool +is_privileged(const pcmk__client_t *c, const char *op) +{ + if ((c == NULL) || pcmk_is_set(c->flags, pcmk__client_privileged)) { + return true; + } else { + crm_warn("Rejecting IPC request '%s' from unprivileged client %s", + pcmk__s(op, ""), pcmk__client_name(c)); + return false; + } +} + +// CRM_OP_REGISTER +static xmlNode * +handle_register_request(pcmk__request_t *request) +{ + xmlNode *reply = create_xml_node(NULL, "reply"); + + CRM_ASSERT(request->ipc_client != NULL); + crm_xml_add(reply, F_STONITH_OPERATION, CRM_OP_REGISTER); + crm_xml_add(reply, F_STONITH_CLIENTID, request->ipc_client->id); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + pcmk__set_request_flags(request, pcmk__request_reuse_options); + return reply; +} + +// STONITH_OP_EXEC +static xmlNode * +handle_agent_request(pcmk__request_t *request) +{ + execute_agent_action(request->xml, &request->result); + if (request->result.execution_status == PCMK_EXEC_PENDING) { + return NULL; + } + return fenced_construct_reply(request->xml, NULL, &request->result); +} + +// STONITH_OP_TIMEOUT_UPDATE +static xmlNode * +handle_update_timeout_request(pcmk__request_t *request) +{ + const char *call_id = crm_element_value(request->xml, F_STONITH_CALLID); + const char *client_id = crm_element_value(request->xml, F_STONITH_CLIENTID); + int op_timeout = 0; + + crm_element_value_int(request->xml, F_STONITH_TIMEOUT, &op_timeout); + do_stonith_async_timeout_update(client_id, call_id, op_timeout); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + return NULL; +} + +// STONITH_OP_QUERY +static xmlNode * +handle_query_request(pcmk__request_t *request) +{ + int timeout = 0; + xmlNode *dev = NULL; + const char *action = NULL; + const char *target = NULL; + const char *client_id = crm_element_value(request->xml, F_STONITH_CLIENTID); + struct st_query_data *query = NULL; + + if (request->peer != NULL) { + // Record it for the future notification + create_remote_stonith_op(client_id, request->xml, TRUE); + } + + /* Delete the DC node RELAY operation. */ + remove_relay_op(request->xml); + + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + + dev = get_xpath_object("//@" F_STONITH_ACTION, request->xml, LOG_NEVER); + if (dev != NULL) { + const char *device = crm_element_value(dev, F_STONITH_DEVICE); + + if (pcmk__str_eq(device, "manual_ack", pcmk__str_casei)) { + return NULL; // No query or reply necessary + } + target = crm_element_value(dev, F_STONITH_TARGET); + action = crm_element_value(dev, F_STONITH_ACTION); + } + + crm_log_xml_trace(request->xml, "Query"); + + query = calloc(1, sizeof(struct st_query_data)); + CRM_ASSERT(query != NULL); + + query->reply = fenced_construct_reply(request->xml, NULL, &request->result); + pcmk__str_update(&query->remote_peer, request->peer); + pcmk__str_update(&query->client_id, client_id); + pcmk__str_update(&query->target, target); + pcmk__str_update(&query->action, action); + query->call_options = request->call_options; + + crm_element_value_int(request->xml, F_STONITH_TIMEOUT, &timeout); + get_capable_devices(target, action, timeout, + pcmk_is_set(query->call_options, st_opt_allow_suicide), + query, stonith_query_capable_device_cb, st_device_supports_none); + return NULL; +} + +// T_STONITH_NOTIFY +static xmlNode * +handle_notify_request(pcmk__request_t *request) +{ + const char *flag_name = NULL; + + CRM_ASSERT(request->ipc_client != NULL); + flag_name = crm_element_value(request->xml, F_STONITH_NOTIFY_ACTIVATE); + if (flag_name != NULL) { + crm_debug("Enabling %s callbacks for client %s", + flag_name, pcmk__request_origin(request)); + pcmk__set_client_flags(request->ipc_client, get_stonith_flag(flag_name)); + } + + flag_name = crm_element_value(request->xml, F_STONITH_NOTIFY_DEACTIVATE); + if (flag_name != NULL) { + crm_debug("Disabling %s callbacks for client %s", + flag_name, pcmk__request_origin(request)); + pcmk__clear_client_flags(request->ipc_client, + get_stonith_flag(flag_name)); + } + + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + pcmk__set_request_flags(request, pcmk__request_reuse_options); + + return pcmk__ipc_create_ack(request->ipc_flags, "ack", NULL, CRM_EX_OK); +} + +// STONITH_OP_RELAY +static xmlNode * +handle_relay_request(pcmk__request_t *request) +{ + xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request->xml, + LOG_TRACE); + + crm_notice("Received forwarded fencing request from " + "%s %s to fence (%s) peer %s", + pcmk__request_origin_type(request), + pcmk__request_origin(request), + crm_element_value(dev, F_STONITH_ACTION), + crm_element_value(dev, F_STONITH_TARGET)); + + if (initiate_remote_stonith_op(NULL, request->xml, FALSE) == NULL) { + fenced_set_protocol_error(&request->result); + return fenced_construct_reply(request->xml, NULL, &request->result); + } + + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL); + return NULL; +} + +// STONITH_OP_FENCE +static xmlNode * +handle_fence_request(pcmk__request_t *request) +{ + if ((request->peer != NULL) || stand_alone) { + fence_locally(request->xml, &request->result); + + } else if (pcmk_is_set(request->call_options, st_opt_manual_ack)) { + switch (fenced_handle_manual_confirmation(request->ipc_client, + request->xml)) { + case pcmk_rc_ok: + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, + NULL); + break; + case EINPROGRESS: + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, + NULL); + break; + default: + fenced_set_protocol_error(&request->result); + break; + } + + } else { + const char *alternate_host = NULL; + xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request->xml, + LOG_TRACE); + const char *target = crm_element_value(dev, F_STONITH_TARGET); + const char *action = crm_element_value(dev, F_STONITH_ACTION); + const char *device = crm_element_value(dev, F_STONITH_DEVICE); + + if (request->ipc_client != NULL) { + int tolerance = 0; + + crm_notice("Client %s wants to fence (%s) %s using %s", + pcmk__request_origin(request), action, + target, (device? device : "any device")); + crm_element_value_int(dev, F_STONITH_TOLERANCE, &tolerance); + if (stonith_check_fence_tolerance(tolerance, target, action)) { + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, + NULL); + return fenced_construct_reply(request->xml, NULL, + &request->result); + } + alternate_host = check_alternate_host(target); + + } else { + crm_notice("Peer %s wants to fence (%s) '%s' with device '%s'", + request->peer, action, target, + (device == NULL)? "(any)" : device); + } + + if (alternate_host != NULL) { + const char *client_id = NULL; + remote_fencing_op_t *op = NULL; + + if (request->ipc_client->id == 0) { + client_id = crm_element_value(request->xml, F_STONITH_CLIENTID); + } else { + client_id = request->ipc_client->id; + } + + /* Create a duplicate fencing operation to relay with the client ID. + * When a query response is received, this operation should be + * deleted to avoid keeping the duplicate around. + */ + op = create_remote_stonith_op(client_id, request->xml, FALSE); + + crm_xml_add(request->xml, F_STONITH_OPERATION, STONITH_OP_RELAY); + crm_xml_add(request->xml, F_STONITH_CLIENTID, + request->ipc_client->id); + crm_xml_add(request->xml, F_STONITH_REMOTE_OP_ID, op->id); + send_cluster_message(crm_get_peer(0, alternate_host), + crm_msg_stonith_ng, request->xml, FALSE); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, + NULL); + + } else if (initiate_remote_stonith_op(request->ipc_client, request->xml, + FALSE) == NULL) { + fenced_set_protocol_error(&request->result); + + } else { + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, + NULL); + } + } + + if (request->result.execution_status == PCMK_EXEC_PENDING) { + return NULL; + } + return fenced_construct_reply(request->xml, NULL, &request->result); +} + +// STONITH_OP_FENCE_HISTORY +static xmlNode * +handle_history_request(pcmk__request_t *request) +{ + xmlNode *reply = NULL; + xmlNode *data = NULL; + + stonith_fence_history(request->xml, &data, request->peer, + request->call_options); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + if (!pcmk_is_set(request->call_options, st_opt_discard_reply)) { + /* When the local node broadcasts its history, it sets + * st_opt_discard_reply and doesn't need a reply. + */ + reply = fenced_construct_reply(request->xml, data, &request->result); + } + free_xml(data); + return reply; +} + +// STONITH_OP_DEVICE_ADD +static xmlNode * +handle_device_add_request(pcmk__request_t *request) +{ + const char *op = crm_element_value(request->xml, F_STONITH_OPERATION); + xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request->xml, + LOG_ERR); + + if (is_privileged(request->ipc_client, op)) { + int rc = stonith_device_register(dev, FALSE); + + pcmk__set_result(&request->result, + ((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR), + stonith__legacy2status(rc), + ((rc == pcmk_ok)? NULL : pcmk_strerror(rc))); + } else { + pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV, + PCMK_EXEC_INVALID, + "Unprivileged users must register device via CIB"); + } + fenced_send_device_notification(op, &request->result, + (dev == NULL)? NULL : ID(dev)); + return fenced_construct_reply(request->xml, NULL, &request->result); +} + +// STONITH_OP_DEVICE_DEL +static xmlNode * +handle_device_delete_request(pcmk__request_t *request) +{ + xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request->xml, + LOG_ERR); + const char *device_id = crm_element_value(dev, XML_ATTR_ID); + const char *op = crm_element_value(request->xml, F_STONITH_OPERATION); + + if (is_privileged(request->ipc_client, op)) { + stonith_device_remove(device_id, false); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + } else { + pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV, + PCMK_EXEC_INVALID, + "Unprivileged users must delete device via CIB"); + } + fenced_send_device_notification(op, &request->result, device_id); + return fenced_construct_reply(request->xml, NULL, &request->result); +} + +// STONITH_OP_LEVEL_ADD +static xmlNode * +handle_level_add_request(pcmk__request_t *request) +{ + char *desc = NULL; + const char *op = crm_element_value(request->xml, F_STONITH_OPERATION); + + if (is_privileged(request->ipc_client, op)) { + fenced_register_level(request->xml, &desc, &request->result); + } else { + unpack_level_request(request->xml, NULL, NULL, NULL, &desc); + pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV, + PCMK_EXEC_INVALID, + "Unprivileged users must add level via CIB"); + } + fenced_send_level_notification(op, &request->result, desc); + free(desc); + return fenced_construct_reply(request->xml, NULL, &request->result); +} + +// STONITH_OP_LEVEL_DEL +static xmlNode * +handle_level_delete_request(pcmk__request_t *request) +{ + char *desc = NULL; + const char *op = crm_element_value(request->xml, F_STONITH_OPERATION); + + if (is_privileged(request->ipc_client, op)) { + fenced_unregister_level(request->xml, &desc, &request->result); + } else { + unpack_level_request(request->xml, NULL, NULL, NULL, &desc); + pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV, + PCMK_EXEC_INVALID, + "Unprivileged users must delete level via CIB"); + } + fenced_send_level_notification(op, &request->result, desc); + free(desc); + return fenced_construct_reply(request->xml, NULL, &request->result); +} + +// CRM_OP_RM_NODE_CACHE +static xmlNode * +handle_cache_request(pcmk__request_t *request) +{ + int node_id = 0; + const char *name = NULL; + + crm_element_value_int(request->xml, XML_ATTR_ID, &node_id); + name = crm_element_value(request->xml, XML_ATTR_UNAME); + reap_crm_member(node_id, name); + pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + return NULL; +} + +static xmlNode * +handle_unknown_request(pcmk__request_t *request) +{ + crm_err("Unknown IPC request %s from %s %s", + request->op, pcmk__request_origin_type(request), + pcmk__request_origin(request)); + pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID, + "Unknown IPC request type '%s' (bug?)", request->op); + return fenced_construct_reply(request->xml, NULL, &request->result); +} + +static void +fenced_register_handlers(void) +{ + pcmk__server_command_t handlers[] = { + { CRM_OP_REGISTER, handle_register_request }, + { STONITH_OP_EXEC, handle_agent_request }, + { STONITH_OP_TIMEOUT_UPDATE, handle_update_timeout_request }, + { STONITH_OP_QUERY, handle_query_request }, + { T_STONITH_NOTIFY, handle_notify_request }, + { STONITH_OP_RELAY, handle_relay_request }, + { STONITH_OP_FENCE, handle_fence_request }, + { STONITH_OP_FENCE_HISTORY, handle_history_request }, + { STONITH_OP_DEVICE_ADD, handle_device_add_request }, + { STONITH_OP_DEVICE_DEL, handle_device_delete_request }, + { STONITH_OP_LEVEL_ADD, handle_level_add_request }, + { STONITH_OP_LEVEL_DEL, handle_level_delete_request }, + { CRM_OP_RM_NODE_CACHE, handle_cache_request }, + { NULL, handle_unknown_request }, + }; + + fenced_handlers = pcmk__register_handlers(handlers); +} + +void +fenced_unregister_handlers(void) +{ + if (fenced_handlers != NULL) { + g_hash_table_destroy(fenced_handlers); + fenced_handlers = NULL; + } +} + +static void +handle_request(pcmk__request_t *request) +{ + xmlNode *reply = NULL; + const char *reason = NULL; + + if (fenced_handlers == NULL) { + fenced_register_handlers(); + } + reply = pcmk__process_request(request, fenced_handlers); + if (reply != NULL) { + if (pcmk_is_set(request->flags, pcmk__request_reuse_options) + && (request->ipc_client != NULL)) { + /* Certain IPC-only commands must reuse the call options from the + * original request rather than the ones set by stonith_send_reply() + * -> do_local_reply(). + */ + pcmk__ipc_send_xml(request->ipc_client, request->ipc_id, reply, + request->ipc_flags); + request->ipc_client->request_id = 0; + } else { + stonith_send_reply(reply, request->call_options, + request->peer, request->ipc_client); + } + free_xml(reply); + } + + reason = request->result.exit_reason; + crm_debug("Processed %s request from %s %s: %s%s%s%s", + request->op, pcmk__request_origin_type(request), + pcmk__request_origin(request), + pcmk_exec_status_str(request->result.execution_status), + (reason == NULL)? "" : " (", + (reason == NULL)? "" : reason, + (reason == NULL)? "" : ")"); +} + +static void +handle_reply(pcmk__client_t *client, xmlNode *request, const char *remote_peer) +{ + // Copy, because request might be freed before we want to log this + char *op = crm_element_value_copy(request, F_STONITH_OPERATION); + + if (pcmk__str_eq(op, STONITH_OP_QUERY, pcmk__str_none)) { + process_remote_stonith_query(request); + } else if (pcmk__str_any_of(op, T_STONITH_NOTIFY, STONITH_OP_FENCE, NULL)) { + fenced_process_fencing_reply(request); + } else { + crm_err("Ignoring unknown %s reply from %s %s", + pcmk__s(op, "untyped"), ((client == NULL)? "peer" : "client"), + ((client == NULL)? remote_peer : pcmk__client_name(client))); + crm_log_xml_warn(request, "UnknownOp"); + free(op); + return; + } + crm_debug("Processed %s reply from %s %s", + op, ((client == NULL)? "peer" : "client"), + ((client == NULL)? remote_peer : pcmk__client_name(client))); + free(op); +} + +/*! + * \internal + * \brief Handle a message from an IPC client or CPG peer + * + * \param[in,out] client If not NULL, IPC client that sent message + * \param[in] id If from IPC client, IPC message ID + * \param[in] flags Message flags + * \param[in,out] message Message XML + * \param[in] remote_peer If not NULL, CPG peer that sent message + */ +void +stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags, + xmlNode *message, const char *remote_peer) +{ + int call_options = st_opt_none; + bool is_reply = false; + + CRM_CHECK(message != NULL, return); + + if (get_xpath_object("//" T_STONITH_REPLY, message, LOG_NEVER) != NULL) { + is_reply = true; + } + crm_element_value_int(message, F_STONITH_CALLOPTS, &call_options); + crm_debug("Processing %ssynchronous %s %s %u from %s %s", + pcmk_is_set(call_options, st_opt_sync_call)? "" : "a", + crm_element_value(message, F_STONITH_OPERATION), + (is_reply? "reply" : "request"), id, + ((client == NULL)? "peer" : "client"), + ((client == NULL)? remote_peer : pcmk__client_name(client))); + + if (pcmk_is_set(call_options, st_opt_sync_call)) { + CRM_ASSERT(client == NULL || client->request_id == id); + } + + if (is_reply) { + handle_reply(client, message, remote_peer); + } else { + pcmk__request_t request = { + .ipc_client = client, + .ipc_id = id, + .ipc_flags = flags, + .peer = remote_peer, + .xml = message, + .call_options = call_options, + .result = PCMK__UNKNOWN_RESULT, + }; + + request.op = crm_element_value_copy(request.xml, F_STONITH_OPERATION); + CRM_CHECK(request.op != NULL, return); + + if (pcmk_is_set(request.call_options, st_opt_sync_call)) { + pcmk__set_request_flags(&request, pcmk__request_sync); + } + + handle_request(&request); + pcmk__reset_request(&request); + } +} diff --git a/daemons/fenced/fenced_history.c b/daemons/fenced/fenced_history.c new file mode 100644 index 0000000..a766477 --- /dev/null +++ b/daemons/fenced/fenced_history.c @@ -0,0 +1,548 @@ +/* + * Copyright 2009-2022 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. + */ + +#include <crm_internal.h> + +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> + +#include <crm/crm.h> +#include <crm/msg_xml.h> +#include <crm/common/ipc.h> +#include <crm/common/ipc_internal.h> +#include <crm/cluster/internal.h> + +#include <crm/stonith-ng.h> +#include <crm/fencing/internal.h> +#include <crm/common/xml.h> +#include <crm/common/xml_internal.h> + +#include <pacemaker-fenced.h> + +#define MAX_STONITH_HISTORY 500 + +/*! + * \internal + * \brief Send a broadcast to all nodes to trigger cleanup or + * history synchronisation + * + * \param[in] history Optional history to be attached + * \param[in] callopts We control cleanup via a flag in the callopts + * \param[in] target Cleanup can be limited to certain fence-targets + */ +static void +stonith_send_broadcast_history(xmlNode *history, + int callopts, + const char *target) +{ + xmlNode *bcast = create_xml_node(NULL, "stonith_command"); + xmlNode *data = create_xml_node(NULL, __func__); + + if (target) { + crm_xml_add(data, F_STONITH_TARGET, target); + } + crm_xml_add(bcast, F_TYPE, T_STONITH_NG); + crm_xml_add(bcast, F_SUBTYPE, "broadcast"); + crm_xml_add(bcast, F_STONITH_OPERATION, STONITH_OP_FENCE_HISTORY); + crm_xml_add_int(bcast, F_STONITH_CALLOPTS, callopts); + if (history) { + add_node_copy(data, history); + } + add_message_xml(bcast, F_STONITH_CALLDATA, data); + send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE); + + free_xml(data); + free_xml(bcast); +} + +static gboolean +stonith_remove_history_entry (gpointer key, + gpointer value, + gpointer user_data) +{ + remote_fencing_op_t *op = value; + const char *target = (const char *) user_data; + + if ((op->state == st_failed) || (op->state == st_done)) { + if ((target) && (strcmp(op->target, target) != 0)) { + return FALSE; + } + return TRUE; + } + + return FALSE; /* don't clean pending operations */ +} + +/*! + * \internal + * \brief Send out a cleanup broadcast or do a local history-cleanup + * + * \param[in] target Cleanup can be limited to certain fence-targets + * \param[in] broadcast Send out a cleanup broadcast + */ +static void +stonith_fence_history_cleanup(const char *target, + gboolean broadcast) +{ + if (broadcast) { + stonith_send_broadcast_history(NULL, + st_opt_cleanup | st_opt_discard_reply, + target); + /* we'll do the local clean when we receive back our own broadcast */ + } else if (stonith_remote_op_list) { + g_hash_table_foreach_remove(stonith_remote_op_list, + stonith_remove_history_entry, + (gpointer) target); + fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL); + } +} + +/* keeping the length of fence-history within bounds + * ================================================= + * + * If things are really running wild a lot of fencing-attempts + * might fill up the hash-map, eventually using up a lot + * of memory and creating huge history-sync messages. + * Before the history being synced across nodes at least + * the reboot of a cluster-node helped keeping the + * history within bounds even though not in a reliable + * manner. + * + * stonith_remote_op_list isn't sorted for time-stamps + * thus it would be kind of expensive to delete e.g. + * the oldest entry if it would grow past MAX_STONITH_HISTORY + * entries. + * It is more efficient to purge MAX_STONITH_HISTORY/2 + * entries whenever the list grows beyond MAX_STONITH_HISTORY. + * (sort for age + purge the MAX_STONITH_HISTORY/2 oldest) + * That done on a per-node-base might raise the + * probability of large syncs to occur. + * Things like introducing a broadcast to purge + * MAX_STONITH_HISTORY/2 entries or not sync above a certain + * threshold coming to mind ... + * Simplest thing though is to purge the full history + * throughout the cluster once MAX_STONITH_HISTORY is reached. + * On the other hand this leads to purging the history in + * situations where it would be handy to have it probably. + */ + + +static int +op_time_sort(const void *a_voidp, const void *b_voidp) +{ + const remote_fencing_op_t **a = (const remote_fencing_op_t **) a_voidp; + const remote_fencing_op_t **b = (const remote_fencing_op_t **) b_voidp; + gboolean a_pending = ((*a)->state != st_failed) && ((*a)->state != st_done); + gboolean b_pending = ((*b)->state != st_failed) && ((*b)->state != st_done); + + if (a_pending && b_pending) { + return 0; + } else if (a_pending) { + return -1; + } else if (b_pending) { + return 1; + } else if ((*b)->completed == (*a)->completed) { + if ((*b)->completed_nsec > (*a)->completed_nsec) { + return 1; + } else if ((*b)->completed_nsec == (*a)->completed_nsec) { + return 0; + } + } else if ((*b)->completed > (*a)->completed) { + return 1; + } + + return -1; +} + + +/*! + * \internal + * \brief Do a local history-trim to MAX_STONITH_HISTORY / 2 entries + * once over MAX_STONITH_HISTORY + */ +void +stonith_fence_history_trim(void) +{ + guint num_ops; + + if (!stonith_remote_op_list) { + return; + } + num_ops = g_hash_table_size(stonith_remote_op_list); + if (num_ops > MAX_STONITH_HISTORY) { + remote_fencing_op_t *ops[num_ops]; + remote_fencing_op_t *op = NULL; + GHashTableIter iter; + int i; + + crm_trace("Fencing History growing beyond limit of %d so purge " + "half of failed/successful attempts", MAX_STONITH_HISTORY); + + /* write all ops into an array */ + i = 0; + g_hash_table_iter_init(&iter, stonith_remote_op_list); + while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) { + ops[i++] = op; + } + /* run quicksort over the array so that we get pending ops + * first and then sorted most recent to oldest + */ + qsort(ops, num_ops, sizeof(remote_fencing_op_t *), op_time_sort); + /* purgest oldest half of the history entries */ + for (i = MAX_STONITH_HISTORY / 2; i < num_ops; i++) { + /* keep pending ops even if they shouldn't fill more than + * half of our buffer + */ + if ((ops[i]->state == st_failed) || (ops[i]->state == st_done)) { + g_hash_table_remove(stonith_remote_op_list, ops[i]->id); + } + } + /* we've just purged valid data from the list so there is no need + * to create a notification - if displayed it can stay + */ + } +} + +/*! + * \internal + * \brief Convert xml fence-history to a hash-table like stonith_remote_op_list + * + * \param[in] history Fence-history in xml + * + * \return Fence-history as hash-table + */ +static GHashTable * +stonith_xml_history_to_list(const xmlNode *history) +{ + xmlNode *xml_op = NULL; + GHashTable *rv = NULL; + + init_stonith_remote_op_hash_table(&rv); + + CRM_LOG_ASSERT(rv != NULL); + + for (xml_op = pcmk__xml_first_child(history); xml_op != NULL; + xml_op = pcmk__xml_next(xml_op)) { + remote_fencing_op_t *op = NULL; + char *id = crm_element_value_copy(xml_op, F_STONITH_REMOTE_OP_ID); + int state; + int exit_status = CRM_EX_OK; + int execution_status = PCMK_EXEC_DONE; + long long completed; + long long completed_nsec = 0L; + + if (!id) { + crm_warn("Malformed fencing history received from peer"); + continue; + } + + crm_trace("Attaching op %s to hashtable", id); + + op = calloc(1, sizeof(remote_fencing_op_t)); + + op->id = id; + op->target = crm_element_value_copy(xml_op, F_STONITH_TARGET); + op->action = crm_element_value_copy(xml_op, F_STONITH_ACTION); + op->originator = crm_element_value_copy(xml_op, F_STONITH_ORIGIN); + op->delegate = crm_element_value_copy(xml_op, F_STONITH_DELEGATE); + op->client_name = crm_element_value_copy(xml_op, F_STONITH_CLIENTNAME); + crm_element_value_ll(xml_op, F_STONITH_DATE, &completed); + op->completed = (time_t) completed; + crm_element_value_ll(xml_op, F_STONITH_DATE_NSEC, &completed_nsec); + op->completed_nsec = completed_nsec; + crm_element_value_int(xml_op, F_STONITH_STATE, &state); + op->state = (enum op_state) state; + + /* @COMPAT We can't use stonith__xe_get_result() here because + * fencers <2.1.3 didn't include results, leading it to assume an error + * status. Instead, set an unknown status in that case. + */ + if ((crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &exit_status) < 0) + || (crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS, + &execution_status) < 0)) { + exit_status = CRM_EX_INDETERMINATE; + execution_status = PCMK_EXEC_UNKNOWN; + } + pcmk__set_result(&op->result, exit_status, execution_status, + crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON)); + pcmk__set_result_output(&op->result, + crm_element_value_copy(xml_op, F_STONITH_OUTPUT), + NULL); + + + g_hash_table_replace(rv, id, op); + CRM_LOG_ASSERT(g_hash_table_lookup(rv, id) != NULL); + } + + return rv; +} + +/*! + * \internal + * \brief Craft xml difference between local fence-history and a history + * coming from remote, and merge the remote history into the local + * + * \param[in,out] remote_history Fence-history as hash-table (may be NULL) + * \param[in] add_id If crafting the answer for an API + * history-request there is no need for the id + * \param[in] target Optionally limit to certain fence-target + * + * \return The fence-history as xml + */ +static xmlNode * +stonith_local_history_diff_and_merge(GHashTable *remote_history, + gboolean add_id, const char *target) +{ + xmlNode *history = NULL; + GHashTableIter iter; + remote_fencing_op_t *op = NULL; + gboolean updated = FALSE; + int cnt = 0; + + if (stonith_remote_op_list) { + char *id = NULL; + + history = create_xml_node(NULL, F_STONITH_HISTORY_LIST); + + g_hash_table_iter_init(&iter, stonith_remote_op_list); + while (g_hash_table_iter_next(&iter, (void **)&id, (void **)&op)) { + xmlNode *entry = NULL; + + if (remote_history) { + remote_fencing_op_t *remote_op = + g_hash_table_lookup(remote_history, op->id); + + if (remote_op) { + if (stonith__op_state_pending(op->state) + && !stonith__op_state_pending(remote_op->state)) { + + crm_debug("Updating outdated pending operation %.8s " + "(state=%s) according to the one (state=%s) from " + "remote peer history", + op->id, stonith_op_state_str(op->state), + stonith_op_state_str(remote_op->state)); + + g_hash_table_steal(remote_history, op->id); + op->id = remote_op->id; + remote_op->id = id; + g_hash_table_iter_replace(&iter, remote_op); + + updated = TRUE; + continue; /* skip outdated entries */ + + } else if (!stonith__op_state_pending(op->state) + && stonith__op_state_pending(remote_op->state)) { + + crm_debug("Broadcasting operation %.8s (state=%s) to " + "update the outdated pending one " + "(state=%s) in remote peer history", + op->id, stonith_op_state_str(op->state), + stonith_op_state_str(remote_op->state)); + + g_hash_table_remove(remote_history, op->id); + + } else { + g_hash_table_remove(remote_history, op->id); + continue; /* skip entries broadcasted already */ + } + } + } + + if (!pcmk__str_eq(target, op->target, pcmk__str_null_matches)) { + continue; + } + + cnt++; + crm_trace("Attaching op %s", op->id); + entry = create_xml_node(history, STONITH_OP_EXEC); + if (add_id) { + crm_xml_add(entry, F_STONITH_REMOTE_OP_ID, op->id); + } + crm_xml_add(entry, F_STONITH_TARGET, op->target); + crm_xml_add(entry, F_STONITH_ACTION, op->action); + crm_xml_add(entry, F_STONITH_ORIGIN, op->originator); + crm_xml_add(entry, F_STONITH_DELEGATE, op->delegate); + crm_xml_add(entry, F_STONITH_CLIENTNAME, op->client_name); + crm_xml_add_ll(entry, F_STONITH_DATE, op->completed); + crm_xml_add_ll(entry, F_STONITH_DATE_NSEC, op->completed_nsec); + crm_xml_add_int(entry, F_STONITH_STATE, op->state); + stonith__xe_set_result(entry, &op->result); + } + } + + if (remote_history) { + init_stonith_remote_op_hash_table(&stonith_remote_op_list); + + updated |= g_hash_table_size(remote_history); + + g_hash_table_iter_init(&iter, remote_history); + while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) { + if (stonith__op_state_pending(op->state) && + pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) { + + crm_warn("Failing pending operation %.8s originated by us but " + "known only from peer history", op->id); + op->state = st_failed; + set_fencing_completed(op); + + /* CRM_EX_EXPIRED + PCMK_EXEC_INVALID prevents finalize_op() + * from setting a delegate + */ + pcmk__set_result(&op->result, CRM_EX_EXPIRED, PCMK_EXEC_INVALID, + "Initiated by earlier fencer " + "process and presumed failed"); + fenced_broadcast_op_result(op, false); + } + + g_hash_table_iter_steal(&iter); + g_hash_table_replace(stonith_remote_op_list, op->id, op); + /* we could trim the history here but if we bail + * out after trim we might miss more recent entries + * of those that might still be in the list + * if we don't bail out trimming once is more + * efficient and memory overhead is minimal as + * we are just moving pointers from one hash to + * another + */ + } + + g_hash_table_destroy(remote_history); /* remove what is left */ + } + + if (updated) { + stonith_fence_history_trim(); + fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL); + } + + if (cnt == 0) { + free_xml(history); + return NULL; + } else { + return history; + } +} + +/*! + * \internal + * \brief Craft xml from the local fence-history + * + * \param[in] add_id If crafting the answer for an API + * history-request there is no need for the id + * \param[in] target Optionally limit to certain fence-target + * + * \return The fence-history as xml + */ +static xmlNode * +stonith_local_history(gboolean add_id, const char *target) +{ + return stonith_local_history_diff_and_merge(NULL, add_id, target); +} + +/*! + * \internal + * \brief Handle fence-history messages (from API or coming in as broadcasts) + * + * \param[in,out] msg Request XML + * \param[out] output Where to set local history, if requested + * \param[in] remote_peer If broadcast, peer that sent it + * \param[in] options Call options from the request + */ +void +stonith_fence_history(xmlNode *msg, xmlNode **output, + const char *remote_peer, int options) +{ + const char *target = NULL; + xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_NEVER); + xmlNode *out_history = NULL; + + if (dev) { + target = crm_element_value(dev, F_STONITH_TARGET); + if (target && (options & st_opt_cs_nodeid)) { + int nodeid; + crm_node_t *node; + + pcmk__scan_min_int(target, &nodeid, 0); + node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY); + if (node) { + target = node->uname; + } + } + } + + if (options & st_opt_cleanup) { + crm_trace("Cleaning up operations on %s in %p", target, + stonith_remote_op_list); + + stonith_fence_history_cleanup(target, + crm_element_value(msg, F_STONITH_CALLID) != NULL); + } else if (options & st_opt_broadcast) { + /* there is no clear sign atm for when a history sync + is done so send a notification for anything + that smells like history-sync + */ + fenced_send_notification(T_STONITH_NOTIFY_HISTORY_SYNCED, NULL, NULL); + if (crm_element_value(msg, F_STONITH_CALLID)) { + /* this is coming from the stonith-API + * + * craft a broadcast with node's history + * so that every node can merge and broadcast + * what it has on top + */ + out_history = stonith_local_history(TRUE, NULL); + crm_trace("Broadcasting history to peers"); + stonith_send_broadcast_history(out_history, + st_opt_broadcast | st_opt_discard_reply, + NULL); + } else if (remote_peer && + !pcmk__str_eq(remote_peer, stonith_our_uname, pcmk__str_casei)) { + xmlNode *history = get_xpath_object("//" F_STONITH_HISTORY_LIST, + msg, LOG_NEVER); + + /* either a broadcast created directly upon stonith-API request + * or a diff as response to such a thing + * + * in both cases it may have a history or not + * if we have differential data + * merge in what we've received and stop + * otherwise broadcast what we have on top + * marking as differential and merge in afterwards + */ + if (!history || !pcmk__xe_attr_is_true(history, F_STONITH_DIFFERENTIAL)) { + GHashTable *received_history = NULL; + + if (history != NULL) { + received_history = stonith_xml_history_to_list(history); + } + out_history = + stonith_local_history_diff_and_merge(received_history, TRUE, NULL); + if (out_history) { + crm_trace("Broadcasting history-diff to peers"); + pcmk__xe_set_bool_attr(out_history, F_STONITH_DIFFERENTIAL, true); + stonith_send_broadcast_history(out_history, + st_opt_broadcast | st_opt_discard_reply, + NULL); + } else { + crm_trace("History-diff is empty - skip broadcast"); + } + } + } else { + crm_trace("Skipping history-query-broadcast (%s%s)" + " we sent ourselves", + remote_peer?"remote-peer=":"local-ipc", + remote_peer?remote_peer:""); + } + } else { + /* plain history request */ + crm_trace("Looking for operations on %s in %p", target, + stonith_remote_op_list); + *output = stonith_local_history(FALSE, target); + } + free_xml(out_history); +} diff --git a/daemons/fenced/fenced_remote.c b/daemons/fenced/fenced_remote.c new file mode 100644 index 0000000..dc67947 --- /dev/null +++ b/daemons/fenced/fenced_remote.c @@ -0,0 +1,2509 @@ +/* + * Copyright 2009-2023 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. + */ + +#include <crm_internal.h> + +#include <sys/param.h> +#include <stdio.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <sys/stat.h> +#include <unistd.h> +#include <sys/utsname.h> + +#include <stdlib.h> +#include <errno.h> +#include <fcntl.h> +#include <ctype.h> +#include <regex.h> + +#include <crm/crm.h> +#include <crm/msg_xml.h> +#include <crm/common/ipc.h> +#include <crm/common/ipc_internal.h> +#include <crm/cluster/internal.h> + +#include <crm/stonith-ng.h> +#include <crm/fencing/internal.h> +#include <crm/common/xml.h> +#include <crm/common/xml_internal.h> + +#include <crm/common/util.h> +#include <pacemaker-fenced.h> + +#define TIMEOUT_MULTIPLY_FACTOR 1.2 + +/* When one fencer queries its peers for devices able to handle a fencing + * request, each peer will reply with a list of such devices available to it. + * Each reply will be parsed into a peer_device_info_t, with each device's + * information kept in a device_properties_t. + */ + +typedef struct device_properties_s { + /* Whether access to this device has been verified */ + gboolean verified; + + /* The remaining members are indexed by the operation's "phase" */ + + /* Whether this device has been executed in each phase */ + gboolean executed[st_phase_max]; + /* Whether this device is disallowed from executing in each phase */ + gboolean disallowed[st_phase_max]; + /* Action-specific timeout for each phase */ + int custom_action_timeout[st_phase_max]; + /* Action-specific maximum random delay for each phase */ + int delay_max[st_phase_max]; + /* Action-specific base delay for each phase */ + int delay_base[st_phase_max]; + /* Group of enum st_device_flags */ + uint32_t device_support_flags; +} device_properties_t; + +typedef struct { + /* Name of peer that sent this result */ + char *host; + /* Only try peers for non-topology based operations once */ + gboolean tried; + /* Number of entries in the devices table */ + int ndevices; + /* Devices available to this host that are capable of fencing the target */ + GHashTable *devices; +} peer_device_info_t; + +GHashTable *stonith_remote_op_list = NULL; + +extern xmlNode *stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data, + int call_options); + +static void request_peer_fencing(remote_fencing_op_t *op, + peer_device_info_t *peer); +static void finalize_op(remote_fencing_op_t *op, xmlNode *data, bool dup); +static void report_timeout_period(remote_fencing_op_t * op, int op_timeout); +static int get_op_total_timeout(const remote_fencing_op_t *op, + const peer_device_info_t *chosen_peer); + +static gint +sort_strings(gconstpointer a, gconstpointer b) +{ + return strcmp(a, b); +} + +static void +free_remote_query(gpointer data) +{ + if (data != NULL) { + peer_device_info_t *peer = data; + + g_hash_table_destroy(peer->devices); + free(peer->host); + free(peer); + } +} + +void +free_stonith_remote_op_list(void) +{ + if (stonith_remote_op_list != NULL) { + g_hash_table_destroy(stonith_remote_op_list); + stonith_remote_op_list = NULL; + } +} + +struct peer_count_data { + const remote_fencing_op_t *op; + gboolean verified_only; + uint32_t support_action_only; + int count; +}; + +/*! + * \internal + * \brief Increment a counter if a device has not been executed yet + * + * \param[in] key Device ID (ignored) + * \param[in] value Device properties + * \param[in,out] user_data Peer count data + */ +static void +count_peer_device(gpointer key, gpointer value, gpointer user_data) +{ + device_properties_t *props = (device_properties_t*)value; + struct peer_count_data *data = user_data; + + if (!props->executed[data->op->phase] + && (!data->verified_only || props->verified) + && ((data->support_action_only == st_device_supports_none) || pcmk_is_set(props->device_support_flags, data->support_action_only))) { + ++(data->count); + } +} + +/*! + * \internal + * \brief Check the number of available devices in a peer's query results + * + * \param[in] op Operation that results are for + * \param[in] peer Peer to count + * \param[in] verified_only Whether to count only verified devices + * \param[in] support_action_only Whether to count only devices that support action + * + * \return Number of devices available to peer that were not already executed + */ +static int +count_peer_devices(const remote_fencing_op_t *op, + const peer_device_info_t *peer, gboolean verified_only, uint32_t support_on_action_only) +{ + struct peer_count_data data; + + data.op = op; + data.verified_only = verified_only; + data.support_action_only = support_on_action_only; + data.count = 0; + if (peer) { + g_hash_table_foreach(peer->devices, count_peer_device, &data); + } + return data.count; +} + +/*! + * \internal + * \brief Search for a device in a query result + * + * \param[in] op Operation that result is for + * \param[in] peer Query result for a peer + * \param[in] device Device ID to search for + * + * \return Device properties if found, NULL otherwise + */ +static device_properties_t * +find_peer_device(const remote_fencing_op_t *op, const peer_device_info_t *peer, + const char *device, uint32_t support_action_only) +{ + device_properties_t *props = g_hash_table_lookup(peer->devices, device); + + if (props && support_action_only != st_device_supports_none && !pcmk_is_set(props->device_support_flags, support_action_only)) { + return NULL; + } + return (props && !props->executed[op->phase] + && !props->disallowed[op->phase])? props : NULL; +} + +/*! + * \internal + * \brief Find a device in a peer's device list and mark it as executed + * + * \param[in] op Operation that peer result is for + * \param[in,out] peer Peer with results to search + * \param[in] device ID of device to mark as done + * \param[in] verified_devices_only Only consider verified devices + * + * \return TRUE if device was found and marked, FALSE otherwise + */ +static gboolean +grab_peer_device(const remote_fencing_op_t *op, peer_device_info_t *peer, + const char *device, gboolean verified_devices_only) +{ + device_properties_t *props = find_peer_device(op, peer, device, + fenced_support_flag(op->action)); + + if ((props == NULL) || (verified_devices_only && !props->verified)) { + return FALSE; + } + + crm_trace("Removing %s from %s (%d remaining)", + device, peer->host, count_peer_devices(op, peer, FALSE, st_device_supports_none)); + props->executed[op->phase] = TRUE; + return TRUE; +} + +static void +clear_remote_op_timers(remote_fencing_op_t * op) +{ + if (op->query_timer) { + g_source_remove(op->query_timer); + op->query_timer = 0; + } + if (op->op_timer_total) { + g_source_remove(op->op_timer_total); + op->op_timer_total = 0; + } + if (op->op_timer_one) { + g_source_remove(op->op_timer_one); + op->op_timer_one = 0; + } +} + +static void +free_remote_op(gpointer data) +{ + remote_fencing_op_t *op = data; + + crm_log_xml_debug(op->request, "Destroying"); + + clear_remote_op_timers(op); + + free(op->id); + free(op->action); + free(op->delegate); + free(op->target); + free(op->client_id); + free(op->client_name); + free(op->originator); + + if (op->query_results) { + g_list_free_full(op->query_results, free_remote_query); + } + if (op->request) { + free_xml(op->request); + op->request = NULL; + } + if (op->devices_list) { + g_list_free_full(op->devices_list, free); + op->devices_list = NULL; + } + g_list_free_full(op->automatic_list, free); + g_list_free(op->duplicates); + + pcmk__reset_result(&op->result); + free(op); +} + +void +init_stonith_remote_op_hash_table(GHashTable **table) +{ + if (*table == NULL) { + *table = pcmk__strkey_table(NULL, free_remote_op); + } +} + +/*! + * \internal + * \brief Return an operation's originally requested action (before any remap) + * + * \param[in] op Operation to check + * + * \return Operation's original action + */ +static const char * +op_requested_action(const remote_fencing_op_t *op) +{ + return ((op->phase > st_phase_requested)? "reboot" : op->action); +} + +/*! + * \internal + * \brief Remap a "reboot" operation to the "off" phase + * + * \param[in,out] op Operation to remap + */ +static void +op_phase_off(remote_fencing_op_t *op) +{ + crm_info("Remapping multiple-device reboot targeting %s to 'off' " + CRM_XS " id=%.8s", op->target, op->id); + op->phase = st_phase_off; + + /* Happily, "off" and "on" are shorter than "reboot", so we can reuse the + * memory allocation at each phase. + */ + strcpy(op->action, "off"); +} + +/*! + * \internal + * \brief Advance a remapped reboot operation to the "on" phase + * + * \param[in,out] op Operation to remap + */ +static void +op_phase_on(remote_fencing_op_t *op) +{ + GList *iter = NULL; + + crm_info("Remapped 'off' targeting %s complete, " + "remapping to 'on' for %s " CRM_XS " id=%.8s", + op->target, op->client_name, op->id); + op->phase = st_phase_on; + strcpy(op->action, "on"); + + /* Skip devices with automatic unfencing, because the cluster will handle it + * when the node rejoins. + */ + for (iter = op->automatic_list; iter != NULL; iter = iter->next) { + GList *match = g_list_find_custom(op->devices_list, iter->data, + sort_strings); + + if (match) { + op->devices_list = g_list_remove(op->devices_list, match->data); + } + } + g_list_free_full(op->automatic_list, free); + op->automatic_list = NULL; + + /* Rewind device list pointer */ + op->devices = op->devices_list; +} + +/*! + * \internal + * \brief Reset a remapped reboot operation + * + * \param[in,out] op Operation to reset + */ +static void +undo_op_remap(remote_fencing_op_t *op) +{ + if (op->phase > 0) { + crm_info("Undoing remap of reboot targeting %s for %s " + CRM_XS " id=%.8s", op->target, op->client_name, op->id); + op->phase = st_phase_requested; + strcpy(op->action, "reboot"); + } +} + +/*! + * \internal + * \brief Create notification data XML for a fencing operation result + * + * \param[in] op Fencer operation that completed + * + * \return Newly created XML to add as notification data + * \note The caller is responsible for freeing the result. + */ +static xmlNode * +fencing_result2xml(const remote_fencing_op_t *op) +{ + xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE); + + crm_xml_add_int(notify_data, "state", op->state); + crm_xml_add(notify_data, F_STONITH_TARGET, op->target); + crm_xml_add(notify_data, F_STONITH_ACTION, op->action); + crm_xml_add(notify_data, F_STONITH_DELEGATE, op->delegate); + crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, op->id); + crm_xml_add(notify_data, F_STONITH_ORIGIN, op->originator); + crm_xml_add(notify_data, F_STONITH_CLIENTID, op->client_id); + crm_xml_add(notify_data, F_STONITH_CLIENTNAME, op->client_name); + + return notify_data; +} + +/*! + * \internal + * \brief Broadcast a fence result notification to all CPG peers + * + * \param[in] op Fencer operation that completed + * \param[in] op_merged Whether this operation is a duplicate of another + */ +void +fenced_broadcast_op_result(const remote_fencing_op_t *op, bool op_merged) +{ + static int count = 0; + xmlNode *bcast = create_xml_node(NULL, T_STONITH_REPLY); + xmlNode *notify_data = fencing_result2xml(op); + + count++; + crm_trace("Broadcasting result to peers"); + crm_xml_add(bcast, F_TYPE, T_STONITH_NOTIFY); + crm_xml_add(bcast, F_SUBTYPE, "broadcast"); + crm_xml_add(bcast, F_STONITH_OPERATION, T_STONITH_NOTIFY); + crm_xml_add_int(bcast, "count", count); + + if (op_merged) { + pcmk__xe_set_bool_attr(bcast, F_STONITH_MERGED, true); + } + + stonith__xe_set_result(notify_data, &op->result); + + add_message_xml(bcast, F_STONITH_CALLDATA, notify_data); + send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE); + free_xml(notify_data); + free_xml(bcast); + + return; +} + +/*! + * \internal + * \brief Reply to a local request originator and notify all subscribed clients + * + * \param[in,out] op Fencer operation that completed + * \param[in,out] data Top-level XML to add notification to + */ +static void +handle_local_reply_and_notify(remote_fencing_op_t *op, xmlNode *data) +{ + xmlNode *notify_data = NULL; + xmlNode *reply = NULL; + pcmk__client_t *client = NULL; + + if (op->notify_sent == TRUE) { + /* nothing to do */ + return; + } + + /* Do notification with a clean data object */ + crm_xml_add_int(data, "state", op->state); + crm_xml_add(data, F_STONITH_TARGET, op->target); + crm_xml_add(data, F_STONITH_OPERATION, op->action); + + reply = fenced_construct_reply(op->request, data, &op->result); + crm_xml_add(reply, F_STONITH_DELEGATE, op->delegate); + + /* Send fencing OP reply to local client that initiated fencing */ + client = pcmk__find_client_by_id(op->client_id); + if (client == NULL) { + crm_trace("Skipping reply to %s: no longer a client", op->client_id); + } else { + do_local_reply(reply, client, op->call_options); + } + + /* bcast to all local clients that the fencing operation happend */ + notify_data = fencing_result2xml(op); + fenced_send_notification(T_STONITH_NOTIFY_FENCE, &op->result, notify_data); + free_xml(notify_data); + fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL); + + /* mark this op as having notify's already sent */ + op->notify_sent = TRUE; + free_xml(reply); +} + +/*! + * \internal + * \brief Finalize all duplicates of a given fencer operation + * + * \param[in,out] op Fencer operation that completed + * \param[in,out] data Top-level XML to add notification to + */ +static void +finalize_op_duplicates(remote_fencing_op_t *op, xmlNode *data) +{ + for (GList *iter = op->duplicates; iter != NULL; iter = iter->next) { + remote_fencing_op_t *other = iter->data; + + if (other->state == st_duplicate) { + other->state = op->state; + crm_debug("Performing duplicate notification for %s@%s: %s " + CRM_XS " id=%.8s", + other->client_name, other->originator, + pcmk_exec_status_str(op->result.execution_status), + other->id); + pcmk__copy_result(&op->result, &other->result); + finalize_op(other, data, true); + + } else { + // Possible if (for example) it timed out already + crm_err("Skipping duplicate notification for %s@%s " + CRM_XS " state=%s id=%.8s", + other->client_name, other->originator, + stonith_op_state_str(other->state), other->id); + } + } +} + +static char * +delegate_from_xml(xmlNode *xml) +{ + xmlNode *match = get_xpath_object("//@" F_STONITH_DELEGATE, xml, LOG_NEVER); + + if (match == NULL) { + return crm_element_value_copy(xml, F_ORIG); + } else { + return crm_element_value_copy(match, F_STONITH_DELEGATE); + } +} + +/*! + * \internal + * \brief Finalize a peer fencing operation + * + * Clean up after a fencing operation completes. This function has two code + * paths: the executioner uses it to broadcast the result to CPG peers, and then + * each peer (including the executioner) uses it to process that broadcast and + * notify its IPC clients of the result. + * + * \param[in,out] op Fencer operation that completed + * \param[in,out] data If not NULL, XML reply of last delegated operation + * \param[in] dup Whether this operation is a duplicate of another + * (in which case, do not broadcast the result) + * + * \note The operation result should be set before calling this function. + */ +static void +finalize_op(remote_fencing_op_t *op, xmlNode *data, bool dup) +{ + int level = LOG_ERR; + const char *subt = NULL; + xmlNode *local_data = NULL; + gboolean op_merged = FALSE; + + CRM_CHECK((op != NULL), return); + + // This is a no-op if timers have already been cleared + clear_remote_op_timers(op); + + if (op->notify_sent) { + // Most likely, this is a timed-out action that eventually completed + crm_notice("Operation '%s'%s%s by %s for %s@%s%s: " + "Result arrived too late " CRM_XS " id=%.8s", + op->action, (op->target? " targeting " : ""), + (op->target? op->target : ""), + (op->delegate? op->delegate : "unknown node"), + op->client_name, op->originator, + (op_merged? " (merged)" : ""), + op->id); + return; + } + + set_fencing_completed(op); + undo_op_remap(op); + + if (data == NULL) { + data = create_xml_node(NULL, "remote-op"); + local_data = data; + + } else if (op->delegate == NULL) { + switch (op->result.execution_status) { + case PCMK_EXEC_NO_FENCE_DEVICE: + break; + + case PCMK_EXEC_INVALID: + if (op->result.exit_status != CRM_EX_EXPIRED) { + op->delegate = delegate_from_xml(data); + } + break; + + default: + op->delegate = delegate_from_xml(data); + break; + } + } + + if (dup || (crm_element_value(data, F_STONITH_MERGED) != NULL)) { + op_merged = true; + } + + /* Tell everyone the operation is done, we will continue + * with doing the local notifications once we receive + * the broadcast back. */ + subt = crm_element_value(data, F_SUBTYPE); + if (!dup && !pcmk__str_eq(subt, "broadcast", pcmk__str_casei)) { + /* Defer notification until the bcast message arrives */ + fenced_broadcast_op_result(op, op_merged); + free_xml(local_data); + return; + } + + if (pcmk__result_ok(&op->result) || dup + || !pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) { + level = LOG_NOTICE; + } + do_crm_log(level, "Operation '%s'%s%s by %s for %s@%s%s: %s (%s%s%s) " + CRM_XS " id=%.8s", op->action, (op->target? " targeting " : ""), + (op->target? op->target : ""), + (op->delegate? op->delegate : "unknown node"), + op->client_name, op->originator, + (op_merged? " (merged)" : ""), + crm_exit_str(op->result.exit_status), + pcmk_exec_status_str(op->result.execution_status), + ((op->result.exit_reason == NULL)? "" : ": "), + ((op->result.exit_reason == NULL)? "" : op->result.exit_reason), + op->id); + + handle_local_reply_and_notify(op, data); + + if (!dup) { + finalize_op_duplicates(op, data); + } + + /* Free non-essential parts of the record + * Keep the record around so we can query the history + */ + if (op->query_results) { + g_list_free_full(op->query_results, free_remote_query); + op->query_results = NULL; + } + if (op->request) { + free_xml(op->request); + op->request = NULL; + } + + free_xml(local_data); +} + +/*! + * \internal + * \brief Finalize a watchdog fencer op after the waiting time expires + * + * \param[in,out] userdata Fencer operation that completed + * + * \return G_SOURCE_REMOVE (which tells glib not to restart timer) + */ +static gboolean +remote_op_watchdog_done(gpointer userdata) +{ + remote_fencing_op_t *op = userdata; + + op->op_timer_one = 0; + + crm_notice("Self-fencing (%s) by %s for %s assumed complete " + CRM_XS " id=%.8s", + op->action, op->target, op->client_name, op->id); + op->state = st_done; + pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + finalize_op(op, NULL, false); + return G_SOURCE_REMOVE; +} + +static gboolean +remote_op_timeout_one(gpointer userdata) +{ + remote_fencing_op_t *op = userdata; + + op->op_timer_one = 0; + + crm_notice("Peer's '%s' action targeting %s for client %s timed out " CRM_XS + " id=%.8s", op->action, op->target, op->client_name, op->id); + pcmk__set_result(&op->result, CRM_EX_ERROR, PCMK_EXEC_TIMEOUT, + "Peer did not return fence result within timeout"); + + // The requested delay has been applied for the first device + if (op->delay > 0) { + op->delay = 0; + crm_trace("Try another device for '%s' action targeting %s " + "for client %s without delay " CRM_XS " id=%.8s", + op->action, op->target, op->client_name, op->id); + } + + // Try another device, if appropriate + request_peer_fencing(op, NULL); + return G_SOURCE_REMOVE; +} + +/*! + * \internal + * \brief Finalize a remote fencer operation that timed out + * + * \param[in,out] op Fencer operation that timed out + * \param[in] reason Readable description of what step timed out + */ +static void +finalize_timed_out_op(remote_fencing_op_t *op, const char *reason) +{ + crm_debug("Action '%s' targeting %s for client %s timed out " + CRM_XS " id=%.8s", + op->action, op->target, op->client_name, op->id); + + if (op->phase == st_phase_on) { + /* A remapped reboot operation timed out in the "on" phase, but the + * "off" phase completed successfully, so quit trying any further + * devices, and return success. + */ + op->state = st_done; + pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + } else { + op->state = st_failed; + pcmk__set_result(&op->result, CRM_EX_ERROR, PCMK_EXEC_TIMEOUT, reason); + } + finalize_op(op, NULL, false); +} + +/*! + * \internal + * \brief Finalize a remote fencer operation that timed out + * + * \param[in,out] userdata Fencer operation that timed out + * + * \return G_SOURCE_REMOVE (which tells glib not to restart timer) + */ +static gboolean +remote_op_timeout(gpointer userdata) +{ + remote_fencing_op_t *op = userdata; + + op->op_timer_total = 0; + + if (op->state == st_done) { + crm_debug("Action '%s' targeting %s for client %s already completed " + CRM_XS " id=%.8s", + op->action, op->target, op->client_name, op->id); + } else { + finalize_timed_out_op(userdata, "Fencing did not complete within a " + "total timeout based on the " + "configured timeout and retries for " + "any devices attempted"); + } + return G_SOURCE_REMOVE; +} + +static gboolean +remote_op_query_timeout(gpointer data) +{ + remote_fencing_op_t *op = data; + + op->query_timer = 0; + + if (op->state == st_done) { + crm_debug("Operation %.8s targeting %s already completed", + op->id, op->target); + } else if (op->state == st_exec) { + crm_debug("Operation %.8s targeting %s already in progress", + op->id, op->target); + } else if (op->query_results) { + // Query succeeded, so attempt the actual fencing + crm_debug("Query %.8s targeting %s complete (state=%s)", + op->id, op->target, stonith_op_state_str(op->state)); + request_peer_fencing(op, NULL); + } else { + crm_debug("Query %.8s targeting %s timed out (state=%s)", + op->id, op->target, stonith_op_state_str(op->state)); + finalize_timed_out_op(op, "No capable peers replied to device query " + "within timeout"); + } + + return G_SOURCE_REMOVE; +} + +static gboolean +topology_is_empty(stonith_topology_t *tp) +{ + int i; + + if (tp == NULL) { + return TRUE; + } + + for (i = 0; i < ST_LEVEL_MAX; i++) { + if (tp->levels[i] != NULL) { + return FALSE; + } + } + return TRUE; +} + +/*! + * \internal + * \brief Add a device to an operation's automatic unfencing list + * + * \param[in,out] op Operation to modify + * \param[in] device Device ID to add + */ +static void +add_required_device(remote_fencing_op_t *op, const char *device) +{ + GList *match = g_list_find_custom(op->automatic_list, device, + sort_strings); + + if (!match) { + op->automatic_list = g_list_prepend(op->automatic_list, strdup(device)); + } +} + +/*! + * \internal + * \brief Remove a device from the automatic unfencing list + * + * \param[in,out] op Operation to modify + * \param[in] device Device ID to remove + */ +static void +remove_required_device(remote_fencing_op_t *op, const char *device) +{ + GList *match = g_list_find_custom(op->automatic_list, device, + sort_strings); + + if (match) { + op->automatic_list = g_list_remove(op->automatic_list, match->data); + } +} + +/* deep copy the device list */ +static void +set_op_device_list(remote_fencing_op_t * op, GList *devices) +{ + GList *lpc = NULL; + + if (op->devices_list) { + g_list_free_full(op->devices_list, free); + op->devices_list = NULL; + } + for (lpc = devices; lpc != NULL; lpc = lpc->next) { + op->devices_list = g_list_append(op->devices_list, strdup(lpc->data)); + } + op->devices = op->devices_list; +} + +/*! + * \internal + * \brief Check whether a node matches a topology target + * + * \param[in] tp Topology table entry to check + * \param[in] node Name of node to check + * + * \return TRUE if node matches topology target + */ +static gboolean +topology_matches(const stonith_topology_t *tp, const char *node) +{ + regex_t r_patt; + + CRM_CHECK(node && tp && tp->target, return FALSE); + switch (tp->kind) { + case fenced_target_by_attribute: + /* This level targets by attribute, so tp->target is a NAME=VALUE pair + * of a permanent attribute applied to targeted nodes. The test below + * relies on the locally cached copy of the CIB, so if fencing needs to + * be done before the initial CIB is received or after a malformed CIB + * is received, then the topology will be unable to be used. + */ + if (node_has_attr(node, tp->target_attribute, tp->target_value)) { + crm_notice("Matched %s with %s by attribute", node, tp->target); + return TRUE; + } + break; + + case fenced_target_by_pattern: + /* This level targets node names matching a pattern, so tp->target + * (and tp->target_pattern) is a regular expression. + */ + if (regcomp(&r_patt, tp->target_pattern, REG_EXTENDED|REG_NOSUB)) { + crm_info("Bad regex '%s' for fencing level", tp->target); + } else { + int status = regexec(&r_patt, node, 0, NULL, 0); + + regfree(&r_patt); + if (status == 0) { + crm_notice("Matched %s with %s by name", node, tp->target); + return TRUE; + } + } + break; + + case fenced_target_by_name: + crm_trace("Testing %s against %s", node, tp->target); + return pcmk__str_eq(tp->target, node, pcmk__str_casei); + + default: + break; + } + crm_trace("No match for %s with %s", node, tp->target); + return FALSE; +} + +stonith_topology_t * +find_topology_for_host(const char *host) +{ + GHashTableIter tIter; + stonith_topology_t *tp = g_hash_table_lookup(topology, host); + + if(tp != NULL) { + crm_trace("Found %s for %s in %d entries", tp->target, host, g_hash_table_size(topology)); + return tp; + } + + g_hash_table_iter_init(&tIter, topology); + while (g_hash_table_iter_next(&tIter, NULL, (gpointer *) & tp)) { + if (topology_matches(tp, host)) { + crm_trace("Found %s for %s in %d entries", tp->target, host, g_hash_table_size(topology)); + return tp; + } + } + + crm_trace("No matches for %s in %d topology entries", host, g_hash_table_size(topology)); + return NULL; +} + +/*! + * \internal + * \brief Set fencing operation's device list to target's next topology level + * + * \param[in,out] op Remote fencing operation to modify + * \param[in] empty_ok If true, an operation without a target (i.e. + * queries) or a target without a topology will get a + * pcmk_rc_ok return value instead of ENODEV + * + * \return Standard Pacemaker return value + */ +static int +advance_topology_level(remote_fencing_op_t *op, bool empty_ok) +{ + stonith_topology_t *tp = NULL; + + if (op->target) { + tp = find_topology_for_host(op->target); + } + if (topology_is_empty(tp)) { + return empty_ok? pcmk_rc_ok : ENODEV; + } + + CRM_ASSERT(tp->levels != NULL); + + stonith__set_call_options(op->call_options, op->id, st_opt_topology); + + /* This is a new level, so undo any remapping left over from previous */ + undo_op_remap(op); + + do { + op->level++; + + } while (op->level < ST_LEVEL_MAX && tp->levels[op->level] == NULL); + + if (op->level < ST_LEVEL_MAX) { + crm_trace("Attempting fencing level %d targeting %s (%d devices) " + "for client %s@%s (id=%.8s)", + op->level, op->target, g_list_length(tp->levels[op->level]), + op->client_name, op->originator, op->id); + set_op_device_list(op, tp->levels[op->level]); + + // The requested delay has been applied for the first fencing level + if (op->level > 1 && op->delay > 0) { + op->delay = 0; + } + + if ((g_list_next(op->devices_list) != NULL) + && pcmk__str_eq(op->action, "reboot", pcmk__str_none)) { + /* A reboot has been requested for a topology level with multiple + * devices. Instead of rebooting the devices sequentially, we will + * turn them all off, then turn them all on again. (Think about + * switched power outlets for redundant power supplies.) + */ + op_phase_off(op); + } + return pcmk_rc_ok; + } + + crm_info("All %sfencing options targeting %s for client %s@%s failed " + CRM_XS " id=%.8s", + (stonith_watchdog_timeout_ms > 0)?"non-watchdog ":"", + op->target, op->client_name, op->originator, op->id); + return ENODEV; +} + +/*! + * \internal + * \brief If fencing operation is a duplicate, merge it into the other one + * + * \param[in,out] op Fencing operation to check + */ +static void +merge_duplicates(remote_fencing_op_t *op) +{ + GHashTableIter iter; + remote_fencing_op_t *other = NULL; + + time_t now = time(NULL); + + g_hash_table_iter_init(&iter, stonith_remote_op_list); + while (g_hash_table_iter_next(&iter, NULL, (void **)&other)) { + const char *other_action = op_requested_action(other); + + if (!strcmp(op->id, other->id)) { + continue; // Don't compare against self + } + if (other->state > st_exec) { + crm_trace("%.8s not duplicate of %.8s: not in progress", + op->id, other->id); + continue; + } + if (!pcmk__str_eq(op->target, other->target, pcmk__str_casei)) { + crm_trace("%.8s not duplicate of %.8s: node %s vs. %s", + op->id, other->id, op->target, other->target); + continue; + } + if (!pcmk__str_eq(op->action, other_action, pcmk__str_none)) { + crm_trace("%.8s not duplicate of %.8s: action %s vs. %s", + op->id, other->id, op->action, other_action); + continue; + } + if (pcmk__str_eq(op->client_name, other->client_name, pcmk__str_casei)) { + crm_trace("%.8s not duplicate of %.8s: same client %s", + op->id, other->id, op->client_name); + continue; + } + if (pcmk__str_eq(other->target, other->originator, pcmk__str_casei)) { + crm_trace("%.8s not duplicate of %.8s: suicide for %s", + op->id, other->id, other->target); + continue; + } + if (!fencing_peer_active(crm_get_peer(0, other->originator))) { + crm_notice("Failing action '%s' targeting %s originating from " + "client %s@%s: Originator is dead " CRM_XS " id=%.8s", + other->action, other->target, other->client_name, + other->originator, other->id); + crm_trace("%.8s not duplicate of %.8s: originator dead", + op->id, other->id); + other->state = st_failed; + continue; + } + if ((other->total_timeout > 0) + && (now > (other->total_timeout + other->created))) { + crm_trace("%.8s not duplicate of %.8s: old (%ld vs. %ld + %d)", + op->id, other->id, now, other->created, + other->total_timeout); + continue; + } + + /* There is another in-flight request to fence the same host + * Piggyback on that instead. If it fails, so do we. + */ + other->duplicates = g_list_append(other->duplicates, op); + if (other->total_timeout == 0) { + other->total_timeout = op->total_timeout = + TIMEOUT_MULTIPLY_FACTOR * get_op_total_timeout(op, NULL); + crm_trace("Best guess as to timeout used for %.8s: %d", + other->id, other->total_timeout); + } + crm_notice("Merging fencing action '%s' targeting %s originating from " + "client %s with identical request from %s@%s " + CRM_XS " original=%.8s duplicate=%.8s total_timeout=%ds", + op->action, op->target, op->client_name, + other->client_name, other->originator, + op->id, other->id, other->total_timeout); + report_timeout_period(op, other->total_timeout); + op->state = st_duplicate; + } +} + +static uint32_t fencing_active_peers(void) +{ + uint32_t count = 0; + crm_node_t *entry; + GHashTableIter gIter; + + g_hash_table_iter_init(&gIter, crm_peer_cache); + while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) { + if(fencing_peer_active(entry)) { + count++; + } + } + return count; +} + +/*! + * \internal + * \brief Process a manual confirmation of a pending fence action + * + * \param[in] client IPC client that sent confirmation + * \param[in,out] msg Request XML with manual confirmation + * + * \return Standard Pacemaker return code + */ +int +fenced_handle_manual_confirmation(const pcmk__client_t *client, xmlNode *msg) +{ + remote_fencing_op_t *op = NULL; + xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR); + + CRM_CHECK(dev != NULL, return EPROTO); + + crm_notice("Received manual confirmation that %s has been fenced", + pcmk__s(crm_element_value(dev, F_STONITH_TARGET), + "unknown target")); + op = initiate_remote_stonith_op(client, msg, TRUE); + if (op == NULL) { + return EPROTO; + } + op->state = st_done; + set_fencing_completed(op); + op->delegate = strdup("a human"); + + // For the fencer's purposes, the fencing operation is done + pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + finalize_op(op, msg, false); + + /* For the requester's purposes, the operation is still pending. The + * actual result will be sent asynchronously via the operation's done_cb(). + */ + return EINPROGRESS; +} + +/*! + * \internal + * \brief Create a new remote stonith operation + * + * \param[in] client ID of local stonith client that initiated the operation + * \param[in] request The request from the client that started the operation + * \param[in] peer TRUE if this operation is owned by another stonith peer + * (an operation owned by one peer is stored on all peers, + * but only the owner executes it; all nodes get the results + * once the owner finishes execution) + */ +void * +create_remote_stonith_op(const char *client, xmlNode *request, gboolean peer) +{ + remote_fencing_op_t *op = NULL; + xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_NEVER); + int call_options = 0; + const char *operation = NULL; + + init_stonith_remote_op_hash_table(&stonith_remote_op_list); + + /* If this operation is owned by another node, check to make + * sure we haven't already created this operation. */ + if (peer && dev) { + const char *op_id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID); + + CRM_CHECK(op_id != NULL, return NULL); + + op = g_hash_table_lookup(stonith_remote_op_list, op_id); + if (op) { + crm_debug("Reusing existing remote fencing op %.8s for %s", + op_id, ((client == NULL)? "unknown client" : client)); + return op; + } + } + + op = calloc(1, sizeof(remote_fencing_op_t)); + CRM_ASSERT(op != NULL); + + crm_element_value_int(request, F_STONITH_TIMEOUT, &(op->base_timeout)); + // Value -1 means disable any static/random fencing delays + crm_element_value_int(request, F_STONITH_DELAY, &(op->delay)); + + if (peer && dev) { + op->id = crm_element_value_copy(dev, F_STONITH_REMOTE_OP_ID); + } else { + op->id = crm_generate_uuid(); + } + + g_hash_table_replace(stonith_remote_op_list, op->id, op); + + op->state = st_query; + op->replies_expected = fencing_active_peers(); + op->action = crm_element_value_copy(dev, F_STONITH_ACTION); + op->originator = crm_element_value_copy(dev, F_STONITH_ORIGIN); + op->delegate = crm_element_value_copy(dev, F_STONITH_DELEGATE); /* May not be set */ + op->created = time(NULL); + + if (op->originator == NULL) { + /* Local or relayed request */ + op->originator = strdup(stonith_our_uname); + } + + CRM_LOG_ASSERT(client != NULL); + if (client) { + op->client_id = strdup(client); + } + + + /* For a RELAY operation, set fenced on the client. */ + operation = crm_element_value(request, F_STONITH_OPERATION); + + if (pcmk__str_eq(operation, STONITH_OP_RELAY, pcmk__str_none)) { + op->client_name = crm_strdup_printf("%s.%lu", crm_system_name, + (unsigned long) getpid()); + } else { + op->client_name = crm_element_value_copy(request, F_STONITH_CLIENTNAME); + } + + op->target = crm_element_value_copy(dev, F_STONITH_TARGET); + op->request = copy_xml(request); /* TODO: Figure out how to avoid this */ + crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options); + op->call_options = call_options; + + crm_element_value_int(request, F_STONITH_CALLID, &(op->client_callid)); + + crm_trace("%s new fencing op %s ('%s' targeting %s for client %s, " + "base timeout %d, %u %s expected)", + (peer && dev)? "Recorded" : "Generated", op->id, op->action, + op->target, op->client_name, op->base_timeout, + op->replies_expected, + pcmk__plural_alt(op->replies_expected, "reply", "replies")); + + if (op->call_options & st_opt_cs_nodeid) { + int nodeid; + crm_node_t *node; + + pcmk__scan_min_int(op->target, &nodeid, 0); + node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY); + + /* Ensure the conversion only happens once */ + stonith__clear_call_options(op->call_options, op->id, st_opt_cs_nodeid); + + if (node && node->uname) { + free(op->target); + op->target = strdup(node->uname); + + } else { + crm_warn("Could not expand nodeid '%s' into a host name", op->target); + } + } + + /* check to see if this is a duplicate operation of another in-flight operation */ + merge_duplicates(op); + + if (op->state != st_duplicate) { + /* kick history readers */ + fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL); + } + + /* safe to trim as long as that doesn't touch pending ops */ + stonith_fence_history_trim(); + + return op; +} + +/*! + * \internal + * \brief Create a peer fencing operation from a request, and initiate it + * + * \param[in] client IPC client that made request (NULL to get from request) + * \param[in] request Request XML + * \param[in] manual_ack Whether this is a manual action confirmation + * + * \return Newly created operation on success, otherwise NULL + */ +remote_fencing_op_t * +initiate_remote_stonith_op(const pcmk__client_t *client, xmlNode *request, + gboolean manual_ack) +{ + int query_timeout = 0; + xmlNode *query = NULL; + const char *client_id = NULL; + remote_fencing_op_t *op = NULL; + const char *relay_op_id = NULL; + const char *operation = NULL; + + if (client) { + client_id = client->id; + } else { + client_id = crm_element_value(request, F_STONITH_CLIENTID); + } + + CRM_LOG_ASSERT(client_id != NULL); + op = create_remote_stonith_op(client_id, request, FALSE); + op->owner = TRUE; + if (manual_ack) { + return op; + } + + CRM_CHECK(op->action, return NULL); + + if (advance_topology_level(op, true) != pcmk_rc_ok) { + op->state = st_failed; + } + + switch (op->state) { + case st_failed: + // advance_topology_level() exhausted levels + pcmk__set_result(&op->result, CRM_EX_ERROR, PCMK_EXEC_ERROR, + "All topology levels failed"); + crm_warn("Could not request peer fencing (%s) targeting %s " + CRM_XS " id=%.8s", op->action, op->target, op->id); + finalize_op(op, NULL, false); + return op; + + case st_duplicate: + crm_info("Requesting peer fencing (%s) targeting %s (duplicate) " + CRM_XS " id=%.8s", op->action, op->target, op->id); + return op; + + default: + crm_notice("Requesting peer fencing (%s) targeting %s " + CRM_XS " id=%.8s state=%s base_timeout=%d", + op->action, op->target, op->id, + stonith_op_state_str(op->state), op->base_timeout); + } + + query = stonith_create_op(op->client_callid, op->id, STONITH_OP_QUERY, + NULL, op->call_options); + + crm_xml_add(query, F_STONITH_REMOTE_OP_ID, op->id); + crm_xml_add(query, F_STONITH_TARGET, op->target); + crm_xml_add(query, F_STONITH_ACTION, op_requested_action(op)); + crm_xml_add(query, F_STONITH_ORIGIN, op->originator); + crm_xml_add(query, F_STONITH_CLIENTID, op->client_id); + crm_xml_add(query, F_STONITH_CLIENTNAME, op->client_name); + crm_xml_add_int(query, F_STONITH_TIMEOUT, op->base_timeout); + + /* In case of RELAY operation, RELAY information is added to the query to delete the original operation of RELAY. */ + operation = crm_element_value(request, F_STONITH_OPERATION); + if (pcmk__str_eq(operation, STONITH_OP_RELAY, pcmk__str_none)) { + relay_op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID); + if (relay_op_id) { + crm_xml_add(query, F_STONITH_REMOTE_OP_ID_RELAY, relay_op_id); + } + } + + send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE); + free_xml(query); + + query_timeout = op->base_timeout * TIMEOUT_MULTIPLY_FACTOR; + op->query_timer = g_timeout_add((1000 * query_timeout), remote_op_query_timeout, op); + + return op; +} + +enum find_best_peer_options { + /*! Skip checking the target peer for capable fencing devices */ + FIND_PEER_SKIP_TARGET = 0x0001, + /*! Only check the target peer for capable fencing devices */ + FIND_PEER_TARGET_ONLY = 0x0002, + /*! Skip peers and devices that are not verified */ + FIND_PEER_VERIFIED_ONLY = 0x0004, +}; + +static peer_device_info_t * +find_best_peer(const char *device, remote_fencing_op_t * op, enum find_best_peer_options options) +{ + GList *iter = NULL; + gboolean verified_devices_only = (options & FIND_PEER_VERIFIED_ONLY) ? TRUE : FALSE; + + if (!device && pcmk_is_set(op->call_options, st_opt_topology)) { + return NULL; + } + + for (iter = op->query_results; iter != NULL; iter = iter->next) { + peer_device_info_t *peer = iter->data; + + crm_trace("Testing result from %s targeting %s with %d device%s: %d %x", + peer->host, op->target, peer->ndevices, + pcmk__plural_s(peer->ndevices), peer->tried, options); + if ((options & FIND_PEER_SKIP_TARGET) && pcmk__str_eq(peer->host, op->target, pcmk__str_casei)) { + continue; + } + if ((options & FIND_PEER_TARGET_ONLY) && !pcmk__str_eq(peer->host, op->target, pcmk__str_casei)) { + continue; + } + + if (pcmk_is_set(op->call_options, st_opt_topology)) { + + if (grab_peer_device(op, peer, device, verified_devices_only)) { + return peer; + } + + } else if (!peer->tried + && count_peer_devices(op, peer, verified_devices_only, + fenced_support_flag(op->action))) { + /* No topology: Use the current best peer */ + crm_trace("Simple fencing"); + return peer; + } + } + + return NULL; +} + +static peer_device_info_t * +stonith_choose_peer(remote_fencing_op_t * op) +{ + const char *device = NULL; + peer_device_info_t *peer = NULL; + uint32_t active = fencing_active_peers(); + + do { + if (op->devices) { + device = op->devices->data; + crm_trace("Checking for someone to fence (%s) %s using %s", + op->action, op->target, device); + } else { + crm_trace("Checking for someone to fence (%s) %s", + op->action, op->target); + } + + /* Best choice is a peer other than the target with verified access */ + peer = find_best_peer(device, op, FIND_PEER_SKIP_TARGET|FIND_PEER_VERIFIED_ONLY); + if (peer) { + crm_trace("Found verified peer %s for %s", peer->host, device?device:"<any>"); + return peer; + } + + if(op->query_timer != 0 && op->replies < QB_MIN(op->replies_expected, active)) { + crm_trace("Waiting before looking for unverified devices to fence %s", op->target); + return NULL; + } + + /* If no other peer has verified access, next best is unverified access */ + peer = find_best_peer(device, op, FIND_PEER_SKIP_TARGET); + if (peer) { + crm_trace("Found best unverified peer %s", peer->host); + return peer; + } + + /* If no other peer can do it, last option is self-fencing + * (which is never allowed for the "on" phase of a remapped reboot) + */ + if (op->phase != st_phase_on) { + peer = find_best_peer(device, op, FIND_PEER_TARGET_ONLY); + if (peer) { + crm_trace("%s will fence itself", peer->host); + return peer; + } + } + + /* Try the next fencing level if there is one (unless we're in the "on" + * phase of a remapped "reboot", because we ignore errors in that case) + */ + } while ((op->phase != st_phase_on) + && pcmk_is_set(op->call_options, st_opt_topology) + && (advance_topology_level(op, false) == pcmk_rc_ok)); + + if ((stonith_watchdog_timeout_ms > 0) + && pcmk__is_fencing_action(op->action) + && pcmk__str_eq(device, STONITH_WATCHDOG_ID, pcmk__str_none) + && node_does_watchdog_fencing(op->target)) { + crm_info("Couldn't contact watchdog-fencing target-node (%s)", + op->target); + /* check_watchdog_fencing_and_wait will log additional info */ + } else { + crm_notice("Couldn't find anyone to fence (%s) %s using %s", + op->action, op->target, (device? device : "any device")); + } + return NULL; +} + +static int +get_device_timeout(const remote_fencing_op_t *op, + const peer_device_info_t *peer, const char *device, + bool with_delay) +{ + device_properties_t *props; + int delay = 0; + + if (!peer || !device) { + return op->base_timeout; + } + + props = g_hash_table_lookup(peer->devices, device); + if (!props) { + return op->base_timeout; + } + + // op->delay < 0 means disable any static/random fencing delays + if (with_delay && op->delay >= 0) { + // delay_base is eventually limited by delay_max + delay = (props->delay_max[op->phase] > 0 ? + props->delay_max[op->phase] : props->delay_base[op->phase]); + } + + return (props->custom_action_timeout[op->phase]? + props->custom_action_timeout[op->phase] : op->base_timeout) + + delay; +} + +struct timeout_data { + const remote_fencing_op_t *op; + const peer_device_info_t *peer; + int total_timeout; +}; + +/*! + * \internal + * \brief Add timeout to a total if device has not been executed yet + * + * \param[in] key GHashTable key (device ID) + * \param[in] value GHashTable value (device properties) + * \param[in,out] user_data Timeout data + */ +static void +add_device_timeout(gpointer key, gpointer value, gpointer user_data) +{ + const char *device_id = key; + device_properties_t *props = value; + struct timeout_data *timeout = user_data; + + if (!props->executed[timeout->op->phase] + && !props->disallowed[timeout->op->phase]) { + timeout->total_timeout += get_device_timeout(timeout->op, timeout->peer, + device_id, true); + } +} + +static int +get_peer_timeout(const remote_fencing_op_t *op, const peer_device_info_t *peer) +{ + struct timeout_data timeout; + + timeout.op = op; + timeout.peer = peer; + timeout.total_timeout = 0; + + g_hash_table_foreach(peer->devices, add_device_timeout, &timeout); + + return (timeout.total_timeout? timeout.total_timeout : op->base_timeout); +} + +static int +get_op_total_timeout(const remote_fencing_op_t *op, + const peer_device_info_t *chosen_peer) +{ + int total_timeout = 0; + stonith_topology_t *tp = find_topology_for_host(op->target); + + if (pcmk_is_set(op->call_options, st_opt_topology) && tp) { + int i; + GList *device_list = NULL; + GList *iter = NULL; + GList *auto_list = NULL; + + if (pcmk__str_eq(op->action, "on", pcmk__str_none) + && (op->automatic_list != NULL)) { + auto_list = g_list_copy(op->automatic_list); + } + + /* Yep, this looks scary, nested loops all over the place. + * Here is what is going on. + * Loop1: Iterate through fencing levels. + * Loop2: If a fencing level has devices, loop through each device + * Loop3: For each device in a fencing level, see what peer owns it + * and what that peer has reported the timeout is for the device. + */ + for (i = 0; i < ST_LEVEL_MAX; i++) { + if (!tp->levels[i]) { + continue; + } + for (device_list = tp->levels[i]; device_list; device_list = device_list->next) { + /* in case of watchdog-device we add the timeout to the budget + regardless of if we got a reply or not + */ + if ((stonith_watchdog_timeout_ms > 0) + && pcmk__is_fencing_action(op->action) + && pcmk__str_eq(device_list->data, STONITH_WATCHDOG_ID, + pcmk__str_none) + && node_does_watchdog_fencing(op->target)) { + total_timeout += stonith_watchdog_timeout_ms / 1000; + continue; + } + + for (iter = op->query_results; iter != NULL; iter = iter->next) { + const peer_device_info_t *peer = iter->data; + + if (auto_list) { + GList *match = g_list_find_custom(auto_list, device_list->data, + sort_strings); + if (match) { + auto_list = g_list_remove(auto_list, match->data); + } + } + + if (find_peer_device(op, peer, device_list->data, + fenced_support_flag(op->action))) { + total_timeout += get_device_timeout(op, peer, + device_list->data, + true); + break; + } + } /* End Loop3: match device with peer that owns device, find device's timeout period */ + } /* End Loop2: iterate through devices at a specific level */ + } /*End Loop1: iterate through fencing levels */ + + //Add only exists automatic_list device timeout + if (auto_list) { + for (iter = auto_list; iter != NULL; iter = iter->next) { + GList *iter2 = NULL; + + for (iter2 = op->query_results; iter2 != NULL; iter = iter2->next) { + peer_device_info_t *peer = iter2->data; + if (find_peer_device(op, peer, iter->data, st_device_supports_on)) { + total_timeout += get_device_timeout(op, peer, + iter->data, true); + break; + } + } + } + } + + g_list_free(auto_list); + + } else if (chosen_peer) { + total_timeout = get_peer_timeout(op, chosen_peer); + } else { + total_timeout = op->base_timeout; + } + + /* Take any requested fencing delay into account to prevent it from eating + * up the total timeout. + */ + return ((total_timeout ? total_timeout : op->base_timeout) + + (op->delay > 0 ? op->delay : 0)); +} + +static void +report_timeout_period(remote_fencing_op_t * op, int op_timeout) +{ + GList *iter = NULL; + xmlNode *update = NULL; + const char *client_node = NULL; + const char *client_id = NULL; + const char *call_id = NULL; + + if (op->call_options & st_opt_sync_call) { + /* There is no reason to report the timeout for a synchronous call. It + * is impossible to use the reported timeout to do anything when the client + * is blocking for the response. This update is only important for + * async calls that require a callback to report the results in. */ + return; + } else if (!op->request) { + return; + } + + crm_trace("Reporting timeout for %s (id=%.8s)", op->client_name, op->id); + client_node = crm_element_value(op->request, F_STONITH_CLIENTNODE); + call_id = crm_element_value(op->request, F_STONITH_CALLID); + client_id = crm_element_value(op->request, F_STONITH_CLIENTID); + if (!client_node || !call_id || !client_id) { + return; + } + + if (pcmk__str_eq(client_node, stonith_our_uname, pcmk__str_casei)) { + // Client is connected to this node, so send update directly to them + do_stonith_async_timeout_update(client_id, call_id, op_timeout); + return; + } + + /* The client is connected to another node, relay this update to them */ + update = stonith_create_op(op->client_callid, op->id, STONITH_OP_TIMEOUT_UPDATE, NULL, 0); + crm_xml_add(update, F_STONITH_REMOTE_OP_ID, op->id); + crm_xml_add(update, F_STONITH_CLIENTID, client_id); + crm_xml_add(update, F_STONITH_CALLID, call_id); + crm_xml_add_int(update, F_STONITH_TIMEOUT, op_timeout); + + send_cluster_message(crm_get_peer(0, client_node), crm_msg_stonith_ng, update, FALSE); + + free_xml(update); + + for (iter = op->duplicates; iter != NULL; iter = iter->next) { + remote_fencing_op_t *dup = iter->data; + + crm_trace("Reporting timeout for duplicate %.8s to client %s", + dup->id, dup->client_name); + report_timeout_period(iter->data, op_timeout); + } +} + +/*! + * \internal + * \brief Advance an operation to the next device in its topology + * + * \param[in,out] op Fencer operation to advance + * \param[in] device ID of device that just completed + * \param[in,out] msg If not NULL, XML reply of last delegated operation + */ +static void +advance_topology_device_in_level(remote_fencing_op_t *op, const char *device, + xmlNode *msg) +{ + /* Advance to the next device at this topology level, if any */ + if (op->devices) { + op->devices = op->devices->next; + } + + /* Handle automatic unfencing if an "on" action was requested */ + if ((op->phase == st_phase_requested) + && pcmk__str_eq(op->action, "on", pcmk__str_none)) { + /* If the device we just executed was required, it's not anymore */ + remove_required_device(op, device); + + /* If there are no more devices at this topology level, run through any + * remaining devices with automatic unfencing + */ + if (op->devices == NULL) { + op->devices = op->automatic_list; + } + } + + if ((op->devices == NULL) && (op->phase == st_phase_off)) { + /* We're done with this level and with required devices, but we had + * remapped "reboot" to "off", so start over with "on". If any devices + * need to be turned back on, op->devices will be non-NULL after this. + */ + op_phase_on(op); + } + + // This function is only called if the previous device succeeded + pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + + if (op->devices) { + /* Necessary devices remain, so execute the next one */ + crm_trace("Next targeting %s on behalf of %s@%s", + op->target, op->client_name, op->originator); + + // The requested delay has been applied for the first device + if (op->delay > 0) { + op->delay = 0; + } + + request_peer_fencing(op, NULL); + } else { + /* We're done with all devices and phases, so finalize operation */ + crm_trace("Marking complex fencing op targeting %s as complete", + op->target); + op->state = st_done; + finalize_op(op, msg, false); + } +} + +static gboolean +check_watchdog_fencing_and_wait(remote_fencing_op_t * op) +{ + if (node_does_watchdog_fencing(op->target)) { + + crm_notice("Waiting %lds for %s to self-fence (%s) for " + "client %s " CRM_XS " id=%.8s", + (stonith_watchdog_timeout_ms / 1000), + op->target, op->action, op->client_name, op->id); + + if (op->op_timer_one) { + g_source_remove(op->op_timer_one); + } + op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms, + remote_op_watchdog_done, op); + return TRUE; + } else { + crm_debug("Skipping fallback to watchdog-fencing as %s is " + "not in host-list", op->target); + } + return FALSE; +} + +/*! + * \internal + * \brief Ask a peer to execute a fencing operation + * + * \param[in,out] op Fencing operation to be executed + * \param[in,out] peer If NULL or topology is in use, choose best peer to + * execute the fencing, otherwise use this peer + */ +static void +request_peer_fencing(remote_fencing_op_t *op, peer_device_info_t *peer) +{ + const char *device = NULL; + int timeout; + + CRM_CHECK(op != NULL, return); + + crm_trace("Action %.8s targeting %s for %s is %s", + op->id, op->target, op->client_name, + stonith_op_state_str(op->state)); + + if ((op->phase == st_phase_on) && (op->devices != NULL)) { + /* We are in the "on" phase of a remapped topology reboot. If this + * device has pcmk_reboot_action="off", or doesn't support the "on" + * action, skip it. + * + * We can't check device properties at this point because we haven't + * chosen a peer for this stage yet. Instead, we check the local node's + * knowledge about the device. If different versions of the fence agent + * are installed on different nodes, there's a chance this could be + * mistaken, but the worst that could happen is we don't try turning the + * node back on when we should. + */ + device = op->devices->data; + if (pcmk__str_eq(fenced_device_reboot_action(device), "off", + pcmk__str_none)) { + crm_info("Not turning %s back on using %s because the device is " + "configured to stay off (pcmk_reboot_action='off')", + op->target, device); + advance_topology_device_in_level(op, device, NULL); + return; + } + if (!fenced_device_supports_on(device)) { + crm_info("Not turning %s back on using %s because the agent " + "doesn't support 'on'", op->target, device); + advance_topology_device_in_level(op, device, NULL); + return; + } + } + + timeout = op->base_timeout; + if ((peer == NULL) && !pcmk_is_set(op->call_options, st_opt_topology)) { + peer = stonith_choose_peer(op); + } + + if (!op->op_timer_total) { + op->total_timeout = TIMEOUT_MULTIPLY_FACTOR * get_op_total_timeout(op, peer); + op->op_timer_total = g_timeout_add(1000 * op->total_timeout, remote_op_timeout, op); + report_timeout_period(op, op->total_timeout); + crm_info("Total timeout set to %d for peer's fencing targeting %s for %s" + CRM_XS "id=%.8s", + op->total_timeout, op->target, op->client_name, op->id); + } + + if (pcmk_is_set(op->call_options, st_opt_topology) && op->devices) { + /* Ignore the caller's peer preference if topology is in use, because + * that peer might not have access to the required device. With + * topology, stonith_choose_peer() removes the device from further + * consideration, so the timeout must be calculated beforehand. + * + * @TODO Basing the total timeout on the caller's preferred peer (above) + * is less than ideal. + */ + peer = stonith_choose_peer(op); + + device = op->devices->data; + /* Fencing timeout sent to peer takes no delay into account. + * The peer will add a dedicated timer for any delay upon + * schedule_stonith_command(). + */ + timeout = get_device_timeout(op, peer, device, false); + } + + if (peer) { + /* Take any requested fencing delay into account to prevent it from eating + * up the timeout. + */ + int timeout_one = (op->delay > 0 ? + TIMEOUT_MULTIPLY_FACTOR * op->delay : 0); + xmlNode *remote_op = stonith_create_op(op->client_callid, op->id, STONITH_OP_FENCE, NULL, 0); + + crm_xml_add(remote_op, F_STONITH_REMOTE_OP_ID, op->id); + crm_xml_add(remote_op, F_STONITH_TARGET, op->target); + crm_xml_add(remote_op, F_STONITH_ACTION, op->action); + crm_xml_add(remote_op, F_STONITH_ORIGIN, op->originator); + crm_xml_add(remote_op, F_STONITH_CLIENTID, op->client_id); + crm_xml_add(remote_op, F_STONITH_CLIENTNAME, op->client_name); + crm_xml_add_int(remote_op, F_STONITH_TIMEOUT, timeout); + crm_xml_add_int(remote_op, F_STONITH_CALLOPTS, op->call_options); + crm_xml_add_int(remote_op, F_STONITH_DELAY, op->delay); + + if (device) { + timeout_one += TIMEOUT_MULTIPLY_FACTOR * + get_device_timeout(op, peer, device, true); + crm_notice("Requesting that %s perform '%s' action targeting %s " + "using %s " CRM_XS " for client %s (%ds)", + peer->host, op->action, op->target, device, + op->client_name, timeout_one); + crm_xml_add(remote_op, F_STONITH_DEVICE, device); + + } else { + timeout_one += TIMEOUT_MULTIPLY_FACTOR * get_peer_timeout(op, peer); + crm_notice("Requesting that %s perform '%s' action targeting %s " + CRM_XS " for client %s (%ds, %lds)", + peer->host, op->action, op->target, op->client_name, + timeout_one, stonith_watchdog_timeout_ms); + } + + op->state = st_exec; + if (op->op_timer_one) { + g_source_remove(op->op_timer_one); + op->op_timer_one = 0; + } + + if (!((stonith_watchdog_timeout_ms > 0) + && (pcmk__str_eq(device, STONITH_WATCHDOG_ID, pcmk__str_none) + || (pcmk__str_eq(peer->host, op->target, pcmk__str_casei) + && pcmk__is_fencing_action(op->action))) + && check_watchdog_fencing_and_wait(op))) { + + /* Some thoughts about self-fencing cases reaching this point: + - Actually check in check_watchdog_fencing_and_wait + shouldn't fail if STONITH_WATCHDOG_ID is + chosen as fencing-device and it being present implies + watchdog-fencing is enabled anyway + - If watchdog-fencing is disabled either in general or for + a specific target - detected in check_watchdog_fencing_and_wait - + for some other kind of self-fencing we can't expect + a success answer but timeout is fine if the node doesn't + come back in between + - Delicate might be the case where we have watchdog-fencing + enabled for a node but the watchdog-fencing-device isn't + explicitly chosen for suicide. Local pe-execution in sbd + may detect the node as unclean and lead to timely suicide. + Otherwise the selection of stonith-watchdog-timeout at + least is questionable. + */ + + /* coming here we're not waiting for watchdog timeout - + thus engage timer with timout evaluated before */ + op->op_timer_one = g_timeout_add((1000 * timeout_one), remote_op_timeout_one, op); + } + + send_cluster_message(crm_get_peer(0, peer->host), crm_msg_stonith_ng, remote_op, FALSE); + peer->tried = TRUE; + free_xml(remote_op); + return; + + } else if (op->phase == st_phase_on) { + /* A remapped "on" cannot be executed, but the node was already + * turned off successfully, so ignore the error and continue. + */ + crm_warn("Ignoring %s 'on' failure (no capable peers) targeting %s " + "after successful 'off'", device, op->target); + advance_topology_device_in_level(op, device, NULL); + return; + + } else if (op->owner == FALSE) { + crm_err("Fencing (%s) targeting %s for client %s is not ours to control", + op->action, op->target, op->client_name); + + } else if (op->query_timer == 0) { + /* We've exhausted all available peers */ + crm_info("No remaining peers capable of fencing (%s) %s for client %s " + CRM_XS " state=%s", op->action, op->target, op->client_name, + stonith_op_state_str(op->state)); + CRM_CHECK(op->state < st_done, return); + finalize_timed_out_op(op, "All nodes failed, or are unable, to " + "fence target"); + + } else if(op->replies >= op->replies_expected || op->replies >= fencing_active_peers()) { + /* if the operation never left the query state, + * but we have all the expected replies, then no devices + * are available to execute the fencing operation. */ + + if(stonith_watchdog_timeout_ms > 0 && pcmk__str_eq(device, + STONITH_WATCHDOG_ID, pcmk__str_null_matches)) { + if (check_watchdog_fencing_and_wait(op)) { + return; + } + } + + if (op->state == st_query) { + crm_info("No peers (out of %d) have devices capable of fencing " + "(%s) %s for client %s " CRM_XS " state=%s", + op->replies, op->action, op->target, op->client_name, + stonith_op_state_str(op->state)); + + pcmk__reset_result(&op->result); + pcmk__set_result(&op->result, CRM_EX_ERROR, + PCMK_EXEC_NO_FENCE_DEVICE, NULL); + } else { + if (pcmk_is_set(op->call_options, st_opt_topology)) { + pcmk__reset_result(&op->result); + pcmk__set_result(&op->result, CRM_EX_ERROR, + PCMK_EXEC_NO_FENCE_DEVICE, NULL); + } + /* ... else use existing result from previous failed attempt + * (topology is not in use, and no devices remain to be attempted). + * Overwriting the result with PCMK_EXEC_NO_FENCE_DEVICE would + * prevent finalize_op() from setting the correct delegate if + * needed. + */ + + crm_info("No peers (out of %d) are capable of fencing (%s) %s " + "for client %s " CRM_XS " state=%s", + op->replies, op->action, op->target, op->client_name, + stonith_op_state_str(op->state)); + } + + op->state = st_failed; + finalize_op(op, NULL, false); + + } else { + crm_info("Waiting for additional peers capable of fencing (%s) %s%s%s " + "for client %s " CRM_XS " id=%.8s", + op->action, op->target, (device? " using " : ""), + (device? device : ""), op->client_name, op->id); + } +} + +/*! + * \internal + * \brief Comparison function for sorting query results + * + * \param[in] a GList item to compare + * \param[in] b GList item to compare + * + * \return Per the glib documentation, "a negative integer if the first value + * comes before the second, 0 if they are equal, or a positive integer + * if the first value comes after the second." + */ +static gint +sort_peers(gconstpointer a, gconstpointer b) +{ + const peer_device_info_t *peer_a = a; + const peer_device_info_t *peer_b = b; + + return (peer_b->ndevices - peer_a->ndevices); +} + +/*! + * \internal + * \brief Determine if all the devices in the topology are found or not + * + * \param[in] op Fencing operation with topology to check + */ +static gboolean +all_topology_devices_found(const remote_fencing_op_t *op) +{ + GList *device = NULL; + GList *iter = NULL; + device_properties_t *match = NULL; + stonith_topology_t *tp = NULL; + gboolean skip_target = FALSE; + int i; + + tp = find_topology_for_host(op->target); + if (!tp) { + return FALSE; + } + if (pcmk__is_fencing_action(op->action)) { + /* Don't count the devices on the target node if we are killing + * the target node. */ + skip_target = TRUE; + } + + for (i = 0; i < ST_LEVEL_MAX; i++) { + for (device = tp->levels[i]; device; device = device->next) { + match = NULL; + for (iter = op->query_results; iter && !match; iter = iter->next) { + peer_device_info_t *peer = iter->data; + + if (skip_target && pcmk__str_eq(peer->host, op->target, pcmk__str_casei)) { + continue; + } + match = find_peer_device(op, peer, device->data, st_device_supports_none); + } + if (!match) { + return FALSE; + } + } + } + + return TRUE; +} + +/*! + * \internal + * \brief Parse action-specific device properties from XML + * + * \param[in] xml XML element containing the properties + * \param[in] peer Name of peer that sent XML (for logs) + * \param[in] device Device ID (for logs) + * \param[in] action Action the properties relate to (for logs) + * \param[in,out] op Fencing operation that properties are being parsed for + * \param[in] phase Phase the properties relate to + * \param[in,out] props Device properties to update + */ +static void +parse_action_specific(const xmlNode *xml, const char *peer, const char *device, + const char *action, remote_fencing_op_t *op, + enum st_remap_phase phase, device_properties_t *props) +{ + props->custom_action_timeout[phase] = 0; + crm_element_value_int(xml, F_STONITH_ACTION_TIMEOUT, + &props->custom_action_timeout[phase]); + if (props->custom_action_timeout[phase]) { + crm_trace("Peer %s with device %s returned %s action timeout %d", + peer, device, action, props->custom_action_timeout[phase]); + } + + props->delay_max[phase] = 0; + crm_element_value_int(xml, F_STONITH_DELAY_MAX, &props->delay_max[phase]); + if (props->delay_max[phase]) { + crm_trace("Peer %s with device %s returned maximum of random delay %d for %s", + peer, device, props->delay_max[phase], action); + } + + props->delay_base[phase] = 0; + crm_element_value_int(xml, F_STONITH_DELAY_BASE, &props->delay_base[phase]); + if (props->delay_base[phase]) { + crm_trace("Peer %s with device %s returned base delay %d for %s", + peer, device, props->delay_base[phase], action); + } + + /* Handle devices with automatic unfencing */ + if (pcmk__str_eq(action, "on", pcmk__str_none)) { + int required = 0; + + crm_element_value_int(xml, F_STONITH_DEVICE_REQUIRED, &required); + if (required) { + crm_trace("Peer %s requires device %s to execute for action %s", + peer, device, action); + add_required_device(op, device); + } + } + + /* If a reboot is remapped to off+on, it's possible that a node is allowed + * to perform one action but not another. + */ + if (pcmk__xe_attr_is_true(xml, F_STONITH_ACTION_DISALLOWED)) { + props->disallowed[phase] = TRUE; + crm_trace("Peer %s is disallowed from executing %s for device %s", + peer, action, device); + } +} + +/*! + * \internal + * \brief Parse one device's properties from peer's XML query reply + * + * \param[in] xml XML node containing device properties + * \param[in,out] op Operation that query and reply relate to + * \param[in,out] peer Peer's device information + * \param[in] device ID of device being parsed + */ +static void +add_device_properties(const xmlNode *xml, remote_fencing_op_t *op, + peer_device_info_t *peer, const char *device) +{ + xmlNode *child; + int verified = 0; + device_properties_t *props = calloc(1, sizeof(device_properties_t)); + int flags = st_device_supports_on; /* Old nodes that don't set the flag assume they support the on action */ + + /* Add a new entry to this peer's devices list */ + CRM_ASSERT(props != NULL); + g_hash_table_insert(peer->devices, strdup(device), props); + + /* Peers with verified (monitored) access will be preferred */ + crm_element_value_int(xml, F_STONITH_DEVICE_VERIFIED, &verified); + if (verified) { + crm_trace("Peer %s has confirmed a verified device %s", + peer->host, device); + props->verified = TRUE; + } + + crm_element_value_int(xml, F_STONITH_DEVICE_SUPPORT_FLAGS, &flags); + props->device_support_flags = flags; + + /* Parse action-specific device properties */ + parse_action_specific(xml, peer->host, device, op_requested_action(op), + op, st_phase_requested, props); + for (child = pcmk__xml_first_child(xml); child != NULL; + child = pcmk__xml_next(child)) { + /* Replies for "reboot" operations will include the action-specific + * values for "off" and "on" in child elements, just in case the reboot + * winds up getting remapped. + */ + if (pcmk__str_eq(ID(child), "off", pcmk__str_none)) { + parse_action_specific(child, peer->host, device, "off", + op, st_phase_off, props); + } else if (pcmk__str_eq(ID(child), "on", pcmk__str_none)) { + parse_action_specific(child, peer->host, device, "on", + op, st_phase_on, props); + } + } +} + +/*! + * \internal + * \brief Parse a peer's XML query reply and add it to operation's results + * + * \param[in,out] op Operation that query and reply relate to + * \param[in] host Name of peer that sent this reply + * \param[in] ndevices Number of devices expected in reply + * \param[in] xml XML node containing device list + * + * \return Newly allocated result structure with parsed reply + */ +static peer_device_info_t * +add_result(remote_fencing_op_t *op, const char *host, int ndevices, + const xmlNode *xml) +{ + peer_device_info_t *peer = calloc(1, sizeof(peer_device_info_t)); + xmlNode *child; + + // cppcheck seems not to understand the abort logic in CRM_CHECK + // cppcheck-suppress memleak + CRM_CHECK(peer != NULL, return NULL); + peer->host = strdup(host); + peer->devices = pcmk__strkey_table(free, free); + + /* Each child element describes one capable device available to the peer */ + for (child = pcmk__xml_first_child(xml); child != NULL; + child = pcmk__xml_next(child)) { + const char *device = ID(child); + + if (device) { + add_device_properties(child, op, peer, device); + } + } + + peer->ndevices = g_hash_table_size(peer->devices); + CRM_CHECK(ndevices == peer->ndevices, + crm_err("Query claimed to have %d device%s but %d found", + ndevices, pcmk__plural_s(ndevices), peer->ndevices)); + + op->query_results = g_list_insert_sorted(op->query_results, peer, sort_peers); + return peer; +} + +/*! + * \internal + * \brief Handle a peer's reply to our fencing query + * + * Parse a query result from XML and store it in the remote operation + * table, and when enough replies have been received, issue a fencing request. + * + * \param[in] msg XML reply received + * + * \return pcmk_ok on success, -errno on error + * + * \note See initiate_remote_stonith_op() for how the XML query was initially + * formed, and stonith_query() for how the peer formed its XML reply. + */ +int +process_remote_stonith_query(xmlNode *msg) +{ + int ndevices = 0; + gboolean host_is_target = FALSE; + gboolean have_all_replies = FALSE; + const char *id = NULL; + const char *host = NULL; + remote_fencing_op_t *op = NULL; + peer_device_info_t *peer = NULL; + uint32_t replies_expected; + xmlNode *dev = get_xpath_object("//@" F_STONITH_REMOTE_OP_ID, msg, LOG_ERR); + + CRM_CHECK(dev != NULL, return -EPROTO); + + id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID); + CRM_CHECK(id != NULL, return -EPROTO); + + dev = get_xpath_object("//@" F_STONITH_AVAILABLE_DEVICES, msg, LOG_ERR); + CRM_CHECK(dev != NULL, return -EPROTO); + crm_element_value_int(dev, F_STONITH_AVAILABLE_DEVICES, &ndevices); + + op = g_hash_table_lookup(stonith_remote_op_list, id); + if (op == NULL) { + crm_debug("Received query reply for unknown or expired operation %s", + id); + return -EOPNOTSUPP; + } + + replies_expected = fencing_active_peers(); + if (op->replies_expected < replies_expected) { + replies_expected = op->replies_expected; + } + if ((++op->replies >= replies_expected) && (op->state == st_query)) { + have_all_replies = TRUE; + } + host = crm_element_value(msg, F_ORIG); + host_is_target = pcmk__str_eq(host, op->target, pcmk__str_casei); + + crm_info("Query result %d of %d from %s for %s/%s (%d device%s) %s", + op->replies, replies_expected, host, + op->target, op->action, ndevices, pcmk__plural_s(ndevices), id); + if (ndevices > 0) { + peer = add_result(op, host, ndevices, dev); + } + + pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + + if (pcmk_is_set(op->call_options, st_opt_topology)) { + /* If we start the fencing before all the topology results are in, + * it is possible fencing levels will be skipped because of the missing + * query results. */ + if (op->state == st_query && all_topology_devices_found(op)) { + /* All the query results are in for the topology, start the fencing ops. */ + crm_trace("All topology devices found"); + request_peer_fencing(op, peer); + + } else if (have_all_replies) { + crm_info("All topology query replies have arrived, continuing (%d expected/%d received) ", + replies_expected, op->replies); + request_peer_fencing(op, NULL); + } + + } else if (op->state == st_query) { + int nverified = count_peer_devices(op, peer, TRUE, + fenced_support_flag(op->action)); + + /* We have a result for a non-topology fencing op that looks promising, + * go ahead and start fencing before query timeout */ + if ((peer != NULL) && !host_is_target && nverified) { + /* we have a verified device living on a peer that is not the target */ + crm_trace("Found %d verified device%s", + nverified, pcmk__plural_s(nverified)); + request_peer_fencing(op, peer); + + } else if (have_all_replies) { + crm_info("All query replies have arrived, continuing (%d expected/%d received) ", + replies_expected, op->replies); + request_peer_fencing(op, NULL); + + } else { + crm_trace("Waiting for more peer results before launching fencing operation"); + } + + } else if ((peer != NULL) && (op->state == st_done)) { + crm_info("Discarding query result from %s (%d device%s): " + "Operation is %s", peer->host, + peer->ndevices, pcmk__plural_s(peer->ndevices), + stonith_op_state_str(op->state)); + } + + return pcmk_ok; +} + +/*! + * \internal + * \brief Handle a peer's reply to a fencing request + * + * Parse a fencing reply from XML, and either finalize the operation + * or attempt another device as appropriate. + * + * \param[in] msg XML reply received + */ +void +fenced_process_fencing_reply(xmlNode *msg) +{ + const char *id = NULL; + const char *device = NULL; + remote_fencing_op_t *op = NULL; + xmlNode *dev = get_xpath_object("//@" F_STONITH_REMOTE_OP_ID, msg, LOG_ERR); + pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; + + CRM_CHECK(dev != NULL, return); + + id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID); + CRM_CHECK(id != NULL, return); + + dev = stonith__find_xe_with_result(msg); + CRM_CHECK(dev != NULL, return); + + stonith__xe_get_result(dev, &result); + + device = crm_element_value(dev, F_STONITH_DEVICE); + + if (stonith_remote_op_list) { + op = g_hash_table_lookup(stonith_remote_op_list, id); + } + + if ((op == NULL) && pcmk__result_ok(&result)) { + /* Record successful fencing operations */ + const char *client_id = crm_element_value(dev, F_STONITH_CLIENTID); + + op = create_remote_stonith_op(client_id, dev, TRUE); + } + + if (op == NULL) { + /* Could be for an event that began before we started */ + /* TODO: Record the op for later querying */ + crm_info("Received peer result of unknown or expired operation %s", id); + pcmk__reset_result(&result); + return; + } + + pcmk__reset_result(&op->result); + op->result = result; // The operation takes ownership of the result + + if (op->devices && device && !pcmk__str_eq(op->devices->data, device, pcmk__str_casei)) { + crm_err("Received outdated reply for device %s (instead of %s) to " + "fence (%s) %s. Operation already timed out at peer level.", + device, (const char *) op->devices->data, op->action, op->target); + return; + } + + if (pcmk__str_eq(crm_element_value(msg, F_SUBTYPE), "broadcast", pcmk__str_casei)) { + if (pcmk__result_ok(&op->result)) { + op->state = st_done; + } else { + op->state = st_failed; + } + finalize_op(op, msg, false); + return; + + } else if (!pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) { + /* If this isn't a remote level broadcast, and we are not the + * originator of the operation, we should not be receiving this msg. */ + crm_err("Received non-broadcast fencing result for operation %.8s " + "we do not own (device %s targeting %s)", + op->id, device, op->target); + return; + } + + if (pcmk_is_set(op->call_options, st_opt_topology)) { + const char *device = NULL; + const char *reason = op->result.exit_reason; + + /* We own the op, and it is complete. broadcast the result to all nodes + * and notify our local clients. */ + if (op->state == st_done) { + finalize_op(op, msg, false); + return; + } + + device = crm_element_value(msg, F_STONITH_DEVICE); + + if ((op->phase == 2) && !pcmk__result_ok(&op->result)) { + /* A remapped "on" failed, but the node was already turned off + * successfully, so ignore the error and continue. + */ + crm_warn("Ignoring %s 'on' failure (%s%s%s) targeting %s " + "after successful 'off'", + device, pcmk_exec_status_str(op->result.execution_status), + (reason == NULL)? "" : ": ", + (reason == NULL)? "" : reason, + op->target); + pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + } else { + crm_notice("Action '%s' targeting %s%s%s on behalf of %s@%s: " + "%s%s%s%s", + op->action, op->target, + ((device == NULL)? "" : " using "), + ((device == NULL)? "" : device), + op->client_name, + op->originator, + pcmk_exec_status_str(op->result.execution_status), + (reason == NULL)? "" : " (", + (reason == NULL)? "" : reason, + (reason == NULL)? "" : ")"); + } + + if (pcmk__result_ok(&op->result)) { + /* An operation completed successfully. Try another device if + * necessary, otherwise mark the operation as done. */ + advance_topology_device_in_level(op, device, msg); + return; + } else { + /* This device failed, time to try another topology level. If no other + * levels are available, mark this operation as failed and report results. */ + if (advance_topology_level(op, false) != pcmk_rc_ok) { + op->state = st_failed; + finalize_op(op, msg, false); + return; + } + } + + } else if (pcmk__result_ok(&op->result) && (op->devices == NULL)) { + op->state = st_done; + finalize_op(op, msg, false); + return; + + } else if ((op->result.execution_status == PCMK_EXEC_TIMEOUT) + && (op->devices == NULL)) { + /* If the operation timed out don't bother retrying other peers. */ + op->state = st_failed; + finalize_op(op, msg, false); + return; + + } else { + /* fall-through and attempt other fencing action using another peer */ + } + + /* Retry on failure */ + crm_trace("Next for %s on behalf of %s@%s (result was: %s)", + op->target, op->originator, op->client_name, + pcmk_exec_status_str(op->result.execution_status)); + request_peer_fencing(op, NULL); +} + +gboolean +stonith_check_fence_tolerance(int tolerance, const char *target, const char *action) +{ + GHashTableIter iter; + time_t now = time(NULL); + remote_fencing_op_t *rop = NULL; + + if (tolerance <= 0 || !stonith_remote_op_list || target == NULL || + action == NULL) { + return FALSE; + } + + g_hash_table_iter_init(&iter, stonith_remote_op_list); + while (g_hash_table_iter_next(&iter, NULL, (void **)&rop)) { + if (strcmp(rop->target, target) != 0) { + continue; + } else if (rop->state != st_done) { + continue; + /* We don't have to worry about remapped reboots here + * because if state is done, any remapping has been undone + */ + } else if (strcmp(rop->action, action) != 0) { + continue; + } else if ((rop->completed + tolerance) < now) { + continue; + } + + crm_notice("Target %s was fenced (%s) less than %ds ago by %s on behalf of %s", + target, action, tolerance, rop->delegate, rop->originator); + return TRUE; + } + return FALSE; +} diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c new file mode 100644 index 0000000..4edda6c --- /dev/null +++ b/daemons/fenced/pacemaker-fenced.c @@ -0,0 +1,1751 @@ +/* + * Copyright 2009-2023 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. + */ + +#include <crm_internal.h> + +#include <sys/param.h> +#include <stdio.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <sys/utsname.h> + +#include <stdlib.h> +#include <errno.h> +#include <fcntl.h> +#include <inttypes.h> // PRIu32, PRIx32 + +#include <crm/crm.h> +#include <crm/msg_xml.h> +#include <crm/common/cmdline_internal.h> +#include <crm/common/ipc.h> +#include <crm/common/ipc_internal.h> +#include <crm/common/output_internal.h> +#include <crm/cluster/internal.h> + +#include <crm/stonith-ng.h> +#include <crm/fencing/internal.h> +#include <crm/common/xml.h> +#include <crm/common/xml_internal.h> + +#include <crm/common/mainloop.h> + +#include <crm/cib/internal.h> +#include <crm/pengine/status.h> +#include <pacemaker-internal.h> + +#include <pacemaker-fenced.h> + +#define SUMMARY "daemon for executing fencing devices in a Pacemaker cluster" + +char *stonith_our_uname = NULL; +long stonith_watchdog_timeout_ms = 0; +GList *stonith_watchdog_targets = NULL; + +static GMainLoop *mainloop = NULL; + +gboolean stand_alone = FALSE; +static gboolean stonith_shutdown_flag = FALSE; + +static qb_ipcs_service_t *ipcs = NULL; +static xmlNode *local_cib = NULL; +static pe_working_set_t *fenced_data_set = NULL; +static const unsigned long long data_set_flags = pe_flag_quick_location + | pe_flag_no_compat + | pe_flag_no_counts; + +static cib_t *cib_api = NULL; + +static pcmk__output_t *logger_out = NULL; +static pcmk__output_t *out = NULL; + +pcmk__supported_format_t formats[] = { + PCMK__SUPPORTED_FORMAT_NONE, + PCMK__SUPPORTED_FORMAT_TEXT, + PCMK__SUPPORTED_FORMAT_XML, + { NULL, NULL, NULL } +}; + +static struct { + bool no_cib_connect; + gchar **log_files; +} options; + +static crm_exit_t exit_code = CRM_EX_OK; + +static void stonith_shutdown(int nsig); +static void stonith_cleanup(void); + +static int32_t +st_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid) +{ + if (stonith_shutdown_flag) { + crm_info("Ignoring new client [%d] during shutdown", + pcmk__client_pid(c)); + return -EPERM; + } + + if (pcmk__new_client(c, uid, gid) == NULL) { + return -EIO; + } + return 0; +} + +/* Exit code means? */ +static int32_t +st_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size) +{ + uint32_t id = 0; + uint32_t flags = 0; + int call_options = 0; + xmlNode *request = NULL; + pcmk__client_t *c = pcmk__find_client(qbc); + const char *op = NULL; + + if (c == NULL) { + crm_info("Invalid client: %p", qbc); + return 0; + } + + request = pcmk__client_data2xml(c, data, &id, &flags); + if (request == NULL) { + pcmk__ipc_send_ack(c, id, flags, "nack", NULL, CRM_EX_PROTOCOL); + return 0; + } + + + op = crm_element_value(request, F_CRM_TASK); + if(pcmk__str_eq(op, CRM_OP_RM_NODE_CACHE, pcmk__str_casei)) { + crm_xml_add(request, F_TYPE, T_STONITH_NG); + crm_xml_add(request, F_STONITH_OPERATION, op); + crm_xml_add(request, F_STONITH_CLIENTID, c->id); + crm_xml_add(request, F_STONITH_CLIENTNAME, pcmk__client_name(c)); + crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname); + + send_cluster_message(NULL, crm_msg_stonith_ng, request, FALSE); + free_xml(request); + return 0; + } + + if (c->name == NULL) { + const char *value = crm_element_value(request, F_STONITH_CLIENTNAME); + + if (value == NULL) { + value = "unknown"; + } + c->name = crm_strdup_printf("%s.%u", value, c->pid); + } + + crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options); + crm_trace("Flags %#08" PRIx32 "/%#08x for command %" PRIu32 + " from client %s", flags, call_options, id, pcmk__client_name(c)); + + if (pcmk_is_set(call_options, st_opt_sync_call)) { + CRM_ASSERT(flags & crm_ipc_client_response); + CRM_LOG_ASSERT(c->request_id == 0); /* This means the client has two synchronous events in-flight */ + c->request_id = id; /* Reply only to the last one */ + } + + crm_xml_add(request, F_STONITH_CLIENTID, c->id); + crm_xml_add(request, F_STONITH_CLIENTNAME, pcmk__client_name(c)); + crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname); + + crm_log_xml_trace(request, "ipc-received"); + stonith_command(c, id, flags, request, NULL); + + free_xml(request); + return 0; +} + +/* Error code means? */ +static int32_t +st_ipc_closed(qb_ipcs_connection_t * c) +{ + pcmk__client_t *client = pcmk__find_client(c); + + if (client == NULL) { + return 0; + } + + crm_trace("Connection %p closed", c); + pcmk__free_client(client); + + /* 0 means: yes, go ahead and destroy the connection */ + return 0; +} + +static void +st_ipc_destroy(qb_ipcs_connection_t * c) +{ + crm_trace("Connection %p destroyed", c); + st_ipc_closed(c); +} + +static void +stonith_peer_callback(xmlNode * msg, void *private_data) +{ + const char *remote_peer = crm_element_value(msg, F_ORIG); + const char *op = crm_element_value(msg, F_STONITH_OPERATION); + + if (pcmk__str_eq(op, "poke", pcmk__str_none)) { + return; + } + + crm_log_xml_trace(msg, "Peer[inbound]"); + stonith_command(NULL, 0, 0, msg, remote_peer); +} + +#if SUPPORT_COROSYNC +static void +stonith_peer_ais_callback(cpg_handle_t handle, + const struct cpg_name *groupName, + uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len) +{ + uint32_t kind = 0; + xmlNode *xml = NULL; + const char *from = NULL; + char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from); + + if(data == NULL) { + return; + } + if (kind == crm_class_cluster) { + xml = string2xml(data); + if (xml == NULL) { + crm_err("Invalid XML: '%.120s'", data); + free(data); + return; + } + crm_xml_add(xml, F_ORIG, from); + /* crm_xml_add_int(xml, F_SEQ, wrapper->id); */ + stonith_peer_callback(xml, NULL); + } + + free_xml(xml); + free(data); + return; +} + +static void +stonith_peer_cs_destroy(gpointer user_data) +{ + crm_crit("Lost connection to cluster layer, shutting down"); + stonith_shutdown(0); +} +#endif + +void +do_local_reply(xmlNode *notify_src, pcmk__client_t *client, int call_options) +{ + /* send callback to originating child */ + int local_rc = pcmk_rc_ok; + int rid = 0; + uint32_t ipc_flags = crm_ipc_server_event; + + if (pcmk_is_set(call_options, st_opt_sync_call)) { + CRM_LOG_ASSERT(client->request_id); + rid = client->request_id; + client->request_id = 0; + ipc_flags = crm_ipc_flags_none; + } + + local_rc = pcmk__ipc_send_xml(client, rid, notify_src, ipc_flags); + if (local_rc == pcmk_rc_ok) { + crm_trace("Sent response %d to client %s", + rid, pcmk__client_name(client)); + } else { + crm_warn("%synchronous reply to client %s failed: %s", + (pcmk_is_set(call_options, st_opt_sync_call)? "S" : "As"), + pcmk__client_name(client), pcmk_rc_str(local_rc)); + } +} + +uint64_t +get_stonith_flag(const char *name) +{ + if (pcmk__str_eq(name, T_STONITH_NOTIFY_FENCE, pcmk__str_casei)) { + return st_callback_notify_fence; + + } else if (pcmk__str_eq(name, STONITH_OP_DEVICE_ADD, pcmk__str_casei)) { + return st_callback_device_add; + + } else if (pcmk__str_eq(name, STONITH_OP_DEVICE_DEL, pcmk__str_casei)) { + return st_callback_device_del; + + } else if (pcmk__str_eq(name, T_STONITH_NOTIFY_HISTORY, pcmk__str_casei)) { + return st_callback_notify_history; + + } else if (pcmk__str_eq(name, T_STONITH_NOTIFY_HISTORY_SYNCED, pcmk__str_casei)) { + return st_callback_notify_history_synced; + + } + return st_callback_unknown; +} + +static void +stonith_notify_client(gpointer key, gpointer value, gpointer user_data) +{ + + xmlNode *update_msg = user_data; + pcmk__client_t *client = value; + const char *type = NULL; + + CRM_CHECK(client != NULL, return); + CRM_CHECK(update_msg != NULL, return); + + type = crm_element_value(update_msg, F_SUBTYPE); + CRM_CHECK(type != NULL, crm_log_xml_err(update_msg, "notify"); return); + + if (client->ipcs == NULL) { + crm_trace("Skipping client with NULL channel"); + return; + } + + if (pcmk_is_set(client->flags, get_stonith_flag(type))) { + int rc = pcmk__ipc_send_xml(client, 0, update_msg, + crm_ipc_server_event); + + if (rc != pcmk_rc_ok) { + crm_warn("%s notification of client %s failed: %s " + CRM_XS " id=%.8s rc=%d", type, pcmk__client_name(client), + pcmk_rc_str(rc), client->id, rc); + } else { + crm_trace("Sent %s notification to client %s", + type, pcmk__client_name(client)); + } + } +} + +void +do_stonith_async_timeout_update(const char *client_id, const char *call_id, int timeout) +{ + pcmk__client_t *client = NULL; + xmlNode *notify_data = NULL; + + if (!timeout || !call_id || !client_id) { + return; + } + + client = pcmk__find_client_by_id(client_id); + if (!client) { + return; + } + + notify_data = create_xml_node(NULL, T_STONITH_TIMEOUT_VALUE); + crm_xml_add(notify_data, F_TYPE, T_STONITH_TIMEOUT_VALUE); + crm_xml_add(notify_data, F_STONITH_CALLID, call_id); + crm_xml_add_int(notify_data, F_STONITH_TIMEOUT, timeout); + + crm_trace("timeout update is %d for client %s and call id %s", timeout, client_id, call_id); + + if (client) { + pcmk__ipc_send_xml(client, 0, notify_data, crm_ipc_server_event); + } + + free_xml(notify_data); +} + +/*! + * \internal + * \brief Notify relevant IPC clients of a fencing operation result + * + * \param[in] type Notification type + * \param[in] result Result of fencing operation (assume success if NULL) + * \param[in] data If not NULL, add to notification as call data + */ +void +fenced_send_notification(const char *type, const pcmk__action_result_t *result, + xmlNode *data) +{ + /* TODO: Standardize the contents of data */ + xmlNode *update_msg = create_xml_node(NULL, "notify"); + + CRM_LOG_ASSERT(type != NULL); + + crm_xml_add(update_msg, F_TYPE, T_STONITH_NOTIFY); + crm_xml_add(update_msg, F_SUBTYPE, type); + crm_xml_add(update_msg, F_STONITH_OPERATION, type); + stonith__xe_set_result(update_msg, result); + + if (data != NULL) { + add_message_xml(update_msg, F_STONITH_CALLDATA, data); + } + + crm_trace("Notifying clients"); + pcmk__foreach_ipc_client(stonith_notify_client, update_msg); + free_xml(update_msg); + crm_trace("Notify complete"); +} + +/*! + * \internal + * \brief Send notifications for a configuration change to subscribed clients + * + * \param[in] op Notification type (STONITH_OP_DEVICE_ADD, + * STONITH_OP_DEVICE_DEL, STONITH_OP_LEVEL_ADD, or + * STONITH_OP_LEVEL_DEL) + * \param[in] result Operation result + * \param[in] desc Description of what changed + * \param[in] active Current number of devices or topologies in use + */ +static void +send_config_notification(const char *op, const pcmk__action_result_t *result, + const char *desc, int active) +{ + xmlNode *notify_data = create_xml_node(NULL, op); + + CRM_CHECK(notify_data != NULL, return); + + crm_xml_add(notify_data, F_STONITH_DEVICE, desc); + crm_xml_add_int(notify_data, F_STONITH_ACTIVE, active); + + fenced_send_notification(op, result, notify_data); + free_xml(notify_data); +} + +/*! + * \internal + * \brief Send notifications for a device change to subscribed clients + * + * \param[in] op Notification type (STONITH_OP_DEVICE_ADD or + * STONITH_OP_DEVICE_DEL) + * \param[in] result Operation result + * \param[in] desc ID of device that changed + */ +void +fenced_send_device_notification(const char *op, + const pcmk__action_result_t *result, + const char *desc) +{ + send_config_notification(op, result, desc, g_hash_table_size(device_list)); +} + +/*! + * \internal + * \brief Send notifications for a topology level change to subscribed clients + * + * \param[in] op Notification type (STONITH_OP_LEVEL_ADD or + * STONITH_OP_LEVEL_DEL) + * \param[in] result Operation result + * \param[in] desc String representation of level (<target>[<level_index>]) + */ +void +fenced_send_level_notification(const char *op, + const pcmk__action_result_t *result, + const char *desc) +{ + send_config_notification(op, result, desc, g_hash_table_size(topology)); +} + +static void +topology_remove_helper(const char *node, int level) +{ + char *desc = NULL; + pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; + xmlNode *data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL); + + crm_xml_add(data, F_STONITH_ORIGIN, __func__); + crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level); + crm_xml_add(data, XML_ATTR_STONITH_TARGET, node); + + fenced_unregister_level(data, &desc, &result); + fenced_send_level_notification(STONITH_OP_LEVEL_DEL, &result, desc); + pcmk__reset_result(&result); + free_xml(data); + free(desc); +} + +static void +remove_cib_device(xmlXPathObjectPtr xpathObj) +{ + int max = numXpathResults(xpathObj), lpc = 0; + + for (lpc = 0; lpc < max; lpc++) { + const char *rsc_id = NULL; + const char *standard = NULL; + xmlNode *match = getXpathResult(xpathObj, lpc); + + CRM_LOG_ASSERT(match != NULL); + if(match != NULL) { + standard = crm_element_value(match, XML_AGENT_ATTR_CLASS); + } + + if (!pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) { + continue; + } + + rsc_id = crm_element_value(match, XML_ATTR_ID); + + stonith_device_remove(rsc_id, true); + } +} + +static void +remove_topology_level(xmlNode *match) +{ + int index = 0; + char *key = NULL; + + CRM_CHECK(match != NULL, return); + + key = stonith_level_key(match, fenced_target_by_unknown); + crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index); + topology_remove_helper(key, index); + free(key); +} + +static void +add_topology_level(xmlNode *match) +{ + char *desc = NULL; + pcmk__action_result_t result = PCMK__UNKNOWN_RESULT; + + CRM_CHECK(match != NULL, return); + + fenced_register_level(match, &desc, &result); + fenced_send_level_notification(STONITH_OP_LEVEL_ADD, &result, desc); + pcmk__reset_result(&result); + free(desc); +} + +static void +remove_fencing_topology(xmlXPathObjectPtr xpathObj) +{ + int max = numXpathResults(xpathObj), lpc = 0; + + for (lpc = 0; lpc < max; lpc++) { + xmlNode *match = getXpathResult(xpathObj, lpc); + + CRM_LOG_ASSERT(match != NULL); + if (match && crm_element_value(match, XML_DIFF_MARKER)) { + /* Deletion */ + int index = 0; + char *target = stonith_level_key(match, fenced_target_by_unknown); + + crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index); + if (target == NULL) { + crm_err("Invalid fencing target in element %s", ID(match)); + + } else if (index <= 0) { + crm_err("Invalid level for %s in element %s", target, ID(match)); + + } else { + topology_remove_helper(target, index); + } + /* } else { Deal with modifications during the 'addition' stage */ + } + } +} + +static void +register_fencing_topology(xmlXPathObjectPtr xpathObj) +{ + int max = numXpathResults(xpathObj), lpc = 0; + + for (lpc = 0; lpc < max; lpc++) { + xmlNode *match = getXpathResult(xpathObj, lpc); + + remove_topology_level(match); + add_topology_level(match); + } +} + +/* Fencing +<diff crm_feature_set="3.0.6"> + <diff-removed> + <fencing-topology> + <fencing-level id="f-p1.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="removed:top"/> + <fencing-level id="f-p1.2" target="pcmk-1" index="2" devices="power" __crm_diff_marker__="removed:top"/> + <fencing-level devices="disk,network" id="f-p2.1"/> + </fencing-topology> + </diff-removed> + <diff-added> + <fencing-topology> + <fencing-level id="f-p.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="added:top"/> + <fencing-level id="f-p2.1" target="pcmk-2" index="1" devices="disk,something"/> + <fencing-level id="f-p3.1" target="pcmk-2" index="2" devices="power" __crm_diff_marker__="added:top"/> + </fencing-topology> + </diff-added> +</diff> +*/ + +static void +fencing_topology_init(void) +{ + xmlXPathObjectPtr xpathObj = NULL; + const char *xpath = "//" XML_TAG_FENCING_LEVEL; + + crm_trace("Full topology refresh"); + free_topology_list(); + init_topology_list(); + + /* Grab everything */ + xpathObj = xpath_search(local_cib, xpath); + register_fencing_topology(xpathObj); + + freeXpathObject(xpathObj); +} + +#define rsc_name(x) x->clone_name?x->clone_name:x->id + +/*! + * \internal + * \brief Check whether our uname is in a resource's allowed node list + * + * \param[in] rsc Resource to check + * + * \return Pointer to node object if found, NULL otherwise + */ +static pe_node_t * +our_node_allowed_for(const pe_resource_t *rsc) +{ + GHashTableIter iter; + pe_node_t *node = NULL; + + if (rsc && stonith_our_uname) { + g_hash_table_iter_init(&iter, rsc->allowed_nodes); + while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { + if (node && strcmp(node->details->uname, stonith_our_uname) == 0) { + break; + } + node = NULL; + } + } + return node; +} + +static void +watchdog_device_update(void) +{ + if (stonith_watchdog_timeout_ms > 0) { + if (!g_hash_table_lookup(device_list, STONITH_WATCHDOG_ID) && + !stonith_watchdog_targets) { + /* getting here watchdog-fencing enabled, no device there yet + and reason isn't stonith_watchdog_targets preventing that + */ + int rc; + xmlNode *xml; + + xml = create_device_registration_xml( + STONITH_WATCHDOG_ID, + st_namespace_internal, + STONITH_WATCHDOG_AGENT, + NULL, /* stonith_device_register will add our + own name as PCMK_STONITH_HOST_LIST param + so we can skip that here + */ + NULL); + rc = stonith_device_register(xml, TRUE); + free_xml(xml); + if (rc != pcmk_ok) { + rc = pcmk_legacy2rc(rc); + exit_code = CRM_EX_FATAL; + crm_crit("Cannot register watchdog pseudo fence agent: %s", + pcmk_rc_str(rc)); + stonith_shutdown(0); + } + } + + } else if (g_hash_table_lookup(device_list, STONITH_WATCHDOG_ID) != NULL) { + /* be silent if no device - todo parameter to stonith_device_remove */ + stonith_device_remove(STONITH_WATCHDOG_ID, true); + } +} + +static void +update_stonith_watchdog_timeout_ms(xmlNode *cib) +{ + long timeout_ms = 0; + xmlNode *stonith_watchdog_xml = NULL; + const char *value = NULL; + + stonith_watchdog_xml = get_xpath_object("//nvpair[@name='stonith-watchdog-timeout']", + cib, LOG_NEVER); + if (stonith_watchdog_xml) { + value = crm_element_value(stonith_watchdog_xml, XML_NVPAIR_ATTR_VALUE); + } + if (value) { + timeout_ms = crm_get_msec(value); + } + + if (timeout_ms < 0) { + timeout_ms = pcmk__auto_watchdog_timeout(); + } + + stonith_watchdog_timeout_ms = timeout_ms; +} + +/*! + * \internal + * \brief If a resource or any of its children are STONITH devices, update their + * definitions given a cluster working set. + * + * \param[in,out] rsc Resource to check + * \param[in,out] data_set Cluster working set with device information + */ +static void +cib_device_update(pe_resource_t *rsc, pe_working_set_t *data_set) +{ + pe_node_t *node = NULL; + const char *value = NULL; + const char *rclass = NULL; + pe_node_t *parent = NULL; + + /* If this is a complex resource, check children rather than this resource itself. */ + if(rsc->children) { + GList *gIter = NULL; + for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { + cib_device_update(gIter->data, data_set); + if(pe_rsc_is_clone(rsc)) { + crm_trace("Only processing one copy of the clone %s", rsc->id); + break; + } + } + return; + } + + /* We only care about STONITH resources. */ + rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS); + if (!pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) { + return; + } + + /* If this STONITH resource is disabled, remove it. */ + if (pe__resource_is_disabled(rsc)) { + crm_info("Device %s has been disabled", rsc->id); + return; + } + + /* if watchdog-fencing is disabled handle any watchdog-fence + resource as if it was disabled + */ + if ((stonith_watchdog_timeout_ms <= 0) && + pcmk__str_eq(rsc->id, STONITH_WATCHDOG_ID, pcmk__str_none)) { + crm_info("Watchdog-fencing disabled thus handling " + "device %s as disabled", rsc->id); + return; + } + + /* Check whether our node is allowed for this resource (and its parent if in a group) */ + node = our_node_allowed_for(rsc); + if (rsc->parent && (rsc->parent->variant == pe_group)) { + parent = our_node_allowed_for(rsc->parent); + } + + if(node == NULL) { + /* Our node is disallowed, so remove the device */ + GHashTableIter iter; + + crm_info("Device %s has been disabled on %s: unknown", rsc->id, stonith_our_uname); + g_hash_table_iter_init(&iter, rsc->allowed_nodes); + while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { + crm_trace("Available: %s = %d", pe__node_name(node), node->weight); + } + + return; + + } else if(node->weight < 0 || (parent && parent->weight < 0)) { + /* Our node (or its group) is disallowed by score, so remove the device */ + int score = (node->weight < 0)? node->weight : parent->weight; + + crm_info("Device %s has been disabled on %s: score=%s", + rsc->id, stonith_our_uname, pcmk_readable_score(score)); + return; + + } else { + /* Our node is allowed, so update the device information */ + int rc; + xmlNode *data; + GHashTable *rsc_params = NULL; + GHashTableIter gIter; + stonith_key_value_t *params = NULL; + + const char *name = NULL; + const char *agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE); + const char *rsc_provides = NULL; + + crm_debug("Device %s is allowed on %s: score=%d", rsc->id, stonith_our_uname, node->weight); + rsc_params = pe_rsc_params(rsc, node, data_set); + get_meta_attributes(rsc->meta, rsc, node, data_set); + + rsc_provides = g_hash_table_lookup(rsc->meta, PCMK_STONITH_PROVIDES); + + g_hash_table_iter_init(&gIter, rsc_params); + while (g_hash_table_iter_next(&gIter, (gpointer *) & name, (gpointer *) & value)) { + if (!name || !value) { + continue; + } + params = stonith_key_value_add(params, name, value); + crm_trace(" %s=%s", name, value); + } + + data = create_device_registration_xml(rsc_name(rsc), st_namespace_any, + agent, params, rsc_provides); + stonith_key_value_freeall(params, 1, 1); + rc = stonith_device_register(data, TRUE); + CRM_ASSERT(rc == pcmk_ok); + free_xml(data); + } +} + +/*! + * \internal + * \brief Update all STONITH device definitions based on current CIB + */ +static void +cib_devices_update(void) +{ + GHashTableIter iter; + stonith_device_t *device = NULL; + + crm_info("Updating devices to version %s.%s.%s", + crm_element_value(local_cib, XML_ATTR_GENERATION_ADMIN), + crm_element_value(local_cib, XML_ATTR_GENERATION), + crm_element_value(local_cib, XML_ATTR_NUMUPDATES)); + + if (fenced_data_set->now != NULL) { + crm_time_free(fenced_data_set->now); + fenced_data_set->now = NULL; + } + fenced_data_set->localhost = stonith_our_uname; + pcmk__schedule_actions(local_cib, data_set_flags, fenced_data_set); + + g_hash_table_iter_init(&iter, device_list); + while (g_hash_table_iter_next(&iter, NULL, (void **)&device)) { + if (device->cib_registered) { + device->dirty = TRUE; + } + } + + /* have list repopulated if cib has a watchdog-fencing-resource + TODO: keep a cached list for queries happening while we are refreshing + */ + g_list_free_full(stonith_watchdog_targets, free); + stonith_watchdog_targets = NULL; + g_list_foreach(fenced_data_set->resources, (GFunc) cib_device_update, fenced_data_set); + + g_hash_table_iter_init(&iter, device_list); + while (g_hash_table_iter_next(&iter, NULL, (void **)&device)) { + if (device->dirty) { + g_hash_table_iter_remove(&iter); + } + } + + fenced_data_set->input = NULL; // Wasn't a copy, so don't let API free it + pe_reset_working_set(fenced_data_set); +} + +static void +update_cib_stonith_devices_v2(const char *event, xmlNode * msg) +{ + xmlNode *change = NULL; + char *reason = NULL; + bool needs_update = FALSE; + xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT); + + for (change = pcmk__xml_first_child(patchset); change != NULL; + change = pcmk__xml_next(change)) { + const char *op = crm_element_value(change, XML_DIFF_OP); + const char *xpath = crm_element_value(change, XML_DIFF_PATH); + const char *shortpath = NULL; + + if ((op == NULL) || + (strcmp(op, "move") == 0) || + strstr(xpath, "/"XML_CIB_TAG_STATUS)) { + continue; + } else if (pcmk__str_eq(op, "delete", pcmk__str_casei) && strstr(xpath, "/"XML_CIB_TAG_RESOURCE)) { + const char *rsc_id = NULL; + char *search = NULL; + char *mutable = NULL; + + if (strstr(xpath, XML_TAG_ATTR_SETS) || + strstr(xpath, XML_TAG_META_SETS)) { + needs_update = TRUE; + pcmk__str_update(&reason, + "(meta) attribute deleted from resource"); + break; + } + pcmk__str_update(&mutable, xpath); + rsc_id = strstr(mutable, "primitive[@" XML_ATTR_ID "=\'"); + if (rsc_id != NULL) { + rsc_id += strlen("primitive[@" XML_ATTR_ID "=\'"); + search = strchr(rsc_id, '\''); + } + if (search != NULL) { + *search = 0; + stonith_device_remove(rsc_id, true); + /* watchdog_device_update called afterwards + to fall back to implicit definition if needed */ + } else { + crm_warn("Ignoring malformed CIB update (resource deletion)"); + } + free(mutable); + + } else if (strstr(xpath, "/"XML_CIB_TAG_RESOURCES) || + strstr(xpath, "/"XML_CIB_TAG_CONSTRAINTS) || + strstr(xpath, "/"XML_CIB_TAG_RSCCONFIG)) { + shortpath = strrchr(xpath, '/'); CRM_ASSERT(shortpath); + reason = crm_strdup_printf("%s %s", op, shortpath+1); + needs_update = TRUE; + break; + } + } + + if(needs_update) { + crm_info("Updating device list from CIB: %s", reason); + cib_devices_update(); + } else { + crm_trace("No updates for device list found in CIB"); + } + free(reason); +} + + +static void +update_cib_stonith_devices_v1(const char *event, xmlNode * msg) +{ + const char *reason = "none"; + gboolean needs_update = FALSE; + xmlXPathObjectPtr xpath_obj = NULL; + + /* process new constraints */ + xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_CONS_TAG_RSC_LOCATION); + if (numXpathResults(xpath_obj) > 0) { + int max = numXpathResults(xpath_obj), lpc = 0; + + /* Safest and simplest to always recompute */ + needs_update = TRUE; + reason = "new location constraint"; + + for (lpc = 0; lpc < max; lpc++) { + xmlNode *match = getXpathResult(xpath_obj, lpc); + + crm_log_xml_trace(match, "new constraint"); + } + } + freeXpathObject(xpath_obj); + + /* process deletions */ + xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_RESOURCE); + if (numXpathResults(xpath_obj) > 0) { + remove_cib_device(xpath_obj); + } + freeXpathObject(xpath_obj); + + /* process additions */ + xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_RESOURCE); + if (numXpathResults(xpath_obj) > 0) { + int max = numXpathResults(xpath_obj), lpc = 0; + + for (lpc = 0; lpc < max; lpc++) { + const char *rsc_id = NULL; + const char *standard = NULL; + xmlNode *match = getXpathResult(xpath_obj, lpc); + + rsc_id = crm_element_value(match, XML_ATTR_ID); + standard = crm_element_value(match, XML_AGENT_ATTR_CLASS); + + if (!pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) { + continue; + } + + crm_trace("Fencing resource %s was added or modified", rsc_id); + reason = "new resource"; + needs_update = TRUE; + } + } + freeXpathObject(xpath_obj); + + if(needs_update) { + crm_info("Updating device list from CIB: %s", reason); + cib_devices_update(); + } +} + +static void +update_cib_stonith_devices(const char *event, xmlNode * msg) +{ + int format = 1; + xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT); + + CRM_ASSERT(patchset); + crm_element_value_int(patchset, "format", &format); + switch(format) { + case 1: + update_cib_stonith_devices_v1(event, msg); + break; + case 2: + update_cib_stonith_devices_v2(event, msg); + break; + default: + crm_warn("Unknown patch format: %d", format); + } +} + +/*! + * \internal + * \brief Check whether a node has a specific attribute name/value + * + * \param[in] node Name of node to check + * \param[in] name Name of an attribute to look for + * \param[in] value The value the named attribute needs to be set to in order to be considered a match + * + * \return TRUE if the locally cached CIB has the specified node attribute + */ +gboolean +node_has_attr(const char *node, const char *name, const char *value) +{ + GString *xpath = NULL; + xmlNode *match; + + CRM_CHECK((local_cib != NULL) && (node != NULL) && (name != NULL) + && (value != NULL), return FALSE); + + /* Search for the node's attributes in the CIB. While the schema allows + * multiple sets of instance attributes, and allows instance attributes to + * use id-ref to reference values elsewhere, that is intended for resources, + * so we ignore that here. + */ + xpath = g_string_sized_new(256); + pcmk__g_strcat(xpath, + "//" XML_CIB_TAG_NODES "/" XML_CIB_TAG_NODE + "[@" XML_ATTR_UNAME "='", node, "']/" XML_TAG_ATTR_SETS + "/" XML_CIB_TAG_NVPAIR + "[@" XML_NVPAIR_ATTR_NAME "='", name, "' " + "and @" XML_NVPAIR_ATTR_VALUE "='", value, "']", NULL); + + match = get_xpath_object((const char *) xpath->str, local_cib, LOG_NEVER); + + g_string_free(xpath, TRUE); + return (match != NULL); +} + +/*! + * \internal + * \brief Check whether a node does watchdog-fencing + * + * \param[in] node Name of node to check + * + * \return TRUE if node found in stonith_watchdog_targets + * or stonith_watchdog_targets is empty indicating + * all nodes are doing watchdog-fencing + */ +gboolean +node_does_watchdog_fencing(const char *node) +{ + return ((stonith_watchdog_targets == NULL) || + pcmk__str_in_list(node, stonith_watchdog_targets, pcmk__str_casei)); +} + + +static void +update_fencing_topology(const char *event, xmlNode * msg) +{ + int format = 1; + const char *xpath; + xmlXPathObjectPtr xpathObj = NULL; + xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT); + + CRM_ASSERT(patchset); + crm_element_value_int(patchset, "format", &format); + + if(format == 1) { + /* Process deletions (only) */ + xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_TAG_FENCING_LEVEL; + xpathObj = xpath_search(msg, xpath); + + remove_fencing_topology(xpathObj); + freeXpathObject(xpathObj); + + /* Process additions and changes */ + xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_TAG_FENCING_LEVEL; + xpathObj = xpath_search(msg, xpath); + + register_fencing_topology(xpathObj); + freeXpathObject(xpathObj); + + } else if(format == 2) { + xmlNode *change = NULL; + int add[] = { 0, 0, 0 }; + int del[] = { 0, 0, 0 }; + + xml_patch_versions(patchset, add, del); + + for (change = pcmk__xml_first_child(patchset); change != NULL; + change = pcmk__xml_next(change)) { + const char *op = crm_element_value(change, XML_DIFF_OP); + const char *xpath = crm_element_value(change, XML_DIFF_PATH); + + if(op == NULL) { + continue; + + } else if(strstr(xpath, "/" XML_TAG_FENCING_LEVEL) != NULL) { + /* Change to a specific entry */ + + crm_trace("Handling %s operation %d.%d.%d for %s", op, add[0], add[1], add[2], xpath); + if(strcmp(op, "move") == 0) { + continue; + + } else if(strcmp(op, "create") == 0) { + add_topology_level(change->children); + + } else if(strcmp(op, "modify") == 0) { + xmlNode *match = first_named_child(change, XML_DIFF_RESULT); + + if(match) { + remove_topology_level(match->children); + add_topology_level(match->children); + } + + } else if(strcmp(op, "delete") == 0) { + /* Nuclear option, all we have is the path and an id... not enough to remove a specific entry */ + crm_info("Re-initializing fencing topology after %s operation %d.%d.%d for %s", + op, add[0], add[1], add[2], xpath); + fencing_topology_init(); + return; + } + + } else if (strstr(xpath, "/" XML_TAG_FENCING_TOPOLOGY) != NULL) { + /* Change to the topology in general */ + crm_info("Re-initializing fencing topology after top-level %s operation %d.%d.%d for %s", + op, add[0], add[1], add[2], xpath); + fencing_topology_init(); + return; + + } else if (strstr(xpath, "/" XML_CIB_TAG_CONFIGURATION)) { + /* Changes to the whole config section, possibly including the topology as a whild */ + if(first_named_child(change, XML_TAG_FENCING_TOPOLOGY) == NULL) { + crm_trace("Nothing for us in %s operation %d.%d.%d for %s.", + op, add[0], add[1], add[2], xpath); + + } else if(strcmp(op, "delete") == 0 || strcmp(op, "create") == 0) { + crm_info("Re-initializing fencing topology after top-level %s operation %d.%d.%d for %s.", + op, add[0], add[1], add[2], xpath); + fencing_topology_init(); + return; + } + + } else { + crm_trace("Nothing for us in %s operation %d.%d.%d for %s", + op, add[0], add[1], add[2], xpath); + } + } + + } else { + crm_warn("Unknown patch format: %d", format); + } +} +static bool have_cib_devices = FALSE; + +static void +update_cib_cache_cb(const char *event, xmlNode * msg) +{ + int rc = pcmk_ok; + long timeout_ms_saved = stonith_watchdog_timeout_ms; + bool need_full_refresh = false; + + if(!have_cib_devices) { + crm_trace("Skipping updates until we get a full dump"); + return; + + } else if(msg == NULL) { + crm_trace("Missing %s update", event); + return; + } + + /* Maintain a local copy of the CIB so that we have full access + * to device definitions, location constraints, and node attributes + */ + if (local_cib != NULL) { + int rc = pcmk_ok; + xmlNode *patchset = NULL; + + crm_element_value_int(msg, F_CIB_RC, &rc); + if (rc != pcmk_ok) { + return; + } + + patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT); + pcmk__output_set_log_level(logger_out, LOG_TRACE); + out->message(out, "xml-patchset", patchset); + rc = xml_apply_patchset(local_cib, patchset, TRUE); + switch (rc) { + case pcmk_ok: + case -pcmk_err_old_data: + break; + case -pcmk_err_diff_resync: + case -pcmk_err_diff_failed: + crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc); + free_xml(local_cib); + local_cib = NULL; + break; + default: + crm_warn("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc); + free_xml(local_cib); + local_cib = NULL; + } + } + + if (local_cib == NULL) { + crm_trace("Re-requesting full CIB"); + rc = cib_api->cmds->query(cib_api, NULL, &local_cib, cib_scope_local | cib_sync_call); + if(rc != pcmk_ok) { + crm_err("Couldn't retrieve the CIB: %s (%d)", pcmk_strerror(rc), rc); + return; + } + CRM_ASSERT(local_cib != NULL); + need_full_refresh = true; + } + + pcmk__refresh_node_caches_from_cib(local_cib); + update_stonith_watchdog_timeout_ms(local_cib); + + if (timeout_ms_saved != stonith_watchdog_timeout_ms) { + need_full_refresh = true; + } + + if (need_full_refresh) { + fencing_topology_init(); + cib_devices_update(); + } else { + // Partial refresh + update_fencing_topology(event, msg); + update_cib_stonith_devices(event, msg); + } + + watchdog_device_update(); +} + +static void +init_cib_cache_cb(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) +{ + crm_info("Updating device list from CIB"); + have_cib_devices = TRUE; + local_cib = copy_xml(output); + + pcmk__refresh_node_caches_from_cib(local_cib); + update_stonith_watchdog_timeout_ms(local_cib); + + fencing_topology_init(); + cib_devices_update(); + watchdog_device_update(); +} + +static void +stonith_shutdown(int nsig) +{ + crm_info("Terminating with %d clients", pcmk__ipc_client_count()); + stonith_shutdown_flag = TRUE; + if (mainloop != NULL && g_main_loop_is_running(mainloop)) { + g_main_loop_quit(mainloop); + } +} + +static void +cib_connection_destroy(gpointer user_data) +{ + if (stonith_shutdown_flag) { + crm_info("Connection to the CIB manager closed"); + return; + } else { + crm_crit("Lost connection to the CIB manager, shutting down"); + } + if (cib_api) { + cib_api->cmds->signoff(cib_api); + } + stonith_shutdown(0); +} + +static void +stonith_cleanup(void) +{ + if (cib_api) { + cib_api->cmds->del_notify_callback(cib_api, T_CIB_DIFF_NOTIFY, update_cib_cache_cb); + cib_api->cmds->signoff(cib_api); + } + + if (ipcs) { + qb_ipcs_destroy(ipcs); + } + + crm_peer_destroy(); + pcmk__client_cleanup(); + free_stonith_remote_op_list(); + free_topology_list(); + free_device_list(); + free_metadata_cache(); + fenced_unregister_handlers(); + + free(stonith_our_uname); + stonith_our_uname = NULL; + + free_xml(local_cib); + local_cib = NULL; +} + +static gboolean +stand_alone_cpg_cb(const gchar *option_name, const gchar *optarg, gpointer data, + GError **error) +{ + stand_alone = FALSE; + options.no_cib_connect = true; + return TRUE; +} + +static void +setup_cib(void) +{ + int rc, retries = 0; + + cib_api = cib_new(); + if (cib_api == NULL) { + crm_err("No connection to the CIB manager"); + return; + } + + do { + sleep(retries); + rc = cib_api->cmds->signon(cib_api, CRM_SYSTEM_STONITHD, cib_command); + } while (rc == -ENOTCONN && ++retries < 5); + + if (rc != pcmk_ok) { + crm_err("Could not connect to the CIB manager: %s (%d)", pcmk_strerror(rc), rc); + + } else if (pcmk_ok != + cib_api->cmds->add_notify_callback(cib_api, T_CIB_DIFF_NOTIFY, update_cib_cache_cb)) { + crm_err("Could not set CIB notification callback"); + + } else { + rc = cib_api->cmds->query(cib_api, NULL, NULL, cib_scope_local); + cib_api->cmds->register_callback(cib_api, rc, 120, FALSE, NULL, "init_cib_cache_cb", + init_cib_cache_cb); + cib_api->cmds->set_connection_dnotify(cib_api, cib_connection_destroy); + crm_info("Watching for fencing topology changes"); + } +} + +struct qb_ipcs_service_handlers ipc_callbacks = { + .connection_accept = st_ipc_accept, + .connection_created = NULL, + .msg_process = st_ipc_dispatch, + .connection_closed = st_ipc_closed, + .connection_destroyed = st_ipc_destroy +}; + +/*! + * \internal + * \brief Callback for peer status changes + * + * \param[in] type What changed + * \param[in] node What peer had the change + * \param[in] data Previous value of what changed + */ +static void +st_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data) +{ + if ((type != crm_status_processes) + && !pcmk_is_set(node->flags, crm_remote_node)) { + /* + * This is a hack until we can send to a nodeid and/or we fix node name lookups + * These messages are ignored in stonith_peer_callback() + */ + xmlNode *query = create_xml_node(NULL, "stonith_command"); + + crm_xml_add(query, F_XML_TAGNAME, "stonith_command"); + crm_xml_add(query, F_TYPE, T_STONITH_NG); + crm_xml_add(query, F_STONITH_OPERATION, "poke"); + + crm_debug("Broadcasting our uname because of node %u", node->id); + send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE); + + free_xml(query); + } +} + +static pcmk__cluster_option_t fencer_options[] = { + /* name, old name, type, allowed values, + * default value, validator, + * short description, + * long description + */ + { + PCMK_STONITH_HOST_ARGUMENT, NULL, "string", NULL, "port", NULL, + N_("Advanced use only: An alternate parameter to supply instead of 'port'"), + N_("some devices do not support the " + "standard 'port' parameter or may provide additional ones. Use " + "this to specify an alternate, device-specific, parameter " + "that should indicate the machine to be fenced. A value of " + "none can be used to tell the cluster not to supply any " + "additional parameters.") + }, + { + PCMK_STONITH_HOST_MAP,NULL, "string", NULL, "", NULL, + N_("A mapping of host names to ports numbers for devices that do not support host names."), + N_("Eg. node1:1;node2:2,3 would tell the cluster to use port 1 for node1 and ports 2 and 3 for node2") + }, + { + PCMK_STONITH_HOST_LIST,NULL, "string", NULL, "", NULL, + N_("Eg. node1,node2,node3"), + N_("A list of machines controlled by " + "this device (Optional unless pcmk_host_list=static-list)") + }, + { + PCMK_STONITH_HOST_CHECK,NULL, "string", NULL, "dynamic-list", NULL, + N_("How to determine which machines are controlled by the device."), + N_("Allowed values: dynamic-list " + "(query the device via the 'list' command), static-list " + "(check the pcmk_host_list attribute), status " + "(query the device via the 'status' command), " + "none (assume every device can fence every " + "machine)") + }, + { + PCMK_STONITH_DELAY_MAX,NULL, "time", NULL, "0s", NULL, + N_("Enable a base delay for fencing actions and specify base delay value."), + N_("Enable a delay of no more than the " + "time specified before executing fencing actions. Pacemaker " + "derives the overall delay by taking the value of " + "pcmk_delay_base and adding a random delay value such " + "that the sum is kept below this maximum.") + }, + { + PCMK_STONITH_DELAY_BASE,NULL, "string", NULL, "0s", NULL, + N_("Enable a base delay for " + "fencing actions and specify base delay value."), + N_("This enables a static delay for " + "fencing actions, which can help avoid \"death matches\" where " + "two nodes try to fence each other at the same time. If " + "pcmk_delay_max is also used, a random delay will be " + "added such that the total delay is kept below that value." + "This can be set to a single time value to apply to any node " + "targeted by this device (useful if a separate device is " + "configured for each target), or to a node map (for example, " + "\"node1:1s;node2:5\") to set a different value per target.") + }, + { + PCMK_STONITH_ACTION_LIMIT,NULL, "integer", NULL, "1", NULL, + N_("The maximum number of actions can be performed in parallel on this device"), + N_("Cluster property concurrent-fencing=true needs to be configured first." + "Then use this to specify the maximum number of actions can be performed in parallel on this device. -1 is unlimited.") + }, + { + "pcmk_reboot_action",NULL, "string", NULL, "reboot", NULL, + N_("Advanced use only: An alternate command to run instead of 'reboot'"), + N_("Some devices do not support the standard commands or may provide additional ones.\n" + "Use this to specify an alternate, device-specific, command that implements the \'reboot\' action.") + }, + { + "pcmk_reboot_timeout",NULL, "time", NULL, "60s", NULL, + N_("Advanced use only: Specify an alternate timeout to use for reboot actions instead of stonith-timeout"), + N_("Some devices need much more/less time to complete than normal." + "Use this to specify an alternate, device-specific, timeout for \'reboot\' actions.") + }, + { + "pcmk_reboot_retries",NULL, "integer", NULL, "2", NULL, + N_("Advanced use only: The maximum number of times to retry the 'reboot' command within the timeout period"), + N_("Some devices do not support multiple connections." + " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining." + " Use this option to alter the number of times Pacemaker retries \'reboot\' actions before giving up.") + }, + { + "pcmk_off_action",NULL, "string", NULL, "off", NULL, + N_("Advanced use only: An alternate command to run instead of \'off\'"), + N_("Some devices do not support the standard commands or may provide additional ones." + "Use this to specify an alternate, device-specific, command that implements the \'off\' action.") + }, + { + "pcmk_off_timeout",NULL, "time", NULL, "60s", NULL, + N_("Advanced use only: Specify an alternate timeout to use for off actions instead of stonith-timeout"), + N_("Some devices need much more/less time to complete than normal." + "Use this to specify an alternate, device-specific, timeout for \'off\' actions.") + }, + { + "pcmk_off_retries",NULL, "integer", NULL, "2", NULL, + N_("Advanced use only: The maximum number of times to retry the 'off' command within the timeout period"), + N_("Some devices do not support multiple connections." + " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining." + " Use this option to alter the number of times Pacemaker retries \'off\' actions before giving up.") + }, + { + "pcmk_on_action",NULL, "string", NULL, "on", NULL, + N_("Advanced use only: An alternate command to run instead of 'on'"), + N_("Some devices do not support the standard commands or may provide additional ones." + "Use this to specify an alternate, device-specific, command that implements the \'on\' action.") + }, + { + "pcmk_on_timeout",NULL, "time", NULL, "60s", NULL, + N_("Advanced use only: Specify an alternate timeout to use for on actions instead of stonith-timeout"), + N_("Some devices need much more/less time to complete than normal." + "Use this to specify an alternate, device-specific, timeout for \'on\' actions.") + }, + { + "pcmk_on_retries",NULL, "integer", NULL, "2", NULL, + N_("Advanced use only: The maximum number of times to retry the 'on' command within the timeout period"), + N_("Some devices do not support multiple connections." + " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining." + " Use this option to alter the number of times Pacemaker retries \'on\' actions before giving up.") + }, + { + "pcmk_list_action",NULL, "string", NULL, "list", NULL, + N_("Advanced use only: An alternate command to run instead of \'list\'"), + N_("Some devices do not support the standard commands or may provide additional ones." + "Use this to specify an alternate, device-specific, command that implements the \'list\' action.") + }, + { + "pcmk_list_timeout",NULL, "time", NULL, "60s", NULL, + N_("Advanced use only: Specify an alternate timeout to use for list actions instead of stonith-timeout"), + N_("Some devices need much more/less time to complete than normal." + "Use this to specify an alternate, device-specific, timeout for \'list\' actions.") + }, + { + "pcmk_list_retries",NULL, "integer", NULL, "2", NULL, + N_("Advanced use only: The maximum number of times to retry the \'list\' command within the timeout period"), + N_("Some devices do not support multiple connections." + " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining." + " Use this option to alter the number of times Pacemaker retries \'list\' actions before giving up.") + }, + { + "pcmk_monitor_action",NULL, "string", NULL, "monitor", NULL, + N_("Advanced use only: An alternate command to run instead of \'monitor\'"), + N_("Some devices do not support the standard commands or may provide additional ones." + "Use this to specify an alternate, device-specific, command that implements the \'monitor\' action.") + }, + { + "pcmk_monitor_timeout",NULL, "time", NULL, "60s", NULL, + N_("Advanced use only: Specify an alternate timeout to use for monitor actions instead of stonith-timeout"), + N_("Some devices need much more/less time to complete than normal.\n" + "Use this to specify an alternate, device-specific, timeout for \'monitor\' actions.") + }, + { + "pcmk_monitor_retries",NULL, "integer", NULL, "2", NULL, + N_("Advanced use only: The maximum number of times to retry the \'monitor\' command within the timeout period"), + N_("Some devices do not support multiple connections." + " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining." + " Use this option to alter the number of times Pacemaker retries \'monitor\' actions before giving up.") + }, + { + "pcmk_status_action",NULL, "string", NULL, "status", NULL, + N_("Advanced use only: An alternate command to run instead of \'status\'"), + N_("Some devices do not support the standard commands or may provide additional ones." + "Use this to specify an alternate, device-specific, command that implements the \'status\' action.") + }, + { + "pcmk_status_timeout",NULL, "time", NULL, "60s", NULL, + N_("Advanced use only: Specify an alternate timeout to use for status actions instead of stonith-timeout"), + N_("Some devices need much more/less time to complete than normal." + "Use this to specify an alternate, device-specific, timeout for \'status\' actions.") + }, + { + "pcmk_status_retries",NULL, "integer", NULL, "2", NULL, + N_("Advanced use only: The maximum number of times to retry the \'status\' command within the timeout period"), + N_("Some devices do not support multiple connections." + " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining." + " Use this option to alter the number of times Pacemaker retries \'status\' actions before giving up.") + }, +}; + +void +fencer_metadata(void) +{ + const char *desc_short = N_("Instance attributes available for all " + "\"stonith\"-class resources"); + const char *desc_long = N_("Instance attributes available for all \"stonith\"-" + "class resources and used by Pacemaker's fence " + "daemon, formerly known as stonithd"); + + gchar *s = pcmk__format_option_metadata("pacemaker-fenced", desc_short, + desc_long, fencer_options, + PCMK__NELEM(fencer_options)); + printf("%s", s); + g_free(s); +} + +static GOptionEntry entries[] = { + { "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &stand_alone, + "Deprecated (will be removed in a future release)", NULL }, + + { "stand-alone-w-cpg", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, + stand_alone_cpg_cb, "Intended for use in regression testing only", NULL }, + + { "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY, + &options.log_files, "Send logs to the additional named logfile", NULL }, + + { NULL } +}; + +static GOptionContext * +build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) +{ + GOptionContext *context = NULL; + + context = pcmk__build_arg_context(args, "text (default), xml", group, + "[metadata]"); + pcmk__add_main_args(context, entries); + return context; +} + +int +main(int argc, char **argv) +{ + int rc = pcmk_rc_ok; + crm_cluster_t *cluster = NULL; + crm_ipc_t *old_instance = NULL; + + GError *error = NULL; + + GOptionGroup *output_group = NULL; + pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY); + gchar **processed_args = pcmk__cmdline_preproc(argv, "l"); + GOptionContext *context = build_arg_context(args, &output_group); + + crm_log_preinit(NULL, argc, argv); + + pcmk__register_formats(output_group, formats); + if (!g_option_context_parse_strv(context, &processed_args, &error)) { + exit_code = CRM_EX_USAGE; + goto done; + } + + rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv); + if (rc != pcmk_rc_ok) { + exit_code = CRM_EX_ERROR; + g_set_error(&error, PCMK__EXITC_ERROR, exit_code, + "Error creating output format %s: %s", + args->output_ty, pcmk_rc_str(rc)); + goto done; + } + + if (args->version) { + out->version(out, false); + goto done; + } + + if ((g_strv_length(processed_args) >= 2) + && pcmk__str_eq(processed_args[1], "metadata", pcmk__str_none)) { + fencer_metadata(); + goto done; + } + + // Open additional log files + pcmk__add_logfiles(options.log_files, out); + + crm_log_init(NULL, LOG_INFO + args->verbosity, TRUE, + (args->verbosity > 0), argc, argv, FALSE); + + crm_notice("Starting Pacemaker fencer"); + + old_instance = crm_ipc_new("stonith-ng", 0); + if (old_instance == NULL) { + /* crm_ipc_new() will have already logged an error message with + * crm_err() + */ + exit_code = CRM_EX_FATAL; + goto done; + } + + if (crm_ipc_connect(old_instance)) { + // IPC endpoint already up + crm_ipc_close(old_instance); + crm_ipc_destroy(old_instance); + crm_err("pacemaker-fenced is already active, aborting startup"); + goto done; + } else { + // Not up or not authentic, we'll proceed either way + crm_ipc_destroy(old_instance); + old_instance = NULL; + } + + mainloop_add_signal(SIGTERM, stonith_shutdown); + + crm_peer_init(); + + fenced_data_set = pe_new_working_set(); + CRM_ASSERT(fenced_data_set != NULL); + + cluster = pcmk_cluster_new(); + + /* Initialize the logger prior to setup_cib(). update_cib_cache_cb() may + * call the "xml-patchset" message function, which needs the logger, after + * setup_cib() has run. + */ + rc = pcmk__log_output_new(&logger_out) != pcmk_rc_ok; + if (rc != pcmk_rc_ok) { + exit_code = CRM_EX_FATAL; + g_set_error(&error, PCMK__EXITC_ERROR, exit_code, + "Error creating output format log: %s", pcmk_rc_str(rc)); + goto done; + } + pe__register_messages(logger_out); + pcmk__register_lib_messages(logger_out); + pcmk__output_set_log_level(logger_out, LOG_TRACE); + fenced_data_set->priv = logger_out; + + if (!stand_alone) { +#if SUPPORT_COROSYNC + if (is_corosync_cluster()) { + cluster->destroy = stonith_peer_cs_destroy; + cluster->cpg.cpg_deliver_fn = stonith_peer_ais_callback; + cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership; + } +#endif // SUPPORT_COROSYNC + + crm_set_status_callback(&st_peer_update_callback); + + if (crm_cluster_connect(cluster) == FALSE) { + exit_code = CRM_EX_FATAL; + crm_crit("Cannot sign in to the cluster... terminating"); + goto done; + } + pcmk__str_update(&stonith_our_uname, cluster->uname); + + if (!options.no_cib_connect) { + setup_cib(); + } + + } else { + pcmk__str_update(&stonith_our_uname, "localhost"); + crm_warn("Stand-alone mode is deprecated and will be removed " + "in a future release"); + } + + init_device_list(); + init_topology_list(); + + pcmk__serve_fenced_ipc(&ipcs, &ipc_callbacks); + + // Create the mainloop and run it... + mainloop = g_main_loop_new(NULL, FALSE); + crm_notice("Pacemaker fencer successfully started and accepting connections"); + g_main_loop_run(mainloop); + +done: + g_strfreev(processed_args); + pcmk__free_arg_context(context); + + g_strfreev(options.log_files); + + stonith_cleanup(); + pcmk_cluster_free(cluster); + pe_free_working_set(fenced_data_set); + + pcmk__output_and_clear_error(&error, out); + + if (logger_out != NULL) { + logger_out->finish(logger_out, exit_code, true, NULL); + pcmk__output_free(logger_out); + } + + if (out != NULL) { + out->finish(out, exit_code, true, NULL); + pcmk__output_free(out); + } + + pcmk__unregister_formats(); + crm_exit(exit_code); +} diff --git a/daemons/fenced/pacemaker-fenced.h b/daemons/fenced/pacemaker-fenced.h new file mode 100644 index 0000000..a3d2e17 --- /dev/null +++ b/daemons/fenced/pacemaker-fenced.h @@ -0,0 +1,315 @@ +/* + * Copyright 2009-2023 the Pacemaker project contributors + * + * This source code is licensed under the GNU General Public License version 2 + * or later (GPLv2+) WITHOUT ANY WARRANTY. + */ + +#include <stdint.h> // uint32_t, uint64_t +#include <crm/common/mainloop.h> + +/*! + * \internal + * \brief Check whether target has already been fenced recently + * + * \param[in] tolerance Number of seconds to look back in time + * \param[in] target Name of node to search for + * \param[in] action Action we want to match + * + * \return TRUE if an equivalent fencing operation took place in the last + * \p tolerance seconds, FALSE otherwise + */ +gboolean stonith_check_fence_tolerance(int tolerance, const char *target, const char *action); + +typedef struct stonith_device_s { + char *id; + char *agent; + char *namespace; + + /*! list of actions that must execute on the target node. Used for unfencing */ + GString *on_target_actions; + GList *targets; + time_t targets_age; + gboolean has_attr_map; + + // Whether target's nodeid should be passed as a parameter to the agent + gboolean include_nodeid; + + /* whether the cluster should automatically unfence nodes with the device */ + gboolean automatic_unfencing; + guint priority; + + uint32_t flags; // Group of enum st_device_flags + + GHashTable *params; + GHashTable *aliases; + GList *pending_ops; + mainloop_timer_t *timer; + crm_trigger_t *work; + xmlNode *agent_metadata; + + /*! A verified device is one that has contacted the + * agent successfully to perform a monitor operation */ + gboolean verified; + + gboolean cib_registered; + gboolean api_registered; + gboolean dirty; +} stonith_device_t; + +/* These values are used to index certain arrays by "phase". Usually an + * operation has only one "phase", so phase is always zero. However, some + * reboots are remapped to "off" then "on", in which case "reboot" will be + * phase 0, "off" will be phase 1 and "on" will be phase 2. + */ +enum st_remap_phase { + st_phase_requested = 0, + st_phase_off = 1, + st_phase_on = 2, + st_phase_max = 3 +}; + +typedef struct remote_fencing_op_s { + /* The unique id associated with this operation */ + char *id; + /*! The node this operation will fence */ + char *target; + /*! The fencing action to perform on the target. (reboot, on, off) */ + char *action; + + /*! When was the fencing action recorded (seconds since epoch) */ + time_t created; + + /*! Marks if the final notifications have been sent to local stonith clients. */ + gboolean notify_sent; + /*! The number of query replies received */ + guint replies; + /*! The number of query replies expected */ + guint replies_expected; + /*! Does this node own control of this operation */ + gboolean owner; + /*! After query is complete, This the high level timer that expires the entire operation */ + guint op_timer_total; + /*! This timer expires the current fencing request. Many fencing + * requests may exist in a single operation */ + guint op_timer_one; + /*! This timer expires the query request sent out to determine + * what nodes are contain what devices, and who those devices can fence */ + guint query_timer; + /*! This is the default timeout to use for each fencing device if no + * custom timeout is received in the query. */ + gint base_timeout; + /*! This is the calculated total timeout an operation can take before + * expiring. This is calculated by adding together all the timeout + * values associated with the devices this fencing operation may call */ + gint total_timeout; + + /*! Requested fencing delay. + * Value -1 means disable any static/random fencing delays. */ + int delay; + + /*! Delegate is the node being asked to perform a fencing action + * on behalf of the node that owns the remote operation. Some operations + * will involve multiple delegates. This value represents the final delegate + * that is used. */ + char *delegate; + /*! The point at which the remote operation completed */ + time_t completed; + //! Group of enum stonith_call_options associated with this operation + uint32_t call_options; + + /*! The current state of the remote operation. This indicates + * what stage the op is in, query, exec, done, duplicate, failed. */ + enum op_state state; + /*! The node that owns the remote operation */ + char *originator; + /*! The local client id that initiated the fencing request */ + char *client_id; + /*! The client's call_id that initiated the fencing request */ + int client_callid; + /*! The name of client that initiated the fencing request */ + char *client_name; + /*! List of the received query results for all the nodes in the cpg group */ + GList *query_results; + /*! The original request that initiated the remote stonith operation */ + xmlNode *request; + + /*! The current topology level being executed */ + guint level; + /*! The current operation phase being executed */ + enum st_remap_phase phase; + + /*! Devices with automatic unfencing (always run if "on" requested, never if remapped) */ + GList *automatic_list; + /*! List of all devices at the currently executing topology level */ + GList *devices_list; + /*! Current entry in the topology device list */ + GList *devices; + + /*! List of duplicate operations attached to this operation. Once this operation + * completes, the duplicate operations will be closed out as well. */ + GList *duplicates; + + /*! The point at which the remote operation completed(nsec) */ + long long completed_nsec; + + /*! The (potentially intermediate) result of the operation */ + pcmk__action_result_t result; +} remote_fencing_op_t; + +void fenced_broadcast_op_result(const remote_fencing_op_t *op, bool op_merged); + +// Fencer-specific client flags +enum st_client_flags { + st_callback_unknown = UINT64_C(0), + st_callback_notify_fence = (UINT64_C(1) << 0), + st_callback_device_add = (UINT64_C(1) << 2), + st_callback_device_del = (UINT64_C(1) << 4), + st_callback_notify_history = (UINT64_C(1) << 5), + st_callback_notify_history_synced = (UINT64_C(1) << 6) +}; + +// How the user specified the target of a topology level +enum fenced_target_by { + fenced_target_by_unknown = -1, // Invalid or not yet parsed + fenced_target_by_name, // By target name + fenced_target_by_pattern, // By a pattern matching target names + fenced_target_by_attribute, // By a node attribute/value on target +}; + +/* + * Complex fencing requirements are specified via fencing topologies. + * A topology consists of levels; each level is a list of fencing devices. + * Topologies are stored in a hash table by node name. When a node needs to be + * fenced, if it has an entry in the topology table, the levels are tried + * sequentially, and the devices in each level are tried sequentially. + * Fencing is considered successful as soon as any level succeeds; + * a level is considered successful if all its devices succeed. + * Essentially, all devices at a given level are "and-ed" and the + * levels are "or-ed". + * + * This structure is used for the topology table entries. + * Topology levels start from 1, so levels[0] is unused and always NULL. + */ +typedef struct stonith_topology_s { + enum fenced_target_by kind; // How target was specified + + /*! Node name regex or attribute name=value for which topology applies */ + char *target; + char *target_value; + char *target_pattern; + char *target_attribute; + + /*! Names of fencing devices at each topology level */ + GList *levels[ST_LEVEL_MAX]; + +} stonith_topology_t; + +void init_device_list(void); +void free_device_list(void); +void init_topology_list(void); +void free_topology_list(void); +void free_stonith_remote_op_list(void); +void init_stonith_remote_op_hash_table(GHashTable **table); +void free_metadata_cache(void); +void fenced_unregister_handlers(void); + +uint64_t get_stonith_flag(const char *name); + +void stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags, + xmlNode *op_request, const char *remote_peer); + +int stonith_device_register(xmlNode *msg, gboolean from_cib); + +void stonith_device_remove(const char *id, bool from_cib); + +char *stonith_level_key(const xmlNode *msg, enum fenced_target_by); +void fenced_register_level(xmlNode *msg, char **desc, + pcmk__action_result_t *result); +void fenced_unregister_level(xmlNode *msg, char **desc, + pcmk__action_result_t *result); + +stonith_topology_t *find_topology_for_host(const char *host); + +void do_local_reply(xmlNode *notify_src, pcmk__client_t *client, + int call_options); + +xmlNode *fenced_construct_reply(const xmlNode *request, xmlNode *data, + const pcmk__action_result_t *result); + +void + do_stonith_async_timeout_update(const char *client, const char *call_id, int timeout); + +void fenced_send_notification(const char *type, + const pcmk__action_result_t *result, + xmlNode *data); +void fenced_send_device_notification(const char *op, + const pcmk__action_result_t *result, + const char *desc); +void fenced_send_level_notification(const char *op, + const pcmk__action_result_t *result, + const char *desc); + +remote_fencing_op_t *initiate_remote_stonith_op(const pcmk__client_t *client, + xmlNode *request, + gboolean manual_ack); + +void fenced_process_fencing_reply(xmlNode *msg); + +int process_remote_stonith_query(xmlNode * msg); + +void *create_remote_stonith_op(const char *client, xmlNode * request, gboolean peer); + +void stonith_fence_history(xmlNode *msg, xmlNode **output, + const char *remote_peer, int options); + +void stonith_fence_history_trim(void); + +bool fencing_peer_active(crm_node_t *peer); + +void set_fencing_completed(remote_fencing_op_t * op); + +int fenced_handle_manual_confirmation(const pcmk__client_t *client, + xmlNode *msg); +void fencer_metadata(void); + +const char *fenced_device_reboot_action(const char *device_id); +bool fenced_device_supports_on(const char *device_id); + +gboolean node_has_attr(const char *node, const char *name, const char *value); + +gboolean node_does_watchdog_fencing(const char *node); + +static inline void +fenced_set_protocol_error(pcmk__action_result_t *result) +{ + pcmk__set_result(result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID, + "Fencer API request missing required information (bug?)"); +} + +/*! + * \internal + * \brief Get the device flag to use with a given action when searching devices + * + * \param[in] action Action to check + * + * \return st_device_supports_on if \p action is "on", otherwise + * st_device_supports_none + */ +static inline uint32_t +fenced_support_flag(const char *action) +{ + if (pcmk__str_eq(action, "on", pcmk__str_none)) { + return st_device_supports_on; + } + return st_device_supports_none; +} + +extern char *stonith_our_uname; +extern gboolean stand_alone; +extern GHashTable *device_list; +extern GHashTable *topology; +extern long stonith_watchdog_timeout_ms; +extern GList *stonith_watchdog_targets; + +extern GHashTable *stonith_remote_op_list; |