Adding upstream version 2.1.6.upstream/2.1.6

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

53 files changed, 21434 insertions, 0 deletions

diff --git a/tools/Makefile.am b/tools/Makefile.am
new file mode 100644
index 0000000..36bd3ae
--- /dev/null
+++ b/tools/Makefile.am
@@ -0,0 +1,165 @@
+#
+# Copyright 2004-2023 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+include $(top_srcdir)/mk/common.mk
+include $(top_srcdir)/mk/man.mk
+
+if BUILD_SYSTEMD
+systemdsystemunit_DATA	= crm_mon.service
+endif
+
+noinst_HEADERS		= crm_mon.h crm_resource.h
+
+pcmkdir			= $(datadir)/$(PACKAGE)
+pcmk_DATA		= report.common report.collector
+
+sbin_SCRIPTS		= crm_report crm_standby crm_master crm_failcount
+if BUILD_CIBSECRETS
+sbin_SCRIPTS		+= cibsecret
+endif
+noinst_SCRIPTS		= cluster-clean		\
+			  cluster-init		\
+			  cluster-helper	\
+			  pcmk_simtimes
+
+EXTRA_DIST		= attrd_updater.8.inc \
+			  cibadmin.8.inc \
+				  crm_attribute.8.inc \
+				  crm_diff.8.inc \
+				  crm_error.8.inc \
+			  crm_mon.8.inc			\
+			  crm_node.8.inc \
+			  crm_resource.8.inc \
+			  crm_rule.8.inc \
+			  crm_shadow.8.inc \
+			  crm_simulate.8.inc \
+			  crm_ticket.8.inc \
+			  crm_verify.8.inc \
+			  crmadmin.8.inc \
+			  fix-manpages \
+			  iso8601.8.inc \
+			  stonith_admin.8.inc
+
+sbin_PROGRAMS		= attrd_updater \
+			  cibadmin \
+			  crmadmin \
+			  crm_simulate \
+			  crm_attribute \
+			  crm_diff \
+			  crm_error \
+			  crm_mon \
+			  crm_node \
+			  crm_resource \
+			  crm_rule \
+			  crm_shadow \
+			  crm_verify \
+			  crm_ticket \
+			  iso8601 \
+			  stonith_admin
+
+## SOURCES
+
+# A few tools are just thin wrappers around crm_attribute.
+# This makes their help get updated when crm_attribute changes
+# (see mk/common.mk).
+MAN8DEPS		= crm_attribute
+
+crmadmin_SOURCES	= crmadmin.c
+crmadmin_LDADD		= $(top_builddir)/lib/pengine/libpe_status.la	\
+			  $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/common/libcrmcommon.la	\
+			  $(top_builddir)/lib/pacemaker/libpacemaker.la
+
+crm_error_SOURCES	= crm_error.c
+crm_error_LDADD		= $(top_builddir)/lib/pacemaker/libpacemaker.la	\
+			  $(top_builddir)/lib/common/libcrmcommon.la
+
+cibadmin_SOURCES	= cibadmin.c
+cibadmin_LDADD		= $(top_builddir)/lib/pacemaker/libpacemaker.la		\
+			  	$(top_builddir)/lib/cib/libcib.la		\
+				$(top_builddir)/lib/common/libcrmcommon.la
+
+crm_shadow_SOURCES	= crm_shadow.c
+crm_shadow_LDADD	= $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/common/libcrmcommon.la
+
+crm_node_SOURCES	= crm_node.c
+crm_node_LDADD		= $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/common/libcrmcommon.la
+
+crm_simulate_SOURCES	= crm_simulate.c
+
+crm_simulate_LDADD	= $(top_builddir)/lib/pengine/libpe_status.la	\
+			  $(top_builddir)/lib/pacemaker/libpacemaker.la	\
+			  $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/common/libcrmcommon.la
+
+crm_diff_SOURCES	= crm_diff.c
+crm_diff_LDADD		= $(top_builddir)/lib/common/libcrmcommon.la
+
+crm_mon_SOURCES		= crm_mon.c crm_mon_curses.c
+crm_mon_LDADD		= $(top_builddir)/lib/pengine/libpe_status.la	\
+			  $(top_builddir)/lib/fencing/libstonithd.la	\
+			  $(top_builddir)/lib/pacemaker/libpacemaker.la	\
+			  $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/common/libcrmcommon.la	\
+			  $(CURSESLIBS)
+
+crm_verify_SOURCES	= crm_verify.c
+crm_verify_LDADD	= $(top_builddir)/lib/pengine/libpe_status.la 	\
+			  $(top_builddir)/lib/pacemaker/libpacemaker.la	\
+			  $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/common/libcrmcommon.la
+
+crm_attribute_SOURCES	= crm_attribute.c
+crm_attribute_LDADD	= $(top_builddir)/lib/pacemaker/libpacemaker.la	\
+			  $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/common/libcrmcommon.la
+
+crm_resource_SOURCES	= crm_resource.c		\
+			  crm_resource_ban.c		\
+			  crm_resource_print.c		\
+			  crm_resource_runtime.c
+crm_resource_LDADD	= $(top_builddir)/lib/pengine/libpe_rules.la  	\
+			  $(top_builddir)/lib/fencing/libstonithd.la	\
+			  $(top_builddir)/lib/lrmd/liblrmd.la 		\
+			  $(top_builddir)/lib/services/libcrmservice.la \
+			  $(top_builddir)/lib/pengine/libpe_status.la 	\
+			  $(top_builddir)/lib/pacemaker/libpacemaker.la	\
+			  $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/common/libcrmcommon.la
+
+crm_rule_SOURCES 	= crm_rule.c
+crm_rule_LDADD		= $(top_builddir)/lib/pacemaker/libpacemaker.la \
+			  $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/pengine/libpe_rules.la   \
+			  $(top_builddir)/lib/pengine/libpe_status.la   \
+			  $(top_builddir)/lib/common/libcrmcommon.la
+
+iso8601_SOURCES		= iso8601.c
+iso8601_LDADD		= $(top_builddir)/lib/common/libcrmcommon.la
+
+attrd_updater_SOURCES	= attrd_updater.c
+attrd_updater_LDADD	= $(top_builddir)/lib/pacemaker/libpacemaker.la	\
+			  $(top_builddir)/lib/common/libcrmcommon.la
+
+crm_ticket_SOURCES	= crm_ticket.c
+crm_ticket_LDADD	= $(top_builddir)/lib/pengine/libpe_rules.la	\
+			  $(top_builddir)/lib/pengine/libpe_status.la	\
+			  $(top_builddir)/lib/pacemaker/libpacemaker.la	\
+			  $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/common/libcrmcommon.la
+
+stonith_admin_SOURCES	= stonith_admin.c
+stonith_admin_LDADD	= $(top_builddir)/lib/pacemaker/libpacemaker.la	\
+			  $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/pengine/libpe_status.la	\
+			  $(top_builddir)/lib/fencing/libstonithd.la \
+			  $(top_builddir)/lib/common/libcrmcommon.la
+
+CLEANFILES = $(man8_MANS)
diff --git a/tools/attrd_updater.8.inc b/tools/attrd_updater.8.inc
new file mode 100644
index 0000000..780e6a4
--- /dev/null
+++ b/tools/attrd_updater.8.inc
@@ -0,0 +1,5 @@
+[synopsis]
+attrd_updater {-n <attribute> | -P <pattern>} <command> [options]
+
+/node attributes/
+.SH OPTIONS
diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c
new file mode 100644
index 0000000..60e4cc7
--- /dev/null
+++ b/tools/attrd_updater.c
@@ -0,0 +1,520 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <libgen.h>
+
+#include <sys/param.h>
+#include <sys/types.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/ipc_attrd_internal.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/output_internal.h>
+#include <crm/common/xml_internal.h>
+
+#include <crm/common/attrd_internal.h>
+
+#include <pcmki/pcmki_output.h>
+
+#define SUMMARY "query and update Pacemaker node attributes"
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+GError *error = NULL;
+bool printed_values = false;
+
+struct {
+    char command;
+    gchar *attr_dampen;
+    gchar *attr_name;
+    gchar *attr_pattern;
+    gchar *attr_node;
+    gchar *attr_set;
+    char *attr_value;
+    uint32_t attr_options;
+    gboolean query_all;
+    gboolean quiet;
+} options = {
+    .attr_options = pcmk__node_attr_none,
+    .command = 'Q',
+};
+
+static gboolean
+command_cb (const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    pcmk__str_update(&options.attr_value, optarg);
+
+    if (pcmk__str_any_of(option_name, "--update-both", "-B", NULL)) {
+        options.command = 'B';
+    } else if (pcmk__str_any_of(option_name, "--delete", "-D", NULL)) {
+        options.command = 'D';
+    } else if (pcmk__str_any_of(option_name, "--query", "-Q", NULL)) {
+        options.command = 'Q';
+    } else if (pcmk__str_any_of(option_name, "--refresh", "-R", NULL)) {
+        options.command = 'R';
+    } else if (pcmk__str_any_of(option_name, "--update", "-U", "-v", NULL)) {
+        options.command = 'U';
+    } else if (pcmk__str_any_of(option_name, "--update-delay", "-Y", NULL)) {
+        options.command = 'Y';
+    }
+
+    return TRUE;
+}
+
+static gboolean
+private_cb (const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_private);
+    return TRUE;
+}
+
+static gboolean
+section_cb (const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    if (pcmk__str_any_of(optarg, "nodes", "forever", NULL)) {
+        pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_perm);
+    } else if (pcmk__str_any_of(optarg, "status", "reboot", NULL)) {
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_perm);
+    } else {
+        g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_USAGE, "Unknown value for --lifetime: %s",
+                    optarg);
+        return FALSE;
+    }
+
+    return TRUE;
+}
+
+static gboolean
+attr_set_type_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_any_of(option_name, "-z", "--utilization", NULL)) {
+        pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_utilization);
+    }
+
+    return TRUE;
+}
+
+static gboolean
+wait_cb (const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    if (pcmk__str_eq(optarg, "no", pcmk__str_none)) {
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
+        return TRUE;
+    } else if (pcmk__str_eq(optarg, PCMK__VALUE_LOCAL, pcmk__str_none)) {
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
+        pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local);
+        return TRUE;
+    } else if (pcmk__str_eq(optarg, PCMK__VALUE_CLUSTER, pcmk__str_none)) {
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
+        pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_sync_cluster);
+        return TRUE;
+    } else {
+        g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                    "--wait= must be one of 'no', 'local', 'cluster'");
+        return FALSE;
+    }
+}
+
+#define INDENT "                              "
+
+static GOptionEntry required_entries[] = {
+    { "name", 'n', 0, G_OPTION_ARG_STRING, &options.attr_name,
+      "The attribute's name",
+      "NAME" },
+
+    { "pattern", 'P', 0, G_OPTION_ARG_STRING, &options.attr_pattern,
+      "Operate on all attributes matching this pattern\n"
+      INDENT "(with -B, -D, -U, or -Y)",
+      "PATTERN"
+    },
+
+    { NULL }
+};
+
+static GOptionEntry command_entries[] = {
+    { "update", 'U', 0, G_OPTION_ARG_CALLBACK, command_cb,
+      "Update attribute's value in pacemaker-attrd. If this causes the value\n"
+      INDENT "to change, it will also be updated in the cluster configuration.",
+      "VALUE" },
+
+    { "update-both", 'B', 0, G_OPTION_ARG_CALLBACK, command_cb,
+      "Update attribute's value and time to wait (dampening) in\n"
+      INDENT "pacemaker-attrd. If this causes the value or dampening to change,\n"
+      INDENT "the attribute will also be written to the cluster configuration,\n"
+      INDENT "so be aware that repeatedly changing the dampening reduces its\n"
+      INDENT "effectiveness.\n"
+      INDENT "Requires -d/--delay",
+      "VALUE" },
+
+    { "update-delay", 'Y', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Update attribute's dampening in pacemaker-attrd. If this causes\n"
+      INDENT "the dampening to change, the attribute will also be written\n"
+      INDENT "to the cluster configuration, so be aware that repeatedly\n"
+      INDENT "changing the dampening reduces its effectiveness.\n"
+      INDENT "Requires -d/--delay",
+      NULL },
+
+    { "query", 'Q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Query the attribute's value from pacemaker-attrd",
+      NULL },
+
+    { "delete", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Unset attribute from pacemaker-attrd. At the moment, there is no way\n"
+      INDENT "to remove an attribute. This option will instead set its value\n"
+      INDENT "to the empty string.",
+      NULL },
+
+    { "refresh", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "(Advanced) Force the pacemaker-attrd daemon to resend all current\n"
+      INDENT "values to the CIB",
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry addl_entries[] = {
+    { "delay", 'd', 0, G_OPTION_ARG_STRING, &options.attr_dampen,
+      "The time to wait (dampening) in seconds for further changes\n"
+      INDENT "before sending to the CIB",
+      "SECONDS" },
+
+    { "set", 's', 0, G_OPTION_ARG_STRING, &options.attr_set,
+      "(Advanced) The attribute set in which to place the value",
+      "SET" },
+
+    { "node", 'N', 0, G_OPTION_ARG_STRING, &options.attr_node,
+      "Set the attribute for the named node (instead of the local one)",
+      "NODE" },
+
+    { "all", 'A', 0, G_OPTION_ARG_NONE, &options.query_all,
+      "Show values of the attribute for all nodes (query only)",
+      NULL },
+
+    { "lifetime", 'l', 0, G_OPTION_ARG_CALLBACK, section_cb,
+      "(Not yet implemented) Lifetime of the node attribute (silently\n"
+      INDENT "ignored by cluster)",
+      "SECTION" },
+
+    { "private", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, private_cb,
+      "If this creates a new attribute, never write the attribute to CIB",
+      NULL },
+
+    { "wait", 'W', 0, G_OPTION_ARG_CALLBACK, wait_cb,
+      "Wait for some event to occur before returning.  Values are 'no' (wait\n"
+      INDENT "only for the attribute daemon to acknowledge the request),\n"
+      INDENT "'local' (wait until the change has propagated to where a local\n"
+      INDENT "query will return the request value, or the value set by a\n"
+      INDENT "later request), or 'cluster' (wait until the change has propagated\n"
+      INDENT "to where a query anywhere on the cluster will return the requested\n"
+      INDENT "value, or the value set by a later request).  Default is 'no'.",
+      "UNTIL" },
+
+    { "utilization", 'z', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
+      "When creating a new attribute, create it as a node utilization attribute\n"
+      INDENT "instead of an instance attribute.  If the attribute already exists,\n"
+      INDENT "its existing type (utilization vs. instance) will be used regardless.\n"
+      INDENT "(with -B, -U, -Y)",
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry deprecated_entries[] = {
+    { "quiet", 'q', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.quiet,
+      NULL,
+      NULL },
+
+    { "update", 'v', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, command_cb,
+      NULL,
+      NULL },
+
+    { "section", 'S', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, section_cb,
+      NULL,
+      NULL },
+
+    { NULL }
+};
+
+static int send_attrd_query(pcmk__output_t *out, const char *attr_name, const char *attr_node,
+                    gboolean query_all);
+static int send_attrd_update(char command, const char *attr_node, const char *attr_name,
+                             const char *attr_value, const char *attr_set,
+                             const char *attr_dampen, uint32_t attr_options);
+
+static bool
+pattern_used_correctly(void)
+{
+    /* --pattern can only be used with:
+     * -B (update-both), -D (delete), -U (update), or -Y (update-delay)
+     */
+    return options.command == 'B' || options.command == 'D' || options.command == 'U' || options.command == 'Y';
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
+
+    pcmk__add_arg_group(context, "required", "Required Arguments:",
+                        "Show required arguments", required_entries);
+    pcmk__add_arg_group(context, "command", "Command:",
+                        "Show command options (mutually exclusive)", command_entries);
+    pcmk__add_arg_group(context, "additional", "Additional Options:",
+                        "Show additional options", addl_entries);
+    pcmk__add_arg_group(context, "deprecated", "Deprecated Options:",
+                        "Show deprecated options", deprecated_entries);
+
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    int rc = pcmk_rc_ok;
+    crm_exit_t exit_code = CRM_EX_OK;
+
+    pcmk__output_t *out = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    GOptionContext *context = build_arg_context(args, &output_group);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "dlnsvBNUS");
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("attrd_updater", args->verbosity);
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    if (options.attr_pattern) {
+        if (options.attr_name) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Error: --name and --pattern cannot be used at the same time");
+            goto done;
+        }
+
+        if (!pattern_used_correctly()) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Error: pattern can only be used with delete or update");
+            goto done;
+        }
+
+        g_free(options.attr_name);
+        options.attr_name = options.attr_pattern;
+        options.attr_options |= pcmk__node_attr_pattern;
+    }
+
+    if (options.command != 'R' && options.attr_name == NULL) {
+        exit_code = CRM_EX_USAGE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Command requires --name or --pattern argument");
+        goto done;
+    } else if ((options.command == 'B'|| options.command == 'Y') && options.attr_dampen == NULL) {
+        out->info(out, "Warning: '%c' command given without required --delay", options.command);
+    }
+
+    pcmk__register_lib_messages(out);
+
+    if (options.command == 'Q') {
+        int rc = send_attrd_query(out, options.attr_name, options.attr_node, options.query_all);
+        exit_code = pcmk_rc2exitc(rc);
+    } else {
+        /* @TODO We don't know whether the specified node is a Pacemaker Remote
+         * node or not, so we can't set pcmk__node_attr_remote when appropriate.
+         * However, it's not a big problem, because pacemaker-attrd will learn
+         * and remember a node's "remoteness".
+         */
+        int rc = send_attrd_update(options.command, options.attr_node,
+                                   options.attr_name, options.attr_value,
+                                   options.attr_set, options.attr_dampen,
+                                   options.attr_options);
+        exit_code = pcmk_rc2exitc(rc);
+    }
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+    g_free(options.attr_dampen);
+    g_free(options.attr_name);
+    g_free(options.attr_node);
+    g_free(options.attr_set);
+    free(options.attr_value);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+
+    pcmk__unregister_formats();
+    crm_exit(exit_code);
+}
+
+/*!
+ * \brief Print the attribute values in a pacemaker-attrd XML query reply
+ *
+ * \param[in,out] out    Output object
+ * \param[in]     reply  List of attribute name/value pairs
+ *
+ * \return true if any values were printed
+ */
+static void
+print_attrd_values(pcmk__output_t *out, const GList *reply)
+{
+    for (const GList *iter = reply; iter != NULL; iter = iter->next) {
+        const pcmk__attrd_query_pair_t *pair = iter->data;
+
+        out->message(out, "attribute", NULL, NULL, pair->name, pair->value,
+                     pair->node);
+        printed_values = true;
+    }
+}
+
+static void
+attrd_event_cb(pcmk_ipc_api_t *attrd_api, enum pcmk_ipc_event event_type,
+               crm_exit_t status, void *event_data, void *user_data)
+{
+    pcmk__output_t *out = (pcmk__output_t *) user_data;
+    pcmk__attrd_api_reply_t *reply = event_data;
+
+    if (event_type != pcmk_ipc_event_reply || status != CRM_EX_OK) {
+        return;
+    }
+
+    /* Print the values from the reply. */
+    if (reply->reply_type == pcmk__attrd_reply_query) {
+        print_attrd_values(out, reply->data.pairs);
+    }
+}
+
+/*!
+ * \brief Submit a query to pacemaker-attrd and print reply
+ *
+ * \param[in,out] out  Output object
+ * \param[in]     attr_name  Name of attribute to be affected by request
+ * \param[in]     attr_node  Name of host to query for (or NULL for localhost)
+ * \param[in]     query_all  If TRUE, ignore attr_node and query all nodes
+ *
+ * \return Standard Pacemaker return code
+ */
+static int
+send_attrd_query(pcmk__output_t *out, const char *attr_name,
+                 const char *attr_node, gboolean query_all)
+{
+    uint32_t options = pcmk__node_attr_none;
+    pcmk_ipc_api_t *attrd_api = NULL;
+    int rc = pcmk_rc_ok;
+
+    // Create attrd IPC object
+    rc = pcmk_new_ipc_api(&attrd_api, pcmk_ipc_attrd);
+    if (rc != pcmk_rc_ok) {
+        g_set_error(&error, PCMK__RC_ERROR, rc,
+                    "Could not connect to attrd: %s", pcmk_rc_str(rc));
+        return ENOTCONN;
+    }
+
+    pcmk_register_ipc_callback(attrd_api, attrd_event_cb, out);
+
+    // Connect to attrd (without main loop)
+    rc = pcmk_connect_ipc(attrd_api, pcmk_ipc_dispatch_sync);
+    if (rc != pcmk_rc_ok) {
+        g_set_error(&error, PCMK__RC_ERROR, rc,
+                    "Could not connect to attrd: %s", pcmk_rc_str(rc));
+        pcmk_free_ipc_api(attrd_api);
+        return rc;
+    }
+
+    /* Decide which node(s) to query */
+    if (query_all == TRUE) {
+        options |= pcmk__node_attr_query_all;
+    }
+
+    rc = pcmk__attrd_api_query(attrd_api, attr_node, attr_name, options);
+
+    if (rc != pcmk_rc_ok) {
+        g_set_error(&error, PCMK__RC_ERROR, rc, "Could not query value of %s: %s (%d)",
+                    attr_name, pcmk_strerror(rc), rc);
+    } else if (!printed_values) {
+        rc = pcmk_rc_schema_validation;
+        g_set_error(&error, PCMK__RC_ERROR, rc,
+                    "Could not query value of %s: attribute does not exist", attr_name);
+    }
+
+    pcmk_disconnect_ipc(attrd_api);
+    pcmk_free_ipc_api(attrd_api);
+
+    return rc;
+}
+
+static int
+send_attrd_update(char command, const char *attr_node, const char *attr_name,
+                  const char *attr_value, const char *attr_set,
+                  const char *attr_dampen, uint32_t attr_options)
+{
+    int rc = pcmk_rc_ok;
+
+    switch (command) {
+        case 'B':
+            rc = pcmk__attrd_api_update(NULL, attr_node, attr_name, attr_value,
+                                        attr_dampen, attr_set, NULL,
+                                        attr_options | pcmk__node_attr_value | pcmk__node_attr_delay);
+            break;
+
+        case 'D':
+            rc = pcmk__attrd_api_delete(NULL, attr_node, attr_name, attr_options);
+            break;
+
+        case 'R':
+            rc = pcmk__attrd_api_refresh(NULL, attr_node);
+            break;
+
+        case 'U':
+            rc = pcmk__attrd_api_update(NULL, attr_node, attr_name, attr_value,
+                                        NULL, attr_set, NULL,
+                                        attr_options | pcmk__node_attr_value);
+            break;
+
+        case 'Y':
+            rc = pcmk__attrd_api_update(NULL, attr_node, attr_name, NULL,
+                                        attr_dampen, attr_set, NULL,
+                                        attr_options | pcmk__node_attr_delay);
+            break;
+    }
+
+    if (rc != pcmk_rc_ok) {
+        g_set_error(&error, PCMK__RC_ERROR, rc, "Could not update %s=%s: %s (%d)",
+                    attr_name, attr_value, pcmk_rc_str(rc), rc);
+    }
+
+    return rc;
+}
diff --git a/tools/cibadmin.8.inc b/tools/cibadmin.8.inc
new file mode 100644
index 0000000..bfd969b
--- /dev/null
+++ b/tools/cibadmin.8.inc
@@ -0,0 +1,5 @@
+[=synopsis]
+cibadmin <command> [options]
+
+/query and edit the Pacemaker configuration/
+.SH OPTIONS
diff --git a/tools/cibadmin.c b/tools/cibadmin.c
new file mode 100644
index 0000000..f80afae
--- /dev/null
+++ b/tools/cibadmin.c
@@ -0,0 +1,954 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include <stdio.h>
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/ipc.h>
+#include <crm/common/xml.h>
+#include <crm/cib/internal.h>
+
+#include <pacemaker-internal.h>
+
+#define SUMMARY "query and edit the Pacemaker configuration"
+
+#define INDENT "                                "
+
+enum cibadmin_section_type {
+    cibadmin_section_all = 0,
+    cibadmin_section_scope,
+    cibadmin_section_xpath,
+};
+
+static int request_id = 0;
+
+static cib_t *the_cib = NULL;
+static GMainLoop *mainloop = NULL;
+static crm_exit_t exit_code = CRM_EX_OK;
+
+static struct {
+    const char *cib_action;
+    int cmd_options;
+    enum cibadmin_section_type section_type;
+    char *cib_section;
+    char *validate_with;
+    gint message_timeout_sec;
+    enum pcmk__acl_render_how acl_render_mode;
+    gchar *cib_user;
+    gchar *dest_node;
+    gchar *input_file;
+    gchar *input_xml;
+    gboolean input_stdin;
+    bool delete_all;
+    gboolean allow_create;
+    gboolean force;
+    gboolean get_node_path;
+    gboolean local;
+    gboolean no_children;
+    gboolean sync_call;
+
+    /* @COMPAT: For "-!" version option. Not advertised nor marked as
+     * deprecated, but accepted.
+     */
+    gboolean extended_version;
+
+    //! \deprecated
+    gboolean no_bcast;
+} options;
+
+int do_init(void);
+static int do_work(xmlNode *input, xmlNode **output);
+void cibadmin_op_callback(xmlNode *msg, int call_id, int rc, xmlNode *output,
+                          void *user_data);
+
+static void
+print_xml_output(xmlNode * xml)
+{
+    char *buffer;
+
+    if (!xml) {
+        return;
+    } else if (xml->type != XML_ELEMENT_NODE) {
+        return;
+    }
+
+    if (pcmk_is_set(options.cmd_options, cib_xpath_address)) {
+        const char *id = crm_element_value(xml, XML_ATTR_ID);
+
+        if (pcmk__str_eq((const char *)xml->name, "xpath-query", pcmk__str_casei)) {
+            xmlNode *child = NULL;
+
+            for (child = xml->children; child; child = child->next) {
+                print_xml_output(child);
+            }
+
+        } else if (id) {
+            printf("%s\n", id);
+        }
+
+    } else {
+        buffer = dump_xml_formatted(xml);
+        fprintf(stdout, "%s", pcmk__s(buffer, "<null>\n"));
+        free(buffer);
+    }
+}
+
+// Upgrade requested but already at latest schema
+static void
+report_schema_unchanged(void)
+{
+    const char *err = pcmk_rc_str(pcmk_rc_schema_unchanged);
+
+    crm_info("Upgrade unnecessary: %s\n", err);
+    printf("Upgrade unnecessary: %s\n", err);
+    exit_code = CRM_EX_OK;
+}
+
+/*!
+ * \internal
+ * \brief Check whether the current CIB action is dangerous
+ * \return true if \p options.cib_action is dangerous, or false otherwise
+ */
+static inline bool
+cib_action_is_dangerous(void)
+{
+    return options.no_bcast || options.delete_all
+           || pcmk__str_any_of(options.cib_action,
+                               PCMK__CIB_REQUEST_UPGRADE,
+                               PCMK__CIB_REQUEST_ERASE,
+                               NULL);
+}
+
+/*!
+ * \internal
+ * \brief Determine whether the given CIB scope is valid for \p cibadmin
+ *
+ * \param[in] scope  Scope to validate
+ *
+ * \return true if \p scope is valid, or false otherwise
+ * \note An invalid scope applies the operation to the entire CIB.
+ */
+static inline bool
+scope_is_valid(const char *scope)
+{
+    return pcmk__str_any_of(scope,
+                            XML_CIB_TAG_CONFIGURATION,
+                            XML_CIB_TAG_NODES,
+                            XML_CIB_TAG_RESOURCES,
+                            XML_CIB_TAG_CONSTRAINTS,
+                            XML_CIB_TAG_CRMCONFIG,
+                            XML_CIB_TAG_RSCCONFIG,
+                            XML_CIB_TAG_OPCONFIG,
+                            XML_CIB_TAG_ACLS,
+                            XML_TAG_FENCING_TOPOLOGY,
+                            XML_CIB_TAG_TAGS,
+                            XML_CIB_TAG_ALERTS,
+                            XML_CIB_TAG_STATUS,
+                            NULL);
+}
+
+static gboolean
+command_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+           GError **error)
+{
+    options.delete_all = false;
+
+    if (pcmk__str_any_of(option_name, "-u", "--upgrade", NULL)) {
+        options.cib_action = PCMK__CIB_REQUEST_UPGRADE;
+
+    } else if (pcmk__str_any_of(option_name, "-Q", "--query", NULL)) {
+        options.cib_action = PCMK__CIB_REQUEST_QUERY;
+
+    } else if (pcmk__str_any_of(option_name, "-E", "--erase", NULL)) {
+        options.cib_action = PCMK__CIB_REQUEST_ERASE;
+
+    } else if (pcmk__str_any_of(option_name, "-B", "--bump", NULL)) {
+        options.cib_action = PCMK__CIB_REQUEST_BUMP;
+
+    } else if (pcmk__str_any_of(option_name, "-C", "--create", NULL)) {
+        options.cib_action = PCMK__CIB_REQUEST_CREATE;
+
+    } else if (pcmk__str_any_of(option_name, "-M", "--modify", NULL)) {
+        options.cib_action = PCMK__CIB_REQUEST_MODIFY;
+
+    } else if (pcmk__str_any_of(option_name, "-P", "--patch", NULL)) {
+        options.cib_action = PCMK__CIB_REQUEST_APPLY_PATCH;
+
+    } else if (pcmk__str_any_of(option_name, "-R", "--replace", NULL)) {
+        options.cib_action = PCMK__CIB_REQUEST_REPLACE;
+
+    } else if (pcmk__str_any_of(option_name, "-D", "--delete", NULL)) {
+        options.cib_action = PCMK__CIB_REQUEST_DELETE;
+
+    } else if (pcmk__str_any_of(option_name, "-d", "--delete-all", NULL)) {
+        options.cib_action = PCMK__CIB_REQUEST_DELETE;
+        options.delete_all = true;
+
+    } else if (pcmk__str_any_of(option_name, "-a", "--empty", NULL)) {
+        options.cib_action = "empty";
+        pcmk__str_update(&options.validate_with, optarg);
+
+    } else if (pcmk__str_any_of(option_name, "-5", "--md5-sum", NULL)) {
+        options.cib_action = "md5-sum";
+
+    } else if (pcmk__str_any_of(option_name, "-6", "--md5-sum-versioned",
+                                NULL)) {
+        options.cib_action = "md5-sum-versioned";
+
+    } else {
+        // Should be impossible
+        return FALSE;
+    }
+
+    return TRUE;
+}
+
+static gboolean
+show_access_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+               GError **error)
+{
+    if (pcmk__str_eq(optarg, "auto", pcmk__str_null_matches)) {
+        options.acl_render_mode = pcmk__acl_render_default;
+
+    } else if (g_strcmp0(optarg, "namespace") == 0) {
+        options.acl_render_mode = pcmk__acl_render_namespace;
+
+    } else if (g_strcmp0(optarg, "text") == 0) {
+        options.acl_render_mode = pcmk__acl_render_text;
+
+    } else if (g_strcmp0(optarg, "color") == 0) {
+        options.acl_render_mode = pcmk__acl_render_color;
+
+    } else {
+        g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                    "Invalid value '%s' for option '%s'",
+                    optarg, option_name);
+        return FALSE;
+    }
+    return TRUE;
+}
+
+static gboolean
+section_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+           GError **error)
+{
+    if (pcmk__str_any_of(option_name, "-o", "--scope", NULL)) {
+        options.section_type = cibadmin_section_scope;
+
+    } else if (pcmk__str_any_of(option_name, "-A", "--xpath", NULL)) {
+        options.section_type = cibadmin_section_xpath;
+
+    } else {
+        // Should be impossible
+        return FALSE;
+    }
+
+    pcmk__str_update(&options.cib_section, optarg);
+    return TRUE;
+}
+
+static GOptionEntry command_entries[] = {
+    { "upgrade", 'u', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Upgrade the configuration to the latest syntax", NULL },
+
+    { "query", 'Q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Query the contents of the CIB", NULL },
+
+    { "erase", 'E', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Erase the contents of the whole CIB", NULL },
+
+    { "bump", 'B', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Increase the CIB's epoch value by 1", NULL },
+
+    { "create", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Create an object in the CIB (will fail if object already exists)",
+      NULL },
+
+    { "modify", 'M', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Find object somewhere in CIB's XML tree and update it (fails if object "
+      "does not exist unless -c is also specified)",
+      NULL },
+
+    { "patch", 'P', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Supply an update in the form of an XML diff (see crm_diff(8))", NULL },
+
+    { "replace", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Recursively replace an object in the CIB", NULL },
+
+    { "delete", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Delete first object matching supplied criteria (for example, "
+      "<" XML_ATTR_OP " " XML_ATTR_ID "=\"rsc1_op1\" "
+          XML_ATTR_NAME "=\"monitor\"/>).\n"
+      INDENT "The XML element name and all attributes must match in order for "
+      "the element to be deleted.",
+      NULL },
+
+    { "delete-all", 'd', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
+      command_cb,
+      "When used with --xpath, remove all matching objects in the "
+      "configuration instead of just the first one",
+      NULL },
+
+    { "empty", 'a', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
+      command_cb,
+      "Output an empty CIB. Accepts an optional schema name argument to use as "
+      "the " XML_ATTR_VALIDATION " value.\n"
+      INDENT "If no schema is given, the latest will be used.",
+      "[schema]" },
+
+    { "md5-sum", '5', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Calculate the on-disk CIB digest", NULL },
+
+    { "md5-sum-versioned", '6', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
+      command_cb, "Calculate an on-the-wire versioned CIB digest", NULL },
+
+    { NULL }
+};
+
+static GOptionEntry data_entries[] = {
+    /* @COMPAT: These arguments should be last-wins. We can have an enum option
+     * that stores the input type, along with a single string option that stores
+     * the XML string for --xml-text, filename for --xml-file, or NULL for
+     * --xml-pipe.
+     */
+    { "xml-text", 'X', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING,
+      &options.input_xml, "Retrieve XML from the supplied string", "value" },
+
+    { "xml-file", 'x', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME,
+      &options.input_file, "Retrieve XML from the named file", "value" },
+
+    { "xml-pipe", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
+      &options.input_stdin, "Retrieve XML from stdin", NULL },
+
+    { NULL }
+};
+
+static GOptionEntry addl_entries[] = {
+    { "force", 'f', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.force,
+      "Force the action to be performed", NULL },
+
+    { "timeout", 't', G_OPTION_FLAG_NONE, G_OPTION_ARG_INT,
+      &options.message_timeout_sec,
+      "Time (in seconds) to wait before declaring the operation failed",
+      "value" },
+
+    { "user", 'U', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.cib_user,
+      "Run the command with permissions of the named user (valid only for the "
+      "root and " CRM_DAEMON_USER " accounts)", "value" },
+
+    { "sync-call", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
+      &options.sync_call, "Wait for call to complete before returning", NULL },
+
+    { "local", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.local,
+      "Command takes effect locally (should be used only for queries)", NULL },
+
+    { "scope", 'o', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, section_cb,
+      "Limit scope of operation to specific section of CIB\n"
+      INDENT "Valid values: " XML_CIB_TAG_CONFIGURATION ", " XML_CIB_TAG_NODES
+      ", " XML_CIB_TAG_RESOURCES ", " XML_CIB_TAG_CONSTRAINTS
+      ", " XML_CIB_TAG_CRMCONFIG ", " XML_CIB_TAG_RSCCONFIG ",\n"
+      INDENT "              " XML_CIB_TAG_OPCONFIG ", " XML_CIB_TAG_ACLS
+      ", " XML_TAG_FENCING_TOPOLOGY ", " XML_CIB_TAG_TAGS
+      ", " XML_CIB_TAG_ALERTS ", " XML_CIB_TAG_STATUS "\n"
+      INDENT "If both --scope/-o and --xpath/-a are specified, the last one to "
+      "appear takes effect",
+      "value" },
+
+    { "xpath", 'A', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, section_cb,
+      "A valid XPath to use instead of --scope/-o\n"
+      INDENT "If both --scope/-o and --xpath/-a are specified, the last one to "
+      "appear takes effect",
+      "value" },
+
+    { "node-path", 'e', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
+      &options.get_node_path,
+      "When performing XPath queries, return paths of any matches found\n"
+      INDENT "(for example, "
+      "\"/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION
+      "/" XML_CIB_TAG_RESOURCES "/" XML_CIB_TAG_INCARNATION
+      "[@" XML_ATTR_ID "='dummy-clone']"
+      "/" XML_CIB_TAG_RESOURCE "[@" XML_ATTR_ID "='dummy']\")",
+      NULL },
+
+    { "show-access", 'S', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
+      show_access_cb,
+      "Whether to use syntax highlighting for ACLs (with -Q/--query and "
+      "-U/--user)\n"
+      INDENT "Allowed values: 'color' (default for terminal), 'text' (plain text, "
+      "default for non-terminal),\n"
+      INDENT "                'namespace', or 'auto' (use default value)\n"
+      INDENT "Default value: 'auto'",
+      "[value]" },
+
+    { "allow-create", 'c', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
+      &options.allow_create,
+      "(Advanced) Allow target of --modify/-M to be created if it does not "
+      "exist",
+      NULL },
+
+    { "no-children", 'n', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
+      &options.no_children,
+      "(Advanced) When querying an object, do not include its children in the "
+      "result",
+      NULL },
+
+    { "node", 'N', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.dest_node,
+      "(Advanced) Send command to the specified host", "value" },
+
+    // @COMPAT: Deprecated
+    { "no-bcast", 'b', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE,
+      &options.no_bcast, "deprecated", NULL },
+
+    // @COMPAT: Deprecated
+    { "host", 'h', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING,
+      &options.dest_node, "deprecated", NULL },
+
+    { NULL }
+};
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args)
+{
+    const char *desc = NULL;
+    GOptionContext *context = NULL;
+
+    GOptionEntry extra_prog_entries[] = {
+        // @COMPAT: Deprecated
+        { "extended-version", '!', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE,
+          &options.extended_version, "deprecated", NULL },
+
+        { NULL }
+    };
+
+    desc = "Examples:\n\n"
+           "Query the configuration from the local node:\n\n"
+           "\t# cibadmin --query --local\n\n"
+           "Query just the cluster options configuration:\n\n"
+           "\t# cibadmin --query --scope " XML_CIB_TAG_CRMCONFIG "\n\n"
+           "Query all '" XML_RSC_ATTR_TARGET_ROLE "' settings:\n\n"
+           "\t# cibadmin --query --xpath "
+               "\"//" XML_CIB_TAG_NVPAIR
+               "[@" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_TARGET_ROLE"']\""
+               "\n\n"
+           "Remove all '" XML_RSC_ATTR_MANAGED "' settings:\n\n"
+           "\t# cibadmin --delete-all --xpath "
+               "\"//" XML_CIB_TAG_NVPAIR
+               "[@" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_MANAGED "']\"\n\n"
+           "Remove the resource named 'old':\n\n"
+           "\t# cibadmin --delete --xml-text "
+               "'<" XML_CIB_TAG_RESOURCE " " XML_ATTR_ID "=\"old\"/>'\n\n"
+           "Remove all resources from the configuration:\n\n"
+           "\t# cibadmin --replace --scope " XML_CIB_TAG_RESOURCES
+               " --xml-text '<" XML_CIB_TAG_RESOURCES "/>'\n\n"
+           "Replace complete configuration with contents of "
+               "$HOME/pacemaker.xml:\n\n"
+           "\t# cibadmin --replace --xml-file $HOME/pacemaker.xml\n\n"
+           "Replace " XML_CIB_TAG_CONSTRAINTS " section of configuration with "
+               "contents of $HOME/constraints.xml:\n\n"
+           "\t# cibadmin --replace --scope " XML_CIB_TAG_CONSTRAINTS
+               " --xml-file $HOME/constraints.xml\n\n"
+           "Increase configuration version to prevent old configurations from "
+               "being loaded accidentally:\n\n"
+           "\t# cibadmin --modify --xml-text "
+               "'<" XML_TAG_CIB " " XML_ATTR_GENERATION_ADMIN
+                   "=\"" XML_ATTR_GENERATION_ADMIN "++\"/>'\n\n"
+           "Edit the configuration with your favorite $EDITOR:\n\n"
+           "\t# cibadmin --query > $HOME/local.xml\n\n"
+           "\t# $EDITOR $HOME/local.xml\n\n"
+           "\t# cibadmin --replace --xml-file $HOME/local.xml\n\n"
+           "Assuming terminal, render configuration in color (green for "
+               "writable, blue for readable, red for\n"
+               "denied) to visualize permissions for user tony:\n\n"
+           "\t# cibadmin --show-access=color --query --user tony | less -r\n\n"
+           "SEE ALSO:\n"
+           " crm(8), pcs(8), crm_shadow(8), crm_diff(8)\n";
+
+    context = pcmk__build_arg_context(args, NULL, NULL, "<command>");
+    g_option_context_set_description(context, desc);
+
+    pcmk__add_main_args(context, extra_prog_entries);
+
+    pcmk__add_arg_group(context, "commands", "Commands:", "Show command help",
+                        command_entries);
+    pcmk__add_arg_group(context, "data", "Data:", "Show data help",
+                        data_entries);
+    pcmk__add_arg_group(context, "additional", "Additional Options:",
+                        "Show additional options", addl_entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    int rc = pcmk_rc_ok;
+    const char *source = NULL;
+    xmlNode *output = NULL;
+    xmlNode *input = NULL;
+    gchar *acl_cred = NULL;
+
+    GError *error = NULL;
+
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "ANSUXhotx");
+    GOptionContext *context = build_arg_context(args);
+
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    if (g_strv_length(processed_args) > 1) {
+        gchar *help = g_option_context_get_help(context, TRUE, NULL);
+        GString *extra = g_string_sized_new(128);
+
+        for (int lpc = 1; processed_args[lpc] != NULL; lpc++) {
+            if (extra->len > 0) {
+                g_string_append_c(extra, ' ');
+            }
+            g_string_append(extra, processed_args[lpc]);
+        }
+
+        exit_code = CRM_EX_USAGE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "non-option ARGV-elements: %s\n\n%s", extra->str, help);
+        g_free(help);
+        g_string_free(extra, TRUE);
+        goto done;
+    }
+
+    if (args->version || options.extended_version) {
+        g_strfreev(processed_args);
+        pcmk__free_arg_context(context);
+
+        /* FIXME: When cibadmin is converted to use formatted output, this can
+         * be replaced by out->version with the appropriate boolean flag.
+         *
+         * options.extended_version is deprecated and will be removed in a
+         * future release.
+         */
+        pcmk__cli_help(options.extended_version? '!' : 'v');
+    }
+
+    /* At LOG_ERR, stderr for CIB calls is rather verbose. Several lines like
+     *
+     * (func@file:line)      error: CIB <op> failures   <XML>
+     *
+     * In cibadmin we explicitly output the XML portion without the prefixes. So
+     * we default to LOG_CRIT.
+     */
+    pcmk__cli_init_logging("cibadmin", 0);
+    set_crm_log_level(LOG_CRIT);
+
+    if (args->verbosity > 0) {
+        cib__set_call_options(options.cmd_options, crm_system_name,
+                              cib_verbose);
+
+        for (int i = 0; i < args->verbosity; i++) {
+            crm_bump_log_level(argc, argv);
+        }
+    }
+
+    if (options.cib_action == NULL) {
+        // @COMPAT: Create a default command if other tools have one
+        gchar *help = g_option_context_get_help(context, TRUE, NULL);
+
+        exit_code = CRM_EX_USAGE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Must specify a command option\n\n%s", help);
+        g_free(help);
+        goto done;
+    }
+
+    if (strcmp(options.cib_action, "empty") == 0) {
+        // Output an empty CIB
+        char *buf = NULL;
+
+        output = createEmptyCib(1);
+        crm_xml_add(output, XML_ATTR_VALIDATION, options.validate_with);
+        buf = dump_xml_formatted(output);
+        fprintf(stdout, "%s", pcmk__s(buf, "<null>\n"));
+        free(buf);
+        goto done;
+    }
+
+    if (cib_action_is_dangerous() && !options.force) {
+        exit_code = CRM_EX_UNSAFE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "The supplied command is considered dangerous. To prevent "
+                    "accidental destruction of the cluster, the --force flag "
+                    "is required in order to proceed.");
+        goto done;
+    }
+
+    if (options.message_timeout_sec < 1) {
+        // Set default timeout
+        options.message_timeout_sec = 30;
+    }
+
+    if (options.section_type == cibadmin_section_xpath) {
+        // Enable getting section by XPath
+        cib__set_call_options(options.cmd_options, crm_system_name,
+                              cib_xpath);
+
+    } else if (options.section_type == cibadmin_section_scope) {
+        if (!scope_is_valid(options.cib_section)) {
+            // @COMPAT: Consider requiring --force to proceed
+            fprintf(stderr,
+                    "Invalid value '%s' for '--scope'. Operation will apply "
+                    "to the entire CIB.\n", options.cib_section);
+        }
+    }
+
+    if (options.allow_create) {
+        // Allow target of --modify/-M to be created if it does not exist
+        cib__set_call_options(options.cmd_options, crm_system_name,
+                              cib_can_create);
+    }
+
+    if (options.delete_all) {
+        // With cibadmin_section_xpath, remove all matching objects
+        cib__set_call_options(options.cmd_options, crm_system_name,
+                              cib_multiple);
+    }
+
+    if (options.get_node_path) {
+        /* Enable getting node path of XPath query matches.
+         * Meaningful only if options.section_type == cibadmin_section_xpath.
+         */
+        cib__set_call_options(options.cmd_options, crm_system_name,
+                              cib_xpath_address);
+    }
+
+    if (options.local) {
+        // Configure command to take effect only locally
+        cib__set_call_options(options.cmd_options, crm_system_name,
+                              cib_scope_local);
+    }
+
+    // @COMPAT: Deprecated option
+    if (options.no_bcast) {
+        // Configure command to take effect only locally and not to broadcast
+        cib__set_call_options(options.cmd_options, crm_system_name,
+                              cib_inhibit_bcast|cib_scope_local);
+    }
+
+    if (options.no_children) {
+        // When querying an object, don't include its children in the result
+        cib__set_call_options(options.cmd_options, crm_system_name,
+                              cib_no_children);
+    }
+
+    if (options.sync_call
+        || (options.acl_render_mode != pcmk__acl_render_none)) {
+        /* Wait for call to complete before returning.
+         *
+         * The ACL render modes work only with sync calls due to differences in
+         * output handling between sync/async. It shouldn't matter to the user
+         * whether the call is synchronous; for a CIB query, we have to wait for
+         * the result in order to display it in any case.
+         */
+        cib__set_call_options(options.cmd_options, crm_system_name,
+                              cib_sync_call);
+    }
+
+    if (options.input_file != NULL) {
+        input = filename2xml(options.input_file);
+        source = options.input_file;
+
+    } else if (options.input_xml != NULL) {
+        input = string2xml(options.input_xml);
+        source = "input string";
+
+    } else if (options.input_stdin) {
+        source = "STDIN";
+        input = stdin2xml();
+
+    } else if (options.acl_render_mode != pcmk__acl_render_none) {
+        char *username = pcmk__uid2username(geteuid());
+        bool required = pcmk_acl_required(username);
+
+        free(username);
+
+        if (required) {
+            if (options.force) {
+                fprintf(stderr, "The supplied command can provide skewed"
+                                 " result since it is run under user that also"
+                                 " gets guarded per ACLs on their own right."
+                                 " Continuing since --force flag was"
+                                 " provided.\n");
+
+            } else {
+                exit_code = CRM_EX_UNSAFE;
+                g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                            "The supplied command can provide skewed result "
+                            "since it is run under user that also gets guarded "
+                            "per ACLs in their own right. To accept the risk "
+                            "of such a possible distortion (without even "
+                            "knowing it at this time), use the --force flag.");
+                goto done;
+            }
+        }
+
+        if (options.cib_user == NULL) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "The supplied command requires -U user specified.");
+            goto done;
+        }
+
+        /* We already stopped/warned ACL-controlled users about consequences.
+         *
+         * Note: acl_cred takes ownership of options.cib_user here.
+         * options.cib_user is set to NULL so that the CIB is obtained as the
+         * user running the cibadmin command. The CIB must be obtained as a user
+         * with full permissions in order to show the CIB correctly annotated
+         * for the options.cib_user's permissions.
+         */
+        acl_cred = options.cib_user;
+        options.cib_user = NULL;
+    }
+
+    if (input != NULL) {
+        crm_log_xml_debug(input, "[admin input]");
+
+    } else if (source != NULL) {
+        exit_code = CRM_EX_CONFIG;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Couldn't parse input from %s.", source);
+        goto done;
+    }
+
+    if (strcmp(options.cib_action, "md5-sum") == 0) {
+        char *digest = NULL;
+
+        if (input == NULL) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Please supply XML to process with -X, -x, or -p");
+            goto done;
+        }
+
+        digest = calculate_on_disk_digest(input);
+        fprintf(stderr, "Digest: ");
+        fprintf(stdout, "%s\n", pcmk__s(digest, "<null>"));
+        free(digest);
+        goto done;
+
+    } else if (strcmp(options.cib_action, "md5-sum-versioned") == 0) {
+        char *digest = NULL;
+        const char *version = NULL;
+
+        if (input == NULL) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Please supply XML to process with -X, -x, or -p");
+            goto done;
+        }
+
+        version = crm_element_value(input, XML_ATTR_CRM_VERSION);
+        digest = calculate_xml_versioned_digest(input, FALSE, TRUE, version);
+        fprintf(stderr, "Versioned (%s) digest: ", version);
+        fprintf(stdout, "%s\n", pcmk__s(digest, "<null>"));
+        free(digest);
+        goto done;
+    }
+
+    rc = do_init();
+    if (rc != pcmk_ok) {
+        rc = pcmk_legacy2rc(rc);
+        exit_code = pcmk_rc2exitc(rc);
+
+        crm_err("Init failed, could not perform requested operations: %s",
+                pcmk_rc_str(rc));
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Init failed, could not perform requested operations: %s",
+                    pcmk_rc_str(rc));
+        goto done;
+    }
+
+    rc = do_work(input, &output);
+    if (rc > 0) {
+        /* wait for the reply by creating a mainloop and running it until
+         * the callbacks are invoked...
+         */
+        request_id = rc;
+
+        the_cib->cmds->register_callback(the_cib, request_id,
+                                         options.message_timeout_sec, FALSE,
+                                         NULL, "cibadmin_op_callback",
+                                         cibadmin_op_callback);
+
+        mainloop = g_main_loop_new(NULL, FALSE);
+
+        crm_trace("%s waiting for reply from the local CIB", crm_system_name);
+
+        crm_info("Starting mainloop");
+        g_main_loop_run(mainloop);
+
+    } else if ((rc == -pcmk_err_schema_unchanged)
+               && (strcmp(options.cib_action,
+                          PCMK__CIB_REQUEST_UPGRADE) == 0)) {
+        report_schema_unchanged();
+
+    } else if (rc < 0) {
+        rc = pcmk_legacy2rc(rc);
+        crm_err("Call failed: %s", pcmk_rc_str(rc));
+        fprintf(stderr, "Call failed: %s\n", pcmk_rc_str(rc));
+
+        if (rc == pcmk_rc_schema_validation) {
+            if (strcmp(options.cib_action, PCMK__CIB_REQUEST_UPGRADE) == 0) {
+                xmlNode *obj = NULL;
+                int version = 0;
+
+                if (the_cib->cmds->query(the_cib, NULL, &obj,
+                                         options.cmd_options) == pcmk_ok) {
+                    update_validation(&obj, &version, 0, TRUE, FALSE);
+                }
+                free_xml(obj);
+
+            } else if (output) {
+                validate_xml_verbose(output);
+            }
+        }
+        exit_code = pcmk_rc2exitc(rc);
+    }
+
+    if ((output != NULL)
+        && (options.acl_render_mode != pcmk__acl_render_none)) {
+
+        xmlDoc *acl_evaled_doc;
+        rc = pcmk__acl_annotate_permissions(acl_cred, output->doc, &acl_evaled_doc);
+        if (rc == pcmk_rc_ok) {
+            xmlChar *rendered = NULL;
+
+            rc = pcmk__acl_evaled_render(acl_evaled_doc,
+                                         options.acl_render_mode, &rendered);
+            if (rc != pcmk_rc_ok) {
+                exit_code = CRM_EX_CONFIG;
+                g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                            "Could not render evaluated access: %s",
+                            pcmk_rc_str(rc));
+                goto done;
+            }
+            printf("%s\n", (char *) rendered);
+            free(rendered);
+
+        } else {
+            exit_code = CRM_EX_CONFIG;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Could not evaluate access per request (%s, error: %s)",
+                        acl_cred, pcmk_rc_str(rc));
+            goto done;
+        }
+
+    } else if (output != NULL) {
+        print_xml_output(output);
+    }
+
+    crm_trace("%s exiting normally", crm_system_name);
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    g_free(options.cib_user);
+    g_free(options.dest_node);
+    g_free(options.input_file);
+    g_free(options.input_xml);
+    free(options.cib_section);
+    free(options.validate_with);
+
+    g_free(acl_cred);
+    free_xml(input);
+    free_xml(output);
+
+    rc = cib__clean_up_connection(&the_cib);
+    if (exit_code == CRM_EX_OK) {
+        exit_code = pcmk_rc2exitc(rc);
+    }
+
+    pcmk__output_and_clear_error(&error, NULL);
+    crm_exit(exit_code);
+}
+
+static int
+do_work(xmlNode *input, xmlNode **output)
+{
+    /* construct the request */
+    the_cib->call_timeout = options.message_timeout_sec;
+    if ((strcmp(options.cib_action, PCMK__CIB_REQUEST_REPLACE) == 0)
+        && pcmk__str_eq(crm_element_name(input), XML_TAG_CIB, pcmk__str_casei)) {
+        xmlNode *status = pcmk_find_cib_element(input, XML_CIB_TAG_STATUS);
+
+        if (status == NULL) {
+            create_xml_node(input, XML_CIB_TAG_STATUS);
+        }
+    }
+
+    crm_trace("Passing \"%s\" to variant_op...", options.cib_action);
+    return cib_internal_op(the_cib, options.cib_action, options.dest_node,
+                           options.cib_section, input, output,
+                           options.cmd_options, options.cib_user);
+}
+
+int
+do_init(void)
+{
+    int rc = pcmk_ok;
+
+    the_cib = cib_new();
+    rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command);
+    if (rc != pcmk_ok) {
+        crm_err("Could not connect to the CIB: %s", pcmk_strerror(rc));
+        fprintf(stderr, "Could not connect to the CIB: %s\n",
+                pcmk_strerror(rc));
+    }
+
+    return rc;
+}
+
+void
+cibadmin_op_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    rc = pcmk_legacy2rc(rc);
+    exit_code = pcmk_rc2exitc(rc);
+
+    if (rc == pcmk_rc_schema_unchanged) {
+        report_schema_unchanged();
+
+    } else if (rc != pcmk_rc_ok) {
+        crm_warn("Call %s failed: %s " CRM_XS " rc=%d",
+                 options.cib_action, pcmk_rc_str(rc), rc);
+        fprintf(stderr, "Call %s failed: %s\n",
+                options.cib_action, pcmk_rc_str(rc));
+        print_xml_output(output);
+
+    } else if ((strcmp(options.cib_action, PCMK__CIB_REQUEST_QUERY) == 0)
+               && (output == NULL)) {
+        crm_err("Query returned no output");
+        crm_log_xml_err(msg, "no output");
+
+    } else if (output == NULL) {
+        crm_info("Call passed");
+
+    } else {
+        crm_info("Call passed");
+        print_xml_output(output);
+    }
+
+    if (call_id == request_id) {
+        g_main_loop_quit(mainloop);
+
+    } else {
+        crm_info("Message was not the response we were looking for (%d vs. %d)",
+                 call_id, request_id);
+    }
+}
diff --git a/tools/cibsecret.in b/tools/cibsecret.in
new file mode 100644
index 0000000..4569863
--- /dev/null
+++ b/tools/cibsecret.in
@@ -0,0 +1,440 @@
+#!@BASH_PATH@
+
+# Copyright 2011-2020 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+# cibsecret
+#
+# Manage the secrets directory (by default, /var/lib/pacemaker/lrm/secrets).
+# Secrets are ASCII files, holding one value per file:
+# <secrets-directory>/<rsc>/<param>
+
+# These constants must track crm_exit_t values
+CRM_EX_OK=0
+CRM_EX_ERROR=1
+CRM_EX_NOT_INSTALLED=5
+CRM_EX_USAGE=64
+CRM_EX_UNAVAILABLE=69
+CRM_EX_OSFILE=72
+CRM_EX_CONFIG=78
+CRM_EX_DIGEST=104
+CRM_EX_NOSUCH=105
+CRM_EX_EXISTS=108
+
+LRM_CIBSECRETS="@LRM_CIBSECRETS_DIR@"
+
+PROG="$(basename "$0")"
+SSH_OPTS="-o StrictHostKeyChecking=no"
+MAGIC="lrm://"
+
+usage() {
+    cat <<EOF
+cibsecret - manage sensitive information in Pacemaker CIB
+
+Usage:
+    $PROG [<options>] <command> [<parameters>]
+
+Options:
+    --help       Show this message, then exit
+    --version    Display version information, then exit
+    -C           Don't read or write the CIB
+
+Commands and their parameters:
+    set <resource-id> <resource-parameter> <value>
+        Set the value of a sensitive resource parameter.
+
+    get <resource-id> <resource-parameter>
+        Display the locally stored value of a sensitive resource parameter.
+
+    check <resource-id> <resource-parameter>
+        Verify that the locally stored value of a sensitive resource parameter
+        matches its locally stored MD5 hash.
+
+    stash <resource-id> <resource-parameter>
+        Make a non-sensitive resource parameter that is already in the CIB
+        sensitive (move its value to a locally stored and protected file).
+        This may not be used with -C.
+
+    unstash <resource-id> <resource-parameter>
+        Make a sensitive resource parameter that is already in the CIB
+        non-sensitive (move its value from the locally stored file to the CIB).
+        This may not be used with -C.
+
+    delete <resource-id> <resource-parameter>
+        Remove a sensitive resource parameter value.
+
+    sync
+        Copy all locally stored secrets to all other nodes.
+
+This command manages sensitive resource parameter values that should not be
+stored directly in Pacemaker's Cluster Information Base (CIB). Such values
+are handled by storing a special string directly in the CIB that tells
+Pacemaker to look in a separate, protected file for the actual value.
+
+The secret files are not encrypted, but protected by file system permissions
+such that only root can read or modify them.
+
+Since the secret files are stored locally, they must be synchronized across all
+cluster nodes. This command handles the synchronization using (in order of
+preference) pssh, pdsh, or ssh, so one of those must be installed. Before
+synchronizing, this command will ping the cluster nodes to determine which are
+alive, using fping if it is installed, otherwise the ping command. Installing
+fping is strongly recommended for better performance.
+
+Known limitations:
+
+    This command can only be run from full cluster nodes (not Pacemaker Remote
+    nodes).
+
+    Changes are not atomic, so the cluster may use different values while a
+    change is in progress. To avoid problems, it is recommended to put the
+    cluster in maintenance mode when making changes with this command.
+
+    Changes in secret values do not trigger an agent reload or restart of the
+    affected resource, since they do not change the CIB. If a response is
+    desired before the next cluster recheck interval, any CIB change (such as
+    setting a node attribute) will trigger it.
+
+    If any node is down when changes to secrets are made, or a new node is
+    later added to the cluster, it may have different values when it joins the
+    cluster, before "$PROG sync" is run. To avoid this, it is recommended to
+    run the sync command (from another node) before starting Pacemaker on the
+    node.
+
+Examples:
+
+    $PROG set ipmi_node1 passwd SecreT_PASS
+
+    $PROG get ipmi_node1 passwd
+
+    $PROG check ipmi_node1 passwd
+
+    $PROG stash ipmi_node2 passwd
+
+    $PROG sync
+EOF
+        exit "$1"
+}
+
+check_usage() {
+    case "$1" in
+        set)        [ "$2" -ne 4 ] && [ "$2" -ne 3 ] && usage 1 ;;
+        get)        [ "$2" -ne 3 ] && usage 1 ;;
+        check)      [ "$2" -ne 3 ] && usage 1 ;;
+        stash)      [ "$2" -ne 3 ] && usage 1 ;;
+        unstash)    [ "$2" -ne 3 ] && usage 1 ;;
+        delete)     [ "$2" -ne 3 ] && usage 1 ;;
+        sync)       [ "$2" -ne 1 ] && usage 1 ;;
+        --help)     usage $CRM_EX_OK ;;
+        --version)  crm_attribute --version; exit $? ;;
+        *)          usage $CRM_EX_USAGE ;;
+    esac
+}
+
+fatal() {
+    rc=$1
+    shift
+    echo "ERROR: $*"
+    exit $rc
+}
+
+warn() {
+    echo "WARNING: $*"
+}
+
+info() {
+    echo "INFO: $*"
+}
+
+check_env() {
+    which md5sum >/dev/null 2>&1 ||
+        fatal $CRM_EX_NOT_INSTALLED "please install md5sum to run $PROG"
+    if which pssh >/dev/null 2>&1; then
+        rsh=pssh_fun
+        rcp_to_from=pscp_fun
+
+        # -q is a SUSE patch not present in upstream pssh
+        PSSH_QUIET_OPTION=""
+        pssh -q 2>&1|grep "no such option: -q" > /dev/null ||
+            PSSH_QUIET_OPTION="-q"
+    elif which pdsh >/dev/null 2>&1; then
+        rsh=pdsh_fun
+        rcp_to_from=pdcp_fun
+    elif which ssh >/dev/null 2>&1; then
+        rsh=ssh_fun
+        rcp_to_from=scp_fun
+    else
+        fatal $CRM_EX_NOT_INSTALLED "please install pssh, pdsh, or ssh to run $PROG"
+    fi
+    ps -ef | grep '[p]acemaker-controld' >/dev/null ||
+        fatal $CRM_EX_UNAVAILABLE "pacemaker not running? $PROG needs pacemaker"
+}
+
+# This must be called (and return success) before calling $rsh or $rcp_to_from
+get_live_peers() {
+    # Get local node name
+    GLP_LOCAL_NODE="$(crm_node -n)"
+    [ $? -eq 0 ] || fatal $CRM_EX_UNAVAILABLE "couldn't get local node name"
+
+    # Get a list of all other cluster nodes
+    GLP_ALL_PEERS="$(crmadmin -N -q)"
+    [ $? -eq 0 ] || fatal $CRM_EX_UNAVAILABLE "couldn't determine cluster nodes"
+    GLP_ALL_PEERS="$(echo "$GLP_ALL_PEERS" | grep -v "^${GLP_LOCAL_NODE}$")"
+
+    # Make a list of those that respond to pings
+    if [ "$(id -u)" = "0" ] && which fping >/dev/null 2>&1; then
+        LIVE_NODES=$(fping -a $GLP_ALL_PEERS 2>/dev/null)
+    else
+        LIVE_NODES=""
+        for GLP_NODE in $GLP_ALL_PEERS; do \
+            ping -c 2 -q "$GLP_NODE" >/dev/null 2>&1 &&
+                LIVE_NODES="$LIVE_NODES $GLP_NODE"
+        done
+    fi
+
+    # Warn the user about any that didn't respond to pings
+    GLP_DOWN="$( (for GLP_NODE in $LIVE_NODES $GLP_ALL_PEERS; do echo "$GLP_NODE"; done) | sort | uniq -u)"
+    if [ "$(echo "$GLP_DOWN" | wc -w)" = "1" ]; then
+        warn "node $GLP_DOWN is down"
+        warn "you'll need to update it using \"$PROG sync\" later"
+    elif [ -n "$GLP_DOWN" ]; then
+        warn "nodes $(echo "$GLP_DOWN" | tr '\n' ' ')are down"
+        warn "you'll need to update them using \"$PROG sync\" later"
+    fi
+
+    if [ "$LIVE_NODES" = "" ]; then
+        info "no other nodes live"
+        return 1
+    fi
+    return 0
+}
+
+pssh_fun() {
+    pssh $PSSH_QUIET_OPTION -i -H "$LIVE_NODES" -x "$SSH_OPTS" -- "$@"
+}
+
+pscp_fun() {
+    PSCP_DEST="$1"
+    shift
+    pscp $PSSH_QUIET_OPTION -H "$LIVE_NODES" -x "-pr" -x "$SSH_OPTS" -- "$@" "$PSCP_DEST"
+}
+
+pdsh_fun() {
+    PDSH_NODES=$(echo "$LIVE_NODES" | tr '[:space:]' ',')
+    export PDSH_SSH_ARGS_APPEND="$SSH_OPTS"
+    pdsh -w "$PDSH_NODES" -- "$@"
+}
+
+pdcp_fun() {
+    PDCP_DEST="$1"
+    shift
+    PDCP_NODES=$(echo "$LIVE_NODES" | tr '[:space:]' ',')
+    export PDSH_SSH_ARGS_APPEND="$SSH_OPTS"
+    pdcp -pr -w "$PDCP_NODES" -- "$@" "$PDCP_DEST"
+}
+
+ssh_fun() {
+    for SSH_NODE in $LIVE_NODES; do
+        ssh $SSH_OPTS "$SSH_NODE" -- "$@" || return
+    done
+}
+
+scp_fun() {
+    SCP_DEST="$1"
+    shift
+    for SCP_NODE in $LIVE_NODES; do
+        scp -pqr $SSH_OPTS "$@" "$SCP_NODE:$SCP_DEST" || return
+    done
+}
+
+# TODO: this procedure should be replaced with csync2
+# provided that csync2 has already been configured
+sync_files() {
+    get_live_peers || return
+    if [ "$cmd" != "delete" ]; then
+        info "syncing $LRM_CIBSECRETS to $(echo "$LIVE_NODES" | tr '\n' ' ') ..."
+    else
+        info "deleting $LRM_CIBSECRETS from $(echo "$LIVE_NODES" | tr '\n' ' ') ..."
+    fi
+    $rsh rm -rf "$LRM_CIBSECRETS" &&
+        $rsh mkdir -p "$(dirname "$LRM_CIBSECRETS")" &&
+        $rcp_to_from "$(dirname "$LRM_CIBSECRETS")" "$LRM_CIBSECRETS"
+}
+
+sync_one() {
+    SO_FILE="$1"
+    get_live_peers || return
+    if [ "$cmd" != "delete" ]; then
+        info "syncing $SO_FILE to $(echo "$LIVE_NODES" | tr '\n' ' ') ..."
+    else
+        info "deleting $SO_FILE from $(echo "$LIVE_NODES" | tr '\n' ' ') ..."
+    fi
+    $rsh mkdir -p "$(dirname "$SO_FILE")" &&
+        if [ -f "$SO_FILE" ]; then
+            $rcp_to_from "$(dirname "$SO_FILE")" "$SO_FILE" "${SO_FILE}.sign"
+        else
+            $rsh rm -f "$SO_FILE" "${SO_FILE}.sign"
+        fi
+}
+
+is_secret() {
+    # assume that the secret is in the CIB if we cannot talk to cib
+    [ "$NO_CRM" ] || test "$1" = "$MAGIC"
+}
+
+check_cib_rsc() {
+    CCR_OUT="$($NO_CRM crm_resource -r "$1" -W 2>&1)" || fatal $CRM_EX_NOSUCH "$CCR_OUT"
+}
+
+get_cib_param() {
+    GCP_RSC="$1"
+    GCP_PARAM="$2"
+    $NO_CRM crm_resource -r "$GCP_RSC" -g "$GCP_PARAM" 2>/dev/null
+}
+
+set_cib_param() {
+    SET_RSC="$1"
+    SET_PARAM="$2"
+    SET_VAL="$3"
+    $NO_CRM crm_resource -r "$SET_RSC" -p "$SET_PARAM" -v "$SET_VAL" 2>/dev/null
+}
+
+remove_cib_param() {
+    RM_RSC="$1"
+    RM_PARAM="$2"
+    $NO_CRM crm_resource -r "$RM_RSC" -d "$RM_PARAM" 2>/dev/null
+}
+
+localfiles() {
+    LF_CMD="$1"
+    LF_RSC="$2"
+    LF_PARAM="$3"
+    LF_VALUE="$4"
+    LF_FILE="$LRM_CIBSECRETS/$LF_RSC/$LF_PARAM"
+    case "$LF_CMD" in
+        get)
+            cat "$LF_FILE" 2>/dev/null
+            true
+            ;;
+
+        getsum)
+            cat "${LF_FILE}.sign" 2>/dev/null
+            true
+            ;;
+
+        set)
+            LF_SUM="$(printf %s "$LF_VALUE" | md5sum)" ||
+                fatal $CRM_EX_ERROR "md5sum failed to produce hash for resource $LF_RSC parameter $LF_PARAM"
+            LF_SUM="$(echo "$LF_SUM" | awk '{print $1}')"
+            mkdir -p "$(dirname "$LF_FILE")" &&
+                echo "$LF_VALUE" > "$LF_FILE" &&
+                echo "$LF_SUM" > "${LF_FILE}.sign" &&
+                sync_one "$LF_FILE"
+            ;;
+
+        remove)
+            rm -f "$LF_FILE" "${LF_FILE}.sign"
+            sync_one "$LF_FILE"
+            ;;
+    esac
+}
+
+cibsecret_set() {
+    CS_VALUE="$1"
+
+    if [ "$2" -ne 4 ]; then
+        read -p "Enter value: " CS_VALUE
+    fi
+
+    check_cib_rsc "$rsc"
+    CIBSET_CURRENT="$(get_cib_param "$rsc" "$param")"
+    [ -z "$NO_CRM" ] &&
+        [ ! -z "$CIBSET_CURRENT" ] &&
+        [ "$CIBSET_CURRENT" != "$MAGIC" ] &&
+        [ "$CIBSET_CURRENT" != "$CS_VALUE" ] &&
+        fatal $CRM_EX_CONFIG "CIB value <$CIBSET_CURRENT> different for $rsc parameter $param; please delete it first"
+    localfiles set "$rsc" "$param" "$CS_VALUE" &&
+        set_cib_param "$rsc" "$param" "$MAGIC"
+}
+
+cibsecret_check() {
+    check_cib_rsc "$rsc"
+    is_secret "$(get_cib_param "$rsc" "$param")" ||
+        fatal $CRM_EX_CONFIG "resource $rsc parameter $param not set as secret, nothing to check"
+    CSC_LOCAL_SUM="$(localfiles getsum "$rsc" "$param")"
+    [ "$CSC_LOCAL_SUM" ] ||
+        fatal $CRM_EX_OSFILE "no MD5 hash for resource $rsc parameter $param"
+    CSC_LOCAL_VALUE="$(localfiles get "$rsc" "$param")"
+    CSC_CALC_SUM="$(printf "%s" "$CSC_LOCAL_VALUE" | md5sum | awk '{print $1}')"
+    [ "$CSC_CALC_SUM" = "$CSC_LOCAL_SUM" ] ||
+        fatal $CRM_EX_DIGEST "MD5 hash mismatch for resource $rsc parameter $param"
+}
+
+cibsecret_get() {
+    cibsecret_check
+    localfiles get "$rsc" "$param"
+}
+
+cibsecret_delete() {
+    check_cib_rsc "$rsc"
+    localfiles remove "$rsc" "$param" && remove_cib_param "$rsc" "$param"
+}
+
+cibsecret_stash() {
+    [ "$NO_CRM" ] && fatal $CRM_EX_USAGE "no access to Pacemaker, stash not supported"
+    check_cib_rsc "$rsc"
+    CIBSTASH_CURRENT="$(get_cib_param "$rsc" "$param")"
+    [ "$CIBSTASH_CURRENT" = "" ] &&
+        fatal $CRM_EX_NOSUCH "nothing to stash for resource $rsc parameter $param"
+    is_secret "$CIBSTASH_CURRENT" &&
+        fatal $CRM_EX_EXISTS "resource $rsc parameter $param already set as secret, nothing to stash"
+    cibsecret_set "$CIBSTASH_CURRENT" 4
+}
+
+cibsecret_unstash() {
+    [ "$NO_CRM" ] && fatal $CRM_EX_USAGE "no access to Pacemaker, unstash not supported"
+    UNSTASH_LOCAL_VALUE="$(localfiles get "$rsc" "$param")"
+    [ "$UNSTASH_LOCAL_VALUE" = "" ] &&
+        fatal $CRM_EX_NOSUCH "nothing to unstash for resource $rsc parameter $param"
+    check_cib_rsc "$rsc"
+    is_secret "$(get_cib_param "$rsc" "$param")" ||
+        warn "resource $rsc parameter $param not set as secret, but we have local value so proceeding anyway"
+    localfiles remove "$rsc" "$param" &&
+        set_cib_param "$rsc" "$param" "$UNSTASH_LOCAL_VALUE"
+}
+
+cibsecret_sync() {
+    sync_files
+}
+
+# Grab arguments
+if [ "$1" = "-C" ]; then
+    NO_CRM=':'
+    shift
+fi
+cmd="$1"
+rsc="$2"
+param="$3"
+value="$4"
+
+# Ensure we have everything we need
+check_usage "$cmd" $#
+check_env
+umask 0077
+
+# for dirname() function (@TODO why are we replacing dirname?)
+. "@OCF_ROOT_DIR@/lib/heartbeat/ocf-shellfuncs"
+
+"cibsecret_$cmd" "$value" $#
+rc=$?
+
+if [ $rc -ne 0 ]; then
+    fatal $CRM_EX_ERROR "$cmd(): failed with rc: $rc"
+fi
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4 textwidth=80:
diff --git a/tools/cluster-clean.in b/tools/cluster-clean.in
new file mode 100755
index 0000000..4b75edf
--- /dev/null
+++ b/tools/cluster-clean.in
@@ -0,0 +1,99 @@
+#!@BASH_PATH@
+#
+# Copyright 2011-2023 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+hosts=
+group=
+kill=0
+while true; do
+    case "$1" in
+	-x) set -x; shift;;
+	-w) for h in $2; do
+		hosts="$hosts -w $h";
+	    done
+	    shift; shift;;
+	-g) group=$2; shift; shift;;
+	--kill) kill=1; shift;;
+	--kill-only) kill=2; shift;;
+	"") break;;
+	*) echo "unknown option: $1"; exit 1;;
+    esac
+done
+
+if [ x"$group" = x -a x"$hosts" = x ]; then
+    group=$CTS_GROUP
+fi
+
+if [ x"$hosts" != x ]; then
+    echo `date` ": Cleaning up hosts:"
+    target=$hosts
+elif [ x"$group" != x ]; then
+    echo `date` ": Cleaning up group: $group"
+    target="-g $group"
+else
+    echo "You didn't specify any nodes to clean up"
+    exit 1
+fi
+cluster-helper --list bullet $target
+
+if [ $kill != 0 ]; then
+    echo "Cleaning processes"
+
+    # Bah. Force systemd to actually look at the process and realize it's dead
+    cluster-helper $target -- "service corosync stop" &> /dev/null &
+    cluster-helper $target -- "service pacemaker stop" &> /dev/null &
+
+    cluster-helper $target -- "killall -q -9 corosync pacemakerd pacemaker-attrd pacemaker-based pacemaker-controld pacemaker-execd pacemaker-fenced pacemaker-remoted pacemaker-schedulerd dlm_controld gfs_controld" &> /dev/null
+    cluster-helper $target -- 'kill -9 `pidof valgrind`' &> /dev/null
+
+    if [ $kill == 2 ]; then
+	exit 0
+    fi
+fi
+
+#logrotate -f $cluster_rotate
+echo "Cleaning files"
+
+log_files=""
+log_files="$log_files 'messages*'"
+log_files="$log_files 'localmessages*'"
+log_files="$log_files 'cluster*.log'"
+log_files="$log_files 'corosync.log*'"
+log_files="$log_files 'pacemaker.log*'"
+log_files="$log_files '*.journal'"
+log_files="$log_files '*.journal~'"
+log_files="$log_files 'secure-*'"
+
+state_files=""
+state_files="$state_files 'cib.xml*'"
+state_files="$state_files 'valgrind-*'"
+state_files="$state_files 'cib-*'"
+state_files="$state_files 'core.*'"
+state_files="$state_files 'cts.*'"
+state_files="$state_files 'pe*.bz2'"
+state_files="$state_files 'fdata-*'"
+
+for f in $log_files; do
+    cluster-helper $target -- "find /var/log -name '$f' -exec rm -f \{\} \;"
+done
+
+for f in $state_files; do
+   cluster-helper $target -- "find /var/lib -name '$f' -exec rm -f \{\} \;"
+done
+
+cluster-helper $target -- "find /dev/shm -name 'qb-*' -exec rm -f \{\} \;"
+cluster-helper $target -- "find @CRM_BLACKBOX_DIR@ -name '*-*' -exec rm -f \{\} \;"
+cluster-helper $target -- "find /tmp -name '*.valgrind' -exec rm -f \{\} \;"
+
+cluster-helper $target -- 'service rsyslog restart' > /dev/null  2>&1
+cluster-helper $target -- 'systemctl restart systemd-journald.socket' > /dev/null  2>&1
+cluster-helper $target -- logger -i -p daemon.info __clean_logs__
+
+#touch $cluster_log
+echo `date` ": Clean complete"
diff --git a/tools/cluster-helper.in b/tools/cluster-helper.in
new file mode 100755
index 0000000..d8dac6e
--- /dev/null
+++ b/tools/cluster-helper.in
@@ -0,0 +1,201 @@
+#!@BASH_PATH@
+#
+# Copyright 2011-2023 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+hosts=
+group=$cluster_name
+user=root
+pdsh=`which pdsh 2>/dev/null`
+ssh=`which qarsh 2>/dev/null`
+scp=`which qacp 2>/dev/null`
+command=list
+format=oneline
+replace="{}"
+
+if [ x$ssh = "x" ]; then
+    ssh=ssh
+    scp=scp
+fi
+
+function helptext() {
+    echo "cluster-helper - A tool for running commands on multiple hosts"
+    echo ""
+    echo "Attempt to use pdsh, qarsh, or ssh (in that order) to execute commands"
+    echo "on multiple hosts"
+    echo ""
+    echo "DSH groups can be configured and specified with -g instead of listing"
+    echo "the individual hosts every time"
+    echo ""
+    echo "Usage: cluster-helper [options] [command]"
+    echo ""
+    echo "Options:"
+    echo "--ssh            Force the use of ssh instead of qarsh even if it available"
+    echo "-g, --group      Specify the group to operate on/with"
+    echo "-w, --host       Specify a host to operate on/with.  May be specified multiple times"
+    echo "-f, --format     Specifiy the output format When listing hosts or group contents"
+    echo "                 Allowed values: [oneline], long, short, pdsh, bullet"
+    echo ""
+    echo ""
+    echo "Commands:"
+    echo "--list   format  List the contents of a group in the specified format"
+    echo "--add    name    Add supplied (-w) hosts to the named group"
+    echo "--create name    Create the named group with the supplied (-w) hosts"
+    echo "--run, --        Treat all subsequent arguments as a command to perform on"
+    echo "                 the specified command on the hosts or group"
+    echo "--xargs          Run the supplied command having replaced any occurrences"
+    echo "                 of {} with the node name"
+    echo ""
+    echo "--copy file(s) host:file   Pass subsequent arguments to scp or qacp"
+    echo "                           Any occurrences of {} are replaced with the node name"
+    echo "--key            Install an ssh key"
+    echo ""
+    exit $1
+}
+
+while true ; do
+    case "$1" in
+	--help|-h|-\?) helptext 0;;
+	-x)  set -x; shift;;
+	--ssh) ssh="ssh"; scp="scp"; pdsh=""; shift;;
+	-g|--group) group="$2"; shift; shift;;
+	-w|--host) for h in $2; do
+		hosts="$hosts $h";
+	    done
+	    shift; shift;;
+	-f|--format) format=$2; shift; shift;;
+	-I) replace=$2; shift; shift;;
+	--list|list) format=$2; command=list; shift; shift;;
+	--add|add) command=group-add; shift;;
+	--create|create) group="$2", command=group-create; shift; shift;;
+	--run|run) command=run; shift;;
+	--copy|copy) command=copy; shift; break ;;
+	--key|key) command=key; shift; break ;;
+	--xargs) command=xargs; shift; break ;;
+	--) command=run; shift; break ;;
+	"") break;;
+	*) helptext 1;;
+    esac
+done	
+
+if [ x"$group" = x -a x"$hosts" = x ]; then
+    group=$CTS_GROUP
+fi
+
+function expand() {
+    fmt=$1
+    if [ x$group != x -a -f ~/.dsh/group/$group ]; then
+	hosts=`cat ~/.dsh/group/$group`
+    elif [ x$group != x ]; then
+	echo "Unknown group: $group" >&2
+	exit 1
+    fi
+    
+    if [ "x$hosts" != x -a $fmt = oneline ]; then
+	echo $hosts
+	
+    elif [ "x$hosts" != x -a $fmt = short ]; then
+	( for h in $hosts; do
+		echo $h | sed 's:\..*::'
+		done ) | tr '\n' ' '
+	echo ""
+	
+    elif [ "x$hosts" != x -a $fmt = pdsh ]; then
+	( for h in $hosts; do
+		echo "-w $h"
+	    done ) | tr '\n' ' '
+	echo ""
+	
+    elif [ "x$hosts" != x -a $fmt = long ]; then
+	for h in $hosts; do
+	    echo $h
+	done
+
+    elif [ "x$hosts" != x -a $fmt = bullet ]; then
+	for h in $hosts; do
+	    echo " * $h"
+	done
+
+    elif [ "x$hosts" != x ]; then
+	echo "Unknown format: $fmt" >&2
+    fi
+}
+
+if [ $command = list ]; then
+    expand $format
+
+elif [ $command = key ]; then
+    hosts=`expand oneline`
+    for h in $hosts; do
+	ssh-copy-id root@$h
+    done
+
+elif [ $command = group-create ]; then
+
+    f=`mktemp`
+    mkdir -p ~/.dsh/group
+
+    if [ -f ~/.dsh/group/$group ]; then
+	echo "Overwriting existing group $group"
+    fi
+
+    for h in $hosts; do
+	echo $h >> $f
+    done
+
+    echo "Creating group $group in ~/.dsh/group"
+    sort -u $f > ~/.dsh/group/$group
+    rm -f $f
+
+elif [ $command = group-add ]; then
+    if [ x$group = x ]; then
+	echo "Please specify a group to append to"
+	exit 1
+    fi
+    
+    f=`mktemp`
+    mkdir -p ~/.dsh/group
+
+    if [ -f ~/.dsh/group/$group ]; then
+	cat ~/.dsh/group/$group > $f
+    fi
+
+    for h in $hosts; do
+	echo $h >> $f
+    done
+
+    echo "Appending hosts to group $group in ~/.dsh/group"
+    sort -u $f > ~/.dsh/group/$group
+    rm -f $f
+
+elif [ $command = run ]; then
+    if [ x$pdsh != x ]; then
+	hosts=`expand pdsh`
+	$pdsh -l $user $hosts -- $*
+
+    else
+	hosts=`expand oneline`
+	for n in $hosts; do
+	    $ssh -l $user $n -- $* < /dev/null
+	done
+	if [ x"$hosts" = x ]; then
+	    echo "No hosts specified"
+	fi
+    fi
+elif [ $command = copy ]; then
+    hosts=`expand oneline`
+    for n in $hosts; do
+	$scp `echo $* | sed 's@'$replace'@'$n'@'`
+    done
+
+elif [ $command = xargs ]; then
+    hosts=`expand oneline`
+    for n in $hosts; do
+	eval `echo $* | sed 's@'$replace'@'$n'@'`
+    done
+fi
diff --git a/tools/cluster-init.in b/tools/cluster-init.in
new file mode 100755
index 0000000..1485c81
--- /dev/null
+++ b/tools/cluster-init.in
@@ -0,0 +1,537 @@
+#!@BASH_PATH@
+#
+# Copyright 2011-2023 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+accept_defaults=0
+do_raw=0
+ETCHOSTS=0
+nodelist=0
+limit=0
+
+pkgs="corosync xinetd nmap abrt-cli fence-agents perl-TimeDate gdb"
+
+transport="multicast"
+inaddr_any="no"
+
+INSTALL=
+cs_conf=
+fence_conf=
+
+dsh_group=0
+if [ ! -z $cluster_name ]; then
+    cluster=$cluster_name
+else
+    cluster=dummy0
+fi
+
+#  Corosync Settings
+cs_port=666
+
+# Settings that work great on nXX
+join=60
+#token=3000
+consensus=1500
+
+# Official settings
+join=2000
+token=5000
+consensus=2500
+
+# Testing
+join=1000
+consensus=7500
+do_debug=off
+
+function ip_for_node() {
+    ping -c 1 $1 | grep "bytes from" | head -n 1 | sed -e 's/.*bytes from//' -e 's/: icmp.*//'  | awk  '{print $NF}' | sed 's:(::' | sed 's:)::'
+#    if [ $do_raw = 1 ]; then
+#	echo $1
+#    else
+#	#host $1 | grep "has address" | head -n 1 | awk '{print $NF}' | sed 's:(::' | sed 's:)::'
+#    fi
+}
+function id_for_node() {
+    ip_for_node $* | tr '.' ' ' | awk '{print $4}'
+}
+function name_for_node() {
+    echo $1 | awk -F. '{print $1}'
+}
+
+function helptext() {
+    echo "cluster-init - Configure cluster communication for the infrastructures supported by Pacemaker"
+    echo ""
+    echo "-g, --group         Specify the group to operate on/with"
+    echo "-w, --host          Specify a host to operate on/with.  May be specified multiple times"
+    echo "-r, --raw-ip        Supplied nodes were listed as their IP addresses"
+    echo ""
+    echo "-c, --corosync      configure for corosync"
+    echo "-C, --nodelist      configure for corosync with a node list"
+    echo "-u, --unicast       configure point-to-point communication instead of multicast"
+    echo ""
+    echo "-I, --install       Install packages"
+    echo ""
+    echo "-d, --debug         Enable debug logging for the cluster"
+    echo "--hosts             Copy the local /etc/hosts file to all nodes"
+    echo "-e, --extra list    Whitespace separated list of extra packages to install"
+    echo "-l, --limit N       Use the first N hosts from the named group" 
+    echo "                    Extra packages to install"
+    exit $1
+}
+
+host_input=""
+while true; do
+    case "$1" in
+	-g) cluster=$2;
+	    shift; shift;;
+	-w|--host)
+	    for h in $2; do
+		host_input="$host_input -w $h";
+	    done
+	    shift; shift;;
+	-w) host_input="$host_input -w $2"
+	    shift; shift;;
+	-r|--raw-ip) do_raw=1;       shift;;
+
+	-d|--debug) do_debug=on; shift;;
+
+	-I|--install) INSTALL=Yes; shift;;
+	--hosts) ETCHOSTS=1; shift;;
+
+	-c|--corosync) CTYPE=corosync; shift;;
+	-C|--nodelist) CTYPE=corosync; nodelist=1; shift;;
+	-u|--unicast) nodelist=1; transport=udpu; inaddr_any="yes"; shift;;
+	-e|--extra) pkgs="$pkgs $2"; shift; shift;;
+	-t|--test)  pkgs="$pkgs valgrind"; shift;;
+	-l|--limit) limit=$2; shift; shift;;
+
+	r*[0-9])
+	    rhel=`echo $1 | sed -e s/rhel// -e s/-// -e s/r//`
+	    pkgs="$pkgs qarsh-server";
+	    case $rhel in
+		7) CTYPE=corosync;;
+	    esac
+	    shift
+	    ;;
+
+	f*[0-9][0-9])
+	    CTYPE=corosync;
+	    shift
+	    ;;
+
+	-y|--yes|--defaults) accept_defaults=1; shift;;
+	-x) set -x; shift;;
+	-\?|--help) helptext 0; shift;;
+	"") break;;
+	*) echo "unknown option: $1"; exit 1;;
+    esac
+done
+
+if [ ! -z $cluster ]; then
+    host_input="-g $cluster"
+    # use the last digit present in the variable (if any)
+    dsh_group=`echo $cluster | sed 's/[^0-9][^0-9]*//g;s/.*\([0-9]\)$/\1/'`
+fi
+
+if [ -z $dsh_group ]; then
+    dsh_group=1
+fi
+
+if [ x = "x$host_input" -a x = "x$cluster" ]; then
+    if [ -d $HOME/.dsh/group ]; then
+        read -p "Please specify a dsh group you'd like to configure as a cluster? [] " -t 60 cluster
+    else
+        read -p "Please specify a whitespace delimetered list of nodes you'd like to configure as a cluster? [] " -t 60 host_list
+
+        for h in $2; do
+	    host_input="$host_input -w $h";
+	done
+    fi
+fi
+
+if [ -z "$host_input" ]; then
+    echo "You didn't specify any nodes or groups to configure"
+    exit 1
+fi
+
+if [ $limit -gt 0 ]; then
+    echo "Using only the first $limit hosts in $cluster group"
+    host_list=`cluster-helper --list bullet $host_input | head -n $limit | tr '\n*' '  '`
+else
+    host_list=`cluster-helper --list short $host_input`
+fi
+num_hosts=`echo $host_list | wc -w`
+
+if [ $num_hosts -gt 9 ]; then
+    cs_port=66
+fi
+
+for h in $host_list; do
+    ping -c 1 -q $h
+    if [ $? != 0 ]; then
+	echo "Using long names..."
+	host_list=`cluster-helper --list long $host_input`
+	break
+    fi
+done
+
+if [ -z $CTYPE ]; then
+    echo ""
+    read -p "Where should Pacemaker obtain membership and quorum from? [corosync] (corosync) " -t 60 CTYPE
+fi
+
+case $CTYPE in
+    corosync) cs_conf="@PCMK__COROSYNC_CONF@" ;;
+esac
+
+function get_defaults()
+{
+    if [ -z $SSH ]; then
+	SSH="No"
+    fi
+
+    if [ -z $SELINUX ]; then
+	SELINUX="No"
+    fi
+
+    if [ -z $IPTABLES ]; then
+	IPTABLES="Yes"
+    fi
+
+    if [ -z $DOMAIN ]; then
+	DOMAIN="No"
+    fi
+    if [ -z $INSTALL ]; then
+	INSTALL="Yes"
+    fi
+    if [ -z $DATE ]; then
+	DATE="No"
+    fi
+}
+
+get_defaults
+if [ $accept_defaults = 0 ]; then
+    echo ""
+    read -p "Shall I install an ssh key to cluster nodes? [$SSH] " -t 60 SSH
+    echo ""
+    echo "SELinux prevent many things, including password-less ssh logins"
+    read -p "Shall I disable selinux? [$SELINUX] " -t 60 SELINUX
+    echo ""
+    echo "Incorrectly configured firewalls will prevent corosync from starting up"
+    read -p "Shall I disable iptables? [$IPTABLES] " -t 60 IPTABLES
+
+    echo ""
+    read -p "Shall I install/update the relevant packages? [$INSTALL] "  -t 60 INSTALL
+
+    echo ""
+    read -p "Shall I sync the date/time? [$DATE] "  -t 60 DATE
+fi
+get_defaults
+
+echo ""
+echo "Detecting possible fencing options"
+if [ -e /etc/cluster/fence_xvm.key ]; then
+    echo "* Found fence_xvm"
+    fence_conf=/etc/cluster/fence_xvm.key
+    pkgs="$pkgs fence-virt"
+fi
+
+if [ ! -z ${OS_AUTH_URL} ]; then
+    echo "* Found openstack credentials"
+    fence_conf=/sbin/fence_openstack
+    pkgs="$pkgs python-novaclient"
+fi
+echo ""
+echo "Beginning cluster configuration"
+echo ""
+
+case $SSH in
+    [Yy][Ee][Ss]|[Yy])
+	for host in $host_list; do
+	    echo "Installing our ssh key on ${host}"
+	    ssh-copy-id root@${host} >/dev/null 2>&1
+	    # Fix selinux labeling
+	    ssh -l root ${host} -- restorecon -R -v .
+	done
+	;;
+esac
+
+case $DATE in
+    [Yy][Ee][Ss]|[Yy])
+	for host in $host_list; do
+	    echo "Setting time on ${host}"
+	    scp /etc/localtime root@${host}:/etc
+	    now=`date +%s`
+	    ssh -l root ${host} -- date -s @$now
+	    echo ""
+	done
+	;;
+esac
+
+init=`mktemp`
+cat<<-END>$init
+verbose=0
+pkgs="$pkgs"
+
+lhost=\`uname -n\`
+lshort=\`echo \$lhost | awk -F. '{print \$1}'\`
+
+log() {
+    printf "%-10s  \$*\n" "\$lshort:" 1>&2
+}
+
+debug() {
+    if [ \$verbose -gt 0 ]; then
+	log "Debug: \$*"
+    fi
+}
+
+info() {
+    log "\$*"
+}
+
+warning() {
+    log "WARN: \$*"
+}
+
+fatal() {
+    log "ERROR: \$*"
+    exit 1
+}
+
+case $SELINUX in
+    [Yy][Ee][Ss]|[Yy])
+	sed -i.sed "s/enforcing/disabled/g" /etc/selinux/config
+	;;
+esac
+
+case $IPTABLES in
+    [Yy][Ee][Ss]|[Yy]|"")
+	service iptables stop
+	chkconfig iptables off
+	service firewalld stop
+	chkconfig firewalld off
+	;;
+esac
+
+case $DOMAIN in
+    [Nn][Oo]|"")
+	;;
+    *.*)
+	if
+            ! grep domain /etc/resolv.conf
+        then
+	    sed -i.sed "s/nameserver/domain\ $DOMAIN\\\nnameserver/g" /etc/resolv.conf
+	fi
+	;;
+    *) echo "Unknown domain: $DOMAIN";;
+esac
+
+case $INSTALL in
+    [Yy][Ee][Ss]|[Yy]|"")
+	info Installing cluster software
+	yum install -y $pkgs pacemaker
+	;;
+esac
+
+info "Configuring services"
+chkconfig xinetd on
+service xinetd start &>/dev/null
+
+chkconfig corosync off &> /dev/null
+mkdir -p /etc/cluster
+
+info "Turning on core files"
+grep -q "unlimited" /etc/bashrc
+if [ $? = 1 ]; then
+    sed -i.sed "s/bashrc/bashrc\\\nulimit\ -c\ unlimited/g" /etc/bashrc
+fi
+
+function patch_cs_config() {
+    test $num_hosts != 2
+    two_node=$?
+
+    priority="info"
+    if [ $do_debug = 1 ]; then
+	priority="debug"
+    fi
+
+    ssh -l root ${host} -- sed -i.sed "s/.*mcastaddr:.*/mcastaddr:\ 226.94.1.1/g" $cs_conf
+    ssh -l root ${host} -- sed -i.sed "s/.*mcastport:.*/mcastport:\ $cs_port$dsh_group/g" $cs_conf
+    ssh -l root ${host} -- sed -i.sed "s/.*bindnetaddr:.*/bindnetaddr:\ $ip/g" $cs_conf
+    ssh -l root ${host} -- sed -i.sed "s/.*syslog_facility:.*/syslog_facility:\ daemon/g" $cs_conf
+    ssh -l root ${host} -- sed -i.sed "s/.*logfile_priority:.*/logfile_priority:\ $priority/g" $cs_conf
+
+    if [ ! -z $token ]; then
+	ssh -l root ${host} -- sed -i.sed "s/.*token:.*/token:\ $token/g" $cs_conf
+    fi
+    if [ ! -z $consensus ]; then
+	ssh -l root ${host} -- sed -i.sed "s/.*consensus:.*/consensus:\ $consensus/g" $cs_conf
+    fi
+    if [ ! -z $join ]; then
+	ssh -l root ${host} -- sed -i.sed "s/^join:.*/join:\ $join/g" $cs_conf
+	ssh -l root ${host} -- sed -i.sed "s/\\\Wjoin:.*/join:\ $join/g" $cs_conf
+    fi
+
+    ssh -l root ${host} -- grep -q "corosync_votequorum" $cs_conf 2>&1 > /dev/null
+    if [ $? -eq 0 ]; then
+        ssh -l root ${host} -- sed -i.sed "s/\\\Wexpected_votes:.*/expected_votes:\ $num_hosts/g" $cs_conf
+        ssh -l root ${host} -- sed -i.sed "s/\\\Wtwo_node:.*/two_node:\ $two_node/g" $cs_conf
+    else
+        printf "%-10s  Wrong quorum provider: installing $cs_conf for corosync instead\n" ${host}
+        create_cs_config
+    fi
+}
+
+function create_cs_config() {
+    cs_tmp=/tmp/cs_conf.$$
+    test $num_hosts != 2
+    two_node=$?
+
+    # Base config
+    priority="info"
+    if [ $do_debug = 1 ]; then
+	priority="debug"
+    fi
+
+    cat <<-END >$cs_tmp
+# Please read the corosync.conf.5 manual page
+totem {
+        version: 2
+
+        # cypto_cipher and crypto_hash: Used for mutual node authentication.
+        # If you choose to enable this, then do remember to create a shared
+        # secret with "corosync-keygen".
+        crypto_cipher: none
+        crypto_hash: none
+
+        # Assign a fixed node id
+        nodeid:         $id
+
+        # Disable encryption
+        secauth:        off
+
+	transport:      $transport
+	inaddr_any:     $inaddr_any
+
+        # interface: define at least one interface to communicate
+        # over. If you define more than one interface stanza, you must
+        # also set rrp_mode.
+        interface {
+                # Rings must be consecutively numbered, starting at 0.
+                ringnumber: 0
+
+                # This is normally the *network* address of the
+                # interface to bind to. This ensures that you can use
+                # identical instances of this configuration file
+                # across all your cluster nodes, without having to
+                # modify this option.
+                bindnetaddr: $ip
+
+                # However, if you have multiple physical network
+                # interfaces configured for the same subnet, then the
+                # network address alone is not sufficient to identify
+                # the interface Corosync should bind to. In that case,
+                # configure the *host* address of the interface
+                # instead:
+                # bindnetaddr: 192.168.1.1
+                # When selecting a multicast address, consider RFC
+                # 2365 (which, among other things, specifies that
+                # 239.255.x.x addresses are left to the discretion of
+                # the network administrator). Do not reuse multicast
+                # addresses across multiple Corosync clusters sharing
+                # the same network.
+
+                # Corosync uses the port you specify here for UDP
+                # messaging, and also the immediately preceding
+                # port. Thus if you set this to 5405, Corosync sends
+                # messages over UDP ports 5405 and 5404.
+                mcastport: $cs_port$dsh_group
+
+                # Time-to-live for cluster communication packets. The
+                # number of hops (routers) that this ring will allow
+                # itself to pass. Note that multicast routing must be
+                # specifically enabled on most network routers.
+                ttl: 1
+                mcastaddr: 226.94.1.1
+        }
+}
+
+logging {
+        debug: off
+        fileline: off
+        to_syslog: yes
+        to_stderr: no
+        syslog_facility: daemon
+        timestamp: on
+        to_logfile: yes
+        logfile: /var/log/corosync.log
+        logfile_priority: $priority
+}
+
+amf {
+        mode: disabled
+}
+
+quorum {
+        provider: corosync_votequorum
+        expected_votes: $num_hosts
+        votes: 1
+        two_node: $two_node
+        wait_for_all: 0
+        last_man_standing: 0
+        auto_tie_breaker: 0
+}
+END
+    scp -q $cs_tmp root@${host}:$cs_conf
+    rm -f $cs_tmp
+}
+
+for host in $host_list; do
+    echo ""
+    echo ""
+    echo "* Configuring $host"
+
+    cs_short_host=`name_for_node $host`
+    ip=`ip_for_node $host`
+    id=`id_for_node $host`
+
+    echo $ip | grep -qis NXDOMAIN
+    if [ $? = 0 ]; then
+	echo "Couldn't find resolve $host to an IP address"
+	exit 1
+    fi
+
+    if [ `uname -n` = $host ]; then
+	bash $init
+    else
+	cat $init | ssh -l root -T $host -- "cat > $init; bash $init"
+    fi
+
+    if [ "x$fence_conf" != x ]; then
+	if [ -e $fence_conf ]; then
+	    scp $fence_conf root@${host}:$fence_conf
+	fi
+    fi
+
+    if [ $ETCHOSTS = 1 ]; then
+	scp /etc/hosts root@${host}:/etc/hosts
+    fi
+
+    ssh -l root ${host} -- grep -q "token:" $cs_conf 2>&1 > /dev/null
+    new_config=$?
+    new_config=1
+
+    if [ $new_config = 0 ]; then
+        printf "%-10s  Updating $cs_conf\n" ${host}:
+        patch_cs_config
+    else
+        printf "%-10s  Installing $cs_conf\n" ${host}:
+        create_cs_config
+    fi
+done
diff --git a/tools/crm_attribute.8.inc b/tools/crm_attribute.8.inc
new file mode 100644
index 0000000..2be8253
--- /dev/null
+++ b/tools/crm_attribute.8.inc
@@ -0,0 +1,5 @@
+[synopsis]
+crm_attribute -n <attribute> <command> [options]
+
+/and node attributes/
+.SH OPTIONS
diff --git a/tools/crm_attribute.c b/tools/crm_attribute.c
new file mode 100644
index 0000000..358b150
--- /dev/null
+++ b/tools/crm_attribute.c
@@ -0,0 +1,883 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdint.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <libgen.h>
+#include <time.h>
+
+#include <sys/param.h>
+#include <sys/types.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/common/ipc.h>
+#include <crm/common/util.h>
+#include <crm/cluster.h>
+
+#include <crm/cib.h>
+#include <crm/cib/internal.h>
+#include <crm/common/attrd_internal.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/ipc_attrd_internal.h>
+#include <crm/common/ipc_controld.h>
+#include <crm/common/output_internal.h>
+#include <sys/utsname.h>
+
+#include <pacemaker-internal.h>
+
+#define SUMMARY "crm_attribute - query and update Pacemaker cluster options and node attributes"
+
+GError *error = NULL;
+crm_exit_t exit_code = CRM_EX_OK;
+uint64_t cib_opts = cib_sync_call;
+
+PCMK__OUTPUT_ARGS("attribute", "const char *", "const char *", "const char *",
+                  "const char *", "const char *")
+static int
+attribute_text(pcmk__output_t *out, va_list args)
+{
+    const char *scope = va_arg(args, const char *);
+    const char *instance = va_arg(args, const char *);
+    const char *name = va_arg(args, const char *);
+    const char *value = va_arg(args, const char *);
+    const char *host G_GNUC_UNUSED = va_arg(args, const char *);
+
+    if (out->quiet) {
+        if (value != NULL) {
+            pcmk__formatted_printf(out, "%s\n", value);
+        }
+    } else {
+        out->info(out, "%s%s %s%s %s%s value=%s",
+                  scope ? "scope=" : "", scope ? scope : "",
+                  instance ? "id=" : "", instance ? instance : "",
+                  name ? "name=" : "", name ? name : "",
+                  value ? value : "(null)");
+    }
+
+    return pcmk_rc_ok;
+}
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+static pcmk__message_entry_t fmt_functions[] = {
+    { "attribute", "text", attribute_text },
+
+    { NULL, NULL, NULL }
+};
+
+struct {
+    char command;
+    gchar *attr_default;
+    gchar *attr_id;
+    gchar *attr_name;
+    uint32_t attr_options;
+    gchar *attr_pattern;
+    char *attr_value;
+    char *dest_node;
+    gchar *dest_uname;
+    gboolean inhibit;
+    gchar *set_name;
+    char *set_type;
+    gchar *type;
+    gboolean promotion_score;
+} options = {
+    .command = 'G',
+    .promotion_score = FALSE
+};
+
+#define INDENT "                               "
+
+static gboolean
+delete_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.command = 'D';
+    pcmk__str_update(&options.attr_value, NULL);
+    return TRUE;
+}
+
+static gboolean
+promotion_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    char *score_name = NULL;
+
+    options.promotion_score = TRUE;
+
+    if (options.attr_name) {
+        g_free(options.attr_name);
+    }
+
+    score_name = pcmk_promotion_score_name(optarg);
+    if (score_name != NULL) {
+        options.attr_name = g_strdup(score_name);
+        free(score_name);
+    } else {
+        options.attr_name = NULL;
+    }
+
+    return TRUE;
+}
+
+static gboolean
+update_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.command = 'u';
+    pcmk__str_update(&options.attr_value, optarg);
+    return TRUE;
+}
+
+static gboolean
+utilization_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (options.type) {
+        g_free(options.type);
+    }
+
+    options.type = g_strdup(XML_CIB_TAG_NODES);
+    pcmk__str_update(&options.set_type, XML_TAG_UTILIZATION);
+    return TRUE;
+}
+
+static gboolean
+value_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.command = 'G';
+    pcmk__str_update(&options.attr_value, NULL);
+    return TRUE;
+}
+
+static gboolean
+wait_cb (const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    if (pcmk__str_eq(optarg, "no", pcmk__str_none)) {
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
+        return TRUE;
+    } else if (pcmk__str_eq(optarg, PCMK__VALUE_LOCAL, pcmk__str_none)) {
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
+        pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local);
+        return TRUE;
+    } else if (pcmk__str_eq(optarg, PCMK__VALUE_CLUSTER, pcmk__str_none)) {
+        pcmk__clear_node_attr_flags(options.attr_options, pcmk__node_attr_sync_local | pcmk__node_attr_sync_cluster);
+        pcmk__set_node_attr_flags(options.attr_options, pcmk__node_attr_sync_cluster);
+        return TRUE;
+    } else {
+        g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                    "--wait= must be one of 'no', 'local', 'cluster'");
+        return FALSE;
+    }
+}
+
+static GOptionEntry selecting_entries[] = {
+    { "id", 'i', 0, G_OPTION_ARG_STRING, &options.attr_id,
+      "(Advanced) Operate on instance of specified attribute with this\n"
+      INDENT "XML ID",
+      "XML_ID"
+    },
+
+    { "name", 'n', 0, G_OPTION_ARG_STRING, &options.attr_name,
+      "Operate on attribute or option with this name.  For queries, this\n"
+      INDENT "is optional, in which case all matching attributes will be\n"
+      INDENT "returned.",
+      "NAME"
+    },
+
+    { "pattern", 'P', 0, G_OPTION_ARG_STRING, &options.attr_pattern,
+      "Operate on all attributes matching this pattern\n"
+      INDENT "(with -v, -D, or -G)",
+      "PATTERN"
+    },
+
+    { "promotion", 'p', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, promotion_cb,
+      "Operate on node attribute used as promotion score for specified\n"
+      INDENT "resource, or resource given in OCF_RESOURCE_INSTANCE environment\n"
+      INDENT "variable if none is specified; this also defaults -l/--lifetime\n"
+      INDENT "to reboot (normally invoked from an OCF resource agent)",
+      "RESOURCE"
+    },
+
+    { "set-name", 's', 0, G_OPTION_ARG_STRING, &options.set_name,
+      "(Advanced) Operate on instance of specified attribute that is\n"
+      INDENT "within set with this XML ID",
+      "NAME"
+    },
+
+    { NULL }
+};
+
+static GOptionEntry command_entries[] = {
+    { "delete", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, delete_cb,
+      "Delete the attribute/option",
+      NULL
+    },
+
+    { "query", 'G', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, value_cb,
+      "Query the current value of the attribute/option.\n"
+      INDENT "See also: -n, -P",
+      NULL
+    },
+
+    { "update", 'v', 0, G_OPTION_ARG_CALLBACK, update_cb,
+      "Update the value of the attribute/option",
+      "VALUE"
+    },
+
+    { NULL }
+};
+
+static GOptionEntry addl_entries[] = {
+    { "default", 'd', 0, G_OPTION_ARG_STRING, &options.attr_default,
+      "(Advanced) Default value to display if none is found in configuration",
+      "VALUE"
+    },
+
+    { "lifetime", 'l', 0, G_OPTION_ARG_STRING, &options.type,
+      "Lifetime of the node attribute.\n"
+      INDENT "Valid values: reboot, forever",
+      "LIFETIME"
+    },
+
+    { "node", 'N', 0, G_OPTION_ARG_STRING, &options.dest_uname,
+      "Set a node attribute for named node (instead of a cluster option).\n"
+      INDENT "See also: -l",
+      "NODE"
+    },
+
+    { "type", 't', 0, G_OPTION_ARG_STRING, &options.type,
+      "Which part of the configuration to update/delete/query the option in.\n"
+      INDENT "Valid values: crm_config, rsc_defaults, op_defaults, tickets",
+      "SECTION"
+    },
+
+    { "wait", 'W', 0, G_OPTION_ARG_CALLBACK, wait_cb,
+      "Wait for some event to occur before returning.  Values are 'no' (wait\n"
+      INDENT "only for the attribute daemon to acknowledge the request),\n"
+      INDENT "'local' (wait until the change has propagated to where a local\n"
+      INDENT "query will return the request value, or the value set by a\n"
+      INDENT "later request), or 'cluster' (wait until the change has propagated\n"
+      INDENT "to where a query anywhere on the cluster will return the requested\n"
+      INDENT "value, or the value set by a later request).  Default is 'no'.\n"
+      INDENT "(with -N, and one of -D or -u)",
+      "UNTIL" },
+
+    { "utilization", 'z', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, utilization_cb,
+      "Set an utilization attribute for the node.",
+      NULL
+    },
+
+    { "inhibit-policy-engine", '!', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.inhibit,
+      NULL, NULL
+    },
+
+    { NULL }
+};
+
+static GOptionEntry deprecated_entries[] = {
+    { "attr-id", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.attr_id,
+      NULL, NULL
+    },
+
+    { "attr-name", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.attr_name,
+      NULL, NULL
+    },
+
+    { "attr-value", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, update_cb,
+      NULL, NULL
+    },
+
+    { "delete-attr", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, delete_cb,
+      NULL, NULL
+    },
+
+    { "get-value", 0, G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, value_cb,
+      NULL, NULL
+    },
+
+    { "node-uname", 'U', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.dest_uname,
+      NULL, NULL
+    },
+
+    { NULL }
+};
+
+static void
+get_node_name_from_local(void)
+{
+    char *hostname = pcmk_hostname();
+
+    g_free(options.dest_uname);
+
+    /* This silliness is so that dest_uname is always a glib-managed
+     * string so we know how to free it later.  pcmk_hostname returns
+     * a newly allocated string via strdup.
+     */
+    options.dest_uname = g_strdup(hostname);
+    free(hostname);
+}
+
+static int
+send_attrd_update(char command, const char *attr_node, const char *attr_name,
+                  const char *attr_value, const char *attr_set,
+                  const char *attr_dampen, uint32_t attr_options)
+{
+    int rc = pcmk_rc_ok;
+    uint32_t opts = attr_options;
+
+    switch (command) {
+        case 'D':
+            rc = pcmk__attrd_api_delete(NULL, attr_node, attr_name, opts);
+            break;
+
+        case 'u':
+            rc = pcmk__attrd_api_update(NULL, attr_node, attr_name,
+                                        attr_value, NULL, attr_set, NULL,
+                                        opts | pcmk__node_attr_value);
+            break;
+    }
+
+    if (rc != pcmk_rc_ok) {
+        g_set_error(&error, PCMK__RC_ERROR, rc, "Could not update %s=%s: %s (%d)",
+                    attr_name, attr_value, pcmk_rc_str(rc), rc);
+    }
+
+    return rc;
+}
+
+struct delete_data_s {
+    pcmk__output_t *out;
+    cib_t *cib;
+};
+
+static int
+delete_attr_on_node(xmlNode *child, void *userdata)
+{
+    struct delete_data_s *dd = (struct delete_data_s *) userdata;
+
+    const char *attr_name = crm_element_value(child, XML_NVPAIR_ATTR_NAME);
+    int rc = pcmk_rc_ok;
+
+    if (!pcmk__str_eq(attr_name, options.attr_pattern, pcmk__str_regex)) {
+        return pcmk_rc_ok;
+    }
+
+    rc = cib__delete_node_attr(dd->out, dd->cib, cib_opts, options.type,
+                               options.dest_node, options.set_type,
+                               options.set_name, options.attr_id,
+                               attr_name, options.attr_value, NULL);
+
+    if (rc == ENXIO) {
+        rc = pcmk_rc_ok;
+    }
+
+    return rc;
+}
+
+static int
+command_delete(pcmk__output_t *out, cib_t *cib)
+{
+    int rc = pcmk_rc_ok;
+
+    xmlNode *result = NULL;
+    bool use_pattern = options.attr_pattern != NULL;
+
+    /* See the comment in command_query regarding xpath and regular expressions. */
+    if (use_pattern) {
+        struct delete_data_s dd = { out, cib };
+
+        rc = cib__get_node_attrs(out, cib, options.type, options.dest_node,
+                                 options.set_type, options.set_name, NULL, NULL,
+                                 NULL, &result);
+
+        if (rc != pcmk_rc_ok) {
+            goto done_deleting;
+        }
+
+        rc = pcmk__xe_foreach_child(result, NULL, delete_attr_on_node, &dd);
+
+        if (rc != pcmk_rc_ok) {
+            goto done_deleting;
+        }
+
+    } else {
+        rc = cib__delete_node_attr(out, cib, cib_opts, options.type, options.dest_node,
+                                   options.set_type, options.set_name, options.attr_id,
+                                   options.attr_name, options.attr_value, NULL);
+    }
+
+done_deleting:
+    free_xml(result);
+
+    if (rc == ENXIO) {
+        /* Nothing to delete...
+         * which means it's not there...
+         * which is what the admin wanted
+         */
+        rc = pcmk_rc_ok;
+    }
+
+    return rc;
+}
+
+struct update_data_s {
+    pcmk__output_t *out;
+    cib_t *cib;
+    int is_remote_node;
+};
+
+static int
+update_attr_on_node(xmlNode *child, void *userdata)
+{
+    struct update_data_s *ud = (struct update_data_s *) userdata;
+
+    const char *attr_name = crm_element_value(child, XML_NVPAIR_ATTR_NAME);
+
+    if (!pcmk__str_eq(attr_name, options.attr_pattern, pcmk__str_regex)) {
+        return pcmk_rc_ok;
+    }
+
+    return cib__update_node_attr(ud->out, ud->cib, cib_opts, options.type,
+                                 options.dest_node, options.set_type,
+                                 options.set_name, options.attr_id,
+                                 attr_name, options.attr_value, NULL,
+                                 ud->is_remote_node ? "remote" : NULL);
+}
+
+static int
+command_update(pcmk__output_t *out, cib_t *cib, int is_remote_node)
+{
+    int rc = pcmk_rc_ok;
+
+    xmlNode *result = NULL;
+    bool use_pattern = options.attr_pattern != NULL;
+
+    CRM_LOG_ASSERT(options.type != NULL);
+    CRM_LOG_ASSERT(options.attr_name != NULL);
+    CRM_LOG_ASSERT(options.attr_value != NULL);
+
+    /* See the comment in command_query regarding xpath and regular expressions. */
+    if (use_pattern) {
+        struct update_data_s ud = { out, cib, is_remote_node };
+
+        rc = cib__get_node_attrs(out, cib, options.type, options.dest_node,
+                                 options.set_type, options.set_name, NULL, NULL,
+                                 NULL, &result);
+
+        if (rc != pcmk_rc_ok) {
+            goto done_updating;
+        }
+
+        rc = pcmk__xe_foreach_child(result, NULL, update_attr_on_node, &ud);
+
+        if (rc != pcmk_rc_ok) {
+            goto done_updating;
+        }
+
+    } else {
+        rc = cib__update_node_attr(out, cib, cib_opts, options.type,
+                                   options.dest_node, options.set_type,
+                                   options.set_name, options.attr_id,
+                                   options.attr_name, options.attr_value,
+                                   NULL, is_remote_node ? "remote" : NULL);
+    }
+
+done_updating:
+    free_xml(result);
+    return rc;
+}
+
+struct output_data_s {
+    pcmk__output_t *out;
+    bool use_pattern;
+    bool did_output;
+};
+
+static int
+output_one_attribute(xmlNode *node, void *userdata)
+{
+    struct output_data_s *od = (struct output_data_s *) userdata;
+
+    const char *name = crm_element_value(node, XML_NVPAIR_ATTR_NAME);
+    const char *value = crm_element_value(node, XML_NVPAIR_ATTR_VALUE);
+    const char *host = crm_element_value(node, PCMK__XA_ATTR_NODE_NAME);
+
+    const char *type = options.type;
+    const char *attr_id = options.attr_id;
+
+    if (od->use_pattern && !pcmk__str_eq(name, options.attr_pattern, pcmk__str_regex)) {
+        return pcmk_rc_ok;
+    }
+
+    od->out->message(od->out, "attribute", type, attr_id, name, value, host);
+    od->did_output = true;
+    crm_info("Read %s='%s' %s%s",
+             pcmk__s(name, "<null>"), pcmk__s(value, ""),
+             options.set_name ? "in " : "", options.set_name ? options.set_name : "");
+
+    return pcmk_rc_ok;
+}
+
+static int
+command_query(pcmk__output_t *out, cib_t *cib)
+{
+    int rc = pcmk_rc_ok;
+
+    xmlNode *result = NULL;
+    bool use_pattern = options.attr_pattern != NULL;
+
+    /* libxml2 doesn't support regular expressions in xpath queries (which is how
+     * cib__get_node_attrs -> find_attr finds attributes).  So instead, we'll just
+     * find all the attributes for a given node here by passing NULL for attr_id
+     * and attr_name, and then later see if they match the given pattern.
+     */
+    if (use_pattern) {
+        rc = cib__get_node_attrs(out, cib, options.type, options.dest_node,
+                                 options.set_type, options.set_name, NULL,
+                                 NULL, NULL, &result);
+    } else {
+        rc = cib__get_node_attrs(out, cib, options.type, options.dest_node,
+                                 options.set_type, options.set_name, options.attr_id,
+                                 options.attr_name, NULL, &result);
+    }
+
+    if (rc == ENXIO && options.attr_default) {
+        /* Make static analysis happy */
+        const char *type = options.type;
+        const char *attr_id = options.attr_id;
+        const char *attr_name = options.attr_name;
+        const char *attr_default = options.attr_default;
+        const char *dest_uname = options.dest_uname;
+
+        out->message(out, "attribute", type, attr_id, attr_name, attr_default,
+                     dest_uname);
+        rc = pcmk_rc_ok;
+
+    } else if (rc != pcmk_rc_ok) {
+        // Don't do anything.
+
+    } else if (xml_has_children(result)) {
+        struct output_data_s od = { out, use_pattern, false };
+
+        pcmk__xe_foreach_child(result, NULL, output_one_attribute, &od);
+
+        if (!od.did_output) {
+            rc = ENXIO;
+        }
+
+    } else {
+        struct output_data_s od = { out, use_pattern, false };
+        output_one_attribute(result, &od);
+    }
+
+    free_xml(result);
+    return rc;
+}
+
+static void
+set_type(void)
+{
+    if (options.type == NULL) {
+        if (options.promotion_score) {
+            // Updating a promotion score node attribute
+            options.type = g_strdup(XML_CIB_TAG_STATUS);
+
+        } else if (options.dest_uname != NULL) {
+            // Updating some other node attribute
+            options.type = g_strdup(XML_CIB_TAG_NODES);
+
+        } else {
+            // Updating cluster options
+            options.type = g_strdup(XML_CIB_TAG_CRMCONFIG);
+        }
+
+    } else if (pcmk__str_eq(options.type, "reboot", pcmk__str_casei)) {
+        options.type = g_strdup(XML_CIB_TAG_STATUS);
+
+    } else if (pcmk__str_eq(options.type, "forever", pcmk__str_casei)) {
+        options.type = g_strdup(XML_CIB_TAG_NODES);
+    }
+}
+
+static bool
+use_attrd(void)
+{
+    /* Only go through the attribute manager for transient attributes, and
+     * then only if we're not using a file as the CIB.
+     */
+    return pcmk__str_eq(options.type, XML_CIB_TAG_STATUS, pcmk__str_casei) &&
+           getenv("CIB_file") == NULL && getenv("CIB_shadow") == NULL;
+}
+
+static bool
+try_ipc_update(void)
+{
+    return use_attrd() && (options.command == 'D' || options.command == 'u');
+}
+
+static bool
+pattern_used_correctly(void)
+{
+    /* --pattern can only be used with:
+     * -G (query), -v (update), or -D (delete)
+     */
+    return options.command == 'G' || options.command == 'u' || options.command == 'D';
+}
+
+static bool
+delete_used_correctly(void)
+{
+    return options.command != 'D' || options.attr_name != NULL || options.attr_pattern != NULL;
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    GOptionEntry extra_prog_entries[] = {
+        { "quiet", 'q', 0, G_OPTION_ARG_NONE, &(args->quiet),
+          "Print only the value on stdout",
+          NULL },
+
+        { "quiet", 'Q', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &(args->quiet),
+          NULL, NULL
+        },
+
+        { NULL }
+    };
+
+    const char *description = "Examples:\n\n"
+                              "Add new node attribute called 'location' with the value of 'office' for host 'myhost':\n\n"
+                              "\tcrm_attribute --node myhost --name location --update office\n\n"
+                              "Query the value of the 'location' node attribute for host 'myhost':\n\n"
+                              "\tcrm_attribute --node myhost --name location --query\n\n"
+                              "Change the value of the 'location' node attribute for host 'myhost':\n\n"
+                              "\tcrm_attribute --node myhost --name location --update backoffice\n\n"
+                              "Delete the 'location' node attribute for host 'myhost':\n\n"
+                              "\tcrm_attribute --node myhost --name location --delete\n\n"
+                              "Query the value of the 'cluster-delay' cluster option:\n\n"
+                              "\tcrm_attribute --type crm_config --name cluster-delay --query\n\n"
+                              "Query value of the 'cluster-delay' cluster option and print only the value:\n\n"
+                              "\tcrm_attribute --type crm_config --name cluster-delay --query --quiet\n\n";
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
+    pcmk__add_main_args(context, extra_prog_entries);
+    g_option_context_set_description(context, description);
+
+    pcmk__add_arg_group(context, "selections", "Selecting attributes:",
+                        "Show selecting options", selecting_entries);
+    pcmk__add_arg_group(context, "command", "Commands:",
+                        "Show command options", command_entries);
+    pcmk__add_arg_group(context, "additional", "Additional options:",
+                        "Show additional options", addl_entries);
+    pcmk__add_arg_group(context, "deprecated", "Deprecated Options:",
+                        "Show deprecated options", deprecated_entries);
+
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    cib_t *the_cib = NULL;
+    int is_remote_node = 0;
+
+    int rc = pcmk_rc_ok;
+
+    pcmk__output_t *out = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "NPUdilnpstv");
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("crm_attribute", args->verbosity);
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    pcmk__register_lib_messages(out);
+    pcmk__register_messages(out, fmt_functions);
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    out->quiet = args->quiet;
+
+    if (options.promotion_score && options.attr_name == NULL) {
+        exit_code = CRM_EX_USAGE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "-p/--promotion must be called from an OCF resource agent "
+                    "or with a resource ID specified");
+        goto done;
+    }
+
+    if (options.inhibit) {
+        crm_warn("Inhibiting notifications for this update");
+        cib__set_call_options(cib_opts, crm_system_name, cib_inhibit_notify);
+    }
+
+    the_cib = cib_new();
+    rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command);
+    rc = pcmk_legacy2rc(rc);
+
+    if (rc != pcmk_rc_ok) {
+        exit_code = pcmk_rc2exitc(rc);
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not connect to the CIB: %s", pcmk_rc_str(rc));
+        goto done;
+    }
+
+    set_type();
+
+    // Use default node if not given (except for cluster options and tickets)
+    if (!pcmk__strcase_any_of(options.type, XML_CIB_TAG_CRMCONFIG, XML_CIB_TAG_TICKETS,
+                              NULL)) {
+        /* If we are being called from a resource agent via the cluster,
+         * the correct local node name will be passed as an environment
+         * variable. Otherwise, we have to ask the cluster.
+         */
+        const char *target = pcmk__node_attr_target(options.dest_uname);
+
+        if (target != NULL) {
+            g_free(options.dest_uname);
+            options.dest_uname = g_strdup(target);
+        } else if (getenv("CIB_file") != NULL && options.dest_uname == NULL) {
+            get_node_name_from_local();
+        }
+
+        if (options.dest_uname == NULL) {
+            char *node_name = NULL;
+
+            rc = pcmk__query_node_name(out, 0, &node_name, 0);
+
+            if (rc != pcmk_rc_ok) {
+                exit_code = pcmk_rc2exitc(rc);
+                free(node_name);
+                goto done;
+            }
+            options.dest_uname = g_strdup(node_name);
+            free(node_name);
+        }
+
+        rc = query_node_uuid(the_cib, options.dest_uname, &options.dest_node, &is_remote_node);
+        rc = pcmk_legacy2rc(rc);
+
+        if (rc != pcmk_rc_ok) {
+            exit_code = pcmk_rc2exitc(rc);
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Could not map name=%s to a UUID", options.dest_uname);
+            goto done;
+        }
+    }
+
+    if (!delete_used_correctly()) {
+        exit_code = CRM_EX_USAGE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Error: must specify attribute name or pattern to delete");
+        goto done;
+    }
+
+    if (options.attr_pattern) {
+        if (options.attr_name) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Error: --name and --pattern cannot be used at the same time");
+            goto done;
+        }
+
+        if (!pattern_used_correctly()) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Error: pattern can only be used with delete, query, or update");
+            goto done;
+        }
+
+        g_free(options.attr_name);
+        options.attr_name = options.attr_pattern;
+        options.attr_options |= pcmk__node_attr_pattern;
+    }
+
+    if (is_remote_node) {
+        options.attr_options |= pcmk__node_attr_remote;
+    }
+
+    if (pcmk__str_eq(options.set_type, XML_TAG_UTILIZATION, pcmk__str_none)) {
+        options.attr_options |= pcmk__node_attr_utilization;
+    }
+
+    if (try_ipc_update() &&
+        (send_attrd_update(options.command, options.dest_uname, options.attr_name,
+                           options.attr_value, options.set_name, NULL, options.attr_options) == pcmk_rc_ok)) {
+        crm_info("Update %s=%s sent via pacemaker-attrd",
+                 options.attr_name, ((options.command == 'D')? "<none>" : options.attr_value));
+
+    } else if (options.command == 'D') {
+        rc = command_delete(out, the_cib);
+
+    } else if (options.command == 'u') {
+        rc = command_update(out, the_cib, is_remote_node);
+
+    } else {
+        rc = command_query(out, the_cib);
+    }
+
+    if (rc == ENOTUNIQ) {
+        exit_code = pcmk_rc2exitc(rc);
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Please choose from one of the matches below and supply the 'id' with --attr-id");
+
+    } else if (rc != pcmk_rc_ok) {
+        exit_code = pcmk_rc2exitc(rc);
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Error performing operation: %s", pcmk_strerror(rc));
+    }
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    free(options.attr_default);
+    g_free(options.attr_id);
+    g_free(options.attr_name);
+    free(options.attr_value);
+    free(options.dest_node);
+    g_free(options.dest_uname);
+    g_free(options.set_name);
+    free(options.set_type);
+    g_free(options.type);
+
+    cib__clean_up_connection(&the_cib);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+
+    pcmk__unregister_formats();
+    return crm_exit(exit_code);
+}
diff --git a/tools/crm_diff.8.inc b/tools/crm_diff.8.inc
new file mode 100644
index 0000000..4229cbf
--- /dev/null
+++ b/tools/crm_diff.8.inc
@@ -0,0 +1,5 @@
+[synopsis]
+crm_diff original_xml operation [options]
+
+/as a patch/
+.SH OPTIONS
diff --git a/tools/crm_diff.c b/tools/crm_diff.c
new file mode 100644
index 0000000..efe2fcf
--- /dev/null
+++ b/tools/crm_diff.c
@@ -0,0 +1,399 @@
+/*
+ * Copyright 2005-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/param.h>
+#include <sys/types.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/output_internal.h>
+#include <crm/common/xml.h>
+#include <crm/common/ipc.h>
+#include <crm/cib.h>
+
+#define SUMMARY "Compare two Pacemaker configurations (in XML format) to produce a custom diff-like output, " \
+                "or apply such an output as a patch"
+
+struct {
+    gboolean apply;
+    gboolean as_cib;
+    gboolean no_version;
+    gboolean raw_1;
+    gboolean raw_2;
+    gboolean use_stdin;
+    char *xml_file_1;
+    char *xml_file_2;
+} options;
+
+gboolean new_string_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean original_string_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean patch_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+
+static GOptionEntry original_xml_entries[] = {
+    { "original", 'o', 0, G_OPTION_ARG_STRING, &options.xml_file_1,
+      "XML is contained in the named file",
+      "FILE" },
+    { "original-string", 'O', 0, G_OPTION_ARG_CALLBACK, original_string_cb,
+      "XML is contained in the supplied string",
+      "STRING" },
+
+    { NULL }
+};
+
+static GOptionEntry operation_entries[] = {
+    { "new", 'n', 0, G_OPTION_ARG_STRING, &options.xml_file_2,
+      "Compare the original XML to the contents of the named file",
+      "FILE" },
+    { "new-string", 'N', 0, G_OPTION_ARG_CALLBACK, new_string_cb,
+      "Compare the original XML with the contents of the supplied string",
+      "STRING" },
+    { "patch", 'p', 0, G_OPTION_ARG_CALLBACK, patch_cb,
+      "Patch the original XML with the contents of the named file",
+      "FILE" },
+
+    { NULL }
+};
+
+static GOptionEntry addl_entries[] = {
+    { "cib", 'c', 0, G_OPTION_ARG_NONE, &options.as_cib,
+      "Compare/patch the inputs as a CIB (includes versions details)",
+      NULL },
+    { "stdin", 's', 0, G_OPTION_ARG_NONE, &options.use_stdin,
+      "",
+      NULL },
+    { "no-version", 'u', 0, G_OPTION_ARG_NONE, &options.no_version,
+      "Generate the difference without versions details",
+      NULL },
+
+    { NULL }
+};
+
+gboolean
+new_string_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.raw_2 = TRUE;
+    pcmk__str_update(&options.xml_file_2, optarg);
+    return TRUE;
+}
+
+gboolean
+original_string_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.raw_1 = TRUE;
+    pcmk__str_update(&options.xml_file_1, optarg);
+    return TRUE;
+}
+
+gboolean
+patch_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.apply = TRUE;
+    pcmk__str_update(&options.xml_file_2, optarg);
+    return TRUE;
+}
+
+static void
+print_patch(xmlNode *patch)
+{
+    char *buffer = dump_xml_formatted(patch);
+
+    printf("%s", pcmk__s(buffer, "<null>\n"));
+    free(buffer);
+    fflush(stdout);
+}
+
+// \return Standard Pacemaker return code
+static int
+apply_patch(xmlNode *input, xmlNode *patch, gboolean as_cib)
+{
+    xmlNode *output = copy_xml(input);
+    int rc = xml_apply_patchset(output, patch, as_cib);
+
+    rc = pcmk_legacy2rc(rc);
+    if (rc != pcmk_rc_ok) {
+        fprintf(stderr, "Could not apply patch: %s\n", pcmk_rc_str(rc));
+        free_xml(output);
+        return rc;
+    }
+
+    if (output != NULL) {
+        const char *version;
+        char *buffer;
+
+        print_patch(output);
+
+        version = crm_element_value(output, XML_ATTR_CRM_VERSION);
+        buffer = calculate_xml_versioned_digest(output, FALSE, TRUE, version);
+        crm_trace("Digest: %s", pcmk__s(buffer, "<null>\n"));
+        free(buffer);
+        free_xml(output);
+    }
+    return pcmk_rc_ok;
+}
+
+static void
+log_patch_cib_versions(xmlNode *patch)
+{
+    int add[] = { 0, 0, 0 };
+    int del[] = { 0, 0, 0 };
+
+    const char *fmt = NULL;
+    const char *digest = NULL;
+
+    xml_patch_versions(patch, add, del);
+    fmt = crm_element_value(patch, "format");
+    digest = crm_element_value(patch, XML_ATTR_DIGEST);
+
+    if (add[2] != del[2] || add[1] != del[1] || add[0] != del[0]) {
+        crm_info("Patch: --- %d.%d.%d %s", del[0], del[1], del[2], fmt);
+        crm_info("Patch: +++ %d.%d.%d %s", add[0], add[1], add[2], digest);
+    }
+}
+
+static void
+strip_patch_cib_version(xmlNode *patch, const char **vfields, size_t nvfields)
+{
+    int format = 1;
+
+    crm_element_value_int(patch, "format", &format);
+    if (format == 2) {
+        xmlNode *version_xml = find_xml_node(patch, "version", FALSE);
+
+        if (version_xml) {
+            free_xml(version_xml);
+        }
+
+    } else {
+        int i = 0;
+
+        const char *tags[] = {
+            XML_TAG_DIFF_REMOVED,
+            XML_TAG_DIFF_ADDED,
+        };
+
+        for (i = 0; i < PCMK__NELEM(tags); i++) {
+            xmlNode *tmp = NULL;
+            int lpc;
+
+            tmp = find_xml_node(patch, tags[i], FALSE);
+            if (tmp) {
+                for (lpc = 0; lpc < nvfields; lpc++) {
+                    xml_remove_prop(tmp, vfields[lpc]);
+                }
+
+                tmp = find_xml_node(tmp, XML_TAG_CIB, FALSE);
+                if (tmp) {
+                    for (lpc = 0; lpc < nvfields; lpc++) {
+                        xml_remove_prop(tmp, vfields[lpc]);
+                    }
+                }
+            }
+        }
+    }
+}
+
+// \return Standard Pacemaker return code
+static int
+generate_patch(xmlNode *object_1, xmlNode *object_2, const char *xml_file_2,
+               gboolean as_cib, gboolean no_version)
+{
+    xmlNode *output = NULL;
+    int rc = pcmk_rc_ok;
+
+    pcmk__output_t *logger_out = NULL;
+    int out_rc = pcmk_rc_no_output;
+    int temp_rc = pcmk_rc_no_output;
+
+    const char *vfields[] = {
+        XML_ATTR_GENERATION_ADMIN,
+        XML_ATTR_GENERATION,
+        XML_ATTR_NUMUPDATES,
+    };
+
+    rc = pcmk__log_output_new(&logger_out);
+    CRM_CHECK(rc == pcmk_rc_ok, return rc);
+
+    /* If we're ignoring the version, make the version information
+     * identical, so it isn't detected as a change. */
+    if (no_version) {
+        int lpc;
+
+        for (lpc = 0; lpc < PCMK__NELEM(vfields); lpc++) {
+            crm_copy_xml_element(object_1, object_2, vfields[lpc]);
+        }
+    }
+
+    xml_track_changes(object_2, NULL, object_2, FALSE);
+    if(as_cib) {
+        xml_calculate_significant_changes(object_1, object_2);
+    } else {
+        xml_calculate_changes(object_1, object_2);
+    }
+    crm_log_xml_debug(object_2, (xml_file_2? xml_file_2: "target"));
+
+    output = xml_create_patchset(0, object_1, object_2, NULL, FALSE);
+
+    pcmk__output_set_log_level(logger_out, LOG_INFO);
+    out_rc = pcmk__xml_show_changes(logger_out, object_2);
+
+    xml_accept_changes(object_2);
+
+    if (output == NULL) {
+        goto done;  // rc == pcmk_rc_ok
+    }
+
+    /* pcmk_rc_error means there's non-empty diff.
+     * @COMPAT: Choose a more descriptive return code, like one that maps to
+     * CRM_EX_DIGEST?
+     */
+    rc = pcmk_rc_error;
+
+    patchset_process_digest(output, object_1, object_2, as_cib);
+
+    if (as_cib) {
+        log_patch_cib_versions(output);
+
+    } else if (no_version) {
+        strip_patch_cib_version(output, vfields, PCMK__NELEM(vfields));
+    }
+
+    pcmk__output_set_log_level(logger_out, LOG_NOTICE);
+    temp_rc = logger_out->message(logger_out, "xml-patchset", output);
+    out_rc = pcmk__output_select_rc(out_rc, temp_rc);
+
+    print_patch(output);
+    free_xml(output);
+
+done:
+    logger_out->finish(logger_out, pcmk_rc2exitc(out_rc), true, NULL);
+    pcmk__output_free(logger_out);
+
+    return rc;
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args) {
+    GOptionContext *context = NULL;
+
+    const char *description = "Examples:\n\n"
+                              "Obtain the two different configuration files by running cibadmin on the two cluster setups to compare:\n\n"
+                              "\t# cibadmin --query > cib-old.xml\n\n"
+                              "\t# cibadmin --query > cib-new.xml\n\n"
+                              "Calculate and save the difference between the two files:\n\n"
+                              "\t# crm_diff --original cib-old.xml --new cib-new.xml > patch.xml\n\n"
+                              "Apply the patch to the original file:\n\n"
+                              "\t# crm_diff --original cib-old.xml --patch patch.xml > updated.xml\n\n"
+                              "Apply the patch to the running cluster:\n\n"
+                              "\t# cibadmin --patch -x patch.xml\n";
+
+    context = pcmk__build_arg_context(args, NULL, NULL, NULL);
+    g_option_context_set_description(context, description);
+
+    pcmk__add_arg_group(context, "xml", "Original XML:",
+                        "Show original XML options", original_xml_entries);
+    pcmk__add_arg_group(context, "operation", "Operation:",
+                        "Show operation options", operation_entries);
+    pcmk__add_arg_group(context, "additional", "Additional Options:",
+                        "Show additional options", addl_entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    xmlNode *object_1 = NULL;
+    xmlNode *object_2 = NULL;
+
+    crm_exit_t exit_code = CRM_EX_OK;
+    GError *error = NULL;
+
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "nopNO");
+    GOptionContext *context = build_arg_context(args);
+
+    int rc = pcmk_rc_ok;
+
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("crm_diff", args->verbosity);
+
+    if (args->version) {
+        g_strfreev(processed_args);
+        pcmk__free_arg_context(context);
+        /* FIXME:  When crm_diff is converted to use formatted output, this can go. */
+        pcmk__cli_help('v');
+    }
+
+    if (options.apply && options.no_version) {
+        fprintf(stderr, "warning: -u/--no-version ignored with -p/--patch\n");
+    } else if (options.as_cib && options.no_version) {
+        fprintf(stderr, "error: -u/--no-version incompatible with -c/--cib\n");
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    if (options.raw_1) {
+        object_1 = string2xml(options.xml_file_1);
+
+    } else if (options.use_stdin) {
+        fprintf(stderr, "Input first XML fragment:");
+        object_1 = stdin2xml();
+
+    } else if (options.xml_file_1 != NULL) {
+        object_1 = filename2xml(options.xml_file_1);
+    }
+
+    if (options.raw_2) {
+        object_2 = string2xml(options.xml_file_2);
+
+    } else if (options.use_stdin) {
+        fprintf(stderr, "Input second XML fragment:");
+        object_2 = stdin2xml();
+
+    } else if (options.xml_file_2 != NULL) {
+        object_2 = filename2xml(options.xml_file_2);
+    }
+
+    if (object_1 == NULL) {
+        fprintf(stderr, "Could not parse the first XML fragment\n");
+        exit_code = CRM_EX_DATAERR;
+        goto done;
+    }
+    if (object_2 == NULL) {
+        fprintf(stderr, "Could not parse the second XML fragment\n");
+        exit_code = CRM_EX_DATAERR;
+        goto done;
+    }
+
+    if (options.apply) {
+        rc = apply_patch(object_1, object_2, options.as_cib);
+    } else {
+        rc = generate_patch(object_1, object_2, options.xml_file_2, options.as_cib, options.no_version);
+    }
+    exit_code = pcmk_rc2exitc(rc);
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+    free(options.xml_file_1);
+    free(options.xml_file_2);
+    free_xml(object_1);
+    free_xml(object_2);
+
+    pcmk__output_and_clear_error(&error, NULL);
+    crm_exit(exit_code);
+}
diff --git a/tools/crm_error.8.inc b/tools/crm_error.8.inc
new file mode 100644
index 0000000..7ef0440
--- /dev/null
+++ b/tools/crm_error.8.inc
@@ -0,0 +1,5 @@
+[synopsis]
+crm_error [options] -- <rc> [...]
+
+/Pacemaker error code/
+.SH OPTIONS
diff --git a/tools/crm_error.c b/tools/crm_error.c
new file mode 100644
index 0000000..8911eae
--- /dev/null
+++ b/tools/crm_error.c
@@ -0,0 +1,175 @@
+/*
+ * Copyright 2012-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include <crm/msg_xml.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/output_internal.h>
+#include <crm/common/strings_internal.h>
+
+#include <crm/crm.h>
+
+#include <pacemaker-internal.h>
+
+#define SUMMARY "crm_error - display name or description of a Pacemaker error code"
+
+struct {
+    gboolean with_name;
+    gboolean do_list;
+    enum pcmk_result_type result_type; // How to interpret result codes
+} options = {
+    .result_type = pcmk_result_legacy,
+};
+
+static gboolean
+result_type_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+               GError **error)
+{
+    if (pcmk__str_any_of(option_name, "--exit", "-X", NULL)) {
+        options.result_type = pcmk_result_exitcode;
+    } else if (pcmk__str_any_of(option_name, "--rc", "-r", NULL)) {
+        options.result_type = pcmk_result_rc;
+    }
+
+    return TRUE;
+}
+
+static GOptionEntry entries[] = {
+    { "name", 'n', 0, G_OPTION_ARG_NONE, &options.with_name,
+      "Show error's name with its description (useful for looking for sources "
+      "of the error in source code)",
+       NULL },
+    { "list", 'l', 0, G_OPTION_ARG_NONE, &options.do_list,
+      "Show all known errors (enabled by default if no rc is specified)",
+      NULL },
+    { "exit", 'X', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, result_type_cb,
+      "Interpret as exit code rather than legacy function return value",
+      NULL },
+    { "rc", 'r', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, result_type_cb,
+      "Interpret as return code rather than legacy function return value",
+      NULL },
+
+    { NULL }
+};
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group,
+                                      "[-- <rc> [<rc>...]]");
+    pcmk__add_main_args(context, entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    crm_exit_t exit_code = CRM_EX_OK;
+    int rc = pcmk_rc_ok;
+
+    pcmk__output_t *out = NULL;
+
+    GError *error = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, NULL);
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("crm_error", args->verbosity);
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Error creating output format %s: %s", args->output_ty,
+                    pcmk_rc_str(rc));
+        goto done;
+    }
+
+    if (g_strv_length(processed_args) < 2) {
+        // If no result codes were specified, list them all
+        options.do_list = TRUE;
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    pcmk__register_lib_messages(out);
+
+    if (options.do_list) {
+        uint32_t flags = pcmk_rc_disp_code|pcmk_rc_disp_desc;
+
+        if (options.with_name) {
+            flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
+                                       "pcmk_rc_disp_flags",
+                                       "pcmk__list_result_codes", flags,
+                                       pcmk_rc_disp_name, "pcmk_rc_disp_name");
+        }
+        pcmk__list_result_codes(out, options.result_type, flags);
+
+    } else {
+        uint32_t flags = pcmk_rc_disp_desc;
+
+        // For text output, print only "[name -] description" by default
+        if (args->verbosity > 0) {
+            flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
+                                       "pcmk_rc_disp_flags",
+                                       "pcmk__show_result_code", flags,
+                                       pcmk_rc_disp_code, "pcmk_rc_disp_code");
+        }
+
+        if (options.with_name) {
+            flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
+                                       "pcmk_rc_disp_flags",
+                                       "pcmk__show_result_code", flags,
+                                       pcmk_rc_disp_name, "pcmk_rc_disp_name");
+        }
+
+        /* Skip #1 because that's the program name. */
+        for (int lpc = 1; processed_args[lpc] != NULL; lpc++) {
+            int code = 0;
+
+            if (pcmk__str_eq(processed_args[lpc], "--", pcmk__str_none)) {
+                continue;
+            }
+            pcmk__scan_min_int(processed_args[lpc], &code, INT_MIN);
+            pcmk__show_result_code(out, code, options.result_type, flags);
+        }
+    }
+
+ done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+    pcmk__unregister_formats();
+    crm_exit(exit_code);
+}
diff --git a/tools/crm_failcount.in b/tools/crm_failcount.in
new file mode 100755
index 0000000..f70fe78
--- /dev/null
+++ b/tools/crm_failcount.in
@@ -0,0 +1,294 @@
+#!@BASH_PATH@
+#
+# Copyright 2009-2018 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+USAGE_TEXT="Usage: crm_failcount <command> [<options>]
+Common options:
+ --help                 Display this text, then exit
+ --version              Display version information, then exit
+ -V, --verbose          Specify multiple times to increase debug output
+ -q, --quiet            Print only the value (if querying)
+
+Commands:
+ -G, --query            Query the current value of the resource's fail count
+ -D, --delete           Delete resource's recorded failures
+
+Additional Options:
+ -r, --resource=value   Name of the resource to use (required)
+ -n, --operation=value  Name of operation to use (instead of all operations)
+ -I, --interval=value   If operation is specified, its interval
+ -N, --node=value       Use failcount on named node (instead of local node)"
+
+
+HELP_TEXT="crm_failcount - Query or delete resource fail counts
+
+$USAGE_TEXT"
+
+# These constants must track crm_exit_t values
+CRM_EX_OK=0
+CRM_EX_USAGE=64
+CRM_EX_NOSUCH=105
+
+exit_usage() {
+	if [ $# -gt 0 ]; then
+		echo "error:" "$@" >&2
+	fi
+	echo
+	echo "$USAGE_TEXT"
+	exit $CRM_EX_USAGE
+}
+
+warn() {
+	echo "warning:" "$@" >&2
+}
+
+interval_re() {
+	echo "^[[:blank:]]*([0-9]+)[[:blank:]]*(${1})[[:blank:]]*$"
+}
+
+# This function should follow crm_get_interval() as closely as possible
+parse_interval() {
+	INT_S="$1"
+
+	INT_8601RE="^P(([0-9]+)Y)?(([0-9]+)M)?(([0-9]+)D)?T?(([0-9]+)H)?(([0-9]+)M)?(([0-9]+)S)?$"
+	
+	if [[ $INT_S =~ $(interval_re "") ]]; then
+		echo $(( ${BASH_REMATCH[1]} * 1000 ))
+
+	elif [[ $INT_S =~ $(interval_re "s|sec") ]]; then
+		echo $(( ${BASH_REMATCH[1]} * 1000 ))
+
+	elif [[ $INT_S =~ $(interval_re "ms|msec") ]]; then
+		echo "${BASH_REMATCH[1]}"
+
+	elif [[ $INT_S =~ $(interval_re "m|min") ]]; then
+		echo $(( ${BASH_REMATCH[1]} * 60000 ))
+
+	elif [[ $INT_S =~ $(interval_re "h|hr") ]]; then
+		echo $(( ${BASH_REMATCH[1]} * 3600000 ))
+
+	elif [[ $INT_S =~ $(interval_re "us|usec") ]]; then
+		echo $(( ${BASH_REMATCH[1]} / 1000 ))
+
+	elif [[ $INT_S =~ ^P([0-9]+)W$ ]]; then
+		echo $(( ${BASH_REMATCH[1]} * 604800000 ))
+
+	elif [[ $INT_S =~ $INT_8601RE ]]; then
+		echo $(( ( ${BASH_REMATCH[2]:-0} * 31536000000 ) \
+			+ ( ${BASH_REMATCH[4]:-0} * 2592000000 ) \
+			+ ( ${BASH_REMATCH[6]:-0} * 86400000 ) \
+			+ ( ${BASH_REMATCH[8]:-0} * 3600000 ) \
+			+ ( ${BASH_REMATCH[10]:-0} * 60000 ) \
+			+ ( ${BASH_REMATCH[12]:-0} * 1000 ) ))
+
+	else
+		warn "Unrecognized interval, using 0"
+		echo "0"
+	fi
+}
+
+query_single_attr() {
+	QSR_TARGET="$1"
+	QSR_ATTR="$2"
+
+	crm_attribute $VERBOSE --quiet --query -t status -d 0 \
+		-N "$QSR_TARGET" -n "$QSR_ATTR"
+}
+
+query_attr_sum() {
+	QAS_TARGET="$1"
+	QAS_PREFIX="$2"
+
+	# Build xpath to match all transient node attributes with prefix
+	QAS_XPATH="/cib/status/node_state[@uname='${QAS_TARGET}']"
+	QAS_XPATH="${QAS_XPATH}/transient_attributes/instance_attributes"
+	QAS_XPATH="${QAS_XPATH}/nvpair[starts-with(@name,'$QAS_PREFIX')]"
+
+	# Query attributes that match xpath
+	# @TODO We ignore stderr because we don't want "no results" to look
+	# like an error, but that also makes $VERBOSE pointless.
+	QAS_ALL=$(cibadmin --query --sync-call --local \
+		--xpath="$QAS_XPATH" 2>/dev/null)
+	QAS_EX=$?
+
+	# "No results" is not an error
+	if [ $QAS_EX -ne $CRM_EX_OK ] && [ $QAS_EX -ne $CRM_EX_NOSUCH ]; then
+		echo "error: could not query CIB for fail counts" >&2
+		exit $QAS_EX
+	fi
+
+	# Extract the attribute values (one per line) from the output
+	QAS_VALUE=$(echo "$QAS_ALL" | sed -n -e \
+		's/.*<nvpair.*value="\([0-9][0-9]*\|INFINITY\)".*>.*/\1/p')
+
+	# Sum the values
+	QAS_SUM=0
+	for i in 0 $QAS_VALUE; do
+		if [ "$i" = "INFINITY" ]; then
+			QAS_SUM="INFINITY"
+			break
+		else
+			QAS_SUM=$(($QAS_SUM + $i))
+		fi
+	done
+	if [ "$QAS_SUM" = "INFINITY" ]; then
+		echo $QAS_SUM
+	elif [ "$QAS_SUM" -ge 1000000 ]; then
+		echo "INFINITY"
+	else
+		echo $QAS_SUM
+	fi
+}
+
+query_failcount() {
+	QF_TARGET="$1"
+	QF_RESOURCE="$2"
+	QF_OPERATION="$3"
+	QF_INTERVAL="$4"
+
+	QF_ATTR_RSC="fail-count-${QF_RESOURCE}"
+
+	if [ -n "$QF_OPERATION" ]; then
+		QF_ATTR_DISPLAY="${QF_ATTR_RSC}#${QF_OPERATION}_${QF_INTERVAL}"
+		QF_COUNT=$(query_single_attr "$QF_TARGET" "$QF_ATTR_DISPLAY")
+	else
+		QF_ATTR_DISPLAY="$QF_ATTR_RSC"
+		QF_COUNT=$(query_attr_sum "$QF_TARGET" "${QF_ATTR_RSC}#")
+	fi
+
+	# @COMPAT attributes set < 1.1.17:
+	# If we didn't find any per-operation failcount,
+	# check whether there is a legacy per-resource failcount.
+	if [ "$QF_COUNT" = "0" ]; then
+		QF_COUNT=$(query_single_attr "$QF_TARGET" "$QF_ATTR_RSC")
+		if [ "$QF_COUNT" != "0" ]; then
+			QF_ATTR_DISPLAY="$QF_ATTR_RSC"
+		fi
+	fi
+
+	# Echo result (comparable to crm_attribute, for backward compatibility)
+	if [ -n "$QUIET" ]; then
+		echo $QF_COUNT
+	else
+		echo "scope=status  name=$QF_ATTR_DISPLAY value=$QF_COUNT"
+	fi
+}
+
+clear_failcount() {
+	CF_TARGET="$1"
+	CF_RESOURCE="$2"
+	CF_OPERATION="$3"
+	CF_INTERVAL="$4"
+
+	if [ -n "$CF_OPERATION" ]; then
+		CF_OPERATION="-n $CF_OPERATION -I ${CF_INTERVAL}ms"
+	fi
+	crm_resource $QUIET $VERBOSE --cleanup \
+		-N "$CF_TARGET" -r "$CF_RESOURCE" $CF_OPERATION
+}
+
+QUIET=""
+VERBOSE=""
+
+command=""
+resource=""
+operation=""
+interval="0"
+target=$(crm_node -n 2>/dev/null)
+
+SHORTOPTS="qDGQVN:U:v:i:l:r:n:I:"
+
+LONGOPTS_COMMON="help,version,verbose,quiet"
+LONGOPTS_COMMANDS="query,delete"
+LONGOPTS_OTHER="resource:,node:,operation:,interval:"
+LONGOPTS_COMPAT="delete-attr,get-value,resource-id:,uname:,lifetime:,attr-value:,attr-id:"
+
+LONGOPTS="$LONGOPTS_COMMON,$LONGOPTS_COMMANDS,$LONGOPTS_OTHER,$LONGOPTS_COMPAT"
+
+TEMP=$(@GETOPT_PATH@ -o $SHORTOPTS --long $LONGOPTS -n crm_failcount -- "$@")
+if [ $? -ne 0 ]; then
+	exit_usage
+fi
+eval set -- "$TEMP" # Quotes around $TEMP are essential
+
+while true ; do
+	case "$1" in
+		--help)
+			echo "$HELP_TEXT"
+			exit $CRM_EX_OK
+			;;
+		--version)
+			crm_attribute --version
+			exit $?
+			;;
+		-q|-Q|--quiet)
+			QUIET="--quiet"
+			shift
+			;;
+		-V|--verbose)
+			VERBOSE="$VERBOSE $1"
+			shift
+			;;
+		-G|--query|--get-value)
+			command="--query"
+			shift
+			;;
+		-D|--delete|--delete-attr)
+			command="--delete"
+			shift
+			;;
+		-r|--resource|--resource-id)
+			resource="$2"
+			shift 2
+			;;
+		-n|--operation)
+			operation="$2"
+			shift 2
+			;;
+		-I|--interval)
+			interval="$2"
+			shift 2
+			;;
+		-N|--node|-U|--uname)
+			target="$2"
+			shift 2
+			;;
+		-v|--attr-value)
+			if [ "$2" = "0" ]; then
+				command="--delete"
+			else
+				warn "ignoring deprecated option '$1' with nonzero value"
+			fi
+			shift 2
+			;;
+		-i|--attr-id|-l|--lifetime)
+			warn "ignoring deprecated option '$1'"
+			shift 2
+			;;
+		--)
+			shift
+			break
+			;;
+		*)
+			exit_usage "unknown option '$1'"
+			;;
+	esac
+done
+
+[ -n "$command" ]  || exit_usage "must specify a command"
+[ -n "$resource" ] || exit_usage "resource name required"
+[ -n "$target" ]   || exit_usage "node name required"
+
+interval=$(parse_interval $interval)
+
+if [ "$command" = "--query" ]; then
+	query_failcount "$target" "$resource" "$operation" "$interval"
+else
+	clear_failcount "$target" "$resource" "$operation" "$interval"
+fi
diff --git a/tools/crm_master.in b/tools/crm_master.in
new file mode 100755
index 0000000..a4769f4
--- /dev/null
+++ b/tools/crm_master.in
@@ -0,0 +1,92 @@
+#!@BASH_PATH@
+#
+# Copyright 2009-2021 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+USAGE_TEXT="Usage: crm_master <command> [<options>]
+
+This command is deprecated. Use crm_attribute with the --promotion option
+instead."
+
+exit_usage() {
+	if [ $# -gt 0 ]; then
+		echo "error:" "$@" >&2
+	fi
+	echo
+	echo "$USAGE_TEXT"
+	exit 1
+}
+
+SHORTOPTS_DEPRECATED="U:Q"
+LONGOPTS_DEPRECATED="uname:,get-value,delete-attr,attr-value:,attr-id:"
+SHORTOPTS="VqGv:DN:l:i:r:"
+LONGOPTS="help,version,verbose,quiet,query,update:,delete,node:,lifetime:,id:,resource:"
+
+TEMP=$(@GETOPT_PATH@ -o ${SHORTOPTS}${SHORTOPTS_DEPRECATED} \
+	--long ${LONGOPTS},${LONGOPTS_DEPRECATED} \
+	-n crm_master -- "$@")
+if [ $? -ne 0 ]; then
+	exit_usage
+fi
+
+eval set -- "$TEMP" # Quotes around $TEMP are essential
+
+# Explicitly set the (usual default) lifetime, so the attribute gets set as a
+# node attribute and not a cluster property.
+options="--lifetime forever"
+
+while true ; do
+	case "$1" in
+		--help) 
+			echo "crm_master - Query, update, or delete a resource's promotion score"
+			echo
+			echo "$USAGE_TEXT"
+			exit 0
+			;;
+		--version)
+			crm_attribute --version
+			exit 0
+			;;
+		--verbose|-V|--quiet|-q|--query|-G|--delete|-D)
+			options="$options $1"
+			shift
+			;;
+		--update|-v|--node|-N|--lifetime|-l|--id|-i)
+			options="$options $1 $2"
+			shift
+			shift
+			;;
+		-r|--resource)
+			OCF_RESOURCE_INSTANCE=$2;
+			shift
+			shift
+			;;
+		--get-value|--delete-attr|-Q) # deprecated
+			options="$options $1"
+			shift
+			;;
+		--uname|-U|--attr-value|--attr-id) # deprecated
+			options="$options $1 $2"
+			shift
+			shift
+			;;
+		--)
+			shift
+			break
+			;;
+		*)
+			exit_usage "unknown option '$1'"
+			;;
+	esac
+done
+
+if [ -z "$OCF_RESOURCE_INSTANCE" ]; then
+	exit_usage "No resource specified"
+fi
+
+crm_attribute -n master-$OCF_RESOURCE_INSTANCE $options
diff --git a/tools/crm_mon.8.inc b/tools/crm_mon.8.inc
new file mode 100644
index 0000000..4bc2d82
--- /dev/null
+++ b/tools/crm_mon.8.inc
@@ -0,0 +1,17 @@
+[synopsis]
+crm_mon [options]
+
+/number of different formats/
+.SH OPTIONS
+
+/less descriptive in output./
+.SH NOTES
+
+/period specification./
+.SH OUTPUT CONTROL
+
+/command line arguments./
+.SH TIME SPECIFICATION
+
+/or --exclude=list./
+.SH INTERACTIVE USE
diff --git a/tools/crm_mon.c b/tools/crm_mon.c
new file mode 100644
index 0000000..c20766c
--- /dev/null
+++ b/tools/crm_mon.c
@@ -0,0 +1,2287 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+
+#include <crm/crm.h>
+
+#include <stdint.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <libgen.h>
+#include <signal.h>
+#include <sys/utsname.h>
+
+#include <crm/msg_xml.h>
+#include <crm/services.h>
+#include <crm/lrmd.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/internal.h>  // pcmk__ends_with_ext()
+#include <crm/common/ipc.h>
+#include <crm/common/mainloop.h>
+#include <crm/common/output.h>
+#include <crm/common/output_internal.h>
+#include <crm/common/results.h>
+#include <crm/common/util.h>
+#include <crm/common/xml.h>
+#include <crm/common/xml_internal.h>
+
+#include <crm/cib/internal.h>
+#include <crm/pengine/status.h>
+#include <crm/pengine/internal.h>
+#include <pacemaker-internal.h>
+#include <crm/stonith-ng.h>
+#include <crm/fencing/internal.h>
+
+#include "crm_mon.h"
+
+#define SUMMARY "Provides a summary of cluster's current state.\n\n" \
+                "Outputs varying levels of detail in a number of different formats."
+
+/*
+ * Definitions indicating which items to print
+ */
+
+static uint32_t show;
+static uint32_t show_opts = pcmk_show_pending;
+
+/*
+ * Definitions indicating how to output
+ */
+
+static mon_output_format_t output_format = mon_output_unset;
+
+/* other globals */
+static GIOChannel *io_channel = NULL;
+static GMainLoop *mainloop = NULL;
+static guint reconnect_timer = 0;
+static mainloop_timer_t *refresh_timer = NULL;
+
+static enum pcmk_pacemakerd_state pcmkd_state = pcmk_pacemakerd_state_invalid;
+static cib_t *cib = NULL;
+static stonith_t *st = NULL;
+static xmlNode *current_cib = NULL;
+
+static GError *error = NULL;
+static pcmk__common_args_t *args = NULL;
+static pcmk__output_t *out = NULL;
+static GOptionContext *context = NULL;
+static gchar **processed_args = NULL;
+
+static time_t last_refresh = 0;
+volatile crm_trigger_t *refresh_trigger = NULL;
+
+static enum pcmk__fence_history fence_history = pcmk__fence_history_none;
+
+int interactive_fence_level = 0;
+
+static pcmk__supported_format_t formats[] = {
+#if CURSES_ENABLED
+    CRM_MON_SUPPORTED_FORMAT_CURSES,
+#endif
+    PCMK__SUPPORTED_FORMAT_HTML,
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+PCMK__OUTPUT_ARGS("crm-mon-disconnected", "const char *",
+                  "enum pcmk_pacemakerd_state")
+static int
+crm_mon_disconnected_default(pcmk__output_t *out, va_list args)
+{
+    return pcmk_rc_no_output;
+}
+
+PCMK__OUTPUT_ARGS("crm-mon-disconnected", "const char *",
+                  "enum pcmk_pacemakerd_state")
+static int
+crm_mon_disconnected_html(pcmk__output_t *out, va_list args)
+{
+    const char *desc = va_arg(args, const char *);
+    enum pcmk_pacemakerd_state state =
+        (enum pcmk_pacemakerd_state) va_arg(args, int);
+
+    if (out->dest != stdout) {
+        out->reset(out);
+    }
+
+    pcmk__output_create_xml_text_node(out, "span", "Not connected to CIB");
+
+    if (desc != NULL) {
+        pcmk__output_create_xml_text_node(out, "span", ": ");
+        pcmk__output_create_xml_text_node(out, "span", desc);
+    }
+
+    if (state != pcmk_pacemakerd_state_invalid) {
+        const char *state_s = pcmk__pcmkd_state_enum2friendly(state);
+
+        pcmk__output_create_xml_text_node(out, "span", " (");
+        pcmk__output_create_xml_text_node(out, "span", state_s);
+        pcmk__output_create_xml_text_node(out, "span", ")");
+    }
+
+    out->finish(out, CRM_EX_DISCONNECT, true, NULL);
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("crm-mon-disconnected", "const char *",
+                  "enum pcmk_pacemakerd_state")
+static int
+crm_mon_disconnected_text(pcmk__output_t *out, va_list args)
+{
+    const char *desc = va_arg(args, const char *);
+    enum pcmk_pacemakerd_state state =
+        (enum pcmk_pacemakerd_state) va_arg(args, int);
+    int rc = pcmk_rc_ok;
+
+    if (out->dest != stdout) {
+        out->reset(out);
+    }
+
+    if (state != pcmk_pacemakerd_state_invalid) {
+        rc = out->info(out, "Not connected to CIB%s%s (%s)",
+                       (desc != NULL)? ": " : "", pcmk__s(desc, ""),
+                       pcmk__pcmkd_state_enum2friendly(state));
+    } else {
+        rc = out->info(out, "Not connected to CIB%s%s",
+                       (desc != NULL)? ": " : "", pcmk__s(desc, ""));
+    }
+
+    out->finish(out, CRM_EX_DISCONNECT, true, NULL);
+    return rc;
+}
+
+PCMK__OUTPUT_ARGS("crm-mon-disconnected", "const char *",
+                  "enum pcmk_pacemakerd_state")
+static int
+crm_mon_disconnected_xml(pcmk__output_t *out, va_list args)
+{
+    const char *desc = va_arg(args, const char *);
+    enum pcmk_pacemakerd_state state =
+        (enum pcmk_pacemakerd_state) va_arg(args, int);
+    const char *state_s = NULL;
+
+    if (out->dest != stdout) {
+        out->reset(out);
+    }
+
+    if (state != pcmk_pacemakerd_state_invalid) {
+        state_s = pcmk_pacemakerd_api_daemon_state_enum2text(state);
+    }
+
+    pcmk__output_create_xml_node(out, "crm-mon-disconnected",
+                                 XML_ATTR_DESC, desc,
+                                 "pacemakerd-state", state_s,
+                                 NULL);
+
+    out->finish(out, CRM_EX_DISCONNECT, true, NULL);
+    return pcmk_rc_ok;
+}
+
+static pcmk__message_entry_t fmt_functions[] = {
+    { "crm-mon-disconnected", "default", crm_mon_disconnected_default },
+    { "crm-mon-disconnected", "html", crm_mon_disconnected_html },
+    { "crm-mon-disconnected", "text", crm_mon_disconnected_text },
+    { "crm-mon-disconnected", "xml", crm_mon_disconnected_xml },
+    { NULL, NULL, NULL },
+};
+
+/* Define exit codes for monitoring-compatible output
+ * For nagios plugins, the possibilities are
+ * OK=0, WARN=1, CRIT=2, and UNKNOWN=3
+ */
+#define MON_STATUS_WARN    CRM_EX_ERROR
+#define MON_STATUS_CRIT    CRM_EX_INVALID_PARAM
+#define MON_STATUS_UNKNOWN CRM_EX_UNIMPLEMENT_FEATURE
+
+#define RECONNECT_MSECS 5000
+
+struct {
+    guint reconnect_ms;
+    enum mon_exec_mode exec_mode;
+    gboolean fence_connect;
+    gboolean print_pending;
+    gboolean show_bans;
+    gboolean watch_fencing;
+    char *pid_file;
+    char *external_agent;
+    char *external_recipient;
+    char *neg_location_prefix;
+    char *only_node;
+    char *only_rsc;
+    GSList *user_includes_excludes;
+    GSList *includes_excludes;
+} options = {
+    .reconnect_ms = RECONNECT_MSECS,
+    .exec_mode = mon_exec_unset,
+    .fence_connect = TRUE,
+};
+
+static crm_exit_t clean_up(crm_exit_t exit_code);
+static void crm_diff_update(const char *event, xmlNode * msg);
+static void clean_up_on_connection_failure(int rc);
+static int mon_refresh_display(gpointer user_data);
+static int setup_cib_connection(void);
+static int setup_fencer_connection(void);
+static int setup_api_connections(void);
+static void mon_st_callback_event(stonith_t * st, stonith_event_t * e);
+static void mon_st_callback_display(stonith_t * st, stonith_event_t * e);
+static void refresh_after_event(gboolean data_updated, gboolean enforce);
+
+static uint32_t
+all_includes(mon_output_format_t fmt) {
+    if (fmt == mon_output_monitor || fmt == mon_output_plain || fmt == mon_output_console) {
+        return ~pcmk_section_options;
+    } else {
+        return pcmk_section_all;
+    }
+}
+
+static uint32_t
+default_includes(mon_output_format_t fmt) {
+    switch (fmt) {
+        case mon_output_monitor:
+        case mon_output_plain:
+        case mon_output_console:
+        case mon_output_html:
+        case mon_output_cgi:
+            return pcmk_section_summary
+                   |pcmk_section_nodes
+                   |pcmk_section_resources
+                   |pcmk_section_failures;
+
+        case mon_output_xml:
+            return all_includes(fmt);
+
+        default:
+            return 0;
+    }
+}
+
+struct {
+    const char *name;
+    uint32_t bit;
+} sections[] = {
+    { "attributes", pcmk_section_attributes },
+    { "bans", pcmk_section_bans },
+    { "counts", pcmk_section_counts },
+    { "dc", pcmk_section_dc },
+    { "failcounts", pcmk_section_failcounts },
+    { "failures", pcmk_section_failures },
+    { PCMK__VALUE_FENCING, pcmk_section_fencing_all },
+    { "fencing-failed", pcmk_section_fence_failed },
+    { "fencing-pending", pcmk_section_fence_pending },
+    { "fencing-succeeded", pcmk_section_fence_worked },
+    { "maint-mode", pcmk_section_maint_mode },
+    { "nodes", pcmk_section_nodes },
+    { "operations", pcmk_section_operations },
+    { "options", pcmk_section_options },
+    { "resources", pcmk_section_resources },
+    { "stack", pcmk_section_stack },
+    { "summary", pcmk_section_summary },
+    { "tickets", pcmk_section_tickets },
+    { "times", pcmk_section_times },
+    { NULL }
+};
+
+static uint32_t
+find_section_bit(const char *name) {
+    for (int i = 0; sections[i].name != NULL; i++) {
+        if (pcmk__str_eq(sections[i].name, name, pcmk__str_casei)) {
+            return sections[i].bit;
+        }
+    }
+
+    return 0;
+}
+
+static gboolean
+apply_exclude(const gchar *excludes, GError **error) {
+    char **parts = NULL;
+    gboolean result = TRUE;
+
+    parts = g_strsplit(excludes, ",", 0);
+    for (char **s = parts; *s != NULL; s++) {
+        uint32_t bit = find_section_bit(*s);
+
+        if (pcmk__str_eq(*s, "all", pcmk__str_none)) {
+            show = 0;
+        } else if (pcmk__str_eq(*s, PCMK__VALUE_NONE, pcmk__str_none)) {
+            show = all_includes(output_format);
+        } else if (bit != 0) {
+            show &= ~bit;
+        } else {
+            g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                        "--exclude options: all, attributes, bans, counts, dc, "
+                        "failcounts, failures, fencing, fencing-failed, "
+                        "fencing-pending, fencing-succeeded, maint-mode, nodes, "
+                        PCMK__VALUE_NONE ", operations, options, resources, "
+                        "stack, summary, tickets, times");
+            result = FALSE;
+            break;
+        }
+    }
+    g_strfreev(parts);
+    return result;
+}
+
+static gboolean
+apply_include(const gchar *includes, GError **error) {
+    char **parts = NULL;
+    gboolean result = TRUE;
+
+    parts = g_strsplit(includes, ",", 0);
+    for (char **s = parts; *s != NULL; s++) {
+        uint32_t bit = find_section_bit(*s);
+
+        if (pcmk__str_eq(*s, "all", pcmk__str_none)) {
+            show = all_includes(output_format);
+        } else if (pcmk__starts_with(*s, "bans")) {
+            show |= pcmk_section_bans;
+            if (options.neg_location_prefix != NULL) {
+                free(options.neg_location_prefix);
+                options.neg_location_prefix = NULL;
+            }
+
+            if (strlen(*s) > 4 && (*s)[4] == ':') {
+                options.neg_location_prefix = strdup(*s+5);
+            }
+        } else if (pcmk__str_any_of(*s, "default", "defaults", NULL)) {
+            show |= default_includes(output_format);
+        } else if (pcmk__str_eq(*s, PCMK__VALUE_NONE, pcmk__str_none)) {
+            show = 0;
+        } else if (bit != 0) {
+            show |= bit;
+        } else {
+            g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                        "--include options: all, attributes, bans[:PREFIX], counts, dc, "
+                        "default, failcounts, failures, fencing, fencing-failed, "
+                        "fencing-pending, fencing-succeeded, maint-mode, nodes, "
+                        PCMK__VALUE_NONE ", operations, options, resources, "
+                        "stack, summary, tickets, times");
+            result = FALSE;
+            break;
+        }
+    }
+    g_strfreev(parts);
+    return result;
+}
+
+static gboolean
+apply_include_exclude(GSList *lst, GError **error) {
+    gboolean rc = TRUE;
+    GSList *node = lst;
+
+    while (node != NULL) {
+        char *s = node->data;
+
+        if (pcmk__starts_with(s, "--include=")) {
+            rc = apply_include(s+10, error);
+        } else if (pcmk__starts_with(s, "-I=")) {
+            rc = apply_include(s+3, error);
+        } else if (pcmk__starts_with(s, "--exclude=")) {
+            rc = apply_exclude(s+10, error);
+        } else if (pcmk__starts_with(s, "-U=")) {
+            rc = apply_exclude(s+3, error);
+        }
+
+        if (rc != TRUE) {
+            break;
+        }
+
+        node = node->next;
+    }
+
+    return rc;
+}
+
+static gboolean
+user_include_exclude_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    char *s = crm_strdup_printf("%s=%s", option_name, optarg);
+
+    options.user_includes_excludes = g_slist_append(options.user_includes_excludes, s);
+    return TRUE;
+}
+
+static gboolean
+include_exclude_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    char *s = crm_strdup_printf("%s=%s", option_name, optarg);
+
+    options.includes_excludes = g_slist_append(options.includes_excludes, s);
+    return TRUE;
+}
+
+static gboolean
+as_cgi_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    pcmk__str_update(&args->output_ty, "html");
+    output_format = mon_output_cgi;
+    options.exec_mode = mon_exec_one_shot;
+    return TRUE;
+}
+
+static gboolean
+as_html_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    pcmk__str_update(&args->output_dest, optarg);
+    pcmk__str_update(&args->output_ty, "html");
+    output_format = mon_output_html;
+    umask(S_IWGRP | S_IWOTH);   // World-readable HTML
+    return TRUE;
+}
+
+static gboolean
+as_simple_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    pcmk__str_update(&args->output_ty, "text");
+    output_format = mon_output_monitor;
+    options.exec_mode = mon_exec_one_shot;
+    return TRUE;
+}
+
+static gboolean
+as_xml_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    pcmk__str_update(&args->output_ty, "xml");
+    output_format = mon_output_legacy_xml;
+    return TRUE;
+}
+
+static gboolean
+fence_history_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    if (optarg == NULL) {
+        interactive_fence_level = 2;
+    } else {
+        pcmk__scan_min_int(optarg, &interactive_fence_level, 0);
+    }
+
+    switch (interactive_fence_level) {
+        case 3:
+            options.fence_connect = TRUE;
+            fence_history = pcmk__fence_history_full;
+            return include_exclude_cb("--include", PCMK__VALUE_FENCING, data,
+                                      err);
+
+        case 2:
+            options.fence_connect = TRUE;
+            fence_history = pcmk__fence_history_full;
+            return include_exclude_cb("--include", PCMK__VALUE_FENCING, data,
+                                      err);
+
+        case 1:
+            options.fence_connect = TRUE;
+            fence_history = pcmk__fence_history_full;
+            return include_exclude_cb("--include", "fencing-failed,fencing-pending", data, err);
+
+        case 0:
+            options.fence_connect = FALSE;
+            fence_history = pcmk__fence_history_none;
+            return include_exclude_cb("--exclude", PCMK__VALUE_FENCING, data,
+                                      err);
+
+        default:
+            g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Fence history must be 0-3");
+            return FALSE;
+    }
+}
+
+static gboolean
+group_by_node_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    show_opts |= pcmk_show_rscs_by_node;
+    return TRUE;
+}
+
+static gboolean
+hide_headers_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    return user_include_exclude_cb("--exclude", "summary", data, err);
+}
+
+static gboolean
+inactive_resources_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    show_opts |= pcmk_show_inactive_rscs;
+    return TRUE;
+}
+
+static gboolean
+no_curses_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    pcmk__str_update(&args->output_ty, "text");
+    output_format = mon_output_plain;
+    return TRUE;
+}
+
+static gboolean
+print_brief_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    show_opts |= pcmk_show_brief;
+    return TRUE;
+}
+
+static gboolean
+print_detail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    show_opts |= pcmk_show_details;
+    return TRUE;
+}
+
+static gboolean
+print_description_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    show_opts |= pcmk_show_description;
+    return TRUE;
+}
+
+static gboolean
+print_timing_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    show_opts |= pcmk_show_timing;
+    return user_include_exclude_cb("--include", "operations", data, err);
+}
+
+static gboolean
+reconnect_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    int rc = crm_get_msec(optarg);
+
+    if (rc == -1) {
+        g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Invalid value for -i: %s", optarg);
+        return FALSE;
+    } else {
+        options.reconnect_ms = crm_parse_interval_spec(optarg);
+
+        if (options.exec_mode != mon_exec_daemonized) {
+            // Reconnect interval applies to daemonized too, so don't override
+            options.exec_mode = mon_exec_update;
+        }
+    }
+
+    return TRUE;
+}
+
+/*!
+ * \internal
+ * \brief Enable one-shot mode
+ *
+ * \param[in]  option_name  Name of option being parsed (ignored)
+ * \param[in]  optarg       Value to be parsed (ignored)
+ * \param[in]  data         User data (ignored)
+ * \param[out] err          Where to store error (ignored)
+ */
+static gboolean
+one_shot_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+            GError **err)
+{
+    options.exec_mode = mon_exec_one_shot;
+    return TRUE;
+}
+
+/*!
+ * \internal
+ * \brief Enable daemonized mode
+ *
+ * \param[in]  option_name  Name of option being parsed (ignored)
+ * \param[in]  optarg       Value to be parsed (ignored)
+ * \param[in]  data         User data (ignored)
+ * \param[out] err          Where to store error (ignored)
+ */
+static gboolean
+daemonize_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+             GError **err)
+{
+    options.exec_mode = mon_exec_daemonized;
+    return TRUE;
+}
+
+static gboolean
+show_attributes_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    return user_include_exclude_cb("--include", "attributes", data, err);
+}
+
+static gboolean
+show_bans_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    if (optarg != NULL) {
+        char *s = crm_strdup_printf("bans:%s", optarg);
+        gboolean rc = user_include_exclude_cb("--include", s, data, err);
+        free(s);
+        return rc;
+    } else {
+        return user_include_exclude_cb("--include", "bans", data, err);
+    }
+}
+
+static gboolean
+show_failcounts_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    return user_include_exclude_cb("--include", "failcounts", data, err);
+}
+
+static gboolean
+show_operations_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    return user_include_exclude_cb("--include", "failcounts,operations", data, err);
+}
+
+static gboolean
+show_tickets_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    return user_include_exclude_cb("--include", "tickets", data, err);
+}
+
+static gboolean
+use_cib_file_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    setenv("CIB_file", optarg, 1);
+    options.exec_mode = mon_exec_one_shot;
+    return TRUE;
+}
+
+#define INDENT "                                    "
+
+/* *INDENT-OFF* */
+static GOptionEntry addl_entries[] = {
+    { "interval", 'i', 0, G_OPTION_ARG_CALLBACK, reconnect_cb,
+      "Update frequency (default is 5 seconds)",
+      "TIMESPEC" },
+
+    { "one-shot", '1', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
+      one_shot_cb,
+      "Display the cluster status once and exit",
+      NULL },
+
+    { "daemonize", 'd', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
+      daemonize_cb,
+      "Run in the background as a daemon.\n"
+      INDENT "Requires at least one of --output-to and --external-agent.",
+      NULL },
+
+    { "pid-file", 'p', 0, G_OPTION_ARG_FILENAME, &options.pid_file,
+      "(Advanced) Daemon pid file location",
+      "FILE" },
+
+    { "external-agent", 'E', 0, G_OPTION_ARG_FILENAME, &options.external_agent,
+      "A program to run when resource operations take place",
+      "FILE" },
+
+    { "external-recipient", 'e', 0, G_OPTION_ARG_STRING, &options.external_recipient,
+      "A recipient for your program (assuming you want the program to send something to someone).",
+      "RCPT" },
+
+    { "watch-fencing", 'W', 0, G_OPTION_ARG_NONE, &options.watch_fencing,
+      "Listen for fencing events. For use with --external-agent.",
+      NULL },
+
+    { "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, use_cib_file_cb,
+      NULL,
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry display_entries[] = {
+    { "include", 'I', 0, G_OPTION_ARG_CALLBACK, user_include_exclude_cb,
+      "A list of sections to include in the output.\n"
+      INDENT "See `Output Control` help for more information.",
+      "SECTION(s)" },
+
+    { "exclude", 'U', 0, G_OPTION_ARG_CALLBACK, user_include_exclude_cb,
+      "A list of sections to exclude from the output.\n"
+      INDENT "See `Output Control` help for more information.",
+      "SECTION(s)" },
+
+    { "node", 0, 0, G_OPTION_ARG_STRING, &options.only_node,
+      "When displaying information about nodes, show only what's related to the given\n"
+      INDENT "node, or to all nodes tagged with the given tag",
+      "NODE" },
+
+    { "resource", 0, 0, G_OPTION_ARG_STRING, &options.only_rsc,
+      "When displaying information about resources, show only what's related to the given\n"
+      INDENT "resource, or to all resources tagged with the given tag",
+      "RSC" },
+
+    { "group-by-node", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, group_by_node_cb,
+      "Group resources by node",
+      NULL },
+
+    { "inactive", 'r', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, inactive_resources_cb,
+      "Display inactive resources",
+      NULL },
+
+    { "failcounts", 'f', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_failcounts_cb,
+      "Display resource fail counts",
+      NULL },
+
+    { "operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_operations_cb,
+      "Display resource operation history",
+      NULL },
+
+    { "timing-details", 't', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_timing_cb,
+      "Display resource operation history with timing details",
+      NULL },
+
+    { "tickets", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_tickets_cb,
+      "Display cluster tickets",
+      NULL },
+
+    { "fence-history", 'm', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, fence_history_cb,
+      "Show fence history:\n"
+      INDENT "0=off, 1=failures and pending (default without option),\n"
+      INDENT "2=add successes (default without value for option),\n"
+      INDENT "3=show full history without reduction to most recent of each flavor",
+      "LEVEL" },
+
+    { "neg-locations", 'L', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, show_bans_cb,
+      "Display negative location constraints [optionally filtered by id prefix]",
+      NULL },
+
+    { "show-node-attributes", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_attributes_cb,
+      "Display node attributes",
+      NULL },
+
+    { "hide-headers", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, hide_headers_cb,
+      "Hide all headers",
+      NULL },
+
+    { "show-detail", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_detail_cb,
+      "Show more details (node IDs, individual clone instances)",
+      NULL },
+
+    { "show-description", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_description_cb,
+      "Show resource descriptions",
+      NULL },
+
+    { "brief", 'b', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_brief_cb,
+      "Brief output",
+      NULL },
+
+    { "pending", 'j', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.print_pending,
+      "Display pending state if 'record-pending' is enabled",
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry deprecated_entries[] = {
+    { "as-html", 'h', G_OPTION_FLAG_FILENAME, G_OPTION_ARG_CALLBACK, as_html_cb,
+      "Write cluster status to the named HTML file.\n"
+      INDENT "Use --output-as=html --output-to=FILE instead.",
+      "FILE" },
+
+    { "as-xml", 'X', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_xml_cb,
+      "Write cluster status as XML to stdout. This will enable one-shot mode.\n"
+      INDENT "Use --output-as=xml instead.",
+      NULL },
+
+    { "simple-status", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
+      as_simple_cb,
+      "Display the cluster status once as a simple one line output\n"
+      INDENT "(suitable for nagios)",
+      NULL },
+
+    { "disable-ncurses", 'N', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, no_curses_cb,
+      "Disable the use of ncurses.\n"
+      INDENT "Use --output-as=text instead.",
+      NULL },
+
+    { "web-cgi", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_cgi_cb,
+      "Web mode with output suitable for CGI (preselected when run as *.cgi).\n"
+      INDENT "Use --output-as=html --html-cgi instead.",
+      NULL },
+
+    { NULL }
+};
+/* *INDENT-ON* */
+
+/* Reconnect to the CIB and fencing agent after reconnect_ms has passed.  This sounds
+ * like it would be more broadly useful, but only ever happens after a disconnect via
+ * mon_cib_connection_destroy.
+ */
+static gboolean
+reconnect_after_timeout(gpointer data)
+{
+#if CURSES_ENABLED
+    if (output_format == mon_output_console) {
+        clear();
+        refresh();
+    }
+#endif
+
+    out->transient(out, "Reconnecting...");
+    if (setup_api_connections() == pcmk_rc_ok) {
+        // Trigger redrawing the screen (needs reconnect_timer == 0)
+        reconnect_timer = 0;
+        refresh_after_event(FALSE, TRUE);
+        return G_SOURCE_REMOVE;
+    }
+
+    out->message(out, "crm-mon-disconnected",
+                 "Latest connection attempt failed", pcmkd_state);
+
+    reconnect_timer = g_timeout_add(options.reconnect_ms,
+                                    reconnect_after_timeout, NULL);
+    return G_SOURCE_REMOVE;
+}
+
+/* Called from various places when we are disconnected from the CIB or from the
+ * fencing agent.  If the CIB connection is still valid, this function will also
+ * attempt to sign off and reconnect.
+ */
+static void
+mon_cib_connection_destroy(gpointer user_data)
+{
+    const char *msg = "Connection to the cluster lost";
+
+    pcmkd_state = pcmk_pacemakerd_state_invalid;
+
+    /* No crm-mon-disconnected message for console; a working implementation
+     * is not currently worth the effort
+     */
+    out->transient(out, "%s", msg);
+
+    out->message(out, "crm-mon-disconnected", msg, pcmkd_state);
+
+    if (refresh_timer != NULL) {
+        /* we'll trigger a refresh after reconnect */
+        mainloop_timer_stop(refresh_timer);
+    }
+    if (reconnect_timer) {
+        /* we'll trigger a new reconnect-timeout at the end */
+        g_source_remove(reconnect_timer);
+        reconnect_timer = 0;
+    }
+
+    /* the client API won't properly reconnect notifications if they are still
+     * in the table - so remove them
+     */
+    stonith_api_delete(st);
+    st = NULL;
+
+    if (cib) {
+        cib->cmds->signoff(cib);
+        reconnect_timer = g_timeout_add(options.reconnect_ms,
+                                        reconnect_after_timeout, NULL);
+    }
+}
+
+/* Signal handler installed into the mainloop for normal program shutdown */
+static void
+mon_shutdown(int nsig)
+{
+    clean_up(CRM_EX_OK);
+}
+
+#if CURSES_ENABLED
+static volatile sighandler_t ncurses_winch_handler;
+
+/* Signal handler installed the regular way (not into the main loop) for when
+ * the screen is resized.  Commonly, this happens when running in an xterm and
+ * the user changes its size.
+ */
+static void
+mon_winresize(int nsig)
+{
+    static int not_done;
+    int lines = 0, cols = 0;
+
+    if (!not_done++) {
+        if (ncurses_winch_handler)
+            /* the original ncurses WINCH signal handler does the
+             * magic of retrieving the new window size;
+             * otherwise, we'd have to use ioctl or tgetent */
+            (*ncurses_winch_handler) (SIGWINCH);
+        getmaxyx(stdscr, lines, cols);
+        resizeterm(lines, cols);
+        /* Alert the mainloop code we'd like the refresh_trigger to run next
+         * time the mainloop gets around to checking.
+         */
+        mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
+    }
+    not_done--;
+}
+#endif
+
+static int
+setup_fencer_connection(void)
+{
+    int rc = pcmk_ok;
+
+    if (options.fence_connect && st == NULL) {
+        st = stonith_api_new();
+    }
+
+    if (!options.fence_connect || st == NULL || st->state != stonith_disconnected) {
+        return rc;
+    }
+
+    rc = st->cmds->connect(st, crm_system_name, NULL);
+    if (rc == pcmk_ok) {
+        crm_trace("Setting up stonith callbacks");
+        if (options.watch_fencing) {
+            st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
+                                            mon_st_callback_event);
+            st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, mon_st_callback_event);
+        } else {
+            st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
+                                            mon_st_callback_display);
+            st->cmds->register_notification(st, T_STONITH_NOTIFY_HISTORY, mon_st_callback_display);
+        }
+    } else {
+        stonith_api_delete(st);
+        st = NULL;
+    }
+
+    return rc;
+}
+
+static int
+setup_cib_connection(void)
+{
+    int rc = pcmk_rc_ok;
+
+    CRM_CHECK(cib != NULL, return EINVAL);
+
+    if (cib->state != cib_disconnected) {
+        // Already connected with notifications registered for CIB updates
+        return rc;
+    }
+
+    rc = cib__signon_query(out, &cib, &current_cib);
+
+    if (rc == pcmk_rc_ok) {
+        rc = pcmk_legacy2rc(cib->cmds->set_connection_dnotify(cib,
+            mon_cib_connection_destroy));
+        if (rc == EPROTONOSUPPORT) {
+            out->err(out,
+                     "CIB client does not support connection loss "
+                     "notifications; crm_mon will be unable to reconnect after "
+                     "connection loss");
+            rc = pcmk_rc_ok;
+        }
+
+        if (rc == pcmk_rc_ok) {
+            cib->cmds->del_notify_callback(cib, T_CIB_DIFF_NOTIFY,
+                                           crm_diff_update);
+            rc = pcmk_legacy2rc(cib->cmds->add_notify_callback(cib,
+                                    T_CIB_DIFF_NOTIFY, crm_diff_update));
+        }
+
+        if (rc != pcmk_rc_ok) {
+            if (rc == EPROTONOSUPPORT) {
+                out->err(out,
+                         "CIB client does not support CIB diff "
+                         "notifications");
+            } else {
+                out->err(out, "CIB diff notification setup failed");
+            }
+
+            out->err(out, "Cannot monitor CIB changes; exiting");
+            cib__clean_up_connection(&cib);
+            stonith_api_delete(st);
+            st = NULL;
+        }
+    }
+    return rc;
+}
+
+/* This is used to set up the fencing options after the interactive UI has been stared.
+ * fence_history_cb can't be used because it builds up a list of includes/excludes that
+ * then have to be processed with apply_include_exclude and that could affect other
+ * things.
+ */
+static void
+set_fencing_options(int level)
+{
+    switch (level) {
+        case 3:
+            options.fence_connect = TRUE;
+            fence_history = pcmk__fence_history_full;
+            show |= pcmk_section_fencing_all;
+            break;
+
+        case 2:
+            options.fence_connect = TRUE;
+            fence_history = pcmk__fence_history_full;
+            show |= pcmk_section_fencing_all;
+            break;
+
+        case 1:
+            options.fence_connect = TRUE;
+            fence_history = pcmk__fence_history_full;
+            show |= pcmk_section_fence_failed | pcmk_section_fence_pending;
+            break;
+
+        default:
+            interactive_fence_level = 0;
+            options.fence_connect = FALSE;
+            fence_history = pcmk__fence_history_none;
+            show &= ~pcmk_section_fencing_all;
+            break;
+    }
+}
+
+static int
+setup_api_connections(void)
+{
+    int rc = pcmk_rc_ok;
+
+    CRM_CHECK(cib != NULL, return EINVAL);
+
+    if (cib->state != cib_disconnected) {
+        return rc;
+    }
+
+    if (cib->variant == cib_native) {
+        rc = pcmk__pacemakerd_status(out, crm_system_name,
+                                     options.reconnect_ms / 2, false,
+                                     &pcmkd_state);
+        if (rc != pcmk_rc_ok) {
+            return rc;
+        }
+
+        switch (pcmkd_state) {
+            case pcmk_pacemakerd_state_running:
+            case pcmk_pacemakerd_state_remote:
+            case pcmk_pacemakerd_state_shutting_down:
+                /* Fencer and CIB may still be available while shutting down or
+                 * running on a Pacemaker Remote node
+                 */
+                break;
+            default:
+                // Fencer and CIB are definitely unavailable
+                return ENOTCONN;
+        }
+
+        setup_fencer_connection();
+    }
+
+    rc = setup_cib_connection();
+    return rc;
+}
+
+#if CURSES_ENABLED
+static const char *
+get_option_desc(char c)
+{
+    const char *desc = "No help available";
+
+    for (GOptionEntry *entry = display_entries; entry != NULL; entry++) {
+        if (entry->short_name == c) {
+            desc = entry->description;
+            break;
+        }
+    }
+    return desc;
+}
+
+#define print_option_help(out, option, condition) \
+    curses_formatted_printf(out, "%c %c: \t%s\n", ((condition)? '*': ' '), option, get_option_desc(option));
+
+/* This function is called from the main loop when there is something to be read
+ * on stdin, like an interactive user's keystroke.  All it does is read the keystroke,
+ * set flags (or show the page showing which keystrokes are valid), and redraw the
+ * screen.  It does not do anything with connections to the CIB or fencing agent
+ * agent what would happen in mon_refresh_display.
+ */
+static gboolean
+detect_user_input(GIOChannel *channel, GIOCondition condition, gpointer user_data)
+{
+    int c;
+    gboolean config_mode = FALSE;
+
+    while (1) {
+
+        /* Get user input */
+        c = getchar();
+
+        switch (c) {
+            case 'm':
+                interactive_fence_level++;
+                if (interactive_fence_level > 3) {
+                    interactive_fence_level = 0;
+                }
+
+                set_fencing_options(interactive_fence_level);
+                break;
+            case 'c':
+                show ^= pcmk_section_tickets;
+                break;
+            case 'f':
+                show ^= pcmk_section_failcounts;
+                break;
+            case 'n':
+                show_opts ^= pcmk_show_rscs_by_node;
+                break;
+            case 'o':
+                show ^= pcmk_section_operations;
+                if (!pcmk_is_set(show, pcmk_section_operations)) {
+                    show_opts &= ~pcmk_show_timing;
+                }
+                break;
+            case 'r':
+                show_opts ^= pcmk_show_inactive_rscs;
+                break;
+            case 'R':
+                show_opts ^= pcmk_show_details;
+#ifdef PCMK__COMPAT_2_0
+                // Keep failed action output the same as 2.0.x
+                show_opts |= pcmk_show_failed_detail;
+#endif
+                break;
+            case 't':
+                show_opts ^= pcmk_show_timing;
+                if (pcmk_is_set(show_opts, pcmk_show_timing)) {
+                    show |= pcmk_section_operations;
+                }
+                break;
+            case 'A':
+                show ^= pcmk_section_attributes;
+                break;
+            case 'L':
+                show ^= pcmk_section_bans;
+                break;
+            case 'D':
+                /* If any header is shown, clear them all, otherwise set them all */
+                if (pcmk_any_flags_set(show, pcmk_section_summary)) {
+                    show &= ~pcmk_section_summary;
+                } else {
+                    show |= pcmk_section_summary;
+                }
+                /* Regardless, we don't show options in console mode. */
+                show &= ~pcmk_section_options;
+                break;
+            case 'b':
+                show_opts ^= pcmk_show_brief;
+                break;
+            case 'j':
+                show_opts ^= pcmk_show_pending;
+                break;
+            case '?':
+                config_mode = TRUE;
+                break;
+            default:
+                /* All other keys just redraw the screen. */
+                goto refresh;
+        }
+
+        if (!config_mode)
+            goto refresh;
+
+        clear();
+        refresh();
+
+        curses_formatted_printf(out, "%s", "Display option change mode\n");
+        print_option_help(out, 'c', pcmk_is_set(show, pcmk_section_tickets));
+        print_option_help(out, 'f', pcmk_is_set(show, pcmk_section_failcounts));
+        print_option_help(out, 'n', pcmk_is_set(show_opts, pcmk_show_rscs_by_node));
+        print_option_help(out, 'o', pcmk_is_set(show, pcmk_section_operations));
+        print_option_help(out, 'r', pcmk_is_set(show_opts, pcmk_show_inactive_rscs));
+        print_option_help(out, 't', pcmk_is_set(show_opts, pcmk_show_timing));
+        print_option_help(out, 'A', pcmk_is_set(show, pcmk_section_attributes));
+        print_option_help(out, 'L', pcmk_is_set(show, pcmk_section_bans));
+        print_option_help(out, 'D', !pcmk_is_set(show, pcmk_section_summary));
+#ifdef PCMK__COMPAT_2_0
+        print_option_help(out, 'R', pcmk_any_flags_set(show_opts, pcmk_show_details & ~pcmk_show_failed_detail));
+#else
+        print_option_help(out, 'R', pcmk_any_flags_set(show_opts, pcmk_show_details));
+#endif
+        print_option_help(out, 'b', pcmk_is_set(show_opts, pcmk_show_brief));
+        print_option_help(out, 'j', pcmk_is_set(show_opts, pcmk_show_pending));
+        curses_formatted_printf(out, "%d m: \t%s\n", interactive_fence_level, get_option_desc('m'));
+        curses_formatted_printf(out, "%s", "\nToggle fields via field letter, type any other key to return\n");
+    }
+
+refresh:
+    refresh_after_event(FALSE, TRUE);
+
+    return TRUE;
+}
+#endif  // CURSES_ENABLED
+
+// Basically crm_signal_handler(SIGCHLD, SIG_IGN) plus the SA_NOCLDWAIT flag
+static void
+avoid_zombies(void)
+{
+    struct sigaction sa;
+
+    memset(&sa, 0, sizeof(struct sigaction));
+    if (sigemptyset(&sa.sa_mask) < 0) {
+        crm_warn("Cannot avoid zombies: %s", pcmk_rc_str(errno));
+        return;
+    }
+    sa.sa_handler = SIG_IGN;
+    sa.sa_flags = SA_RESTART|SA_NOCLDWAIT;
+    if (sigaction(SIGCHLD, &sa, NULL) < 0) {
+        crm_warn("Cannot avoid zombies: %s", pcmk_rc_str(errno));
+    }
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    GOptionEntry extra_prog_entries[] = {
+        { "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
+          "Be less descriptive in output.",
+          NULL },
+
+        { NULL }
+    };
+
+#if CURSES_ENABLED
+    const char *fmts = "console (default), html, text, xml, none";
+#else
+    const char *fmts = "text (default), html, xml, none";
+#endif // CURSES_ENABLED
+    const char *desc = NULL;
+
+    desc = "Notes:\n\n"
+           "If this program is called as crm_mon.cgi, --output-as=html and\n"
+           "--html-cgi are automatically added to the command line\n"
+           "arguments.\n\n"
+
+           "Time Specification:\n\n"
+           "The TIMESPEC in any command line option can be specified in many\n"
+           "different formats. It can be an integer number of seconds, a\n"
+           "number plus units (us/usec/ms/msec/s/sec/m/min/h/hr), or an ISO\n"
+           "8601 period specification.\n\n"
+
+           "Output Control:\n\n"
+           "By default, a particular set of sections are written to the\n"
+           "output destination. The default varies based on the output\n"
+           "format: XML includes all sections by default, while other output\n"
+           "formats include less. This set can be modified with the --include\n"
+           "and --exclude command line options. Each option may be passed\n"
+           "multiple times, and each can specify a comma-separated list of\n"
+           "sections. The options are applied to the default set, in order\n"
+           "from left to right as they are passed on the command line. For a\n"
+           "list of valid sections, pass --include=list or --exclude=list.\n\n"
+
+           "Interactive Use:\n\n"
+#if CURSES_ENABLED
+           "When run interactively, crm_mon can be told to hide and show\n"
+           "various sections of output. To see a help screen explaining the\n"
+           "options, press '?'. Any key stroke aside from those listed will\n"
+           "cause the screen to refresh.\n\n"
+#else
+           "The local installation of Pacemaker was built without support for\n"
+           "interactive (console) mode. A curses library must be available at\n"
+           "build time to support interactive mode.\n\n"
+#endif // CURSES_ENABLED
+
+           "Examples:\n\n"
+#if CURSES_ENABLED
+           "Display the cluster status on the console with updates as they\n"
+           "occur:\n\n"
+           "\tcrm_mon\n\n"
+#endif // CURSES_ENABLED
+
+           "Display the cluster status once and exit:\n\n"
+           "\tcrm_mon -1\n\n"
+
+           "Display the cluster status, group resources by node, and include\n"
+           "inactive resources in the list:\n\n"
+           "\tcrm_mon --group-by-node --inactive\n\n"
+
+           "Start crm_mon as a background daemon and have it write the\n"
+           "cluster status to an HTML file:\n\n"
+           "\tcrm_mon --daemonize --output-as html "
+           "--output-to /path/to/docroot/filename.html\n\n"
+
+           "Display the cluster status as XML:\n\n"
+           "\tcrm_mon --output-as xml\n\n";
+
+    context = pcmk__build_arg_context(args, fmts, group, NULL);
+    pcmk__add_main_args(context, extra_prog_entries);
+    g_option_context_set_description(context, desc);
+
+    pcmk__add_arg_group(context, "display", "Display Options:",
+                        "Show display options", display_entries);
+    pcmk__add_arg_group(context, "additional", "Additional Options:",
+                        "Show additional options", addl_entries);
+    pcmk__add_arg_group(context, "deprecated", "Deprecated Options:",
+                        "Show deprecated options", deprecated_entries);
+
+    return context;
+}
+
+/* If certain format options were specified, we want to set some extra
+ * options.  We can just process these like they were given on the
+ * command line.
+ */
+static void
+add_output_args(void) {
+    GError *err = NULL;
+
+    if (output_format == mon_output_plain) {
+        if (!pcmk__force_args(context, &err, "%s --text-fancy", g_get_prgname())) {
+            g_propagate_error(&error, err);
+            clean_up(CRM_EX_USAGE);
+        }
+    } else if (output_format == mon_output_cgi) {
+        if (!pcmk__force_args(context, &err, "%s --html-cgi", g_get_prgname())) {
+            g_propagate_error(&error, err);
+            clean_up(CRM_EX_USAGE);
+        }
+    } else if (output_format == mon_output_xml) {
+        if (!pcmk__force_args(context, &err, "%s --xml-simple-list --xml-substitute", g_get_prgname())) {
+            g_propagate_error(&error, err);
+            clean_up(CRM_EX_USAGE);
+        }
+    } else if (output_format == mon_output_legacy_xml) {
+        output_format = mon_output_xml;
+        if (!pcmk__force_args(context, &err, "%s --xml-legacy --xml-substitute", g_get_prgname())) {
+            g_propagate_error(&error, err);
+            clean_up(CRM_EX_USAGE);
+        }
+    }
+}
+
+/*!
+ * \internal
+ * \brief Set output format based on \p --output-as arguments and mode arguments
+ *
+ * When the deprecated output format arguments (\p --as-cgi, \p --as-html,
+ * \p --simple-status, \p --as-xml) are parsed, callback functions set
+ * \p output_format (and the umask if appropriate). If none of the deprecated
+ * arguments were specified, this function does the same based on the current
+ * \p --output-as arguments and the \p --one-shot and \p --daemonize arguments.
+ *
+ * \param[in,out] args  Command line arguments
+ */
+static void
+reconcile_output_format(pcmk__common_args_t *args)
+{
+    if (output_format != mon_output_unset) {
+        /* One of the deprecated arguments was used, and we're finished. Note
+         * that this means the deprecated arguments take precedence.
+         */
+        return;
+    }
+
+    if (pcmk__str_eq(args->output_ty, "none", pcmk__str_none)) {
+        output_format = mon_output_none;
+
+    } else if (pcmk__str_eq(args->output_ty, "html", pcmk__str_none)) {
+        output_format = mon_output_html;
+        umask(S_IWGRP | S_IWOTH);   // World-readable HTML
+
+    } else if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
+        output_format = mon_output_xml;
+
+#if CURSES_ENABLED
+    } else if (pcmk__str_eq(args->output_ty, "console",
+                            pcmk__str_null_matches)) {
+        /* Console is the default format if no conflicting options are given.
+         *
+         * Use text output instead if one of the following conditions is met:
+         * * We've requested daemonized or one-shot mode (console output is
+         *   incompatible with modes other than mon_exec_update)
+         * * We requested the version, which is effectively one-shot
+         * * We specified a non-stdout output destination (console mode is
+         *   compatible only with stdout)
+         */
+        if ((options.exec_mode == mon_exec_daemonized)
+            || (options.exec_mode == mon_exec_one_shot)
+            || args->version
+            || !pcmk__str_eq(args->output_dest, "-", pcmk__str_null_matches)) {
+
+            pcmk__str_update(&args->output_ty, "text");
+            output_format = mon_output_plain;
+        } else {
+            pcmk__str_update(&args->output_ty, "console");
+            output_format = mon_output_console;
+            crm_enable_stderr(FALSE);
+        }
+#endif // CURSES_ENABLED
+
+    } else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches)) {
+        /* Text output was explicitly requested, or it's the default because
+         * curses is not enabled
+         */
+        pcmk__str_update(&args->output_ty, "text");
+        output_format = mon_output_plain;
+    }
+
+    // Otherwise, invalid format. Let pcmk__output_new() throw an error.
+}
+
+/*!
+ * \internal
+ * \brief Set execution mode to the output format's default if appropriate
+ *
+ * \param[in,out] args  Command line arguments
+ */
+static void
+set_default_exec_mode(const pcmk__common_args_t *args)
+{
+    if (output_format == mon_output_console) {
+        /* Update is the only valid mode for console, but set here instead of
+         * reconcile_output_format() for isolation and consistency
+         */
+        options.exec_mode = mon_exec_update;
+
+    } else if (options.exec_mode == mon_exec_unset) {
+        // Default to one-shot mode for all other formats
+        options.exec_mode = mon_exec_one_shot;
+
+    } else if ((options.exec_mode == mon_exec_update)
+               && pcmk__str_eq(args->output_dest, "-",
+                               pcmk__str_null_matches)) {
+        // If not using console format, update mode cannot be used with stdout
+        options.exec_mode = mon_exec_one_shot;
+    }
+}
+
+static void
+clean_up_on_connection_failure(int rc)
+{
+    if (output_format == mon_output_monitor) {
+        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "CLUSTER CRIT: Connection to cluster failed: %s",
+                    pcmk_rc_str(rc));
+        clean_up(MON_STATUS_CRIT);
+    } else if (rc == ENOTCONN) {
+        if (pcmkd_state == pcmk_pacemakerd_state_remote) {
+            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: remote-node not connected to cluster");
+        } else {
+            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: cluster is not available on this node");
+        }
+    } else {
+        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Connection to cluster failed: %s", pcmk_rc_str(rc));
+    }
+
+    clean_up(pcmk_rc2exitc(rc));
+}
+
+static void
+one_shot(void)
+{
+    int rc = pcmk__status(out, cib, fence_history, show, show_opts,
+                          options.only_node, options.only_rsc,
+                          options.neg_location_prefix,
+                          output_format == mon_output_monitor, 0);
+
+    if (rc == pcmk_rc_ok) {
+        clean_up(pcmk_rc2exitc(rc));
+    } else {
+        clean_up_on_connection_failure(rc);
+    }
+}
+
+static void
+exit_on_invalid_cib(void)
+{
+    if (cib != NULL) {
+        return;
+    }
+
+    // Shouldn't really be possible
+    g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Invalid CIB source");
+    clean_up(CRM_EX_ERROR);
+}
+
+int
+main(int argc, char **argv)
+{
+    int rc = pcmk_rc_ok;
+    GOptionGroup *output_group = NULL;
+
+    args = pcmk__new_common_args(SUMMARY);
+    context = build_arg_context(args, &output_group);
+    pcmk__register_formats(output_group, formats);
+
+    options.pid_file = strdup("/tmp/ClusterMon.pid");
+    pcmk__cli_init_logging("crm_mon", 0);
+
+    // Avoid needing to wait for subprocesses forked for -E/--external-agent
+    avoid_zombies();
+
+    if (pcmk__ends_with_ext(argv[0], ".cgi")) {
+        output_format = mon_output_cgi;
+        options.exec_mode = mon_exec_one_shot;
+    }
+
+    processed_args = pcmk__cmdline_preproc(argv, "ehimpxEILU");
+
+    fence_history_cb("--fence-history", "1", NULL, NULL);
+
+    /* Set an HTML title regardless of what format we will eventually use.  This can't
+     * be done in add_output_args.  That function is called after command line
+     * arguments are processed in the next block, which means it'll override whatever
+     * title the user provides.  Doing this here means the user can give their own
+     * title on the command line.
+     */
+    if (!pcmk__force_args(context, &error, "%s --html-title \"Cluster Status\"",
+                          g_get_prgname())) {
+        return clean_up(CRM_EX_USAGE);
+    }
+
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        return clean_up(CRM_EX_USAGE);
+    }
+
+    for (int i = 0; i < args->verbosity; i++) {
+        crm_bump_log_level(argc, argv);
+    }
+
+    if (!args->version) {
+        if (args->quiet) {
+            include_exclude_cb("--exclude", "times", NULL, NULL);
+        }
+
+        if (options.watch_fencing) {
+            fence_history_cb("--fence-history", "0", NULL, NULL);
+            options.fence_connect = TRUE;
+        }
+
+        /* create the cib-object early to be able to do further
+         * decisions based on the cib-source
+         */
+        cib = cib_new();
+
+        exit_on_invalid_cib();
+
+        switch (cib->variant) {
+            case cib_native:
+                // Everything (fencer, CIB, pcmkd status) should be available
+                break;
+
+            case cib_file:
+                // Live fence history is not meaningful
+                fence_history_cb("--fence-history", "0", NULL, NULL);
+
+                /* Notifications are unsupported; nothing to monitor
+                 * @COMPAT: Let setup_cib_connection() handle this by exiting?
+                 */
+                options.exec_mode = mon_exec_one_shot;
+                break;
+
+            case cib_remote:
+                // We won't receive any fencing updates
+                fence_history_cb("--fence-history", "0", NULL, NULL);
+                break;
+
+            default:
+                /* something is odd */
+                exit_on_invalid_cib();
+                break;
+        }
+
+        if ((options.exec_mode == mon_exec_daemonized)
+            && !options.external_agent
+            && pcmk__str_eq(args->output_dest, "-", pcmk__str_null_matches)) {
+
+            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                        "--daemonize requires at least one of --output-to "
+                        "(with value not set to '-') and --external-agent");
+            return clean_up(CRM_EX_USAGE);
+        }
+    }
+
+    reconcile_output_format(args);
+    set_default_exec_mode(args);
+    add_output_args();
+
+    /* output_format MUST NOT BE CHANGED AFTER THIS POINT. */
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        return clean_up(CRM_EX_ERROR);
+    }
+
+    /* If we had a valid format for pcmk__output_new(), output_format should be
+     * set by now.
+     */
+    CRM_ASSERT(output_format != mon_output_unset);
+
+    if (options.exec_mode == mon_exec_daemonized) {
+        if (!options.external_agent && (output_format == mon_output_none)) {
+            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                        "--daemonize requires --external-agent if used with "
+                        "--output-as=none");
+            return clean_up(CRM_EX_USAGE);
+        }
+        crm_enable_stderr(FALSE);
+        cib_delete(cib);
+        cib = NULL;
+        pcmk__daemonize(crm_system_name, options.pid_file);
+        cib = cib_new();
+        exit_on_invalid_cib();
+    }
+
+    show = default_includes(output_format);
+
+    /* Apply --include/--exclude flags we used internally.  There's no error reporting
+     * here because this would be a programming error.
+     */
+    apply_include_exclude(options.includes_excludes, &error);
+
+    /* And now apply any --include/--exclude flags the user gave on the command line.
+     * These are done in a separate pass from the internal ones because we want to
+     * make sure whatever the user specifies overrides whatever we do.
+     */
+    if (!apply_include_exclude(options.user_includes_excludes, &error)) {
+        return clean_up(CRM_EX_USAGE);
+    }
+
+    /* Sync up the initial value of interactive_fence_level with whatever was set with
+     * --include/--exclude= options.
+     */
+    if (pcmk_all_flags_set(show, pcmk_section_fencing_all)) {
+        interactive_fence_level = 3;
+    } else if (pcmk_is_set(show, pcmk_section_fence_worked)) {
+        interactive_fence_level = 2;
+    } else if (pcmk_any_flags_set(show, pcmk_section_fence_failed | pcmk_section_fence_pending)) {
+        interactive_fence_level = 1;
+    } else {
+        interactive_fence_level = 0;
+    }
+
+    pcmk__register_lib_messages(out);
+    crm_mon_register_messages(out);
+    pe__register_messages(out);
+    stonith__register_messages(out);
+
+    // Messages internal to this file, nothing curses-specific
+    pcmk__register_messages(out, fmt_functions);
+
+    if (args->version) {
+        out->version(out, false);
+        return clean_up(CRM_EX_OK);
+    }
+
+    /* Extra sanity checks when in CGI mode */
+    if (output_format == mon_output_cgi) {
+        if (cib->variant == cib_file) {
+            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode used with CIB file");
+            return clean_up(CRM_EX_USAGE);
+        } else if (options.external_agent != NULL) {
+            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode cannot be used with --external-agent");
+            return clean_up(CRM_EX_USAGE);
+        } else if (options.exec_mode == mon_exec_daemonized) {
+            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode cannot be used with -d");
+            return clean_up(CRM_EX_USAGE);
+        }
+    }
+
+    if (output_format == mon_output_xml) {
+        show_opts |= pcmk_show_inactive_rscs | pcmk_show_timing;
+    }
+
+    if ((output_format == mon_output_html || output_format == mon_output_cgi) &&
+        out->dest != stdout) {
+        pcmk__html_add_header("meta", "http-equiv", "refresh", "content",
+                              pcmk__itoa(options.reconnect_ms / 1000), NULL);
+    }
+
+#ifdef PCMK__COMPAT_2_0
+    // Keep failed action output the same as 2.0.x
+    show_opts |= pcmk_show_failed_detail;
+#endif
+
+    crm_info("Starting %s", crm_system_name);
+
+    cib__set_output(cib, out);
+
+    if (options.exec_mode == mon_exec_one_shot) {
+        one_shot();
+    }
+
+    out->message(out, "crm-mon-disconnected",
+                 "Waiting for initial connection", pcmkd_state);
+    do {
+        out->transient(out, "Connecting to cluster...");
+        rc = setup_api_connections();
+
+        if (rc != pcmk_rc_ok) {
+            if ((rc == ENOTCONN) || (rc == ECONNREFUSED)) {
+                out->transient(out, "Connection failed. Retrying in %ums...",
+                               options.reconnect_ms);
+            }
+
+            // Give some time to view all output even if we won't retry
+            pcmk__sleep_ms(options.reconnect_ms);
+#if CURSES_ENABLED
+            if (output_format == mon_output_console) {
+                clear();
+                refresh();
+            }
+#endif
+        }
+    } while ((rc == ENOTCONN) || (rc == ECONNREFUSED));
+
+    if (rc != pcmk_rc_ok) {
+        clean_up_on_connection_failure(rc);
+    }
+
+    set_fencing_options(interactive_fence_level);
+    mon_refresh_display(NULL);
+
+    mainloop = g_main_loop_new(NULL, FALSE);
+
+    mainloop_add_signal(SIGTERM, mon_shutdown);
+    mainloop_add_signal(SIGINT, mon_shutdown);
+#if CURSES_ENABLED
+    if (output_format == mon_output_console) {
+        ncurses_winch_handler = crm_signal_handler(SIGWINCH, mon_winresize);
+        if (ncurses_winch_handler == SIG_DFL ||
+            ncurses_winch_handler == SIG_IGN || ncurses_winch_handler == SIG_ERR)
+            ncurses_winch_handler = NULL;
+
+        io_channel = g_io_channel_unix_new(STDIN_FILENO);
+        g_io_add_watch(io_channel, G_IO_IN, detect_user_input, NULL);
+    }
+#endif
+
+    /* When refresh_trigger->trigger is set to TRUE, call mon_refresh_display.  In
+     * this file, that is anywhere mainloop_set_trigger is called.
+     */
+    refresh_trigger = mainloop_add_trigger(G_PRIORITY_LOW, mon_refresh_display, NULL);
+
+    g_main_loop_run(mainloop);
+    g_main_loop_unref(mainloop);
+
+    if (io_channel != NULL) {
+        g_io_channel_shutdown(io_channel, TRUE, NULL);
+    }
+
+    crm_info("Exiting %s", crm_system_name);
+
+    return clean_up(CRM_EX_OK);
+}
+
+static int
+send_custom_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
+                 int status, const char *desc)
+{
+    pid_t pid;
+
+    /*setenv needs chars, these are ints */
+    char *rc_s = pcmk__itoa(rc);
+    char *status_s = pcmk__itoa(status);
+    char *target_rc_s = pcmk__itoa(target_rc);
+
+    crm_debug("Sending external notification to '%s' via '%s'", options.external_recipient, options.external_agent);
+
+    if(rsc) {
+        setenv("CRM_notify_rsc", rsc, 1);
+    }
+    if (options.external_recipient) {
+        setenv("CRM_notify_recipient", options.external_recipient, 1);
+    }
+    setenv("CRM_notify_node", node, 1);
+    setenv("CRM_notify_task", task, 1);
+    setenv("CRM_notify_desc", desc, 1);
+    setenv("CRM_notify_rc", rc_s, 1);
+    setenv("CRM_notify_target_rc", target_rc_s, 1);
+    setenv("CRM_notify_status", status_s, 1);
+
+    pid = fork();
+    if (pid == -1) {
+        crm_perror(LOG_ERR, "notification fork() failed.");
+    }
+    if (pid == 0) {
+        /* crm_debug("notification: I am the child. Executing the nofitication program."); */
+        execl(options.external_agent, options.external_agent, NULL);
+        exit(CRM_EX_ERROR);
+    }
+
+    crm_trace("Finished running custom notification program '%s'.", options.external_agent);
+    free(target_rc_s);
+    free(status_s);
+    free(rc_s);
+    return 0;
+}
+
+static int
+handle_rsc_op(xmlNode *xml, void *userdata)
+{
+    const char *node_id = (const char *) userdata;
+    int rc = -1;
+    int status = -1;
+    int target_rc = -1;
+    gboolean notify = TRUE;
+
+    char *rsc = NULL;
+    char *task = NULL;
+    const char *desc = NULL;
+    const char *magic = NULL;
+    const char *id = NULL;
+    const char *node = NULL;
+
+    xmlNode *n = xml;
+    xmlNode * rsc_op = xml;
+
+    if(strcmp((const char*)xml->name, XML_LRM_TAG_RSC_OP) != 0) {
+        pcmk__xe_foreach_child(xml, NULL, handle_rsc_op, (void *) node_id);
+        return pcmk_rc_ok;
+    }
+
+    id = pe__xe_history_key(rsc_op);
+
+    magic = crm_element_value(rsc_op, XML_ATTR_TRANSITION_MAGIC);
+    if (magic == NULL) {
+        /* non-change */
+        return pcmk_rc_ok;
+    }
+
+    if (!decode_transition_magic(magic, NULL, NULL, NULL, &status, &rc,
+                                 &target_rc)) {
+        crm_err("Invalid event %s detected for %s", magic, id);
+        return pcmk_rc_ok;
+    }
+
+    if (parse_op_key(id, &rsc, &task, NULL) == FALSE) {
+        crm_err("Invalid event detected for %s", id);
+        goto bail;
+    }
+
+    node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
+
+    while (n != NULL && !pcmk__str_eq(XML_CIB_TAG_STATE, TYPE(n), pcmk__str_casei)) {
+        n = n->parent;
+    }
+
+    if(node == NULL && n) {
+        node = crm_element_value(n, XML_ATTR_UNAME);
+    }
+
+    if (node == NULL && n) {
+        node = ID(n);
+    }
+
+    if (node == NULL) {
+        node = node_id;
+    }
+
+    if (node == NULL) {
+        crm_err("No node detected for event %s (%s)", magic, id);
+        goto bail;
+    }
+
+    /* look up where we expected it to be? */
+    desc = pcmk_rc_str(pcmk_rc_ok);
+    if ((status == PCMK_EXEC_DONE) && (target_rc == rc)) {
+        crm_notice("%s of %s on %s completed: %s", task, rsc, node, desc);
+        if (rc == PCMK_OCF_NOT_RUNNING) {
+            notify = FALSE;
+        }
+
+    } else if (status == PCMK_EXEC_DONE) {
+        desc = services_ocf_exitcode_str(rc);
+        crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
+
+    } else {
+        desc = pcmk_exec_status_str(status);
+        crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
+    }
+
+    if (notify && options.external_agent) {
+        send_custom_trap(node, rsc, task, target_rc, rc, status, desc);
+    }
+
+  bail:
+    free(rsc);
+    free(task);
+    return pcmk_rc_ok;
+}
+
+/* This function is just a wrapper around mainloop_set_trigger so that it can be
+ * called from a mainloop directly.  It's simply another way of ensuring the screen
+ * gets redrawn.
+ */
+static gboolean
+mon_trigger_refresh(gpointer user_data)
+{
+    mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
+    return FALSE;
+}
+
+static int
+handle_op_for_node(xmlNode *xml, void *userdata)
+{
+    const char *node = crm_element_value(xml, XML_ATTR_UNAME);
+
+    if (node == NULL) {
+        node = ID(xml);
+    }
+
+    handle_rsc_op(xml, (void *) node);
+    return pcmk_rc_ok;
+}
+
+static void
+crm_diff_update_v2(const char *event, xmlNode * msg)
+{
+    xmlNode *change = NULL;
+    xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
+
+    for (change = pcmk__xml_first_child(diff); change != NULL;
+         change = pcmk__xml_next(change)) {
+        const char *name = NULL;
+        const char *op = crm_element_value(change, XML_DIFF_OP);
+        const char *xpath = crm_element_value(change, XML_DIFF_PATH);
+        xmlNode *match = NULL;
+        const char *node = NULL;
+
+        if(op == NULL) {
+            continue;
+
+        } else if(strcmp(op, "create") == 0) {
+            match = change->children;
+
+        } else if(strcmp(op, "move") == 0) {
+            continue;
+
+        } else if(strcmp(op, "delete") == 0) {
+            continue;
+
+        } else if(strcmp(op, "modify") == 0) {
+            match = first_named_child(change, XML_DIFF_RESULT);
+            if(match) {
+                match = match->children;
+            }
+        }
+
+        if(match) {
+            name = (const char *)match->name;
+        }
+
+        crm_trace("Handling %s operation for %s %p, %s", op, xpath, match, name);
+        if(xpath == NULL) {
+            /* Version field, ignore */
+
+        } else if(name == NULL) {
+            crm_debug("No result for %s operation to %s", op, xpath);
+            CRM_ASSERT(strcmp(op, "delete") == 0 || strcmp(op, "move") == 0);
+
+        } else if(strcmp(name, XML_TAG_CIB) == 0) {
+            pcmk__xe_foreach_child(first_named_child(match, XML_CIB_TAG_STATUS),
+                                   NULL, handle_op_for_node, NULL);
+
+        } else if(strcmp(name, XML_CIB_TAG_STATUS) == 0) {
+            pcmk__xe_foreach_child(match, NULL, handle_op_for_node, NULL);
+
+        } else if(strcmp(name, XML_CIB_TAG_STATE) == 0) {
+            node = crm_element_value(match, XML_ATTR_UNAME);
+            if (node == NULL) {
+                node = ID(match);
+            }
+            handle_rsc_op(match, (void *) node);
+
+        } else if(strcmp(name, XML_CIB_TAG_LRM) == 0) {
+            node = ID(match);
+            handle_rsc_op(match, (void *) node);
+
+        } else if(strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
+            char *local_node = pcmk__xpath_node_id(xpath, "lrm");
+
+            handle_rsc_op(match, local_node);
+            free(local_node);
+
+        } else if(strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
+            char *local_node = pcmk__xpath_node_id(xpath, "lrm");
+
+            handle_rsc_op(match, local_node);
+            free(local_node);
+
+        } else if(strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
+            char *local_node = pcmk__xpath_node_id(xpath, "lrm");
+
+            handle_rsc_op(match, local_node);
+            free(local_node);
+
+        } else {
+            crm_trace("Ignoring %s operation for %s %p, %s", op, xpath, match, name);
+        }
+    }
+}
+
+static void
+crm_diff_update_v1(const char *event, xmlNode * msg)
+{
+    /* Process operation updates */
+    xmlXPathObject *xpathObj = xpath_search(msg,
+                                            "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED
+                                            "//" XML_LRM_TAG_RSC_OP);
+    int lpc = 0, max = numXpathResults(xpathObj);
+
+    for (lpc = 0; lpc < max; lpc++) {
+        xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
+
+        handle_rsc_op(rsc_op, NULL);
+    }
+    freeXpathObject(xpathObj);
+}
+
+static void
+crm_diff_update(const char *event, xmlNode * msg)
+{
+    int rc = -1;
+    static bool stale = FALSE;
+    gboolean cib_updated = FALSE;
+    xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
+
+    out->progress(out, false);
+
+    if (current_cib != NULL) {
+        rc = xml_apply_patchset(current_cib, diff, TRUE);
+
+        switch (rc) {
+            case -pcmk_err_diff_resync:
+            case -pcmk_err_diff_failed:
+                crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
+                free_xml(current_cib); current_cib = NULL;
+                break;
+            case pcmk_ok:
+                cib_updated = TRUE;
+                break;
+            default:
+                crm_notice("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
+                free_xml(current_cib); current_cib = NULL;
+        }
+    }
+
+    if (current_cib == NULL) {
+        crm_trace("Re-requesting the full cib");
+        cib->cmds->query(cib, NULL, &current_cib, cib_scope_local | cib_sync_call);
+    }
+
+    if (options.external_agent) {
+        int format = 0;
+        crm_element_value_int(diff, "format", &format);
+        switch(format) {
+            case 1:
+                crm_diff_update_v1(event, msg);
+                break;
+            case 2:
+                crm_diff_update_v2(event, msg);
+                break;
+            default:
+                crm_err("Unknown patch format: %d", format);
+        }
+    }
+
+    if (current_cib == NULL) {
+        if(!stale) {
+            out->info(out, "--- Stale data ---");
+        }
+        stale = TRUE;
+        return;
+    }
+
+    stale = FALSE;
+    refresh_after_event(cib_updated, FALSE);
+}
+
+static int
+mon_refresh_display(gpointer user_data)
+{
+    int rc = pcmk_rc_ok;
+
+    last_refresh = time(NULL);
+
+    if (output_format == mon_output_none) {
+        return G_SOURCE_REMOVE;
+    }
+
+    if (fence_history == pcmk__fence_history_full &&
+        !pcmk_all_flags_set(show, pcmk_section_fencing_all) &&
+        output_format != mon_output_xml) {
+        fence_history = pcmk__fence_history_reduced;
+    }
+
+    // Get an up-to-date pacemakerd status for the cluster summary
+    if (cib->variant == cib_native) {
+        pcmk__pacemakerd_status(out, crm_system_name, options.reconnect_ms / 2,
+                                false, &pcmkd_state);
+    }
+
+    if (out->dest != stdout) {
+        out->reset(out);
+    }
+
+    rc = pcmk__output_cluster_status(out, st, cib, current_cib, pcmkd_state,
+                                     fence_history, show, show_opts,
+                                     options.only_node,options.only_rsc,
+                                     options.neg_location_prefix,
+                                     output_format == mon_output_monitor);
+
+    if (output_format == mon_output_monitor && rc != pcmk_rc_ok) {
+        clean_up(MON_STATUS_WARN);
+        return G_SOURCE_REMOVE;
+    } else if (rc == pcmk_rc_schema_validation) {
+        clean_up(CRM_EX_CONFIG);
+        return G_SOURCE_REMOVE;
+    }
+
+    if (out->dest != stdout) {
+        out->finish(out, CRM_EX_OK, true, NULL);
+    }
+
+    return G_SOURCE_CONTINUE;
+}
+
+/* This function is called for fencing events (see setup_fencer_connection() for
+ * which ones) when --watch-fencing is used on the command line
+ */
+static void
+mon_st_callback_event(stonith_t * st, stonith_event_t * e)
+{
+    if (st->state == stonith_disconnected) {
+        /* disconnect cib as well and have everything reconnect */
+        mon_cib_connection_destroy(NULL);
+    } else if (options.external_agent) {
+        char *desc = stonith__event_description(e);
+
+        send_custom_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
+        free(desc);
+    }
+}
+
+/* Cause the screen to be redrawn (via mainloop_set_trigger) when various conditions are met:
+ *
+ * - If the last update occurred more than reconnect_ms ago (defaults to 5s, but
+ *   can be changed via the -i command line option), or
+ * - After every 10 CIB updates, or
+ * - If it's been 2s since the last update
+ *
+ * This function sounds like it would be more broadly useful, but it is only called when a
+ * fencing event is received or a CIB diff occurrs.
+ */
+static void
+refresh_after_event(gboolean data_updated, gboolean enforce)
+{
+    static int updates = 0;
+    time_t now = time(NULL);
+
+    if (data_updated) {
+        updates++;
+    }
+
+    if(refresh_timer == NULL) {
+        refresh_timer = mainloop_timer_add("refresh", 2000, FALSE, mon_trigger_refresh, NULL);
+    }
+
+    if (reconnect_timer > 0) {
+        /* we will receive a refresh request after successful reconnect */
+        mainloop_timer_stop(refresh_timer);
+        return;
+    }
+
+    /* as we're not handling initial failure of fencer-connection as
+     * fatal give it a retry here
+     * not getting here if cib-reconnection is already on the way
+     */
+    setup_fencer_connection();
+
+    if (enforce ||
+        ((now - last_refresh) > (options.reconnect_ms / 1000)) ||
+        updates >= 10) {
+        mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
+        mainloop_timer_stop(refresh_timer);
+        updates = 0;
+
+    } else {
+        mainloop_timer_start(refresh_timer);
+    }
+}
+
+/* This function is called for fencing events (see setup_fencer_connection() for
+ * which ones) when --watch-fencing is NOT used on the command line
+ */
+static void
+mon_st_callback_display(stonith_t * st, stonith_event_t * e)
+{
+    if (st->state == stonith_disconnected) {
+        /* disconnect cib as well and have everything reconnect */
+        mon_cib_connection_destroy(NULL);
+    } else {
+        out->progress(out, false);
+        refresh_after_event(TRUE, FALSE);
+    }
+}
+
+/*
+ * De-init ncurses, disconnect from the CIB manager, disconnect fencing,
+ * deallocate memory and show usage-message if requested.
+ *
+ * We don't actually return, but nominally returning crm_exit_t allows a usage
+ * like "return clean_up(exit_code);" which helps static analysis understand the
+ * code flow.
+ */
+static crm_exit_t
+clean_up(crm_exit_t exit_code)
+{
+    /* Quitting crm_mon is much more complicated than it ought to be. */
+
+    /* (1) Close connections, free things, etc. */
+    cib__clean_up_connection(&cib);
+    stonith_api_delete(st);
+    free(options.neg_location_prefix);
+    free(options.only_node);
+    free(options.only_rsc);
+    free(options.pid_file);
+    g_slist_free_full(options.includes_excludes, free);
+
+    g_strfreev(processed_args);
+
+    /* (2) If this is abnormal termination and we're in curses mode, shut down
+     * curses first.  Any messages displayed to the screen before curses is shut
+     * down will be lost because doing the shut down will also restore the
+     * screen to whatever it looked like before crm_mon was started.
+     */
+    if (((error != NULL) || (exit_code == CRM_EX_USAGE))
+        && (output_format == mon_output_console)
+        && (out != NULL)) {
+
+        out->finish(out, exit_code, false, NULL);
+        pcmk__output_free(out);
+        out = NULL;
+    }
+
+    /* (3) If this is a command line usage related failure, print the usage
+     * message.
+     */
+    if (exit_code == CRM_EX_USAGE && (output_format == mon_output_console || output_format == mon_output_plain)) {
+        char *help = g_option_context_get_help(context, TRUE, NULL);
+
+        fprintf(stderr, "%s", help);
+        g_free(help);
+    }
+
+    pcmk__free_arg_context(context);
+
+    /* (4) If this is any kind of error, print the error out and exit.  Make
+     * sure to handle situations both before and after formatted output is
+     * set up.  We want errors to appear formatted if at all possible.
+     */
+    if (error != NULL) {
+        if (out != NULL) {
+            out->err(out, "%s: %s", g_get_prgname(), error->message);
+            out->finish(out, exit_code, true, NULL);
+            pcmk__output_free(out);
+        } else {
+            fprintf(stderr, "%s: %s\n", g_get_prgname(), error->message);
+        }
+
+        g_clear_error(&error);
+        crm_exit(exit_code);
+    }
+
+    /* (5) Print formatted output to the screen if we made it far enough in
+     * crm_mon to be able to do so.
+     */
+    if (out != NULL) {
+        if (options.exec_mode != mon_exec_daemonized) {
+            out->finish(out, exit_code, true, NULL);
+        }
+
+        pcmk__output_free(out);
+        pcmk__unregister_formats();
+    }
+
+    crm_exit(exit_code);
+}
diff --git a/tools/crm_mon.h b/tools/crm_mon.h
new file mode 100644
index 0000000..a505f50
--- /dev/null
+++ b/tools/crm_mon.h
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2019-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef CRM_MON__H
+#define CRM_MON__H
+
+#include <crm_internal.h>
+
+#include <glib.h>
+
+#include <crm/common/output_internal.h>
+#include <crm/pengine/pe_types.h>
+#include <crm/stonith-ng.h>
+
+/*
+ * The man pages for both curses and ncurses suggest inclusion of "curses.h".
+ * We believe the following to be acceptable and portable.
+ */
+
+#  if defined(HAVE_LIBNCURSES) || defined(HAVE_LIBCURSES)
+#    if defined(HAVE_NCURSES_H) && !defined(HAVE_INCOMPATIBLE_PRINTW)
+#      include <ncurses.h>
+#      define CURSES_ENABLED 1
+#    elif defined(HAVE_NCURSES_NCURSES_H) && !defined(HAVE_INCOMPATIBLE_PRINTW)
+#      include <ncurses/ncurses.h>
+#      define CURSES_ENABLED 1
+#    elif defined(HAVE_CURSES_H) && !defined(HAVE_INCOMPATIBLE_PRINTW)
+#      include <curses.h>
+#      define CURSES_ENABLED 1
+#    elif defined(HAVE_CURSES_CURSES_H) && !defined(HAVE_INCOMPATIBLE_PRINTW)
+#      include <curses/curses.h>
+#      define CURSES_ENABLED 1
+#    else
+#      define CURSES_ENABLED 0
+#    endif
+#  else
+#    define CURSES_ENABLED 0
+#  endif
+
+typedef enum mon_output_format_e {
+    mon_output_unset,
+    mon_output_none,
+    mon_output_monitor,
+    mon_output_plain,
+    mon_output_console,
+    mon_output_xml,
+    mon_output_legacy_xml,
+    mon_output_html,
+    mon_output_cgi
+} mon_output_format_t;
+
+enum mon_exec_mode {
+    mon_exec_unset,
+    mon_exec_daemonized,
+    mon_exec_one_shot,
+    mon_exec_update,
+};
+
+void crm_mon_register_messages(pcmk__output_t *out);
+
+#if CURSES_ENABLED
+pcmk__output_t *crm_mon_mk_curses_output(char **argv);
+void curses_formatted_printf(pcmk__output_t *out, const char *format, ...) G_GNUC_PRINTF(2, 3);
+void curses_formatted_vprintf(pcmk__output_t *out, const char *format, va_list args) G_GNUC_PRINTF(2, 0);
+void curses_indented_printf(pcmk__output_t *out, const char *format, ...) G_GNUC_PRINTF(2, 3);
+void curses_indented_vprintf(pcmk__output_t *out, const char *format, va_list args) G_GNUC_PRINTF(2, 0);
+
+extern GOptionEntry crm_mon_curses_output_entries[];
+#define CRM_MON_SUPPORTED_FORMAT_CURSES { "console", crm_mon_mk_curses_output, crm_mon_curses_output_entries }
+#endif
+
+#endif
diff --git a/tools/crm_mon.service.in b/tools/crm_mon.service.in
new file mode 100644
index 0000000..7ae6649
--- /dev/null
+++ b/tools/crm_mon.service.in
@@ -0,0 +1,17 @@
+[Unit]
+Description=Daemon for pacemaker monitor
+Documentation=man:crm_mon
+After=pacemaker.service
+
+[Service]
+Type=forking
+EnvironmentFile=-@CONFIGDIR@/crm_mon
+ExecStart=@sbindir@/crm_mon $OPTIONS
+Restart=always
+
+# crm_perror() writes directly to stderr, so ignore it here to avoid
+# double-logging with the wrong format
+StandardError=null
+
+[Install]
+WantedBy=multi-user.target
diff --git a/tools/crm_mon.upstart.in b/tools/crm_mon.upstart.in
new file mode 100644
index 0000000..eb4c956
--- /dev/null
+++ b/tools/crm_mon.upstart.in
@@ -0,0 +1,35 @@
+# crm_mon - Daemon for pacemaker monitor
+#
+#
+
+kill timeout 3600
+respawn
+respawn limit 10 3600
+
+expect fork
+
+env prog=crm_mon
+env sysconf=@CONFIGDIR@/crm_mon
+env rpm_lockdir=@localstatedir@/lock/subsys
+env deb_lockdir=@localstatedir@/lock
+
+
+script
+    [ -f "$sysconf" ] && . "$sysconf"
+    exec $prog $OPTIONS
+end script
+
+post-start script
+    [ -f "$sysconf" ] && . "$sysconf"
+    [ -z "$LOCK_FILE" -a -d "$rpm_lockdir" ] && LOCK_FILE="$rpm_lockdir/$prog"
+    [ -z "$LOCK_FILE" -a -d "$deb_lockdir" ] && LOCK_FILE="$deb_lockdir/$prog"
+    touch "$LOCK_FILE"
+end script
+
+post-stop script
+    [ -f "$sysconf" ] && . "$sysconf"
+    [ -z "$LOCK_FILE" -a -d "$rpm_lockdir" ] && LOCK_FILE="$rpm_lockdir/$prog"
+    [ -z "$LOCK_FILE" -a -d "$deb_lockdir" ] && LOCK_FILE="$deb_lockdir/$prog"
+    rm -f "$LOCK_FILE"
+end script
+
diff --git a/tools/crm_mon_curses.c b/tools/crm_mon_curses.c
new file mode 100644
index 0000000..769c7c9
--- /dev/null
+++ b/tools/crm_mon_curses.c
@@ -0,0 +1,490 @@
+/*
+ * Copyright 2019-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <crm/crm.h>
+#include <crm/common/output.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/stonith-ng.h>
+#include <crm/fencing/internal.h>
+#include <crm/pengine/internal.h>
+#include <glib.h>
+#include <pacemaker-internal.h>
+
+#include "crm_mon.h"
+
+#if CURSES_ENABLED
+
+GOptionEntry crm_mon_curses_output_entries[] = {
+    { NULL }
+};
+
+typedef struct curses_list_data_s {
+    unsigned int len;
+    char *singular_noun;
+    char *plural_noun;
+} curses_list_data_t;
+
+typedef struct private_data_s {
+    GQueue *parent_q;
+} private_data_t;
+
+static void
+curses_free_priv(pcmk__output_t *out) {
+    private_data_t *priv = NULL;
+
+    if (out == NULL || out->priv == NULL) {
+        return;
+    }
+
+    priv = out->priv;
+
+    g_queue_free(priv->parent_q);
+    free(priv);
+    out->priv = NULL;
+}
+
+static bool
+curses_init(pcmk__output_t *out) {
+    private_data_t *priv = NULL;
+
+    CRM_ASSERT(out != NULL);
+
+    /* If curses_init was previously called on this output struct, just return. */
+    if (out->priv != NULL) {
+        return true;
+    } else {
+        out->priv = calloc(1, sizeof(private_data_t));
+        if (out->priv == NULL) {
+            return false;
+        }
+
+        priv = out->priv;
+    }
+
+    priv->parent_q = g_queue_new();
+
+    initscr();
+    cbreak();
+    noecho();
+
+    return true;
+}
+
+static void
+curses_finish(pcmk__output_t *out, crm_exit_t exit_status, bool print, void **copy_dest) {
+    CRM_ASSERT(out != NULL);
+
+    echo();
+    nocbreak();
+    endwin();
+}
+
+static void
+curses_reset(pcmk__output_t *out) {
+    CRM_ASSERT(out != NULL);
+
+    curses_free_priv(out);
+    curses_init(out);
+}
+
+static void
+curses_subprocess_output(pcmk__output_t *out, int exit_status,
+                         const char *proc_stdout, const char *proc_stderr) {
+    CRM_ASSERT(out != NULL);
+
+    if (proc_stdout != NULL) {
+        printw("%s\n", proc_stdout);
+    }
+
+    if (proc_stderr != NULL) {
+        printw("%s\n", proc_stderr);
+    }
+
+    clrtoeol();
+    refresh();
+}
+
+/* curses_version is defined in curses.h, so we can't use that name here.
+ * This function is empty because we create a text object instead of a console
+ * object if version is requested, so this is never called.
+ */
+static void
+curses_ver(pcmk__output_t *out, bool extended) {
+    CRM_ASSERT(out != NULL);
+}
+
+G_GNUC_PRINTF(2, 3)
+static void
+curses_error(pcmk__output_t *out, const char *format, ...) {
+    va_list ap;
+
+    CRM_ASSERT(out != NULL);
+
+    /* Informational output does not get indented, to separate it from other
+     * potentially indented list output.
+     */
+    va_start(ap, format);
+    vw_printw(stdscr, format, ap);
+    va_end(ap);
+
+    /* Add a newline. */
+    addch('\n');
+
+    clrtoeol();
+    refresh();
+    sleep(2);
+}
+
+G_GNUC_PRINTF(2, 3)
+static int
+curses_info(pcmk__output_t *out, const char *format, ...) {
+    va_list ap;
+
+    CRM_ASSERT(out != NULL);
+
+    if (out->is_quiet(out)) {
+        return pcmk_rc_no_output;
+    }
+
+    /* Informational output does not get indented, to separate it from other
+     * potentially indented list output.
+     */
+    va_start(ap, format);
+    vw_printw(stdscr, format, ap);
+    va_end(ap);
+
+    /* Add a newline. */
+    addch('\n');
+
+    clrtoeol();
+    refresh();
+    return pcmk_rc_ok;
+}
+
+static void
+curses_output_xml(pcmk__output_t *out, const char *name, const char *buf) {
+    CRM_ASSERT(out != NULL);
+    curses_indented_printf(out, "%s", buf);
+}
+
+G_GNUC_PRINTF(4, 5)
+static void
+curses_begin_list(pcmk__output_t *out, const char *singular_noun, const char *plural_noun,
+                  const char *format, ...) {
+    private_data_t *priv = NULL;
+    curses_list_data_t *new_list = NULL;
+    va_list ap;
+
+    CRM_ASSERT(out != NULL && out->priv != NULL);
+    priv = out->priv;
+
+    /* Empty formats can be used to create a new level of indentation, but without
+     * displaying some sort of list header.  In that case we need to not do any of
+     * this stuff. vw_printw will act weird if told to print a NULL.
+     */
+    if (format != NULL) {
+        va_start(ap, format);
+
+        curses_indented_vprintf(out, format, ap);
+        printw(":\n");
+
+        va_end(ap);
+    }
+
+    new_list = calloc(1, sizeof(curses_list_data_t));
+    new_list->len = 0;
+    pcmk__str_update(&new_list->singular_noun, singular_noun);
+    pcmk__str_update(&new_list->plural_noun, plural_noun);
+
+    g_queue_push_tail(priv->parent_q, new_list);
+}
+
+G_GNUC_PRINTF(3, 4)
+static void
+curses_list_item(pcmk__output_t *out, const char *id, const char *format, ...) {
+    va_list ap;
+
+    CRM_ASSERT(out != NULL);
+
+    va_start(ap, format);
+
+    if (id != NULL) {
+        curses_indented_printf(out, "%s: ", id);
+        vw_printw(stdscr, format, ap);
+    } else {
+        curses_indented_vprintf(out, format, ap);
+    }
+
+    addch('\n');
+    va_end(ap);
+
+    out->increment_list(out);
+}
+
+static void
+curses_increment_list(pcmk__output_t *out) {
+    private_data_t *priv = NULL;
+    gpointer tail;
+
+    CRM_ASSERT(out != NULL && out->priv != NULL);
+    priv = out->priv;
+
+    tail = g_queue_peek_tail(priv->parent_q);
+    CRM_ASSERT(tail != NULL);
+    ((curses_list_data_t *) tail)->len++;
+}
+
+static void
+curses_end_list(pcmk__output_t *out) {
+    private_data_t *priv = NULL;
+    curses_list_data_t *node = NULL;
+
+    CRM_ASSERT(out != NULL && out->priv != NULL);
+    priv = out->priv;
+
+    node = g_queue_pop_tail(priv->parent_q);
+
+    if (node->singular_noun != NULL && node->plural_noun != NULL) {
+        if (node->len == 1) {
+            curses_indented_printf(out, "%d %s found\n", node->len, node->singular_noun);
+        } else {
+            curses_indented_printf(out, "%d %s found\n", node->len, node->plural_noun);
+        }
+    }
+
+    free(node);
+}
+
+static bool
+curses_is_quiet(pcmk__output_t *out) {
+    CRM_ASSERT(out != NULL);
+    return out->quiet;
+}
+
+static void
+curses_spacer(pcmk__output_t *out) {
+    CRM_ASSERT(out != NULL);
+    addch('\n');
+}
+
+static void
+curses_progress(pcmk__output_t *out, bool end) {
+    CRM_ASSERT(out != NULL);
+
+    if (end) {
+        printw(".\n");
+    } else {
+        addch('.');
+    }
+}
+
+static void
+curses_prompt(const char *prompt, bool do_echo, char **dest)
+{
+    int rc = OK;
+
+    CRM_ASSERT(prompt != NULL && dest != NULL);
+
+    /* This is backwards from the text version of this function on purpose.  We
+     * disable echo by default in curses_init, so we need to enable it here if
+     * asked for.
+     */
+    if (do_echo) {
+        rc = echo();
+    }
+
+    if (rc == OK) {
+        printw("%s: ", prompt);
+
+        if (*dest != NULL) {
+            free(*dest);
+        }
+
+        *dest = calloc(1, 1024);
+        /* On older systems, scanw is defined as taking a char * for its first argument,
+         * while newer systems rightly want a const char *.  Accomodate both here due
+         * to building with -Werror.
+         */
+        rc = scanw((NCURSES_CONST char *) "%1023s", *dest);
+        addch('\n');
+    }
+
+    if (rc < 1) {
+        free(*dest);
+        *dest = NULL;
+    }
+
+    if (do_echo) {
+        noecho();
+    }
+}
+
+pcmk__output_t *
+crm_mon_mk_curses_output(char **argv) {
+    pcmk__output_t *retval = calloc(1, sizeof(pcmk__output_t));
+
+    if (retval == NULL) {
+        return NULL;
+    }
+
+    retval->fmt_name = "console";
+    retval->request = pcmk__quote_cmdline(argv);
+
+    retval->init = curses_init;
+    retval->free_priv = curses_free_priv;
+    retval->finish = curses_finish;
+    retval->reset = curses_reset;
+
+    retval->register_message = pcmk__register_message;
+    retval->message = pcmk__call_message;
+
+    retval->subprocess_output = curses_subprocess_output;
+    retval->version = curses_ver;
+    retval->err = curses_error;
+    retval->info = curses_info;
+    retval->transient = curses_info;
+    retval->output_xml = curses_output_xml;
+
+    retval->begin_list = curses_begin_list;
+    retval->list_item = curses_list_item;
+    retval->increment_list = curses_increment_list;
+    retval->end_list = curses_end_list;
+
+    retval->is_quiet = curses_is_quiet;
+    retval->spacer = curses_spacer;
+    retval->progress = curses_progress;
+    retval->prompt = curses_prompt;
+
+    return retval;
+}
+
+G_GNUC_PRINTF(2, 0)
+void
+curses_formatted_vprintf(pcmk__output_t *out, const char *format, va_list args) {
+    vw_printw(stdscr, format, args);
+
+    clrtoeol();
+    refresh();
+}
+
+G_GNUC_PRINTF(2, 3)
+void
+curses_formatted_printf(pcmk__output_t *out, const char *format, ...) {
+    va_list ap;
+
+    va_start(ap, format);
+    curses_formatted_vprintf(out, format, ap);
+    va_end(ap);
+}
+
+G_GNUC_PRINTF(2, 0)
+void
+curses_indented_vprintf(pcmk__output_t *out, const char *format, va_list args) {
+    int level = 0;
+    private_data_t *priv = NULL;
+
+    CRM_ASSERT(out != NULL && out->priv != NULL);
+
+    priv = out->priv;
+
+    level = g_queue_get_length(priv->parent_q);
+
+    for (int i = 0; i < level; i++) {
+        printw("  ");
+    }
+
+    if (level > 0) {
+        printw("* ");
+    }
+
+    curses_formatted_vprintf(out, format, args);
+}
+
+G_GNUC_PRINTF(2, 3)
+void
+curses_indented_printf(pcmk__output_t *out, const char *format, ...) {
+    va_list ap;
+
+    va_start(ap, format);
+    curses_indented_vprintf(out, format, ap);
+    va_end(ap);
+}
+
+PCMK__OUTPUT_ARGS("maint-mode", "unsigned long long int")
+static int
+cluster_maint_mode_console(pcmk__output_t *out, va_list args) {
+    unsigned long long flags = va_arg(args, unsigned long long);
+
+    if (pcmk_is_set(flags, pe_flag_maintenance_mode)) {
+        curses_formatted_printf(out, "\n              *** Resource management is DISABLED ***\n");
+        curses_formatted_printf(out, "  The cluster will not attempt to start, stop or recover services\n");
+        return pcmk_rc_ok;
+    } else if (pcmk_is_set(flags, pe_flag_stop_everything)) {
+        curses_formatted_printf(out, "\n    *** Resource management is DISABLED ***\n");
+        curses_formatted_printf(out, "  The cluster will keep all resources stopped\n");
+        return pcmk_rc_ok;
+    } else {
+        return pcmk_rc_no_output;
+    }
+}
+
+PCMK__OUTPUT_ARGS("cluster-status", "pe_working_set_t *",
+                  "enum pcmk_pacemakerd_state", "crm_exit_t",
+                  "stonith_history_t *", "enum pcmk__fence_history", "uint32_t",
+                  "uint32_t", "const char *", "GList *", "GList *")
+static int
+cluster_status_console(pcmk__output_t *out, va_list args) {
+    int rc = pcmk_rc_no_output;
+
+    clear();
+    rc = pcmk__cluster_status_text(out, args);
+    refresh();
+    return rc;
+}
+
+PCMK__OUTPUT_ARGS("stonith-event", "stonith_history_t *", "bool", "bool",
+                  "const char *", "uint32_t")
+static int
+stonith_event_console(pcmk__output_t *out, va_list args)
+{
+    stonith_history_t *event = va_arg(args, stonith_history_t *);
+    bool full_history = va_arg(args, int);
+    bool completed_only G_GNUC_UNUSED = va_arg(args, int);
+    const char *succeeded = va_arg(args, const char *);
+    uint32_t show_opts = va_arg(args, uint32_t);
+
+    gchar *desc = stonith__history_description(event, full_history, succeeded,
+                                               show_opts);
+
+
+    curses_indented_printf(out, "%s\n", desc);
+    g_free(desc);
+    return pcmk_rc_ok;
+}
+
+static pcmk__message_entry_t fmt_functions[] = {
+    { "cluster-status", "console", cluster_status_console },
+    { "maint-mode", "console", cluster_maint_mode_console },
+    { "stonith-event", "console", stonith_event_console },
+
+    { NULL, NULL, NULL }
+};
+
+#endif
+
+void
+crm_mon_register_messages(pcmk__output_t *out) {
+#if CURSES_ENABLED
+    pcmk__register_messages(out, fmt_functions);
+#endif
+}
diff --git a/tools/crm_node.8.inc b/tools/crm_node.8.inc
new file mode 100644
index 0000000..84cef12
--- /dev/null
+++ b/tools/crm_node.8.inc
@@ -0,0 +1,5 @@
+[synopsis]
+crm_node [command] [options]
+
+/level node information/
+.SH OPTIONS
diff --git a/tools/crm_node.c b/tools/crm_node.c
new file mode 100644
index 0000000..ac2a190
--- /dev/null
+++ b/tools/crm_node.c
@@ -0,0 +1,601 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <sys/types.h>
+
+#include <crm/crm.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/output_internal.h>
+#include <crm/common/mainloop.h>
+#include <crm/msg_xml.h>
+#include <crm/cib.h>
+#include <crm/cib/internal.h>
+#include <crm/common/ipc_controld.h>
+#include <crm/common/attrd_internal.h>
+
+#define SUMMARY "crm_node - Tool for displaying low-level node information"
+
+struct {
+    gboolean corosync;
+    gboolean dangerous_cmd;
+    gboolean force_flag;
+    char command;
+    int nodeid;
+    char *target_uname;
+} options = {
+    .command = '\0',
+    .force_flag = FALSE
+};
+
+gboolean command_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean name_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+
+static GMainLoop *mainloop = NULL;
+static crm_exit_t exit_code = CRM_EX_OK;
+
+#define INDENT "                           "
+
+static GOptionEntry command_entries[] = {
+    { "cluster-id", 'i', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display this node's cluster id",
+      NULL },
+    { "list", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display all known members (past and present) of this cluster",
+      NULL },
+    { "name", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the name used by the cluster for this node",
+      NULL },
+    { "partition", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the members of this partition",
+      NULL },
+    { "quorum", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display a 1 if our partition has quorum, 0 if not",
+      NULL },
+    { "name-for-id", 'N', 0, G_OPTION_ARG_CALLBACK, name_cb,
+      "Display the name used by the cluster for the node with the specified ID",
+      "ID" },
+    { "remove", 'R', 0, G_OPTION_ARG_CALLBACK, remove_cb,
+      "(Advanced) Remove the (stopped) node with the specified name from Pacemaker's\n"
+      INDENT "configuration and caches (the node must already have been removed from\n"
+      INDENT "the underlying cluster stack configuration",
+      "NAME" },
+
+    { NULL }
+};
+
+static GOptionEntry addl_entries[] = {
+    { "force", 'f', 0, G_OPTION_ARG_NONE, &options.force_flag,
+      NULL,
+      NULL },
+#if SUPPORT_COROSYNC
+    /* Unused and deprecated */
+    { "corosync", 'C', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.corosync,
+      NULL,
+      NULL },
+#endif
+
+    // @TODO add timeout option for when IPC replies are needed
+
+    { NULL }
+};
+
+gboolean
+command_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_eq("-i", option_name, pcmk__str_casei) || pcmk__str_eq("--cluster-id", option_name, pcmk__str_casei)) {
+        options.command = 'i';
+    } else if (pcmk__str_eq("-l", option_name, pcmk__str_casei) || pcmk__str_eq("--list", option_name, pcmk__str_casei)) {
+        options.command = 'l';
+    } else if (pcmk__str_eq("-n", option_name, pcmk__str_casei) || pcmk__str_eq("--name", option_name, pcmk__str_casei)) {
+        options.command = 'n';
+    } else if (pcmk__str_eq("-p", option_name, pcmk__str_casei) || pcmk__str_eq("--partition", option_name, pcmk__str_casei)) {
+        options.command = 'p';
+    } else if (pcmk__str_eq("-q", option_name, pcmk__str_casei) || pcmk__str_eq("--quorum", option_name, pcmk__str_casei)) {
+        options.command = 'q';
+    } else {
+        g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Unknown param passed to command_cb: %s\n", option_name);
+        return FALSE;
+    }
+
+    return TRUE;
+}
+
+gboolean
+name_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.command = 'N';
+    pcmk__scan_min_int(optarg, &(options.nodeid), 0);
+    return TRUE;
+}
+
+gboolean
+remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (optarg == NULL) {
+        crm_err("-R option requires an argument");
+        g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "-R option requires an argument");
+        return FALSE;
+    }
+
+    options.command = 'R';
+    options.dangerous_cmd = TRUE;
+    pcmk__str_update(&options.target_uname, optarg);
+    return TRUE;
+}
+
+static gint
+sort_node(gconstpointer a, gconstpointer b)
+{
+    const pcmk_controld_api_node_t *node_a = a;
+    const pcmk_controld_api_node_t *node_b = b;
+
+    return pcmk__numeric_strcasecmp((node_a->uname? node_a->uname : ""),
+                                    (node_b->uname? node_b->uname : ""));
+}
+
+static void
+controller_event_cb(pcmk_ipc_api_t *controld_api,
+                    enum pcmk_ipc_event event_type, crm_exit_t status,
+                    void *event_data, void *user_data)
+{
+    pcmk_controld_api_reply_t *reply = event_data;
+
+    switch (event_type) {
+        case pcmk_ipc_event_disconnect:
+            if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
+                fprintf(stderr, "error: Lost connection to controller\n");
+            }
+            goto done;
+            break;
+
+        case pcmk_ipc_event_reply:
+            break;
+
+        default:
+            return;
+    }
+
+    if (status != CRM_EX_OK) {
+        fprintf(stderr, "error: Bad reply from controller: %s\n",
+                crm_exit_str(status));
+        goto done;
+    }
+
+    // Parse desired info from reply and display to user
+    switch (options.command) {
+        case 'i':
+            if (reply->reply_type != pcmk_controld_reply_info) {
+                fprintf(stderr,
+                        "error: Unknown reply type %d from controller\n",
+                        reply->reply_type);
+                goto done;
+            }
+            if (reply->data.node_info.id == 0) {
+                fprintf(stderr,
+                        "error: Controller reply did not contain node ID\n");
+                exit_code = CRM_EX_PROTOCOL;
+                goto done;
+            }
+            printf("%d\n", reply->data.node_info.id);
+            break;
+
+        case 'n':
+        case 'N':
+            if (reply->reply_type != pcmk_controld_reply_info) {
+                fprintf(stderr,
+                        "error: Unknown reply type %d from controller\n",
+                        reply->reply_type);
+                goto done;
+            }
+            if (reply->data.node_info.uname == NULL) {
+                fprintf(stderr, "Node is not known to cluster\n");
+                exit_code = CRM_EX_NOHOST;
+                goto done;
+            }
+            printf("%s\n", reply->data.node_info.uname);
+            break;
+
+        case 'q':
+            if (reply->reply_type != pcmk_controld_reply_info) {
+                fprintf(stderr,
+                        "error: Unknown reply type %d from controller\n",
+                        reply->reply_type);
+                goto done;
+            }
+            printf("%d\n", reply->data.node_info.have_quorum);
+            if (!(reply->data.node_info.have_quorum)) {
+                exit_code = CRM_EX_QUORUM;
+                goto done;
+            }
+            break;
+
+        case 'l':
+        case 'p':
+            if (reply->reply_type != pcmk_controld_reply_nodes) {
+                fprintf(stderr,
+                        "error: Unknown reply type %d from controller\n",
+                        reply->reply_type);
+                goto done;
+            }
+            reply->data.nodes = g_list_sort(reply->data.nodes, sort_node);
+            for (GList *node_iter = reply->data.nodes;
+                 node_iter != NULL; node_iter = node_iter->next) {
+
+                pcmk_controld_api_node_t *node = node_iter->data;
+                const char *uname = (node->uname? node->uname : "");
+                const char *state = (node->state? node->state : "");
+
+                if (options.command == 'l') {
+                    printf("%lu %s %s\n",
+                           (unsigned long) node->id, uname, state);
+
+                // i.e. CRM_NODE_MEMBER, but we don't want to include cluster.h
+                } else if (!strcmp(state, "member")) {
+                    printf("%s ", uname);
+                }
+            }
+            if (options.command == 'p') {
+                printf("\n");
+            }
+            break;
+
+        default:
+            fprintf(stderr, "internal error: Controller reply not expected\n");
+            exit_code = CRM_EX_SOFTWARE;
+            goto done;
+    }
+
+    // Success
+    exit_code = CRM_EX_OK;
+done:
+    pcmk_disconnect_ipc(controld_api);
+    pcmk_quit_main_loop(mainloop, 10);
+}
+
+static void
+run_controller_mainloop(uint32_t nodeid, bool list_nodes)
+{
+    pcmk_ipc_api_t *controld_api = NULL;
+    int rc;
+
+    // Set disconnect exit code to handle unexpected disconnects
+    exit_code = CRM_EX_DISCONNECT;
+
+    // Create controller IPC object
+    rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
+    if (rc != pcmk_rc_ok) {
+        fprintf(stderr, "error: Could not connect to controller: %s\n",
+                pcmk_rc_str(rc));
+        return;
+    }
+    pcmk_register_ipc_callback(controld_api, controller_event_cb, NULL);
+
+    // Connect to controller
+    rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main);
+    if (rc != pcmk_rc_ok) {
+        fprintf(stderr, "error: Could not connect to controller: %s\n",
+                pcmk_rc_str(rc));
+        exit_code = pcmk_rc2exitc(rc);
+        return;
+    }
+
+    if (list_nodes) {
+        rc = pcmk_controld_api_list_nodes(controld_api);
+    } else {
+        rc = pcmk_controld_api_node_info(controld_api, nodeid);
+    }
+    if (rc != pcmk_rc_ok) {
+        fprintf(stderr, "error: Could not ping controller: %s\n",
+                pcmk_rc_str(rc));
+        pcmk_disconnect_ipc(controld_api);
+        exit_code = pcmk_rc2exitc(rc);
+        return;
+    }
+
+    // Run main loop to get controller reply via controller_event_cb()
+    mainloop = g_main_loop_new(NULL, FALSE);
+    g_main_loop_run(mainloop);
+    g_main_loop_unref(mainloop);
+    mainloop = NULL;
+    pcmk_free_ipc_api(controld_api);
+}
+
+static void
+print_node_name(void)
+{
+    // Check environment first (i.e. when called by resource agent)
+    const char *name = getenv("OCF_RESKEY_" CRM_META "_" XML_LRM_ATTR_TARGET);
+
+    if (name != NULL) {
+        printf("%s\n", name);
+        exit_code = CRM_EX_OK;
+        return;
+
+    } else {
+        /* Otherwise ask the controller.
+         * FIXME: Use pcmk__query_node_name() after conversion to formatted
+         * output.
+         */
+        run_controller_mainloop(0, false);
+    }
+}
+
+static int
+cib_remove_node(long id, const char *name)
+{
+    int rc;
+    cib_t *cib = NULL;
+    xmlNode *node = NULL;
+    xmlNode *node_state = NULL;
+
+    crm_trace("Removing %s from the CIB", name);
+
+    if(name == NULL && id == 0) {
+        return -ENOTUNIQ;
+    }
+
+    node = create_xml_node(NULL, XML_CIB_TAG_NODE);
+    node_state = create_xml_node(NULL, XML_CIB_TAG_STATE);
+
+    crm_xml_add(node, XML_ATTR_UNAME, name);
+    crm_xml_add(node_state, XML_ATTR_UNAME, name);
+    if (id > 0) {
+        crm_xml_set_id(node, "%ld", id);
+        crm_xml_add(node_state, XML_ATTR_ID, ID(node));
+    }
+
+    cib = cib_new();
+    cib->cmds->signon(cib, crm_system_name, cib_command);
+
+    rc = cib->cmds->remove(cib, XML_CIB_TAG_NODES, node, cib_sync_call);
+    if (rc != pcmk_ok) {
+        printf("Could not remove %s[%ld] from " XML_CIB_TAG_NODES ": %s",
+                name, id, pcmk_strerror(rc));
+    }
+    rc = cib->cmds->remove(cib, XML_CIB_TAG_STATUS, node_state, cib_sync_call);
+    if (rc != pcmk_ok) {
+        printf("Could not remove %s[%ld] from " XML_CIB_TAG_STATUS ": %s",
+                name, id, pcmk_strerror(rc));
+    }
+
+    cib__clean_up_connection(&cib);
+    return rc;
+}
+
+static int
+controller_remove_node(const char *node_name, long nodeid)
+{
+    pcmk_ipc_api_t *controld_api = NULL;
+    int rc;
+
+    // Create controller IPC object
+    rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
+    if (rc != pcmk_rc_ok) {
+        fprintf(stderr, "error: Could not connect to controller: %s\n",
+                pcmk_rc_str(rc));
+        return ENOTCONN;
+    }
+
+    // Connect to controller (without main loop)
+    rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_sync);
+    if (rc != pcmk_rc_ok) {
+        fprintf(stderr, "error: Could not connect to controller: %s\n",
+                pcmk_rc_str(rc));
+        pcmk_free_ipc_api(controld_api);
+        return rc;
+    }
+
+    rc = pcmk_ipc_purge_node(controld_api, node_name, nodeid);
+    if (rc != pcmk_rc_ok) {
+        fprintf(stderr,
+                "error: Could not clear node from controller's cache: %s\n",
+                pcmk_rc_str(rc));
+    }
+
+    pcmk_free_ipc_api(controld_api);
+    return pcmk_rc_ok;
+}
+
+static int
+tools_remove_node_cache(const char *node_name, long nodeid, const char *target)
+{
+    int rc = -1;
+    crm_ipc_t *conn = NULL;
+    xmlNode *cmd = NULL;
+
+    conn = crm_ipc_new(target, 0);
+    if (!conn) {
+        return -ENOTCONN;
+    }
+    if (!crm_ipc_connect(conn)) {
+        crm_perror(LOG_ERR, "Connection to %s failed", target);
+        crm_ipc_destroy(conn);
+        return -ENOTCONN;
+    }
+
+    crm_trace("Removing %s[%ld] from the %s membership cache",
+              node_name, nodeid, target);
+
+    if(pcmk__str_eq(target, T_ATTRD, pcmk__str_casei)) {
+        cmd = create_xml_node(NULL, __func__);
+
+        crm_xml_add(cmd, F_TYPE, T_ATTRD);
+        crm_xml_add(cmd, F_ORIG, crm_system_name);
+
+        crm_xml_add(cmd, PCMK__XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE);
+
+        pcmk__xe_add_node(cmd, node_name, nodeid);
+
+    } else { // Fencer or pacemakerd
+        cmd = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, target,
+                             crm_system_name, NULL);
+        if (nodeid > 0) {
+            crm_xml_set_id(cmd, "%ld", nodeid);
+        }
+        crm_xml_add(cmd, XML_ATTR_UNAME, node_name);
+    }
+
+    rc = crm_ipc_send(conn, cmd, 0, 0, NULL);
+    crm_debug("%s peer cache cleanup for %s (%ld): %d",
+              target, node_name, nodeid, rc);
+
+    if (rc > 0) {
+        // @TODO Should this be done just once after all the rest?
+        rc = cib_remove_node(nodeid, node_name);
+    }
+
+    if (conn) {
+        crm_ipc_close(conn);
+        crm_ipc_destroy(conn);
+    }
+    free_xml(cmd);
+    return rc > 0 ? 0 : rc;
+}
+
+static void
+remove_node(const char *target_uname)
+{
+    int rc;
+    int d = 0;
+    long nodeid = 0;
+    const char *node_name = NULL;
+    char *endptr = NULL;
+    const char *daemons[] = {
+        "stonith-ng",
+        T_ATTRD,
+        CRM_SYSTEM_MCP,
+    };
+
+    // Check whether node was specified by name or numeric ID
+    errno = 0;
+    nodeid = strtol(target_uname, &endptr, 10);
+    if ((errno != 0) || (endptr == target_uname) || (*endptr != '\0')
+        || (nodeid <= 0)) {
+        // It's not a positive integer, so assume it's a node name
+        nodeid = 0;
+        node_name = target_uname;
+    }
+
+    rc = controller_remove_node(node_name, nodeid);
+    if (rc != pcmk_rc_ok) {
+        exit_code = pcmk_rc2exitc(rc);
+        return;
+    }
+
+    for (d = 0; d < PCMK__NELEM(daemons); d++) {
+        if (tools_remove_node_cache(node_name, nodeid, daemons[d])) {
+            crm_err("Failed to connect to %s to remove node '%s'",
+                    daemons[d], target_uname);
+            exit_code = CRM_EX_ERROR;
+            return;
+        }
+    }
+    exit_code = CRM_EX_OK;
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup *group) {
+    GOptionContext *context = NULL;
+
+    GOptionEntry extra_prog_entries[] = {
+        { "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
+          "Be less descriptive in output.",
+          NULL },
+
+        { NULL }
+    };
+
+    context = pcmk__build_arg_context(args, NULL, &group, NULL);
+
+    /* Add the -q option, which cannot be part of the globally supported options
+     * because some tools use that flag for something else.
+     */
+    pcmk__add_main_args(context, extra_prog_entries);
+
+    pcmk__add_arg_group(context, "commands", "Commands:",
+                        "Show command help", command_entries);
+    pcmk__add_arg_group(context, "additional", "Additional Options:",
+                        "Show additional options", addl_entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    GError *error = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "NR");
+    GOptionContext *context = build_arg_context(args, output_group);
+
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("crm_node", args->verbosity);
+
+    if (args->version) {
+        g_strfreev(processed_args);
+        pcmk__free_arg_context(context);
+        /* FIXME:  When crm_node is converted to use formatted output, this can go. */
+        pcmk__cli_help('v');
+    }
+
+    if (options.command == 0) {
+        char *help = g_option_context_get_help(context, TRUE, NULL);
+
+        fprintf(stderr, "%s", help);
+        g_free(help);
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    if (options.dangerous_cmd && options.force_flag == FALSE) {
+        fprintf(stderr, "The supplied command is considered dangerous."
+                "  To prevent accidental destruction of the cluster,"
+                " the --force flag is required in order to proceed.\n");
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    switch (options.command) {
+        case 'n':
+            print_node_name();
+            break;
+        case 'R':
+            remove_node(options.target_uname);
+            break;
+        case 'i':
+        case 'q':
+        case 'N':
+            /* FIXME: Use pcmk__query_node_name() after conversion to formatted
+             * output
+             */
+            run_controller_mainloop(options.nodeid, false);
+            break;
+        case 'l':
+        case 'p':
+            run_controller_mainloop(0, true);
+            break;
+        default:
+            break;
+    }
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    pcmk__output_and_clear_error(&error, NULL);
+    return crm_exit(exit_code);
+}
diff --git a/tools/crm_report.in b/tools/crm_report.in
new file mode 100644
index 0000000..1818879
--- /dev/null
+++ b/tools/crm_report.in
@@ -0,0 +1,481 @@
+#!/bin/sh
+#
+# Copyright 2010-2019 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+TEMP=`@GETOPT_PATH@			\
+    -o hv?xl:f:t:n:T:L:p:c:dSCu:D:MVse:	\
+    --long help,corosync,cts:,cts-log:,dest:,node:,nodes:,from:,to:,sos-mode,logfile:,as-directory,single-node,cluster:,user:,max-depth:,version,features,rsh:	\
+    -n 'crm_report' -- "$@"`
+# The quotes around $TEMP are essential
+eval set -- "$TEMP"
+
+progname=$(basename "$0")
+rsh="ssh -T"
+tests=""
+nodes=""
+compress=1
+cluster="any"
+ssh_user="root"
+search_logs=1
+sos_mode=0
+report_data=`dirname $0`
+maxdepth=5
+
+extra_logs=""
+sanitize_patterns="passw.*"
+log_patterns="CRIT: ERROR:"
+
+usage() {
+cat<<EOF
+$progname - Create archive of everything needed when reporting cluster problems
+
+
+Usage: $progname [options] [DEST]
+
+Required option:
+  -f, --from TIME       time prior to problems beginning
+                        (as "YYYY-M-D H:M:S" including the quotes)
+
+Options:
+  -V                    increase verbosity (may be specified multiple times)
+  -h, --help            display this message
+  -v, --version         display software version
+  --features            display software features
+  -t, --to TIME         time at which all problems were resolved
+                        (as "YYYY-M-D H:M:S" including the quotes; default "now")
+  -T, --cts TEST        CTS test or set of tests to extract
+  --cts-log             CTS master logfile
+  -n, --nodes NODES     node names for this cluster (only needed if cluster is
+                        not active on this host; accepts -n "a b" or -n a -n b)
+  -M                    do not search for cluster logs
+  -l, --logfile FILE    log file to collect (in addition to detected logs if -M
+                        is not specified; may be specified multiple times)
+  -p PATT               additional regular expression to match variables to be
+                        masked in output (default: "passw.*")
+  -L PATT               additional regular expression to match in log files for
+                        analysis (default: $log_patterns)
+  -S, --single-node     don't attempt to collect data from other nodes
+  -c, --cluster TYPE    force the cluster type instead of detecting
+                        (currently only corosync is supported)
+  -C, --corosync        force the cluster type to be corosync
+  -u, --user USER       username to use when collecting data from other nodes
+                        (default root)
+  -D, --max-depth       search depth to use when attempting to locate files
+  -e, --rsh             command to use to run commands on other nodes
+                        (default ssh -T)
+  -d, --as-directory    leave result as a directory tree instead of archiving
+  --sos-mode            use defaults suitable for being called by sosreport tool
+                        (behavior subject to change and not useful to end users)
+  DEST, --dest DEST     custom destination directory or file name
+
+$progname works best when run from a cluster node on a running cluster,
+but can be run from a stopped cluster node or a Pacemaker Remote node.
+
+If neither --nodes nor --single-node is given, $progname will guess the
+node list, but may have trouble detecting Pacemaker Remote nodes.
+Unless --single-node is given, the node names (whether specified by --nodes
+or detected automatically) must be resolvable and reachable via the command
+specified by -e/--rsh using the user specified by -u/--user.
+
+Examples:
+   $progname -f "2011-12-14 13:05:00" unexplained-apache-failure
+   $progname -f 2011-12-14 -t 2011-12-15 something-that-took-multiple-days
+   $progname -f 13:05:00   -t 13:12:00   brief-outage
+EOF
+}
+
+case "$1" in
+    -v|--version)   echo "$progname @VERSION@-@BUILD_VERSION@"; exit 0;;
+    --features)     echo "@VERSION@-@BUILD_VERSION@: @PCMK_FEATURES@"; exit 0;;
+    --|-h|--help) usage; exit 0;;
+esac
+
+# Prefer helpers in the same directory if they exist, to simplify development
+if [ ! -f $report_data/report.common ]; then
+    report_data=@datadir@/@PACKAGE@
+else
+    echo "Using local helpers"
+fi
+
+. $report_data/report.common
+
+while true; do
+    case "$1" in
+	-x) set -x; shift;;
+	-V) verbose=`expr $verbose + 1`; shift;;
+	-T|--cts) tests="$tests $2"; shift; shift;;
+	   --cts-log) ctslog="$2"; shift; shift;;
+	-f|--from) start_time=`get_time "$2"`; shift; shift;;
+	-t|--to) end_time=`get_time "$2"`; shift; shift;;
+	-n|--node|--nodes) nodes="$nodes $2"; shift; shift;;
+	-S|--single-node) nodes="$host"; shift;;
+	-l|--logfile) extra_logs="$extra_logs $2"; shift; shift;;
+	-p) sanitize_patterns="$sanitize_patterns $2"; shift; shift;;
+	-L) log_patterns="$log_patterns `echo $2 | sed 's/ /\\\W/g'`"; shift; shift;;
+	-d|--as-directory) compress=0; shift;;
+	-C|--corosync)  cluster="corosync";  shift;;
+	-c|--cluster)   cluster="$2"; shift; shift;;
+	-e|--rsh)       rsh="$2";     shift; shift;;
+	-u|--user)      ssh_user="$2"; shift; shift;;
+        -D|--max-depth)     maxdepth="$2"; shift; shift;;
+	-M) search_logs=0; shift;;
+        --sos-mode) sos_mode=1; nodes="$host"; shift;;
+	--dest) DESTDIR=$2; shift; shift;;
+	--) if [ ! -z $2 ]; then DESTDIR=$2; fi; break;;
+	-h|--help) usage; exit 0;;
+	# Options for compatibility with hb_report
+	-s) shift;;
+
+	*) echo "Unknown argument: $1"; usage; exit 1;;
+    esac
+done
+
+
+collect_data() {
+    label="$1"
+    start=`expr $2 - 10`
+    end=`expr $3 + 10`
+    masterlog=$4
+
+    if [ "x$DESTDIR" != x ]; then
+	echo $DESTDIR | grep -e "^/" -qs
+	if [ $? = 0 ]; then
+	    l_base=$DESTDIR
+	else
+	    l_base="`pwd`/$DESTDIR"
+	fi
+	debug "Using custom scratch dir: $l_base"
+	r_base=`basename $l_base`
+    else
+	l_base=$HOME/$label
+	r_base=$label
+    fi
+
+    if [ -e $l_base ]; then
+	fatal "Output directory $l_base already exists, specify an alternate name with --dest"
+    fi
+    mkdir -p $l_base
+
+    if [ "x$masterlog" != "x" ]; then
+	dumplogset "$masterlog" $start $end > "$l_base/$HALOG_F"
+    fi
+
+    for node in $nodes; do
+	cat <<EOF >$l_base/.env
+LABEL="$label"
+REPORT_HOME="$r_base"
+REPORT_MASTER="$host"
+REPORT_TARGET="$node"
+LOG_START=$start
+LOG_END=$end
+REMOVE=1
+SANITIZE="$sanitize_patterns"
+CLUSTER=$cluster
+LOG_PATTERNS="$log_patterns"
+EXTRA_LOGS="$extra_logs"
+SEARCH_LOGS=$search_logs
+SOS_MODE=$sos_mode
+verbose=$verbose
+maxdepth=$maxdepth
+EOF
+
+	if [ $host = $node ]; then
+	    cat <<EOF >>$l_base/.env
+REPORT_HOME="$l_base"
+EOF
+	    cat $l_base/.env $report_data/report.common $report_data/report.collector > $l_base/collector
+	    bash $l_base/collector
+	else
+	    cat $l_base/.env $report_data/report.common $report_data/report.collector \
+		| $rsh -l $ssh_user $node -- "mkdir -p $r_base; cat > $r_base/collector; bash $r_base/collector" | (cd $l_base && tar mxf -)
+	fi
+    done
+
+    analyze $l_base > $l_base/$ANALYSIS_F
+    if [ -f $l_base/$HALOG_F ]; then
+	node_events $l_base/$HALOG_F > $l_base/$EVENTS_F
+    fi
+
+    for node in $nodes; do
+	cat $l_base/$node/$ANALYSIS_F >> $l_base/$ANALYSIS_F
+	if [ -s $l_base/$node/$EVENTS_F ]; then
+	    cat $l_base/$node/$EVENTS_F >> $l_base/$EVENTS_F
+	elif [ -s $l_base/$HALOG_F ]; then
+	    awk "\$4==\"$nodes\"" $l_base/$EVENTS_F >> $l_base/$n/$EVENTS_F
+	fi
+    done
+
+    log " "
+    if [ $compress = 1 ]; then
+	fname=`shrink $l_base`
+	rm -rf $l_base
+	log "Collected results are available in $fname"
+	log " "
+	log "Please create a bug entry at"
+	log "    @BUG_URL@"
+	log "Include a description of your problem and attach this tarball"
+	log " "
+	log "Thank you for taking time to create this report."
+    else
+	log "Collected results are available in $l_base"
+    fi
+    log " "
+}
+
+#
+# check if files have same content in the cluster
+#
+cibdiff() {
+    d1=$(dirname $1)
+    d2=$(dirname $2)
+
+    if [ -f "$d1/RUNNING" ] && [ ! -f "$d2/RUNNING" ]; then
+        DIFF_OK=0
+    elif [ -f "$d1/STOPPED" ] && [ ! -f "$d2/STOPPED" ]; then
+        DIFF_OK=0
+    else
+        DIFF_OK=1
+    fi
+
+    if [ $DIFF_OK -eq 1 ]; then
+	if which crm_diff > /dev/null 2>&1; then
+	    crm_diff -c -n $1 -o $2
+	else
+	    info "crm_diff(8) not found, cannot diff CIBs"
+	fi
+    else
+	echo "can't compare cibs from running and stopped systems"
+    fi
+}
+
+diffcheck() {
+    [ -f "$1" ] || {
+	echo "$1 does not exist"
+	return 1
+    }
+    [ -f "$2" ] || {
+	echo "$2 does not exist"
+	return 1
+    }
+    case $(basename "$1") in
+        $CIB_F)  cibdiff $1 $2 ;;
+        *)       diff -u $1 $2 ;;
+    esac
+}
+
+#
+# remove duplicates if files are same, make links instead
+#
+consolidate() {
+    for n in $nodes; do
+	if [ -f $1/$2 ]; then
+	    rm $1/$n/$2
+	else
+	    mv $1/$n/$2 $1
+	fi
+	ln -s ../$2 $1/$n
+    done
+}
+
+analyze_one() {
+    rc=0
+    node0=""
+    for n in $nodes; do
+	if [ "$node0" ]; then
+	    diffcheck $1/$node0/$2 $1/$n/$2
+	    rc=$(($rc+$?))
+	else
+	    node0=$n
+	fi
+    done
+    return $rc
+}
+
+analyze() {
+    flist="$MEMBERSHIP_F $CIB_F $CRM_MON_F $SYSINFO_F"
+    for f in $flist; do
+	printf "Diff $f... "
+	ls $1/*/$f >/dev/null 2>&1 || {
+	    echo "no $1/*/$f :/"
+	    continue
+	}
+	if analyze_one $1 $f; then
+	    echo "OK"
+	    [ "$f" != $CIB_F ] && consolidate $1 $f
+	else
+	    echo ""
+	fi
+    done
+}
+
+do_cts() {
+    test_sets=`echo $tests | tr ',' ' '`
+    for test_set in $test_sets; do
+
+	start_time=0
+	start_test=`echo $test_set | tr '-' ' ' | awk '{print $1}'`
+
+	end_time=0
+	end_test=`echo $test_set | tr '-' ' ' | awk '{print $2}'`
+
+	if [ x$end_test = x ]; then
+	    msg="Extracting test $start_test"
+	    label="CTS-$start_test-`date +"%b-%d-%Y"`"
+	    end_test=`expr $start_test + 1`
+	else
+	    msg="Extracting tests $start_test to $end_test"
+	    label="CTS-$start_test-$end_test-`date +"%b-%d-%Y"`"
+	    end_test=`expr $end_test + 1`
+	fi
+
+	if [ $start_test = 0 ]; then
+	    start_pat="BEGINNING [0-9].* TESTS"
+	else
+	    start_pat="Running test.*\[ *$start_test\]"
+	fi
+
+	if [ x$ctslog = x ]; then
+	    ctslog=`findmsg 1 "$start_pat"`
+
+	    if [ x$ctslog = x ]; then
+		fatal "No CTS control file detected"
+	    else
+		log "Using CTS control file: $ctslog"
+	    fi
+	fi
+
+	line=`grep -n "$start_pat" $ctslog | tail -1 | sed 's/:.*//'`
+	if [ ! -z "$line" ]; then
+	    start_time=`linetime $ctslog $line`
+	fi
+
+	line=`grep -n "Running test.*\[ *$end_test\]" $ctslog | tail -1 | sed 's/:.*//'`
+	if [ ! -z "$line" ]; then
+	    end_time=`linetime $ctslog $line`
+	fi
+
+	if [ -z "$nodes" ]; then
+	    nodes=`grep CTS: $ctslog | grep -v debug: | grep " \* " | sed s:.*\\\*::g | sort -u  | tr '\\n' ' '`
+	    log "Calculated node list: $nodes"
+	fi
+
+	if [ $end_time -lt $start_time ]; then
+	    debug "Test didn't complete, grabbing everything up to now"
+	    end_time=`date +%s`
+	fi
+
+	if [ $start_time != 0 ];then
+	    log "$msg (`time2str $start_time` to `time2str $end_time`)"
+	    collect_data $label $start_time $end_time $ctslog
+	else
+	    fatal "$msg failed: not found"
+	fi
+    done
+}
+
+node_names_from_xml() {
+    awk '
+      /uname/ {
+            for( i=1; i<=NF; i++ )
+                    if( $i~/^uname=/ ) {
+                            sub("uname=.","",$i);
+                            sub("\".*","",$i);
+                            print $i;
+                            next;
+                    }
+      }
+    ' | tr '\n' ' '
+}
+
+getnodes() {
+    cluster="$1"
+
+    # 1. Live (cluster nodes or Pacemaker Remote nodes)
+    # TODO: This will not detect Pacemaker Remote nodes unless they
+    # have ever had a permanent node attribute set, because it only
+    # searches the nodes section. It should also search the config
+    # for resources that create Pacemaker Remote nodes.
+    cib_nodes=$(cibadmin -Ql -o nodes 2>/dev/null)
+    if [ $? -eq 0 ]; then
+	debug "Querying CIB for nodes"
+        echo "$cib_nodes" | node_names_from_xml
+        return
+    fi
+
+    # 2. Saved
+    if [ -f "@CRM_CONFIG_DIR@/cib.xml" ]; then
+	debug "Querying on-disk CIB for nodes"
+        grep "node " "@CRM_CONFIG_DIR@/cib.xml" | node_names_from_xml
+        return
+    fi
+
+    # 3. logs
+    # TODO: Look for something like crm_update_peer
+}
+
+if [ $compress -eq 1 ]; then
+    require_tar
+fi
+
+if [ "x$tests" != "x" ]; then
+    do_cts
+
+elif [ "x$start_time" != "x" ]; then
+    masterlog=""
+
+    if [ -z "$sanitize_patterns" ]; then
+	log "WARNING: The tarball produced by this program may contain"
+	log "         sensitive information such as passwords."
+	log ""
+	log "We will attempt to remove such information if you use the"
+	log "-p option. For example: -p \"pass.*\" -p \"user.*\""
+	log ""
+	log "However, doing this may reduce the ability for the recipients"
+	log "to diagnose issues and generally provide assistance."
+	log ""
+	log "IT IS YOUR RESPONSIBILITY TO PROTECT SENSITIVE DATA FROM EXPOSURE"
+	log ""
+    fi
+
+    # If user didn't specify a cluster stack, make a best guess if possible.
+    if [ -z "$cluster" ] || [ "$cluster" = "any" ]; then
+        cluster=$(get_cluster_type)
+    fi
+
+    # If user didn't specify node(s), make a best guess if possible.
+    if [ -z "$nodes" ]; then
+	nodes=`getnodes $cluster`
+        if [ -n "$nodes" ]; then
+            log "Calculated node list: $nodes"
+        else
+            fatal "Cannot determine nodes; specify --nodes or --single-node"
+        fi
+    fi
+
+    if
+	echo $nodes | grep -qs $host
+    then
+	debug "We are a cluster node"
+    else
+	debug "We are a log master"
+	masterlog=`findmsg 1 "pacemaker-controld\\|CTS"`
+    fi
+
+
+    if [ -z $end_time ]; then
+	end_time=`perl -e 'print time()'`
+    fi
+    label="pcmk-`date +"%a-%d-%b-%Y"`"
+    log "Collecting data from $nodes (`time2str $start_time` to `time2str $end_time`)"
+    collect_data $label $start_time $end_time $masterlog
+else
+    fatal "Not sure what to do, no tests or time ranges to extract"
+fi
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4 textwidth=80:
diff --git a/tools/crm_resource.8.inc b/tools/crm_resource.8.inc
new file mode 100644
index 0000000..a92fac6
--- /dev/null
+++ b/tools/crm_resource.8.inc
@@ -0,0 +1,5 @@
+[synopsis]
+crm_resource <query>|<command> [options]
+
+/Pacemaker cluster resources/
+.SH OPTIONS
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
new file mode 100644
index 0000000..f351c26
--- /dev/null
+++ b/tools/crm_resource.c
@@ -0,0 +1,2182 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm_resource.h>
+#include <crm/lrmd_internal.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/ipc_attrd_internal.h>
+#include <crm/common/lists_internal.h>
+#include <crm/common/output.h>
+#include <pacemaker-internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <libgen.h>
+#include <time.h>
+
+#include <crm/crm.h>
+#include <crm/stonith-ng.h>
+#include <crm/common/ipc_controld.h>
+#include <crm/cib/internal.h>
+
+#define SUMMARY "crm_resource - perform tasks related to Pacemaker cluster resources"
+
+enum rsc_command {
+    cmd_none = 0,           // No command option given (yet)
+    cmd_ban,
+    cmd_cleanup,
+    cmd_clear,
+    cmd_colocations,
+    cmd_cts,
+    cmd_delete,
+    cmd_delete_param,
+    cmd_digests,
+    cmd_execute_agent,
+    cmd_fail,
+    cmd_get_param,
+    cmd_get_property,
+    cmd_list_active_ops,
+    cmd_list_agents,
+    cmd_list_all_ops,
+    cmd_list_alternatives,
+    cmd_list_instances,
+    cmd_list_providers,
+    cmd_list_resources,
+    cmd_list_standards,
+    cmd_locate,
+    cmd_metadata,
+    cmd_move,
+    cmd_query_raw_xml,
+    cmd_query_xml,
+    cmd_refresh,
+    cmd_restart,
+    cmd_set_param,
+    cmd_set_property,
+    cmd_wait,
+    cmd_why,
+};
+
+struct {
+    enum rsc_command rsc_cmd;     // crm_resource command to perform
+
+    // Infrastructure that given command needs to work
+    gboolean require_cib;         // Whether command requires CIB IPC
+    int cib_options;              // Options to use with CIB IPC calls
+    gboolean require_crmd;        // Whether command requires controller IPC
+    gboolean require_dataset;     // Whether command requires populated data set
+    gboolean require_resource;    // Whether command requires resource specified
+    gboolean require_node;        // Whether command requires node specified
+    int find_flags;               // Flags to use when searching for resource
+
+    // Command-line option values
+    gchar *rsc_id;                // Value of --resource
+    gchar *rsc_type;              // Value of --resource-type
+    gboolean force;               // --force was given
+    gboolean clear_expired;       // --expired was given
+    gboolean recursive;           // --recursive was given
+    gboolean promoted_role_only;  // --promoted was given
+    gchar *host_uname;            // Value of --node
+    gchar *interval_spec;         // Value of --interval
+    gchar *move_lifetime;         // Value of --lifetime
+    gchar *operation;             // Value of --operation
+    const char *attr_set_type;    // Instance, meta, utilization, or element attribute
+    gchar *prop_id;               // --nvpair (attribute XML ID)
+    char *prop_name;              // Attribute name
+    gchar *prop_set;              // --set-name (attribute block XML ID)
+    gchar *prop_value;            // --parameter-value (attribute value)
+    int timeout_ms;               // Parsed from --timeout value
+    char *agent_spec;             // Standard and/or provider and/or agent
+    gchar *xml_file;              // Value of (deprecated) --xml-file
+    int check_level;              // Optional value of --validate or --force-check
+
+    // Resource configuration specified via command-line arguments
+    gboolean cmdline_config;      // Resource configuration was via arguments
+    char *v_agent;                // Value of --agent
+    char *v_class;                // Value of --class
+    char *v_provider;             // Value of --provider
+    GHashTable *cmdline_params;   // Resource parameters specified
+
+    // Positional command-line arguments
+    gchar **remainder;            // Positional arguments as given
+    GHashTable *override_params;  // Resource parameter values that override config
+} options = {
+    .attr_set_type = XML_TAG_ATTR_SETS,
+    .check_level = -1,
+    .cib_options = cib_sync_call,
+    .require_cib = TRUE,
+    .require_dataset = TRUE,
+    .require_resource = TRUE,
+};
+
+#if 0
+// @COMPAT @TODO enable this at next backward compatibility break
+#define SET_COMMAND(cmd) do {                                               \
+        if (options.rsc_cmd != cmd_none) {                                  \
+            g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,             \
+                        "Only one command option may be specified");        \
+            return FALSE;                                                   \
+        }                                                                   \
+        options.rsc_cmd = (cmd);                                            \
+    } while (0)
+#else
+#define SET_COMMAND(cmd) do {                                               \
+        if (options.rsc_cmd != cmd_none) {                                  \
+            reset_options();                                                \
+        }                                                                   \
+        options.rsc_cmd = (cmd);                                            \
+    } while (0)
+#endif
+
+gboolean agent_provider_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean attr_set_type_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean class_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean cleanup_refresh_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean delete_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean expired_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean list_agents_cb(const gchar *option_name, const gchar *optarg,
+                        gpointer data, GError **error);
+gboolean list_providers_cb(const gchar *option_name, const gchar *optarg,
+                           gpointer data, GError **error);
+gboolean list_standards_cb(const gchar *option_name, const gchar *optarg,
+                           gpointer data, GError **error);
+gboolean list_alternatives_cb(const gchar *option_name, const gchar *optarg,
+                              gpointer data, GError **error);
+gboolean metadata_cb(const gchar *option_name, const gchar *optarg,
+                     gpointer data, GError **error);
+gboolean option_cb(const gchar *option_name, const gchar *optarg,
+                   gpointer data, GError **error);
+gboolean fail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean flag_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean get_param_prop_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean list_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean set_delete_param_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean set_prop_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean timeout_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean validate_or_force_cb(const gchar *option_name, const gchar *optarg,
+                              gpointer data, GError **error);
+gboolean restart_cb(const gchar *option_name, const gchar *optarg,
+                    gpointer data, GError **error);
+gboolean digests_cb(const gchar *option_name, const gchar *optarg,
+                    gpointer data, GError **error);
+gboolean wait_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean why_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+
+static crm_exit_t exit_code = CRM_EX_OK;
+static pcmk__output_t *out = NULL;
+static pcmk__common_args_t *args = NULL;
+
+// Things that should be cleaned up on exit
+static GError *error = NULL;
+static GMainLoop *mainloop = NULL;
+static cib_t *cib_conn = NULL;
+static pcmk_ipc_api_t *controld_api = NULL;
+static pe_working_set_t *data_set = NULL;
+
+#define MESSAGE_TIMEOUT_S 60
+
+#define INDENT "                                    "
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+// Clean up and exit
+static crm_exit_t
+bye(crm_exit_t ec)
+{
+    pcmk__output_and_clear_error(&error, out);
+
+    if (out != NULL) {
+        out->finish(out, ec, true, NULL);
+        pcmk__output_free(out);
+    }
+    pcmk__unregister_formats();
+
+    if (cib_conn != NULL) {
+        cib_t *save_cib_conn = cib_conn;
+
+        cib_conn = NULL; // Ensure we can't free this twice
+        cib__clean_up_connection(&save_cib_conn);
+    }
+
+    if (controld_api != NULL) {
+        pcmk_ipc_api_t *save_controld_api = controld_api;
+
+        controld_api = NULL; // Ensure we can't free this twice
+        pcmk_free_ipc_api(save_controld_api);
+    }
+
+    if (mainloop != NULL) {
+        g_main_loop_unref(mainloop);
+        mainloop = NULL;
+    }
+
+    pe_free_working_set(data_set);
+    data_set = NULL;
+    crm_exit(ec);
+    return ec;
+}
+
+static void
+quit_main_loop(crm_exit_t ec)
+{
+    exit_code = ec;
+    if (mainloop != NULL) {
+        GMainLoop *mloop = mainloop;
+
+        mainloop = NULL; // Don't re-enter this block
+        pcmk_quit_main_loop(mloop, 10);
+        g_main_loop_unref(mloop);
+    }
+}
+
+static gboolean
+resource_ipc_timeout(gpointer data)
+{
+    // Start with newline because "Waiting for ..." message doesn't have one
+    if (error != NULL) {
+        g_clear_error(&error);
+    }
+
+    g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_TIMEOUT,
+                _("Aborting because no messages received in %d seconds"), MESSAGE_TIMEOUT_S);
+
+    quit_main_loop(CRM_EX_TIMEOUT);
+    return FALSE;
+}
+
+static void
+controller_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
+                          crm_exit_t status, void *event_data, void *user_data)
+{
+    switch (event_type) {
+        case pcmk_ipc_event_disconnect:
+            if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
+                crm_info("Connection to controller was terminated");
+            }
+            quit_main_loop(exit_code);
+            break;
+
+        case pcmk_ipc_event_reply:
+            if (status != CRM_EX_OK) {
+                out->err(out, "Error: bad reply from controller: %s",
+                         crm_exit_str(status));
+                pcmk_disconnect_ipc(api);
+                quit_main_loop(status);
+            } else {
+                if ((pcmk_controld_api_replies_expected(api) == 0)
+                    && mainloop && g_main_loop_is_running(mainloop)) {
+                    out->info(out, "... got reply (done)");
+                    crm_debug("Got all the replies we expected");
+                    pcmk_disconnect_ipc(api);
+                    quit_main_loop(CRM_EX_OK);
+                } else {
+                    out->info(out, "... got reply");
+                }
+            }
+            break;
+
+        default:
+            break;
+    }
+}
+
+static void
+start_mainloop(pcmk_ipc_api_t *capi)
+{
+    unsigned int count = pcmk_controld_api_replies_expected(capi);
+
+    if (count > 0) {
+        out->info(out, "Waiting for %u %s from the controller",
+                  count, pcmk__plural_alt(count, "reply", "replies"));
+        exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects
+        mainloop = g_main_loop_new(NULL, FALSE);
+        g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL);
+        g_main_loop_run(mainloop);
+    }
+}
+
+static int
+compare_id(gconstpointer a, gconstpointer b)
+{
+    return strcmp((const char *)a, (const char *)b);
+}
+
+static GList *
+build_constraint_list(xmlNode *root)
+{
+    GList *retval = NULL;
+    xmlNode *cib_constraints = NULL;
+    xmlXPathObjectPtr xpathObj = NULL;
+    int ndx = 0;
+
+    cib_constraints = pcmk_find_cib_element(root, XML_CIB_TAG_CONSTRAINTS);
+    xpathObj = xpath_search(cib_constraints, "//" XML_CONS_TAG_RSC_LOCATION);
+
+    for (ndx = 0; ndx < numXpathResults(xpathObj); ndx++) {
+        xmlNode *match = getXpathResult(xpathObj, ndx);
+        retval = g_list_insert_sorted(retval, (gpointer) ID(match), compare_id);
+    }
+
+    freeXpathObject(xpathObj);
+    return retval;
+}
+
+/* short option letters still available: eEJkKXyYZ */
+
+static GOptionEntry query_entries[] = {
+    { "list", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
+      "List all cluster resources with status",
+      NULL },
+    { "list-raw", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
+      "List IDs of all instantiated resources (individual members\n"
+      INDENT "rather than groups etc.)",
+      NULL },
+    { "list-cts", 'c', G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
+      NULL,
+      NULL },
+    { "list-operations", 'O', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
+      "List active resource operations, optionally filtered by\n"
+      INDENT "--resource and/or --node",
+      NULL },
+    { "list-all-operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
+      "List all resource operations, optionally filtered by\n"
+      INDENT "--resource and/or --node",
+      NULL },
+    { "list-standards", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
+      list_standards_cb,
+      "List supported standards",
+      NULL },
+    { "list-ocf-providers", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
+      list_providers_cb,
+      "List all available OCF providers",
+      NULL },
+    { "list-agents", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
+      list_agents_cb,
+      "List all agents available for the named standard and/or provider",
+      "STD:PROV" },
+    { "list-ocf-alternatives", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
+      list_alternatives_cb,
+      "List all available providers for the named OCF agent",
+      "AGENT" },
+    { "show-metadata", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
+      metadata_cb,
+      "Show the metadata for the named class:provider:agent",
+      "SPEC" },
+    { "query-xml", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
+      "Show XML configuration of resource (after any template expansion)",
+      NULL },
+    { "query-xml-raw", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
+      "Show XML configuration of resource (before any template expansion)",
+      NULL },
+    { "get-parameter", 'g', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, get_param_prop_cb,
+      "Display named parameter for resource (use instance attribute\n"
+      INDENT "unless --element, --meta, or --utilization is specified)",
+      "PARAM" },
+    { "get-property", 'G', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, get_param_prop_cb,
+      "Display named property of resource ('class', 'type', or 'provider') "
+      "(requires --resource)",
+      "PROPERTY" },
+    { "locate", 'W', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
+      "Show node(s) currently running resource",
+      NULL },
+    { "constraints", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
+      "Display the location and colocation constraints that apply to a\n"
+      INDENT "resource, and if --recursive is specified, to the resources\n"
+      INDENT "directly or indirectly involved in those colocations.\n"
+      INDENT "If the named resource is part of a group, or a clone or\n"
+      INDENT "bundle instance, constraints for the collective resource\n"
+      INDENT "will be shown unless --force is given.",
+      NULL },
+    { "stack", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
+      "Equivalent to --constraints --recursive",
+      NULL },
+    { "why", 'Y', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, why_cb,
+      "Show why resources are not running, optionally filtered by\n"
+      INDENT "--resource and/or --node",
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry command_entries[] = {
+    { "validate", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
+      validate_or_force_cb,
+      "Validate resource configuration by calling agent's validate-all\n"
+      INDENT "action. The configuration may be specified either by giving an\n"
+      INDENT "existing resource name with -r, or by specifying --class,\n"
+      INDENT "--agent, and --provider arguments, along with any number of\n"
+      INDENT "--option arguments. An optional LEVEL argument can be given\n"
+      INDENT "to control the level of checking performed.",
+      "LEVEL" },
+    { "cleanup", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
+      "If resource has any past failures, clear its history and fail\n"
+      INDENT "count. Optionally filtered by --resource, --node, --operation\n"
+      INDENT "and --interval (otherwise all). --operation and --interval\n"
+      INDENT "apply to fail counts, but entire history is always clear, to\n"
+      INDENT "allow current state to be rechecked. If the named resource is\n"
+      INDENT "part of a group, or one numbered instance of a clone or bundled\n"
+      INDENT "resource, the clean-up applies to the whole collective resource\n"
+      INDENT "unless --force is given.",
+      NULL },
+    { "refresh", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
+      "Delete resource's history (including failures) so its current state\n"
+      INDENT "is rechecked. Optionally filtered by --resource and --node\n"
+      INDENT "(otherwise all). If the named resource is part of a group, or one\n"
+      INDENT "numbered instance of a clone or bundled resource, the refresh\n"
+      INDENT "applies to the whole collective resource unless --force is given.",
+      NULL },
+    { "set-parameter", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, set_delete_param_cb,
+      "Set named parameter for resource (requires -v). Use instance\n"
+      INDENT "attribute unless --element, --meta, or --utilization is "
+      "specified.",
+      "PARAM" },
+    { "delete-parameter", 'd', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, set_delete_param_cb,
+      "Delete named parameter for resource. Use instance attribute\n"
+      INDENT "unless --element, --meta or, --utilization is specified.",
+      "PARAM" },
+    { "set-property", 'S', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, set_prop_cb,
+      "Set named property of resource ('class', 'type', or 'provider') "
+      "(requires -r, -t, -v)",
+      "PROPERTY" },
+
+    { NULL }
+};
+
+static GOptionEntry location_entries[] = {
+    { "move", 'M', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
+      "Create a constraint to move resource. If --node is specified,\n"
+      INDENT "the constraint will be to move to that node, otherwise it\n"
+      INDENT "will be to ban the current node. Unless --force is specified\n"
+      INDENT "this will return an error if the resource is already running\n"
+      INDENT "on the specified node. If --force is specified, this will\n"
+      INDENT "always ban the current node.\n"
+      INDENT "Optional: --lifetime, --promoted. NOTE: This may prevent the\n"
+      INDENT "resource from running on its previous location until the\n"
+      INDENT "implicit constraint expires or is removed with --clear.",
+      NULL },
+    { "ban", 'B', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
+      "Create a constraint to keep resource off a node.\n"
+      INDENT "Optional: --node, --lifetime, --promoted.\n"
+      INDENT "NOTE: This will prevent the resource from running on the\n"
+      INDENT "affected node until the implicit constraint expires or is\n"
+      INDENT "removed with --clear. If --node is not specified, it defaults\n"
+      INDENT "to the node currently running the resource for primitives\n"
+      INDENT "and groups, or the promoted instance of promotable clones with\n"
+      INDENT "promoted-max=1 (all other situations result in an error as\n"
+      INDENT "there is no sane default).",
+      NULL },
+    { "clear", 'U', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
+      "Remove all constraints created by the --ban and/or --move\n"
+      INDENT "commands. Requires: --resource. Optional: --node, --promoted,\n"
+      INDENT "--expired. If --node is not specified, all constraints created\n"
+      INDENT "by --ban and --move will be removed for the named resource. If\n"
+      INDENT "--node and --force are specified, any constraint created by\n"
+      INDENT "--move will be cleared, even if it is not for the specified\n"
+      INDENT "node. If --expired is specified, only those constraints whose\n"
+      INDENT "lifetimes have expired will be removed.",
+      NULL },
+    { "expired", 'e', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, expired_cb,
+      "Modifies the --clear argument to remove constraints with\n"
+      INDENT "expired lifetimes.",
+      NULL },
+    { "lifetime", 'u', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.move_lifetime,
+      "Lifespan (as ISO 8601 duration) of created constraints (with\n"
+      INDENT "-B, -M) see https://en.wikipedia.org/wiki/ISO_8601#Durations)",
+      "TIMESPEC" },
+    { "promoted", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
+      &options.promoted_role_only,
+      "Limit scope of command to promoted role (with -B, -M, -U). For\n"
+      INDENT "-B and -M, previously promoted instances may remain\n"
+      INDENT "active in the unpromoted role.",
+      NULL },
+
+    // Deprecated since 2.1.0
+    { "master", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
+      &options.promoted_role_only,
+      "Deprecated: Use --promoted instead", NULL },
+
+    { NULL }
+};
+
+static GOptionEntry advanced_entries[] = {
+    { "delete", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, delete_cb,
+      "(Advanced) Delete a resource from the CIB. Required: -t",
+      NULL },
+    { "fail", 'F', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, fail_cb,
+      "(Advanced) Tell the cluster this resource has failed",
+      NULL },
+    { "restart", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, restart_cb,
+      "(Advanced) Tell the cluster to restart this resource and\n"
+      INDENT "anything that depends on it",
+      NULL },
+    { "wait", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, wait_cb,
+      "(Advanced) Wait until the cluster settles into a stable state",
+      NULL },
+    { "digests", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, digests_cb,
+      "(Advanced) Show parameter hashes that Pacemaker uses to detect\n"
+      INDENT "configuration changes (only accurate if there is resource\n"
+      INDENT "history on the specified node). Required: --resource, --node.\n"
+      INDENT "Optional: any NAME=VALUE parameters will be used to override\n"
+      INDENT "the configuration (to see what the hash would be with those\n"
+      INDENT "changes).",
+      NULL },
+    { "force-demote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
+      validate_or_force_cb,
+      "(Advanced) Bypass the cluster and demote a resource on the local\n"
+      INDENT "node. Unless --force is specified, this will refuse to do so if\n"
+      INDENT "the cluster believes the resource is a clone instance already\n"
+      INDENT "running on the local node.",
+      NULL },
+    { "force-stop", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
+      validate_or_force_cb,
+      "(Advanced) Bypass the cluster and stop a resource on the local node",
+      NULL },
+    { "force-start", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
+      validate_or_force_cb,
+      "(Advanced) Bypass the cluster and start a resource on the local\n"
+      INDENT "node. Unless --force is specified, this will refuse to do so if\n"
+      INDENT "the cluster believes the resource is a clone instance already\n"
+      INDENT "running on the local node.",
+      NULL },
+    { "force-promote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
+      validate_or_force_cb,
+      "(Advanced) Bypass the cluster and promote a resource on the local\n"
+      INDENT "node. Unless --force is specified, this will refuse to do so if\n"
+      INDENT "the cluster believes the resource is a clone instance already\n"
+      INDENT "running on the local node.",
+      NULL },
+    { "force-check", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
+      validate_or_force_cb,
+      "(Advanced) Bypass the cluster and check the state of a resource on\n"
+      INDENT "the local node. An optional LEVEL argument can be given\n"
+      INDENT "to control the level of checking performed.",
+      "LEVEL" },
+
+    { NULL }
+};
+
+static GOptionEntry addl_entries[] = {
+    { "node", 'N', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.host_uname,
+      "Node name",
+      "NAME" },
+    { "recursive", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.recursive,
+      "Follow colocation chains when using --set-parameter or --constraints",
+      NULL },
+    { "resource-type", 't', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_type,
+      "Resource XML element (primitive, group, etc.) (with -D)",
+      "ELEMENT" },
+    { "parameter-value", 'v', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_value,
+      "Value to use with -p",
+      "PARAM" },
+    { "meta", 'm', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
+      "Use resource meta-attribute instead of instance attribute\n"
+      INDENT "(with -p, -g, -d)",
+      NULL },
+    { "utilization", 'z', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
+      "Use resource utilization attribute instead of instance attribute\n"
+      INDENT "(with -p, -g, -d)",
+      NULL },
+    { "element", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
+      "Use resource element attribute instead of instance attribute\n"
+      INDENT "(with -p, -g, -d)",
+      NULL },
+    { "operation", 'n', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.operation,
+      "Operation to clear instead of all (with -C -r)",
+      "OPERATION" },
+    { "interval", 'I', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.interval_spec,
+      "Interval of operation to clear (default 0) (with -C -r -n)",
+      "N" },
+    { "class", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, class_cb,
+      "The standard the resource agent conforms to (for example, ocf).\n"
+      INDENT "Use with --agent, --provider, --option, and --validate.",
+      "CLASS" },
+    { "agent", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, agent_provider_cb,
+      "The agent to use (for example, IPaddr). Use with --class,\n"
+      INDENT "--provider, --option, and --validate.",
+      "AGENT" },
+    { "provider", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, agent_provider_cb,
+      "The vendor that supplies the resource agent (for example,\n"
+      INDENT "heartbeat). Use with --class, --agent, --option, and --validate.",
+      "PROVIDER" },
+    { "option", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, option_cb,
+      "Specify a device configuration parameter as NAME=VALUE (may be\n"
+      INDENT "specified multiple times). Use with --validate and without the\n"
+      INDENT "-r option.",
+      "PARAM" },
+    { "set-name", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_set,
+      "(Advanced) XML ID of attributes element to use (with -p, -d)",
+      "ID" },
+    { "nvpair", 'i', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_id,
+      "(Advanced) XML ID of nvpair element to use (with -p, -d)",
+      "ID" },
+    { "timeout", 'T', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, timeout_cb,
+      "(Advanced) Abort if command does not finish in this time (with\n"
+      INDENT "--restart, --wait, --force-*)",
+      "N" },
+    { "force", 'f', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.force,
+      "Force the action to be performed. See help for individual commands for\n"
+      INDENT "additional behavior.",
+      NULL },
+    { "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_FILENAME, &options.xml_file,
+      NULL,
+      "FILE" },
+    { "host-uname", 'H', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.host_uname,
+      NULL,
+      "HOST" },
+
+    { NULL }
+};
+
+static void
+reset_options(void) {
+    options.require_crmd = FALSE;
+    options.require_node = FALSE;
+
+    options.require_cib = TRUE;
+    options.require_dataset = TRUE;
+    options.require_resource = TRUE;
+
+    options.find_flags = 0;
+}
+
+gboolean
+agent_provider_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.cmdline_config = TRUE;
+    options.require_resource = FALSE;
+
+    if (pcmk__str_eq(option_name, "--provider", pcmk__str_casei)) {
+        pcmk__str_update(&options.v_provider, optarg);
+    } else {
+        pcmk__str_update(&options.v_agent, optarg);
+    }
+
+    return TRUE;
+}
+
+gboolean
+attr_set_type_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_any_of(option_name, "-m", "--meta", NULL)) {
+        options.attr_set_type = XML_TAG_META_SETS;
+    } else if (pcmk__str_any_of(option_name, "-z", "--utilization", NULL)) {
+        options.attr_set_type = XML_TAG_UTILIZATION;
+    } else if (pcmk__str_eq(option_name, "--element", pcmk__str_casei)) {
+        options.attr_set_type = ATTR_SET_ELEMENT;
+    }
+    return TRUE;
+}
+
+gboolean
+class_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    pcmk__str_update(&options.v_class, optarg);
+    options.cmdline_config = TRUE;
+    options.require_resource = FALSE;
+    return TRUE;
+}
+
+gboolean
+cleanup_refresh_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_any_of(option_name, "-C", "--cleanup", NULL)) {
+        SET_COMMAND(cmd_cleanup);
+    } else {
+        SET_COMMAND(cmd_refresh);
+    }
+
+    options.require_resource = FALSE;
+    if (getenv("CIB_file") == NULL) {
+        options.require_crmd = TRUE;
+    }
+    options.find_flags = pe_find_renamed|pe_find_anon;
+    return TRUE;
+}
+
+gboolean
+delete_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    SET_COMMAND(cmd_delete);
+    options.require_dataset = FALSE;
+    options.find_flags = pe_find_renamed|pe_find_any;
+    return TRUE;
+}
+
+gboolean
+expired_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.clear_expired = TRUE;
+    options.require_resource = FALSE;
+    return TRUE;
+}
+
+static void
+get_agent_spec(const gchar *optarg)
+{
+    options.require_cib = FALSE;
+    options.require_dataset = FALSE;
+    options.require_resource = FALSE;
+    pcmk__str_update(&options.agent_spec, optarg);
+}
+
+gboolean
+list_agents_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+               GError **error)
+{
+    SET_COMMAND(cmd_list_agents);
+    get_agent_spec(optarg);
+    return TRUE;
+}
+
+gboolean
+list_providers_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+                  GError **error)
+{
+    SET_COMMAND(cmd_list_providers);
+    get_agent_spec(optarg);
+    return TRUE;
+}
+
+gboolean
+list_standards_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+                  GError **error)
+{
+    SET_COMMAND(cmd_list_standards);
+    options.require_cib = FALSE;
+    options.require_dataset = FALSE;
+    options.require_resource = FALSE;
+    return TRUE;
+}
+
+gboolean
+list_alternatives_cb(const gchar *option_name, const gchar *optarg,
+                     gpointer data, GError **error)
+{
+    SET_COMMAND(cmd_list_alternatives);
+    get_agent_spec(optarg);
+    return TRUE;
+}
+
+gboolean
+metadata_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+            GError **error)
+{
+    SET_COMMAND(cmd_metadata);
+    get_agent_spec(optarg);
+    return TRUE;
+}
+
+gboolean
+option_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+          GError **error)
+{
+    char *name = NULL;
+    char *value = NULL;
+
+    if (pcmk__scan_nvpair(optarg, &name, &value) != 2) {
+        return FALSE;
+    }
+    if (options.cmdline_params == NULL) {
+        options.cmdline_params = pcmk__strkey_table(free, free);
+    }
+    g_hash_table_replace(options.cmdline_params, name, value);
+    return TRUE;
+}
+
+gboolean
+fail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    SET_COMMAND(cmd_fail);
+    options.require_crmd = TRUE;
+    options.require_node = TRUE;
+    return TRUE;
+}
+
+gboolean
+flag_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_any_of(option_name, "-U", "--clear", NULL)) {
+        SET_COMMAND(cmd_clear);
+        options.find_flags = pe_find_renamed|pe_find_anon;
+    } else if (pcmk__str_any_of(option_name, "-B", "--ban", NULL)) {
+        SET_COMMAND(cmd_ban);
+        options.find_flags = pe_find_renamed|pe_find_anon;
+    } else if (pcmk__str_any_of(option_name, "-M", "--move", NULL)) {
+        SET_COMMAND(cmd_move);
+        options.find_flags = pe_find_renamed|pe_find_anon;
+    } else if (pcmk__str_any_of(option_name, "-q", "--query-xml", NULL)) {
+        SET_COMMAND(cmd_query_xml);
+        options.find_flags = pe_find_renamed|pe_find_any;
+    } else if (pcmk__str_any_of(option_name, "-w", "--query-xml-raw", NULL)) {
+        SET_COMMAND(cmd_query_raw_xml);
+        options.find_flags = pe_find_renamed|pe_find_any;
+    } else if (pcmk__str_any_of(option_name, "-W", "--locate", NULL)) {
+        SET_COMMAND(cmd_locate);
+        options.find_flags = pe_find_renamed|pe_find_anon;
+
+    } else if (pcmk__str_any_of(option_name, "-a", "--constraints", NULL)) {
+        SET_COMMAND(cmd_colocations);
+        options.find_flags = pe_find_renamed|pe_find_anon;
+
+    } else if (pcmk__str_any_of(option_name, "-A", "--stack", NULL)) {
+        SET_COMMAND(cmd_colocations);
+        options.find_flags = pe_find_renamed|pe_find_anon;
+        options.recursive = TRUE;
+    }
+
+    return TRUE;
+}
+
+gboolean
+get_param_prop_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_any_of(option_name, "-g", "--get-parameter", NULL)) {
+        SET_COMMAND(cmd_get_param);
+    } else {
+        SET_COMMAND(cmd_get_property);
+    }
+
+    pcmk__str_update(&options.prop_name, optarg);
+    options.find_flags = pe_find_renamed|pe_find_any;
+    return TRUE;
+}
+
+gboolean
+list_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_any_of(option_name, "-c", "--list-cts", NULL)) {
+        SET_COMMAND(cmd_cts);
+    } else if (pcmk__str_any_of(option_name, "-L", "--list", NULL)) {
+        SET_COMMAND(cmd_list_resources);
+    } else if (pcmk__str_any_of(option_name, "-l", "--list-raw", NULL)) {
+        SET_COMMAND(cmd_list_instances);
+    } else if (pcmk__str_any_of(option_name, "-O", "--list-operations", NULL)) {
+        SET_COMMAND(cmd_list_active_ops);
+    } else {
+        SET_COMMAND(cmd_list_all_ops);
+    }
+
+    options.require_resource = FALSE;
+    return TRUE;
+}
+
+gboolean
+set_delete_param_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_any_of(option_name, "-p", "--set-parameter", NULL)) {
+        SET_COMMAND(cmd_set_param);
+    } else {
+        SET_COMMAND(cmd_delete_param);
+    }
+
+    pcmk__str_update(&options.prop_name, optarg);
+    options.find_flags = pe_find_renamed|pe_find_any;
+    return TRUE;
+}
+
+gboolean
+set_prop_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    SET_COMMAND(cmd_set_property);
+    options.require_dataset = FALSE;
+    pcmk__str_update(&options.prop_name, optarg);
+    options.find_flags = pe_find_renamed|pe_find_any;
+    return TRUE;
+}
+
+gboolean
+timeout_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.timeout_ms = crm_get_msec(optarg);
+    return TRUE;
+}
+
+gboolean
+validate_or_force_cb(const gchar *option_name, const gchar *optarg,
+                     gpointer data, GError **error)
+{
+    SET_COMMAND(cmd_execute_agent);
+    if (options.operation) {
+        g_free(options.operation);
+    }
+    options.operation = g_strdup(option_name + 2); // skip "--"
+    options.find_flags = pe_find_renamed|pe_find_anon;
+    if (options.override_params == NULL) {
+        options.override_params = pcmk__strkey_table(free, free);
+    }
+
+    if (optarg != NULL) {
+        if (pcmk__scan_min_int(optarg, &options.check_level, 0) != pcmk_rc_ok) {
+            g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM,
+                        _("Invalid check level setting: %s"), optarg);
+            return FALSE;
+        }
+    }
+
+    return TRUE;
+}
+
+gboolean
+restart_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+           GError **error)
+{
+    SET_COMMAND(cmd_restart);
+    options.find_flags = pe_find_renamed|pe_find_anon;
+    return TRUE;
+}
+
+gboolean
+digests_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+           GError **error)
+{
+    SET_COMMAND(cmd_digests);
+    options.find_flags = pe_find_renamed|pe_find_anon;
+    if (options.override_params == NULL) {
+        options.override_params = pcmk__strkey_table(free, free);
+    }
+    options.require_node = TRUE;
+    options.require_dataset = TRUE;
+    return TRUE;
+}
+
+gboolean
+wait_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    SET_COMMAND(cmd_wait);
+    options.require_resource = FALSE;
+    options.require_dataset = FALSE;
+    return TRUE;
+}
+
+gboolean
+why_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    SET_COMMAND(cmd_why);
+    options.require_resource = FALSE;
+    options.find_flags = pe_find_renamed|pe_find_anon;
+    return TRUE;
+}
+
+static int
+ban_or_move(pcmk__output_t *out, pe_resource_t *rsc, const char *move_lifetime)
+{
+    int rc = pcmk_rc_ok;
+    pe_node_t *current = NULL;
+    unsigned int nactive = 0;
+
+    CRM_CHECK(rsc != NULL, return EINVAL);
+
+    current = pe__find_active_requires(rsc, &nactive);
+
+    if (nactive == 1) {
+        rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime, NULL,
+                              cib_conn, options.cib_options, options.promoted_role_only);
+
+    } else if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
+        int count = 0;
+        GList *iter = NULL;
+
+        current = NULL;
+        for(iter = rsc->children; iter; iter = iter->next) {
+            pe_resource_t *child = (pe_resource_t *)iter->data;
+            enum rsc_role_e child_role = child->fns->state(child, TRUE);
+
+            if (child_role == RSC_ROLE_PROMOTED) {
+                count++;
+                current = pe__current_node(child);
+            }
+        }
+
+        if(count == 1 && current) {
+            rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime, NULL,
+                                  cib_conn, options.cib_options, options.promoted_role_only);
+
+        } else {
+            rc = EINVAL;
+            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                        _("Resource '%s' not moved: active in %d locations (promoted in %d).\n"
+                        "To prevent '%s' from running on a specific location, "
+                        "specify a node."
+                        "To prevent '%s' from being promoted at a specific "
+                        "location, specify a node and the --promoted option."),
+                        options.rsc_id, nactive, count, options.rsc_id, options.rsc_id);
+        }
+
+    } else {
+        rc = EINVAL;
+        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                    _("Resource '%s' not moved: active in %d locations.\n"
+                    "To prevent '%s' from running on a specific location, "
+                    "specify a node."),
+                    options.rsc_id, nactive, options.rsc_id);
+    }
+
+    return rc;
+}
+
+static void
+cleanup(pcmk__output_t *out, pe_resource_t *rsc, pe_node_t *node)
+{
+    int rc = pcmk_rc_ok;
+
+    if (options.force == FALSE) {
+        rsc = uber_parent(rsc);
+    }
+
+    crm_debug("Erasing failures of %s (%s requested) on %s",
+              rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
+    rc = cli_resource_delete(controld_api, options.host_uname, rsc, options.operation,
+                             options.interval_spec, TRUE, data_set, options.force);
+
+    if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
+        // Show any reasons why resource might stay stopped
+        cli_resource_check(out, rsc, node);
+    }
+
+    if (rc == pcmk_rc_ok) {
+        start_mainloop(controld_api);
+    }
+}
+
+static int
+clear_constraints(pcmk__output_t *out, xmlNodePtr *cib_xml_copy)
+{
+    GList *before = NULL;
+    GList *after = NULL;
+    GList *remaining = NULL;
+    GList *ele = NULL;
+    pe_node_t *dest = NULL;
+    int rc = pcmk_rc_ok;
+
+    if (!out->is_quiet(out)) {
+        before = build_constraint_list(data_set->input);
+    }
+
+    if (options.clear_expired) {
+        rc = cli_resource_clear_all_expired(data_set->input, cib_conn, options.cib_options,
+                                            options.rsc_id, options.host_uname,
+                                            options.promoted_role_only);
+
+    } else if (options.host_uname) {
+        dest = pe_find_node(data_set->nodes, options.host_uname);
+        if (dest == NULL) {
+            rc = pcmk_rc_node_unknown;
+            if (!out->is_quiet(out)) {
+                g_list_free(before);
+            }
+            return rc;
+        }
+        rc = cli_resource_clear(options.rsc_id, dest->details->uname, NULL,
+                                cib_conn, options.cib_options, TRUE, options.force);
+
+    } else {
+        rc = cli_resource_clear(options.rsc_id, NULL, data_set->nodes,
+                                cib_conn, options.cib_options, TRUE, options.force);
+    }
+
+    if (!out->is_quiet(out)) {
+        rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local | cib_sync_call);
+        rc = pcmk_legacy2rc(rc);
+
+        if (rc != pcmk_rc_ok) {
+            g_set_error(&error, PCMK__RC_ERROR, rc,
+                        _("Could not get modified CIB: %s\n"), pcmk_strerror(rc));
+            g_list_free(before);
+            free_xml(*cib_xml_copy);
+            *cib_xml_copy = NULL;
+            return rc;
+        }
+
+        data_set->input = *cib_xml_copy;
+        cluster_status(data_set);
+
+        after = build_constraint_list(data_set->input);
+        remaining = pcmk__subtract_lists(before, after, (GCompareFunc) strcmp);
+
+        for (ele = remaining; ele != NULL; ele = ele->next) {
+            out->info(out, "Removing constraint: %s", (char *) ele->data);
+        }
+
+        g_list_free(before);
+        g_list_free(after);
+        g_list_free(remaining);
+    }
+
+    return rc;
+}
+
+static int
+delete(void)
+{
+    int rc = pcmk_rc_ok;
+    xmlNode *msg_data = NULL;
+
+    if (options.rsc_type == NULL) {
+        rc = ENXIO;
+        g_set_error(&error, PCMK__RC_ERROR, rc,
+                    _("You need to specify a resource type with -t"));
+        return rc;
+    }
+
+    msg_data = create_xml_node(NULL, options.rsc_type);
+    crm_xml_add(msg_data, XML_ATTR_ID, options.rsc_id);
+
+    rc = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_RESOURCES, msg_data,
+                                options.cib_options);
+    rc = pcmk_legacy2rc(rc);
+    free_xml(msg_data);
+    return rc;
+}
+
+static int
+list_agents(pcmk__output_t *out, const char *agent_spec)
+{
+    int rc = pcmk_rc_ok;
+    char *provider = strchr(agent_spec, ':');
+    lrmd_t *lrmd_conn = NULL;
+    lrmd_list_t *list = NULL;
+
+    rc = lrmd__new(&lrmd_conn, NULL, NULL, 0);
+    if (rc != pcmk_rc_ok) {
+        goto error;
+    }
+
+    if (provider) {
+        *provider++ = 0;
+    }
+
+    rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, agent_spec, provider);
+
+    if (rc > 0) {
+        rc = out->message(out, "agents-list", list, agent_spec, provider);
+    } else {
+        rc = pcmk_rc_error;
+    }
+
+error:
+    if (rc != pcmk_rc_ok) {
+        if (provider == NULL) {
+            g_set_error(&error, PCMK__RC_ERROR, rc,
+                        _("No agents found for standard '%s'"), agent_spec);
+        } else {
+            g_set_error(&error, PCMK__RC_ERROR, rc,
+                        _("No agents found for standard '%s' and provider '%s'"),
+                        agent_spec, provider);
+        }
+    }
+
+    lrmd_api_delete(lrmd_conn);
+    return rc;
+}
+
+static int
+list_providers(pcmk__output_t *out, const char *agent_spec)
+{
+    int rc;
+    const char *text = NULL;
+    lrmd_t *lrmd_conn = NULL;
+    lrmd_list_t *list = NULL;
+
+    rc = lrmd__new(&lrmd_conn, NULL, NULL, 0);
+    if (rc != pcmk_rc_ok) {
+        goto error;
+    }
+
+    switch (options.rsc_cmd) {
+        case cmd_list_alternatives:
+            rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, agent_spec, &list);
+
+            if (rc > 0) {
+                rc = out->message(out, "alternatives-list", list, agent_spec);
+            } else {
+                rc = pcmk_rc_error;
+            }
+
+            text = "OCF providers";
+            break;
+        case cmd_list_standards:
+            rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);
+
+            if (rc > 0) {
+                rc = out->message(out, "standards-list", list);
+            } else {
+                rc = pcmk_rc_error;
+            }
+
+            text = "standards";
+            break;
+        case cmd_list_providers:
+            rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, agent_spec, &list);
+
+            if (rc > 0) {
+                rc = out->message(out, "providers-list", list, agent_spec);
+            } else {
+                rc = pcmk_rc_error;
+            }
+
+            text = "OCF providers";
+            break;
+        default:
+            g_set_error(&error, PCMK__RC_ERROR, pcmk_rc_error, "Bug");
+            lrmd_api_delete(lrmd_conn);
+            return pcmk_rc_error;
+    }
+
+error:
+    if (rc != pcmk_rc_ok) {
+        if (agent_spec != NULL) {
+            rc = ENXIO;
+            g_set_error(&error, PCMK__RC_ERROR, rc,
+                        _("No %s found for %s"), text, agent_spec);
+
+        } else {
+            rc = ENXIO;
+            g_set_error(&error, PCMK__RC_ERROR, rc,
+                        _("No %s found"), text);
+        }
+    }
+
+    lrmd_api_delete(lrmd_conn);
+    return rc;
+}
+
+static int
+populate_working_set(xmlNodePtr *cib_xml_copy)
+{
+    int rc = pcmk_rc_ok;
+
+    if (options.xml_file != NULL) {
+        *cib_xml_copy = filename2xml(options.xml_file);
+        if (*cib_xml_copy == NULL) {
+            rc = pcmk_rc_cib_corrupt;
+        }
+    } else {
+        rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local | cib_sync_call);
+        rc = pcmk_legacy2rc(rc);
+    }
+
+    if (rc == pcmk_rc_ok) {
+        data_set = pe_new_working_set();
+        if (data_set == NULL) {
+            rc = ENOMEM;
+        } else {
+            pe__set_working_set_flags(data_set,
+                                      pe_flag_no_counts|pe_flag_no_compat);
+            data_set->priv = out;
+            rc = update_working_set_xml(data_set, cib_xml_copy);
+        }
+    }
+
+    if (rc != pcmk_rc_ok) {
+        free_xml(*cib_xml_copy);
+        *cib_xml_copy = NULL;
+        return rc;
+    }
+
+    cluster_status(data_set);
+    return pcmk_rc_ok;
+}
+
+static int
+refresh(pcmk__output_t *out)
+{
+    int rc = pcmk_rc_ok;
+    const char *router_node = options.host_uname;
+    int attr_options = pcmk__node_attr_none;
+
+    if (options.host_uname) {
+        pe_node_t *node = pe_find_node(data_set->nodes, options.host_uname);
+
+        if (pe__is_guest_or_remote_node(node)) {
+            node = pe__current_node(node->details->remote_rsc);
+            if (node == NULL) {
+                rc = ENXIO;
+                g_set_error(&error, PCMK__RC_ERROR, rc,
+                            _("No cluster connection to Pacemaker Remote node %s detected"),
+                            options.host_uname);
+                return rc;
+            }
+            router_node = node->details->uname;
+            attr_options |= pcmk__node_attr_remote;
+        }
+    }
+
+    if (controld_api == NULL) {
+        out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
+                  options.host_uname? options.host_uname : "all nodes");
+        rc = pcmk_rc_ok;
+        return rc;
+    }
+
+    crm_debug("Re-checking the state of all resources on %s", options.host_uname?options.host_uname:"all nodes");
+
+    rc = pcmk__attrd_api_clear_failures(NULL, options.host_uname, NULL,
+                                        NULL, NULL, NULL, attr_options);
+
+    if (pcmk_controld_api_reprobe(controld_api, options.host_uname,
+                                  router_node) == pcmk_rc_ok) {
+        start_mainloop(controld_api);
+    }
+
+    return rc;
+}
+
+static void
+refresh_resource(pcmk__output_t *out, pe_resource_t *rsc, pe_node_t *node)
+{
+    int rc = pcmk_rc_ok;
+
+    if (options.force == FALSE) {
+        rsc = uber_parent(rsc);
+    }
+
+    crm_debug("Re-checking the state of %s (%s requested) on %s",
+              rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
+    rc = cli_resource_delete(controld_api, options.host_uname, rsc, NULL, 0,
+                             FALSE, data_set, options.force);
+
+    if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
+        // Show any reasons why resource might stay stopped
+        cli_resource_check(out, rsc, node);
+    }
+
+    if (rc == pcmk_rc_ok) {
+        start_mainloop(controld_api);
+    }
+}
+
+static int
+set_property(void)
+{
+    int rc = pcmk_rc_ok;
+    xmlNode *msg_data = NULL;
+
+    if (pcmk__str_empty(options.rsc_type)) {
+        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                    _("Must specify -t with resource type"));
+        rc = ENXIO;
+        return rc;
+
+    } else if (pcmk__str_empty(options.prop_value)) {
+        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                    _("Must supply -v with new value"));
+        rc = ENXIO;
+        return rc;
+    }
+
+    CRM_LOG_ASSERT(options.prop_name != NULL);
+
+    msg_data = create_xml_node(NULL, options.rsc_type);
+    crm_xml_add(msg_data, XML_ATTR_ID, options.rsc_id);
+    crm_xml_add(msg_data, options.prop_name, options.prop_value);
+
+    rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data,
+                                options.cib_options);
+    rc = pcmk_legacy2rc(rc);
+    free_xml(msg_data);
+
+    return rc;
+}
+
+static int
+show_metadata(pcmk__output_t *out, const char *agent_spec)
+{
+    int rc = pcmk_rc_ok;
+    char *standard = NULL;
+    char *provider = NULL;
+    char *type = NULL;
+    char *metadata = NULL;
+    lrmd_t *lrmd_conn = NULL;
+
+    rc = lrmd__new(&lrmd_conn, NULL, NULL, 0);
+    if (rc != pcmk_rc_ok) {
+        g_set_error(&error, PCMK__RC_ERROR, rc,
+                    _("Could not create executor connection"));
+        lrmd_api_delete(lrmd_conn);
+        return rc;
+    }
+
+    rc = crm_parse_agent_spec(agent_spec, &standard, &provider, &type);
+    rc = pcmk_legacy2rc(rc);
+
+    if (rc == pcmk_rc_ok) {
+        rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard,
+                                           provider, type,
+                                           &metadata, 0);
+        rc = pcmk_legacy2rc(rc);
+
+        if (metadata) {
+            out->output_xml(out, "metadata", metadata);
+            free(metadata);
+        } else {
+            /* We were given a validly formatted spec, but it doesn't necessarily
+             * match up with anything that exists.  Use ENXIO as the return code
+             * here because that maps to an exit code of CRM_EX_NOSUCH, which
+             * probably is the most common reason to get here.
+             */
+            rc = ENXIO;
+            g_set_error(&error, PCMK__RC_ERROR, rc,
+                        _("Metadata query for %s failed: %s"),
+                        agent_spec, pcmk_rc_str(rc));
+        }
+    } else {
+        rc = ENXIO;
+        g_set_error(&error, PCMK__RC_ERROR, rc,
+                    _("'%s' is not a valid agent specification"), agent_spec);
+    }
+
+    lrmd_api_delete(lrmd_conn);
+    return rc;
+}
+
+static void
+validate_cmdline_config(void)
+{
+    // Cannot use both --resource and command-line resource configuration
+    if (options.rsc_id != NULL) {
+        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                    _("--resource cannot be used with --class, --agent, and --provider"));
+
+    // Not all commands support command-line resource configuration
+    } else if (options.rsc_cmd != cmd_execute_agent) {
+        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                    _("--class, --agent, and --provider can only be used with "
+                    "--validate and --force-*"));
+
+    // Not all of --class, --agent, and --provider need to be given.  Not all
+    // classes support the concept of a provider.  Check that what we were given
+    // is valid.
+    } else if (pcmk__str_eq(options.v_class, "stonith", pcmk__str_none)) {
+        if (options.v_provider != NULL) {
+            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                        _("stonith does not support providers"));
+
+        } else if (stonith_agent_exists(options.v_agent, 0) == FALSE) {
+            g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                        _("%s is not a known stonith agent"), options.v_agent ? options.v_agent : "");
+        }
+
+    } else if (resources_agent_exists(options.v_class, options.v_provider, options.v_agent) == FALSE) {
+        g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
+                    _("%s:%s:%s is not a known resource"),
+                    options.v_class ? options.v_class : "",
+                    options.v_provider ? options.v_provider : "",
+                    options.v_agent ? options.v_agent : "");
+    }
+
+    if (error != NULL) {
+        return;
+    }
+
+    if (options.cmdline_params == NULL) {
+        options.cmdline_params = pcmk__strkey_table(free, free);
+    }
+    options.require_resource = FALSE;
+    options.require_dataset = FALSE;
+    options.require_cib = FALSE;
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    GOptionEntry extra_prog_entries[] = {
+        { "quiet", 'Q', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &(args->quiet),
+          "Be less descriptive in output.",
+          NULL },
+        { "resource", 'r', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_id,
+          "Resource ID",
+          "ID" },
+        { G_OPTION_REMAINING, 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING_ARRAY, &options.remainder,
+          NULL,
+          NULL },
+
+        { NULL }
+    };
+
+    const char *description = "Examples:\n\n"
+                              "List the available OCF agents:\n\n"
+                              "\t# crm_resource --list-agents ocf\n\n"
+                              "List the available OCF agents from the linux-ha project:\n\n"
+                              "\t# crm_resource --list-agents ocf:heartbeat\n\n"
+                              "Move 'myResource' to a specific node:\n\n"
+                              "\t# crm_resource --resource myResource --move --node altNode\n\n"
+                              "Allow (but not force) 'myResource' to move back to its original "
+                              "location:\n\n"
+                              "\t# crm_resource --resource myResource --clear\n\n"
+                              "Stop 'myResource' (and anything that depends on it):\n\n"
+                              "\t# crm_resource --resource myResource --set-parameter target-role "
+                              "--meta --parameter-value Stopped\n\n"
+                              "Tell the cluster not to manage 'myResource' (the cluster will not "
+                              "attempt to start or stop the\n"
+                              "resource under any circumstances; useful when performing maintenance "
+                              "tasks on a resource):\n\n"
+                              "\t# crm_resource --resource myResource --set-parameter is-managed "
+                              "--meta --parameter-value false\n\n"
+                              "Erase the operation history of 'myResource' on 'aNode' (the cluster "
+                              "will 'forget' the existing\n"
+                              "resource state, including any errors, and attempt to recover the"
+                              "resource; useful when a resource\n"
+                              "had failed permanently and has been repaired by an administrator):\n\n"
+                              "\t# crm_resource --resource myResource --cleanup --node aNode\n\n";
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
+    g_option_context_set_description(context, description);
+
+    /* Add the -Q option, which cannot be part of the globally supported options
+     * because some tools use that flag for something else.
+     */
+    pcmk__add_main_args(context, extra_prog_entries);
+
+    pcmk__add_arg_group(context, "queries", "Queries:",
+                        "Show query help", query_entries);
+    pcmk__add_arg_group(context, "commands", "Commands:",
+                        "Show command help", command_entries);
+    pcmk__add_arg_group(context, "locations", "Locations:",
+                        "Show location help", location_entries);
+    pcmk__add_arg_group(context, "advanced", "Advanced:",
+                        "Show advanced option help", advanced_entries);
+    pcmk__add_arg_group(context, "additional", "Additional Options:",
+                        "Show additional options", addl_entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    xmlNode *cib_xml_copy = NULL;
+    pe_resource_t *rsc = NULL;
+    pe_node_t *node = NULL;
+    int rc = pcmk_rc_ok;
+
+    GOptionGroup *output_group = NULL;
+    gchar **processed_args = NULL;
+    GOptionContext *context = NULL;
+
+    /*
+     * Parse command line arguments
+     */
+
+    args = pcmk__new_common_args(SUMMARY);
+    processed_args = pcmk__cmdline_preproc(argv, "GHINSTdginpstuvx");
+    context = build_arg_context(args, &output_group);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("crm_resource", args->verbosity);
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, _("Error creating output format %s: %s"),
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    pe__register_messages(out);
+    crm_resource_register_messages(out);
+    lrmd__register_messages(out);
+    pcmk__register_lib_messages(out);
+
+    out->quiet = args->quiet;
+
+    crm_log_args(argc, argv);
+
+    /*
+     * Validate option combinations
+     */
+
+    // If the user didn't explicitly specify a command, list resources
+    if (options.rsc_cmd == cmd_none) {
+        options.rsc_cmd = cmd_list_resources;
+        options.require_resource = FALSE;
+    }
+
+    // --expired without --clear/-U doesn't make sense
+    if (options.clear_expired && (options.rsc_cmd != cmd_clear)) {
+        exit_code = CRM_EX_USAGE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, _("--expired requires --clear or -U"));
+        goto done;
+    }
+
+    if ((options.remainder != NULL) && (options.override_params != NULL)) {
+        // Commands that use positional arguments will create override_params
+        for (gchar **s = options.remainder; *s; s++) {
+            char *name = calloc(1, strlen(*s));
+            char *value = calloc(1, strlen(*s));
+            int rc = sscanf(*s, "%[^=]=%s", name, value);
+
+            if (rc == 2) {
+                g_hash_table_replace(options.override_params, name, value);
+
+            } else {
+                exit_code = CRM_EX_USAGE;
+                g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                            _("Error parsing '%s' as a name=value pair"),
+                            argv[optind]);
+                free(value);
+                free(name);
+                goto done;
+            }
+        }
+
+    } else if (options.remainder != NULL) {
+        gchar **strv = NULL;
+        gchar *msg = NULL;
+        int i = 1;
+        int len = 0;
+
+        for (gchar **s = options.remainder; *s; s++) {
+            len++;
+        }
+
+        CRM_ASSERT(len > 0);
+
+        /* Add 1 for the strv[0] string below, and add another 1 for the NULL
+         * at the end of the array so g_strjoinv knows when to stop.
+         */
+        strv = calloc(len+2, sizeof(char *));
+        strv[0] = strdup("non-option ARGV-elements:\n");
+
+        for (gchar **s = options.remainder; *s; s++) {
+            strv[i] = crm_strdup_printf("[%d of %d] %s\n", i, len, *s);
+            i++;
+        }
+
+        strv[i] = NULL;
+
+        exit_code = CRM_EX_USAGE;
+        msg = g_strjoinv("", strv);
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "%s", msg);
+        g_free(msg);
+
+        /* Don't try to free the last element, which is just NULL. */
+        for(i = 0; i < len+1; i++) {
+            free(strv[i]);
+        }
+        free(strv);
+
+        goto done;
+    }
+
+    if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
+        /* Kind of a hack to display XML lists using a real tag instead of <list>.  This just
+         * saves from having to write custom messages to build the lists around all these things
+         */
+        switch (options.rsc_cmd) {
+            case cmd_execute_agent:
+            case cmd_list_resources:
+            case cmd_query_xml:
+            case cmd_query_raw_xml:
+            case cmd_list_active_ops:
+            case cmd_list_all_ops:
+            case cmd_colocations:
+                pcmk__force_args(context, &error, "%s --xml-simple-list --xml-substitute", g_get_prgname());
+                break;
+
+            default:
+                pcmk__force_args(context, &error, "%s --xml-substitute", g_get_prgname());
+                break;
+        }
+    } else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches)) {
+        if ((options.rsc_cmd == cmd_colocations) ||
+            options.rsc_cmd == cmd_list_resources) {
+            pcmk__force_args(context, &error, "%s --text-fancy", g_get_prgname());
+        }
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    if (options.cmdline_config) {
+        /* A resource configuration was given on the command line. Sanity-check
+         * the values and set error if they don't make sense.
+         */
+        validate_cmdline_config();
+        if (error != NULL) {
+            exit_code = CRM_EX_USAGE;
+            goto done;
+        }
+
+    } else if (options.cmdline_params != NULL) {
+        // @COMPAT @TODO error out here when we can break backward compatibility
+        g_hash_table_destroy(options.cmdline_params);
+        options.cmdline_params = NULL;
+    }
+
+    if (options.require_resource && (options.rsc_id == NULL)) {
+        exit_code = CRM_EX_USAGE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    _("Must supply a resource id with -r"));
+        goto done;
+    }
+    if (options.require_node && (options.host_uname == NULL)) {
+        exit_code = CRM_EX_USAGE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    _("Must supply a node name with -N"));
+        goto done;
+    }
+
+    /*
+     * Set up necessary connections
+     */
+
+    if (options.find_flags && options.rsc_id) {
+        options.require_dataset = TRUE;
+    }
+
+    // Establish a connection to the CIB if needed
+    if (options.require_cib) {
+        cib_conn = cib_new();
+        if ((cib_conn == NULL) || (cib_conn->cmds == NULL)) {
+            exit_code = CRM_EX_DISCONNECT;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        _("Could not create CIB connection"));
+            goto done;
+        }
+        rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
+        rc = pcmk_legacy2rc(rc);
+        if (rc != pcmk_rc_ok) {
+            exit_code = pcmk_rc2exitc(rc);
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        _("Could not connect to the CIB: %s"), pcmk_rc_str(rc));
+            goto done;
+        }
+    }
+
+    /* Populate working set from XML file if specified or CIB query otherwise */
+    if (options.require_dataset) {
+        rc = populate_working_set(&cib_xml_copy);
+        if (rc != pcmk_rc_ok) {
+            exit_code = pcmk_rc2exitc(rc);
+            goto done;
+        }
+    }
+
+    // If command requires that resource exist if specified, find it
+    if (options.find_flags && options.rsc_id) {
+        rsc = pe_find_resource_with_flags(data_set->resources, options.rsc_id,
+                                          options.find_flags);
+        if (rsc == NULL) {
+            exit_code = CRM_EX_NOSUCH;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        _("Resource '%s' not found"), options.rsc_id);
+            goto done;
+        }
+
+        /* The --ban, --clear, --move, and --restart commands do not work with
+         * instances of clone resourcs.
+         */
+        if (strchr(options.rsc_id, ':') != NULL && pe_rsc_is_clone(rsc->parent) &&
+            (options.rsc_cmd == cmd_ban || options.rsc_cmd == cmd_clear ||
+             options.rsc_cmd == cmd_move || options.rsc_cmd == cmd_restart)) {
+            exit_code = CRM_EX_INVALID_PARAM;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        _("Cannot operate on clone resource instance '%s'"), options.rsc_id);
+            goto done;
+        }
+    }
+
+    // If user supplied a node name, check whether it exists
+    if ((options.host_uname != NULL) && (data_set != NULL)) {
+        node = pe_find_node(data_set->nodes, options.host_uname);
+
+        if (node == NULL) {
+            exit_code = CRM_EX_NOSUCH;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        _("Node '%s' not found"), options.host_uname);
+            goto done;
+        }
+    }
+
+    // Establish a connection to the controller if needed
+    if (options.require_crmd) {
+        rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
+        if (rc != pcmk_rc_ok) {
+            exit_code = pcmk_rc2exitc(rc);
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        _("Error connecting to the controller: %s"), pcmk_rc_str(rc));
+            goto done;
+        }
+        pcmk_register_ipc_callback(controld_api, controller_event_callback,
+                                   NULL);
+        rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main);
+        if (rc != pcmk_rc_ok) {
+            exit_code = pcmk_rc2exitc(rc);
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        _("Error connecting to the controller: %s"), pcmk_rc_str(rc));
+            goto done;
+        }
+    }
+
+    /*
+     * Handle requested command
+     */
+
+    switch (options.rsc_cmd) {
+        case cmd_list_resources: {
+            GList *all = NULL;
+            all = g_list_prepend(all, (gpointer) "*");
+            rc = out->message(out, "resource-list", data_set,
+                              pcmk_show_inactive_rscs | pcmk_show_rsc_only | pcmk_show_pending,
+                              true, all, all, false);
+            g_list_free(all);
+
+            if (rc == pcmk_rc_no_output) {
+                rc = ENXIO;
+            }
+            break;
+        }
+
+        case cmd_list_instances:
+            rc = out->message(out, "resource-names-list", data_set->resources);
+
+            if (rc != pcmk_rc_ok) {
+                rc = ENXIO;
+            }
+
+            break;
+
+        case cmd_list_standards:
+        case cmd_list_providers:
+        case cmd_list_alternatives:
+            rc = list_providers(out, options.agent_spec);
+            break;
+
+        case cmd_list_agents:
+            rc = list_agents(out, options.agent_spec);
+            break;
+
+        case cmd_metadata:
+            rc = show_metadata(out, options.agent_spec);
+            break;
+
+        case cmd_restart:
+            /* We don't pass data_set because rsc needs to stay valid for the
+             * entire lifetime of cli_resource_restart(), but it will reset and
+             * update the working set multiple times, so it needs to use its own
+             * copy.
+             */
+            rc = cli_resource_restart(out, rsc, node, options.move_lifetime,
+                                      options.timeout_ms, cib_conn,
+                                      options.cib_options, options.promoted_role_only,
+                                      options.force);
+            break;
+
+        case cmd_wait:
+            rc = wait_till_stable(out, options.timeout_ms, cib_conn);
+            break;
+
+        case cmd_execute_agent:
+            if (options.cmdline_config) {
+                exit_code = cli_resource_execute_from_params(out, NULL,
+                    options.v_class, options.v_provider, options.v_agent,
+                    options.operation, options.cmdline_params,
+                    options.override_params, options.timeout_ms,
+                    args->verbosity, options.force, options.check_level);
+            } else {
+                exit_code = cli_resource_execute(rsc, options.rsc_id,
+                    options.operation, options.override_params,
+                    options.timeout_ms, cib_conn, data_set,
+                    args->verbosity, options.force, options.check_level);
+            }
+            goto done;
+
+        case cmd_digests:
+            node = pe_find_node(data_set->nodes, options.host_uname);
+            if (node == NULL) {
+                rc = pcmk_rc_node_unknown;
+            } else {
+                rc = pcmk__resource_digests(out, rsc, node,
+                                            options.override_params);
+            }
+            break;
+
+        case cmd_colocations:
+            rc = out->message(out, "locations-and-colocations", rsc, data_set,
+                              options.recursive, (bool) options.force);
+            break;
+
+        case cmd_cts:
+            rc = pcmk_rc_ok;
+            g_list_foreach(data_set->resources, (GFunc) cli_resource_print_cts, out);
+            cli_resource_print_cts_constraints(data_set);
+            break;
+
+        case cmd_fail:
+            rc = cli_resource_fail(controld_api, options.host_uname,
+                                   options.rsc_id, data_set);
+            if (rc == pcmk_rc_ok) {
+                start_mainloop(controld_api);
+            }
+            break;
+
+        case cmd_list_active_ops:
+            rc = cli_resource_print_operations(options.rsc_id,
+                                               options.host_uname, TRUE,
+                                               data_set);
+            break;
+
+        case cmd_list_all_ops:
+            rc = cli_resource_print_operations(options.rsc_id,
+                                               options.host_uname, FALSE,
+                                               data_set);
+            break;
+
+        case cmd_locate: {
+            GList *nodes = cli_resource_search(rsc, options.rsc_id, data_set);
+            rc = out->message(out, "resource-search-list", nodes, options.rsc_id);
+            g_list_free_full(nodes, free);
+            break;
+        }
+
+        case cmd_query_xml:
+            rc = cli_resource_print(rsc, data_set, true);
+            break;
+
+        case cmd_query_raw_xml:
+            rc = cli_resource_print(rsc, data_set, false);
+            break;
+
+        case cmd_why:
+            if ((options.host_uname != NULL) && (node == NULL)) {
+                rc = pcmk_rc_node_unknown;
+            } else {
+                rc = out->message(out, "resource-reasons-list",
+                                  data_set->resources, rsc, node);
+            }
+            break;
+
+        case cmd_clear:
+            rc = clear_constraints(out, &cib_xml_copy);
+            break;
+
+        case cmd_move:
+            if (options.host_uname == NULL) {
+                rc = ban_or_move(out, rsc, options.move_lifetime);
+            } else {
+                rc = cli_resource_move(rsc, options.rsc_id, options.host_uname,
+                                       options.move_lifetime, cib_conn,
+                                       options.cib_options, data_set,
+                                       options.promoted_role_only,
+                                       options.force);
+            }
+
+            if (rc == EINVAL) {
+                exit_code = CRM_EX_USAGE;
+                goto done;
+            }
+
+            break;
+
+        case cmd_ban:
+            if (options.host_uname == NULL) {
+                rc = ban_or_move(out, rsc, options.move_lifetime);
+            } else if (node == NULL) {
+                rc = pcmk_rc_node_unknown;
+            } else {
+                rc = cli_resource_ban(out, options.rsc_id, node->details->uname,
+                                      options.move_lifetime, NULL, cib_conn,
+                                      options.cib_options,
+                                      options.promoted_role_only);
+            }
+
+            if (rc == EINVAL) {
+                exit_code = CRM_EX_USAGE;
+                goto done;
+            }
+
+            break;
+
+        case cmd_get_property:
+            rc = out->message(out, "property-list", rsc, options.prop_name);
+            if (rc == pcmk_rc_no_output) {
+                rc = ENXIO;
+            }
+
+            break;
+
+        case cmd_set_property:
+            rc = set_property();
+            break;
+
+        case cmd_get_param: {
+            unsigned int count = 0;
+            GHashTable *params = NULL;
+            pe_node_t *current = rsc->fns->active_node(rsc, &count, NULL);
+            bool free_params = true;
+            const char* value = NULL;
+
+            if (count > 1) {
+                out->err(out, "%s is active on more than one node,"
+                         " returning the default value for %s", rsc->id,
+                         pcmk__s(options.prop_name, "unspecified property"));
+                current = NULL;
+            }
+
+            crm_debug("Looking up %s in %s", options.prop_name, rsc->id);
+
+            if (pcmk__str_eq(options.attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_none)) {
+                params = pe_rsc_params(rsc, current, data_set);
+                free_params = false;
+
+                value = g_hash_table_lookup(params, options.prop_name);
+
+            } else if (pcmk__str_eq(options.attr_set_type, XML_TAG_META_SETS, pcmk__str_none)) {
+                params = pcmk__strkey_table(free, free);
+                get_meta_attributes(params, rsc, current, data_set);
+
+                value = g_hash_table_lookup(params, options.prop_name);
+
+            } else if (pcmk__str_eq(options.attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
+
+                value = crm_element_value(rsc->xml, options.prop_name);
+                free_params = false;
+
+            } else {
+                params = pcmk__strkey_table(free, free);
+                pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_UTILIZATION, NULL, params,
+                                           NULL, FALSE, data_set);
+
+                value = g_hash_table_lookup(params, options.prop_name);
+            }
+
+            rc = out->message(out, "attribute-list", rsc, options.prop_name, value);
+            if (free_params) {
+                g_hash_table_destroy(params);
+            }
+
+            break;
+        }
+
+        case cmd_set_param:
+            if (pcmk__str_empty(options.prop_value)) {
+                exit_code = CRM_EX_USAGE;
+                g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                            _("You need to supply a value with the -v option"));
+                goto done;
+            }
+
+            /* coverity[var_deref_model] False positive */
+            rc = cli_resource_update_attribute(rsc, options.rsc_id,
+                                               options.prop_set,
+                                               options.attr_set_type,
+                                               options.prop_id,
+                                               options.prop_name,
+                                               options.prop_value,
+                                               options.recursive, cib_conn,
+                                               options.cib_options,
+                                               options.force);
+            break;
+
+        case cmd_delete_param:
+            /* coverity[var_deref_model] False positive */
+            rc = cli_resource_delete_attribute(rsc, options.rsc_id,
+                                               options.prop_set,
+                                               options.attr_set_type,
+                                               options.prop_id,
+                                               options.prop_name, cib_conn,
+                                               options.cib_options,
+                                               options.force);
+            break;
+
+        case cmd_cleanup:
+            if (rsc == NULL) {
+                rc = cli_cleanup_all(controld_api, options.host_uname,
+                                     options.operation, options.interval_spec,
+                                     data_set);
+                if (rc == pcmk_rc_ok) {
+                    start_mainloop(controld_api);
+                }
+            } else {
+                cleanup(out, rsc, node);
+            }
+            break;
+
+        case cmd_refresh:
+            if (rsc == NULL) {
+                rc = refresh(out);
+            } else {
+                refresh_resource(out, rsc, node);
+            }
+            break;
+
+        case cmd_delete:
+            rc = delete();
+            break;
+
+        default:
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        _("Unimplemented command: %d"), (int) options.rsc_cmd);
+            goto done;
+    }
+
+    /* Convert rc into an exit code. */
+    if (rc != pcmk_rc_ok && rc != pcmk_rc_no_output) {
+        exit_code = pcmk_rc2exitc(rc);
+    }
+
+    /*
+     * Clean up and exit
+     */
+
+done:
+    /* When we get here, exit_code has been set one of two ways - either at one of
+     * the spots where there's a "goto done" (which itself could have happened either
+     * directly or by calling pcmk_rc2exitc), or just up above after any of the break
+     * statements.
+     *
+     * Thus, we can use just exit_code here to decide what to do.
+     */
+    if (exit_code != CRM_EX_OK && exit_code != CRM_EX_USAGE) {
+        if (error != NULL) {
+            char *msg = crm_strdup_printf("%s\nError performing operation: %s",
+                                          error->message, crm_exit_str(exit_code));
+            g_clear_error(&error);
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "%s", msg);
+            free(msg);
+        } else {
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        _("Error performing operation: %s"), crm_exit_str(exit_code));
+        }
+    }
+
+    g_free(options.host_uname);
+    g_free(options.interval_spec);
+    g_free(options.move_lifetime);
+    g_free(options.operation);
+    g_free(options.prop_id);
+    free(options.prop_name);
+    g_free(options.prop_set);
+    g_free(options.prop_value);
+    g_free(options.rsc_id);
+    g_free(options.rsc_type);
+    free(options.agent_spec);
+    free(options.v_agent);
+    free(options.v_class);
+    free(options.v_provider);
+    g_free(options.xml_file);
+    g_strfreev(options.remainder);
+
+    if (options.override_params != NULL) {
+        g_hash_table_destroy(options.override_params);
+    }
+
+    /* options.cmdline_params does not need to be destroyed here.  See the
+     * comments in cli_resource_execute_from_params.
+     */
+
+    g_strfreev(processed_args);
+    g_option_context_free(context);
+
+    return bye(exit_code);
+}
diff --git a/tools/crm_resource.h b/tools/crm_resource.h
new file mode 100644
index 0000000..dcd6c3d
--- /dev/null
+++ b/tools/crm_resource.h
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdint.h>
+#include <stdbool.h>
+
+#include <crm/crm.h>
+
+#include <crm/msg_xml.h>
+#include <crm/services.h>
+#include <crm/common/xml.h>
+#include <crm/common/mainloop.h>
+#include <crm/common/output_internal.h>
+
+#include <crm/cib.h>
+#include <crm/common/attrd_internal.h>
+#include <crm/pengine/rules.h>
+#include <crm/pengine/status.h>
+#include <crm/pengine/internal.h>
+#include <pacemaker-internal.h>
+
+#define ATTR_SET_ELEMENT "attr_set_element"
+
+typedef struct node_info_s {
+    const char *node_name;
+    bool promoted;
+} node_info_t;
+
+enum resource_check_flags {
+    rsc_remain_stopped  = (1 << 0),
+    rsc_unpromotable    = (1 << 1),
+    rsc_unmanaged       = (1 << 2),
+    rsc_locked          = (1 << 3),
+    rsc_node_health     = (1 << 4),
+};
+
+typedef struct resource_checks_s {
+    pe_resource_t *rsc;     // Resource being checked
+    uint32_t flags;         // Group of enum resource_check_flags
+    const char *lock_node;  // Node that resource is shutdown-locked to, if any
+} resource_checks_t;
+
+resource_checks_t *cli_check_resource(pe_resource_t *rsc, char *role_s, char *managed);
+
+/* ban */
+int cli_resource_prefer(pcmk__output_t *out, const char *rsc_id, const char *host,
+                        const char *move_lifetime, cib_t * cib_conn, int cib_options,
+                        gboolean promoted_role_only);
+int cli_resource_ban(pcmk__output_t *out, const char *rsc_id, const char *host,
+                     const char *move_lifetime, GList *allnodes, cib_t * cib_conn,
+                     int cib_options, gboolean promoted_role_only);
+int cli_resource_clear(const char *rsc_id, const char *host, GList *allnodes,
+                       cib_t * cib_conn, int cib_options, bool clear_ban_constraints, gboolean force);
+int cli_resource_clear_all_expired(xmlNode *root, cib_t *cib_conn, int cib_options,
+                                   const char *rsc, const char *node, gboolean promoted_role_only);
+
+/* print */
+void cli_resource_print_cts(pe_resource_t * rsc, pcmk__output_t *out);
+void cli_resource_print_cts_constraints(pe_working_set_t * data_set);
+
+int cli_resource_print(pe_resource_t *rsc, pe_working_set_t *data_set, bool expanded);
+int cli_resource_print_operations(const char *rsc_id, const char *host_uname,
+                                  bool active, pe_working_set_t * data_set);
+
+/* runtime */
+int cli_resource_check(pcmk__output_t *out, pe_resource_t *rsc,
+                       pe_node_t *node);
+int cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
+                      const char *rsc_id, pe_working_set_t *data_set);
+GList *cli_resource_search(pe_resource_t *rsc, const char *requested_name,
+                             pe_working_set_t *data_set);
+int cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
+                        const pe_resource_t *rsc, const char *operation,
+                        const char *interval_spec, bool just_failures,
+                        pe_working_set_t *data_set, gboolean force);
+int cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
+                    const char *operation, const char *interval_spec,
+                    pe_working_set_t *data_set);
+int cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc,
+                         const pe_node_t *node, const char *move_lifetime,
+                         int timeout_ms, cib_t *cib, int cib_options,
+                         gboolean promoted_role_only, gboolean force);
+int cli_resource_move(const pe_resource_t *rsc, const char *rsc_id,
+                      const char *host_name, const char *move_lifetime,
+                      cib_t *cib, int cib_options, pe_working_set_t *data_set,
+                      gboolean promoted_role_only, gboolean force);
+crm_exit_t cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
+                                            const char *rsc_class, const char *rsc_prov,
+                                            const char *rsc_type, const char *rsc_action,
+                                            GHashTable *params, GHashTable *override_hash,
+                                            int timeout_ms, int resource_verbose,
+                                            gboolean force, int check_level);
+crm_exit_t cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
+                                const char *rsc_action, GHashTable *override_hash,
+                                int timeout_ms, cib_t *cib, pe_working_set_t *data_set,
+                                int resource_verbose, gboolean force, int check_level);
+
+int cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
+                                  const char *attr_set, const char *attr_set_type,
+                                  const char *attr_id, const char *attr_name,
+                                  const char *attr_value, gboolean recursive,
+                                  cib_t *cib, int cib_options, gboolean force);
+int cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name,
+                                  const char *attr_set, const char *attr_set_type,
+                                  const char *attr_id, const char *attr_name,
+                                  cib_t *cib, int cib_options, gboolean force);
+
+int update_working_set_xml(pe_working_set_t *data_set, xmlNode **xml);
+int wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib);
+
+bool resource_is_running_on(pe_resource_t *rsc, const char *host);
+
+void crm_resource_register_messages(pcmk__output_t *out);
diff --git a/tools/crm_resource_ban.c b/tools/crm_resource_ban.c
new file mode 100644
index 0000000..b1edac8
--- /dev/null
+++ b/tools/crm_resource_ban.c
@@ -0,0 +1,505 @@
+/*
+ * Copyright 2004-2021 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm_resource.h>
+
+static char *
+parse_cli_lifetime(pcmk__output_t *out, const char *move_lifetime)
+{
+    char *later_s = NULL;
+    crm_time_t *now = NULL;
+    crm_time_t *later = NULL;
+    crm_time_t *duration = NULL;
+
+    if (move_lifetime == NULL) {
+        return NULL;
+    }
+
+    duration = crm_time_parse_duration(move_lifetime);
+    if (duration == NULL) {
+        out->err(out, "Invalid duration specified: %s\n"
+                      "Please refer to https://en.wikipedia.org/wiki/ISO_8601#Durations "
+                      "for examples of valid durations", move_lifetime);
+        return NULL;
+    }
+
+    now = crm_time_new(NULL);
+    later = crm_time_add(now, duration);
+    if (later == NULL) {
+        out->err(out, "Unable to add %s to current time\n"
+                      "Please report to " PACKAGE_BUGREPORT " as possible bug",
+                      move_lifetime);
+        crm_time_free(now);
+        crm_time_free(duration);
+        return NULL;
+    }
+
+    crm_time_log(LOG_INFO, "now     ", now,
+                 crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
+    crm_time_log(LOG_INFO, "later   ", later,
+                 crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
+    crm_time_log(LOG_INFO, "duration", duration, crm_time_log_date | crm_time_log_timeofday);
+    later_s = crm_time_as_string(later, crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
+    out->info(out, "Migration will take effect until: %s", later_s);
+
+    crm_time_free(duration);
+    crm_time_free(later);
+    crm_time_free(now);
+    return later_s;
+}
+
+static const char *
+promoted_role_name(void)
+{
+    /* This is a judgment call for what string to use. @TODO Ideally we'd
+     * use the legacy string if the DC only supports that, and the new one
+     * otherwise. Basing it on --enable-compat-2.0 is a decent guess.
+     */
+#ifdef PCMK__COMPAT_2_0
+        return RSC_ROLE_PROMOTED_LEGACY_S;
+#else
+        return RSC_ROLE_PROMOTED_S;
+#endif
+}
+
+// \return Standard Pacemaker return code
+int
+cli_resource_ban(pcmk__output_t *out, const char *rsc_id, const char *host,
+                 const char *move_lifetime, GList *allnodes, cib_t * cib_conn,
+                 int cib_options, gboolean promoted_role_only)
+{
+    char *later_s = NULL;
+    int rc = pcmk_rc_ok;
+    xmlNode *fragment = NULL;
+    xmlNode *location = NULL;
+
+    if(host == NULL) {
+        GList *n = allnodes;
+        for(; n && rc == pcmk_rc_ok; n = n->next) {
+            pe_node_t *target = n->data;
+
+            rc = cli_resource_ban(out, rsc_id, target->details->uname, move_lifetime,
+                                  NULL, cib_conn, cib_options, promoted_role_only);
+        }
+        return rc;
+    }
+
+    later_s = parse_cli_lifetime(out, move_lifetime);
+    if(move_lifetime && later_s == NULL) {
+        return EINVAL;
+    }
+
+    fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS);
+
+    location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
+    crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host);
+
+    out->info(out, "WARNING: Creating rsc_location constraint '%s' with a "
+                   "score of -INFINITY for resource %s on %s.\n\tThis will "
+                   "prevent %s from %s on %s until the constraint is removed "
+                   "using the clear option or by editing the CIB with an "
+                   "appropriate tool\n\tThis will be the case even if %s "
+                   "is the last node in the cluster",
+                   ID(location), rsc_id, host, rsc_id,
+                   (promoted_role_only? "being promoted" : "running"),
+                   host, host);
+
+    crm_xml_add(location, XML_LOC_ATTR_SOURCE, rsc_id);
+    if(promoted_role_only) {
+        crm_xml_add(location, XML_RULE_ATTR_ROLE, promoted_role_name());
+    } else {
+        crm_xml_add(location, XML_RULE_ATTR_ROLE, RSC_ROLE_STARTED_S);
+    }
+
+    if (later_s == NULL) {
+        /* Short form */
+        crm_xml_add(location, XML_CIB_TAG_NODE, host);
+        crm_xml_add(location, XML_RULE_ATTR_SCORE, CRM_MINUS_INFINITY_S);
+
+    } else {
+        xmlNode *rule = create_xml_node(location, XML_TAG_RULE);
+        xmlNode *expr = create_xml_node(rule, XML_TAG_EXPRESSION);
+
+        crm_xml_set_id(rule, "cli-ban-%s-on-%s-rule", rsc_id, host);
+        crm_xml_add(rule, XML_RULE_ATTR_SCORE, CRM_MINUS_INFINITY_S);
+        crm_xml_add(rule, XML_RULE_ATTR_BOOLEAN_OP, "and");
+
+        crm_xml_set_id(expr, "cli-ban-%s-on-%s-expr", rsc_id, host);
+        crm_xml_add(expr, XML_EXPR_ATTR_ATTRIBUTE, CRM_ATTR_UNAME);
+        crm_xml_add(expr, XML_EXPR_ATTR_OPERATION, "eq");
+        crm_xml_add(expr, XML_EXPR_ATTR_VALUE, host);
+        crm_xml_add(expr, XML_EXPR_ATTR_TYPE, "string");
+
+        expr = create_xml_node(rule, "date_expression");
+        crm_xml_set_id(expr, "cli-ban-%s-on-%s-lifetime", rsc_id, host);
+        crm_xml_add(expr, "operation", "lt");
+        crm_xml_add(expr, "end", later_s);
+    }
+
+    crm_log_xml_notice(fragment, "Modify");
+    rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment,
+                                cib_options);
+    rc = pcmk_legacy2rc(rc);
+
+    free_xml(fragment);
+    free(later_s);
+    return rc;
+}
+
+// \return Standard Pacemaker return code
+int
+cli_resource_prefer(pcmk__output_t *out,const char *rsc_id, const char *host,
+                    const char *move_lifetime, cib_t * cib_conn, int cib_options,
+                    gboolean promoted_role_only)
+{
+    char *later_s = parse_cli_lifetime(out, move_lifetime);
+    int rc = pcmk_rc_ok;
+    xmlNode *location = NULL;
+    xmlNode *fragment = NULL;
+
+    if(move_lifetime && later_s == NULL) {
+        return EINVAL;
+    }
+
+    if(cib_conn == NULL) {
+        free(later_s);
+        return ENOTCONN;
+    }
+
+    fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS);
+
+    location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
+    crm_xml_set_id(location, "cli-prefer-%s", rsc_id);
+
+    crm_xml_add(location, XML_LOC_ATTR_SOURCE, rsc_id);
+    if(promoted_role_only) {
+        crm_xml_add(location, XML_RULE_ATTR_ROLE, promoted_role_name());
+    } else {
+        crm_xml_add(location, XML_RULE_ATTR_ROLE, RSC_ROLE_STARTED_S);
+    }
+
+    if (later_s == NULL) {
+        /* Short form */
+        crm_xml_add(location, XML_CIB_TAG_NODE, host);
+        crm_xml_add(location, XML_RULE_ATTR_SCORE, CRM_INFINITY_S);
+
+    } else {
+        xmlNode *rule = create_xml_node(location, XML_TAG_RULE);
+        xmlNode *expr = create_xml_node(rule, XML_TAG_EXPRESSION);
+
+        crm_xml_set_id(rule, "cli-prefer-rule-%s", rsc_id);
+        crm_xml_add(rule, XML_RULE_ATTR_SCORE, CRM_INFINITY_S);
+        crm_xml_add(rule, XML_RULE_ATTR_BOOLEAN_OP, "and");
+
+        crm_xml_set_id(expr, "cli-prefer-expr-%s", rsc_id);
+        crm_xml_add(expr, XML_EXPR_ATTR_ATTRIBUTE, CRM_ATTR_UNAME);
+        crm_xml_add(expr, XML_EXPR_ATTR_OPERATION, "eq");
+        crm_xml_add(expr, XML_EXPR_ATTR_VALUE, host);
+        crm_xml_add(expr, XML_EXPR_ATTR_TYPE, "string");
+
+        expr = create_xml_node(rule, "date_expression");
+        crm_xml_set_id(expr, "cli-prefer-lifetime-end-%s", rsc_id);
+        crm_xml_add(expr, "operation", "lt");
+        crm_xml_add(expr, "end", later_s);
+    }
+
+    crm_log_xml_info(fragment, "Modify");
+    rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment,
+                                cib_options);
+    rc = pcmk_legacy2rc(rc);
+
+    free_xml(fragment);
+    free(later_s);
+    return rc;
+}
+
+/* Nodes can be specified two different ways in the CIB, so we have two different
+ * functions to try clearing out any constraints on them:
+ *
+ * (1) The node could be given by attribute=/value= in an expression XML node.
+ * That's what resource_clear_node_in_expr handles.  That XML looks like this:
+ *
+ * <rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started">
+ *   <rule id="cli-prefer-rule-dummy" score="INFINITY" boolean-op="and">
+ *     <expression id="cli-prefer-expr-dummy" attribute="#uname" operation="eq" value="test02" type="string"/>
+ *     <date_expression id="cli-prefer-lifetime-end-dummy" operation="lt" end="2018-12-12 14:05:37 -05:00"/>
+ *   </rule>
+ * </rsc_location>
+ *
+ * (2) The mode could be given by node= in an rsc_location XML node.  That's
+ * what resource_clear_node_in_location handles.  That XML looks like this:
+ *
+ * <rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
+ *
+ * \return Standard Pacemaker return code
+ */
+static int
+resource_clear_node_in_expr(const char *rsc_id, const char *host, cib_t * cib_conn,
+                            int cib_options)
+{
+    int rc = pcmk_rc_ok;
+    char *xpath_string = NULL;
+
+#define XPATH_FMT                                                       \
+    "//" XML_CONS_TAG_RSC_LOCATION "[@" XML_ATTR_ID "='cli-prefer-%s']" \
+    "[" XML_TAG_RULE                                                    \
+        "[@" XML_ATTR_ID "='cli-prefer-rule-%s']"                       \
+        "/" XML_TAG_EXPRESSION                                          \
+        "[@" XML_EXPR_ATTR_ATTRIBUTE "='#uname' "                       \
+        "and @" XML_EXPR_ATTR_VALUE "='%s']"                            \
+    "]"
+
+    xpath_string = crm_strdup_printf(XPATH_FMT, rsc_id, rsc_id, host);
+
+    rc = cib_conn->cmds->remove(cib_conn, xpath_string, NULL, cib_xpath | cib_options);
+    if (rc == -ENXIO) {
+        rc = pcmk_rc_ok;
+    } else {
+        rc = pcmk_legacy2rc(rc);
+    }
+
+    free(xpath_string);
+    return rc;
+}
+
+// \return Standard Pacemaker return code
+static int
+resource_clear_node_in_location(const char *rsc_id, const char *host, cib_t * cib_conn,
+                                int cib_options, bool clear_ban_constraints, gboolean force)
+{
+    int rc = pcmk_rc_ok;
+    xmlNode *fragment = NULL;
+    xmlNode *location = NULL;
+
+    fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS);
+
+    if (clear_ban_constraints == TRUE) {
+        location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
+        crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host);
+    }
+
+    location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
+    crm_xml_set_id(location, "cli-prefer-%s", rsc_id);
+    if (force == FALSE) {
+        crm_xml_add(location, XML_CIB_TAG_NODE, host);
+    }
+
+    crm_log_xml_info(fragment, "Delete");
+    rc = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment, cib_options);
+    if (rc == -ENXIO) {
+        rc = pcmk_rc_ok;
+    } else {
+        rc = pcmk_legacy2rc(rc);
+    }
+
+    free_xml(fragment);
+    return rc;
+}
+
+// \return Standard Pacemaker return code
+int
+cli_resource_clear(const char *rsc_id, const char *host, GList *allnodes, cib_t * cib_conn,
+                   int cib_options, bool clear_ban_constraints, gboolean force)
+{
+    int rc = pcmk_rc_ok;
+
+    if(cib_conn == NULL) {
+        return ENOTCONN;
+    }
+
+    if (host) {
+        rc = resource_clear_node_in_expr(rsc_id, host, cib_conn, cib_options);
+
+        /* rc does not tell us whether the previous operation did anything, only
+         * whether it failed or not.  Thus, as long as it did not fail, we need
+         * to try the second clear method.
+         */
+        if (rc == pcmk_rc_ok) {
+            rc = resource_clear_node_in_location(rsc_id, host, cib_conn,
+                                                 cib_options, clear_ban_constraints,
+                                                 force);
+        }
+
+    } else {
+        GList *n = allnodes;
+
+        /* Iterate over all nodes, attempting to clear the constraint from each.
+         * On the first error, abort.
+         */
+        for(; n; n = n->next) {
+            pe_node_t *target = n->data;
+
+            rc = cli_resource_clear(rsc_id, target->details->uname, NULL,
+                                    cib_conn, cib_options, clear_ban_constraints,
+                                    force);
+            if (rc != pcmk_rc_ok) {
+                break;
+            }
+        }
+    }
+
+    return rc;
+}
+
+static void
+build_clear_xpath_string(GString *buf, const xmlNode *constraint_node,
+                         const char *rsc, const char *node,
+                         bool promoted_role_only)
+{
+    const char *cons_id = ID(constraint_node);
+    const char *cons_rsc = crm_element_value(constraint_node,
+                                             XML_LOC_ATTR_SOURCE);
+    GString *rsc_role_substr = NULL;
+
+    CRM_ASSERT(buf != NULL);
+    g_string_truncate(buf, 0);
+
+    if (!pcmk__starts_with(cons_id, "cli-ban-")
+        && !pcmk__starts_with(cons_id, "cli-prefer-")) {
+        return;
+    }
+
+    g_string_append(buf, "//" XML_CONS_TAG_RSC_LOCATION);
+
+    if ((node != NULL) || (rsc != NULL) || promoted_role_only) {
+        g_string_append_c(buf, '[');
+
+        if (node != NULL) {
+            pcmk__g_strcat(buf, "@" XML_CIB_TAG_NODE "='", node, "'", NULL);
+
+            if (promoted_role_only || (rsc != NULL)) {
+                g_string_append(buf, " and ");
+            }
+        }
+
+        if ((rsc != NULL) && promoted_role_only) {
+            rsc_role_substr = g_string_sized_new(64);
+            pcmk__g_strcat(rsc_role_substr,
+                           "@" XML_LOC_ATTR_SOURCE "='", rsc, "' "
+                           "and @" XML_RULE_ATTR_ROLE "='",
+                           promoted_role_name(), "'", NULL);
+
+        } else if (rsc != NULL) {
+            rsc_role_substr = g_string_sized_new(64);
+            pcmk__g_strcat(rsc_role_substr,
+                           "@" XML_LOC_ATTR_SOURCE "='", rsc, "'", NULL);
+
+        } else if (promoted_role_only) {
+            rsc_role_substr = g_string_sized_new(64);
+            pcmk__g_strcat(rsc_role_substr,
+                           "@" XML_RULE_ATTR_ROLE "='", promoted_role_name(),
+                           "'", NULL);
+        }
+
+        if (rsc_role_substr != NULL) {
+            g_string_append(buf, rsc_role_substr->str);
+        }
+        g_string_append_c(buf, ']');
+    }
+
+    if (node != NULL) {
+        g_string_append(buf, "|//" XML_CONS_TAG_RSC_LOCATION);
+
+        if (rsc_role_substr != NULL) {
+            pcmk__g_strcat(buf, "[", rsc_role_substr, "]", NULL);
+        }
+        pcmk__g_strcat(buf,
+                       "/" XML_TAG_RULE "[" XML_TAG_EXPRESSION
+                       "[@" XML_EXPR_ATTR_ATTRIBUTE "='" CRM_ATTR_UNAME "' "
+                       "and @" XML_EXPR_ATTR_VALUE "='", node, "']]", NULL);
+    }
+
+    g_string_append(buf, "//" PCMK_XE_DATE_EXPRESSION "[@" XML_ATTR_ID "='");
+    if (pcmk__starts_with(cons_id, "cli-ban-")) {
+        pcmk__g_strcat(buf, cons_id, "-lifetime']", NULL);
+
+    } else {    // starts with "cli-prefer-"
+        pcmk__g_strcat(buf,
+                       "cli-prefer-lifetime-end-", cons_rsc, "']", NULL);
+    }
+
+    if (rsc_role_substr != NULL) {
+        g_string_free(rsc_role_substr, TRUE);
+    }
+}
+
+// \return Standard Pacemaker return code
+int
+cli_resource_clear_all_expired(xmlNode *root, cib_t *cib_conn, int cib_options,
+                               const char *rsc, const char *node, gboolean promoted_role_only)
+{
+    GString *buf = NULL;
+    xmlXPathObject *xpathObj = NULL;
+    xmlNode *cib_constraints = NULL;
+    crm_time_t *now = crm_time_new(NULL);
+    int i;
+    int rc = pcmk_rc_ok;
+
+    cib_constraints = pcmk_find_cib_element(root, XML_CIB_TAG_CONSTRAINTS);
+    xpathObj = xpath_search(cib_constraints, "//" XML_CONS_TAG_RSC_LOCATION);
+
+    for (i = 0; i < numXpathResults(xpathObj); i++) {
+        xmlNode *constraint_node = getXpathResult(xpathObj, i);
+        xmlNode *date_expr_node = NULL;
+        crm_time_t *end = NULL;
+
+        if (buf == NULL) {
+            buf = g_string_sized_new(1024);
+        }
+
+        build_clear_xpath_string(buf, constraint_node, rsc, node,
+                                 promoted_role_only);
+        if (buf->len == 0) {
+            continue;
+        }
+
+        date_expr_node = get_xpath_object((const char *) buf->str,
+                                          constraint_node, LOG_DEBUG);
+        if (date_expr_node == NULL) {
+            continue;
+        }
+
+        /* And then finally, see if the date expression is expired.  If so,
+         * clear the constraint.
+         */
+        end = crm_time_new(crm_element_value(date_expr_node, "end"));
+
+        if (crm_time_compare(now, end) == 1) {
+            xmlNode *fragment = NULL;
+            xmlNode *location = NULL;
+
+            fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS);
+            location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
+            crm_xml_set_id(location, "%s", ID(constraint_node));
+            crm_log_xml_info(fragment, "Delete");
+
+            rc = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_CONSTRAINTS,
+                                        fragment, cib_options);
+            rc = pcmk_legacy2rc(rc);
+
+            if (rc != pcmk_rc_ok) {
+                goto done;
+            }
+
+            free_xml(fragment);
+        }
+
+        crm_time_free(end);
+    }
+
+done:
+    if (buf != NULL) {
+        g_string_free(buf, TRUE);
+    }
+    freeXpathObject(xpathObj);
+    crm_time_free(now);
+    return rc;
+}
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
new file mode 100644
index 0000000..c1be53c
--- /dev/null
+++ b/tools/crm_resource_print.c
@@ -0,0 +1,818 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdint.h>
+
+#include <crm_resource.h>
+#include <crm/common/lists_internal.h>
+#include <crm/common/output.h>
+#include <crm/common/results.h>
+
+#define cons_string(x) x?x:"NA"
+static int
+print_constraint(xmlNode *xml_obj, void *userdata)
+{
+    pe_working_set_t *data_set = (pe_working_set_t *) userdata;
+    pcmk__output_t *out = data_set->priv;
+    xmlNode *lifetime = NULL;
+    const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
+
+    if (id == NULL) {
+        return pcmk_rc_ok;
+    }
+
+    // @COMPAT lifetime is deprecated
+    lifetime = first_named_child(xml_obj, "lifetime");
+    if (pe_evaluate_rules(lifetime, NULL, data_set->now, NULL) == FALSE) {
+        return pcmk_rc_ok;
+    }
+
+    if (!pcmk__str_eq(XML_CONS_TAG_RSC_DEPEND, crm_element_name(xml_obj), pcmk__str_casei)) {
+        return pcmk_rc_ok;
+    }
+
+    out->info(out, "Constraint %s %s %s %s %s %s %s",
+              crm_element_name(xml_obj),
+              cons_string(crm_element_value(xml_obj, XML_ATTR_ID)),
+              cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE)),
+              cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET)),
+              cons_string(crm_element_value(xml_obj, XML_RULE_ATTR_SCORE)),
+              cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE)),
+              cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE)));
+
+    return pcmk_rc_ok;
+}
+
+void
+cli_resource_print_cts_constraints(pe_working_set_t * data_set)
+{
+    pcmk__xe_foreach_child(pcmk_find_cib_element(data_set->input, XML_CIB_TAG_CONSTRAINTS),
+                           NULL, print_constraint, data_set);
+}
+
+void
+cli_resource_print_cts(pe_resource_t * rsc, pcmk__output_t *out)
+{
+    const char *host = NULL;
+    bool needs_quorum = TRUE;
+    const char *rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
+    const char *rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
+    const char *rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
+    pe_node_t *node = pe__current_node(rsc);
+
+    if (pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
+        needs_quorum = FALSE;
+    } else {
+        // @TODO check requires in resource meta-data and rsc_defaults
+    }
+
+    if (node != NULL) {
+        host = node->details->uname;
+    }
+
+    out->info(out, "Resource: %s %s %s %s %s %s %s %s %d %lld %#.16llx",
+              crm_element_name(rsc->xml), rsc->id,
+              rsc->clone_name ? rsc->clone_name : rsc->id, rsc->parent ? rsc->parent->id : "NA",
+              rprov ? rprov : "NA", rclass, rtype, host ? host : "NA", needs_quorum, rsc->flags,
+              rsc->flags);
+
+    g_list_foreach(rsc->children, (GFunc) cli_resource_print_cts, out);
+}
+
+// \return Standard Pacemaker return code
+int
+cli_resource_print_operations(const char *rsc_id, const char *host_uname,
+                              bool active, pe_working_set_t * data_set)
+{
+    pcmk__output_t *out = data_set->priv;
+    int rc = pcmk_rc_no_output;
+    GList *ops = find_operations(rsc_id, host_uname, active, data_set);
+
+    if (!ops) {
+        return rc;
+    }
+
+    out->begin_list(out, NULL, NULL, "Resource Operations");
+    rc = pcmk_rc_ok;
+
+    for (GList *lpc = ops; lpc != NULL; lpc = lpc->next) {
+        xmlNode *xml_op = (xmlNode *) lpc->data;
+        out->message(out, "node-and-op", data_set, xml_op);
+    }
+
+    out->end_list(out);
+    return rc;
+}
+
+// \return Standard Pacemaker return code
+int
+cli_resource_print(pe_resource_t *rsc, pe_working_set_t *data_set, bool expanded)
+{
+    pcmk__output_t *out = data_set->priv;
+    uint32_t show_opts = pcmk_show_pending;
+    GList *all = NULL;
+
+    all = g_list_prepend(all, (gpointer) "*");
+
+    out->begin_list(out, NULL, NULL, "Resource Config");
+    out->message(out, crm_map_element_name(rsc->xml), show_opts, rsc, all, all);
+    out->message(out, "resource-config", rsc, !expanded);
+    out->end_list(out);
+
+    g_list_free(all);
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("attribute-list", "pe_resource_t *", "const char *", "const char *")
+static int
+attribute_list_default(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    const char *attr = va_arg(args, char *);
+    const char *value = va_arg(args, const char *);
+
+    if (value != NULL) {
+        out->begin_list(out, NULL, NULL, "Attributes");
+        out->list_item(out, attr, "%s", value);
+        out->end_list(out);
+        return pcmk_rc_ok;
+    } else {
+        out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
+    }
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("agent-status", "int", "const char *", "const char *", "const char *",
+                  "const char *", "const char *", "crm_exit_t", "const char *")
+static int
+agent_status_default(pcmk__output_t *out, va_list args) {
+    int status = va_arg(args, int);
+    const char *action = va_arg(args, const char *);
+    const char *name = va_arg(args, const char *);
+    const char *class = va_arg(args, const char *);
+    const char *provider = va_arg(args, const char *);
+    const char *type = va_arg(args, const char *);
+    crm_exit_t rc = va_arg(args, crm_exit_t);
+    const char *exit_reason = va_arg(args, const char *);
+
+    if (status == PCMK_EXEC_DONE) {
+        /* Operation <action> [for <resource>] (<class>[:<provider>]:<agent>)
+         * returned <exit-code> (<exit-description>[: <exit-reason>])
+         */
+        out->info(out, "Operation %s%s%s (%s%s%s:%s) returned %d (%s%s%s)",
+                  action,
+                  ((name == NULL)? "" : " for "), ((name == NULL)? "" : name),
+                  class,
+                  ((provider == NULL)? "" : ":"),
+                  ((provider == NULL)? "" : provider),
+                  type, (int) rc, services_ocf_exitcode_str((int) rc),
+                  ((exit_reason == NULL)? "" : ": "),
+                  ((exit_reason == NULL)? "" : exit_reason));
+    } else {
+        /* Operation <action> [for <resource>] (<class>[:<provider>]:<agent>)
+         * could not be executed (<execution-status>[: <exit-reason>])
+         */
+        out->err(out,
+                 "Operation %s%s%s (%s%s%s:%s) could not be executed (%s%s%s)",
+                 action,
+                 ((name == NULL)? "" : " for "), ((name == NULL)? "" : name),
+                 class,
+                 ((provider == NULL)? "" : ":"),
+                 ((provider == NULL)? "" : provider),
+                 type, pcmk_exec_status_str(status),
+                 ((exit_reason == NULL)? "" : ": "),
+                 ((exit_reason == NULL)? "" : exit_reason));
+    }
+
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("agent-status", "int", "const char *", "const char *", "const char *",
+                  "const char *", "const char *", "crm_exit_t", "const char *")
+static int
+agent_status_xml(pcmk__output_t *out, va_list args) {
+    int status = va_arg(args, int);
+    const char *action G_GNUC_UNUSED = va_arg(args, const char *);
+    const char *name G_GNUC_UNUSED = va_arg(args, const char *);
+    const char *class G_GNUC_UNUSED = va_arg(args, const char *);
+    const char *provider G_GNUC_UNUSED = va_arg(args, const char *);
+    const char *type G_GNUC_UNUSED = va_arg(args, const char *);
+    crm_exit_t rc = va_arg(args, crm_exit_t);
+    const char *exit_reason = va_arg(args, const char *);
+
+    char *exit_str = pcmk__itoa(rc);
+    char *status_str = pcmk__itoa(status);
+
+    pcmk__output_create_xml_node(out, "agent-status",
+                                 "code", exit_str,
+                                 "message", services_ocf_exitcode_str((int) rc),
+                                 "execution_code", status_str,
+                                 "execution_message", pcmk_exec_status_str(status),
+                                 "reason", exit_reason,
+                                 NULL);
+
+    free(exit_str);
+    free(status_str);
+
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("attribute-list", "pe_resource_t *", "const char *", "const char *")
+static int
+attribute_list_text(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    const char *attr = va_arg(args, char *);
+    const char *value = va_arg(args, const char *);
+
+    if (value != NULL) {
+        pcmk__formatted_printf(out, "%s\n", value);
+        return pcmk_rc_ok;
+    } else {
+        out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
+    }
+    return pcmk_rc_ok;
+}
+PCMK__OUTPUT_ARGS("override", "const char *", "const char *", "const char *")
+static int
+override_default(pcmk__output_t *out, va_list args) {
+    const char *rsc_name = va_arg(args, const char *);
+    const char *name = va_arg(args, const char *);
+    const char *value = va_arg(args, const char *);
+
+    if (rsc_name == NULL) {
+        out->list_item(out, NULL, "Overriding the cluster configuration with '%s' = '%s'",
+                       name, value);
+    } else {
+        out->list_item(out, NULL, "Overriding the cluster configuration for '%s' with '%s' = '%s'",
+                       rsc_name, name, value);
+    }
+
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("override", "const char *", "const char *", "const char *")
+static int
+override_xml(pcmk__output_t *out, va_list args) {
+    const char *rsc_name = va_arg(args, const char *);
+    const char *name = va_arg(args, const char *);
+    const char *value = va_arg(args, const char *);
+
+    xmlNodePtr node = pcmk__output_create_xml_node(out, "override",
+                                                   "name", name,
+                                                   "value", value,
+                                                   NULL);
+
+    if (rsc_name != NULL) {
+        crm_xml_add(node, "rsc", rsc_name);
+    }
+
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("property-list", "pe_resource_t *", "const char *")
+static int
+property_list_default(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    const char *attr = va_arg(args, char *);
+
+    const char *value = crm_element_value(rsc->xml, attr);
+
+    if (value != NULL) {
+        out->begin_list(out, NULL, NULL, "Properties");
+        out->list_item(out, attr, "%s", value);
+        out->end_list(out);
+    }
+
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("property-list", "pe_resource_t *", "const char *")
+static int
+property_list_text(pcmk__output_t *out, va_list args) {
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    const char *attr = va_arg(args, const char *);
+
+    const char *value = crm_element_value(rsc->xml, attr);
+
+    if (value != NULL) {
+        pcmk__formatted_printf(out, "%s\n", value);
+    }
+
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char *", "const char *",
+                  "const char *", "const char *", "const char *", "GHashTable *",
+                  "crm_exit_t", "int", "const char *", "const char *", "const char *")
+static int
+resource_agent_action_default(pcmk__output_t *out, va_list args) {
+    int verbose = va_arg(args, int);
+
+    const char *class = va_arg(args, const char *);
+    const char *provider = va_arg(args, const char *);
+    const char *type = va_arg(args, const char *);
+    const char *rsc_name = va_arg(args, const char *);
+    const char *action = va_arg(args, const char *);
+    GHashTable *overrides = va_arg(args, GHashTable *);
+    crm_exit_t rc = va_arg(args, crm_exit_t);
+    int status = va_arg(args, int);
+    const char *exit_reason = va_arg(args, const char *);
+    const char *stdout_data = va_arg(args, const char *);
+    const char *stderr_data = va_arg(args, const char *);
+
+    if (overrides) {
+        GHashTableIter iter;
+        const char *name = NULL;
+        const char *value = NULL;
+
+        out->begin_list(out, NULL, NULL, "overrides");
+
+        g_hash_table_iter_init(&iter, overrides);
+        while (g_hash_table_iter_next(&iter, (gpointer *) &name, (gpointer *) &value)) {
+            out->message(out, "override", rsc_name, name, value);
+        }
+
+        out->end_list(out);
+    }
+
+    out->message(out, "agent-status", status, action, rsc_name, class, provider,
+                 type, rc, exit_reason);
+
+    /* hide output for validate-all if not in verbose */
+    if (verbose == 0 && pcmk__str_eq(action, "validate-all", pcmk__str_casei)) {
+        return pcmk_rc_ok;
+    }
+
+    if (stdout_data || stderr_data) {
+        xmlNodePtr doc = NULL;
+
+        if (stdout_data != NULL) {
+            doc = string2xml(stdout_data);
+        }
+        if (doc != NULL) {
+            out->output_xml(out, "command", stdout_data);
+            xmlFreeNode(doc);
+        } else {
+            out->subprocess_output(out, rc, stdout_data, stderr_data);
+        }
+    }
+
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char *", "const char *",
+                  "const char *", "const char *", "const char *", "GHashTable *",
+                  "crm_exit_t", "int", "const char *", "const char *", "const char *")
+static int
+resource_agent_action_xml(pcmk__output_t *out, va_list args) {
+    int verbose G_GNUC_UNUSED = va_arg(args, int);
+
+    const char *class = va_arg(args, const char *);
+    const char *provider = va_arg(args, const char *);
+    const char *type = va_arg(args, const char *);
+    const char *rsc_name = va_arg(args, const char *);
+    const char *action = va_arg(args, const char *);
+    GHashTable *overrides = va_arg(args, GHashTable *);
+    crm_exit_t rc = va_arg(args, crm_exit_t);
+    int status = va_arg(args, int);
+    const char *exit_reason = va_arg(args, const char *);
+    const char *stdout_data = va_arg(args, const char *);
+    const char *stderr_data = va_arg(args, const char *);
+
+    xmlNodePtr node = pcmk__output_xml_create_parent(out, "resource-agent-action",
+                                                     "action", action,
+                                                     "class", class,
+                                                     "type", type,
+                                                     NULL);
+
+    if (rsc_name) {
+        crm_xml_add(node, "rsc", rsc_name);
+    }
+
+    if (provider) {
+        crm_xml_add(node, "provider", provider);
+    }
+
+    if (overrides) {
+        GHashTableIter iter;
+        const char *name = NULL;
+        const char *value = NULL;
+
+        out->begin_list(out, NULL, NULL, "overrides");
+
+        g_hash_table_iter_init(&iter, overrides);
+        while (g_hash_table_iter_next(&iter, (gpointer *) &name, (gpointer *) &value)) {
+            out->message(out, "override", rsc_name, name, value);
+        }
+
+        out->end_list(out);
+    }
+
+    out->message(out, "agent-status", status, action, rsc_name, class, provider,
+                 type, rc, exit_reason);
+
+    if (stdout_data || stderr_data) {
+        xmlNodePtr doc = NULL;
+
+        if (stdout_data != NULL) {
+            doc = string2xml(stdout_data);
+        }
+        if (doc != NULL) {
+            out->output_xml(out, "command", stdout_data);
+            xmlFreeNode(doc);
+        } else {
+            out->subprocess_output(out, rc, stdout_data, stderr_data);
+        }
+    }
+
+    pcmk__output_xml_pop_parent(out);
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
+static int
+resource_check_list_default(pcmk__output_t *out, va_list args) {
+    resource_checks_t *checks = va_arg(args, resource_checks_t *);
+
+    const pe_resource_t *parent = pe__const_top_resource(checks->rsc, false);
+
+    if (checks->flags == 0) {
+        return pcmk_rc_no_output;
+    }
+
+    out->begin_list(out, NULL, NULL, "Resource Checks");
+
+    if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
+        out->list_item(out, "check", "Configuration specifies '%s' should remain stopped",
+                       parent->id);
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
+        out->list_item(out, "check", "Configuration specifies '%s' should not be promoted",
+                       parent->id);
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
+        out->list_item(out, "check", "Configuration prevents cluster from stopping or starting unmanaged '%s'",
+                       parent->id);
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_locked)) {
+        out->list_item(out, "check", "'%s' is locked to node %s due to shutdown",
+                       parent->id, checks->lock_node);
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_node_health)) {
+        out->list_item(out, "check",
+                       "'%s' cannot run on unhealthy nodes due to "
+                       PCMK__OPT_NODE_HEALTH_STRATEGY "='%s'",
+                       parent->id,
+                       pe_pref(checks->rsc->cluster->config_hash,
+                               PCMK__OPT_NODE_HEALTH_STRATEGY));
+    }
+
+    out->end_list(out);
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
+static int
+resource_check_list_xml(pcmk__output_t *out, va_list args) {
+    resource_checks_t *checks = va_arg(args, resource_checks_t *);
+
+    const pe_resource_t *parent = pe__const_top_resource(checks->rsc, false);
+
+    xmlNodePtr node = pcmk__output_create_xml_node(out, "check",
+                                                   "id", parent->id,
+                                                   NULL);
+
+    if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
+        pcmk__xe_set_bool_attr(node, "remain_stopped", true);
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
+        pcmk__xe_set_bool_attr(node, "promotable", false);
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
+        pcmk__xe_set_bool_attr(node, "unmanaged", true);
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_locked)) {
+        crm_xml_add(node, "locked-to", checks->lock_node);
+    }
+
+    if (pcmk_is_set(checks->flags, rsc_node_health)) {
+        pcmk__xe_set_bool_attr(node, "unhealthy", true);
+    }
+
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("resource-search-list", "GList *", "const char *")
+static int
+resource_search_list_default(pcmk__output_t *out, va_list args)
+{
+    GList *nodes = va_arg(args, GList *);
+    const char *requested_name = va_arg(args, const char *);
+
+    bool printed = false;
+    int rc = pcmk_rc_no_output;
+
+    if (!out->is_quiet(out) && nodes == NULL) {
+        out->err(out, "resource %s is NOT running", requested_name);
+        return rc;
+    }
+
+    for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
+        node_info_t *ni = (node_info_t *) lpc->data;
+
+        if (!printed) {
+            out->begin_list(out, NULL, NULL, "Nodes");
+            printed = true;
+            rc = pcmk_rc_ok;
+        }
+
+        if (out->is_quiet(out)) {
+            out->list_item(out, "node", "%s", ni->node_name);
+        } else {
+            const char *role_text = "";
+
+            if (ni->promoted) {
+#ifdef PCMK__COMPAT_2_0
+                role_text = " " RSC_ROLE_PROMOTED_LEGACY_S;
+#else
+                role_text = " " RSC_ROLE_PROMOTED_S;
+#endif
+            }
+            out->list_item(out, "node", "resource %s is running on: %s%s",
+                           requested_name, ni->node_name, role_text);
+        }
+    }
+
+    if (printed) {
+        out->end_list(out);
+    }
+
+    return rc;
+}
+
+PCMK__OUTPUT_ARGS("resource-search-list", "GList *", "const char *")
+static int
+resource_search_list_xml(pcmk__output_t *out, va_list args)
+{
+    GList *nodes = va_arg(args, GList *);
+    const char *requested_name = va_arg(args, const char *);
+
+    pcmk__output_xml_create_parent(out, "nodes",
+                                   "resource", requested_name,
+                                   NULL);
+
+    for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
+        node_info_t *ni = (node_info_t *) lpc->data;
+        xmlNodePtr sub_node = pcmk__output_create_xml_text_node(out, "node", ni->node_name);
+
+        if (ni->promoted) {
+            crm_xml_add(sub_node, "state", "promoted");
+        }
+    }
+
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("resource-reasons-list", "GList *", "pe_resource_t *",
+                  "pe_node_t *")
+static int
+resource_reasons_list_default(pcmk__output_t *out, va_list args)
+{
+    GList *resources = va_arg(args, GList *);
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    pe_node_t *node = va_arg(args, pe_node_t *);
+
+    const char *host_uname = (node == NULL)? NULL : node->details->uname;
+
+    out->begin_list(out, NULL, NULL, "Resource Reasons");
+
+    if ((rsc == NULL) && (host_uname == NULL)) {
+        GList *lpc = NULL;
+        GList *hosts = NULL;
+
+        for (lpc = resources; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+            rsc->fns->location(rsc, &hosts, TRUE);
+
+            if (hosts == NULL) {
+                out->list_item(out, "reason", "Resource %s is not running", rsc->id);
+            } else {
+                out->list_item(out, "reason", "Resource %s is running", rsc->id);
+            }
+
+            cli_resource_check(out, rsc, NULL);
+            g_list_free(hosts);
+            hosts = NULL;
+        }
+
+    } else if ((rsc != NULL) && (host_uname != NULL)) {
+        if (resource_is_running_on(rsc, host_uname)) {
+            out->list_item(out, "reason", "Resource %s is running on host %s",
+                           rsc->id, host_uname);
+        } else {
+            out->list_item(out, "reason", "Resource %s is not running on host %s",
+                           rsc->id, host_uname);
+        }
+
+        cli_resource_check(out, rsc, node);
+
+    } else if ((rsc == NULL) && (host_uname != NULL)) {
+        const char* host_uname =  node->details->uname;
+        GList *allResources = node->details->allocated_rsc;
+        GList *activeResources = node->details->running_rsc;
+        GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
+        GList *lpc = NULL;
+
+        for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+            out->list_item(out, "reason", "Resource %s is running on host %s",
+                           rsc->id, host_uname);
+            cli_resource_check(out, rsc, node);
+        }
+
+        for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+            out->list_item(out, "reason", "Resource %s is assigned to host %s but not running",
+                           rsc->id, host_uname);
+            cli_resource_check(out, rsc, node);
+        }
+
+        g_list_free(allResources);
+        g_list_free(activeResources);
+        g_list_free(unactiveResources);
+
+    } else if ((rsc != NULL) && (host_uname == NULL)) {
+        GList *hosts = NULL;
+
+        rsc->fns->location(rsc, &hosts, TRUE);
+        out->list_item(out, "reason", "Resource %s is %srunning",
+                       rsc->id, (hosts? "" : "not "));
+        cli_resource_check(out, rsc, NULL);
+        g_list_free(hosts);
+    }
+
+    out->end_list(out);
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("resource-reasons-list", "GList *", "pe_resource_t *",
+                  "pe_node_t *")
+static int
+resource_reasons_list_xml(pcmk__output_t *out, va_list args)
+{
+    GList *resources = va_arg(args, GList *);
+    pe_resource_t *rsc = va_arg(args, pe_resource_t *);
+    pe_node_t *node = va_arg(args, pe_node_t *);
+
+    const char *host_uname = (node == NULL)? NULL : node->details->uname;
+
+    xmlNodePtr xml_node = pcmk__output_xml_create_parent(out, "reason", NULL);
+
+    if ((rsc == NULL) && (host_uname == NULL)) {
+        GList *lpc = NULL;
+        GList *hosts = NULL;
+
+        pcmk__output_xml_create_parent(out, "resources", NULL);
+
+        for (lpc = resources; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+
+            rsc->fns->location(rsc, &hosts, TRUE);
+
+            pcmk__output_xml_create_parent(out, "resource",
+                                           "id", rsc->id,
+                                           "running", pcmk__btoa(hosts != NULL),
+                                           NULL);
+
+            cli_resource_check(out, rsc, NULL);
+            pcmk__output_xml_pop_parent(out);
+            g_list_free(hosts);
+            hosts = NULL;
+        }
+
+        pcmk__output_xml_pop_parent(out);
+
+    } else if ((rsc != NULL) && (host_uname != NULL)) {
+        if (resource_is_running_on(rsc, host_uname)) {
+            crm_xml_add(xml_node, "running_on", host_uname);
+        }
+
+        cli_resource_check(out, rsc, node);
+
+    } else if ((rsc == NULL) && (host_uname != NULL)) {
+        const char* host_uname =  node->details->uname;
+        GList *allResources = node->details->allocated_rsc;
+        GList *activeResources = node->details->running_rsc;
+        GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
+        GList *lpc = NULL;
+
+        pcmk__output_xml_create_parent(out, "resources", NULL);
+
+        for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+
+            pcmk__output_xml_create_parent(out, "resource",
+                                           "id", rsc->id,
+                                           "running", "true",
+                                           "host", host_uname,
+                                           NULL);
+
+            cli_resource_check(out, rsc, node);
+            pcmk__output_xml_pop_parent(out);
+        }
+
+        for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
+            pe_resource_t *rsc = (pe_resource_t *) lpc->data;
+
+            pcmk__output_xml_create_parent(out, "resource",
+                                           "id", rsc->id,
+                                           "running", "false",
+                                           "host", host_uname,
+                                           NULL);
+
+            cli_resource_check(out, rsc, node);
+            pcmk__output_xml_pop_parent(out);
+        }
+
+        pcmk__output_xml_pop_parent(out);
+        g_list_free(allResources);
+        g_list_free(activeResources);
+        g_list_free(unactiveResources);
+
+    } else if ((rsc != NULL) && (host_uname == NULL)) {
+        GList *hosts = NULL;
+
+        rsc->fns->location(rsc, &hosts, TRUE);
+        crm_xml_add(xml_node, "running", pcmk__btoa(hosts != NULL));
+        cli_resource_check(out, rsc, NULL);
+        g_list_free(hosts);
+    }
+
+    return pcmk_rc_ok;
+}
+
+static void
+add_resource_name(pe_resource_t *rsc, pcmk__output_t *out) {
+    if (rsc->children == NULL) {
+        out->list_item(out, "resource", "%s", rsc->id);
+    } else {
+        g_list_foreach(rsc->children, (GFunc) add_resource_name, out);
+    }
+}
+
+PCMK__OUTPUT_ARGS("resource-names-list", "GList *")
+static int
+resource_names(pcmk__output_t *out, va_list args) {
+    GList *resources = va_arg(args, GList *);
+
+    if (resources == NULL) {
+        out->err(out, "NO resources configured\n");
+        return pcmk_rc_no_output;
+    }
+
+    out->begin_list(out, NULL, NULL, "Resource Names");
+    g_list_foreach(resources, (GFunc) add_resource_name, out);
+    out->end_list(out);
+    return pcmk_rc_ok;
+}
+
+static pcmk__message_entry_t fmt_functions[] = {
+    { "agent-status", "default", agent_status_default },
+    { "agent-status", "xml", agent_status_xml },
+    { "attribute-list", "default", attribute_list_default },
+    { "attribute-list", "text", attribute_list_text },
+    { "override", "default", override_default },
+    { "override", "xml", override_xml },
+    { "property-list", "default", property_list_default },
+    { "property-list", "text", property_list_text },
+    { "resource-agent-action", "default", resource_agent_action_default },
+    { "resource-agent-action", "xml", resource_agent_action_xml },
+    { "resource-check-list", "default", resource_check_list_default },
+    { "resource-check-list", "xml", resource_check_list_xml },
+    { "resource-search-list", "default", resource_search_list_default },
+    { "resource-search-list", "xml", resource_search_list_xml },
+    { "resource-reasons-list", "default", resource_reasons_list_default },
+    { "resource-reasons-list", "xml", resource_reasons_list_xml },
+    { "resource-names-list", "default", resource_names },
+
+    { NULL, NULL, NULL }
+};
+
+void
+crm_resource_register_messages(pcmk__output_t *out) {
+    pcmk__register_messages(out, fmt_functions);
+}
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
new file mode 100644
index 0000000..f25dbbc
--- /dev/null
+++ b/tools/crm_resource_runtime.c
@@ -0,0 +1,2178 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm_resource.h>
+#include <crm/common/ipc_attrd_internal.h>
+#include <crm/common/ipc_controld.h>
+#include <crm/common/lists_internal.h>
+#include <crm/services_internal.h>
+
+static GList *
+build_node_info_list(const pe_resource_t *rsc)
+{
+    GList *retval = NULL;
+
+    for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
+        const pe_resource_t *child = (const pe_resource_t *) iter->data;
+
+        for (const GList *iter2 = child->running_on;
+             iter2 != NULL; iter2 = iter2->next) {
+
+            const pe_node_t *node = (const pe_node_t *) iter2->data;
+            node_info_t *ni = calloc(1, sizeof(node_info_t));
+
+            ni->node_name = node->details->uname;
+            ni->promoted = pcmk_is_set(rsc->flags, pe_rsc_promotable) &&
+                           child->fns->state(child, TRUE) == RSC_ROLE_PROMOTED;
+
+            retval = g_list_prepend(retval, ni);
+        }
+    }
+
+    return retval;
+}
+
+GList *
+cli_resource_search(pe_resource_t *rsc, const char *requested_name,
+                    pe_working_set_t *data_set)
+{
+    GList *retval = NULL;
+    const pe_resource_t *parent = pe__const_top_resource(rsc, false);
+
+    if (pe_rsc_is_clone(rsc)) {
+        retval = build_node_info_list(rsc);
+
+    /* The anonymous clone children's common ID is supplied */
+    } else if (pe_rsc_is_clone(parent)
+               && !pcmk_is_set(rsc->flags, pe_rsc_unique)
+               && rsc->clone_name
+               && pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_casei)
+               && !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {
+
+        retval = build_node_info_list(parent);
+
+    } else if (rsc->running_on != NULL) {
+        for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
+            pe_node_t *node = (pe_node_t *) iter->data;
+            node_info_t *ni = calloc(1, sizeof(node_info_t));
+            ni->node_name = node->details->uname;
+            ni->promoted = (rsc->fns->state(rsc, TRUE) == RSC_ROLE_PROMOTED);
+
+            retval = g_list_prepend(retval, ni);
+        }
+    }
+
+    return retval;
+}
+
+// \return Standard Pacemaker return code
+static int
+find_resource_attr(pcmk__output_t *out, cib_t * the_cib, const char *attr,
+                   const char *rsc, const char *attr_set_type, const char *set_name,
+                   const char *attr_id, const char *attr_name, char **value)
+{
+    int rc = pcmk_rc_ok;
+    xmlNode *xml_search = NULL;
+    GString *xpath = NULL;
+    const char *xpath_base = NULL;
+
+    if(value) {
+        *value = NULL;
+    }
+
+    if(the_cib == NULL) {
+        return ENOTCONN;
+    }
+
+    xpath_base = pcmk_cib_xpath_for(XML_CIB_TAG_RESOURCES);
+    if (xpath_base == NULL) {
+        crm_err(XML_CIB_TAG_RESOURCES " CIB element not known (bug?)");
+        return ENOMSG;
+    }
+
+    xpath = g_string_sized_new(1024);
+    pcmk__g_strcat(xpath,
+                   xpath_base, "//*[@" XML_ATTR_ID "=\"", rsc, "\"]", NULL);
+
+    if (attr_set_type != NULL) {
+        pcmk__g_strcat(xpath, "/", attr_set_type, NULL);
+        if (set_name != NULL) {
+            pcmk__g_strcat(xpath, "[@" XML_ATTR_ID "=\"", set_name, "\"]",
+                           NULL);
+        }
+    }
+
+    g_string_append(xpath, "//" XML_CIB_TAG_NVPAIR "[");
+    if (attr_id != NULL) {
+        pcmk__g_strcat(xpath, "@" XML_ATTR_ID "=\"", attr_id, "\"", NULL);
+    }
+
+    if (attr_name != NULL) {
+        if (attr_id != NULL) {
+            g_string_append(xpath, " and ");
+        }
+        pcmk__g_strcat(xpath, "@" XML_NVPAIR_ATTR_NAME "=\"", attr_name, "\"",
+                       NULL);
+    }
+    g_string_append_c(xpath, ']');
+
+    rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
+                              cib_sync_call | cib_scope_local | cib_xpath);
+    rc = pcmk_legacy2rc(rc);
+
+    if (rc != pcmk_rc_ok) {
+        goto done;
+    }
+
+    crm_log_xml_debug(xml_search, "Match");
+    if (xml_has_children(xml_search)) {
+        xmlNode *child = NULL;
+
+        rc = ENOTUNIQ;
+        out->info(out, "Multiple attributes match name=%s", attr_name);
+
+        for (child = pcmk__xml_first_child(xml_search); child != NULL;
+             child = pcmk__xml_next(child)) {
+            out->info(out, "  Value: %s \t(id=%s)",
+                      crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
+        }
+
+        out->spacer(out);
+
+    } else if(value) {
+        pcmk__str_update(value, crm_element_value(xml_search, attr));
+    }
+
+  done:
+    g_string_free(xpath, TRUE);
+    free_xml(xml_search);
+    return rc;
+}
+
+/* PRIVATE. Use the find_matching_attr_resources instead. */
+static void
+find_matching_attr_resources_recursive(pcmk__output_t *out, GList/* <pe_resource_t*> */ ** result,
+                                       pe_resource_t * rsc, const char * rsc_id,
+                                       const char * attr_set, const char * attr_set_type,
+                                       const char * attr_id, const char * attr_name,
+                                       cib_t * cib, const char * cmd, int depth)
+{
+    int rc = pcmk_rc_ok;
+    char *lookup_id = clone_strip(rsc->id);
+    char *local_attr_id = NULL;
+
+    /* visit the children */
+    for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
+        find_matching_attr_resources_recursive(out, result, (pe_resource_t*)gIter->data,
+                                               rsc_id, attr_set, attr_set_type,
+                                               attr_id, attr_name, cib, cmd, depth+1);
+        /* do it only once for clones */
+        if(pe_clone == rsc->variant) {
+            break;
+        }
+    }
+
+    rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
+                            attr_set, attr_id, attr_name, &local_attr_id);
+    /* Post-order traversal. 
+     * The root is always on the list and it is the last item. */
+    if((0 == depth) || (pcmk_rc_ok == rc)) {
+        /* push the head */
+        *result = g_list_append(*result, rsc);
+    }
+
+    free(local_attr_id);
+    free(lookup_id);
+}
+
+
+/* The result is a linearized pre-ordered tree of resources. */
+static GList/*<pe_resource_t*>*/ *
+find_matching_attr_resources(pcmk__output_t *out, pe_resource_t * rsc,
+                             const char * rsc_id, const char * attr_set,
+                             const char * attr_set_type, const char * attr_id,
+                             const char * attr_name, cib_t * cib, const char * cmd,
+                             gboolean force)
+{
+    int rc = pcmk_rc_ok;
+    char *lookup_id = NULL;
+    char *local_attr_id = NULL;
+    GList * result = NULL;
+    /* If --force is used, update only the requested resource (clone or primitive).
+     * Otherwise, if the primitive has the attribute, use that.
+     * Otherwise use the clone. */
+    if(force == TRUE) {
+        return g_list_append(result, rsc);
+    }
+    if(rsc->parent && pe_clone == rsc->parent->variant) {
+        int rc = pcmk_rc_ok;
+        char *local_attr_id = NULL;
+        rc = find_resource_attr(out, cib, XML_ATTR_ID, rsc_id, attr_set_type,
+                                attr_set, attr_id, attr_name, &local_attr_id);
+        free(local_attr_id);
+
+        if(rc != pcmk_rc_ok) {
+            rsc = rsc->parent;
+            out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
+                      cmd, attr_name, rsc->id, rsc_id);
+        }
+        return g_list_append(result, rsc);
+    } else if(rsc->parent == NULL && rsc->children && pe_clone == rsc->variant) {
+        pe_resource_t *child = rsc->children->data;
+
+        if(child->variant == pe_native) {
+            lookup_id = clone_strip(child->id); /* Could be a cloned group! */
+            rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
+                                    attr_set, attr_id, attr_name, &local_attr_id);
+
+            if(rc == pcmk_rc_ok) {
+                rsc = child;
+                out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
+                          attr_name, lookup_id, cmd, rsc_id);
+            }
+
+            free(local_attr_id);
+            free(lookup_id);
+        }
+        return g_list_append(result, rsc);
+    }
+    /* If the resource is a group ==> children inherit the attribute if defined. */
+    find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
+                                           attr_set_type, attr_id, attr_name,
+                                           cib, cmd, 0);
+    return result;
+}
+
+// \return Standard Pacemaker return code
+int
+cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
+                              const char *attr_set, const char *attr_set_type,
+                              const char *attr_id, const char *attr_name,
+                              const char *attr_value, gboolean recursive,
+                              cib_t *cib, int cib_options, gboolean force)
+{
+    pcmk__output_t *out = rsc->cluster->priv;
+    int rc = pcmk_rc_ok;
+
+    char *found_attr_id = NULL;
+
+    GList/*<pe_resource_t*>*/ *resources = NULL;
+    const char *top_id = pe__const_top_resource(rsc, false)->id;
+
+    if ((attr_id == NULL) && !force) {
+        find_resource_attr(out, cib, XML_ATTR_ID, top_id, NULL, NULL, NULL,
+                           attr_name, NULL);
+    }
+
+    if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
+        if (!force) {
+            rc = find_resource_attr(out, cib, XML_ATTR_ID, top_id,
+                                    XML_TAG_META_SETS, attr_set, attr_id,
+                                    attr_name, &found_attr_id);
+            if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
+                out->err(out,
+                         "WARNING: There is already a meta attribute "
+                         "for '%s' called '%s' (id=%s)",
+                         top_id, attr_name, found_attr_id);
+                out->err(out,
+                         "         Delete '%s' first or use the force option "
+                         "to override", found_attr_id);
+            }
+            free(found_attr_id);
+            if (rc == pcmk_rc_ok) {
+                return ENOTUNIQ;
+            }
+        }
+        resources = g_list_append(resources, rsc);
+
+    } else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
+        crm_xml_add(rsc->xml, attr_name, attr_value);
+        CRM_ASSERT(cib != NULL);
+        rc = cib->cmds->replace(cib, XML_CIB_TAG_RESOURCES, rsc->xml,
+                                cib_options);
+        rc = pcmk_legacy2rc(rc);
+        if (rc == pcmk_rc_ok) {
+            out->info(out, "Set attribute: name=%s value=%s",
+                      attr_name, attr_value);
+        }
+        return rc;
+
+    } else {
+        resources = find_matching_attr_resources(out, rsc, requested_name,
+                                                 attr_set, attr_set_type,
+                                                 attr_id, attr_name, cib,
+                                                 "update", force);
+    }
+
+    /* If the user specified attr_set or attr_id, the intent is to modify a
+     * single resource, which will be the last item in the list.
+     */
+    if ((attr_set != NULL) || (attr_id != NULL)) {
+        GList *last = g_list_last(resources);
+
+        resources = g_list_remove_link(resources, last);
+        g_list_free(resources);
+        resources = last;
+    }
+
+    for (GList *iter = resources; iter != NULL; iter = iter->next) {
+        char *lookup_id = NULL;
+        char *local_attr_set = NULL;
+        const char *rsc_attr_id = attr_id;
+        const char *rsc_attr_set = attr_set;
+
+        xmlNode *xml_top = NULL;
+        xmlNode *xml_obj = NULL;
+        found_attr_id = NULL;
+
+        rsc = (pe_resource_t *) iter->data;
+
+        lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
+        rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
+                                attr_set, attr_id, attr_name, &found_attr_id);
+
+        switch (rc) {
+            case pcmk_rc_ok:
+                crm_debug("Found a match for name=%s: id=%s",
+                          attr_name, found_attr_id);
+                rsc_attr_id = found_attr_id;
+                break;
+
+            case ENXIO:
+                if (rsc_attr_set == NULL) {
+                    local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
+                                                       attr_set_type);
+                    rsc_attr_set = local_attr_set;
+                }
+                if (rsc_attr_id == NULL) {
+                    found_attr_id = crm_strdup_printf("%s-%s",
+                                                      rsc_attr_set, attr_name);
+                    rsc_attr_id = found_attr_id;
+                }
+
+                xml_top = create_xml_node(NULL, crm_element_name(rsc->xml));
+                crm_xml_add(xml_top, XML_ATTR_ID, lookup_id);
+
+                xml_obj = create_xml_node(xml_top, attr_set_type);
+                crm_xml_add(xml_obj, XML_ATTR_ID, rsc_attr_set);
+                break;
+
+            default:
+                free(lookup_id);
+                free(found_attr_id);
+                g_list_free(resources);
+                return rc;
+        }
+
+        xml_obj = crm_create_nvpair_xml(xml_obj, rsc_attr_id, attr_name,
+                                        attr_value);
+        if (xml_top == NULL) {
+            xml_top = xml_obj;
+        }
+
+        crm_log_xml_debug(xml_top, "Update");
+
+        rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top,
+                               cib_options);
+        rc = pcmk_legacy2rc(rc);
+        if (rc == pcmk_rc_ok) {
+            out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s",
+                      lookup_id, found_attr_id,
+                      ((rsc_attr_set == NULL)? "" : " set="),
+                      pcmk__s(rsc_attr_set, ""),
+                      ((attr_name == NULL)? "" : " name="),
+                      pcmk__s(attr_name, ""), attr_value);
+        }
+
+        free_xml(xml_top);
+
+        free(lookup_id);
+        free(found_attr_id);
+        free(local_attr_set);
+
+        if (recursive
+            && pcmk__str_eq(attr_set_type, XML_TAG_META_SETS,
+                            pcmk__str_casei)) {
+            GList *lpc = NULL;
+            static bool need_init = true;
+
+            if (need_init) {
+                need_init = false;
+                pcmk__unpack_constraints(rsc->cluster);
+                pe__clear_resource_flags_on_all(rsc->cluster,
+                                                pe_rsc_detect_loop);
+            }
+
+            /* We want to set the attribute only on resources explicitly
+             * colocated with this one, so we use rsc->rsc_cons_lhs directly
+             * rather than the with_this_colocations() method.
+             */
+            pe__set_resource_flags(rsc, pe_rsc_detect_loop);
+            for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
+                pcmk__colocation_t *cons = (pcmk__colocation_t *) lpc->data;
+
+                crm_debug("Checking %s %d", cons->id, cons->score);
+                if (!pcmk_is_set(cons->dependent->flags, pe_rsc_detect_loop)
+                    && (cons->score > 0)) {
+                    crm_debug("Setting %s=%s for dependent resource %s",
+                              attr_name, attr_value, cons->dependent->id);
+                    cli_resource_update_attribute(cons->dependent,
+                                                  cons->dependent->id, NULL,
+                                                  attr_set_type, NULL,
+                                                  attr_name, attr_value,
+                                                  recursive, cib, cib_options,
+                                                  force);
+                }
+            }
+        }
+    }
+    g_list_free(resources);
+    return rc;
+}
+
+// \return Standard Pacemaker return code
+int
+cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name,
+                              const char *attr_set, const char *attr_set_type,
+                              const char *attr_id, const char *attr_name,
+                              cib_t *cib, int cib_options, gboolean force)
+{
+    pcmk__output_t *out = rsc->cluster->priv;
+    int rc = pcmk_rc_ok;
+    GList/*<pe_resource_t*>*/ *resources = NULL;
+
+    if ((attr_id == NULL) && !force) {
+        find_resource_attr(out, cib, XML_ATTR_ID,
+                           pe__const_top_resource(rsc, false)->id, NULL,
+                           NULL, NULL, attr_name, NULL);
+    }
+
+    if (pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
+        resources = find_matching_attr_resources(out, rsc, requested_name,
+                                                 attr_set, attr_set_type,
+                                                 attr_id, attr_name, cib,
+                                                 "delete", force);
+
+    } else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
+        xml_remove_prop(rsc->xml, attr_name);
+        CRM_ASSERT(cib != NULL);
+        rc = cib->cmds->replace(cib, XML_CIB_TAG_RESOURCES, rsc->xml,
+                                cib_options);
+        rc = pcmk_legacy2rc(rc);
+        if (rc == pcmk_rc_ok) {
+            out->info(out, "Deleted attribute: %s", attr_name);
+        }
+        return rc;
+
+    } else {
+        resources = g_list_append(resources, rsc);
+    }
+
+    for (GList *iter = resources; iter != NULL; iter = iter->next) {
+        char *lookup_id = NULL;
+        xmlNode *xml_obj = NULL;
+        char *found_attr_id = NULL;
+        const char *rsc_attr_id = attr_id;
+
+        rsc = (pe_resource_t *) iter->data;
+
+        lookup_id = clone_strip(rsc->id);
+        rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
+                                attr_set, attr_id, attr_name, &found_attr_id);
+        switch (rc) {
+            case pcmk_rc_ok:
+                break;
+
+            case ENXIO:
+                free(lookup_id);
+                rc = pcmk_rc_ok;
+                continue;
+
+            default:
+                free(lookup_id);
+                g_list_free(resources);
+                return rc;
+        }
+
+        if (rsc_attr_id == NULL) {
+            rsc_attr_id = found_attr_id;
+        }
+
+        xml_obj = crm_create_nvpair_xml(NULL, rsc_attr_id, attr_name, NULL);
+        crm_log_xml_debug(xml_obj, "Delete");
+
+        CRM_ASSERT(cib);
+        rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj,
+                               cib_options);
+        rc = pcmk_legacy2rc(rc);
+
+        if (rc == pcmk_rc_ok) {
+            out->info(out, "Deleted '%s' option: id=%s%s%s%s%s",
+                      lookup_id, found_attr_id,
+                      ((attr_set == NULL)? "" : " set="),
+                      pcmk__s(attr_set, ""),
+                      ((attr_name == NULL)? "" : " name="),
+                      pcmk__s(attr_name, ""));
+        }
+
+        free(lookup_id);
+        free_xml(xml_obj);
+        free(found_attr_id);
+    }
+    g_list_free(resources);
+    return rc;
+}
+
+// \return Standard Pacemaker return code
+static int
+send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
+                const char *host_uname, const char *rsc_id, pe_working_set_t *data_set)
+{
+    pcmk__output_t *out = data_set->priv;
+    const char *router_node = host_uname;
+    const char *rsc_api_id = NULL;
+    const char *rsc_long_id = NULL;
+    const char *rsc_class = NULL;
+    const char *rsc_provider = NULL;
+    const char *rsc_type = NULL;
+    bool cib_only = false;
+    pe_resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
+
+    if (rsc == NULL) {
+        out->err(out, "Resource %s not found", rsc_id);
+        return ENXIO;
+
+    } else if (rsc->variant != pe_native) {
+        out->err(out, "We can only process primitive resources, not %s", rsc_id);
+        return EINVAL;
+    }
+
+    rsc_class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
+    rsc_provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
+    rsc_type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
+    if ((rsc_class == NULL) || (rsc_type == NULL)) {
+        out->err(out, "Resource %s does not have a class and type", rsc_id);
+        return EINVAL;
+    }
+
+    {
+        pe_node_t *node = pe_find_node(data_set->nodes, host_uname);
+
+        if (node == NULL) {
+            out->err(out, "Node %s not found", host_uname);
+            return pcmk_rc_node_unknown;
+        }
+
+        if (!(node->details->online)) {
+            if (do_fail_resource) {
+                out->err(out, "Node %s is not online", host_uname);
+                return ENOTCONN;
+            } else {
+                cib_only = true;
+            }
+        }
+        if (!cib_only && pe__is_guest_or_remote_node(node)) {
+            node = pe__current_node(node->details->remote_rsc);
+            if (node == NULL) {
+                out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
+                         host_uname);
+                return ENOTCONN;
+            }
+            router_node = node->details->uname;
+        }
+    }
+
+    if (rsc->clone_name) {
+        rsc_api_id = rsc->clone_name;
+        rsc_long_id = rsc->id;
+    } else {
+        rsc_api_id = rsc->id;
+    }
+    if (do_fail_resource) {
+        return pcmk_controld_api_fail(controld_api, host_uname, router_node,
+                                      rsc_api_id, rsc_long_id,
+                                      rsc_class, rsc_provider, rsc_type);
+    } else {
+        return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
+                                         rsc_api_id, rsc_long_id, rsc_class,
+                                         rsc_provider, rsc_type, cib_only);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Get resource name as used in failure-related node attributes
+ *
+ * \param[in] rsc  Resource to check
+ *
+ * \return Newly allocated string containing resource's fail name
+ * \note The caller is responsible for freeing the result.
+ */
+static inline char *
+rsc_fail_name(const pe_resource_t *rsc)
+{
+    const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);
+
+    return pcmk_is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
+}
+
+// \return Standard Pacemaker return code
+static int
+clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
+                  const char *rsc_id, pe_working_set_t *data_set)
+{
+    int rc = pcmk_rc_ok;
+
+    /* Erase the resource's entire LRM history in the CIB, even if we're only
+     * clearing a single operation's fail count. If we erased only entries for a
+     * single operation, we might wind up with a wrong idea of the current
+     * resource state, and we might not re-probe the resource.
+     */
+    rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, data_set);
+    if (rc != pcmk_rc_ok) {
+        return rc;
+    }
+
+    crm_trace("Processing %d mainloop inputs",
+              pcmk_controld_api_replies_expected(controld_api));
+    while (g_main_context_iteration(NULL, FALSE)) {
+        crm_trace("Processed mainloop input, %d still remaining",
+                  pcmk_controld_api_replies_expected(controld_api));
+    }
+    return rc;
+}
+
+// \return Standard Pacemaker return code
+static int
+clear_rsc_failures(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
+                   const char *node_name, const char *rsc_id, const char *operation,
+                   const char *interval_spec, pe_working_set_t *data_set)
+{
+    int rc = pcmk_rc_ok;
+    const char *failed_value = NULL;
+    const char *failed_id = NULL;
+    const char *interval_ms_s = NULL;
+    GHashTable *rscs = NULL;
+    GHashTableIter iter;
+
+    /* Create a hash table to use as a set of resources to clean. This lets us
+     * clean each resource only once (per node) regardless of how many failed
+     * operations it has.
+     */
+    rscs = pcmk__strkey_table(NULL, NULL);
+
+    // Normalize interval to milliseconds for comparison to history entry
+    if (operation) {
+        interval_ms_s = crm_strdup_printf("%u",
+                                          crm_parse_interval_spec(interval_spec));
+    }
+
+    for (xmlNode *xml_op = pcmk__xml_first_child(data_set->failed);
+         xml_op != NULL;
+         xml_op = pcmk__xml_next(xml_op)) {
+
+        failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
+        if (failed_id == NULL) {
+            // Malformed history entry, should never happen
+            continue;
+        }
+
+        // No resource specified means all resources match
+        if (rsc_id) {
+            pe_resource_t *fail_rsc = pe_find_resource_with_flags(data_set->resources,
+                                                                  failed_id,
+                                                                  pe_find_renamed|pe_find_anon);
+
+            if (!fail_rsc || !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
+                continue;
+            }
+        }
+
+        // Host name should always have been provided by this point
+        failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
+        if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
+            continue;
+        }
+
+        // No operation specified means all operations match
+        if (operation) {
+            failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
+            if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
+                continue;
+            }
+
+            // Interval (if operation was specified) defaults to 0 (not all)
+            failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
+            if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
+                continue;
+            }
+        }
+
+        g_hash_table_add(rscs, (gpointer) failed_id);
+    }
+
+    g_hash_table_iter_init(&iter, rscs);
+    while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
+        crm_debug("Erasing failures of %s on %s", failed_id, node_name);
+        rc = clear_rsc_history(controld_api, node_name, failed_id, data_set);
+        if (rc != pcmk_rc_ok) {
+            return rc;
+        }
+    }
+    g_hash_table_destroy(rscs);
+    return rc;
+}
+
+// \return Standard Pacemaker return code
+static int
+clear_rsc_fail_attrs(const pe_resource_t *rsc, const char *operation,
+                     const char *interval_spec, const pe_node_t *node)
+{
+    int rc = pcmk_rc_ok;
+    int attr_options = pcmk__node_attr_none;
+    char *rsc_name = rsc_fail_name(rsc);
+
+    if (pe__is_guest_or_remote_node(node)) {
+        attr_options |= pcmk__node_attr_remote;
+    }
+
+    rc = pcmk__attrd_api_clear_failures(NULL, node->details->uname, rsc_name,
+                                        operation, interval_spec, NULL,
+                                        attr_options);
+    free(rsc_name);
+    return rc;
+}
+
+// \return Standard Pacemaker return code
+int
+cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
+                    const pe_resource_t *rsc, const char *operation,
+                    const char *interval_spec, bool just_failures,
+                    pe_working_set_t *data_set, gboolean force)
+{
+    pcmk__output_t *out = data_set->priv;
+    int rc = pcmk_rc_ok;
+    pe_node_t *node = NULL;
+
+    if (rsc == NULL) {
+        return ENXIO;
+
+    } else if (rsc->children) {
+
+        for (const GList *lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
+            const pe_resource_t *child = (const pe_resource_t *) lpc->data;
+
+            rc = cli_resource_delete(controld_api, host_uname, child, operation,
+                                     interval_spec, just_failures, data_set, force);
+            if (rc != pcmk_rc_ok) {
+                return rc;
+            }
+        }
+        return pcmk_rc_ok;
+
+    } else if (host_uname == NULL) {
+        GList *lpc = NULL;
+        GList *nodes = g_hash_table_get_values(rsc->known_on);
+
+        if(nodes == NULL && force) {
+            nodes = pcmk__copy_node_list(data_set->nodes, false);
+
+        } else if(nodes == NULL && rsc->exclusive_discover) {
+            GHashTableIter iter;
+            pe_node_t *node = NULL;
+
+            g_hash_table_iter_init(&iter, rsc->allowed_nodes);
+            while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
+                if(node->weight >= 0) {
+                    nodes = g_list_prepend(nodes, node);
+                }
+            }
+
+        } else if(nodes == NULL) {
+            nodes = g_hash_table_get_values(rsc->allowed_nodes);
+        }
+
+        for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
+            node = (pe_node_t *) lpc->data;
+
+            if (node->details->online) {
+                rc = cli_resource_delete(controld_api, node->details->uname, rsc,
+                                         operation, interval_spec, just_failures,
+                                         data_set, force);
+            }
+            if (rc != pcmk_rc_ok) {
+                g_list_free(nodes);
+                return rc;
+            }
+        }
+
+        g_list_free(nodes);
+        return pcmk_rc_ok;
+    }
+
+    node = pe_find_node(data_set->nodes, host_uname);
+
+    if (node == NULL) {
+        out->err(out, "Unable to clean up %s because node %s not found",
+                 rsc->id, host_uname);
+        return ENODEV;
+    }
+
+    if (!node->details->rsc_discovery_enabled) {
+        out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
+                 rsc->id, host_uname);
+        return EOPNOTSUPP;
+    }
+
+    if (controld_api == NULL) {
+        out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
+                 rsc->id, host_uname);
+        return pcmk_rc_ok;
+    }
+
+    rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
+    if (rc != pcmk_rc_ok) {
+        out->err(out, "Unable to clean up %s failures on %s: %s",
+                 rsc->id, host_uname, pcmk_rc_str(rc));
+        return rc;
+    }
+
+    if (just_failures) {
+        rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
+                                interval_spec, data_set);
+    } else {
+        rc = clear_rsc_history(controld_api, host_uname, rsc->id, data_set);
+    }
+    if (rc != pcmk_rc_ok) {
+        out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
+                 rsc->id, host_uname, pcmk_strerror(rc));
+    } else {
+        out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
+    }
+    return rc;
+}
+
+// \return Standard Pacemaker return code
+int
+cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
+                const char *operation, const char *interval_spec,
+                pe_working_set_t *data_set)
+{
+    pcmk__output_t *out = data_set->priv;
+    int rc = pcmk_rc_ok;
+    int attr_options = pcmk__node_attr_none;
+    const char *display_name = node_name? node_name : "all nodes";
+
+    if (controld_api == NULL) {
+        out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
+                  display_name);
+        return rc;
+    }
+
+    if (node_name) {
+        pe_node_t *node = pe_find_node(data_set->nodes, node_name);
+
+        if (node == NULL) {
+            out->err(out, "Unknown node: %s", node_name);
+            return ENXIO;
+        }
+        if (pe__is_guest_or_remote_node(node)) {
+            attr_options |= pcmk__node_attr_remote;
+        }
+    }
+
+    rc = pcmk__attrd_api_clear_failures(NULL, node_name, NULL, operation,
+                                        interval_spec, NULL, attr_options);
+    if (rc != pcmk_rc_ok) {
+        out->err(out, "Unable to clean up all failures on %s: %s",
+                 display_name, pcmk_rc_str(rc));
+        return rc;
+    }
+
+    if (node_name) {
+        rc = clear_rsc_failures(out, controld_api, node_name, NULL,
+                                operation, interval_spec, data_set);
+        if (rc != pcmk_rc_ok) {
+            out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
+                     node_name, pcmk_strerror(rc));
+            return rc;
+        }
+    } else {
+        for (GList *iter = data_set->nodes; iter; iter = iter->next) {
+            pe_node_t *node = (pe_node_t *) iter->data;
+
+            rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
+                                    operation, interval_spec, data_set);
+            if (rc != pcmk_rc_ok) {
+                out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
+                         pcmk_strerror(rc));
+                return rc;
+            }
+        }
+    }
+
+    out->info(out, "Cleaned up all resources on %s", display_name);
+    return rc;
+}
+
+static void
+check_role(resource_checks_t *checks)
+{
+    const char *role_s = g_hash_table_lookup(checks->rsc->meta,
+                                             XML_RSC_ATTR_TARGET_ROLE);
+
+    if (role_s == NULL) {
+        return;
+    }
+    switch (text2role(role_s)) {
+        case RSC_ROLE_STOPPED:
+            checks->flags |= rsc_remain_stopped;
+            break;
+
+        case RSC_ROLE_UNPROMOTED:
+            if (pcmk_is_set(pe__const_top_resource(checks->rsc, false)->flags,
+                            pe_rsc_promotable)) {
+                checks->flags |= rsc_unpromotable;
+            }
+            break;
+
+        default:
+            break;
+    }
+}
+
+static void
+check_managed(resource_checks_t *checks)
+{
+    const char *managed_s = g_hash_table_lookup(checks->rsc->meta,
+                                                XML_RSC_ATTR_MANAGED);
+
+    if ((managed_s != NULL) && !crm_is_true(managed_s)) {
+        checks->flags |= rsc_unmanaged;
+    }
+}
+
+static void
+check_locked(resource_checks_t *checks)
+{
+    if (checks->rsc->lock_node != NULL) {
+        checks->flags |= rsc_locked;
+        checks->lock_node = checks->rsc->lock_node->details->uname;
+    }
+}
+
+static bool
+node_is_unhealthy(pe_node_t *node)
+{
+    switch (pe__health_strategy(node->details->data_set)) {
+        case pcmk__health_strategy_none:
+            break;
+
+        case pcmk__health_strategy_no_red:
+            if (pe__node_health(node) < 0) {
+                return true;
+            }
+            break;
+
+        case pcmk__health_strategy_only_green:
+            if (pe__node_health(node) <= 0) {
+                return true;
+            }
+            break;
+
+        case pcmk__health_strategy_progressive:
+        case pcmk__health_strategy_custom:
+            /* @TODO These are finite scores, possibly with rules, and possibly
+             * combining with other scores, so attributing these as a cause is
+             * nontrivial.
+             */
+            break;
+    }
+    return false;
+}
+
+static void
+check_node_health(resource_checks_t *checks, pe_node_t *node)
+{
+    if (node == NULL) {
+        GHashTableIter iter;
+        bool allowed = false;
+        bool all_nodes_unhealthy = true;
+
+        g_hash_table_iter_init(&iter, checks->rsc->allowed_nodes);
+        while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
+            allowed = true;
+            if (!node_is_unhealthy(node)) {
+                all_nodes_unhealthy = false;
+                break;
+            }
+        }
+        if (allowed && all_nodes_unhealthy) {
+            checks->flags |= rsc_node_health;
+        }
+
+    } else if (node_is_unhealthy(node)) {
+        checks->flags |= rsc_node_health;
+    }
+}
+
+int
+cli_resource_check(pcmk__output_t *out, pe_resource_t *rsc, pe_node_t *node)
+{
+    resource_checks_t checks = { .rsc = rsc };
+
+    check_role(&checks);
+    check_managed(&checks);
+    check_locked(&checks);
+    check_node_health(&checks, node);
+
+    return out->message(out, "resource-check-list", &checks);
+}
+
+// \return Standard Pacemaker return code
+int
+cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
+                  const char *rsc_id, pe_working_set_t *data_set)
+{
+    crm_notice("Failing %s on %s", rsc_id, host_uname);
+    return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, data_set);
+}
+
+static GHashTable *
+generate_resource_params(pe_resource_t *rsc, pe_node_t *node,
+                         pe_working_set_t *data_set)
+{
+    GHashTable *params = NULL;
+    GHashTable *meta = NULL;
+    GHashTable *combined = NULL;
+    GHashTableIter iter;
+    char *key = NULL;
+    char *value = NULL;
+
+    combined = pcmk__strkey_table(free, free);
+
+    params = pe_rsc_params(rsc, node, data_set);
+    if (params != NULL) {
+        g_hash_table_iter_init(&iter, params);
+        while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
+            g_hash_table_insert(combined, strdup(key), strdup(value));
+        }
+    }
+
+    meta = pcmk__strkey_table(free, free);
+    get_meta_attributes(meta, rsc, node, data_set);
+    if (meta != NULL) {
+        g_hash_table_iter_init(&iter, meta);
+        while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
+            char *crm_name = crm_meta_name(key);
+
+            g_hash_table_insert(combined, crm_name, strdup(value));
+        }
+        g_hash_table_destroy(meta);
+    }
+
+    return combined;
+}
+
+bool resource_is_running_on(pe_resource_t *rsc, const char *host)
+{
+    bool found = true;
+    GList *hIter = NULL;
+    GList *hosts = NULL;
+
+    if (rsc == NULL) {
+        return false;
+    }
+
+    rsc->fns->location(rsc, &hosts, TRUE);
+    for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
+        pe_node_t *node = (pe_node_t *) hIter->data;
+
+        if (pcmk__strcase_any_of(host, node->details->uname, node->details->id, NULL)) {
+            crm_trace("Resource %s is running on %s\n", rsc->id, host);
+            goto done;
+        }
+    }
+
+    if (host != NULL) {
+        crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
+        found = false;
+
+    } else if(host == NULL && hosts == NULL) {
+        crm_trace("Resource %s is not running\n", rsc->id);
+        found = false;
+    }
+
+  done:
+    g_list_free(hosts);
+    return found;
+}
+
+/*!
+ * \internal
+ * \brief Create a list of all resources active on host from a given list
+ *
+ * \param[in] host      Name of host to check whether resources are active
+ * \param[in] rsc_list  List of resources to check
+ *
+ * \return New list of resources from list that are active on host
+ */
+static GList *
+get_active_resources(const char *host, GList *rsc_list)
+{
+    GList *rIter = NULL;
+    GList *active = NULL;
+
+    for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
+        pe_resource_t *rsc = (pe_resource_t *) rIter->data;
+
+        /* Expand groups to their members, because if we're restarting a member
+         * other than the first, we can't otherwise tell which resources are
+         * stopping and starting.
+         */
+        if (rsc->variant == pe_group) {
+            active = g_list_concat(active,
+                                   get_active_resources(host, rsc->children));
+        } else if (resource_is_running_on(rsc, host)) {
+            active = g_list_append(active, strdup(rsc->id));
+        }
+    }
+    return active;
+}
+
+static void dump_list(GList *items, const char *tag) 
+{
+    int lpc = 0;
+    GList *item = NULL;
+
+    for (item = items; item != NULL; item = item->next) {
+        crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
+        lpc++;
+    }
+}
+
+static void display_list(pcmk__output_t *out, GList *items, const char *tag)
+{
+    GList *item = NULL;
+
+    for (item = items; item != NULL; item = item->next) {
+        out->info(out, "%s%s", tag, (const char *)item->data);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Upgrade XML to latest schema version and use it as working set input
+ *
+ * This also updates the working set timestamp to the current time.
+ *
+ * \param[in,out] data_set  Working set instance to update
+ * \param[in,out] xml       XML to use as input
+ *
+ * \return Standard Pacemaker return code
+ * \note On success, caller is responsible for freeing memory allocated for
+ *       data_set->now.
+ * \todo This follows the example of other callers of cli_config_update()
+ *       and returns ENOKEY ("Required key not available") if that fails,
+ *       but perhaps pcmk_rc_schema_validation would be better in that case.
+ */
+int
+update_working_set_xml(pe_working_set_t *data_set, xmlNode **xml)
+{
+    if (cli_config_update(xml, NULL, FALSE) == FALSE) {
+        return ENOKEY;
+    }
+    data_set->input = *xml;
+    data_set->now = crm_time_new(NULL);
+    return pcmk_rc_ok;
+}
+
+/*!
+ * \internal
+ * \brief Update a working set's XML input based on a CIB query
+ *
+ * \param[in] data_set   Data set instance to initialize
+ * \param[in] cib        Connection to the CIB manager
+ *
+ * \return Standard Pacemaker return code
+ * \note On success, caller is responsible for freeing memory allocated for
+ *       data_set->input and data_set->now.
+ */
+static int
+update_working_set_from_cib(pcmk__output_t *out, pe_working_set_t * data_set,
+                            cib_t *cib)
+{
+    xmlNode *cib_xml_copy = NULL;
+    int rc = pcmk_rc_ok;
+
+    rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
+    rc = pcmk_legacy2rc(rc);
+
+    if (rc != pcmk_rc_ok) {
+        out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_strerror(rc), rc);
+        return rc;
+    }
+    rc = update_working_set_xml(data_set, &cib_xml_copy);
+    if (rc != pcmk_rc_ok) {
+        out->err(out, "Could not upgrade the current CIB XML");
+        free_xml(cib_xml_copy);
+        return rc;
+    }
+
+    return rc;
+}
+
+// \return Standard Pacemaker return code
+static int
+update_dataset(cib_t *cib, pe_working_set_t * data_set, bool simulate)
+{
+    char *pid = NULL;
+    char *shadow_file = NULL;
+    cib_t *shadow_cib = NULL;
+    int rc = pcmk_rc_ok;
+
+    pcmk__output_t *out = data_set->priv;
+
+    pe_reset_working_set(data_set);
+    pe__set_working_set_flags(data_set, pe_flag_no_counts|pe_flag_no_compat);
+    rc = update_working_set_from_cib(out, data_set, cib);
+    if (rc != pcmk_rc_ok) {
+        return rc;
+    }
+
+    if(simulate) {
+        bool prev_quiet = false;
+
+        pid = pcmk__getpid_s();
+        shadow_cib = cib_shadow_new(pid);
+        shadow_file = get_shadow_file(pid);
+
+        if (shadow_cib == NULL) {
+            out->err(out, "Could not create shadow cib: '%s'", pid);
+            rc = ENXIO;
+            goto done;
+        }
+
+        rc = write_xml_file(data_set->input, shadow_file, FALSE);
+
+        if (rc < 0) {
+            out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
+            goto done;
+        }
+
+        rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
+        rc = pcmk_legacy2rc(rc);
+
+        if (rc != pcmk_rc_ok) {
+            out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_strerror(rc), rc);
+            goto done;
+        }
+
+        pcmk__schedule_actions(data_set->input,
+                               pe_flag_no_counts|pe_flag_no_compat, data_set);
+
+        prev_quiet = out->is_quiet(out);
+        out->quiet = true;
+        pcmk__simulate_transition(data_set, shadow_cib, NULL);
+        out->quiet = prev_quiet;
+
+        rc = update_dataset(shadow_cib, data_set, false);
+
+    } else {
+        cluster_status(data_set);
+    }
+
+  done:
+    /* Do not free data_set->input here, we need rsc->xml to be valid later on */
+    cib_delete(shadow_cib);
+    free(pid);
+
+    if(shadow_file) {
+        unlink(shadow_file);
+        free(shadow_file);
+    }
+
+    return rc;
+}
+
+static int
+max_delay_for_resource(pe_working_set_t * data_set, pe_resource_t *rsc) 
+{
+    int delay = 0;
+    int max_delay = 0;
+
+    if(rsc && rsc->children) {
+        GList *iter = NULL;
+
+        for(iter = rsc->children; iter; iter = iter->next) {
+            pe_resource_t *child = (pe_resource_t *)iter->data;
+
+            delay = max_delay_for_resource(data_set, child);
+            if(delay > max_delay) {
+                double seconds = delay / 1000.0;
+                crm_trace("Calculated new delay of %.1fs due to %s", seconds, child->id);
+                max_delay = delay;
+            }
+        }
+
+    } else if(rsc) {
+        char *key = crm_strdup_printf("%s_%s_0", rsc->id, RSC_STOP);
+        pe_action_t *stop = custom_action(rsc, key, RSC_STOP, NULL, TRUE, FALSE, data_set);
+        const char *value = g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT);
+        long long result_ll;
+
+        if ((pcmk__scan_ll(value, &result_ll, -1LL) == pcmk_rc_ok)
+            && (result_ll >= 0) && (result_ll <= INT_MAX)) {
+            max_delay = (int) result_ll;
+        } else {
+            max_delay = -1;
+        }
+        pe_free_action(stop);
+    }
+
+    return max_delay;
+}
+
+static int
+max_delay_in(pe_working_set_t * data_set, GList *resources) 
+{
+    int max_delay = 0;
+    GList *item = NULL;
+
+    for (item = resources; item != NULL; item = item->next) {
+        int delay = 0;
+        pe_resource_t *rsc = pe_find_resource(data_set->resources, (const char *)item->data);
+
+        delay = max_delay_for_resource(data_set, rsc);
+
+        if(delay > max_delay) {
+            double seconds = delay / 1000.0;
+            crm_trace("Calculated new delay of %.1fs due to %s", seconds, rsc->id);
+            max_delay = delay;
+        }
+    }
+
+    return 5 + (max_delay / 1000);
+}
+
+#define waiting_for_starts(d, r, h) ((d != NULL) || \
+                                    (!resource_is_running_on((r), (h))))
+
+/*!
+ * \internal
+ * \brief Restart a resource (on a particular host if requested).
+ *
+ * \param[in,out] out                 Output object
+ * \param[in,out] rsc                 The resource to restart
+ * \param[in]     node                Node to restart resource on (NULL for all)
+ * \param[in]     move_lifetime       If not NULL, how long constraint should
+ *                                    remain in effect (as ISO 8601 string)
+ * \param[in]     timeout_ms          Consider failed if actions do not complete
+ *                                    in this time (specified in milliseconds,
+ *                                    but a two-second granularity is actually
+ *                                    used; if 0, it will be calculated based on
+ *                                    the resource timeout)
+ * \param[in,out] cib                 Connection to the CIB manager
+ * \param[in]     cib_options         Group of enum cib_call_options flags to
+ *                                    use with CIB calls
+ * \param[in]     promoted_role_only  If true, limit to promoted instances
+ * \param[in]     force               If true, apply only to requested instance
+ *                                    if part of a collective resource
+ *
+ * \return Standard Pacemaker return code (exits on certain failures)
+ */
+int
+cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc,
+                     const pe_node_t *node, const char *move_lifetime,
+                     int timeout_ms, cib_t *cib, int cib_options,
+                     gboolean promoted_role_only, gboolean force)
+{
+    int rc = pcmk_rc_ok;
+    int lpc = 0;
+    int before = 0;
+    int step_timeout_s = 0;
+    int sleep_interval = 2;
+    int timeout = timeout_ms / 1000;
+
+    bool stop_via_ban = false;
+    char *rsc_id = NULL;
+    char *lookup_id = NULL;
+    char *orig_target_role = NULL;
+
+    GList *list_delta = NULL;
+    GList *target_active = NULL;
+    GList *current_active = NULL;
+    GList *restart_target_active = NULL;
+
+    pe_working_set_t *data_set = NULL;
+    pe_resource_t *parent = uber_parent(rsc);
+
+    bool running = false;
+    const char *id = rsc->clone_name ? rsc->clone_name : rsc->id;
+    const char *host = node ? node->details->uname : NULL;
+
+    /* If the implicit resource or primitive resource of a bundle is given, operate on the
+     * bundle itself instead.
+     */
+    if (pe_rsc_is_bundled(rsc)) {
+        rsc = parent->parent;
+    }
+
+    running = resource_is_running_on(rsc, host);
+
+    if (pe_rsc_is_clone(parent) && !running) {
+        if (pe_rsc_is_unique_clone(parent)) {
+            lookup_id = strdup(rsc->id);
+        } else {
+            lookup_id = clone_strip(rsc->id);
+        }
+
+        rsc = parent->fns->find_rsc(parent, lookup_id, node, pe_find_any|pe_find_current);
+        free(lookup_id);
+        running = resource_is_running_on(rsc, host);
+    }
+
+    if (!running) {
+        if (host) {
+            out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
+        } else {
+            out->err(out, "%s is not running anywhere and so cannot be restarted", id);
+        }
+        return ENXIO;
+    }
+
+    rsc_id = strdup(rsc->id);
+
+    if (pe_rsc_is_unique_clone(parent)) {
+        lookup_id = strdup(rsc->id);
+    } else {
+        lookup_id = clone_strip(rsc->id);
+    }
+
+    if (host) {
+        if (pe_rsc_is_clone(rsc) || pe_bundle_replicas(rsc)) {
+            stop_via_ban = true;
+        } else if (pe_rsc_is_clone(parent)) {
+            stop_via_ban = true;
+            free(lookup_id);
+            lookup_id = strdup(parent->id);
+        }
+    }
+
+    /*
+      grab full cib
+      determine originally active resources
+      disable or ban
+      poll cib and watch for affected resources to get stopped
+      without --timeout, calculate the stop timeout for each step and wait for that
+      if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
+      if everything stopped, re-enable or un-ban
+      poll cib and watch for affected resources to get started
+      without --timeout, calculate the start timeout for each step and wait for that
+      if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
+      report success
+
+      Optimizations:
+      - use constraints to determine ordered list of affected resources
+      - Allow a --no-deps option (aka. --force-restart)
+    */
+
+    data_set = pe_new_working_set();
+    if (data_set == NULL) {
+        crm_perror(LOG_ERR, "Could not allocate working set");
+        rc = ENOMEM;
+        goto done;
+    }
+
+    data_set->priv = out;
+    rc = update_dataset(cib, data_set, false);
+
+    if(rc != pcmk_rc_ok) {
+        out->err(out, "Could not get new resource list: %s (%d)", pcmk_strerror(rc), rc);
+        goto done;
+    }
+
+    restart_target_active = get_active_resources(host, data_set->resources);
+    current_active = get_active_resources(host, data_set->resources);
+
+    dump_list(current_active, "Origin");
+
+    if (stop_via_ban) {
+        /* Stop the clone or bundle instance by banning it from the host */
+        out->quiet = true;
+        rc = cli_resource_ban(out, lookup_id, host, move_lifetime, NULL, cib,
+                              cib_options, promoted_role_only);
+
+    } else {
+        /* Stop the resource by setting target-role to Stopped.
+         * Remember any existing target-role so we can restore it later
+         * (though it only makes any difference if it's Unpromoted).
+         */
+
+        find_resource_attr(out, cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
+                           NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
+        rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
+                                           NULL, XML_RSC_ATTR_TARGET_ROLE,
+                                           RSC_STOPPED, FALSE, cib, cib_options,
+                                           force);
+    }
+    if(rc != pcmk_rc_ok) {
+        out->err(out, "Could not set target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
+        if (current_active != NULL) {
+            g_list_free_full(current_active, free);
+            current_active = NULL;
+        }
+        if (restart_target_active != NULL) {
+            g_list_free_full(restart_target_active, free);
+            restart_target_active = NULL;
+        }
+        goto done;
+    }
+
+    rc = update_dataset(cib, data_set, true);
+    if(rc != pcmk_rc_ok) {
+        out->err(out, "Could not determine which resources would be stopped");
+        goto failure;
+    }
+
+    target_active = get_active_resources(host, data_set->resources);
+    dump_list(target_active, "Target");
+
+    list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
+    out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
+    display_list(out, list_delta, " * ");
+
+    step_timeout_s = timeout / sleep_interval;
+    while (list_delta != NULL) {
+        before = g_list_length(list_delta);
+        if(timeout_ms == 0) {
+            step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
+        }
+
+        /* We probably don't need the entire step timeout */
+        for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
+            sleep(sleep_interval);
+            if(timeout) {
+                timeout -= sleep_interval;
+                crm_trace("%ds remaining", timeout);
+            }
+            rc = update_dataset(cib, data_set, FALSE);
+            if(rc != pcmk_rc_ok) {
+                out->err(out, "Could not determine which resources were stopped");
+                goto failure;
+            }
+
+            if (current_active != NULL) {
+                g_list_free_full(current_active, free);
+                current_active = NULL;
+            }
+            current_active = get_active_resources(host, data_set->resources);
+            g_list_free(list_delta);
+            list_delta = NULL;
+            list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
+            dump_list(current_active, "Current");
+            dump_list(list_delta, "Delta");
+        }
+
+        crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
+        if(before == g_list_length(list_delta)) {
+            /* aborted during stop phase, print the contents of list_delta */
+            out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
+            display_list(out, list_delta, " * ");
+            rc = ETIME;
+            goto failure;
+        }
+
+    }
+
+    if (stop_via_ban) {
+        rc = cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
+
+    } else if (orig_target_role) {
+        rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
+                                           NULL, XML_RSC_ATTR_TARGET_ROLE,
+                                           orig_target_role, FALSE, cib,
+                                           cib_options, force);
+        free(orig_target_role);
+        orig_target_role = NULL;
+    } else {
+        rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
+                                           NULL, XML_RSC_ATTR_TARGET_ROLE, cib,
+                                           cib_options, force);
+    }
+
+    if(rc != pcmk_rc_ok) {
+        out->err(out, "Could not unset target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
+        goto done;
+    }
+
+    if (target_active != NULL) {
+        g_list_free_full(target_active, free);
+        target_active = NULL;
+    }
+    target_active = restart_target_active;
+    list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
+    out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
+    display_list(out, list_delta, " * ");
+
+    step_timeout_s = timeout / sleep_interval;
+    while (waiting_for_starts(list_delta, rsc, host)) {
+        before = g_list_length(list_delta);
+        if(timeout_ms == 0) {
+            step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
+        }
+
+        /* We probably don't need the entire step timeout */
+        for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {
+
+            sleep(sleep_interval);
+            if(timeout) {
+                timeout -= sleep_interval;
+                crm_trace("%ds remaining", timeout);
+            }
+
+            rc = update_dataset(cib, data_set, false);
+            if(rc != pcmk_rc_ok) {
+                out->err(out, "Could not determine which resources were started");
+                goto failure;
+            }
+
+            if (current_active != NULL) {
+                g_list_free_full(current_active, free);
+                current_active = NULL;
+            }
+
+            /* It's OK if dependent resources moved to a different node,
+             * so we check active resources on all nodes.
+             */
+            current_active = get_active_resources(NULL, data_set->resources);
+            g_list_free(list_delta);
+            list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
+            dump_list(current_active, "Current");
+            dump_list(list_delta, "Delta");
+        }
+
+        if(before == g_list_length(list_delta)) {
+            /* aborted during start phase, print the contents of list_delta */
+            out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
+            display_list(out, list_delta, " * ");
+            rc = ETIME;
+            goto failure;
+        }
+
+    }
+
+    rc = pcmk_rc_ok;
+    goto done;
+
+  failure:
+    if (stop_via_ban) {
+        cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
+    } else if (orig_target_role) {
+        cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
+                                      XML_RSC_ATTR_TARGET_ROLE, orig_target_role,
+                                      FALSE, cib, cib_options, force);
+        free(orig_target_role);
+    } else {
+        cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
+                                      NULL, XML_RSC_ATTR_TARGET_ROLE, cib,
+                                      cib_options, force);
+    }
+
+done:
+    if (list_delta != NULL) {
+        g_list_free(list_delta);
+    }
+    if (current_active != NULL) {
+        g_list_free_full(current_active, free);
+    }
+    if (target_active != NULL && (target_active != restart_target_active)) {
+        g_list_free_full(target_active, free);
+    }
+    if (restart_target_active != NULL) {
+        g_list_free_full(restart_target_active, free);
+    }
+    free(rsc_id);
+    free(lookup_id);
+    pe_free_working_set(data_set);
+    return rc;
+}
+
+static inline bool
+action_is_pending(const pe_action_t *action)
+{
+    if (pcmk_any_flags_set(action->flags, pe_action_optional|pe_action_pseudo)
+        || !pcmk_is_set(action->flags, pe_action_runnable)
+        || pcmk__str_eq("notify", action->task, pcmk__str_casei)) {
+        return false;
+    }
+    return true;
+}
+
+/*!
+ * \internal
+ * \brief Check whether any actions in a list are pending
+ *
+ * \param[in] actions   List of actions to check
+ *
+ * \return true if any actions in the list are pending, otherwise false
+ */
+static bool
+actions_are_pending(const GList *actions)
+{
+    for (const GList *action = actions; action != NULL; action = action->next) {
+        const pe_action_t *a = (const pe_action_t *) action->data;
+
+        if (action_is_pending(a)) {
+            crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
+            return true;
+        }
+    }
+    return false;
+}
+
+static void
+print_pending_actions(pcmk__output_t *out, GList *actions)
+{
+    GList *action;
+
+    out->info(out, "Pending actions:");
+    for (action = actions; action != NULL; action = action->next) {
+        pe_action_t *a = (pe_action_t *) action->data;
+
+        if (!action_is_pending(a)) {
+            continue;
+        }
+
+        if (a->node) {
+            out->info(out, "\tAction %d: %s\ton %s",
+                      a->id, a->uuid, pe__node_name(a->node));
+        } else {
+            out->info(out, "\tAction %d: %s", a->id, a->uuid);
+        }
+    }
+}
+
+/* For --wait, timeout (in seconds) to use if caller doesn't specify one */
+#define WAIT_DEFAULT_TIMEOUT_S (60 * 60)
+
+/* For --wait, how long to sleep between cluster state checks */
+#define WAIT_SLEEP_S (2)
+
+/*!
+ * \internal
+ * \brief Wait until all pending cluster actions are complete
+ *
+ * This waits until either the CIB's transition graph is idle or a timeout is
+ * reached.
+ *
+ * \param[in,out] out          Output object
+ * \param[in]     timeout_ms   Consider failed if actions do not complete in
+ *                             this time (specified in milliseconds, but
+ *                             one-second granularity is actually used; if 0, a
+ *                             default will be used)
+ * \param[in,out] cib          Connection to the CIB manager
+ *
+ * \return Standard Pacemaker return code
+ */
+int
+wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib)
+{
+    pe_working_set_t *data_set = NULL;
+    int rc = pcmk_rc_ok;
+    int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
+    time_t expire_time = time(NULL) + timeout_s;
+    time_t time_diff;
+    bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet
+
+    data_set = pe_new_working_set();
+    if (data_set == NULL) {
+        return ENOMEM;
+    }
+
+    do {
+
+        /* Abort if timeout is reached */
+        time_diff = expire_time - time(NULL);
+        if (time_diff > 0) {
+            crm_info("Waiting up to %lld seconds for cluster actions to complete", (long long) time_diff);
+        } else {
+            print_pending_actions(out, data_set->actions);
+            pe_free_working_set(data_set);
+            return ETIME;
+        }
+        if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
+            sleep(WAIT_SLEEP_S);
+        }
+
+        /* Get latest transition graph */
+        pe_reset_working_set(data_set);
+        rc = update_working_set_from_cib(out, data_set, cib);
+        if (rc != pcmk_rc_ok) {
+            pe_free_working_set(data_set);
+            return rc;
+        }
+        pcmk__schedule_actions(data_set->input,
+                               pe_flag_no_counts|pe_flag_no_compat, data_set);
+
+        if (!printed_version_warning) {
+            /* If the DC has a different version than the local node, the two
+             * could come to different conclusions about what actions need to be
+             * done. Warn the user in this case.
+             *
+             * @TODO A possible long-term solution would be to reimplement the
+             * wait as a new controller operation that would be forwarded to the
+             * DC. However, that would have potential problems of its own.
+             */
+            const char *dc_version = g_hash_table_lookup(data_set->config_hash,
+                                                         "dc-version");
+
+            if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
+                out->info(out, "warning: wait option may not work properly in "
+                          "mixed-version cluster");
+                printed_version_warning = true;
+            }
+        }
+
+    } while (actions_are_pending(data_set->actions));
+
+    pe_free_working_set(data_set);
+    return rc;
+}
+
+static const char *
+get_action(const char *rsc_action) {
+    const char *action = NULL;
+
+    if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
+        action = "validate-all";
+
+    } else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
+        action = "monitor";
+
+    } else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
+                                    "force-demote", "force-promote", NULL)) {
+        action = rsc_action+6;
+    } else {
+        action = rsc_action;
+    }
+
+    return action;
+}
+
+/*!
+ * \brief Set up environment variables as expected by resource agents
+ *
+ * When the cluster executes resource agents, it adds certain environment
+ * variables (directly or via resource meta-attributes) expected by some
+ * resource agents. Add the essential ones that many resource agents expect, so
+ * the behavior is the same for command-line execution.
+ *
+ * \param[in,out] params       Resource parameters that will be passed to agent
+ * \param[in]     timeout_ms   Action timeout (in milliseconds)
+ * \param[in]     check_level  OCF check level
+ * \param[in]     verbosity    Verbosity level
+ */
+static void
+set_agent_environment(GHashTable *params, int timeout_ms, int check_level,
+                      int verbosity)
+{
+    g_hash_table_insert(params, strdup("CRM_meta_timeout"),
+                        crm_strdup_printf("%d", timeout_ms));
+
+    g_hash_table_insert(params, strdup(XML_ATTR_CRM_VERSION),
+                        strdup(CRM_FEATURE_SET));
+
+    if (check_level >= 0) {
+        char *level = crm_strdup_printf("%d", check_level);
+
+        setenv("OCF_CHECK_LEVEL", level, 1);
+        free(level);
+    }
+
+    setenv("HA_debug", (verbosity > 0)? "1" : "0", 1);
+    if (verbosity > 1) {
+        setenv("OCF_TRACE_RA", "1", 1);
+    }
+
+    /* A resource agent using the standard ocf-shellfuncs library will not print
+     * messages to stderr if it doesn't have a controlling terminal (e.g. if
+     * crm_resource is called via script or ssh). This forces it to do so.
+     */
+    setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
+}
+
+/*!
+ * \internal
+ * \brief Apply command-line overrides to resource parameters
+ *
+ * \param[in,out] params     Parameters to be passed to agent
+ * \param[in]     overrides  Parameters to override (or NULL if none)
+ */
+static void
+apply_overrides(GHashTable *params, GHashTable *overrides)
+{
+    if (overrides != NULL) {
+        GHashTableIter iter;
+        char *name = NULL;
+        char *value = NULL;
+
+        g_hash_table_iter_init(&iter, overrides);
+        while (g_hash_table_iter_next(&iter, (gpointer *) &name,
+                                      (gpointer *) &value)) {
+            g_hash_table_replace(params, strdup(name), strdup(value));
+        }
+    }
+}
+
+crm_exit_t
+cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
+                                 const char *rsc_class, const char *rsc_prov,
+                                 const char *rsc_type, const char *rsc_action,
+                                 GHashTable *params, GHashTable *override_hash,
+                                 int timeout_ms, int resource_verbose, gboolean force,
+                                 int check_level)
+{
+    const char *class = rsc_class;
+    const char *action = get_action(rsc_action);
+    crm_exit_t exit_code = CRM_EX_OK;
+    svc_action_t *op = NULL;
+
+    // If no timeout was provided, use the same default as the cluster
+    if (timeout_ms == 0) {
+        timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
+    }
+
+    set_agent_environment(params, timeout_ms, check_level, resource_verbose);
+    apply_overrides(params, override_hash);
+
+    op = services__create_resource_action(rsc_name? rsc_name : "test",
+                                          rsc_class, rsc_prov, rsc_type, action,
+                                          0, timeout_ms, params, 0);
+    if (op == NULL) {
+        out->err(out, "Could not execute %s using %s%s%s:%s: %s",
+                 action, rsc_class, (rsc_prov? ":" : ""),
+                 (rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
+        g_hash_table_destroy(params);
+        return CRM_EX_OSERR;
+    }
+
+    if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
+        class = resources_find_service_class(rsc_type);
+    }
+    if (!pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_OCF,
+                              PCMK_RESOURCE_CLASS_LSB, NULL)) {
+        services__format_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
+                                "Manual execution of the %s standard is "
+                                "unsupported", pcmk__s(class, "unspecified"));
+    }
+
+    if (op->rc != PCMK_OCF_UNKNOWN) {
+        exit_code = op->rc;
+        goto done;
+    }
+
+    services_action_sync(op);
+
+    // Map results to OCF codes for consistent reporting to user
+    {
+        enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);
+
+        // Cast variable instead of function return to keep compilers happy
+        exit_code = (crm_exit_t) ocf_code;
+    }
+
+done:
+    out->message(out, "resource-agent-action", resource_verbose, rsc_class,
+                 rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
+                 exit_code, op->status, services__exit_reason(op),
+                 op->stdout_data, op->stderr_data);
+    services_action_free(op);
+    return exit_code;
+}
+
+crm_exit_t
+cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
+                     const char *rsc_action, GHashTable *override_hash,
+                     int timeout_ms, cib_t * cib, pe_working_set_t *data_set,
+                     int resource_verbose, gboolean force, int check_level)
+{
+    pcmk__output_t *out = data_set->priv;
+    crm_exit_t exit_code = CRM_EX_OK;
+    const char *rid = NULL;
+    const char *rtype = NULL;
+    const char *rprov = NULL;
+    const char *rclass = NULL;
+    GHashTable *params = NULL;
+
+    if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
+                                    "force-promote", NULL)) {
+        if(pe_rsc_is_clone(rsc)) {
+            GList *nodes = cli_resource_search(rsc, requested_name, data_set);
+            if(nodes != NULL && force == FALSE) {
+                out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
+                         rsc_action, rsc->id);
+                out->err(out, "Try setting target-role=Stopped first or specifying "
+                         "the force option");
+                return CRM_EX_UNSAFE;
+            }
+
+            g_list_free_full(nodes, free);
+        }
+    }
+
+    if(pe_rsc_is_clone(rsc)) {
+        /* Grab the first child resource in the hope it's not a group */
+        rsc = rsc->children->data;
+    }
+
+    if(rsc->variant == pe_group) {
+        out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
+        return CRM_EX_UNIMPLEMENT_FEATURE;
+    } else if (rsc->variant == pe_container || pe_rsc_is_bundled(rsc)) {
+        out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
+        return CRM_EX_UNIMPLEMENT_FEATURE;
+    }
+
+    rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
+    rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
+    rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
+
+    params = generate_resource_params(rsc, NULL /* @TODO use local node */,
+                                      data_set);
+
+    if (timeout_ms == 0) {
+        timeout_ms = pe_get_configured_timeout(rsc, get_action(rsc_action), data_set);
+    }
+
+    rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;
+
+    exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
+                                                 params, override_hash, timeout_ms,
+                                                 resource_verbose, force, check_level);
+    return exit_code;
+}
+
+// \return Standard Pacemaker return code
+int
+cli_resource_move(const pe_resource_t *rsc, const char *rsc_id,
+                  const char *host_name, const char *move_lifetime, cib_t *cib,
+                  int cib_options, pe_working_set_t *data_set,
+                  gboolean promoted_role_only, gboolean force)
+{
+    pcmk__output_t *out = data_set->priv;
+    int rc = pcmk_rc_ok;
+    unsigned int count = 0;
+    pe_node_t *current = NULL;
+    pe_node_t *dest = pe_find_node(data_set->nodes, host_name);
+    bool cur_is_dest = false;
+
+    if (dest == NULL) {
+        return pcmk_rc_node_unknown;
+    }
+
+    if (promoted_role_only && !pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
+        const pe_resource_t *p = pe__const_top_resource(rsc, false);
+
+        if (pcmk_is_set(p->flags, pe_rsc_promotable)) {
+            out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
+            rsc_id = p->id;
+            rsc = p;
+
+        } else {
+            out->info(out, "Ignoring --promoted option: %s is not promotable",
+                      rsc_id);
+            promoted_role_only = FALSE;
+        }
+    }
+
+    current = pe__find_active_requires(rsc, &count);
+
+    if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
+        unsigned int promoted_count = 0;
+        pe_node_t *promoted_node = NULL;
+
+        for (const GList *iter = rsc->children; iter; iter = iter->next) {
+            const pe_resource_t *child = (const pe_resource_t *) iter->data;
+            enum rsc_role_e child_role = child->fns->state(child, TRUE);
+
+            if (child_role == RSC_ROLE_PROMOTED) {
+                rsc = child;
+                promoted_node = pe__current_node(child);
+                promoted_count++;
+            }
+        }
+        if (promoted_role_only || (promoted_count != 0)) {
+            count = promoted_count;
+            current = promoted_node;
+        }
+
+    }
+
+    if (count > 1) {
+        if (pe_rsc_is_clone(rsc)) {
+            current = NULL;
+        } else {
+            return pcmk_rc_multiple;
+        }
+    }
+
+    if (current && (current->details == dest->details)) {
+        cur_is_dest = true;
+        if (force) {
+            crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
+                     rsc_id, promoted_role_only?"promoted":"active",
+                     pe__node_name(dest));
+        } else {
+            return pcmk_rc_already;
+        }
+    }
+
+    /* Clear any previous prefer constraints across all nodes. */
+    cli_resource_clear(rsc_id, NULL, data_set->nodes, cib, cib_options, false, force);
+
+    /* Clear any previous ban constraints on 'dest'. */
+    cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib,
+                       cib_options, TRUE, force);
+
+    /* Record an explicit preference for 'dest' */
+    rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
+                             cib, cib_options, promoted_role_only);
+
+    crm_trace("%s%s now prefers %s%s",
+              rsc->id, (promoted_role_only? " (promoted)" : ""),
+              pe__node_name(dest), force?"(forced)":"");
+
+    /* only ban the previous location if current location != destination location.
+     * it is possible to use -M to enforce a location without regard of where the
+     * resource is currently located */
+    if (force && !cur_is_dest) {
+        /* Ban the original location if possible */
+        if(current) {
+            (void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
+                                   NULL, cib, cib_options, promoted_role_only);
+
+        } else if(count > 1) {
+            out->info(out, "Resource '%s' is currently %s in %d locations. "
+                      "One may now move to %s",
+                      rsc_id, (promoted_role_only? "promoted" : "active"),
+                      count, pe__node_name(dest));
+            out->info(out, "To prevent '%s' from being %s at a specific location, "
+                      "specify a node.",
+                      rsc_id, (promoted_role_only? "promoted" : "active"));
+
+        } else {
+            crm_trace("Not banning %s from its current location: not active", rsc_id);
+        }
+    }
+
+    return rc;
+}
diff --git a/tools/crm_rule.8.inc b/tools/crm_rule.8.inc
new file mode 100644
index 0000000..2b6ac89
--- /dev/null
+++ b/tools/crm_rule.8.inc
@@ -0,0 +1,8 @@
+[synopsis]
+crm_rule mode [options]
+
+/state of rules/
+.SH OPTIONS
+
+/multiple times/
+.SH NOTICE
diff --git a/tools/crm_rule.c b/tools/crm_rule.c
new file mode 100644
index 0000000..5cdc118
--- /dev/null
+++ b/tools/crm_rule.c
@@ -0,0 +1,227 @@
+/*
+ * Copyright 2019-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/output_internal.h>
+#include <crm/common/iso8601.h>
+#include <crm/msg_xml.h>
+#include <crm/pengine/rules_internal.h>
+#include <crm/pengine/status.h>
+#include <pacemaker-internal.h>
+
+#include <sys/stat.h>
+
+#define SUMMARY "evaluate rules from the Pacemaker configuration"
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+enum crm_rule_mode {
+    crm_rule_mode_none,
+    crm_rule_mode_check
+};
+
+struct {
+    char *date;
+    char *input_xml;
+    enum crm_rule_mode mode;
+    gchar **rules;
+} options = {
+    .mode = crm_rule_mode_none
+};
+
+static gboolean mode_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+
+static GOptionEntry mode_entries[] = {
+    { "check", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, mode_cb,
+      "Check whether a rule is in effect",
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry data_entries[] = {
+    { "xml-text", 'X', 0, G_OPTION_ARG_STRING, &options.input_xml,
+      "Use argument for XML (or stdin if '-')",
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry addl_entries[] = {
+    { "date", 'd', 0, G_OPTION_ARG_STRING, &options.date,
+      "Whether the rule is in effect on a given date",
+      NULL },
+    { "rule", 'r', 0, G_OPTION_ARG_STRING_ARRAY, &options.rules,
+      "The ID of the rule to check (may be specified multiple times)",
+      NULL },
+
+    { NULL }
+};
+
+static gboolean
+mode_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (strcmp(option_name, "c")) {
+        options.mode = crm_rule_mode_check;
+    }
+
+    return TRUE;
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
+
+    pcmk__add_arg_group(context, "modes", "Modes (mutually exclusive):",
+                        "Show modes of operation", mode_entries);
+    pcmk__add_arg_group(context, "data", "Data:",
+                        "Show data options", data_entries);
+    pcmk__add_arg_group(context, "additional", "Additional Options:",
+                        "Show additional options", addl_entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    crm_time_t *rule_date = NULL;
+    xmlNode *input = NULL;
+
+    int rc = pcmk_rc_ok;
+    crm_exit_t exit_code = CRM_EX_OK;
+
+    GError *error = NULL;
+
+    pcmk__output_t *out = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    GOptionContext *context = build_arg_context(args, &output_group);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "drX");
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("crm_rule", args->verbosity);
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    pcmk__register_lib_messages(out);
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    /* Check command line arguments before opening a connection to
+     * the CIB manager or doing anything else important.
+     */
+    switch(options.mode) {
+        case crm_rule_mode_check:
+            if (options.rules == NULL) {
+                exit_code = CRM_EX_USAGE;
+                g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "--check requires use of --rule=");
+                goto done;
+            }
+
+            break;
+
+        default:
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "No mode operation given");
+            goto done;
+            break;
+    }
+
+    /* Set up some defaults. */
+    rule_date = crm_time_new(options.date);
+    if (rule_date == NULL) {
+        if (options.date != NULL) {
+            exit_code = CRM_EX_DATAERR;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Invalid date specified: '%s'", options.date);
+
+        } else {
+            // Should never happen
+            exit_code = CRM_EX_OSERR;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "No --date given and can't determine current date");
+        }
+        goto done;
+    }
+
+    // Parse the input XML specified by the command-line options, if any
+    if (pcmk__str_eq(options.input_xml, "-", pcmk__str_casei)) {
+        input = stdin2xml();
+
+        if (input == NULL) {
+            exit_code = CRM_EX_DATAERR;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Couldn't parse input from STDIN\n");
+            goto done;
+        }
+    } else if (options.input_xml != NULL) {
+        input = string2xml(options.input_xml);
+
+        if (input == NULL) {
+            exit_code = CRM_EX_DATAERR;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Couldn't parse input string: %s\n", options.input_xml);
+            goto done;
+        }
+    }
+
+    /* Now do whichever operation mode was asked for.  There's only one at the
+     * moment so this looks a little silly, but I expect there will be more
+     * modes in the future.
+     */
+    switch(options.mode) {
+        case crm_rule_mode_check:
+            rc = pcmk__check_rules(out, input, rule_date,
+                                   (const char **) options.rules);
+            exit_code = pcmk_rc2exitc(rc);
+            break;
+
+        default:
+            break;
+    }
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    crm_time_free(rule_date);
+    free_xml(input);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+
+    pcmk__unregister_formats();
+    return crm_exit(exit_code);
+}
diff --git a/tools/crm_shadow.8.inc b/tools/crm_shadow.8.inc
new file mode 100644
index 0000000..4c15a95
--- /dev/null
+++ b/tools/crm_shadow.8.inc
@@ -0,0 +1,5 @@
+[=synopsis]
+crm_shadow <query>|<command> [options]
+
+/for side effects./
+.SH OPTIONS
diff --git a/tools/crm_shadow.c b/tools/crm_shadow.c
new file mode 100644
index 0000000..ef69502
--- /dev/null
+++ b/tools/crm_shadow.c
@@ -0,0 +1,1322 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <unistd.h>
+
+#include <sys/param.h>
+#include <crm/crm.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <crm/msg_xml.h>
+
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/ipc.h>
+#include <crm/common/output_internal.h>
+#include <crm/common/xml.h>
+
+#include <crm/cib.h>
+#include <crm/cib/internal.h>
+
+#define SUMMARY "perform Pacemaker configuration changes in a sandbox\n\n"  \
+                "This command sets up an environment in which "             \
+                "configuration tools (cibadmin,\n"                          \
+                "crm_resource, etc.) work offline instead of against a "    \
+                "live cluster, allowing\n"                                  \
+                "changes to be previewed and tested for side effects."
+
+#define INDENT "                              "
+
+enum shadow_command {
+    shadow_cmd_none = 0,
+    shadow_cmd_which,
+    shadow_cmd_display,
+    shadow_cmd_diff,
+    shadow_cmd_file,
+    shadow_cmd_create,
+    shadow_cmd_create_empty,
+    shadow_cmd_commit,
+    shadow_cmd_delete,
+    shadow_cmd_edit,
+    shadow_cmd_reset,
+    shadow_cmd_switch,
+};
+
+/*!
+ * \internal
+ * \enum shadow_disp_flags
+ * \brief Bit flags to control which fields of shadow CIB info are displayed
+ *
+ * \note Ignored for XML output.
+ */
+enum shadow_disp_flags {
+    shadow_disp_instance = (1 << 0),
+    shadow_disp_file     = (1 << 1),
+    shadow_disp_content  = (1 << 2),
+    shadow_disp_diff     = (1 << 3),
+};
+
+static crm_exit_t exit_code = CRM_EX_OK;
+
+static struct {
+    enum shadow_command cmd;
+    int cmd_options;
+    char *instance;
+    gboolean force;
+    gboolean batch;
+    gboolean full_upload;
+    gchar *validate_with;
+} options = {
+    .cmd_options = cib_sync_call,
+};
+
+/*!
+ * \internal
+ * \brief Display an instruction to the user
+ *
+ * \param[in,out] out  Output object
+ * \param[in]     ...  Message arguments
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note The variadic message arguments are of the following format:
+ *       -# Instructional message
+ */
+PCMK__OUTPUT_ARGS("instruction", "const char *")
+static int
+instruction_default(pcmk__output_t *out, va_list args)
+{
+    const char *msg = va_arg(args, const char *);
+
+    if (msg == NULL) {
+        return pcmk_rc_no_output;
+    }
+    return out->info(out, "%s", msg);
+}
+
+/*!
+ * \internal
+ * \brief Display an instruction to the user
+ *
+ * \param[in,out] out  Output object
+ * \param[in]     ...  Message arguments
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note The variadic message arguments are of the following format:
+ *       -# Instructional message
+ */
+PCMK__OUTPUT_ARGS("instruction", "const char *")
+static int
+instruction_xml(pcmk__output_t *out, va_list args)
+{
+    const char *msg = va_arg(args, const char *);
+
+    if (msg == NULL) {
+        return pcmk_rc_no_output;
+    }
+    pcmk__output_create_xml_text_node(out, "instruction", msg);
+    return pcmk_rc_ok;
+}
+
+/*!
+ * \internal
+ * \brief Display information about a shadow CIB instance
+ *
+ * \param[in,out] out  Output object
+ * \param[in]     ...  Message arguments
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note The variadic message arguments are of the following format:
+ *       -# Instance name (can be \p NULL)
+ *       -# Shadow file name (can be \p NULL)
+ *       -# Shadow file content (can be \p NULL)
+ *       -# Patchset containing the changes in the shadow CIB (can be \p NULL)
+ *       -# Group of \p shadow_disp_flags indicating which fields to display
+ */
+PCMK__OUTPUT_ARGS("shadow", "const char *", "const char *", "xmlNodePtr",
+                  "xmlNodePtr", "enum shadow_disp_flags")
+static int
+shadow_default(pcmk__output_t *out, va_list args)
+{
+    const char *instance = va_arg(args, const char *);
+    const char *filename = va_arg(args, const char *);
+    xmlNodePtr content = va_arg(args, xmlNodePtr);
+    xmlNodePtr diff = va_arg(args, xmlNodePtr);
+    enum shadow_disp_flags flags = (enum shadow_disp_flags) va_arg(args, int);
+
+    int rc = pcmk_rc_no_output;
+
+    if (pcmk_is_set(flags, shadow_disp_instance)) {
+        rc = out->info(out, "Instance: %s", pcmk__s(instance, "<unknown>"));
+    }
+    if (pcmk_is_set(flags, shadow_disp_file)) {
+        rc = out->info(out, "File name: %s", pcmk__s(filename, "<unknown>"));
+    }
+    if (pcmk_is_set(flags, shadow_disp_content)) {
+        rc = out->info(out, "Content:");
+
+        if (content != NULL) {
+            char *buf = pcmk__trim(dump_xml_formatted_with_text(content));
+
+            if (!pcmk__str_empty(buf)) {
+                out->info(out, "%s", buf);
+            }
+            free(buf);
+
+        } else {
+            out->info(out, "<unknown>");
+        }
+    }
+    if (pcmk_is_set(flags, shadow_disp_diff)) {
+        rc = out->info(out, "Diff:");
+
+        if (diff != NULL) {
+            out->message(out, "xml-patchset", diff);
+        } else {
+            out->info(out, "<empty>");
+        }
+    }
+
+    return rc;
+}
+
+/*!
+ * \internal
+ * \brief Display information about a shadow CIB instance
+ *
+ * \param[in,out] out  Output object
+ * \param[in]     ...  Message arguments
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note The variadic message arguments are of the following format:
+ *       -# Instance name (can be \p NULL)
+ *       -# Shadow file name (can be \p NULL)
+ *       -# Shadow file content (can be \p NULL)
+ *       -# Patchset containing the changes in the shadow CIB (can be \p NULL)
+ *       -# Group of \p shadow_disp_flags indicating which fields to display
+ */
+PCMK__OUTPUT_ARGS("shadow", "const char *", "const char *", "xmlNodePtr",
+                  "xmlNodePtr", "enum shadow_disp_flags")
+static int
+shadow_text(pcmk__output_t *out, va_list args)
+{
+    if (!out->is_quiet(out)) {
+        return shadow_default(out, args);
+
+    } else {
+        const char *instance = va_arg(args, const char *);
+        const char *filename = va_arg(args, const char *);
+        xmlNodePtr content = va_arg(args, xmlNodePtr);
+        xmlNodePtr diff = va_arg(args, xmlNodePtr);
+        enum shadow_disp_flags flags = (enum shadow_disp_flags) va_arg(args, int);
+
+        int rc = pcmk_rc_no_output;
+        bool quiet_orig = out->quiet;
+
+        /* We have to disable quiet mode for the "xml-patchset" message if we
+         * call it, so we might as well do so for this whole section.
+         */
+        out->quiet = false;
+
+        if (pcmk_is_set(flags, shadow_disp_instance) && (instance != NULL)) {
+            rc = out->info(out, "%s", instance);
+        }
+        if (pcmk_is_set(flags, shadow_disp_file) && (filename != NULL)) {
+            rc = out->info(out, "%s", filename);
+        }
+        if (pcmk_is_set(flags, shadow_disp_content) && (content != NULL)) {
+            char *buf = pcmk__trim(dump_xml_formatted_with_text(content));
+
+            rc = out->info(out, "%s", pcmk__trim(buf));
+            free(buf);
+        }
+        if (pcmk_is_set(flags, shadow_disp_diff) && (diff != NULL)) {
+            rc = out->message(out, "xml-patchset", diff);
+        }
+
+        out->quiet = quiet_orig;
+        return rc;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Display information about a shadow CIB instance
+ *
+ * \param[in,out] out  Output object
+ * \param[in]     ...  Message arguments
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note The variadic message arguments are of the following format:
+ *       -# Instance name (can be \p NULL)
+ *       -# Shadow file name (can be \p NULL)
+ *       -# Shadow file content (can be \p NULL)
+ *       -# Patchset containing the changes in the shadow CIB (can be \p NULL)
+ *       -# Group of \p shadow_disp_flags indicating which fields to display
+ *          (ignored)
+ */
+PCMK__OUTPUT_ARGS("shadow", "const char *", "const char *", "xmlNodePtr",
+                  "xmlNodePtr", "enum shadow_disp_flags")
+static int
+shadow_xml(pcmk__output_t *out, va_list args)
+{
+    const char *instance = va_arg(args, const char *);
+    const char *filename = va_arg(args, const char *);
+    xmlNodePtr content = va_arg(args, xmlNodePtr);
+    xmlNodePtr diff = va_arg(args, xmlNodePtr);
+    enum shadow_disp_flags flags G_GNUC_UNUSED =
+        (enum shadow_disp_flags) va_arg(args, int);
+
+    pcmk__output_xml_create_parent(out, "shadow",
+                                   "instance", instance,
+                                   "file", filename,
+                                   NULL);
+
+    if (content != NULL) {
+        char *buf = dump_xml_formatted_with_text(content);
+
+        out->output_xml(out, "content", buf);
+        free(buf);
+    }
+
+    if (diff != NULL) {
+        out->message(out, "xml-patchset", diff);
+    }
+
+    pcmk__output_xml_pop_parent(out);
+    return pcmk_rc_ok;
+}
+
+static const pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+static const pcmk__message_entry_t fmt_functions[] = {
+    { "instruction", "default", instruction_default },
+    { "instruction", "xml", instruction_xml },
+    { "shadow", "default", shadow_default },
+    { "shadow", "text", shadow_text },
+    { "shadow", "xml", shadow_xml },
+
+    { NULL, NULL, NULL }
+};
+
+/*!
+ * \internal
+ * \brief Set the error when \p --force is not passed with a dangerous command
+ *
+ * \param[in]  reason         Why command is dangerous
+ * \param[in]  for_shadow     If true, command is dangerous to the shadow file.
+ *                            Otherwise, command is dangerous to the active
+ *                            cluster.
+ * \param[in]  show_mismatch  If true and the supplied shadow instance is not
+ *                            the same as the active shadow instance, report
+ *                            this
+ * \param[out] error          Where to store error
+ */
+static void
+set_danger_error(const char *reason, bool for_shadow, bool show_mismatch,
+                 GError **error)
+{
+    const char *active = getenv("CIB_shadow");
+    char *full = NULL;
+
+    if (show_mismatch
+        && !pcmk__str_eq(active, options.instance, pcmk__str_null_matches)) {
+
+        full = crm_strdup_printf("%s.\nAdditionally, the supplied shadow "
+                                 "instance (%s) is not the same as the active "
+                                 "one (%s)",
+                                reason, options.instance, active);
+        reason = full;
+    }
+
+    g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                "%s%sTo prevent accidental destruction of the %s, the --force "
+                "flag is required in order to proceed.",
+                pcmk__s(reason, ""), ((reason != NULL)? ".\n" : ""),
+                (for_shadow? "shadow file" : "cluster"));
+    free(full);
+}
+
+/*!
+ * \internal
+ * \brief Get the active shadow instance from the environment
+ *
+ * This sets \p options.instance to the value of the \p CIB_shadow env variable.
+ *
+ * \param[out] error  Where to store error
+ */
+static int
+get_instance_from_env(GError **error)
+{
+    int rc = pcmk_rc_ok;
+
+    pcmk__str_update(&options.instance, getenv("CIB_shadow"));
+    if (options.instance == NULL) {
+        rc = ENXIO;
+        exit_code = pcmk_rc2exitc(rc);
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "No active shadow configuration defined");
+    }
+    return rc;
+}
+
+/*!
+ * \internal
+ * \brief Validate that the shadow file does or does not exist, as appropriate
+ *
+ * \param[in]  filename      Absolute path of shadow file
+ * \param[in]  should_exist  Whether the shadow file is expected to exist
+ * \param[out] error         Where to store error
+ *
+ * \return Standard Pacemaker return code
+ */
+static int
+check_file_exists(const char *filename, bool should_exist, GError **error)
+{
+    struct stat buf;
+
+    if (!should_exist && (stat(filename, &buf) == 0)) {
+        char *reason = crm_strdup_printf("A shadow instance '%s' already "
+                                         "exists", options.instance);
+
+        exit_code = CRM_EX_CANTCREAT;
+        set_danger_error(reason, true, false, error);
+        free(reason);
+        return EEXIST;
+    }
+
+    if (should_exist && (stat(filename, &buf) < 0)) {
+        // @COMPAT: Use pcmk_rc2exitc(errno)?
+        exit_code = CRM_EX_NOSUCH;
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not access shadow instance '%s': %s",
+                    options.instance, strerror(errno));
+        return errno;
+    }
+
+    return pcmk_rc_ok;
+}
+
+/*!
+ * \internal
+ * \brief Connect to the "real" (non-shadow) CIB
+ *
+ * \param[out] real_cib  Where to store CIB connection
+ * \param[out] error     Where to store error
+ *
+ * \return Standard Pacemaker return code
+ */
+static int
+connect_real_cib(cib_t **real_cib, GError **error)
+{
+    int rc = pcmk_rc_ok;
+
+    *real_cib = cib_new_no_shadow();
+    if (*real_cib == NULL) {
+        rc = ENOMEM;
+        exit_code = pcmk_rc2exitc(rc);
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not create a CIB connection object");
+        return rc;
+    }
+
+    rc = (*real_cib)->cmds->signon(*real_cib, crm_system_name, cib_command);
+    rc = pcmk_legacy2rc(rc);
+    if (rc != pcmk_rc_ok) {
+        exit_code = pcmk_rc2exitc(rc);
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not connect to CIB: %s", pcmk_rc_str(rc));
+    }
+    return rc;
+}
+
+/*!
+ * \internal
+ * \brief Query the "real" (non-shadow) CIB and store the result
+ *
+ * \param[out]    output    Where to store query output
+ * \param[out]    error     Where to store error
+ *
+ * \return Standard Pacemaker return code
+ */
+static int
+query_real_cib(xmlNode **output, GError **error)
+{
+    cib_t *real_cib = NULL;
+    int rc = connect_real_cib(&real_cib, error);
+
+    if (rc != pcmk_rc_ok) {
+        goto done;
+    }
+
+    rc = real_cib->cmds->query(real_cib, NULL, output, options.cmd_options);
+    rc = pcmk_legacy2rc(rc);
+    if (rc != pcmk_rc_ok) {
+        exit_code = pcmk_rc2exitc(rc);
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not query the non-shadow CIB: %s", pcmk_rc_str(rc));
+    }
+
+done:
+    cib_delete(real_cib);
+    return rc;
+}
+
+/*!
+ * \internal
+ * \brief Read XML from the given file
+ *
+ * \param[in]  filename  Path of input file
+ * \param[out] output    Where to store XML read from \p filename
+ * \param[out] error     Where to store error
+ *
+ * \return Standard Pacemaker return code
+ */
+static int
+read_xml(const char *filename, xmlNode **output, GError **error)
+{
+    int rc = pcmk_rc_ok;
+
+    *output = filename2xml(filename);
+    if (*output == NULL) {
+        rc = pcmk_rc_no_input;
+        exit_code = pcmk_rc2exitc(rc);
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not parse XML from input file '%s'", filename);
+    }
+    return rc;
+}
+
+/*!
+ * \internal
+ * \brief Write the shadow XML to a file
+ *
+ * \param[in,out] xml       Shadow XML
+ * \param[in]     filename  Name of destination file
+ * \param[in]     reset     Whether the write is a reset (for logging only)
+ * \param[out]    error     Where to store error
+ */
+static int
+write_shadow_file(xmlNode *xml, const char *filename, bool reset,
+                  GError **error)
+{
+    int rc = write_xml_file(xml, filename, FALSE);
+
+    if (rc < 0) {
+        rc = pcmk_legacy2rc(rc);
+        exit_code = pcmk_rc2exitc(rc);
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not %s the shadow instance '%s': %s",
+                    reset? "reset" : "create", options.instance,
+                    pcmk_rc_str(rc));
+        return rc;
+    }
+    return pcmk_rc_ok;
+}
+
+/*!
+ * \internal
+ * \brief Create a shell prompt based on the given shadow instance name
+ *
+ * \return Newly created prompt
+ *
+ * \note The caller is responsible for freeing the return value using \p free().
+ */
+static inline char *
+get_shadow_prompt(void)
+{
+    return crm_strdup_printf("shadow[%.40s] # ", options.instance);
+}
+
+/*!
+ * \internal
+ * \brief Set up environment variables for a shadow instance
+ *
+ * \param[in,out] out      Output object
+ * \param[in]     do_switch  If true, switch to an existing instance (logging
+ *                           only)
+ * \param[out]    error      Where to store error
+ */
+static void
+shadow_setup(pcmk__output_t *out, bool do_switch, GError **error)
+{
+    const char *active = getenv("CIB_shadow");
+    const char *prompt = getenv("PS1");
+    const char *shell = getenv("SHELL");
+    char *new_prompt = get_shadow_prompt();
+
+    if (pcmk__str_eq(active, options.instance, pcmk__str_none)
+        && pcmk__str_eq(new_prompt, prompt, pcmk__str_none)) {
+        // CIB_shadow and prompt environment variables are already set up
+        goto done;
+    }
+
+    if (!options.batch && (shell != NULL)) {
+        out->info(out, "Setting up shadow instance");
+        setenv("PS1", new_prompt, 1);
+        setenv("CIB_shadow", options.instance, 1);
+
+        out->message(out, "instruction",
+                     "Press Ctrl+D to exit the crm_shadow shell");
+
+        if (pcmk__str_eq(shell, "(^|/)bash$", pcmk__str_regex)) {
+            execl(shell, shell, "--norc", "--noprofile", NULL);
+        } else {
+            execl(shell, shell, NULL);
+        }
+
+        exit_code = pcmk_rc2exitc(errno);
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Failed to launch shell '%s': %s",
+                    shell, pcmk_rc_str(errno));
+
+    } else {
+        char *msg = NULL;
+        const char *prefix = "A new shadow instance was created. To begin "
+                             "using it";
+
+        if (do_switch) {
+            prefix = "To switch to the named shadow instance";
+        }
+
+        msg = crm_strdup_printf("%s, enter the following into your shell:\n"
+                                "\texport CIB_shadow=%s",
+                                prefix, options.instance);
+        out->message(out, "instruction", msg);
+        free(msg);
+    }
+
+done:
+    free(new_prompt);
+}
+
+/*!
+ * \internal
+ * \brief Remind the user to clean up the shadow environment
+ *
+ * \param[in,out] out  Output object
+ */
+static void
+shadow_teardown(pcmk__output_t *out)
+{
+    const char *active = getenv("CIB_shadow");
+    const char *prompt = getenv("PS1");
+
+    if (pcmk__str_eq(active, options.instance, pcmk__str_none)) {
+        char *our_prompt = get_shadow_prompt();
+
+        if (pcmk__str_eq(prompt, our_prompt, pcmk__str_none)) {
+            out->message(out, "instruction",
+                         "Press Ctrl+D to exit the crm_shadow shell");
+
+        } else {
+            out->message(out, "instruction",
+                         "Remember to unset the CIB_shadow variable by "
+                         "entering the following into your shell:\n"
+                         "\tunset CIB_shadow");
+        }
+        free(our_prompt);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Commit the shadow file contents to the active cluster
+ *
+ * \param[out] error  Where to store error
+ */
+static void
+commit_shadow_file(GError **error)
+{
+    char *filename = NULL;
+    cib_t *real_cib = NULL;
+
+    xmlNodePtr input = NULL;
+    xmlNodePtr section_xml = NULL;
+    const char *section = NULL;
+
+    int rc = pcmk_rc_ok;
+
+    if (!options.force) {
+        const char *reason = "The commit command overwrites the active cluster "
+                             "configuration";
+
+        exit_code = CRM_EX_USAGE;
+        set_danger_error(reason, false, true, error);
+        return;
+    }
+
+    filename = get_shadow_file(options.instance);
+    if (check_file_exists(filename, true, error) != pcmk_rc_ok) {
+        goto done;
+    }
+
+    if (connect_real_cib(&real_cib, error) != pcmk_rc_ok) {
+        goto done;
+    }
+
+    if (read_xml(filename, &input, error) != pcmk_rc_ok) {
+        goto done;
+    }
+
+    section_xml = input;
+
+    if (!options.full_upload) {
+        section = XML_CIB_TAG_CONFIGURATION;
+        section_xml = first_named_child(input, section);
+    }
+
+    rc = real_cib->cmds->replace(real_cib, section, section_xml,
+                                 options.cmd_options);
+    rc = pcmk_legacy2rc(rc);
+
+    if (rc != pcmk_rc_ok) {
+        exit_code = pcmk_rc2exitc(rc);
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not commit shadow instance '%s' to the CIB: %s",
+                    options.instance, pcmk_rc_str(rc));
+    }
+
+done:
+    free(filename);
+    cib_delete(real_cib);
+    free_xml(input);
+}
+
+/*!
+ * \internal
+ * \brief Create a new empty shadow instance
+ *
+ * \param[in,out] out    Output object
+ * \param[out]    error  Where to store error
+ *
+ * \note If \p --force is given, we try to write the file regardless of whether
+ *       it already exists.
+ */
+static void
+create_shadow_empty(pcmk__output_t *out, GError **error)
+{
+    char *filename = get_shadow_file(options.instance);
+    xmlNode *output = NULL;
+
+    if (!options.force
+        && (check_file_exists(filename, false, error) != pcmk_rc_ok)) {
+        goto done;
+    }
+
+    output = createEmptyCib(0);
+    crm_xml_add(output, XML_ATTR_VALIDATION, options.validate_with);
+    out->info(out, "Created new %s configuration",
+              crm_element_value(output, XML_ATTR_VALIDATION));
+
+    if (write_shadow_file(output, filename, false, error) != pcmk_rc_ok) {
+        goto done;
+    }
+    shadow_setup(out, false, error);
+
+done:
+    free(filename);
+    free_xml(output);
+}
+
+/*!
+ * \internal
+ * \brief Create a shadow instance based on the active CIB
+ *
+ * \param[in,out] out    Output object
+ * \param[in]     reset  If true, overwrite the given existing shadow instance.
+ *                       Otherwise, create a new shadow instance with the given
+ *                       name.
+ * \param[out]    error  Where to store error
+ *
+ * \note If \p --force is given, we try to write the file regardless of whether
+ *       it already exists.
+ */
+static void
+create_shadow_from_cib(pcmk__output_t *out, bool reset, GError **error)
+{
+    char *filename = get_shadow_file(options.instance);
+    xmlNode *output = NULL;
+
+    if (!options.force) {
+        if (reset) {
+            /* @COMPAT: Reset is dangerous to the shadow file, but to preserve
+             * compatibility we can't require --force unless there's a mismatch.
+             * At a compatibility break, call set_danger_error() with for_shadow
+             * and show_mismatch set to true.
+             */
+            const char *local = getenv("CIB_shadow");
+
+            if (!pcmk__str_eq(local, options.instance, pcmk__str_null_matches)) {
+                exit_code = CRM_EX_USAGE;
+                g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                            "The supplied shadow instance (%s) is not the same "
+                            "as the active one (%s).\n"
+                            "To prevent accidental destruction of the shadow "
+                            "file, the --force flag is required in order to "
+                            "proceed.",
+                            options.instance, local);
+                goto done;
+            }
+        }
+
+        if (check_file_exists(filename, reset, error) != pcmk_rc_ok) {
+            goto done;
+        }
+    }
+
+    if (query_real_cib(&output, error) != pcmk_rc_ok) {
+        goto done;
+    }
+
+    if (write_shadow_file(output, filename, reset, error) != pcmk_rc_ok) {
+        goto done;
+    }
+    shadow_setup(out, false, error);
+
+done:
+    free(filename);
+    free_xml(output);
+}
+
+/*!
+ * \internal
+ * \brief Delete the shadow file
+ *
+ * \param[in,out] out  Output object
+ * \param[out]    error  Where to store error
+ */
+static void
+delete_shadow_file(pcmk__output_t *out, GError **error)
+{
+    char *filename = NULL;
+
+    if (!options.force) {
+        const char *reason = "The delete command removes the specified shadow "
+                             "file";
+
+        exit_code = CRM_EX_USAGE;
+        set_danger_error(reason, true, true, error);
+        return;
+    }
+
+    filename = get_shadow_file(options.instance);
+
+    if ((unlink(filename) < 0) && (errno != ENOENT)) {
+        exit_code = pcmk_rc2exitc(errno);
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not remove shadow instance '%s': %s",
+                    options.instance, strerror(errno));
+    } else {
+        shadow_teardown(out);
+    }
+    free(filename);
+}
+
+/*!
+ * \internal
+ * \brief Open the shadow file in a text editor
+ *
+ * \param[out] error  Where to store error
+ *
+ * \note The \p EDITOR environment variable must be set.
+ */
+static void
+edit_shadow_file(GError **error)
+{
+    char *filename = NULL;
+    const char *editor = NULL;
+
+    if (get_instance_from_env(error) != pcmk_rc_ok) {
+        return;
+    }
+
+    filename = get_shadow_file(options.instance);
+    if (check_file_exists(filename, true, error) != pcmk_rc_ok) {
+        goto done;
+    }
+
+    editor = getenv("EDITOR");
+    if (editor == NULL) {
+        exit_code = CRM_EX_NOT_CONFIGURED;
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "No value for EDITOR defined");
+        goto done;
+    }
+
+    execlp(editor, "--", filename, NULL);
+    exit_code = CRM_EX_OSFILE;
+    g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                "Could not invoke EDITOR (%s %s): %s",
+                editor, filename, strerror(errno));
+
+done:
+    free(filename);
+}
+
+/*!
+ * \internal
+ * \brief Show the contents of the active shadow instance
+ *
+ * \param[in,out] out    Output object
+ * \param[out]    error  Where to store error
+ */
+static void
+show_shadow_contents(pcmk__output_t *out, GError **error)
+{
+    char *filename = NULL;
+
+    if (get_instance_from_env(error) != pcmk_rc_ok) {
+        return;
+    }
+
+    filename = get_shadow_file(options.instance);
+
+    if (check_file_exists(filename, true, error) == pcmk_rc_ok) {
+        xmlNode *output = NULL;
+        bool quiet_orig = out->quiet;
+
+        if (read_xml(filename, &output, error) != pcmk_rc_ok) {
+            goto done;
+        }
+
+        out->quiet = true;
+        out->message(out, "shadow",
+                     options.instance, NULL, output, NULL, shadow_disp_content);
+        out->quiet = quiet_orig;
+
+        free_xml(output);
+    }
+
+done:
+    free(filename);
+}
+
+/*!
+ * \internal
+ * \brief Show the changes in the active shadow instance
+ *
+ * \param[in,out] out    Output object
+ * \param[out]    error  Where to store error
+ */
+static void
+show_shadow_diff(pcmk__output_t *out, GError **error)
+{
+    char *filename = NULL;
+    xmlNodePtr old_config = NULL;
+    xmlNodePtr new_config = NULL;
+    xmlNodePtr diff = NULL;
+    pcmk__output_t *logger_out = NULL;
+    bool quiet_orig = out->quiet;
+    int rc = pcmk_rc_ok;
+
+    if (get_instance_from_env(error) != pcmk_rc_ok) {
+        return;
+    }
+
+    filename = get_shadow_file(options.instance);
+    if (check_file_exists(filename, true, error) != pcmk_rc_ok) {
+        goto done;
+    }
+
+    if (query_real_cib(&old_config, error) != pcmk_rc_ok) {
+        goto done;
+    }
+
+    if (read_xml(filename, &new_config, error) != pcmk_rc_ok) {
+        goto done;
+    }
+    xml_track_changes(new_config, NULL, new_config, false);
+    xml_calculate_changes(old_config, new_config);
+    diff = xml_create_patchset(0, old_config, new_config, NULL, false);
+
+    rc = pcmk__log_output_new(&logger_out);
+    if (rc != pcmk_rc_ok) {
+        exit_code = pcmk_rc2exitc(rc);
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not create logger object: %s", pcmk_rc_str(rc));
+        goto done;
+    }
+    pcmk__output_set_log_level(logger_out, LOG_INFO);
+    rc = pcmk__xml_show_changes(logger_out, new_config);
+    logger_out->finish(logger_out, pcmk_rc2exitc(rc), true, NULL);
+    pcmk__output_free(logger_out);
+
+    xml_accept_changes(new_config);
+
+    out->quiet = true;
+    out->message(out, "shadow",
+                 options.instance, NULL, NULL, diff, shadow_disp_diff);
+    out->quiet = quiet_orig;
+
+    if (diff != NULL) {
+        /* @COMPAT: Exit with CRM_EX_DIGEST? This is not really an error; we
+         * just want to indicate that there are differences (as the diff command
+         * does).
+         */
+        exit_code = CRM_EX_ERROR;
+    }
+
+done:
+    free(filename);
+    free_xml(old_config);
+    free_xml(new_config);
+    free_xml(diff);
+}
+
+/*!
+ * \internal
+ * \brief Show the absolute path of the active shadow instance
+ *
+ * \param[in,out] out    Output object
+ * \param[out]    error  Where to store error
+ */
+static void
+show_shadow_filename(pcmk__output_t *out, GError **error)
+{
+    if (get_instance_from_env(error) == pcmk_rc_ok) {
+        char *filename = get_shadow_file(options.instance);
+        bool quiet_orig = out->quiet;
+
+        out->quiet = true;
+        out->message(out, "shadow",
+                     options.instance, filename, NULL, NULL, shadow_disp_file);
+        out->quiet = quiet_orig;
+
+        free(filename);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Show the active shadow instance
+ *
+ * \param[in,out] out    Output object
+ * \param[out]    error  Where to store error
+ */
+static void
+show_shadow_instance(pcmk__output_t *out, GError **error)
+{
+    if (get_instance_from_env(error) == pcmk_rc_ok) {
+        bool quiet_orig = out->quiet;
+
+        out->quiet = true;
+        out->message(out, "shadow",
+                     options.instance, NULL, NULL, NULL, shadow_disp_instance);
+        out->quiet = quiet_orig;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Switch to the given shadow instance
+ *
+ * \param[in,out] out    Output object
+ * \param[out]    error  Where to store error
+ */
+static void
+switch_shadow_instance(pcmk__output_t *out, GError **error)
+{
+    char *filename = NULL;
+
+    filename = get_shadow_file(options.instance);
+    if (check_file_exists(filename, true, error) == pcmk_rc_ok) {
+        shadow_setup(out, true, error);
+    }
+    free(filename);
+}
+
+static gboolean
+command_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+           GError **error)
+{
+    if (pcmk__str_any_of(option_name, "-w", "--which", NULL)) {
+        options.cmd = shadow_cmd_which;
+
+    } else if (pcmk__str_any_of(option_name, "-p", "--display", NULL)) {
+        options.cmd = shadow_cmd_display;
+
+    } else if (pcmk__str_any_of(option_name, "-d", "--diff", NULL)) {
+        options.cmd = shadow_cmd_diff;
+
+    } else if (pcmk__str_any_of(option_name, "-F", "--file", NULL)) {
+        options.cmd = shadow_cmd_file;
+
+    } else if (pcmk__str_any_of(option_name, "-c", "--create", NULL)) {
+        options.cmd = shadow_cmd_create;
+
+    } else if (pcmk__str_any_of(option_name, "-e", "--create-empty", NULL)) {
+        options.cmd = shadow_cmd_create_empty;
+
+    } else if (pcmk__str_any_of(option_name, "-C", "--commit", NULL)) {
+        options.cmd = shadow_cmd_commit;
+
+    } else if (pcmk__str_any_of(option_name, "-D", "--delete", NULL)) {
+        options.cmd = shadow_cmd_delete;
+
+    } else if (pcmk__str_any_of(option_name, "-E", "--edit", NULL)) {
+        options.cmd = shadow_cmd_edit;
+
+    } else if (pcmk__str_any_of(option_name, "-r", "--reset", NULL)) {
+        options.cmd = shadow_cmd_reset;
+
+    } else if (pcmk__str_any_of(option_name, "-s", "--switch", NULL)) {
+        options.cmd = shadow_cmd_switch;
+
+    } else {
+        // Should be impossible
+        return FALSE;
+    }
+
+    // optarg may be NULL and that's okay
+    pcmk__str_update(&options.instance, optarg);
+    return TRUE;
+}
+
+static GOptionEntry query_entries[] = {
+    { "which", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Indicate the active shadow copy", NULL },
+
+    { "display", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the contents of the active shadow copy", NULL },
+
+    { "diff", 'd', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the changes in the active shadow copy", NULL },
+
+    { "file", 'F', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the location of the active shadow copy file", NULL },
+
+    { NULL }
+};
+
+static GOptionEntry command_entries[] = {
+    { "create", 'c', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, command_cb,
+      "Create the named shadow copy of the active cluster configuration",
+      "name" },
+
+    { "create-empty", 'e', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
+      command_cb,
+      "Create the named shadow copy with an empty cluster configuration.\n"
+      INDENT "Optional: --validate-with", "name" },
+
+    { "commit", 'C', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, command_cb,
+      "Upload the contents of the named shadow copy to the cluster", "name" },
+
+    { "delete", 'D', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, command_cb,
+      "Delete the contents of the named shadow copy", "name" },
+
+    { "edit", 'E', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Edit the contents of the active shadow copy with your favorite $EDITOR",
+      NULL },
+
+    { "reset", 'r', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, command_cb,
+      "Recreate named shadow copy from the active cluster configuration",
+      "name" },
+
+    { "switch", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, command_cb,
+      "(Advanced) Switch to the named shadow copy", "name" },
+
+    { NULL }
+};
+
+static GOptionEntry addl_entries[] = {
+    { "force", 'f', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.force,
+      "(Advanced) Force the action to be performed", NULL },
+
+    { "batch", 'b', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.batch,
+      "(Advanced) Don't spawn a new shell", NULL },
+
+    { "all", 'a', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.full_upload,
+      "(Advanced) Upload entire CIB, including status, with --commit", NULL },
+
+    { "validate-with", 'v', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING,
+      &options.validate_with,
+      "(Advanced) Create an older configuration version", NULL },
+
+    { NULL }
+};
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group)
+{
+    const char *desc = NULL;
+    GOptionContext *context = NULL;
+
+    desc = "Examples:\n\n"
+           "Create a blank shadow configuration:\n\n"
+           "\t# crm_shadow --create-empty myShadow\n\n"
+           "Create a shadow configuration from the running cluster\n\n"
+           "\t# crm_shadow --create myShadow\n\n"
+           "Display the current shadow configuration:\n\n"
+           "\t# crm_shadow --display\n\n"
+           "Discard the current shadow configuration (named myShadow):\n\n"
+           "\t# crm_shadow --delete myShadow --force\n\n"
+           "Upload current shadow configuration (named myShadow) to running "
+           "cluster:\n\n"
+           "\t# crm_shadow --commit myShadow\n\n";
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group,
+                                      "<query>|<command>");
+    g_option_context_set_description(context, desc);
+
+    pcmk__add_arg_group(context, "queries", "Queries:",
+                        "Show query help", query_entries);
+    pcmk__add_arg_group(context, "commands", "Commands:",
+                        "Show command help", command_entries);
+    pcmk__add_arg_group(context, "additional", "Additional Options:",
+                        "Show additional options", addl_entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    int rc = pcmk_rc_ok;
+    pcmk__output_t *out = NULL;
+
+    GError *error = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "CDcersv");
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    crm_log_preinit(NULL, argc, argv);
+
+    pcmk__register_formats(output_group, formats);
+
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Error creating output format %s: %s", args->output_ty,
+                    pcmk_rc_str(rc));
+        goto done;
+    }
+
+    if (g_strv_length(processed_args) > 1) {
+        gchar *help = g_option_context_get_help(context, TRUE, NULL);
+        GString *extra = g_string_sized_new(128);
+
+        for (int lpc = 1; processed_args[lpc] != NULL; lpc++) {
+            if (extra->len > 0) {
+                g_string_append_c(extra, ' ');
+            }
+            g_string_append(extra, processed_args[lpc]);
+        }
+
+        exit_code = CRM_EX_USAGE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "non-option ARGV-elements: %s\n\n%s", extra->str, help);
+        g_free(help);
+        g_string_free(extra, TRUE);
+        goto done;
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    pcmk__register_messages(out, fmt_functions);
+
+    if (options.cmd == shadow_cmd_none) {
+        // @COMPAT: Create a default command if other tools have one
+        gchar *help = g_option_context_get_help(context, TRUE, NULL);
+
+        exit_code = CRM_EX_USAGE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Must specify a query or command option\n\n%s", help);
+        g_free(help);
+        goto done;
+    }
+
+    pcmk__cli_init_logging("crm_shadow", args->verbosity);
+
+    if (args->verbosity > 0) {
+        cib__set_call_options(options.cmd_options, crm_system_name,
+                              cib_verbose);
+    }
+
+    // Run the command
+    switch (options.cmd) {
+        case shadow_cmd_commit:
+            commit_shadow_file(&error);
+            break;
+        case shadow_cmd_create:
+            create_shadow_from_cib(out, false, &error);
+            break;
+        case shadow_cmd_create_empty:
+            create_shadow_empty(out, &error);
+            break;
+        case shadow_cmd_reset:
+            create_shadow_from_cib(out, true, &error);
+            break;
+        case shadow_cmd_delete:
+            delete_shadow_file(out, &error);
+            break;
+        case shadow_cmd_diff:
+            show_shadow_diff(out, &error);
+            break;
+        case shadow_cmd_display:
+            show_shadow_contents(out, &error);
+            break;
+        case shadow_cmd_edit:
+            edit_shadow_file(&error);
+            break;
+        case shadow_cmd_file:
+            show_shadow_filename(out, &error);
+            break;
+        case shadow_cmd_switch:
+            switch_shadow_instance(out, &error);
+            break;
+        case shadow_cmd_which:
+            show_shadow_instance(out, &error);
+            break;
+        default:
+            // Should never reach this point
+            break;
+    }
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    free(options.instance);
+    g_free(options.validate_with);
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+
+    crm_exit(exit_code);
+}
diff --git a/tools/crm_simulate.8.inc b/tools/crm_simulate.8.inc
new file mode 100644
index 0000000..8f37f5b
--- /dev/null
+++ b/tools/crm_simulate.8.inc
@@ -0,0 +1,8 @@
+[synopsis]
+crm_simulate <data source> <operation> [options]
+
+/response to events/
+.SH OPTIONS
+
+/only essential output/
+.SH OPERATION SPECIFICATION
diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c
new file mode 100644
index 0000000..932c5bd
--- /dev/null
+++ b/tools/crm_simulate.c
@@ -0,0 +1,587 @@
+/*
+ * Copyright 2009-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdint.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <time.h>
+
+#include <sys/stat.h>
+#include <sys/param.h>
+#include <sys/types.h>
+#include <dirent.h>
+
+#include <crm/crm.h>
+#include <crm/cib.h>
+#include <crm/cib/internal.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/output_internal.h>
+#include <crm/common/output.h>
+#include <crm/common/util.h>
+#include <crm/common/iso8601.h>
+#include <crm/pengine/status.h>
+#include <pacemaker-internal.h>
+#include <pacemaker.h>
+
+#define SUMMARY "crm_simulate - simulate a Pacemaker cluster's response to events"
+
+struct {
+    char *dot_file;
+    char *graph_file;
+    gchar *input_file;
+    pcmk_injections_t *injections;
+    unsigned int flags;
+    gchar *output_file;
+    long long repeat;
+    gboolean store;
+    gchar *test_dir;
+    char *use_date;
+    char *xml_file;
+} options = {
+    .flags = pcmk_sim_show_pending | pcmk_sim_sanitized,
+    .repeat = 1
+};
+
+uint32_t section_opts = 0;
+char *temp_shadow = NULL;
+crm_exit_t exit_code = CRM_EX_OK;
+
+#define INDENT "                                   "
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+static gboolean
+all_actions_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.flags |= pcmk_sim_all_actions;
+    return TRUE;
+}
+
+static gboolean
+attrs_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    section_opts |= pcmk_section_attributes;
+    return TRUE;
+}
+
+static gboolean
+failcounts_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    section_opts |= pcmk_section_failcounts | pcmk_section_failures;
+    return TRUE;
+}
+
+static gboolean
+in_place_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.store = TRUE;
+    options.flags |= pcmk_sim_process | pcmk_sim_simulate;
+    return TRUE;
+}
+
+static gboolean
+live_check_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (options.xml_file) {
+        free(options.xml_file);
+    }
+
+    options.xml_file = NULL;
+    options.flags &= ~pcmk_sim_sanitized;
+    return TRUE;
+}
+
+static gboolean
+node_down_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.injections->node_down = g_list_append(options.injections->node_down, g_strdup(optarg));
+    return TRUE;
+}
+
+static gboolean
+node_fail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.injections->node_fail = g_list_append(options.injections->node_fail, g_strdup(optarg));
+    return TRUE;
+}
+
+static gboolean
+node_up_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    pcmk__simulate_node_config = true;
+    options.injections->node_up = g_list_append(options.injections->node_up, g_strdup(optarg));
+    return TRUE;
+}
+
+static gboolean
+op_fail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.flags |= pcmk_sim_process | pcmk_sim_simulate;
+    options.injections->op_fail = g_list_append(options.injections->op_fail, g_strdup(optarg));
+    return TRUE;
+}
+
+static gboolean
+op_inject_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.injections->op_inject = g_list_append(options.injections->op_inject, g_strdup(optarg));
+    return TRUE;
+}
+
+static gboolean
+pending_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.flags |= pcmk_sim_show_pending;
+    return TRUE;
+}
+
+static gboolean
+process_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.flags |= pcmk_sim_process;
+    return TRUE;
+}
+
+static gboolean
+quorum_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    pcmk__str_update(&options.injections->quorum, optarg);
+    return TRUE;
+}
+
+static gboolean
+save_dotfile_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.flags |= pcmk_sim_process;
+    pcmk__str_update(&options.dot_file, optarg);
+    return TRUE;
+}
+
+static gboolean
+save_graph_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.flags |= pcmk_sim_process;
+    pcmk__str_update(&options.graph_file, optarg);
+    return TRUE;
+}
+
+static gboolean
+show_scores_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.flags |= pcmk_sim_process | pcmk_sim_show_scores;
+    return TRUE;
+}
+
+static gboolean
+simulate_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.flags |= pcmk_sim_process | pcmk_sim_simulate;
+    return TRUE;
+}
+
+static gboolean
+ticket_activate_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.injections->ticket_activate = g_list_append(options.injections->ticket_activate, g_strdup(optarg));
+    return TRUE;
+}
+
+static gboolean
+ticket_grant_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.injections->ticket_grant = g_list_append(options.injections->ticket_grant, g_strdup(optarg));
+    return TRUE;
+}
+
+static gboolean
+ticket_revoke_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.injections->ticket_revoke = g_list_append(options.injections->ticket_revoke, g_strdup(optarg));
+    return TRUE;
+}
+
+static gboolean
+ticket_standby_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.injections->ticket_standby = g_list_append(options.injections->ticket_standby, g_strdup(optarg));
+    return TRUE;
+}
+
+static gboolean
+utilization_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.flags |= pcmk_sim_process | pcmk_sim_show_utilization;
+    return TRUE;
+}
+
+static gboolean
+watchdog_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    pcmk__str_update(&options.injections->watchdog, optarg);
+    return TRUE;
+}
+
+static gboolean
+xml_file_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    pcmk__str_update(&options.xml_file, optarg);
+    options.flags |= pcmk_sim_sanitized;
+    return TRUE;
+}
+
+static gboolean
+xml_pipe_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    pcmk__str_update(&options.xml_file, "-");
+    options.flags |= pcmk_sim_sanitized;
+    return TRUE;
+}
+
+static GOptionEntry operation_entries[] = {
+    { "run", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, process_cb,
+      "Process the supplied input and show what actions the cluster will take in response",
+      NULL },
+    { "simulate", 'S', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, simulate_cb,
+      "Like --run, but also simulate taking those actions and show the resulting new status",
+      NULL },
+    { "in-place", 'X', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, in_place_cb,
+      "Like --simulate, but also store the results back to the input file",
+      NULL },
+    { "show-attrs", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attrs_cb,
+      "Show node attributes",
+      NULL },
+    { "show-failcounts", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, failcounts_cb,
+      "Show resource fail counts",
+      NULL },
+    { "show-scores", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_scores_cb,
+      "Show allocation scores",
+      NULL },
+    { "show-utilization", 'U', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, utilization_cb,
+      "Show utilization information",
+      NULL },
+    { "profile", 'P', 0, G_OPTION_ARG_FILENAME, &options.test_dir,
+      "Process all the XML files in the named directory to create profiling data",
+      "DIR" },
+    { "repeat", 'N', 0, G_OPTION_ARG_INT, &options.repeat,
+      "With --profile, repeat each test N times and print timings",
+      "N" },
+    /* Deprecated */
+    { "pending", 'j', G_OPTION_FLAG_NO_ARG|G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, pending_cb,
+      "Display pending state if 'record-pending' is enabled",
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry synthetic_entries[] = {
+    { "node-up", 'u', 0, G_OPTION_ARG_CALLBACK, node_up_cb,
+      "Simulate bringing a node online",
+      "NODE" },
+    { "node-down", 'd', 0, G_OPTION_ARG_CALLBACK, node_down_cb,
+      "Simulate taking a node offline",
+      "NODE" },
+    { "node-fail", 'f', 0, G_OPTION_ARG_CALLBACK, node_fail_cb,
+      "Simulate a node failing",
+      "NODE" },
+    { "op-inject", 'i', 0, G_OPTION_ARG_CALLBACK, op_inject_cb,
+      "Generate a failure for the cluster to react to in the simulation.\n"
+      INDENT "See `Operation Specification` help for more information.",
+      "OPSPEC" },
+    { "op-fail", 'F', 0, G_OPTION_ARG_CALLBACK, op_fail_cb,
+      "If the specified task occurs during the simulation, have it fail with return code ${rc}.\n"
+      INDENT "The transition will normally stop at the failed action.\n"
+      INDENT "Save the result with --save-output and re-run with --xml-file.\n"
+      INDENT "See `Operation Specification` help for more information.",
+      "OPSPEC" },
+    { "set-datetime", 't', 0, G_OPTION_ARG_STRING, &options.use_date,
+      "Set date/time (ISO 8601 format, see https://en.wikipedia.org/wiki/ISO_8601)",
+      "DATETIME" },
+    { "quorum", 'q', 0, G_OPTION_ARG_CALLBACK, quorum_cb,
+      "Set to '1' (or 'true') to indicate cluster has quorum",
+      "QUORUM" },
+    { "watchdog", 'w', 0, G_OPTION_ARG_CALLBACK, watchdog_cb,
+      "Set to '1' (or 'true') to indicate cluster has an active watchdog device",
+      "DEVICE" },
+    { "ticket-grant", 'g', 0, G_OPTION_ARG_CALLBACK, ticket_grant_cb,
+      "Simulate granting a ticket",
+      "TICKET" },
+    { "ticket-revoke", 'r', 0, G_OPTION_ARG_CALLBACK, ticket_revoke_cb,
+      "Simulate revoking a ticket",
+      "TICKET" },
+    { "ticket-standby", 'b', 0, G_OPTION_ARG_CALLBACK, ticket_standby_cb,
+      "Simulate making a ticket standby",
+      "TICKET" },
+    { "ticket-activate", 'e', 0, G_OPTION_ARG_CALLBACK, ticket_activate_cb,
+      "Simulate activating a ticket",
+      "TICKET" },
+
+    { NULL }
+};
+
+static GOptionEntry artifact_entries[] = {
+    { "save-input", 'I', 0, G_OPTION_ARG_FILENAME, &options.input_file,
+      "Save the input configuration to the named file",
+      "FILE" },
+    { "save-output", 'O', 0, G_OPTION_ARG_FILENAME, &options.output_file,
+      "Save the output configuration to the named file",
+      "FILE" },
+    { "save-graph", 'G', 0, G_OPTION_ARG_CALLBACK, save_graph_cb,
+      "Save the transition graph (XML format) to the named file",
+      "FILE" },
+    { "save-dotfile", 'D', 0, G_OPTION_ARG_CALLBACK, save_dotfile_cb,
+      "Save the transition graph (DOT format) to the named file",
+      "FILE" },
+    { "all-actions", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, all_actions_cb,
+      "Display all possible actions in DOT graph (even if not part of transition)",
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry source_entries[] = {
+    { "live-check", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, live_check_cb,
+      "Connect to CIB manager and use the current CIB contents as input",
+      NULL },
+    { "xml-file", 'x', 0, G_OPTION_ARG_CALLBACK, xml_file_cb,
+      "Retrieve XML from the named file",
+      "FILE" },
+    { "xml-pipe", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, xml_pipe_cb,
+      "Retrieve XML from stdin",
+      NULL },
+
+    { NULL }
+};
+
+static int
+setup_input(pcmk__output_t *out, const char *input, const char *output,
+            GError **error)
+{
+    int rc = pcmk_rc_ok;
+    xmlNode *cib_object = NULL;
+    char *local_output = NULL;
+
+    if (input == NULL) {
+        /* Use live CIB */
+        rc = cib__signon_query(out, NULL, &cib_object);
+        if (rc != pcmk_rc_ok) {
+            // cib__signon_query() outputs any relevant error
+            return rc;
+        }
+
+    } else if (pcmk__str_eq(input, "-", pcmk__str_casei)) {
+        cib_object = filename2xml(NULL);
+
+    } else {
+        cib_object = filename2xml(input);
+    }
+
+    if (pcmk_find_cib_element(cib_object, XML_CIB_TAG_STATUS) == NULL) {
+        create_xml_node(cib_object, XML_CIB_TAG_STATUS);
+    }
+
+    if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
+        free_xml(cib_object);
+        return pcmk_rc_transform_failed;
+    }
+
+    if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
+        free_xml(cib_object);
+        return pcmk_rc_schema_validation;
+    }
+
+    if (output == NULL) {
+        char *pid = pcmk__getpid_s();
+
+        local_output = get_shadow_file(pid);
+        temp_shadow = strdup(local_output);
+        output = local_output;
+        free(pid);
+    }
+
+    rc = write_xml_file(cib_object, output, FALSE);
+    free_xml(cib_object);
+    cib_object = NULL;
+
+    if (rc < 0) {
+        rc = pcmk_legacy2rc(rc);
+        g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_CANTCREAT,
+                    "Could not create '%s': %s", output, pcmk_rc_str(rc));
+        return rc;
+    } else {
+        setenv("CIB_file", output, 1);
+        free(local_output);
+        return pcmk_rc_ok;
+    }
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    GOptionEntry extra_prog_entries[] = {
+        { "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
+          "Display only essential output",
+          NULL },
+
+        { NULL }
+    };
+
+    const char *description = "Operation Specification:\n\n"
+                              "The OPSPEC in any command line option is of the form\n"
+                              "${resource}_${task}_${interval_in_ms}@${node}=${rc}\n"
+                              "(memcached_monitor_20000@bart.example.com=7, for example).\n"
+                              "${rc} is an OCF return code.  For more information on these\n"
+                              "return codes, refer to https://clusterlabs.org/pacemaker/doc/2.1/Pacemaker_Administration/html/agents.html#ocf-return-codes\n\n"
+                              "Examples:\n\n"
+                              "Pretend a recurring monitor action found memcached stopped on node\n"
+                              "fred.example.com and, during recovery, that the memcached stop\n"
+                              "action failed:\n\n"
+                              "\tcrm_simulate -LS --op-inject memcached:0_monitor_20000@bart.example.com=7 "
+                              "--op-fail memcached:0_stop_0@fred.example.com=1 --save-output /tmp/memcached-test.xml\n\n"
+                              "Now see what the reaction to the stop failed would be:\n\n"
+                              "\tcrm_simulate -S --xml-file /tmp/memcached-test.xml\n\n";
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
+    pcmk__add_main_args(context, extra_prog_entries);
+    g_option_context_set_description(context, description);
+
+    pcmk__add_arg_group(context, "operations", "Operations:",
+                        "Show operations options", operation_entries);
+    pcmk__add_arg_group(context, "synthetic", "Synthetic Cluster Events:",
+                        "Show synthetic cluster event options", synthetic_entries);
+    pcmk__add_arg_group(context, "artifact", "Artifact Options:",
+                        "Show artifact options", artifact_entries);
+    pcmk__add_arg_group(context, "source", "Data Source:",
+                        "Show data source options", source_entries);
+
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    int rc = pcmk_rc_ok;
+    pe_working_set_t *data_set = NULL;
+    pcmk__output_t *out = NULL;
+
+    GError *error = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "bdefgiqrtuwxDFGINOP");
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    options.injections = calloc(1, sizeof(pcmk_injections_t));
+    if (options.injections == NULL) {
+        rc = ENOMEM;
+        goto done;
+    }
+
+    /* This must come before g_option_context_parse_strv. */
+    options.xml_file = strdup("-");
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("crm_simulate", args->verbosity);
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        fprintf(stderr, "Error creating output format %s: %s\n",
+                args->output_ty, pcmk_rc_str(rc));
+        exit_code = CRM_EX_ERROR;
+        goto done;
+    }
+
+    if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches) &&
+        !pcmk_is_set(options.flags, pcmk_sim_show_scores) &&
+        !pcmk_is_set(options.flags, pcmk_sim_show_utilization)) {
+        pcmk__force_args(context, &error, "%s --text-fancy", g_get_prgname());
+    } else if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
+        pcmk__force_args(context, &error, "%s --xml-simple-list --xml-substitute", g_get_prgname());
+    }
+
+    pe__register_messages(out);
+    pcmk__register_lib_messages(out);
+
+    out->quiet = args->quiet;
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    if (args->verbosity > 0) {
+        options.flags |= pcmk_sim_verbose;
+
+#ifdef PCMK__COMPAT_2_0
+        /* Redirect stderr to stdout so we can grep the output */
+        close(STDERR_FILENO);
+        dup2(STDOUT_FILENO, STDERR_FILENO);
+#endif
+    }
+
+    data_set = pe_new_working_set();
+    if (data_set == NULL) {
+        rc = ENOMEM;
+        g_set_error(&error, PCMK__RC_ERROR, rc, "Could not allocate working set");
+        goto done;
+    }
+
+    if (pcmk_is_set(options.flags, pcmk_sim_show_scores)) {
+        pe__set_working_set_flags(data_set, pe_flag_show_scores);
+    }
+    if (pcmk_is_set(options.flags, pcmk_sim_show_utilization)) {
+        pe__set_working_set_flags(data_set, pe_flag_show_utilization);
+    }
+    pe__set_working_set_flags(data_set, pe_flag_no_compat);
+
+    if (options.test_dir != NULL) {
+        data_set->priv = out;
+        pcmk__profile_dir(options.test_dir, options.repeat, data_set, options.use_date);
+        rc = pcmk_rc_ok;
+        goto done;
+    }
+
+    rc = setup_input(out, options.xml_file,
+                     options.store? options.xml_file : options.output_file,
+                     &error);
+    if (rc != pcmk_rc_ok) {
+        goto done;
+    }
+
+    rc = pcmk__simulate(data_set, out, options.injections, options.flags, section_opts,
+                        options.use_date, options.input_file, options.graph_file,
+                        options.dot_file);
+
+  done:
+    pcmk__output_and_clear_error(&error, NULL);
+
+    /* There sure is a lot to free in options. */
+    free(options.dot_file);
+    free(options.graph_file);
+    g_free(options.input_file);
+    g_free(options.output_file);
+    g_free(options.test_dir);
+    free(options.use_date);
+    free(options.xml_file);
+
+    pcmk_free_injections(options.injections);
+    pcmk__free_arg_context(context);
+    g_strfreev(processed_args);
+
+    if (data_set) {
+        pe_free_working_set(data_set);
+    }
+
+    fflush(stderr);
+
+    if (temp_shadow) {
+        unlink(temp_shadow);
+        free(temp_shadow);
+    }
+
+    if (rc != pcmk_rc_ok) {
+        exit_code = pcmk_rc2exitc(rc);
+    }
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+
+    pcmk__unregister_formats();
+    crm_exit(exit_code);
+}
diff --git a/tools/crm_standby.in b/tools/crm_standby.in
new file mode 100755
index 0000000..0911b9d
--- /dev/null
+++ b/tools/crm_standby.in
@@ -0,0 +1,158 @@
+#!@BASH_PATH@
+#
+# Copyright 2009-2018 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+USAGE_TEXT="Usage: crm_standby <command> [options]
+
+Common options:
+ --help                 Display this text, then exit
+ --version              Display version information, then exit
+ -V, --verbose          Specify multiple times to increase debug output
+ -q, --quiet            Print only the standby status (if querying)
+
+Commands:
+ -G, --query            Query the current value of standby mode (on/off)
+ -v, --update=VALUE     Update the value of standby mode (on/off)
+ -D, --delete           Let standby mode use default value
+
+Additional Options:
+ -N, --node=NODE        Operate on the named node instead of the current one
+ -l, --lifetime=VALUE   Until when should the setting take effect
+                        (valid values: reboot, forever)
+ -i, --id=VALUE         (Advanced) XML ID used to identify standby attribute"
+
+HELP_TEXT="crm_standby - Query, enable, or disable standby mode for a node
+
+Nodes in standby mode may not host cluster resources.
+
+$USAGE_TEXT
+"
+
+exit_usage() {
+    if [ $# -gt 0 ]; then
+        echo "error:" "$@" >&2
+    fi
+    echo
+    echo "$USAGE_TEXT"
+    exit 1
+}
+
+op=""
+options=""
+lifetime=0
+target=""
+
+SHORTOPTS_DEPRECATED="U:Q"
+LONGOPTS_DEPRECATED="uname:,get-value,delete-attr,attr-value:,attr-id:"
+SHORTOPTS="VqGv:DN:l:i:"
+LONGOPTS="help,version,verbose,quiet,query,update:,delete,node:,lifetime:,id:"
+
+TEMP=$(@GETOPT_PATH@ -o ${SHORTOPTS}${SHORTOPTS_DEPRECATED} \
+    --long ${LONGOPTS},${LONGOPTS_DEPRECATED} \
+    -n crm_standby -- "$@")
+if [ $? -ne 0 ]; then
+    exit_usage
+fi
+
+eval set -- "$TEMP" # Quotes around $TEMP are essential
+
+while true ; do
+    case "$1" in
+        --help)
+            echo "$HELP_TEXT"
+            exit 0
+            ;;
+        --version)
+            crm_attribute --version
+            exit 0
+            ;;
+        -q|--quiet|-V|--verbose|-Q)
+            options="$options $1"
+            shift
+            ;;
+        -N|--node|-U|--uname)
+            target="$2"
+            shift
+            shift
+            ;;
+        -G|--query|--get-value)
+            options="$options --query"
+            op=g
+            shift
+            ;;
+        -v|--update|--attr-value)
+            options="$options --update $2"
+            op=u
+            shift
+            shift
+            ;;
+        -D|--delete|--delete-attr)
+            options="$options --delete"
+            op=d
+            shift
+            ;;
+        -l|--lifetime)
+            options="$options --lifetime $2"
+            lifetime=1
+            shift
+            shift
+            ;;
+        -i|--id|--attr-id)
+            options="$options --id $2"
+            shift
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *)
+            exit_usage "unknown option '$1'"
+            ;;
+    esac
+done
+
+# It's important to call cluster commands only after arguments are processed,
+# so --version and --help work without problems even if those commands don't.
+if [ "$target" = "" ]; then
+    target=$(crm_node -n)
+fi
+
+options="-N $target -n standby $options"
+if [ x$op = x ]; then
+    options="$options -G"; op=g
+fi
+
+# If the user didn't explicitly specify a lifetime ...
+if [ $lifetime -eq 0 ]; then
+    case $op in
+        g)
+            # For query, report the forever entry if one exists, otherwise
+            # report the reboot entry if one exists, otherwise report off.
+            crm_attribute $options -l forever >/dev/null 2>&1
+            if [ $? -eq 0 ]; then
+                options="$options -l forever"
+            else
+                options="$options -l reboot -d off"
+            fi
+            ;;
+        u)
+            # For update, default to updating the forever entry.
+            options="$options -l forever"
+            ;;
+        d)
+            # For delete, default to deleting both forever and reboot entries.
+            crm_attribute $options -l forever
+            crm_attribute $options -l reboot
+            exit 0
+            ;;
+    esac
+fi
+
+crm_attribute $options
diff --git a/tools/crm_ticket.8.inc b/tools/crm_ticket.8.inc
new file mode 100644
index 0000000..5ae5124
--- /dev/null
+++ b/tools/crm_ticket.8.inc
@@ -0,0 +1,5 @@
+[synopsis]
+crm_ticket <query>|<command> [options]
+
+/and deleted./
+.SH OPTIONS
diff --git a/tools/crm_ticket.c b/tools/crm_ticket.c
new file mode 100644
index 0000000..c451e8a
--- /dev/null
+++ b/tools/crm_ticket.c
@@ -0,0 +1,1007 @@
+/*
+ * Copyright 2012-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+
+#include <crm/crm.h>
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <libgen.h>
+
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/common/ipc.h>
+#include <crm/common/cmdline_internal.h>
+
+#include <crm/cib.h>
+#include <crm/cib/internal.h>
+#include <crm/pengine/rules.h>
+#include <crm/pengine/status.h>
+
+#include <pacemaker-internal.h>
+
+GError *error = NULL;
+
+#define SUMMARY "Perform tasks related to cluster tickets\n\n" \
+                "Allows ticket attributes to be queried, modified and deleted."
+
+struct {
+    gchar *attr_default;
+    gchar *attr_id;
+    char *attr_name;
+    char *attr_value;
+    gboolean force;
+    char *get_attr_name;
+    gboolean quiet;
+    gchar *set_name;
+    char ticket_cmd;
+    gchar *ticket_id;
+    gchar *xml_file;
+} options = {
+    .ticket_cmd = 'S'
+};
+
+GList *attr_delete;
+GHashTable *attr_set;
+bool modified = false;
+int cib_options = cib_sync_call;
+
+#define INDENT "                               "
+
+static gboolean
+attr_value_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    pcmk__str_update(&options.attr_value, optarg);
+
+    if (!options.attr_name || !options.attr_value) {
+        return TRUE;
+    }
+
+    g_hash_table_insert(attr_set, strdup(options.attr_name), strdup(options.attr_value));
+    pcmk__str_update(&options.attr_name, NULL);
+    pcmk__str_update(&options.attr_value, NULL);
+
+    modified = true;
+
+    return TRUE;
+}
+
+static gboolean
+command_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    if (pcmk__str_any_of(option_name, "--info", "-l", NULL)) {
+        options.ticket_cmd = 'l';
+    } else if (pcmk__str_any_of(option_name, "--details", "-L", NULL)) {
+        options.ticket_cmd = 'L';
+    } else if (pcmk__str_any_of(option_name, "--raw", "-w", NULL)) {
+        options.ticket_cmd = 'w';
+    } else if (pcmk__str_any_of(option_name, "--query-xml", "-q", NULL)) {
+        options.ticket_cmd = 'q';
+    } else if (pcmk__str_any_of(option_name, "--constraints", "-c", NULL)) {
+        options.ticket_cmd = 'c';
+    } else if (pcmk__str_any_of(option_name, "--cleanup", "-C", NULL)) {
+        options.ticket_cmd = 'C';
+    }
+
+    return TRUE;
+}
+
+static gboolean
+delete_attr_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    attr_delete = g_list_append(attr_delete, strdup(optarg));
+    modified = true;
+    return TRUE;
+}
+
+static gboolean
+get_attr_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    pcmk__str_update(&options.get_attr_name, optarg);
+    options.ticket_cmd = 'G';
+    return TRUE;
+}
+
+static gboolean
+grant_standby_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    if (pcmk__str_any_of(option_name, "--grant", "-g", NULL)) {
+        g_hash_table_insert(attr_set, strdup("granted"), strdup("true"));
+        modified = true;
+    } else if (pcmk__str_any_of(option_name, "--revoke", "-r", NULL)) {
+        g_hash_table_insert(attr_set, strdup("granted"), strdup("false"));
+        modified = true;
+    } else if (pcmk__str_any_of(option_name, "--standby", "-s", NULL)) {
+        g_hash_table_insert(attr_set, strdup("standby"), strdup("true"));
+        modified = true;
+    } else if (pcmk__str_any_of(option_name, "--activate", "-a", NULL)) {
+        g_hash_table_insert(attr_set, strdup("standby"), strdup("false"));
+        modified = true;
+    }
+
+    return TRUE;
+}
+
+static gboolean
+set_attr_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    pcmk__str_update(&options.attr_name, optarg);
+
+    if (!options.attr_name || !options.attr_value) {
+        return TRUE;
+    }
+
+    g_hash_table_insert(attr_set, strdup(options.attr_name), strdup(options.attr_value));
+    pcmk__str_update(&options.attr_name, NULL);
+    pcmk__str_update(&options.attr_value, NULL);
+
+    modified = true;
+
+    return TRUE;
+}
+
+static GOptionEntry query_entries[] = {
+    { "info", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the information of ticket(s)",
+      NULL },
+
+    { "details", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the details of ticket(s)",
+      NULL },
+
+    { "raw", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the IDs of ticket(s)",
+      NULL },
+
+    { "query-xml", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Query the XML of ticket(s)",
+      NULL },
+
+    { "constraints", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the rsc_ticket constraints that apply to ticket(s)",
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry command_entries[] = {
+    { "grant", 'g', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, grant_standby_cb,
+      "Grant a ticket to this cluster site",
+      NULL },
+
+    { "revoke", 'r', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, grant_standby_cb,
+      "Revoke a ticket from this cluster site",
+      NULL },
+
+    { "standby", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, grant_standby_cb,
+      "Tell this cluster site this ticket is standby",
+      NULL },
+
+    { "activate", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, grant_standby_cb,
+      "Tell this cluster site this ticket is active",
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry advanced_entries[] = {
+    { "get-attr", 'G', 0, G_OPTION_ARG_CALLBACK, get_attr_cb,
+      "Display the named attribute for a ticket",
+      "ATTRIBUTE" },
+
+    { "set-attr", 'S', 0, G_OPTION_ARG_CALLBACK, set_attr_cb,
+      "Set the named attribute for a ticket",
+      "ATTRIBUTE" },
+
+    { "delete-attr", 'D', 0, G_OPTION_ARG_CALLBACK, delete_attr_cb,
+      "Delete the named attribute for a ticket",
+      "ATTRIBUTE" },
+
+    { "cleanup", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Delete all state of a ticket at this cluster site",
+      NULL },
+
+    { NULL}
+};
+
+static GOptionEntry addl_entries[] = {
+    { "attr-value", 'v', 0, G_OPTION_ARG_CALLBACK, attr_value_cb,
+      "Attribute value to use with -S",
+      "VALUE" },
+
+    { "default", 'd', 0, G_OPTION_ARG_STRING, &options.attr_default,
+      "(Advanced) Default attribute value to display if none is found\n"
+      INDENT "(for use with -G)",
+      "VALUE" },
+
+    { "force", 'f', 0, G_OPTION_ARG_NONE, &options.force,
+      "(Advanced) Force the action to be performed",
+      NULL },
+
+    { "ticket", 't', 0, G_OPTION_ARG_STRING, &options.ticket_id,
+      "Ticket ID",
+      "ID" },
+
+    { "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.xml_file,
+      NULL,
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry deprecated_entries[] = {
+    { "set-name", 'n', 0, G_OPTION_ARG_STRING, &options.set_name,
+      "(Advanced) ID of the instance_attributes object to change",
+      "ID" },
+
+    { "nvpair", 'i', 0, G_OPTION_ARG_STRING, &options.attr_id,
+      "(Advanced) ID of the nvpair object to change/delete",
+      "ID" },
+
+    { "quiet", 'Q', 0, G_OPTION_ARG_NONE, &options.quiet,
+      "Print only the value on stdout",
+      NULL },
+
+    { NULL }
+};
+
+static pe_ticket_t *
+find_ticket(gchar *ticket_id, pe_working_set_t * data_set)
+{
+    return g_hash_table_lookup(data_set->tickets, ticket_id);
+}
+
+static void
+print_date(time_t time)
+{
+    int lpc = 0;
+    char date_str[26];
+
+    asctime_r(localtime(&time), date_str);
+    for (; lpc < 26; lpc++) {
+        if (date_str[lpc] == '\n') {
+            date_str[lpc] = 0;
+        }
+    }
+    fprintf(stdout, "'%s'", date_str);
+}
+
+static void
+print_ticket(pe_ticket_t * ticket, bool raw, bool details)
+{
+    if (raw) {
+        fprintf(stdout, "%s\n", ticket->id);
+        return;
+    }
+
+    fprintf(stdout, "%s\t%s %s",
+            ticket->id, ticket->granted ? "granted" : "revoked",
+            ticket->standby ? "[standby]" : "         ");
+
+    if (details && g_hash_table_size(ticket->state) > 0) {
+        GHashTableIter iter;
+        const char *name = NULL;
+        const char *value = NULL;
+        int lpc = 0;
+
+        fprintf(stdout, " (");
+
+        g_hash_table_iter_init(&iter, ticket->state);
+        while (g_hash_table_iter_next(&iter, (void **)&name, (void **)&value)) {
+            if (lpc > 0) {
+                fprintf(stdout, ", ");
+            }
+            fprintf(stdout, "%s=", name);
+            if (pcmk__str_any_of(name, "last-granted", "expires", NULL)) {
+                long long time_ll;
+
+                pcmk__scan_ll(value, &time_ll, 0);
+                print_date((time_t) time_ll);
+            } else {
+                fprintf(stdout, "%s", value);
+            }
+            lpc++;
+        }
+
+        fprintf(stdout, ")\n");
+
+    } else {
+        if (ticket->last_granted > -1) {
+            fprintf(stdout, " last-granted=");
+            print_date(ticket->last_granted);
+        }
+        fprintf(stdout, "\n");
+    }
+
+    return;
+}
+
+static void
+print_ticket_list(pe_working_set_t * data_set, bool raw, bool details)
+{
+    GHashTableIter iter;
+    pe_ticket_t *ticket = NULL;
+
+    g_hash_table_iter_init(&iter, data_set->tickets);
+
+    while (g_hash_table_iter_next(&iter, NULL, (void **)&ticket)) {
+        print_ticket(ticket, raw, details);
+    }
+}
+
+static int
+find_ticket_state(cib_t * the_cib, gchar *ticket_id, xmlNode ** ticket_state_xml)
+{
+    int rc = pcmk_rc_ok;
+    xmlNode *xml_search = NULL;
+
+    GString *xpath = NULL;
+
+    CRM_ASSERT(ticket_state_xml != NULL);
+    *ticket_state_xml = NULL;
+
+    xpath = g_string_sized_new(1024);
+    g_string_append(xpath,
+                    "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS
+                    "/" XML_CIB_TAG_TICKETS);
+
+    if (ticket_id != NULL) {
+        pcmk__g_strcat(xpath,
+                       "/" XML_CIB_TAG_TICKET_STATE
+                       "[@" XML_ATTR_ID "=\"", ticket_id, "\"]", NULL);
+    }
+
+    rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
+                              cib_sync_call | cib_scope_local | cib_xpath);
+    rc = pcmk_legacy2rc(rc);
+    g_string_free(xpath, TRUE);
+
+    if (rc != pcmk_rc_ok) {
+        return rc;
+    }
+
+    crm_log_xml_debug(xml_search, "Match");
+    if (xml_has_children(xml_search)) {
+        if (ticket_id) {
+            fprintf(stdout, "Multiple ticket_states match ticket_id=%s\n", ticket_id);
+        }
+        *ticket_state_xml = xml_search;
+    } else {
+        *ticket_state_xml = xml_search;
+    }
+    return rc;
+}
+
+static int
+find_ticket_constraints(cib_t * the_cib, gchar *ticket_id, xmlNode ** ticket_cons_xml)
+{
+    int rc = pcmk_rc_ok;
+    xmlNode *xml_search = NULL;
+
+    GString *xpath = NULL;
+    const char *xpath_base = NULL;
+
+    CRM_ASSERT(ticket_cons_xml != NULL);
+    *ticket_cons_xml = NULL;
+
+    xpath_base = pcmk_cib_xpath_for(XML_CIB_TAG_CONSTRAINTS);
+    if (xpath_base == NULL) {
+        crm_err(XML_CIB_TAG_CONSTRAINTS " CIB element not known (bug?)");
+        return -ENOMSG;
+    }
+
+    xpath = g_string_sized_new(1024);
+    pcmk__g_strcat(xpath, xpath_base, "/" XML_CONS_TAG_RSC_TICKET, NULL);
+
+    if (ticket_id != NULL) {
+        pcmk__g_strcat(xpath,
+                       "[@" XML_TICKET_ATTR_TICKET "=\"", ticket_id, "\"]",
+                       NULL);
+    }
+
+    rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
+                              cib_sync_call | cib_scope_local | cib_xpath);
+    rc = pcmk_legacy2rc(rc);
+    g_string_free(xpath, TRUE);
+
+    if (rc != pcmk_rc_ok) {
+        return rc;
+    }
+
+    crm_log_xml_debug(xml_search, "Match");
+    *ticket_cons_xml = xml_search;
+
+    return rc;
+}
+
+static int
+dump_ticket_xml(cib_t * the_cib, gchar *ticket_id)
+{
+    int rc = pcmk_rc_ok;
+    xmlNode *state_xml = NULL;
+
+    rc = find_ticket_state(the_cib, ticket_id, &state_xml);
+
+    if (state_xml == NULL) {
+        return rc;
+    }
+
+    fprintf(stdout, "State XML:\n");
+    if (state_xml) {
+        char *state_xml_str = NULL;
+
+        state_xml_str = dump_xml_formatted(state_xml);
+        fprintf(stdout, "\n%s", pcmk__s(state_xml_str, "<null>\n"));
+        free_xml(state_xml);
+        free(state_xml_str);
+    }
+
+    return rc;
+}
+
+static int
+dump_constraints(cib_t * the_cib, gchar *ticket_id)
+{
+    int rc = pcmk_rc_ok;
+    xmlNode *cons_xml = NULL;
+    char *cons_xml_str = NULL;
+
+    rc = find_ticket_constraints(the_cib, ticket_id, &cons_xml);
+
+    if (cons_xml == NULL) {
+        return rc;
+    }
+
+    cons_xml_str = dump_xml_formatted(cons_xml);
+    fprintf(stdout, "Constraints XML:\n\n%s",
+            pcmk__s(cons_xml_str, "<null>\n"));
+    free_xml(cons_xml);
+    free(cons_xml_str);
+
+    return rc;
+}
+
+static int
+get_ticket_state_attr(gchar *ticket_id, const char *attr_name, const char **attr_value,
+                      pe_working_set_t * data_set)
+{
+    pe_ticket_t *ticket = NULL;
+
+    CRM_ASSERT(attr_value != NULL);
+    *attr_value = NULL;
+
+    ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
+    if (ticket == NULL) {
+        return ENXIO;
+    }
+
+    *attr_value = g_hash_table_lookup(ticket->state, attr_name);
+    if (*attr_value == NULL) {
+        return ENXIO;
+    }
+
+    return pcmk_rc_ok;
+}
+
+static void
+ticket_warning(gchar *ticket_id, const char *action)
+{
+    GString *warning = g_string_sized_new(1024);
+    const char *word = NULL;
+
+    CRM_ASSERT(action != NULL);
+
+    if (strcmp(action, "grant") == 0) {
+        pcmk__g_strcat(warning,
+                       "This command cannot help you verify whether '",
+                       ticket_id,
+                       "' has been already granted elsewhere.\n", NULL);
+        word = "to";
+
+    } else {
+        pcmk__g_strcat(warning,
+                       "Revoking '", ticket_id, "' can trigger the specified "
+                       "'loss-policy'(s) relating to '", ticket_id, "'.\n\n"
+                       "You can check that with:\n"
+                       "crm_ticket --ticket ", ticket_id, " --constraints\n\n"
+                       "Otherwise before revoking '", ticket_id, "', "
+                       "you may want to make '", ticket_id, "' "
+                       "standby with:\n"
+                       "crm_ticket --ticket ", ticket_id, " --standby\n\n",
+                       NULL);
+        word = "from";
+    }
+
+    pcmk__g_strcat(warning,
+                   "If you really want to ", action, " '", ticket_id, "' ",
+                   word, " this site now, and you know what you are doing,\n"
+                   "please specify --force.", NULL);
+
+    fprintf(stdout, "%s\n", (const char *) warning->str);
+
+    g_string_free(warning, TRUE);
+}
+
+static bool
+allow_modification(gchar *ticket_id)
+{
+    const char *value = NULL;
+    GList *list_iter = NULL;
+
+    if (options.force) {
+        return true;
+    }
+
+    if (g_hash_table_lookup_extended(attr_set, "granted", NULL, (gpointer *) & value)) {
+        if (crm_is_true(value)) {
+            ticket_warning(ticket_id, "grant");
+            return false;
+
+        } else {
+            ticket_warning(ticket_id, "revoke");
+            return false;
+        }
+    }
+
+    for(list_iter = attr_delete; list_iter; list_iter = list_iter->next) {
+        const char *key = (const char *)list_iter->data;
+
+        if (pcmk__str_eq(key, "granted", pcmk__str_casei)) {
+            ticket_warning(ticket_id, "revoke");
+            return false;
+        }
+    }
+
+    return true;
+}
+
+static int
+modify_ticket_state(gchar * ticket_id, cib_t * cib, pe_working_set_t * data_set)
+{
+    int rc = pcmk_rc_ok;
+    xmlNode *xml_top = NULL;
+    xmlNode *ticket_state_xml = NULL;
+    bool found = false;
+
+    GList *list_iter = NULL;
+    GHashTableIter hash_iter;
+
+    char *key = NULL;
+    char *value = NULL;
+
+    pe_ticket_t *ticket = NULL;
+
+    rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);
+    if (rc == pcmk_rc_ok) {
+        crm_debug("Found a match state for ticket: id=%s", ticket_id);
+        xml_top = ticket_state_xml;
+        found = true;
+
+    } else if (rc != ENXIO) {
+        return rc;
+
+    } else if (g_hash_table_size(attr_set) == 0){
+        return pcmk_rc_ok;
+
+    } else {
+        xmlNode *xml_obj = NULL;
+
+        xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
+        xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
+        ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
+        crm_xml_add(ticket_state_xml, XML_ATTR_ID, ticket_id);
+    }
+
+    for(list_iter = attr_delete; list_iter; list_iter = list_iter->next) {
+        const char *key = (const char *)list_iter->data;
+        xml_remove_prop(ticket_state_xml, key);
+    }
+
+    ticket = find_ticket(ticket_id, data_set);
+
+    g_hash_table_iter_init(&hash_iter, attr_set);
+    while (g_hash_table_iter_next(&hash_iter, (gpointer *) & key, (gpointer *) & value)) {
+        crm_xml_add(ticket_state_xml, key, value);
+
+        if (pcmk__str_eq(key, "granted", pcmk__str_casei)
+            && (ticket == NULL || ticket->granted == FALSE)
+            && crm_is_true(value)) {
+
+            char *now = pcmk__ttoa(time(NULL));
+
+            crm_xml_add(ticket_state_xml, "last-granted", now);
+            free(now);
+        }
+    }
+
+    if (found && (attr_delete != NULL)) {
+        crm_log_xml_debug(xml_top, "Replace");
+        rc = cib->cmds->replace(cib, XML_CIB_TAG_STATUS, ticket_state_xml, cib_options);
+        rc = pcmk_legacy2rc(rc);
+
+    } else {
+        crm_log_xml_debug(xml_top, "Update");
+        rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top, cib_options);
+        rc = pcmk_legacy2rc(rc);
+    }
+
+    free_xml(xml_top);
+    return rc;
+}
+
+static int
+delete_ticket_state(gchar *ticket_id, cib_t * cib)
+{
+    xmlNode *ticket_state_xml = NULL;
+
+    int rc = pcmk_rc_ok;
+
+    rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);
+
+    if (rc == ENXIO) {
+        return pcmk_rc_ok;
+
+    } else if (rc != pcmk_rc_ok) {
+        return rc;
+    }
+
+    crm_log_xml_debug(ticket_state_xml, "Delete");
+
+    rc = cib->cmds->remove(cib, XML_CIB_TAG_STATUS, ticket_state_xml, cib_options);
+    rc = pcmk_legacy2rc(rc);
+
+    if (rc == pcmk_rc_ok) {
+        fprintf(stdout, "Cleaned up %s\n", ticket_id);
+    }
+
+    free_xml(ticket_state_xml);
+    return rc;
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args) {
+    GOptionContext *context = NULL;
+
+    const char *description = "Examples:\n\n"
+                              "Display the info of tickets:\n\n"
+                              "\tcrm_ticket --info\n\n"
+                              "Display the detailed info of tickets:\n\n"
+                              "\tcrm_ticket --details\n\n"
+                              "Display the XML of 'ticketA':\n\n"
+                              "\tcrm_ticket --ticket ticketA --query-xml\n\n"
+                              "Display the rsc_ticket constraints that apply to 'ticketA':\n\n"
+                              "\tcrm_ticket --ticket ticketA --constraints\n\n"
+                              "Grant 'ticketA' to this cluster site:\n\n"
+                              "\tcrm_ticket --ticket ticketA --grant\n\n"
+                              "Revoke 'ticketA' from this cluster site:\n\n"
+                              "\tcrm_ticket --ticket ticketA --revoke\n\n"
+                              "Make 'ticketA' standby (the cluster site will treat a granted\n"
+                              "'ticketA' as 'standby', and the dependent resources will be\n"
+                              "stopped or demoted gracefully without triggering loss-policies):\n\n"
+                              "\tcrm_ticket --ticket ticketA --standby\n\n"
+                              "Activate 'ticketA' from being standby:\n\n"
+                              "\tcrm_ticket --ticket ticketA --activate\n\n"
+                              "Get the value of the 'granted' attribute for 'ticketA':\n\n"
+                              "\tcrm_ticket --ticket ticketA --get-attr granted\n\n"
+                              "Set the value of the 'standby' attribute for 'ticketA':\n\n"
+                              "\tcrm_ticket --ticket ticketA --set-attr standby --attr-value true\n\n"
+                              "Delete the 'granted' attribute for 'ticketA':\n\n"
+                              "\tcrm_ticket --ticket ticketA --delete-attr granted\n\n"
+                              "Erase the operation history of 'ticketA' at this cluster site,\n"
+                              "causing the cluster site to 'forget' the existing ticket state:\n\n"
+                              "\tcrm_ticket --ticket ticketA --cleanup\n\n";
+
+    context = pcmk__build_arg_context(args, NULL, NULL, NULL);
+    g_option_context_set_description(context, description);
+
+    pcmk__add_arg_group(context, "queries", "Queries:",
+                        "Show queries", query_entries);
+    pcmk__add_arg_group(context, "commands", "Commands:",
+                        "Show command options", command_entries);
+    pcmk__add_arg_group(context, "advanced", "Advanced Options:",
+                        "Show advanced options", advanced_entries);
+    pcmk__add_arg_group(context, "additional", "Additional Options:",
+                        "Show additional options", addl_entries);
+    pcmk__add_arg_group(context, "deprecated", "Deprecated Options:",
+                        "Show deprecated options", deprecated_entries);
+
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    pe_working_set_t *data_set = NULL;
+    xmlNode *cib_xml_copy = NULL;
+
+    cib_t *cib_conn = NULL;
+    crm_exit_t exit_code = CRM_EX_OK;
+    int rc = pcmk_rc_ok;
+
+    pcmk__common_args_t *args = NULL;
+    GOptionContext *context = NULL;
+    gchar **processed_args = NULL;
+
+    attr_set = pcmk__strkey_table(free, free);
+    attr_delete = NULL;
+
+    args = pcmk__new_common_args(SUMMARY);
+    context = build_arg_context(args);
+    processed_args = pcmk__cmdline_preproc(argv, "dintvxCDGS");
+
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("crm_ticket", args->verbosity);
+
+    if (args->version) {
+        g_strfreev(processed_args);
+        pcmk__free_arg_context(context);
+        /* FIXME:  When crm_ticket is converted to use formatted output, this can go. */
+        pcmk__cli_help('v');
+    }
+
+    data_set = pe_new_working_set();
+    if (data_set == NULL) {
+        crm_perror(LOG_CRIT, "Could not allocate working set");
+        exit_code = CRM_EX_OSERR;
+        goto done;
+    }
+    pe__set_working_set_flags(data_set, pe_flag_no_counts|pe_flag_no_compat);
+
+    cib_conn = cib_new();
+    if (cib_conn == NULL) {
+        exit_code = CRM_EX_DISCONNECT;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Could not connect to the CIB manager");
+        goto done;
+    }
+
+    rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
+    rc = pcmk_legacy2rc(rc);
+
+    if (rc != pcmk_rc_ok) {
+        exit_code = pcmk_rc2exitc(rc);
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Could not connect to the CIB: %s",
+                    pcmk_rc_str(rc));
+        goto done;
+    }
+
+    if (options.xml_file != NULL) {
+        cib_xml_copy = filename2xml(options.xml_file);
+
+    } else {
+        rc = cib_conn->cmds->query(cib_conn, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
+        rc = pcmk_legacy2rc(rc);
+
+        if (rc != pcmk_rc_ok) {
+            exit_code = pcmk_rc2exitc(rc);
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Could not get local CIB: %s",
+                        pcmk_rc_str(rc));
+            goto done;
+        }
+    }
+
+    if (!cli_config_update(&cib_xml_copy, NULL, FALSE)) {
+        exit_code = CRM_EX_CONFIG;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not update local CIB to latest schema version");
+        goto done;
+    }
+
+    data_set->input = cib_xml_copy;
+    data_set->now = crm_time_new(NULL);
+
+    cluster_status(data_set);
+
+    /* For recording the tickets that are referenced in rsc_ticket constraints
+     * but have never been granted yet. */
+    pcmk__unpack_constraints(data_set);
+
+    if (options.ticket_cmd == 'l' || options.ticket_cmd == 'L' || options.ticket_cmd == 'w') {
+        bool raw = false;
+        bool details = false;
+
+        if (options.ticket_cmd == 'L') {
+            details = true;
+        } else if (options.ticket_cmd == 'w') {
+            raw = true;
+        }
+
+        if (options.ticket_id) {
+            pe_ticket_t *ticket = find_ticket(options.ticket_id, data_set);
+
+            if (ticket == NULL) {
+                exit_code = CRM_EX_NOSUCH;
+                g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                            "No such ticket '%s'", options.ticket_id);
+                goto done;
+            }
+            print_ticket(ticket, raw, details);
+
+        } else {
+            print_ticket_list(data_set, raw, details);
+        }
+
+    } else if (options.ticket_cmd == 'q') {
+        rc = dump_ticket_xml(cib_conn, options.ticket_id);
+        exit_code = pcmk_rc2exitc(rc);
+
+        if (rc != pcmk_rc_ok) {
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Could not query ticket XML: %s", pcmk_rc_str(rc));
+        }
+
+    } else if (options.ticket_cmd == 'c') {
+        rc = dump_constraints(cib_conn, options.ticket_id);
+        exit_code = pcmk_rc2exitc(rc);
+
+        if (rc != pcmk_rc_ok) {
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Could not show ticket constraints: %s", pcmk_rc_str(rc));
+        }
+
+    } else if (options.ticket_cmd == 'G') {
+        const char *value = NULL;
+
+        if (options.ticket_id == NULL) {
+            exit_code = CRM_EX_NOSUCH;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Must supply ticket ID with -t");
+            goto done;
+        }
+
+        rc = get_ticket_state_attr(options.ticket_id, options.get_attr_name, &value, data_set);
+        if (rc == pcmk_rc_ok) {
+            fprintf(stdout, "%s\n", value);
+        } else if (rc == ENXIO && options.attr_default) {
+            fprintf(stdout, "%s\n", options.attr_default);
+            rc = pcmk_rc_ok;
+        }
+        exit_code = pcmk_rc2exitc(rc);
+
+    } else if (options.ticket_cmd == 'C') {
+        if (options.ticket_id == NULL) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Must supply ticket ID with -t");
+            goto done;
+        }
+
+        if (options.force == FALSE) {
+            pe_ticket_t *ticket = NULL;
+
+            ticket = find_ticket(options.ticket_id, data_set);
+            if (ticket == NULL) {
+                exit_code = CRM_EX_NOSUCH;
+                g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                            "No such ticket '%s'", options.ticket_id);
+                goto done;
+            }
+
+            if (ticket->granted) {
+                ticket_warning(options.ticket_id, "revoke");
+                exit_code = CRM_EX_INSUFFICIENT_PRIV;
+                goto done;
+            }
+        }
+
+        rc = delete_ticket_state(options.ticket_id, cib_conn);
+        exit_code = pcmk_rc2exitc(rc);
+
+        if (rc != pcmk_rc_ok) {
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Could not clean up ticket: %s", pcmk_rc_str(rc));
+        }
+
+    } else if (modified) {
+        if (options.ticket_id == NULL) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Must supply ticket ID with -t");
+            goto done;
+        }
+
+        if (options.attr_value
+            && (pcmk__str_empty(options.attr_name))) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Must supply attribute name with -S for -v %s", options.attr_value);
+            goto done;
+        }
+
+        if (options.attr_name
+            && (pcmk__str_empty(options.attr_value))) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Must supply attribute value with -v for -S %s", options.attr_value);
+            goto done;
+        }
+
+        if (!allow_modification(options.ticket_id)) {
+            exit_code = CRM_EX_INSUFFICIENT_PRIV;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Ticket modification not allowed");
+            goto done;
+        }
+
+        rc = modify_ticket_state(options.ticket_id, cib_conn, data_set);
+        exit_code = pcmk_rc2exitc(rc);
+
+        if (rc != pcmk_rc_ok) {
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Could not modify ticket: %s", pcmk_rc_str(rc));
+        }
+
+    } else if (options.ticket_cmd == 'S') {
+        /* Correct usage was handled in the "if (modified)" block above, so
+         * this is just for reporting usage errors
+         */
+
+        if (pcmk__str_empty(options.attr_name)) {
+            // We only get here if ticket_cmd was left as default
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Must supply a command");
+            goto done;
+        }
+
+        if (options.ticket_id == NULL) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Must supply ticket ID with -t");
+            goto done;
+        }
+
+        if (pcmk__str_empty(options.attr_value)) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Must supply value with -v for -S %s", options.attr_name);
+            goto done;
+        }
+
+    } else {
+        exit_code = CRM_EX_USAGE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Unknown command: %c", options.ticket_cmd);
+    }
+
+ done:
+    if (attr_set) {
+        g_hash_table_destroy(attr_set);
+    }
+    attr_set = NULL;
+
+    if (attr_delete) {
+        g_list_free_full(attr_delete, free);
+    }
+    attr_delete = NULL;
+
+    pe_free_working_set(data_set);
+    data_set = NULL;
+
+    cib__clean_up_connection(&cib_conn);
+
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+    g_free(options.attr_default);
+    g_free(options.attr_id);
+    free(options.attr_name);
+    free(options.attr_value);
+    free(options.get_attr_name);
+    g_free(options.set_name);
+    g_free(options.ticket_id);
+    g_free(options.xml_file);
+
+    pcmk__output_and_clear_error(&error, NULL);
+
+    crm_exit(exit_code);
+}
diff --git a/tools/crm_verify.8.inc b/tools/crm_verify.8.inc
new file mode 100644
index 0000000..27b0a55
--- /dev/null
+++ b/tools/crm_verify.8.inc
@@ -0,0 +1,5 @@
+[=synopsis]
+crm_verify [<options>]
+
+/reported as warnings./
+.SH OPTIONS
diff --git a/tools/crm_verify.c b/tools/crm_verify.c
new file mode 100644
index 0000000..43b09da
--- /dev/null
+++ b/tools/crm_verify.c
@@ -0,0 +1,285 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include <crm/crm.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/output_internal.h>
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <libgen.h>
+#include <glib.h>
+
+#include <crm/common/xml.h>
+#include <crm/common/util.h>
+#include <crm/msg_xml.h>
+#include <crm/cib.h>
+#include <crm/cib/internal.h>
+#include <crm/pengine/status.h>
+#include <pacemaker-internal.h>
+
+const char *SUMMARY = "Check a Pacemaker configuration for errors\n\n"
+                      "Check the well-formedness of a complete Pacemaker XML configuration,\n"
+                      "its conformance to the configured schema, and the presence of common\n"
+                      "misconfigurations. Problems reported as errors must be fixed before the\n"
+                      "cluster will work properly. It is left to the administrator to decide\n"
+                      "whether to fix problems reported as warnings.";
+
+struct {
+    char *cib_save;
+    gboolean use_live_cib;
+    char *xml_file;
+    gboolean xml_stdin;
+    char *xml_string;
+} options;
+
+static GOptionEntry data_entries[] = {
+    { "live-check", 'L', 0, G_OPTION_ARG_NONE,
+      &options.use_live_cib, "Check the configuration used by the running cluster",
+      NULL },
+    { "xml-file", 'x', 0, G_OPTION_ARG_FILENAME,
+      &options.xml_file, "Check the configuration in the named file",
+      "FILE" },
+    { "xml-pipe", 'p', 0, G_OPTION_ARG_NONE,
+      &options.xml_stdin, "Check the configuration piped in via stdin",
+      NULL },
+    { "xml-text", 'X', 0, G_OPTION_ARG_STRING,
+      &options.xml_string, "Check the configuration in the supplied string",
+      "XML" },
+
+    { NULL }
+};
+
+static GOptionEntry addl_entries[] = {
+    { "save-xml", 'S', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME,
+      &options.cib_save, "Save verified XML to named file (most useful with -L)",
+      "FILE" },
+
+    { NULL }
+};
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    const char *description = "Examples:\n\n"
+                              "Check the consistency of the configuration in the running cluster:\n\n"
+                              "\tcrm_verify --live-check\n\n"
+                              "Check the consistency of the configuration in a given file and "
+                              "produce verbose output:\n\n"
+                              "\tcrm_verify --xml-file file.xml --verbose\n\n";
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
+    g_option_context_set_description(context, description);
+
+    pcmk__add_arg_group(context, "data", "Data sources:",
+                        "Show data options", data_entries);
+    pcmk__add_arg_group(context, "additional", "Additional options:",
+                        "Show additional options", addl_entries);
+
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    xmlNode *cib_object = NULL;
+    xmlNode *status = NULL;
+
+    pe_working_set_t *data_set = NULL;
+    const char *xml_tag = NULL;
+
+    int rc = pcmk_rc_ok;
+    crm_exit_t exit_code = CRM_EX_OK;
+
+    GError *error = NULL;
+
+    pcmk__output_t *out = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "xSX");
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("crm_verify", args->verbosity);
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    pcmk__register_lib_messages(out);
+
+    crm_info("=#=#=#=#= Getting XML =#=#=#=#=");
+
+    if (options.use_live_cib) {
+        crm_info("Reading XML from: live cluster");
+        rc = cib__signon_query(out, NULL, &cib_object);
+
+        if (rc != pcmk_rc_ok) {
+            // cib__signon_query() outputs any relevant error
+            goto done;
+        }
+
+    } else if (options.xml_file != NULL) {
+        cib_object = filename2xml(options.xml_file);
+        if (cib_object == NULL) {
+            rc = ENODATA;
+            g_set_error(&error, PCMK__RC_ERROR, rc, "Couldn't parse input file: %s", options.xml_file);
+            goto done;
+        }
+
+    } else if (options.xml_string != NULL) {
+        cib_object = string2xml(options.xml_string);
+        if (cib_object == NULL) {
+            rc = ENODATA;
+            g_set_error(&error, PCMK__RC_ERROR, rc, "Couldn't parse input string: %s", options.xml_string);
+            goto done;
+        }
+    } else if (options.xml_stdin) {
+        cib_object = stdin2xml();
+        if (cib_object == NULL) {
+            rc = ENODATA;
+            g_set_error(&error, PCMK__RC_ERROR, rc, "Couldn't parse input from STDIN.");
+            goto done;
+        }
+
+    } else {
+        rc = ENODATA;
+        g_set_error(&error, PCMK__RC_ERROR, rc,
+                    "No configuration source specified.  Use --help for usage information.");
+        goto done;
+    }
+
+    xml_tag = crm_element_name(cib_object);
+    if (!pcmk__str_eq(xml_tag, XML_TAG_CIB, pcmk__str_casei)) {
+        rc = EBADMSG;
+        g_set_error(&error, PCMK__RC_ERROR, rc,
+                    "This tool can only check complete configurations (i.e. those starting with <cib>).");
+        goto done;
+    }
+
+    if (options.cib_save != NULL) {
+        write_xml_file(cib_object, options.cib_save, FALSE);
+    }
+
+    status = pcmk_find_cib_element(cib_object, XML_CIB_TAG_STATUS);
+    if (status == NULL) {
+        create_xml_node(cib_object, XML_CIB_TAG_STATUS);
+    }
+
+    if (validate_xml(cib_object, NULL, FALSE) == FALSE) {
+        pcmk__config_err("CIB did not pass schema validation");
+        free_xml(cib_object);
+        cib_object = NULL;
+
+    } else if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
+        crm_config_error = TRUE;
+        free_xml(cib_object);
+        cib_object = NULL;
+        out->err(out, "The cluster will NOT be able to use this configuration.\n"
+                 "Please manually update the configuration to conform to the %s syntax.",
+                 xml_latest_schema());
+    }
+
+    data_set = pe_new_working_set();
+    if (data_set == NULL) {
+        rc = errno;
+        crm_perror(LOG_CRIT, "Unable to allocate working set");
+        goto done;
+    }
+    data_set->priv = out;
+
+    /* Process the configuration to set crm_config_error/crm_config_warning.
+     *
+     * @TODO Some parts of the configuration are unpacked only when needed (for
+     * example, action configuration), so we aren't necessarily checking those.
+     */
+    if (cib_object != NULL) {
+        unsigned long long flags = pe_flag_no_counts|pe_flag_no_compat;
+
+        if ((status == NULL) && !options.use_live_cib) {
+            // No status available, so do minimal checks
+            flags |= pe_flag_check_config;
+        }
+        pcmk__schedule_actions(cib_object, flags, data_set);
+    }
+    pe_free_working_set(data_set);
+
+    if (crm_config_error) {
+        rc = pcmk_rc_schema_validation;
+
+        if (args->verbosity > 0) {
+            g_set_error(&error, PCMK__RC_ERROR, rc,
+                        "Errors found during check: config not valid");
+        } else {
+            g_set_error(&error, PCMK__RC_ERROR, rc,
+                        "Errors found during check: config not valid\n-V may provide more details");
+        }
+
+    } else if (crm_config_warning) {
+        rc = pcmk_rc_schema_validation;
+
+        if (args->verbosity > 0) {
+            g_set_error(&error, PCMK__RC_ERROR, rc,
+                        "Warnings found during check: config may not be valid");
+        } else {
+            g_set_error(&error, PCMK__RC_ERROR, rc,
+                        "Warnings found during check: config may not be valid\n-V may provide more details");
+        }
+    }
+
+  done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+    free(options.cib_save);
+    free(options.xml_file);
+    free(options.xml_string);
+
+    if (exit_code == CRM_EX_OK) {
+        exit_code = pcmk_rc2exitc(rc);
+    }
+
+    pcmk__output_and_clear_error(&error, NULL);
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+
+    pcmk__unregister_formats();
+    crm_exit(exit_code);
+}
diff --git a/tools/crmadmin.8.inc b/tools/crmadmin.8.inc
new file mode 100644
index 0000000..ab5efbe
--- /dev/null
+++ b/tools/crmadmin.8.inc
@@ -0,0 +1,5 @@
+[synopsis]
+crmadmin [options] [node]
+
+/the Pacemaker controller/
+.SH OPTIONS
diff --git a/tools/crmadmin.c b/tools/crmadmin.c
new file mode 100644
index 0000000..0b400ae
--- /dev/null
+++ b/tools/crmadmin.c
@@ -0,0 +1,275 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <stdlib.h>             // atoi()
+
+#include <glib.h>               // gboolean, GMainLoop, etc.
+#include <libxml/tree.h>        // xmlNode
+
+#include <pacemaker-internal.h>
+
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/output_internal.h>
+
+#define SUMMARY "query and manage the Pacemaker controller"
+
+static enum {
+    cmd_none,
+    cmd_health,
+    cmd_whois_dc,
+    cmd_list_nodes,
+    cmd_pacemakerd_health,
+} command = cmd_none;
+
+struct {
+    gboolean health;
+    gint timeout;
+    char *optarg;
+    char *ipc_name;
+    gboolean bash_export;
+} options = {
+    .optarg = NULL,
+    .ipc_name = NULL,
+    .bash_export = FALSE
+};
+
+gboolean command_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+
+static GOptionEntry command_options[] = {
+    { "status", 'S', 0, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the status of the specified node."
+      "\n                             Result is state of node's internal finite state"
+      "\n                             machine, which can be useful for debugging",
+      "NODE"
+    },
+    { "pacemakerd", 'P', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the status of local pacemakerd."
+      "\n                             Result is the state of the sub-daemons watched"
+      "\n                             by pacemakerd.",
+      NULL
+    },
+    { "dc_lookup", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the uname of the node co-ordinating the cluster."
+      "\n                             This is an internal detail rarely useful to"
+      "\n                             administrators except when deciding on which"
+      "\n                             node to examine the logs.",
+      NULL
+    },
+    { "nodes", 'N', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, command_cb,
+      "Display the uname of all member nodes [optionally filtered by type (comma-separated)]"
+      "\n                             Types: all (default), cluster, guest, remote",
+      "TYPE"
+    },
+    { "health", 'H', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.health,
+      NULL,
+      NULL
+    },
+
+    { NULL }
+};
+
+static GOptionEntry additional_options[] = {
+    { "timeout", 't', 0, G_OPTION_ARG_CALLBACK, command_cb,
+      "Time  to wait before declaring the operation"
+      "\n                             failed",
+      "TIMESPEC"
+    },
+    { "bash-export", 'B', 0, G_OPTION_ARG_NONE, &options.bash_export,
+      "Display nodes as shell commands of the form 'export uname=uuid'"
+      "\n                             (valid with -N/--nodes)",
+    },
+    { "ipc-name", 'i', 0, G_OPTION_ARG_STRING, &options.ipc_name,
+      "Name to use for ipc instead of 'crmadmin' (with -P/--pacemakerd).",
+      "NAME"
+    },
+
+    { NULL }
+};
+
+gboolean
+command_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error)
+{
+    if (!strcmp(option_name, "--status") || !strcmp(option_name, "-S")) {
+        command = cmd_health;
+        crm_trace("Option %c => %s", 'S', optarg);
+    }
+
+    if (!strcmp(option_name, "--pacemakerd") || !strcmp(option_name, "-P")) {
+        command = cmd_pacemakerd_health;
+    }
+
+    if (!strcmp(option_name, "--dc_lookup") || !strcmp(option_name, "-D")) {
+        command = cmd_whois_dc;
+    }
+
+    if (!strcmp(option_name, "--nodes") || !strcmp(option_name, "-N")) {
+        command = cmd_list_nodes;
+    }
+
+    if (!strcmp(option_name, "--timeout") || !strcmp(option_name, "-t")) {
+        options.timeout = crm_parse_interval_spec(optarg);
+        if (errno == EINVAL) {
+            return FALSE;
+        }
+        return TRUE;
+    }
+
+    pcmk__str_update(&options.optarg, optarg);
+    return TRUE;
+}
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    const char *description = "Notes:\n\n"
+                              "Time Specification:\n\n"
+                              "The TIMESPEC in any command line option can be specified in many different\n"
+                              "formats.  It can be just an integer number of seconds, a number plus units\n"
+                              "(ms/msec/us/usec/s/sec/m/min/h/hr), or an ISO 8601 period specification.\n\n"
+                              "Report bugs to " PCMK__BUG_URL;
+
+    GOptionEntry extra_prog_entries[] = {
+        { "quiet", 'q', 0, G_OPTION_ARG_NONE, &(args->quiet),
+          "Display only the essential query information",
+          NULL },
+
+        { NULL }
+    };
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
+    g_option_context_set_description(context, description);
+
+    /* Add the -q option, which cannot be part of the globally supported options
+     * because some tools use that flag for something else.
+     */
+    pcmk__add_main_args(context, extra_prog_entries);
+
+    pcmk__add_arg_group(context, "command", "Commands:",
+                        "Show command options", command_options);
+    pcmk__add_arg_group(context, "additional", "Additional Options:",
+                        "Show additional options", additional_options);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    crm_exit_t exit_code = CRM_EX_OK;
+    int rc;
+    int argerr = 0;
+
+    GError *error = NULL;
+
+    pcmk__output_t *out = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "itKNS");
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("crmadmin", args->verbosity);
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    pcmk__register_lib_messages(out);
+
+    out->quiet = args->quiet;
+
+    if (!pcmk__force_args(context, &error, "%s --xml-simple-list --xml-substitute", g_get_prgname())) {
+        goto done;
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    if (options.health) {
+        out->err(out, "Cluster-wide health option not supported");
+        ++argerr;
+    }
+
+    if (command == cmd_none) {
+        out->err(out, "error: Must specify a command option");
+        ++argerr;
+    }
+
+    if (argerr) {
+        char *help = g_option_context_get_help(context, TRUE, NULL);
+
+        out->err(out, "%s", help);
+        g_free(help);
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    switch (command) {
+        case cmd_health:
+            rc = pcmk__controller_status(out, options.optarg,
+                                         (unsigned int) options.timeout);
+            break;
+        case cmd_pacemakerd_health:
+            rc = pcmk__pacemakerd_status(out, options.ipc_name,
+                                         (unsigned int) options.timeout, true,
+                                         NULL);
+            break;
+        case cmd_list_nodes:
+            rc = pcmk__list_nodes(out, options.optarg, options.bash_export);
+            break;
+        case cmd_whois_dc:
+            rc = pcmk__designated_controller(out,
+                                             (unsigned int) options.timeout);
+            break;
+        case cmd_none:
+            rc = pcmk_rc_error;
+            break;
+    }
+
+    if (rc != pcmk_rc_ok) {
+        out->err(out, "error: Command failed: %s", pcmk_rc_str(rc));
+        exit_code = pcmk_rc2exitc(rc);
+    }
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+    pcmk__unregister_formats();
+    return crm_exit(exit_code);
+}
diff --git a/tools/fix-manpages b/tools/fix-manpages
new file mode 100644
index 0000000..f1f6f0d
--- /dev/null
+++ b/tools/fix-manpages
@@ -0,0 +1,33 @@
+# Because tools/*.8.inc include a synopsis, the following line removes
+# a redundant Usage: header from the man page and the couple lines after
+# it.
+/.SS "Usage:"/,+3d
+
+# The tools/*.8.inc files also include some additional section headers
+# on a per-tool basis.  These section headers will get printed out as
+# .SH lines, but then the header from the --help-all output will also
+# get turned into groff.  For instance, the following will be in the
+# man page for NOTES:
+#
+# .SH NOTES
+# .PP
+# Notes:
+# .PP
+#
+# The following block looks for any of those additional headers.  The
+# 'n' command puts the next line in the pattern space, the two 'N'
+# commands append the next two lines, and then the 'd' command deletes
+# them.  So basically, this just deletes
+#
+# .PP
+#  Notes:
+# .PP
+#
+# This leaves the --help-all output looking good and removes redundant
+# stuff from the man page.  Feel free to add additional headers here.
+# Not all tools will have all headers.
+/.SH NOTES\|.SH INTERACTIVE USE\|.SH OPERATION SPECIFICATION\|.SH OUTPUT CONTROL\|.SH TIME SPECIFICATION/{ n
+N
+N
+d
+}
diff --git a/tools/iso8601.8.inc b/tools/iso8601.8.inc
new file mode 100644
index 0000000..faa7f76
--- /dev/null
+++ b/tools/iso8601.8.inc
@@ -0,0 +1,5 @@
+[synopsis]
+iso8601 <command> [options]
+
+/dates and times/
+.SH OPTIONS
diff --git a/tools/iso8601.c b/tools/iso8601.c
new file mode 100644
index 0000000..e53bca0
--- /dev/null
+++ b/tools/iso8601.c
@@ -0,0 +1,282 @@
+/*
+ * Copyright 2005-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include <crm/crm.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/iso8601.h>
+#include <crm/common/util.h>  /* CRM_ASSERT */
+#include <unistd.h>
+
+#define SUMMARY "Display and parse ISO 8601 dates and times"
+
+struct {
+    char *date_time_s;
+    gchar *duration_s;
+    gchar *expected_s;
+    gchar *period_s;
+    int print_options;
+} options;
+
+#define INDENT "                              "
+
+static gboolean
+date_now_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_any_of(option_name, "--now", "-n", NULL)) {
+        pcmk__str_update(&options.date_time_s, "now");
+    } else if (pcmk__str_any_of(option_name, "--date", "-d", NULL)) {
+        pcmk__str_update(&options.date_time_s, optarg);
+    }
+
+    return TRUE;
+}
+
+static gboolean
+modifier_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_any_of(option_name, "--seconds", "-s", NULL)) {
+        options.print_options |= crm_time_seconds;
+    } else if (pcmk__str_any_of(option_name, "--epoch", "-S", NULL)) {
+        options.print_options |= crm_time_epoch;
+    } else if (pcmk__str_any_of(option_name, "--local", "-L", NULL)) {
+        options.print_options |= crm_time_log_with_timezone;
+    } else if (pcmk__str_any_of(option_name, "--ordinal", "-O", NULL)) {
+        options.print_options |= crm_time_ordinal;
+    } else if (pcmk__str_any_of(option_name, "--week", "-W", NULL)) {
+        options.print_options |= crm_time_weeks;
+    }
+
+    return TRUE;
+}
+
+static GOptionEntry command_entries[] = {
+    { "now", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, date_now_cb,
+      "Display the current date/time",
+      NULL },
+
+    { "date", 'd', 0, G_OPTION_ARG_CALLBACK, date_now_cb,
+      "Parse an ISO 8601 date/time (for example,\n"
+      INDENT "'2019-09-24 00:30:00 +01:00' or '2019-040')",
+      "DATE" },
+
+    { "period", 'p', 0, G_OPTION_ARG_STRING, &options.period_s,
+      "Parse an ISO 8601 period (interval) with start time (for example,\n"
+      INDENT "'2005-040/2005-043')",
+      "PERIOD" },
+
+    { "duration", 'D', 0, G_OPTION_ARG_STRING, &options.duration_s,
+      "Parse an ISO 8601 duration (for example, 'P1M')",
+      "DURATION" },
+
+    { "expected", 'E', 0, G_OPTION_ARG_STRING, &options.expected_s,
+      "Exit with error status if result does not match this text.\n"
+      INDENT "Requires: -n or -d",
+      "TEXT" },
+
+    { NULL }
+};
+
+static GOptionEntry modifier_entries[] = {
+    { "seconds", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, modifier_cb,
+      "Show result as a seconds since 0000-001 00:00:00Z",
+      NULL },
+
+    { "epoch", 'S', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, modifier_cb,
+      "Show result as a seconds since EPOCH (1970-001 00:00:00Z)",
+      NULL },
+
+    { "local", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, modifier_cb,
+      "Show result as a 'local' date/time",
+      NULL },
+
+    { "ordinal", 'O', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, modifier_cb,
+      "Show result as an 'ordinal' date/time",
+      NULL },
+
+    { "week", 'W', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, modifier_cb,
+      "Show result as an 'calendar week' date/time",
+      NULL },
+
+    { NULL }
+};
+
+static void
+log_time_period(int log_level, crm_time_period_t * dtp, int flags)
+{
+    char *start = crm_time_as_string(dtp->start, flags);
+    char *end = crm_time_as_string(dtp->end, flags);
+
+    CRM_ASSERT(start != NULL && end != NULL);
+    do_crm_log(log_level, "Period: %s to %s", start, end);
+    free(start);
+    free(end);
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args) {
+    GOptionContext *context = NULL;
+
+    const char *description = "For more information on the ISO 8601 standard, see " \
+                              "https://en.wikipedia.org/wiki/ISO_8601";
+
+    context = pcmk__build_arg_context(args, NULL, NULL, NULL);
+    g_option_context_set_description(context, description);
+
+    pcmk__add_arg_group(context, "commands", "Commands:",
+                        "Show command options", command_entries);
+    pcmk__add_arg_group(context, "modifiers", "Output modifiers:",
+                        "Show output modifiers", modifier_entries);
+
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    crm_exit_t exit_code = CRM_EX_OK;
+    crm_time_t *duration = NULL;
+    crm_time_t *date_time = NULL;
+
+    GError *error = NULL;
+
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    GOptionContext *context = build_arg_context(args);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "dpDE");
+
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("iso8601", args->verbosity);
+
+    if (args->version) {
+        g_strfreev(processed_args);
+        pcmk__free_arg_context(context);
+        /* FIXME:  When iso8601 is converted to use formatted output, this can go. */
+        pcmk__cli_help('v');
+    }
+
+    if (pcmk__str_eq("now", options.date_time_s, pcmk__str_casei)) {
+        date_time = crm_time_new(NULL);
+
+        if (date_time == NULL) {
+            exit_code = CRM_EX_SOFTWARE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Internal error: couldn't determine 'now'!");
+            goto done;
+        }
+
+        crm_time_log(LOG_TRACE, "Current date/time", date_time,
+                     crm_time_ordinal | crm_time_log_date | crm_time_log_timeofday);
+        crm_time_log(LOG_STDOUT, "Current date/time", date_time,
+                     options.print_options | crm_time_log_date | crm_time_log_timeofday);
+
+    } else if (options.date_time_s) {
+        date_time = crm_time_new(options.date_time_s);
+
+        if (date_time == NULL) {
+            exit_code = CRM_EX_INVALID_PARAM;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Invalid date/time specified: %s", options.date_time_s);
+            goto done;
+        }
+
+        crm_time_log(LOG_TRACE, "Date", date_time,
+                     crm_time_ordinal | crm_time_log_date | crm_time_log_timeofday);
+        crm_time_log(LOG_STDOUT, "Date", date_time,
+                     options.print_options | crm_time_log_date | crm_time_log_timeofday);
+    }
+
+    if (options.duration_s) {
+        duration = crm_time_parse_duration(options.duration_s);
+
+        if (duration == NULL) {
+            exit_code = CRM_EX_INVALID_PARAM;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Invalid duration specified: %s", options.duration_s);
+            goto done;
+        }
+
+        crm_time_log(LOG_TRACE, "Duration", duration, crm_time_log_duration);
+        crm_time_log(LOG_STDOUT, "Duration", duration,
+                     options.print_options | crm_time_log_duration);
+    }
+
+    if (options.period_s) {
+        crm_time_period_t *period = crm_time_parse_period(options.period_s);
+
+        if (period == NULL) {
+            exit_code = CRM_EX_INVALID_PARAM;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Invalid interval specified: %s", options.period_s);
+            goto done;
+        }
+
+        log_time_period(LOG_TRACE, period,
+                        options.print_options | crm_time_log_date | crm_time_log_timeofday);
+        log_time_period(LOG_STDOUT, period,
+                        options.print_options | crm_time_log_date | crm_time_log_timeofday);
+        crm_time_free_period(period);
+    }
+
+    if (date_time && duration) {
+        crm_time_t *later = crm_time_add(date_time, duration);
+
+        if (later == NULL) {
+            exit_code = CRM_EX_SOFTWARE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Unable to calculate ending time of %s plus %s",
+                        options.date_time_s, options.duration_s);
+            goto done;
+        }
+
+        crm_time_log(LOG_TRACE, "Duration ends at", later,
+                     crm_time_ordinal | crm_time_log_date | crm_time_log_timeofday);
+        crm_time_log(LOG_STDOUT, "Duration ends at", later,
+                     options.print_options | crm_time_log_date | crm_time_log_timeofday |
+                     crm_time_log_with_timezone);
+
+        if (options.expected_s) {
+            char *dt_s = crm_time_as_string(later,
+                                            options.print_options | crm_time_log_date |
+                                            crm_time_log_timeofday);
+            if (!pcmk__str_eq(options.expected_s, dt_s, pcmk__str_casei)) {
+                exit_code = CRM_EX_ERROR;
+                goto done;
+            }
+            free(dt_s);
+        }
+        crm_time_free(later);
+
+    } else if (date_time && options.expected_s) {
+        char *dt_s = crm_time_as_string(date_time,
+                                        options.print_options | crm_time_log_date | crm_time_log_timeofday);
+
+        if (!pcmk__str_eq(options.expected_s, dt_s, pcmk__str_casei)) {
+            exit_code = CRM_EX_ERROR;
+            goto done;
+        }
+        free(dt_s);
+    }
+
+done:
+    crm_time_free(date_time);
+    crm_time_free(duration);
+
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    free(options.date_time_s);
+    g_free(options.duration_s);
+    g_free(options.expected_s);
+    g_free(options.period_s);
+
+    pcmk__output_and_clear_error(&error, NULL);
+    crm_exit(exit_code);
+}
diff --git a/tools/pcmk_simtimes.in b/tools/pcmk_simtimes.in
new file mode 100644
index 0000000..c8b0af6
--- /dev/null
+++ b/tools/pcmk_simtimes.in
@@ -0,0 +1,159 @@
+#!@PYTHON@
+""" Timing comparisons for crm_simulate profiling output
+"""
+
+__copyright__ = "Copyright 2019-2023 the Pacemaker project contributors"
+__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
+
+import io
+import re
+import sys
+import errno
+import argparse
+import os
+
+# These imports allow running from a source checkout after running `make`.
+# Note that while this doesn't necessarily mean it will successfully run tests,
+# but being able to see --help output can be useful.
+if os.path.exists("@abs_top_srcdir@/python"):
+    sys.path.insert(0, "@abs_top_srcdir@/python")
+
+if os.path.exists("@abs_top_builddir@/python") and "@abs_top_builddir@" != "@abs_top_srcdir@":
+    sys.path.insert(0, "@abs_top_builddir@/python")
+
+from pacemaker.exitstatus import ExitStatus
+
+DESC = """Compare timings from crm_simulate profiling output"""
+
+BEFORE_HELP = """Output of "crm_simulate --profile cts/scheduler --repeat <N>" from earlier Pacemaker build"""
+
+# line like: * Testing cts/scheduler/xml/1360.xml ... 0.07 secs
+PATTERN = r"""^\s*\*\s+Testing\s+.*/([^/]+)\.xml\s+\.+\s+([.0-9]+)\s+secs\s*$"""
+
+def parse_args(argv=sys.argv):
+    """ Parse command-line arguments """
+
+    parser = argparse.ArgumentParser(description=DESC)
+
+    parser.add_argument('-V', '--verbose', action='count',
+                        help='Increase verbosity')
+
+    parser.add_argument('-p', '--threshold-percent', type=float, default=0,
+                        help="Don't show tests with less than this percentage difference in times")
+
+    parser.add_argument('-s', '--threshold-seconds', type=float, default=0,
+                        help="Don't show tests with less than this seconds difference in times")
+
+    parser.add_argument('-S', '--sort', choices=['test', 'before', 'after', 'diff', 'percent'],
+                        default='test', help="Sort results by this column")
+
+    parser.add_argument('-r', '--reverse', action='store_true',
+                        help="Sort results in descending order")
+
+    parser.add_argument('before_file', metavar='BEFORE',
+                        type=argparse.FileType('r'),
+                        help=BEFORE_HELP)
+
+    parser.add_argument('after_file', metavar='AFTER',
+                        type=argparse.FileType('r'),
+                        help='Output of same command from later Pacemaker build')
+
+    return parser.parse_args(argv[1:])
+
+
+def extract_times(infile):
+    """ Extract test names and times into hash table from file """
+
+    result = {}
+    for line in infile:
+        match = re.search(PATTERN, line)
+        if match is not None:
+            result[match.group(1)] = match.group(2)
+    return result
+
+
+def compare_test(test, before, after, args):
+    """ Compare one test's timings """
+
+    try:
+        before_time = float(before[test])
+    except KeyError:
+        if args.verbose > 0:
+            print("No previous test " + test + " to compare")
+        return None
+
+    after_time = float(after[test])
+
+    time_diff = after_time - before_time
+    time_diff_percent = (time_diff / before_time) * 100
+
+    if ((abs(time_diff) >= args.threshold_seconds)
+        and (abs(time_diff_percent) >= args.threshold_percent)):
+        return { 'test': test,
+                 'before': before_time,
+                 'after': after_time,
+                 'diff': time_diff,
+                 'percent': time_diff_percent
+               }
+    return None
+
+def sort_diff(result):
+    """ Sort two test results by time difference """
+
+    global sort_field
+
+    return result[sort_field]
+
+
+def print_results(results, sort_reverse):
+    """ Output the comparison results """
+
+    if results == []:
+        return
+
+    # Sort and print test differences
+    results.sort(reverse=sort_reverse, key=sort_diff)
+    for result in results:
+        print("%-40s %6.2fs vs %6.2fs (%+.2fs = %+6.2f%%)" % (result['test'],
+              result['before'], result['after'], result['diff'],
+              result['percent']))
+
+    # Print average differences
+    diff_total = sum(d['diff'] for d in results)
+    percent_total = sum(d['percent'] for d in results)
+    nresults = len(results)
+    print("\nAverages: %+.2fs %+6.2f%%" % ((diff_total / nresults),
+          (percent_total / nresults)))
+
+
+if __name__ == "__main__":
+
+    global sort_field
+
+    try:
+        args = parse_args()
+        if args.verbose is None:
+            args.verbose = 0
+
+        before = extract_times(args.before_file)
+        after = extract_times(args.after_file)
+        sort_field = args.sort
+
+        # Build a list of test differences
+        results = []
+        for test in after.keys():
+            result = compare_test(test, before, after, args)
+            if result is not None:
+                results = results + [ result ]
+
+        print_results(results, sort_reverse=args.reverse)
+
+    except KeyboardInterrupt:
+        pass
+    except IOError as e:
+        if e.errno != errno.EPIPE:
+            raise
+
+    sys.exit(ExitStatus.OK)
+
+# vim: set filetype=python expandtab tabstop=4 softtabstop=4 shiftwidth=4 textwidth=120:
diff --git a/tools/report.collector.in b/tools/report.collector.in
new file mode 100644
index 0000000..315b785
--- /dev/null
+++ b/tools/report.collector.in
@@ -0,0 +1,885 @@
+#
+# Originally based on hb_report
+# Copyright 2007 Dejan Muhamedagic <dmuhamedagic@suse.de>
+# Later changes copyright 2010-2018 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+if
+    echo $REPORT_HOME | grep -qs '^/'
+then
+    debug "Using full path to working directory: $REPORT_HOME"
+else
+    REPORT_HOME="$HOME/$REPORT_HOME"
+    debug "Canonicalizing working directory path: $REPORT_HOME"
+fi
+
+detect_host
+
+#
+# find files newer than a and older than b
+#
+isnumber() {
+    echo "$*" | grep -qs '^[0-9][0-9]*$'
+}
+
+touchfile() {
+    t=`mktemp` &&
+        perl -e "\$file=\"$t\"; \$tm=$1;" -e 'utime $tm, $tm, $file;' &&
+        echo $t
+}
+
+find_files_clean() {
+    [ -z "$from_stamp" ] || rm -f "$from_stamp"
+    [ -z "$to_stamp" ] || rm -f "$to_stamp"
+    from_stamp=""
+    to_stamp=""
+}
+
+find_files() {
+    dirs=
+    from_time=$2
+    to_time=$3
+    for d in $1; do
+        if [ -d $d ]; then
+	   dirs="$dirs $d"
+	fi
+    done
+
+    if [ x"$dirs" = x ]; then
+	return
+    fi
+
+    isnumber "$from_time" && [ "$from_time" -gt 0 ] || {
+	warning "sorry, can't find files in [ $1 ] based on time if you don't supply time"
+	return
+    }
+    trap find_files_clean 0
+    if ! from_stamp=`touchfile $from_time`; then
+	warning "sorry, can't create temporary file for find_files"
+	return
+    fi
+    findexp="-newer $from_stamp"
+    if isnumber "$to_time" && [ "$to_time" -gt 0 ]; then
+	if ! to_stamp=`touchfile $to_time`; then
+	    warning "sorry, can't create temporary file for find_files"
+	    find_files_clean
+	    return
+	fi
+	findexp="$findexp ! -newer $to_stamp"
+    fi
+    find $dirs -type f $findexp
+    find_files_clean
+    trap "" 0
+}
+
+#
+# check permissions of files/dirs
+#
+pl_checkperms() {
+    perl -e '
+	# check permissions and ownership
+	# uid and gid are numeric
+	# everything must match exactly
+	# no error checking! (file should exist, etc)
+	($filename, $perms, $in_uid, $in_gid) = @ARGV;
+	($mode,$uid,$gid) = (stat($filename))[2,4,5];
+	$p=sprintf("%04o", $mode & 07777);
+	$p ne $perms and exit(1);
+	$uid ne $in_uid and exit(1);
+	$gid ne $in_gid and exit(1);
+    ' $*
+}
+
+num_id() {
+    getent $1 $2 | awk -F: '{print $3}'
+}
+
+chk_id() {
+    [ "$2" ] && return 0
+    echo "$1: id not found"
+    return 1
+}
+
+check_perms() {
+    while read type f p uid gid; do
+        if [ ! -e "$f" ]; then
+            echo "$f doesn't exist"
+            continue
+        elif [ ! -$type "$f" ]; then
+            echo "$f has wrong type"
+            continue
+        fi
+	n_uid=`num_id passwd $uid`
+	chk_id "$uid" "$n_uid" || continue
+	n_gid=`num_id group $gid`
+	chk_id "$gid" "$n_gid" || continue
+	pl_checkperms $f $p $n_uid $n_gid || {
+	    echo "wrong permissions or ownership for $f:"
+	    ls -ld $f
+	}
+    done
+}
+
+#
+# coredumps
+#
+findbinary() {
+    random_binary=`which cat 2>/dev/null` # suppose we are lucky
+    binary=`gdb $random_binary $1 < /dev/null 2>/dev/null |
+	grep 'Core was generated' | awk '{print $5}' |
+	sed "s/^.//;s/[.':]*$//"`
+    if [ x = x"$binary" ]; then
+	debug "Could not detect the program name for core $1 from the gdb output; will try with file(1)"
+	binary=$(file $1 | awk '/from/{
+			for( i=1; i<=NF; i++ )
+				if( $i == "from" ) {
+					print $(i+1)
+					break
+				}
+			}')
+	binary=`echo $binary | tr -d "'"`
+	binary=$(echo $binary | tr -d '`')
+	if [ "$binary" ]; then
+	    binary=`which $binary 2>/dev/null`
+	fi
+    fi
+    if [ x = x"$binary" ]; then
+	warning "Could not find the program path for core $1"
+	return
+    fi
+    fullpath=`which $binary 2>/dev/null`
+    if [ x = x"$fullpath" ]; then
+	if [ -x $CRM_DAEMON_DIR/$binary ]; then
+	    echo $CRM_DAEMON_DIR/$binary
+	    debug "Found the program at $CRM_DAEMON_DIR/$binary for core $1"
+	else
+	    warning "Could not find the program path for core $1"
+	fi
+    else
+	echo $fullpath
+	debug "Found the program at $fullpath for core $1"
+    fi
+}
+
+getbt() {
+    which gdb > /dev/null 2>&1 || {
+	warning "Please install gdb to get backtraces"
+	return
+    }
+    for corefile; do
+	absbinpath=`findbinary $corefile`
+	[ x = x"$absbinpath" ] && continue
+	echo "====================== start backtrace ======================"
+	ls -l $corefile
+	# Summary first...
+	gdb -batch -n -quiet -ex ${BT_OPTS:-"thread apply all bt"} -ex quit \
+	    $absbinpath $corefile 2>/dev/null
+	echo "====================== start detail ======================"
+	# Now the unreadable details...
+	gdb -batch -n -quiet -ex ${BT_OPTS:-"thread apply all bt full"} -ex quit \
+	    $absbinpath $corefile 2>/dev/null
+	echo "======================= end backtrace ======================="
+    done
+}
+
+dump_status_and_config() {
+    crm_mon -1 2>&1 | grep -v '^Last upd' > $target/$CRM_MON_F
+    cibadmin -Ql 2>/dev/null > $target/${CIB_F}.live
+}
+
+getconfig() {
+    cluster=$1; shift;
+    target=$1; shift;
+
+    for cf in $*; do
+	if [ -e "$cf" ]; then
+	    cp -a "$cf" $target/
+	fi
+    done
+
+    if is_running pacemaker-controld; then
+        dump_status_and_config
+        crm_node -p > "$target/$MEMBERSHIP_F" 2>&1
+	echo "$host" > $target/RUNNING
+
+    elif is_running pacemaker-remoted; then
+        dump_status_and_config
+        echo "$host" > $target/RUNNING
+
+    # Pre-2.0.0 daemon name in case we're collecting on a mixed-version cluster
+    elif is_running pacemaker_remoted; then
+        dump_status_and_config
+        echo "$host" > $target/RUNNING
+
+    else
+	echo "$host" > $target/STOPPED
+    fi
+}
+
+get_readable_cib() {
+    target="$1"; shift;
+
+    if [ -f "$target/$CIB_F" ]; then
+        crm_verify -V -x "$target/$CIB_F" >"$target/$CRM_VERIFY_F" 2>&1
+        if which crm >/dev/null 2>&1 ; then
+            CIB_file="$target/$CIB_F" crm configure show >"$target/$CIB_TXT_F" 2>&1
+        elif which pcs >/dev/null 2>&1 ; then
+            pcs config -f "$target/$CIB_F" >"$target/$CIB_TXT_F" 2>&1
+        fi
+    fi
+}
+
+#
+# remove values of sensitive attributes
+#
+# this is not proper xml parsing, but it will work under the
+# circumstances
+sanitize_xml_attrs() {
+    sed $(
+	for patt in $SANITIZE; do
+	    echo "-e /name=\"$patt\"/s/value=\"[^\"]*\"/value=\"****\"/"
+	done
+    )
+}
+
+sanitize_hacf() {
+    awk '
+	$1=="stonith_host"{ for( i=5; i<=NF; i++ ) $i="****"; }
+	{print}
+	'
+}
+
+sanitize_one_clean() {
+    [ -z "$tmp" ] || rm -f "$tmp"
+    tmp=""
+    [ -z "$ref" ] || rm -f "$ref"
+    ref=""
+}
+
+sanitize() {
+    file=$1
+    compress=""
+    if [ -z "$SANITIZE" ]; then
+	return
+    fi
+    echo $file | grep -qs 'gz$' && compress=gzip
+    echo $file | grep -qs 'bz2$' && compress=bzip2
+    if [ "$compress" ]; then
+	decompress="$compress -dc"
+    else
+	compress=cat
+	decompress=cat
+    fi
+    trap sanitize_one_clean 0
+    tmp=`mktemp`
+    ref=`mktemp`
+    if [ -z "$tmp" -o -z "$ref" ]; then
+	sanitize_one_clean
+	fatal "cannot create temporary files"
+    fi
+    touch -r $file $ref  # save the mtime
+    if [ "`basename $file`" = ha.cf ]; then
+	sanitize_hacf
+    else
+	$decompress | sanitize_xml_attrs | $compress
+    fi < $file > $tmp
+    mv $tmp $file
+	# note: cleaning $tmp up is still needed even after it's renamed
+	# because its temp directory is still there.
+
+	touch -r $ref $file
+	sanitize_one_clean
+	trap "" 0
+}
+
+#
+# get some system info
+#
+distro() {
+    if
+	which lsb_release >/dev/null 2>&1
+    then
+	lsb_release -d | sed -e 's/^Description:\s*//'
+	debug "Using lsb_release for distribution info"
+	return
+    fi
+
+    relf=`ls /etc/debian_version 2>/dev/null` ||
+    relf=`ls /etc/slackware-version 2>/dev/null` ||
+    relf=`ls -d /etc/*-release 2>/dev/null` && {
+	for f in $relf; do
+	    test -f $f && {
+		echo "`ls $f` `cat $f`"
+		debug "Found `echo $relf | tr '\n' ' '` distribution release file(s)"
+		return
+	    }
+	done
+    }
+    warning "No lsb_release, no /etc/*-release, no /etc/debian_version: no distro information"
+}
+
+pkg_ver() {
+    if which dpkg >/dev/null 2>&1 ; then
+	pkg_mgr="deb"
+    elif which rpm >/dev/null 2>&1 ; then
+	pkg_mgr="rpm"
+    elif which pkg_info >/dev/null 2>&1 ; then
+	pkg_mgr="pkg_info"
+    elif which pkginfo >/dev/null 2>&1 ; then
+	pkg_mgr="pkginfo"
+    else
+	warning "Unknown package manager"
+	return
+    fi
+    debug "The package manager is: $pkg_mgr"
+    echo "The package manager is: $pkg_mgr"
+
+    echo "Installed packages:"
+    case $pkg_mgr in
+	deb)
+	    dpkg-query -f '${Package} ${Version} ${Architecture}\n' -W | sort
+            echo
+	    for pkg in $*; do
+		if dpkg-query -W $pkg 2>/dev/null ; then
+		    debug "Verifying installation of: $pkg"
+		    echo "Verifying installation of: $pkg"
+		    debsums -s $pkg 2>/dev/null
+		fi
+	    done
+	    ;;
+	rpm)
+	    rpm -qa --qf '%{name} %{version}-%{release} - %{distribution} %{arch}\n' | sort
+            echo
+	    for pkg in $*; do
+		if rpm -q $pkg >/dev/null 2>&1 ; then
+		    debug "Verifying installation of: $pkg"
+		    echo "Verifying installation of: $pkg"
+		    rpm --verify $pkg 2>&1
+		fi
+	    done
+	    ;;
+	pkg_info)
+	    pkg_info
+	    ;;
+	pkginfo)
+	    pkginfo | awk '{print $3}'  # format?
+	    ;;
+    esac
+}
+
+getbacktraces() {
+    debug "Looking for backtraces: $*"
+    flist=$(
+	for f in `find_files "$CRM_CORE_DIRS" $1 $2`; do
+	    bf=`basename $f`
+	    test `expr match $bf core` -gt 0 &&
+	    echo $f
+	done)
+    if [ "$flist" ]; then
+	for core in $flist; do
+	    log "Found core file: `ls -al $core`"
+	done
+
+	# Make a copy of them in case we need more data later
+	# Luckily they compress well
+	mkdir cores >/dev/null 2>&1
+	cp -a $flist cores/
+	shrink cores
+	rm -rf cores
+
+	# Now get as much as we can from them automagically
+	for f in $flist; do
+	    getbt $f
+        done
+    fi
+}
+
+getpeinputs() {
+    if [ -n "$PE_STATE_DIR" ]; then
+        flist=$(
+            find_files "$PE_STATE_DIR" "$1" "$2" | sed "s,`dirname $PE_STATE_DIR`/,,g"
+        )
+        if [ "$flist" ]; then
+            (cd $(dirname "$PE_STATE_DIR") && tar cf - $flist) | (cd "$3" && tar xf -)
+            debug "found `echo $flist | wc -w` scheduler input files in $PE_STATE_DIR"
+        fi
+    fi
+}
+
+getblackboxes() {
+    flist=$(
+	find_files $BLACKBOX_DIR $1 $2
+    )
+
+    for bb in $flist; do
+        bb_short=`basename $bb`
+	qb-blackbox $bb > $3/${bb_short}.blackbox 2>&1
+	info "Extracting contents of blackbox: $bb_short"
+    done
+}
+
+#
+# some basic system info and stats
+#
+sys_info() {
+    cluster=$1; shift
+    echo "Platform: `uname`"
+    echo "Kernel release: `uname -r`"
+    echo "Architecture: `uname -m`"
+    if [ `uname` = Linux ]; then
+	echo "Distribution: `distro`"
+    fi
+
+    echo
+    cibadmin --version 2>&1 | head -1
+    cibadmin -! 2>&1
+    case $cluster in
+	corosync)
+	    /usr/sbin/corosync -v 2>&1 | head -1
+	    ;;
+    esac
+
+    # Cluster glue version hash (if available)
+    stonith -V 2>/dev/null
+
+    # Resource agents version hash
+    echo "resource-agents: `grep 'Build version:' /usr/lib/ocf/resource.d/heartbeat/.ocf-shellfuncs`"
+
+    echo
+    pkg_ver $*
+}
+
+sys_stats() {
+    set -x
+    uname -n
+    uptime
+    ps axf
+    ps auxw
+    top -b -n 1
+    ifconfig -a
+    ip addr list
+    netstat -i
+    arp -an
+    test -d /proc && {
+	cat /proc/cpuinfo
+    }
+    lsscsi
+    lspci
+    lsblk
+    mount
+    df
+    set +x
+}
+
+dlm_dump() {
+    if which dlm_tool >/dev/null 2>&1 ; then
+      if is_running dlm_controld; then
+	echo "--- Lockspace overview:"
+	dlm_tool ls -n
+
+	echo "---Lockspace history:"
+	dlm_tool dump
+
+	echo "---Lockspace status:"
+	dlm_tool status
+	dlm_tool status -v
+
+	echo "---Lockspace config:"
+	dlm_tool dump_config
+
+	dlm_tool log_plock
+
+	dlm_tool ls | grep name |
+	while read X N ; do
+	    echo "--- Lockspace $N:"
+	    dlm_tool lockdump "$N"
+	    dlm_tool lockdebug -svw "$N"
+	done
+      fi
+    fi
+}
+
+drbd_info() {
+    test -f /proc/drbd && {
+        echo "--- /proc/drbd:"
+        cat /proc/drbd 2>&1
+        echo
+    }
+
+    if which drbdadm >/dev/null 2>&1; then
+        echo "--- drbdadm dump:"
+        if [ -z "$SANITIZE"]; then
+            drbdadm dump 2>&1
+        else
+            drbdadm dump 2>&1 | sed "s/\(shared-secret[ 	]*\"\)[^\"]*\";/\1****\";/"
+        fi
+        echo
+
+        echo "--- drbdadm status:"
+        drbdadm status 2>&1
+        echo
+
+        echo "--- drbdadm show-gi:"
+        for res in $(drbdsetup status | grep -e ^\\S | awk '{ print $1 }'); do
+            echo "$res:"
+            drbdadm show-gi $res 2>&1
+            echo
+        done
+    fi
+
+    if which drbd-overview >/dev/null 2>&1; then
+        echo "--- drbd-overview:"
+        drbd-overview 2>&1
+        echo
+    fi
+
+    if which drbdsetup >/dev/null 2>&1; then
+        echo "--- drbdsetup status:"
+        drbdsetup status --verbose --statistics 2>&1
+        echo
+
+        echo "--- drbdsetup events2:"
+        drbdsetup events2 --timestamps --statistics --now 2>&1
+        echo
+    fi
+}
+
+iscfvarset() {
+    test "`getcfvar $1 $2`"
+}
+
+iscfvartrue() {
+    getcfvar $1 $2 $3 | grep -E -qsi "^(true|y|yes|on|1)"
+}
+
+iscfvarfalse() {
+    getcfvar $1 $2 $3 | grep -E -qsi "^(false|n|no|off|0)"
+}
+
+find_syslog() {
+    priority="$1"
+
+    # Always include system logs (if we can find them)
+    msg="Mark:pcmk:`perl -e 'print time()'`"
+    logger -p "$priority" "$msg" >/dev/null 2>&1
+
+    # Force buffer flush
+    killall -HUP rsyslogd >/dev/null 2>&1
+
+    sleep 2 # Give syslog time to catch up in case it's busy
+    findmsg 1 "$msg"
+}
+
+get_logfiles_cs() {
+    if [ ! -f "$cf_file" ]; then
+        return
+    fi
+
+    debug "Reading $cf_type log settings from $cf_file"
+
+    # The default value of to_syslog is yes.
+    if ! iscfvarfalse $cf_type to_syslog "$cf_file"; then
+        facility_cs=$(getcfvar $cf_type syslog_facility "$cf_file")
+        if [ -z "$facility_cs" ]; then
+            facility_cs="daemon"
+        fi
+
+        find_syslog "$facility_cs.info"
+    fi
+    if [ "$SOS_MODE" = "1" ]; then
+        return
+    fi
+
+    if iscfvartrue $cf_type to_logfile "$cf_file"; then
+        logfile=$(getcfvar $cf_type logfile "$cf_file")
+        if [ -f "$logfile" ]; then
+            debug "Log settings found for cluster type $cf_type: $logfile"
+            echo "$logfile"
+        fi
+    fi
+}
+
+get_logfiles() {
+    cf_type=$1
+    cf_file="$2"
+
+    case $cf_type in
+        corosync) get_logfiles_cs;;
+    esac
+
+    . @CONFIGDIR@/pacemaker
+
+    facility="$PCMK_logfacility"
+    if [ -z "$facility" ]; then
+        facility="daemon"
+    fi
+    if [ "$facility" != "$facility_cs" ]&&[ "$facility" != none ]; then
+        find_syslog "$facility.notice"
+    fi
+    if [ "$SOS_MODE" = "1" ]; then
+        return
+    fi
+
+    logfile="$PCMK_logfile"
+    if [ "$logfile" != none ]; then
+        if [ -z "$logfile" ]; then
+            for logfile in "@CRM_LOG_DIR@/pacemaker.log" "/var/log/pacemaker.log"; do
+                if [ -f "$logfile" ]; then
+                    debug "Log settings not found for Pacemaker, assuming $logfile"
+                    echo "$logfile"
+                    break
+                fi
+            done
+
+        elif [ -f "$logfile" ]; then
+            debug "Log settings found for Pacemaker: $logfile"
+            echo "$logfile"
+        fi
+    fi
+
+    # Look for detail logs:
+
+    # - initial pacemakerd logs and tracing might go to a different file
+    pattern="Starting Pacemaker"
+
+    # - make sure we get something from the scheduler
+    pattern="$pattern\\|Calculated transition"
+
+    # - cib and pacemaker-execd updates
+    # (helpful on non-DC nodes and when cluster has been up for a long time)
+    pattern="$pattern\\|cib_perform_op\\|process_lrm_event"
+
+    # - pacemaker_remote might use a different file
+    pattern="$pattern\\|pacemaker[-_]remoted:"
+
+    findmsg 3 "$pattern"
+}
+
+essential_files() {
+	cat<<EOF
+d $PE_STATE_DIR 0750 hacluster haclient
+d $CRM_CONFIG_DIR 0750 hacluster haclient
+d $CRM_STATE_DIR 0750 hacluster haclient
+EOF
+}
+
+# Trim leading and ending whitespace (using only POSIX expressions)
+trim() {
+    TRIM_S="$1"
+
+    TRIM_S="${TRIM_S#"${TRIM_S%%[![:space:]]*}"}"
+    TRIM_S="${TRIM_S%"${TRIM_S##*[![:space:]]}"}"
+    echo -n "$TRIM_S"
+}
+
+collect_logs() {
+    CL_START="$1"
+    shift
+    CL_END="$1"
+    shift
+    CL_LOGFILES="$@"
+
+    which journalctl > /dev/null 2>&1
+    if [ $? -eq 0 ]; then
+        cl_have_journald=1
+    else
+        cl_have_journald=0
+    fi
+
+    cl_lognames="$CL_LOGFILES"
+    if [ $cl_have_journald -eq 1 ]; then
+        cl_lognames="$cl_lognames journalctl"
+    fi
+    cl_lognames=$(trim "$cl_lognames")
+    if [ -z "$cl_lognames" ]; then
+        return
+    fi
+
+    # YYYY-MM-DD HH:MM:SS
+    cl_start_ymd=$(date -d @${CL_START} +"%F %T")
+    cl_end_ymd=$(date -d @${CL_END} +"%F %T")
+
+    debug "Gathering logs from $cl_start_ymd to $cl_end_ymd:"
+    debug "   $cl_lognames"
+
+    # Remove our temporary file if we get interrupted here
+    trap '[ -z "$cl_pattfile" ] || rm -f "$cl_pattfile"' 0
+
+    # Create a temporary file with patterns to grep for
+    cl_pattfile=$(mktemp) || fatal "cannot create temporary files"
+    for cl_pattern in $LOG_PATTERNS; do
+        echo "$cl_pattern"
+    done > $cl_pattfile
+
+    echo "Log pattern matches from $REPORT_TARGET:" > $ANALYSIS_F
+    if [ -n "$CL_LOGFILES" ]; then
+        for cl_logfile in $CL_LOGFILES; do
+            cl_extract="$(basename $cl_logfile).extract.txt"
+
+            if [ ! -f "$cl_logfile" ]; then
+                # Not a file
+                continue
+
+            elif [ -f "$cl_extract" ]; then
+                # We already have it
+                continue
+            fi
+
+            dumplogset "$cl_logfile" $LOG_START $LOG_END > "$cl_extract"
+            sanitize "$cl_extract"
+
+            grep -f "$cl_pattfile" "$cl_extract" >> $ANALYSIS_F
+        done
+    fi
+
+    # Collect systemd logs if present
+    if [ $cl_have_journald -eq 1 ]; then
+        journalctl --since "$cl_start_ymd" --until "$cl_end_ymd" > journal.log
+        grep -f "$cl_pattfile" journal.log >> $ANALYSIS_F
+    fi
+
+    rm -f $cl_pattfile
+    trap "" 0
+}
+
+require_tar
+
+debug "Initializing $REPORT_TARGET subdir"
+if [ "$REPORT_MASTER" != "$REPORT_TARGET" ]; then
+  if [ -e $REPORT_HOME/$REPORT_TARGET ]; then
+    warning "Directory $REPORT_HOME/$REPORT_TARGET already exists, using /tmp/$$/$REPORT_TARGET instead"
+    REPORT_HOME=/tmp/$$
+  fi
+fi
+
+mkdir -p $REPORT_HOME/$REPORT_TARGET
+cd $REPORT_HOME/$REPORT_TARGET
+
+case $CLUSTER in
+    any) cluster=`get_cluster_type`;;
+    *) cluster=$CLUSTER;;
+esac
+
+cluster_cf=`find_cluster_cf $cluster`
+
+# If cluster stack is still "any", this might be a Pacemaker Remote node,
+# so don't complain in that case.
+if [ -z "$cluster_cf" ] && [ $cluster != "any" ]; then
+   warning "Could not determine the location of your cluster configuration"
+fi
+
+if [ "$SEARCH_LOGS" = "1" ]; then
+    logfiles=$(get_logfiles "$cluster" "$cluster_cf" | sort -u)
+fi
+logfiles="$(trim "$logfiles $EXTRA_LOGS")"
+
+if [ -z "$logfiles" ]; then
+    which journalctl > /dev/null 2>&1
+    if [ $? -eq 0 ]; then
+        info "Systemd journal will be only log collected"
+    else
+        info "No logs will be collected"
+    fi
+    info "No log files found or specified with --logfile /some/path"
+fi
+
+debug "Config: $cluster ($cluster_cf) $logfiles"
+
+sys_info $cluster $PACKAGES > $SYSINFO_F
+essential_files $cluster | check_perms  > $PERMISSIONS_F 2>&1
+getconfig $cluster "$REPORT_HOME/$REPORT_TARGET" "$cluster_cf" "$CRM_CONFIG_DIR/$CIB_F" "/etc/drbd.conf" "/etc/drbd.d" "/etc/booth"
+
+getpeinputs    $LOG_START $LOG_END $REPORT_HOME/$REPORT_TARGET
+getbacktraces  $LOG_START $LOG_END > $REPORT_HOME/$REPORT_TARGET/$BT_F
+getblackboxes  $LOG_START $LOG_END $REPORT_HOME/$REPORT_TARGET
+
+case $cluster in
+    corosync)
+	if is_running corosync; then
+            corosync-blackbox >corosync-blackbox-live.txt 2>&1
+#           corosync-fplay > corosync-blackbox.txt
+            tool=`pickfirst corosync-objctl corosync-cmapctl`
+            case $tool in
+                *objctl)  $tool -a > corosync.dump  2>/dev/null;;
+                *cmapctl) $tool    > corosync.dump  2>/dev/null;;
+            esac
+            corosync-quorumtool -s -i > corosync.quorum 2>&1
+	fi
+	;;
+esac
+
+dc=`crm_mon -1 2>/dev/null | awk '/Current DC/ {print $3}'`
+if [ "$REPORT_TARGET" = "$dc" ]; then
+    echo "$REPORT_TARGET" > DC
+fi
+
+dlm_dump  > $DLM_DUMP_F 2>&1
+sys_stats > $SYSSTATS_F 2>&1
+drbd_info > $DRBD_INFO_F 2>&1
+
+debug "Sanitizing files: $SANITIZE"
+#
+# replace sensitive info with '****'
+#
+cf=""
+if [ ! -z "$cluster_cf" ]; then
+   cf=`basename $cluster_cf`
+fi
+for f in "$cf" "$CIB_F" "$CIB_F.live" pengine/*; do
+    if [ -f "$f" ]; then
+	sanitize "$f"
+    fi
+done
+
+# For convenience, generate human-readable version of CIB and any XML errors
+# in it (AFTER sanitizing, so we don't need to sanitize this output).
+# sosreport does this itself, so we do not need to when run by sosreport.
+if [ "$SOS_MODE" != "1" ]; then
+    get_readable_cib "$REPORT_HOME/$REPORT_TARGET"
+fi
+
+collect_logs "$LOG_START" "$LOG_END" $logfiles
+
+# Purge files containing no information
+for f in `ls -1`; do
+    if [ -d "$f" ]; then
+	continue
+    elif [ ! -s "$f" ]; then
+        case $f in
+	    *core*) log "Detected empty core file: $f";;
+	    *)	    debug "Removing empty file: `ls -al $f`"
+		    rm -f $f
+		    ;;
+	esac
+    fi
+done
+
+# Parse for events
+for l in $logfiles; do
+    b="$(basename $l).extract.txt"
+    node_events "$b" > $EVENTS_F
+
+    # Link the first logfile to a standard name if it doesn't yet exist
+    if [ -e "$b" -a ! -e "$HALOG_F" ]; then
+	ln -s "$b" "$HALOG_F"
+    fi
+done
+
+if [ -e $REPORT_HOME/.env ]; then
+    debug "Localhost: $REPORT_MASTER $REPORT_TARGET"
+
+elif [ "$REPORT_MASTER" != "$REPORT_TARGET" ]; then
+    debug "Streaming report back to $REPORT_MASTER"
+    (cd $REPORT_HOME && tar cf - $REPORT_TARGET)
+    if [ "$REMOVE" = "1" ]; then
+	cd
+	rm -rf $REPORT_HOME
+    fi
+fi
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4 textwidth=80:
diff --git a/tools/report.common.in b/tools/report.common.in
new file mode 100644
index 0000000..aa90c05
--- /dev/null
+++ b/tools/report.common.in
@@ -0,0 +1,890 @@
+#
+# Originally based on hb_report
+# Copyright 2007 Dejan Muhamedagic <dmuhamedagic@suse.de>
+# Later changes copyright 2010-2022 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+host=`uname -n`
+shorthost=`echo $host | sed s:\\\\..*::`
+if [ -z $verbose ]; then
+    verbose=0
+fi
+
+# Target Files
+EVENTS_F=events.txt
+ANALYSIS_F=analysis.txt
+HALOG_F=cluster-log.txt
+BT_F=backtraces.txt
+SYSINFO_F=sysinfo.txt
+SYSSTATS_F=sysstats.txt
+DLM_DUMP_F=dlm_dump.txt
+CRM_MON_F=crm_mon.txt
+MEMBERSHIP_F=members.txt
+CRM_VERIFY_F=crm_verify.txt
+PERMISSIONS_F=permissions.txt
+CIB_F=cib.xml
+CIB_TXT_F=cib.txt
+DRBD_INFO_F=drbd_info.txt
+
+EVENT_PATTERNS="
+state		do_state_transition
+membership	pcmk_peer_update.*(lost|memb):
+quorum		(crmd|pacemaker-controld).*crm_update_quorum
+pause		Process.pause.detected
+resources	(lrmd|pacemaker-execd).*rsc:(start|stop)
+stonith		te_fence_node|fenced.*(requests|(Succeeded|Failed).to.|result=)
+start_stop	shutdown.decision|Corosync.Cluster.Engine|corosync.*Initializing.transport|Executive.Service.RELEASE|crm_shutdown:.Requesting.shutdown|pcmk_shutdown:.Shutdown.complete
+"
+
+# superset of all packages of interest on all distros
+# (the package manager will be used to validate the installation
+# of any of these packages that are installed)
+PACKAGES="pacemaker pacemaker-libs pacemaker-cluster-libs libpacemaker3
+pacemaker-remote pacemaker-pygui pacemaker-pymgmt pymgmt-client
+corosync corosynclib libcorosync4
+resource-agents cluster-glue-libs cluster-glue libglue2 ldirectord
+ocfs2-tools ocfs2-tools-o2cb ocfs2console
+ocfs2-kmp-default ocfs2-kmp-pae ocfs2-kmp-xen ocfs2-kmp-debug ocfs2-kmp-trace
+drbd drbd-kmp-xen drbd-kmp-pae drbd-kmp-default drbd-kmp-debug drbd-kmp-trace
+drbd-pacemaker drbd-utils drbd-bash-completion drbd-xen
+lvm2 lvm2-clvm cmirrord
+libdlm libdlm2 libdlm3
+hawk ruby lighttpd
+kernel-default kernel-pae kernel-xen
+glibc
+"
+
+# Potential locations of system log and cluster daemon logs
+SYSLOGS="
+    /var/log/*
+    /var/logs/*
+    /var/syslog/*
+    /var/adm/*
+    /var/log/ha/*
+    /var/log/cluster/*
+    /var/log/pacemaker/*
+"
+
+# Whether pacemaker-remoted was found (0 = yes, 1 = no, -1 = haven't looked yet)
+REMOTED_STATUS=-1
+
+#
+# keep the user posted
+#
+record() {
+    if [ x != x"$REPORT_HOME" -a -d "${REPORT_HOME}/$shorthost" ]; then
+        rec="${REPORT_HOME}/$shorthost/report.out"
+
+    elif [ x != x"${l_base}" -a -d "${l_base}" ]; then
+        rec="${l_base}/report.summary"
+
+    else
+        rec="/dev/null"
+    fi
+    printf "%-10s  $*\n" "$shorthost:" 2>&1 >> "${rec}"
+}
+
+log() {
+    printf "%-10s  $*\n" "$shorthost:" 1>&2
+    record "$*"
+}
+
+debug() {
+    if [ $verbose -gt 0 ]; then
+	log "Debug: $*"
+    else
+        record "Debug: $*"
+    fi
+}
+
+info() {
+    log "$*"
+}
+
+warning() {
+    log "WARN: $*"
+}
+
+fatal() {
+    log "ERROR: $*"
+    exit 1
+}
+
+require_tar() {
+    which tar >/dev/null 2>&1
+    if [ $? -ne 0 ]; then
+        fatal "Required program 'tar' not found, please install and re-run"
+    fi
+}
+
+# check if process of given substring in its name does exist;
+# only look for processes originated by user 0 (by UID), "@CRM_DAEMON_USER@"
+# or effective user running this script, and/or group 0 (by GID),
+# "@CRM_DAEMON_GROUP@" or one of the groups the effective user belongs to
+# (there's no business in probing any other processes)
+is_running() {
+    ps -G "0 $(getent group '@CRM_DAEMON_GROUP@' 2>/dev/null | cut -d: -f3) $(id -G)" \
+       -u "0 @CRM_DAEMON_USER@ $(id -u)" -f \
+      | grep -Eqs  $(echo "$1" | sed -e 's/^\(.\)/[\1]/')
+}
+
+has_remoted() {
+    if [ $REMOTED_STATUS -eq -1 ]; then
+        REMOTED_STATUS=1
+        if which pacemaker-remoted >/dev/null 2>&1; then
+            REMOTED_STATUS=0
+        # Check for pre-2.0.0 daemon name in case we have mixed-version cluster
+        elif which pacemaker_remoted >/dev/null 2>&1; then
+            REMOTED_STATUS=0
+        elif [ -x "@sbindir@/pacemaker-remoted" ]; then
+            REMOTED_STATUS=0
+        elif [ -x "@sbindir@/pacemaker_remoted" ]; then
+            REMOTED_STATUS=0
+        else
+            # @TODO: the binary might be elsewhere,
+            # but a global search is too expensive
+            for d in /{usr,opt}/{local/,}{s,}bin; do
+                if [ -x "${d}/pacemaker-remoted" ]; then
+                    REMOTED_STATUS=0
+                elif [ -x "${d}/pacemaker_remoted" ]; then
+                    REMOTED_STATUS=0
+                fi
+            done
+        fi
+    fi
+    return $REMOTED_STATUS
+}
+
+# found_dir <description> <dirname>
+found_dir() {
+    echo "$2"
+    info "Pacemaker $1 found in: $2"
+}
+
+detect_daemon_dir() {
+    info "Searching for where Pacemaker daemons live... this may take a while"
+
+    for d in \
+        {/usr,/usr/local,/opt/local,@exec_prefix@}/{libexec,lib64,lib}/pacemaker
+    do
+        # pacemaker and pacemaker-cts packages can install to daemon directory,
+        # so check for a file from each
+        if [ -e $d/pacemaker-schedulerd ] || [ -e $d/cts-exec-helper ]; then
+            found_dir "daemons" "$d"
+            return
+        fi
+    done
+
+    # Pacemaker Remote nodes don't need to install daemons
+    if has_remoted; then
+        info "Pacemaker daemons not found (this appears to be a Pacemaker Remote node)"
+        return
+    fi
+
+    for f in $(find / -maxdepth $maxdepth -type f -name pacemaker-schedulerd -o -name cts-exec-helper); do
+        d=$(dirname "$f")
+        found_dir "daemons" "$d"
+        return
+    done
+
+    fatal "Pacemaker daemons not found (nonstandard installation?)"
+}
+
+detect_cib_dir() {
+    d="${local_state_dir}/lib/pacemaker/cib" 
+    if [ -f "$d/cib.xml" ]; then
+        found_dir "config files" "$d"
+        return
+    fi
+
+    # Pacemaker Remote nodes don't need a CIB
+    if has_remoted; then
+        info "Pacemaker config not found (this appears to be a Pacemaker Remote node)"
+        return
+    fi
+
+    info "Searching for where Pacemaker keeps config information... this may take a while"
+    # TODO: What about false positives where someone copied the CIB?
+    for f in $(find / -maxdepth $maxdepth -type f -name cib.xml); do
+        d=$(dirname $f)
+        found_dir "config files" "$d"
+        return
+    done
+
+    warning "Pacemaker config not found (nonstandard installation?)"
+}
+
+detect_state_dir() {
+    if [ -n "$CRM_CONFIG_DIR" ]; then
+        # Assume new layout
+        # $local_state_dir/lib/pacemaker/(cib,pengine,blackbox,cores)
+        dirname "$CRM_CONFIG_DIR"
+
+    # Pacemaker Remote nodes might not have a CRM_CONFIG_DIR
+    elif [ -d "$local_state_dir/lib/pacemaker" ]; then
+        echo $local_state_dir/lib/pacemaker
+    fi
+}
+
+detect_pe_dir() {
+    config_root="$1"
+
+    d="$config_root/pengine"
+    if [ -d "$d" ]; then
+        found_dir "scheduler inputs" "$d"
+        return
+    fi
+
+    if has_remoted; then
+        info "Pacemaker scheduler inputs not found (this appears to be a Pacemaker Remote node)"
+        return
+    fi
+
+    info "Searching for where Pacemaker keeps scheduler inputs... this may take a while"
+    for d in $(find / -maxdepth $maxdepth -type d -name pengine); do
+        found_dir "scheduler inputs" "$d"
+        return
+    done
+
+    fatal "Pacemaker scheduler inputs not found (nonstandard installation?)"
+}
+
+detect_host() {
+    local_state_dir=@localstatedir@
+
+    if [ -d $local_state_dir/run ]; then
+	CRM_STATE_DIR=$local_state_dir/run/crm
+    else
+        info "Searching for where Pacemaker keeps runtime data... this may take a while"
+	for d in `find / -maxdepth $maxdepth -type d -name run`; do
+	    local_state_dir=`dirname $d`
+	    CRM_STATE_DIR=$d/crm
+	    break
+	done
+	info "Found: $CRM_STATE_DIR"
+    fi
+    debug "Machine runtime directory: $local_state_dir"
+    debug "Pacemaker runtime data located in: $CRM_STATE_DIR"
+
+    CRM_DAEMON_DIR=$(detect_daemon_dir)
+    CRM_CONFIG_DIR=$(detect_cib_dir)
+    config_root=$(detect_state_dir)
+
+    # Older versions had none
+    BLACKBOX_DIR=$config_root/blackbox
+    debug "Pacemaker blackboxes (if any) located in: $BLACKBOX_DIR"
+
+    PE_STATE_DIR=$(detect_pe_dir "$config_root")
+
+    CRM_CORE_DIRS=""
+    for d in $config_root/cores $local_state_dir/lib/corosync; do
+	if [ -d $d ]; then
+	    CRM_CORE_DIRS="$CRM_CORE_DIRS $d"
+	fi
+    done
+    debug "Core files located under: $CRM_CORE_DIRS"
+}
+
+time2str() {
+	perl -e "use POSIX; print strftime('%x %X',localtime($1));"
+}
+
+get_time() {
+	perl -e "\$time=\"$*\";" -e '
+	$unix_tm = 0;
+	eval "use Date::Parse";
+	if (index($time, ":") < 0) {
+	} elsif (!$@) {
+		$unix_tm = str2time($time);
+	} else {
+		eval "use Date::Manip";
+		if (!$@) {
+			$unix_tm = UnixDate(ParseDateString($time), "%s");
+		}
+	}
+	if ($unix_tm != "") {
+		print int($unix_tm);
+	} else {
+		print "";
+	}
+	'
+}
+
+get_time_syslog() {
+    awk '{print $1,$2,$3}'
+}
+
+get_time_legacy() {
+    awk '{print $2}' | sed 's/_/ /'
+}
+
+get_time_iso8601() {
+    awk '{print $1}'
+}
+
+get_time_format_for_string() {
+    l="$*"
+    t=$(get_time `echo $l | get_time_syslog`)
+    if [ "x$t" != x ]; then
+	echo syslog
+	return
+    fi
+
+    t=$(get_time `echo $l | get_time_iso8601`)
+    if [ "x$t" != x ]; then
+	echo iso8601
+	return
+    fi
+
+    t=$(get_time `echo $l | get_time_legacy`)
+    if [ "x$t" != x ]; then
+	echo legacy
+	return
+    fi
+}
+
+get_time_format() {
+    t=0 l="" func=""
+    trycnt=10
+    while [ $trycnt -gt 0 ] && read l; do
+	func=$(get_time_format_for_string $l)
+	if [ "x$func" != x ]; then
+	    break
+	fi
+	trycnt=$(($trycnt-1))
+    done
+    #debug "Logfile uses the $func time format"
+    echo $func
+}
+
+get_time_from_line() {
+    GTFL_FORMAT="$1"
+    shift
+    if [ "$GTFL_FORMAT" = "" ]; then
+        GTFL_FORMAT=$(get_time_format_for_string "$@")
+    fi
+    case $GTFL_FORMAT in
+        syslog|legacy|iso8601)
+            get_time $(echo "$@" | get_time_${GTFL_FORMAT})
+            ;;
+        *)
+            warning "Unknown time format in: $@"
+            ;;
+    esac
+}
+
+get_first_time() {
+    l=""
+    format=$1
+    while read l; do
+        ts=$(get_time_from_line "$format" "$l")
+	if [ "x$ts" != x ]; then
+	    echo "$ts"
+	    return
+	fi
+    done
+}
+
+get_last_time() {
+    l=""
+    best=`date +%s` # Now
+    format=$1
+    while read l; do
+        ts=$(get_time_from_line "$format" "$l")
+	if [ "x$ts" != x ]; then
+	    best=$ts
+	fi
+    done
+    echo $best
+}
+
+linetime() {
+    get_time_from_line "" $(tail -n +$2 $1 | grep -a ":[0-5][0-9]:" | head -n 1)
+}
+
+#
+# findmsg <max> <pattern>
+#
+# Print the names of up to <max> system logs that contain <pattern>,
+# ordered by most recently modified.
+#
+findmsg() {
+    max=$1
+    pattern="$2"
+    found=0
+
+    # List all potential system logs ordered by most recently modified.
+    candidates=$(ls -1td $SYSLOGS 2>/dev/null)
+    if [ -z "$candidates" ]; then
+        debug "No system logs found to search for pattern \'$pattern\'"
+        return
+    fi
+
+    # Portable way to handle files with spaces in their names.
+    SAVE_IFS=$IFS
+    IFS="
+"
+
+    # Check each log file for matches.
+    logfiles=""
+    for f in $candidates; do
+        local cat=""
+
+        # We only care about readable files with something in them.
+        if [ ! -f "$f" ] || [ ! -r "$f" ] || [ ! -s "$f" ] ; then
+            continue
+        fi
+
+        cat=$(find_decompressor "$f")
+
+        # We want to avoid grepping through potentially huge binary logs such
+        # as lastlog. However, control characters sometimes find their way into
+        # text logs, so we use a heuristic of more than 256 nonprintable
+        # characters in the file's first kilobyte.
+        if [ $($cat "$f" 2>/dev/null | head -c 1024 | tr -d '[:print:][:space:]' | wc -c) -gt 256 ]
+        then
+            continue
+        fi
+
+        # Our patterns are ASCII, so we can use LC_ALL="C" to speed up grep
+        $cat "$f" 2>/dev/null | LC_ALL="C" grep -q -e "$pattern"
+        if [ $? -eq 0 ]; then
+
+            # Add this file to the list of hits
+            # (using newline as separator to handle spaces in names).
+            if [ -z "$logfiles" ]; then
+                logfiles="$f"
+            else
+                logfiles="$logfiles
+$f"
+            fi
+
+            # If we have enough hits, print them and return.
+            found=$(($found+1))
+            if [ $found -ge $max ]; then
+                break
+            fi
+        fi
+    done 2>/dev/null
+    IFS=$SAVE_IFS
+    if [ -z "$logfiles" ]; then
+        debug "Pattern \'$pattern\' not found in any system logs"
+    else
+        debug "Pattern \'$pattern\' found in: [ $logfiles ]"
+        echo "$logfiles"
+    fi
+}
+
+node_events() {
+  if [ -e $1 ]; then
+    Epatt=`echo "$EVENT_PATTERNS" |
+      while read title p; do [ -n "$p" ] && echo -n "|$p"; done |
+      sed 's/.//'
+      `
+    grep -E "$Epatt" $1
+  fi
+}
+
+pickfirst() {
+    for x; do
+	which $x >/dev/null 2>&1 && {
+	    echo $x
+	    return 0
+	}
+    done
+    return 1
+}
+
+shrink() {
+    olddir=$PWD
+    dir=`dirname $1`
+    base=`basename $1`
+
+    target=$1.tar
+    tar_options="cf"
+
+    variant=`pickfirst bzip2 gzip xz false`
+    case $variant in
+	bz*)
+	    tar_options="jcf"
+	    target="$target.bz2"
+	    ;;
+	gz*)
+	    tar_options="zcf"
+	    target="$target.gz"
+	    ;;
+	xz*)
+	    tar_options="Jcf"
+	    target="$target.xz"
+	    ;;
+	*)
+	    warning "Could not find a compression program, the resulting tarball may be huge"
+	    ;;
+    esac
+
+    if [ -e $target ]; then
+	fatal "Destination $target already exists, specify an alternate name with --dest"
+    fi
+
+    cd $dir  >/dev/null 2>&1
+    tar $tar_options $target $base >/dev/null 2>&1
+    if [ $? -ne 0 ]; then
+        fatal "Could not archive $base, please investigate and collect manually"
+    fi
+    cd $olddir  >/dev/null 2>&1
+
+    echo $target
+}
+
+findln_by_time() {
+    local logf=$1
+    local tm=$2
+    local first=1
+
+    # Some logs can be massive (over 1,500,000,000 lines have been seen in the wild) 
+    # Even just 'wc -l' on these files can take 10+ minutes 
+
+    local fileSize=`ls -lh "$logf" | awk '{ print $5 }' | grep -ie G`
+    if [ x$fileSize != x ]; then
+        warning "$logf is ${fileSize} in size and could take many hours to process. Skipping."
+        return
+    fi
+
+    local last=`wc -l < $logf`
+    while [ $first -le $last ]; do
+	mid=$((($last+$first)/2))
+	trycnt=10
+	while [ $trycnt -gt 0 ]; do
+	    tmid=`linetime $logf $mid`
+	    [ "$tmid" ] && break
+	    warning "cannot extract time: $logf:$mid; will try the next one"
+	    trycnt=$(($trycnt-1))
+			# shift the whole first-last segment
+	    first=$(($first-1))
+	    last=$(($last-1))
+	    mid=$((($last+$first)/2))
+	done
+	if [ -z "$tmid" ]; then
+	    warning "giving up on log..."
+	    return
+	fi
+	if [ $tmid -gt $tm ]; then
+	    last=$(($mid-1))
+	elif [ $tmid -lt $tm ]; then
+	    first=$(($mid+1))
+	else
+	    break
+	fi
+    done
+    echo $mid
+}
+
+dumplog() {
+    local logf=$1
+    local from_line=$2
+    local to_line=$3
+    [ "$from_line" ] ||
+    return
+    tail -n +$from_line $logf |
+    if [ "$to_line" ]; then
+	head -$(($to_line-$from_line+1))
+    else
+	cat
+    fi
+}
+
+#
+# find log/set of logs which are interesting for us
+#
+#
+# find log slices
+#
+
+find_decompressor() {
+    case $1 in
+        *bz2) echo "bzip2 -dc" ;;
+        *gz)  echo "gzip -dc" ;;
+        *xz)  echo "xz -dc" ;;
+        *)    echo "cat" ;;
+    esac
+}
+
+#
+# check if the log contains a piece of our segment
+#
+is_our_log() {
+	local logf=$1
+	local from_time=$2
+	local to_time=$3
+
+	local cat=`find_decompressor $logf`
+	local format=`$cat $logf | get_time_format`
+	local first_time=`$cat $logf | head -10 | get_first_time $format`
+	local last_time=`$cat $logf | tail -10 | get_last_time $format`
+
+	if [ x = "x$first_time" -o x = "x$last_time" ]; then
+	    warning "Skipping bad logfile '$1': Could not determine log dates"
+	    return 0 # skip (empty log?)
+	fi
+	if [ $from_time -gt $last_time ]; then
+		# we shouldn't get here anyway if the logs are in order
+		return 2 # we're past good logs; exit
+	fi
+	if [ $from_time -ge $first_time ]; then
+		return 3 # this is the last good log
+	fi
+	# have to go further back
+	if [ x = "x$to_time" -o $to_time -ge $first_time ]; then
+		return 1 # include this log
+	else
+		return 0 # don't include this log
+	fi
+}
+#
+# go through archived logs (timewise backwards) and see if there
+# are lines belonging to us
+# (we rely on untouched log files, i.e. that modify time
+# hasn't been changed)
+#
+arch_logs() {
+	local logf=$1
+	local from_time=$2
+	local to_time=$3
+
+	# look for files such as: ha-log-20090308 or
+	# ha-log-20090308.gz (.bz2) or ha-log.0, etc
+	ls -t $logf $logf*[0-9z] 2>/dev/null |
+	while read next_log; do
+		is_our_log $next_log $from_time $to_time
+		case $? in
+		0) ;;  # noop, continue
+		1) echo $next_log  # include log and continue
+			debug "Found log $next_log"
+			;;
+		2) break;; # don't go through older logs!
+		3) echo $next_log  # include log and continue
+			debug "Found log $next_log"
+			break
+			;; # don't go through older logs!
+		esac
+	done
+}
+
+#
+# print part of the log
+#
+drop_tmp_file() {
+	[ -z "$tmp" ] || rm -f "$tmp"
+}
+
+print_logseg() {
+	local logf=$1
+	local from_time=$2
+	local to_time=$3
+
+	# uncompress to a temp file (if necessary)
+	local cat=`find_decompressor $logf`
+	if [ "$cat" != "cat" ]; then
+		tmp=`mktemp`
+		$cat $logf > $tmp
+		trap drop_tmp_file 0
+		sourcef=$tmp
+	else
+		sourcef=$logf
+		tmp=""
+	fi
+
+	if [ "$from_time" = 0 ]; then
+		FROM_LINE=1
+	else
+		FROM_LINE=`findln_by_time $sourcef $from_time`
+	fi
+	if [ -z "$FROM_LINE" ]; then
+		warning "couldn't find line for time $from_time; corrupt log file?"
+		return
+	fi
+
+	TO_LINE=""
+	if [ "$to_time" != 0 ]; then
+		TO_LINE=`findln_by_time $sourcef $to_time`
+		if [ -z "$TO_LINE" ]; then
+			warning "couldn't find line for time $to_time; corrupt log file?"
+			return
+		fi
+		if [ $FROM_LINE -lt $TO_LINE ]; then
+		    dumplog $sourcef $FROM_LINE $TO_LINE
+		    log "Including segment [$FROM_LINE-$TO_LINE] from $logf"
+		else
+		    debug "Empty segment [$FROM_LINE-$TO_LINE] from $logf"
+		fi
+	else
+	    dumplog $sourcef $FROM_LINE $TO_LINE
+	    log "Including all logs after line $FROM_LINE from $logf"
+	fi
+	drop_tmp_file
+	trap "" 0
+}
+
+#
+# find log/set of logs which are interesting for us
+#
+dumplogset() {
+	local logf=$1
+	local from_time=$2
+	local to_time=$3
+
+	local logf_set=`arch_logs $logf $from_time $to_time`
+	if [ x = "x$logf_set" ]; then
+		return
+	fi
+
+	local num_logs=`echo "$logf_set" | wc -l`
+	local oldest=`echo $logf_set | awk '{print $NF}'`
+	local newest=`echo $logf_set | awk '{print $1}'`
+	local mid_logfiles=`echo $logf_set | awk '{for(i=NF-1; i>1; i--) print $i}'`
+
+	# the first logfile: from $from_time to $to_time (or end)
+	# logfiles in the middle: all
+	# the last logfile: from beginning to $to_time (or end)
+	case $num_logs in
+	1) print_logseg $newest $from_time $to_time;;
+	*)
+		print_logseg $oldest $from_time 0
+		for f in $mid_logfiles; do
+		    `find_decompressor $f` $f
+		    debug "including complete $f logfile"
+		done
+		print_logseg $newest 0 $to_time
+	;;
+	esac
+}
+
+# cut out a stanza
+getstanza() {
+	awk -v name="$1" '
+	!in_stanza && NF==2 && /^[a-z][a-z]*[[:space:]]*{/ { # stanza start
+		if ($1 == name)
+			in_stanza = 1
+	}
+	in_stanza { print }
+	in_stanza && NF==1 && $1 == "}" { exit }
+	'
+}
+# supply stanza in $1 and variable name in $2
+# (stanza is optional)
+getcfvar() {
+    cf_type=$1; shift;
+    cf_var=$1; shift;
+    cf_file=$*
+
+    [ -f "$cf_file" ] || return
+    case $cf_type in
+	corosync)
+	    sed 's/#.*//' < $cf_file |
+	        if [ $# -eq 2 ]; then
+			getstanza "$cf_var"
+			shift 1
+		else
+			cat
+		fi |
+		awk -v varname="$cf_var" '
+		NF==2 && match($1,varname":$")==1 { print $2; exit; }
+		'
+	;;
+    esac
+}
+
+pickfirst() {
+    for x; do
+	which $x >/dev/null 2>&1 && {
+	    echo $x
+	    return 0
+	}
+    done
+    return 1
+}
+
+#
+# figure out the cluster type, depending on the process list
+# and existence of configuration files
+#
+get_cluster_type() {
+    if is_running corosync; then
+	tool=`pickfirst corosync-objctl corosync-cmapctl`
+	case $tool in
+	    *objctl) quorum=`$tool -a | grep quorum.provider | sed 's/.*=\s*//'`;;
+	    *cmapctl) quorum=`$tool | grep quorum.provider | sed 's/.*=\s*//'`;;
+	esac
+        stack="corosync"
+
+    # Now we're guessing...
+
+    # TODO: Technically these could be anywhere :-/
+    elif [ -f "@PCMK__COROSYNC_CONF@" ]; then
+	stack="corosync"
+
+    else
+        # We still don't know. This might be a Pacemaker Remote node,
+        # or the configuration might be in a nonstandard location.
+        stack="any"
+    fi
+
+    debug "Detected the '$stack' cluster stack"
+    echo $stack
+}
+
+find_cluster_cf() {
+    case $1 in
+	corosync)
+	    best_size=0
+	    best_file=""
+
+	    # TODO: Technically these could be anywhere :-/
+	    for cf in "@PCMK__COROSYNC_CONF@"; do
+		if [ -f $cf ]; then
+		    size=`wc -l $cf | awk '{print $1}'`
+		    if [ $size -gt $best_size ]; then
+			best_size=$size
+			best_file=$cf
+		    fi
+		fi
+	    done
+	    if [ -z "$best_file" ]; then
+		debug "Looking for corosync configuration file. This may take a while..."
+		for f in `find / -maxdepth $maxdepth -type f -name corosync.conf`; do
+		    best_file=$f
+		    break
+		done
+	    fi
+	    debug "Located corosync config file: $best_file"
+	    echo "$best_file"
+	    ;;
+	any)
+	    # Cluster type is undetermined. Don't complain, because this
+	    # might be a Pacemaker Remote node.
+	    ;;
+	*)
+	    warning "Unknown cluster type: $1"
+	    ;;
+    esac
+}
+
+#
+# check for the major prereq for a) parameter parsing and b)
+# parsing logs
+#
+t=`get_time "12:00"`
+if [ "$t" = "" ]; then
+	fatal "please install the perl Date::Parse module (perl-DateTime-Format-DateParse on Fedora/Red Hat)"
+fi
+
+# Override any locale settings so collected output is in a common language
+LC_ALL="C"
+export LC_ALL
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4 textwidth=80:
diff --git a/tools/stonith_admin.8.inc b/tools/stonith_admin.8.inc
new file mode 100644
index 0000000..681ed1f
--- /dev/null
+++ b/tools/stonith_admin.8.inc
@@ -0,0 +1,5 @@
+[=synopsis]
+stonith_admin <command> [<options>]
+
+/Access the Pacemaker fencing API/
+.SH OPTIONS
diff --git a/tools/stonith_admin.c b/tools/stonith_admin.c
new file mode 100644
index 0000000..1077de7
--- /dev/null
+++ b/tools/stonith_admin.c
@@ -0,0 +1,674 @@
+/*
+ * Copyright 2009-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/utsname.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/ipc.h>
+#include <crm/cluster/internal.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/output_internal.h>
+
+#include <crm/stonith-ng.h>
+#include <crm/fencing/internal.h>
+#include <crm/cib.h>
+#include <crm/pengine/status.h>
+
+#include <crm/common/xml.h>
+#include <pacemaker-internal.h>
+
+#define SUMMARY "stonith_admin - Access the Pacemaker fencing API"
+
+char action = 0;
+
+struct {
+    gboolean as_nodeid;
+    gboolean broadcast;
+    gboolean cleanup;
+    gboolean installed;
+    gboolean metadata;
+    gboolean registered;
+    gboolean validate_cfg;
+    stonith_key_value_t *devices;
+    stonith_key_value_t *params;
+    int fence_level;
+    int timeout ;
+    int tolerance;
+    int delay;
+    char *agent;
+    char *confirm_host;
+    char *fence_host;
+    char *history;
+    char *last_fenced;
+    char *query;
+    char *reboot_host;
+    char *register_dev;
+    char *register_level;
+    char *targets;
+    char *terminate;
+    char *unfence_host;
+    char *unregister_dev;
+    char *unregister_level;
+} options = {
+    .timeout = 120,
+    .delay = 0
+};
+
+gboolean add_env_params(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean add_stonith_device(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean add_stonith_params(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean add_tolerance(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+gboolean set_tag(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
+
+#define INDENT "                                    "
+
+/* *INDENT-OFF* */
+static GOptionEntry defn_entries[] = {
+    { "register", 'R', 0, G_OPTION_ARG_STRING, &options.register_dev,
+      "Register the named stonith device. Requires: --agent.\n"
+      INDENT "Optional: --option, --env-option.",
+      "DEVICE" },
+    { "deregister", 'D', 0, G_OPTION_ARG_STRING, &options.unregister_dev,
+      "De-register the named stonith device.",
+      "DEVICE" },
+    { "register-level", 'r', 0, G_OPTION_ARG_STRING, &options.register_level,
+      "Register a stonith level for the named target,\n"
+      INDENT "specified as one of NAME, @PATTERN, or ATTR=VALUE.\n"
+      INDENT "Requires: --index and one or more --device entries.",
+      "TARGET" },
+    { "deregister-level", 'd', 0, G_OPTION_ARG_STRING, &options.unregister_level,
+      "Unregister a stonith level for the named target,\n"
+      INDENT "specified as for --register-level. Requires: --index",
+      "TARGET" },
+
+    { NULL }
+};
+
+static GOptionEntry query_entries[] = {
+    { "list", 'l', 0, G_OPTION_ARG_STRING, &options.terminate,
+      "List devices that can terminate the specified host.\n"
+      INDENT "Optional: --timeout",
+      "HOST" },
+    { "list-registered", 'L', 0, G_OPTION_ARG_NONE, &options.registered,
+      "List all registered devices. Optional: --timeout.",
+      NULL },
+    { "list-installed", 'I', 0, G_OPTION_ARG_NONE, &options.installed,
+      "List all installed devices. Optional: --timeout.",
+      NULL },
+    { "list-targets", 's', 0, G_OPTION_ARG_STRING, &options.targets,
+      "List the targets that can be fenced by the\n"
+      INDENT "named device. Optional: --timeout.",
+      "DEVICE" },
+    { "metadata", 'M', 0, G_OPTION_ARG_NONE, &options.metadata,
+      "Show agent metadata. Requires: --agent.\n"
+      INDENT "Optional: --timeout.",
+      NULL },
+    { "query", 'Q', 0, G_OPTION_ARG_STRING, &options.query,
+      "Check the named device's status. Optional: --timeout.",
+      "DEVICE" },
+    { "history", 'H', 0, G_OPTION_ARG_STRING, &options.history,
+      "Show last successful fencing operation for named node\n"
+      INDENT "(or '*' for all nodes). Optional: --timeout, --cleanup,\n"
+      INDENT "--quiet (show only the operation's epoch timestamp),\n"
+      INDENT "--verbose (show all recorded and pending operations),\n"
+      INDENT "--broadcast (update history from all nodes available).",
+      "NODE" },
+    { "last", 'h', 0, G_OPTION_ARG_STRING, &options.last_fenced,
+      "Indicate when the named node was last fenced.\n"
+      INDENT "Optional: --as-node-id.",
+      "NODE" },
+    { "validate", 'K', 0, G_OPTION_ARG_NONE, &options.validate_cfg,
+      "Validate a fence device configuration.\n"
+      INDENT "Requires: --agent. Optional: --option, --env-option,\n"
+      INDENT "--quiet (print no output, only return status).",
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry fence_entries[] = {
+    { "fence", 'F', 0, G_OPTION_ARG_STRING, &options.fence_host,
+      "Fence named host. Optional: --timeout, --tolerance, --delay.",
+      "HOST" },
+    { "unfence", 'U', 0, G_OPTION_ARG_STRING, &options.unfence_host,
+      "Unfence named host. Optional: --timeout, --tolerance, --delay.",
+      "HOST" },
+    { "reboot", 'B', 0, G_OPTION_ARG_STRING, &options.reboot_host,
+      "Reboot named host. Optional: --timeout, --tolerance, --delay.",
+      "HOST" },
+    { "confirm", 'C', 0, G_OPTION_ARG_STRING, &options.confirm_host,
+      "Tell cluster that named host is now safely down.",
+      "HOST", },
+
+    { NULL }
+};
+
+static GOptionEntry addl_entries[] = {
+    { "cleanup", 'c', 0, G_OPTION_ARG_NONE, &options.cleanup,
+      "Cleanup wherever appropriate. Requires --history.",
+      NULL },
+    { "broadcast", 'b', 0, G_OPTION_ARG_NONE, &options.broadcast,
+      "Broadcast wherever appropriate.",
+      NULL },
+    { "agent", 'a', 0, G_OPTION_ARG_STRING, &options.agent,
+      "The agent to use (for example, fence_xvm;\n"
+      INDENT "with --register, --metadata, --validate).",
+      "AGENT" },
+    { "option", 'o', 0, G_OPTION_ARG_CALLBACK, add_stonith_params,
+      "Specify a device configuration parameter as NAME=VALUE\n"
+      INDENT "(may be specified multiple times; with --register,\n"
+      INDENT "--validate).",
+      "PARAM" },
+    { "env-option", 'e', 0, G_OPTION_ARG_CALLBACK, add_env_params,
+      "Specify a device configuration parameter with the\n"
+      INDENT "specified name, using the value of the\n"
+      INDENT "environment variable of the same name prefixed with\n"
+      INDENT "OCF_RESKEY_ (may be specified multiple times;\n"
+      INDENT "with --register, --validate).",
+      "PARAM" },
+    { "tag", 'T', 0, G_OPTION_ARG_CALLBACK, set_tag,
+      "Identify fencing operations in logs with the specified\n"
+      INDENT "tag; useful when multiple entities might invoke\n"
+      INDENT "stonith_admin (used with most commands).",
+      "TAG" },
+    { "device", 'v', 0, G_OPTION_ARG_CALLBACK, add_stonith_device,
+      "Device ID (with --register-level, device to associate with\n"
+      INDENT "a given host and level; may be specified multiple times)"
+#if SUPPORT_CIBSECRETS
+      "\n" INDENT "(with --validate, name to use to load CIB secrets)"
+#endif
+      ".",
+      "DEVICE" },
+    { "index", 'i', 0, G_OPTION_ARG_INT, &options.fence_level,
+      "The stonith level (1-9) (with --register-level,\n"
+      INDENT "--deregister-level).",
+      "LEVEL" },
+    { "timeout", 't', 0, G_OPTION_ARG_INT, &options.timeout,
+      "Operation timeout in seconds (default 120;\n"
+      INDENT "used with most commands).",
+      "SECONDS" },
+    { "delay", 'y', 0, G_OPTION_ARG_INT, &options.delay,
+      "Apply a fencing delay in seconds. Any static/random delays from\n"
+      INDENT "pcmk_delay_base/max will be added, otherwise all\n"
+      INDENT "disabled with the value -1\n"
+      INDENT "(default 0; with --fence, --reboot, --unfence).",
+      "SECONDS" },
+    { "as-node-id", 'n', 0, G_OPTION_ARG_NONE, &options.as_nodeid,
+      "(Advanced) The supplied node is the corosync node ID\n"
+      INDENT "(with --last).",
+      NULL },
+    { "tolerance", 0, 0, G_OPTION_ARG_CALLBACK, add_tolerance,
+      "(Advanced) Do nothing if an equivalent --fence request\n"
+      INDENT "succeeded less than this many seconds earlier\n"
+      INDENT "(with --fence, --unfence, --reboot).",
+      "SECONDS" },
+
+    { NULL }
+};
+/* *INDENT-ON* */
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_HTML,
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+static const int st_opts = st_opt_sync_call | st_opt_allow_suicide;
+
+static char *name = NULL;
+
+gboolean
+add_env_params(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    char *key = crm_strdup_printf("OCF_RESKEY_%s", optarg);
+    const char *env = getenv(key);
+    gboolean retval = TRUE;
+
+    if (env == NULL) {
+        g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Invalid option: -e %s", optarg);
+        retval = FALSE;
+    } else {
+        crm_info("Got: '%s'='%s'", optarg, env);
+        options.params = stonith_key_value_add(options.params, optarg, env);
+    }
+
+    free(key);
+    return retval;
+}
+
+gboolean
+add_stonith_device(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.devices = stonith_key_value_add(options.devices, NULL, optarg);
+    return TRUE;
+}
+
+gboolean
+add_tolerance(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.tolerance = crm_get_msec(optarg) / 1000;
+    return TRUE;
+}
+
+gboolean
+add_stonith_params(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    char *name = NULL;
+    char *value = NULL;
+    int rc = 0;
+    gboolean retval = TRUE;
+
+    crm_info("Scanning: -o %s", optarg);
+
+    rc = pcmk__scan_nvpair(optarg, &name, &value);
+
+    if (rc != 2) {
+        rc = pcmk_legacy2rc(rc);
+        g_set_error(error, PCMK__RC_ERROR, rc, "Invalid option: -o %s: %s", optarg, pcmk_rc_str(rc));
+        retval = FALSE;
+    } else {
+        crm_info("Got: '%s'='%s'", name, value);
+        options.params = stonith_key_value_add(options.params, name, value);
+    }
+
+    free(name);
+    free(value);
+    return retval;
+}
+
+gboolean
+set_tag(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    free(name);
+    name = crm_strdup_printf("%s.%s", crm_system_name, optarg);
+    return TRUE;
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    GOptionEntry extra_prog_entries[] = {
+        { "quiet", 'q', 0, G_OPTION_ARG_NONE, &(args->quiet),
+          "Be less descriptive in output.",
+          NULL },
+
+        { NULL }
+    };
+
+    context = pcmk__build_arg_context(args, "text (default), html, xml", group, NULL);
+
+    /* Add the -q option, which cannot be part of the globally supported options
+     * because some tools use that flag for something else.
+     */
+    pcmk__add_main_args(context, extra_prog_entries);
+
+    pcmk__add_arg_group(context, "definition", "Device Definition Commands:",
+                        "Show device definition help", defn_entries);
+    pcmk__add_arg_group(context, "queries", "Queries:",
+                        "Show query help", query_entries);
+    pcmk__add_arg_group(context, "fence", "Fencing Commands:",
+                        "Show fence help", fence_entries);
+    pcmk__add_arg_group(context, "additional", "Additional Options:",
+                        "Show additional options", addl_entries);
+    return context;
+}
+
+// \return Standard Pacemaker return code
+static int
+request_fencing(stonith_t *st, const char *target, const char *command,
+                GError **error)
+{
+    char *reason = NULL;
+    int rc = pcmk__request_fencing(st, target, command, name,
+                                   options.timeout * 1000,
+                                   options.tolerance * 1000,
+                                   options.delay, &reason);
+
+    if (rc != pcmk_rc_ok) {
+        const char *rc_str = pcmk_rc_str(rc);
+        const char *what = (strcmp(command, "on") == 0)? "unfence" : "fence";
+
+        // If reason is identical to return code string, don't display it twice
+        if (pcmk__str_eq(rc_str, reason, pcmk__str_none)) {
+            free(reason);
+            reason = NULL;
+        }
+
+        g_set_error(error, PCMK__RC_ERROR, rc,
+                    "Couldn't %s %s: %s%s%s%s",
+                    what, target, rc_str,
+                    ((reason == NULL)? "" : " ("),
+                    ((reason == NULL)? "" : reason),
+                    ((reason == NULL)? "" : ")"));
+    }
+    free(reason);
+    return rc;
+}
+
+int
+main(int argc, char **argv)
+{
+    int rc = 0;
+    crm_exit_t exit_code = CRM_EX_OK;
+    bool no_connect = false;
+    bool required_agent = false;
+
+    char *target = NULL;
+    const char *device = NULL;
+    stonith_t *st = NULL;
+
+    GError *error = NULL;
+
+    pcmk__output_t *out = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "adehilorstvyBCDFHQRTU");
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("stonith_admin", args->verbosity);
+
+    if (name == NULL) {
+        name = strdup(crm_system_name);
+    }
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    stonith__register_messages(out);
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    if (options.validate_cfg) {
+        required_agent = true;
+        no_connect = true;
+        action = 'K';
+    }
+
+    if (options.installed) {
+        no_connect = true;
+        action = 'I';
+    }
+
+    if (options.registered) {
+        action = 'L';
+    }
+
+    if (options.register_dev != NULL) {
+        required_agent = true;
+        action = 'R';
+        device = options.register_dev;
+    }
+
+    if (options.query != NULL) {
+        action = 'Q';
+        device = options.query;
+    }
+
+    if (options.unregister_dev != NULL) {
+        action = 'D';
+        device = options.unregister_dev;
+    }
+
+    if (options.targets != NULL) {
+        action = 's';
+        device = options.targets;
+    }
+
+    if (options.terminate != NULL) {
+        action = 'L';
+        target = options.terminate;
+    }
+
+    if (options.metadata) {
+        no_connect = true;
+        required_agent = true;
+        action = 'M';
+    }
+
+    if (options.reboot_host != NULL) {
+        no_connect = true;
+        action = 'B';
+        target = options.reboot_host;
+        crm_log_args(argc, argv);
+    }
+
+    if (options.fence_host != NULL) {
+        no_connect = true;
+        action = 'F';
+        target = options.fence_host;
+        crm_log_args(argc, argv);
+    }
+
+    if (options.unfence_host != NULL) {
+        no_connect = true;
+        action = 'U';
+        target = options.unfence_host;
+        crm_log_args(argc, argv);
+    }
+
+    if (options.confirm_host != NULL) {
+        action = 'C';
+        target = options.confirm_host;
+        crm_log_args(argc, argv);
+    }
+
+    if (options.last_fenced != NULL) {
+        action = 'h';
+        target = options.last_fenced;
+    }
+
+    if (options.history != NULL) {
+        action = 'H';
+        target = options.history;
+    }
+
+    if (options.register_level != NULL) {
+        action = 'r';
+        target = options.register_level;
+    }
+
+    if (options.unregister_level != NULL) {
+        action = 'd';
+        target = options.unregister_level;
+    }
+
+    if (action == 0) {
+        char *help = g_option_context_get_help(context, TRUE, NULL);
+
+        out->err(out, "%s", help);
+        g_free(help);
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    if (required_agent && options.agent == NULL) {
+        char *help = g_option_context_get_help(context, TRUE, NULL);
+
+        out->err(out, "Please specify an agent to query using -a,--agent [value]");
+        out->err(out, "%s", help);
+        g_free(help);
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    out->quiet = args->quiet;
+
+    st = stonith_api_new();
+    if (st == NULL) {
+        rc = -ENOMEM;
+    } else if (!no_connect) {
+        rc = st->cmds->connect(st, name, NULL);
+    }
+    if (rc < 0) {
+        out->err(out, "Could not connect to fencer: %s", pcmk_strerror(rc));
+        exit_code = CRM_EX_DISCONNECT;
+        goto done;
+    }
+
+    switch (action) {
+        case 'I':
+            rc = pcmk__fence_installed(out, st, options.timeout*1000);
+            if (rc != pcmk_rc_ok) {
+                out->err(out, "Failed to list installed devices: %s", pcmk_strerror(rc));
+            }
+
+            break;
+
+        case 'L':
+            rc = pcmk__fence_registered(out, st, target, options.timeout*1000);
+            if (rc != pcmk_rc_ok) {
+                out->err(out, "Failed to list registered devices: %s", pcmk_strerror(rc));
+            }
+
+            break;
+
+        case 'Q':
+            rc = st->cmds->monitor(st, st_opts, device, options.timeout);
+            if (rc != pcmk_rc_ok) {
+                rc = st->cmds->list(st, st_opts, device, NULL, options.timeout);
+            }
+            rc = pcmk_legacy2rc(rc);
+            break;
+
+        case 's':
+            rc = pcmk__fence_list_targets(out, st, device, options.timeout*1000);
+            if (rc != pcmk_rc_ok) {
+                out->err(out, "Couldn't list targets: %s", pcmk_strerror(rc));
+            }
+
+            break;
+
+        case 'R':
+            rc = st->cmds->register_device(st, st_opts, device, NULL, options.agent,
+                                           options.params);
+            rc = pcmk_legacy2rc(rc);
+            if (rc != pcmk_rc_ok) {
+                out->err(out, "Can't register device %s using agent %s: %s",
+                         device, options.agent, pcmk_rc_str(rc));
+            }
+            break;
+
+        case 'D':
+            rc = st->cmds->remove_device(st, st_opts, device);
+            rc = pcmk_legacy2rc(rc);
+            if (rc != pcmk_rc_ok) {
+                out->err(out, "Can't unregister device %s: %s",
+                         device, pcmk_rc_str(rc));
+            }
+            break;
+
+        case 'd':
+            rc = pcmk__fence_unregister_level(st, target, options.fence_level);
+            if (rc != pcmk_rc_ok) {
+                out->err(out, "Can't unregister topology level %d for %s: %s",
+                         options.fence_level, target, pcmk_rc_str(rc));
+            }
+            break;
+
+        case 'r':
+            rc = pcmk__fence_register_level(st, target, options.fence_level, options.devices);
+            if (rc != pcmk_rc_ok) {
+                out->err(out, "Can't register topology level %d for %s: %s",
+                         options.fence_level, target, pcmk_rc_str(rc));
+            }
+            break;
+
+        case 'M':
+            rc = pcmk__fence_metadata(out, st, options.agent, options.timeout*1000);
+            if (rc != pcmk_rc_ok) {
+                out->err(out, "Can't get fence agent meta-data: %s",
+                         pcmk_rc_str(rc));
+            }
+
+            break;
+
+        case 'C':
+            rc = st->cmds->confirm(st, st_opts, target);
+            rc = pcmk_legacy2rc(rc);
+            break;
+
+        case 'B':
+            rc = request_fencing(st, target, "reboot", &error);
+            break;
+
+        case 'F':
+            rc = request_fencing(st, target, "off", &error);
+            break;
+
+        case 'U':
+            rc = request_fencing(st, target, "on", &error);
+            break;
+
+        case 'h':
+            rc = pcmk__fence_last(out, target, options.as_nodeid);
+            break;
+
+        case 'H':
+            rc = pcmk__fence_history(out, st, target, options.timeout*1000, args->verbosity,
+                                     options.broadcast, options.cleanup);
+            break;
+
+        case 'K':
+            device = options.devices ? options.devices->key : NULL;
+            rc = pcmk__fence_validate(out, st, options.agent, device, options.params,
+                                        options.timeout*1000);
+            break;
+    }
+
+    crm_info("Command returned: %s (%d)", pcmk_rc_str(rc), rc);
+    exit_code = pcmk_rc2exitc(rc);
+
+  done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+    pcmk__unregister_formats();
+    free(name);
+    stonith_key_value_freeall(options.params, 1, 1);
+
+    if (st != NULL) {
+        st->cmds->disconnect(st);
+        stonith_api_delete(st);
+    }
+
+    return exit_code;
+}