Adding upstream version 2.1.6.upstream/2.1.6

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

100 files changed, 47254 insertions, 0 deletions

diff --git a/daemons/Makefile.am b/daemons/Makefile.am
new file mode 100644
index 0000000..743320b
--- /dev/null
+++ b/daemons/Makefile.am
@@ -0,0 +1,11 @@
+#
+# Copyright 2018 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+MAINTAINERCLEANFILES	= Makefile.in
+SUBDIRS			= based schedulerd attrd controld execd fenced pacemakerd
diff --git a/daemons/attrd/Makefile.am b/daemons/attrd/Makefile.am
new file mode 100644
index 0000000..6bb81c4
--- /dev/null
+++ b/daemons/attrd/Makefile.am
@@ -0,0 +1,48 @@
+#
+# Copyright 2004-2022 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+include $(top_srcdir)/mk/common.mk
+
+halibdir	= $(CRM_DAEMON_DIR)
+
+halib_PROGRAMS	= pacemaker-attrd
+
+noinst_HEADERS  = pacemaker-attrd.h
+
+pacemaker_attrd_CFLAGS	= $(CFLAGS_HARDENED_EXE)
+pacemaker_attrd_LDFLAGS	= $(LDFLAGS_HARDENED_EXE)
+
+pacemaker_attrd_LDADD	= $(top_builddir)/lib/cluster/libcrmcluster.la	\
+			  $(top_builddir)/lib/pengine/libpe_rules.la	\
+			  $(top_builddir)/lib/common/libcrmcommon.la	\
+			  $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/lrmd/liblrmd.la		\
+			  $(CLUSTERLIBS)
+
+pacemaker_attrd_SOURCES	= attrd_alerts.c 	\
+						  attrd_attributes.c \
+						  attrd_cib.c 		\
+						  attrd_corosync.c 	\
+						  attrd_elections.c \
+						  attrd_ipc.c 		\
+						  attrd_messages.c 		\
+						  attrd_sync.c 		\
+						  attrd_utils.c 	\
+						  pacemaker-attrd.c
+
+clean-generic:
+	rm -f *.log *.debug *.xml *~
+
+if BUILD_LEGACY_LINKS
+install-exec-hook:
+	cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f attrd && $(LN_S) pacemaker-attrd attrd
+
+uninstall-hook:
+	cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f attrd
+endif
diff --git a/daemons/attrd/attrd_alerts.c b/daemons/attrd/attrd_alerts.c
new file mode 100644
index 0000000..b694891
--- /dev/null
+++ b/daemons/attrd/attrd_alerts.c
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2015-2021 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include <crm/crm.h>
+#include <crm/cib/internal.h>
+#include <crm/msg_xml.h>
+#include <crm/cluster/internal.h>
+#include <crm/cluster/election_internal.h>
+#include <crm/common/alerts_internal.h>
+#include <crm/pengine/rules_internal.h>
+#include <crm/lrmd_internal.h>
+#include "pacemaker-attrd.h"
+
+static GList *attrd_alert_list = NULL;
+
+static void
+attrd_lrmd_callback(lrmd_event_data_t * op)
+{
+    CRM_CHECK(op != NULL, return);
+    switch (op->type) {
+        case lrmd_event_disconnect:
+            crm_info("Lost connection to executor");
+            attrd_lrmd_disconnect();
+            break;
+        default:
+            break;
+    }
+}
+
+static lrmd_t *
+attrd_lrmd_connect(void)
+{
+    if (the_lrmd == NULL) {
+        the_lrmd = lrmd_api_new();
+        the_lrmd->cmds->set_callback(the_lrmd, attrd_lrmd_callback);
+    }
+
+    if (!the_lrmd->cmds->is_connected(the_lrmd)) {
+        const unsigned int max_attempts = 10;
+        int ret = -ENOTCONN;
+
+        for (int fails = 0; fails < max_attempts; ++fails) {
+            ret = the_lrmd->cmds->connect(the_lrmd, T_ATTRD, NULL);
+            if (ret == pcmk_ok) {
+                break;
+            }
+
+            crm_debug("Could not connect to executor, %d tries remaining",
+                      (max_attempts - fails));
+            /* @TODO We don't want to block here with sleep, but we should wait
+             * some time between connection attempts. We could possibly add a
+             * timer with a callback, but then we'd likely need an alert queue.
+             */
+        }
+
+        if (ret != pcmk_ok) {
+            attrd_lrmd_disconnect();
+        }
+    }
+
+    return the_lrmd;
+}
+
+void
+attrd_lrmd_disconnect(void) {
+    if (the_lrmd) {
+        lrmd_t *conn = the_lrmd;
+
+        the_lrmd = NULL; /* in case we're called recursively */
+        lrmd_api_delete(conn); /* will disconnect if necessary */
+    }
+}
+
+static void
+config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    xmlNode *crmalerts = NULL;
+
+    if (rc == -ENXIO) {
+        crm_debug("Local CIB has no alerts section");
+        return;
+    } else if (rc != pcmk_ok) {
+        crm_notice("Could not query local CIB: %s", pcmk_strerror(rc));
+        return;
+    }
+
+    crmalerts = output;
+    if (crmalerts && !pcmk__str_eq(crm_element_name(crmalerts), XML_CIB_TAG_ALERTS, pcmk__str_none)) {
+        crmalerts = first_named_child(crmalerts, XML_CIB_TAG_ALERTS);
+    }
+    if (!crmalerts) {
+        crm_notice("CIB query result has no " XML_CIB_TAG_ALERTS " section");
+        return;
+    }
+
+    pe_free_alert_list(attrd_alert_list);
+    attrd_alert_list = pe_unpack_alerts(crmalerts);
+}
+
+#define XPATH_ALERTS \
+    "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS
+
+gboolean
+attrd_read_options(gpointer user_data)
+{
+    int call_id;
+
+    CRM_CHECK(the_cib != NULL, return TRUE);
+
+    call_id = the_cib->cmds->query(the_cib, XPATH_ALERTS, NULL,
+                                   cib_xpath | cib_scope_local);
+
+    the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE, NULL,
+                                          "config_query_callback",
+                                          config_query_callback, free);
+
+    crm_trace("Querying the CIB... call %d", call_id);
+    return TRUE;
+}
+
+void
+attrd_cib_updated_cb(const char *event, xmlNode * msg)
+{
+    if (!attrd_shutting_down() && pcmk__alert_in_patchset(msg, false)) {
+        mainloop_set_trigger(attrd_config_read);
+    }
+}
+
+int
+attrd_send_attribute_alert(const char *node, int nodeid,
+                           const char *attr, const char *value)
+{
+    if (attrd_alert_list == NULL) {
+        return pcmk_ok;
+    }
+    return lrmd_send_attribute_alert(attrd_lrmd_connect(), attrd_alert_list,
+                                     node, nodeid, attr, value);
+}
diff --git a/daemons/attrd/attrd_attributes.c b/daemons/attrd/attrd_attributes.c
new file mode 100644
index 0000000..516ced7
--- /dev/null
+++ b/daemons/attrd/attrd_attributes.c
@@ -0,0 +1,188 @@
+/*
+ * Copyright 2013-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <errno.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <glib.h>
+
+#include <crm/msg_xml.h>
+#include <crm/common/logging.h>
+#include <crm/common/results.h>
+#include <crm/common/strings_internal.h>
+#include <crm/common/xml.h>
+
+#include "pacemaker-attrd.h"
+
+static attribute_t *
+attrd_create_attribute(xmlNode *xml)
+{
+    int dampen = 0;
+    const char *value = crm_element_value(xml, PCMK__XA_ATTR_DAMPENING);
+    attribute_t *a = calloc(1, sizeof(attribute_t));
+
+    CRM_ASSERT(a != NULL);
+
+    a->id      = crm_element_value_copy(xml, PCMK__XA_ATTR_NAME);
+    a->set_id  = crm_element_value_copy(xml, PCMK__XA_ATTR_SET);
+    a->set_type = crm_element_value_copy(xml, PCMK__XA_ATTR_SET_TYPE);
+    a->uuid    = crm_element_value_copy(xml, PCMK__XA_ATTR_UUID);
+    a->values = pcmk__strikey_table(NULL, attrd_free_attribute_value);
+
+    crm_element_value_int(xml, PCMK__XA_ATTR_IS_PRIVATE, &a->is_private);
+
+    a->user = crm_element_value_copy(xml, PCMK__XA_ATTR_USER);
+    crm_trace("Performing all %s operations as user '%s'", a->id, a->user);
+
+    if (value != NULL) {
+        dampen = crm_get_msec(value);
+    }
+    crm_trace("Created attribute %s with %s write delay", a->id,
+              (a->timeout_ms == 0)? "no" : pcmk__readable_interval(a->timeout_ms));
+
+    if(dampen > 0) {
+        a->timeout_ms = dampen;
+        a->timer = attrd_add_timer(a->id, a->timeout_ms, a);
+    } else if (dampen < 0) {
+        crm_warn("Ignoring invalid delay %s for attribute %s", value, a->id);
+    }
+
+    g_hash_table_replace(attributes, a->id, a);
+    return a;
+}
+
+static int
+attrd_update_dampening(attribute_t *a, xmlNode *xml, const char *attr)
+{
+    const char *dvalue = crm_element_value(xml, PCMK__XA_ATTR_DAMPENING);
+    int dampen = 0;
+
+    if (dvalue == NULL) {
+        crm_warn("Could not update %s: peer did not specify value for delay",
+                 attr);
+        return EINVAL;
+    }
+
+    dampen = crm_get_msec(dvalue);
+    if (dampen < 0) {
+        crm_warn("Could not update %s: invalid delay value %dms (%s)",
+                 attr, dampen, dvalue);
+        return EINVAL;
+    }
+
+    if (a->timeout_ms != dampen) {
+        mainloop_timer_del(a->timer);
+        a->timeout_ms = dampen;
+        if (dampen > 0) {
+            a->timer = attrd_add_timer(attr, a->timeout_ms, a);
+            crm_info("Update attribute %s delay to %dms (%s)",
+                     attr, dampen, dvalue);
+        } else {
+            a->timer = NULL;
+            crm_info("Update attribute %s to remove delay", attr);
+        }
+
+        /* If dampening changed, do an immediate write-out,
+         * otherwise repeated dampening changes would prevent write-outs
+         */
+        attrd_write_or_elect_attribute(a);
+    }
+
+    return pcmk_rc_ok;
+}
+
+GHashTable *attributes = NULL;
+
+/*!
+ * \internal
+ * \brief Create an XML representation of an attribute for use in peer messages
+ *
+ * \param[in,out] parent      Create attribute XML as child element of this
+ * \param[in]     a           Attribute to represent
+ * \param[in]     v           Attribute value to represent
+ * \param[in]     force_write If true, value should be written even if unchanged
+ *
+ * \return XML representation of attribute
+ */
+xmlNode *
+attrd_add_value_xml(xmlNode *parent, const attribute_t *a,
+                    const attribute_value_t *v, bool force_write)
+{
+    xmlNode *xml = create_xml_node(parent, __func__);
+
+    crm_xml_add(xml, PCMK__XA_ATTR_NAME, a->id);
+    crm_xml_add(xml, PCMK__XA_ATTR_SET, a->set_id);
+    crm_xml_add(xml, PCMK__XA_ATTR_UUID, a->uuid);
+    crm_xml_add(xml, PCMK__XA_ATTR_USER, a->user);
+    pcmk__xe_add_node(xml, v->nodename, v->nodeid);
+    if (v->is_remote != 0) {
+        crm_xml_add_int(xml, PCMK__XA_ATTR_IS_REMOTE, 1);
+    }
+    crm_xml_add(xml, PCMK__XA_ATTR_VALUE, v->current);
+    crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, a->timeout_ms / 1000);
+    crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, a->is_private);
+    crm_xml_add_int(xml, PCMK__XA_ATTR_FORCE, force_write);
+
+    return xml;
+}
+
+void
+attrd_clear_value_seen(void)
+{
+    GHashTableIter aIter;
+    GHashTableIter vIter;
+    attribute_t *a;
+    attribute_value_t *v = NULL;
+
+    g_hash_table_iter_init(&aIter, attributes);
+    while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
+        g_hash_table_iter_init(&vIter, a->values);
+        while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
+            v->seen = FALSE;
+            crm_trace("Clear seen flag %s[%s] = %s.", a->id, v->nodename, v->current);
+        }
+    }
+}
+
+attribute_t *
+attrd_populate_attribute(xmlNode *xml, const char *attr)
+{
+    attribute_t *a = NULL;
+    bool update_both = false;
+
+    const char *op = crm_element_value(xml, PCMK__XA_TASK);
+
+    // NULL because PCMK__ATTRD_CMD_SYNC_RESPONSE has no PCMK__XA_TASK
+    update_both = pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE_BOTH,
+                               pcmk__str_null_matches);
+
+    // Look up or create attribute entry
+    a = g_hash_table_lookup(attributes, attr);
+    if (a == NULL) {
+        if (update_both || pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE, pcmk__str_none)) {
+            a = attrd_create_attribute(xml);
+        } else {
+            crm_warn("Could not update %s: attribute not found", attr);
+            return NULL;
+        }
+    }
+
+    // Update attribute dampening
+    if (update_both || pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE_DELAY, pcmk__str_none)) {
+        int rc = attrd_update_dampening(a, xml, attr);
+
+        if (rc != pcmk_rc_ok || !update_both) {
+            return NULL;
+        }
+    }
+
+    return a;
+}
diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c
new file mode 100644
index 0000000..928c013
--- /dev/null
+++ b/daemons/attrd/attrd_cib.c
@@ -0,0 +1,380 @@
+/*
+ * Copyright 2013-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <errno.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <glib.h>
+
+#include <crm/msg_xml.h>
+#include <crm/common/logging.h>
+#include <crm/common/results.h>
+#include <crm/common/strings_internal.h>
+#include <crm/common/xml.h>
+
+#include "pacemaker-attrd.h"
+
+static int last_cib_op_done = 0;
+
+static gboolean
+attribute_timer_cb(gpointer data)
+{
+    attribute_t *a = data;
+    crm_trace("Dampen interval expired for %s", a->id);
+    attrd_write_or_elect_attribute(a);
+    return FALSE;
+}
+
+static void
+attrd_cib_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data)
+{
+    int level = LOG_ERR;
+    GHashTableIter iter;
+    const char *peer = NULL;
+    attribute_value_t *v = NULL;
+
+    char *name = user_data;
+    attribute_t *a = g_hash_table_lookup(attributes, name);
+
+    if(a == NULL) {
+        crm_info("Attribute %s no longer exists", name);
+        return;
+    }
+
+    a->update = 0;
+    if (rc == pcmk_ok && call_id < 0) {
+        rc = call_id;
+    }
+
+    switch (rc) {
+        case pcmk_ok:
+            level = LOG_INFO;
+            last_cib_op_done = call_id;
+            if (a->timer && !a->timeout_ms) {
+                // Remove temporary dampening for failed writes
+                mainloop_timer_del(a->timer);
+                a->timer = NULL;
+            }
+            break;
+
+        case -pcmk_err_diff_failed:    /* When an attr changes while the CIB is syncing */
+        case -ETIME:           /* When an attr changes while there is a DC election */
+        case -ENXIO:           /* When an attr changes while the CIB is syncing a
+                                *   newer config from a node that just came up
+                                */
+            level = LOG_WARNING;
+            break;
+    }
+
+    do_crm_log(level, "CIB update %d result for %s: %s " CRM_XS " rc=%d",
+               call_id, a->id, pcmk_strerror(rc), rc);
+
+    g_hash_table_iter_init(&iter, a->values);
+    while (g_hash_table_iter_next(&iter, (gpointer *) & peer, (gpointer *) & v)) {
+        do_crm_log(level, "* %s[%s]=%s", a->id, peer, v->requested);
+        free(v->requested);
+        v->requested = NULL;
+        if (rc != pcmk_ok) {
+            a->changed = true; /* Attempt write out again */
+        }
+    }
+
+    if (a->changed && attrd_election_won()) {
+        if (rc == pcmk_ok) {
+            /* We deferred a write of a new update because this update was in
+             * progress. Write out the new value without additional delay.
+             */
+            attrd_write_attribute(a, false);
+
+        /* We're re-attempting a write because the original failed; delay
+         * the next attempt so we don't potentially flood the CIB manager
+         * and logs with a zillion attempts per second.
+         *
+         * @TODO We could elect a new writer instead. However, we'd have to
+         * somehow downgrade our vote, and we'd still need something like this
+         * if all peers similarly fail to write this attribute (which may
+         * indicate a corrupted attribute entry rather than a CIB issue).
+         */
+        } else if (a->timer) {
+            // Attribute has a dampening value, so use that as delay
+            if (!mainloop_timer_running(a->timer)) {
+                crm_trace("Delayed re-attempted write for %s by %s",
+                          name, pcmk__readable_interval(a->timeout_ms));
+                mainloop_timer_start(a->timer);
+            }
+        } else {
+            /* Set a temporary dampening of 2 seconds (timer will continue
+             * to exist until the attribute's dampening gets set or the
+             * write succeeds).
+             */
+            a->timer = attrd_add_timer(a->id, 2000, a);
+            mainloop_timer_start(a->timer);
+        }
+    }
+}
+
+static void
+build_update_element(xmlNode *parent, attribute_t *a, const char *nodeid, const char *value)
+{
+    const char *set = NULL;
+    xmlNode *xml_obj = NULL;
+
+    xml_obj = create_xml_node(parent, XML_CIB_TAG_STATE);
+    crm_xml_add(xml_obj, XML_ATTR_ID, nodeid);
+
+    xml_obj = create_xml_node(xml_obj, XML_TAG_TRANSIENT_NODEATTRS);
+    crm_xml_add(xml_obj, XML_ATTR_ID, nodeid);
+
+    if (pcmk__str_eq(a->set_type, XML_TAG_ATTR_SETS, pcmk__str_null_matches)) {
+        xml_obj = create_xml_node(xml_obj, XML_TAG_ATTR_SETS);
+    } else if (pcmk__str_eq(a->set_type, XML_TAG_UTILIZATION, pcmk__str_none)) {
+        xml_obj = create_xml_node(xml_obj, XML_TAG_UTILIZATION);
+    } else {
+        crm_err("Unknown set type attribute: %s", a->set_type);
+    }
+
+    if (a->set_id) {
+        crm_xml_set_id(xml_obj, "%s", a->set_id);
+    } else {
+        crm_xml_set_id(xml_obj, "%s-%s", XML_CIB_TAG_STATUS, nodeid);
+    }
+    set = ID(xml_obj);
+
+    xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_NVPAIR);
+    if (a->uuid) {
+        crm_xml_set_id(xml_obj, "%s", a->uuid);
+    } else {
+        crm_xml_set_id(xml_obj, "%s-%s", set, a->id);
+    }
+    crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, a->id);
+
+    if(value) {
+        crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, value);
+
+    } else {
+        crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, "");
+        crm_xml_add(xml_obj, "__delete__", XML_NVPAIR_ATTR_VALUE);
+    }
+}
+
+static void
+send_alert_attributes_value(attribute_t *a, GHashTable *t)
+{
+    int rc = 0;
+    attribute_value_t *at = NULL;
+    GHashTableIter vIter;
+
+    g_hash_table_iter_init(&vIter, t);
+
+    while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & at)) {
+        rc = attrd_send_attribute_alert(at->nodename, at->nodeid,
+                                        a->id, at->current);
+        crm_trace("Sent alerts for %s[%s]=%s: nodeid=%d rc=%d",
+                  a->id, at->nodename, at->current, at->nodeid, rc);
+    }
+}
+
+static void
+set_alert_attribute_value(GHashTable *t, attribute_value_t *v)
+{
+    attribute_value_t *a_v = NULL;
+    a_v = calloc(1, sizeof(attribute_value_t));
+    CRM_ASSERT(a_v != NULL);
+
+    a_v->nodeid = v->nodeid;
+    a_v->nodename = strdup(v->nodename);
+    pcmk__str_update(&a_v->current, v->current);
+
+    g_hash_table_replace(t, a_v->nodename, a_v);
+}
+
+mainloop_timer_t *
+attrd_add_timer(const char *id, int timeout_ms, attribute_t *attr)
+{
+   return mainloop_timer_add(id, timeout_ms, FALSE, attribute_timer_cb, attr);
+}
+
+void
+attrd_write_attribute(attribute_t *a, bool ignore_delay)
+{
+    int private_updates = 0, cib_updates = 0;
+    xmlNode *xml_top = NULL;
+    attribute_value_t *v = NULL;
+    GHashTableIter iter;
+    enum cib_call_options flags = cib_none;
+    GHashTable *alert_attribute_value = NULL;
+
+    if (a == NULL) {
+        return;
+    }
+
+    /* If this attribute will be written to the CIB ... */
+    if (!stand_alone && !a->is_private) {
+
+        /* Defer the write if now's not a good time */
+        CRM_CHECK(the_cib != NULL, return);
+        if (a->update && (a->update < last_cib_op_done)) {
+            crm_info("Write out of '%s' continuing: update %d considered lost", a->id, a->update);
+            a->update = 0; // Don't log this message again
+
+        } else if (a->update) {
+            crm_info("Write out of '%s' delayed: update %d in progress", a->id, a->update);
+            return;
+
+        } else if (mainloop_timer_running(a->timer)) {
+            if (ignore_delay) {
+                /* 'refresh' forces a write of the current value of all attributes
+                 * Cancel any existing timers, we're writing it NOW
+                 */
+                mainloop_timer_stop(a->timer);
+                crm_debug("Write out of '%s': timer is running but ignore delay", a->id);
+            } else {
+                crm_info("Write out of '%s' delayed: timer is running", a->id);
+                return;
+            }
+        }
+
+        /* Initialize the status update XML */
+        xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
+    }
+
+    /* Attribute will be written shortly, so clear changed flag */
+    a->changed = false;
+
+    /* We will check all peers' uuids shortly, so initialize this to false */
+    a->unknown_peer_uuids = false;
+
+    /* Attribute will be written shortly, so clear forced write flag */
+    a->force_write = FALSE;
+
+    /* Make the table for the attribute trap */
+    alert_attribute_value = pcmk__strikey_table(NULL, attrd_free_attribute_value);
+
+    /* Iterate over each peer value of this attribute */
+    g_hash_table_iter_init(&iter, a->values);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & v)) {
+        crm_node_t *peer = crm_get_peer_full(v->nodeid, v->nodename, CRM_GET_PEER_ANY);
+
+        /* If the value's peer info does not correspond to a peer, ignore it */
+        if (peer == NULL) {
+            crm_notice("Cannot update %s[%s]=%s because peer not known",
+                       a->id, v->nodename, v->current);
+            continue;
+        }
+
+        /* If we're just learning the peer's node id, remember it */
+        if (peer->id && (v->nodeid == 0)) {
+            crm_trace("Learned ID %u for node %s", peer->id, v->nodename);
+            v->nodeid = peer->id;
+        }
+
+        /* If this is a private attribute, no update needs to be sent */
+        if (stand_alone || a->is_private) {
+            private_updates++;
+            continue;
+        }
+
+        /* If the peer is found, but its uuid is unknown, defer write */
+        if (peer->uuid == NULL) {
+            a->unknown_peer_uuids = true;
+            crm_notice("Cannot update %s[%s]=%s because peer UUID not known "
+                       "(will retry if learned)",
+                       a->id, v->nodename, v->current);
+            continue;
+        }
+
+        /* Add this value to status update XML */
+        crm_debug("Updating %s[%s]=%s (peer known as %s, UUID %s, ID %u/%u)",
+                  a->id, v->nodename, v->current,
+                  peer->uname, peer->uuid, peer->id, v->nodeid);
+        build_update_element(xml_top, a, peer->uuid, v->current);
+        cib_updates++;
+
+        /* Preservation of the attribute to transmit alert */
+        set_alert_attribute_value(alert_attribute_value, v);
+
+        free(v->requested);
+        v->requested = NULL;
+        if (v->current) {
+            v->requested = strdup(v->current);
+        } else {
+            /* Older attrd versions don't know about the cib_mixed_update
+             * flag so make sure it goes to the local cib which does
+             */
+            cib__set_call_options(flags, crm_system_name,
+                                  cib_mixed_update|cib_scope_local);
+        }
+    }
+
+    if (private_updates) {
+        crm_info("Processed %d private change%s for %s, id=%s, set=%s",
+                 private_updates, pcmk__plural_s(private_updates),
+                 a->id, pcmk__s(a->uuid, "n/a"), pcmk__s(a->set_id, "n/a"));
+    }
+    if (cib_updates) {
+        crm_log_xml_trace(xml_top, __func__);
+
+        a->update = cib_internal_op(the_cib, PCMK__CIB_REQUEST_MODIFY, NULL,
+                                    XML_CIB_TAG_STATUS, xml_top, NULL, flags,
+                                    a->user);
+
+        crm_info("Sent CIB request %d with %d change%s for %s (id %s, set %s)",
+                 a->update, cib_updates, pcmk__plural_s(cib_updates),
+                 a->id, pcmk__s(a->uuid, "n/a"), pcmk__s(a->set_id, "n/a"));
+
+        the_cib->cmds->register_callback_full(the_cib, a->update,
+                                              CIB_OP_TIMEOUT_S, FALSE,
+                                              strdup(a->id),
+                                              "attrd_cib_callback",
+                                              attrd_cib_callback, free);
+        /* Transmit alert of the attribute */
+        send_alert_attributes_value(a, alert_attribute_value);
+    }
+
+    g_hash_table_destroy(alert_attribute_value);
+    free_xml(xml_top);
+}
+
+void
+attrd_write_attributes(bool all, bool ignore_delay)
+{
+    GHashTableIter iter;
+    attribute_t *a = NULL;
+
+    crm_debug("Writing out %s attributes", all? "all" : "changed");
+    g_hash_table_iter_init(&iter, attributes);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & a)) {
+        if (!all && a->unknown_peer_uuids) {
+            // Try writing this attribute again, in case peer ID was learned
+            a->changed = true;
+        } else if (a->force_write) {
+            /* If the force_write flag is set, write the attribute. */
+            a->changed = true;
+        }
+
+        if(all || a->changed) {
+            /* When forced write flag is set, ignore delay. */
+            attrd_write_attribute(a, (a->force_write ? true : ignore_delay));
+        } else {
+            crm_trace("Skipping unchanged attribute %s", a->id);
+        }
+    }
+}
+
+void
+attrd_write_or_elect_attribute(attribute_t *a)
+{
+    if (attrd_election_won()) {
+        attrd_write_attribute(a, false);
+    } else {
+        attrd_start_election_if_needed();
+    }
+}
diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c
new file mode 100644
index 0000000..ef205e6
--- /dev/null
+++ b/daemons/attrd/attrd_corosync.c
@@ -0,0 +1,620 @@
+/*
+ * Copyright 2013-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <errno.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <crm/cluster.h>
+#include <crm/cluster/internal.h>
+#include <crm/common/logging.h>
+#include <crm/common/results.h>
+#include <crm/common/strings_internal.h>
+#include <crm/msg_xml.h>
+
+#include "pacemaker-attrd.h"
+
+extern crm_exit_t attrd_exit_status;
+
+static xmlNode *
+attrd_confirmation(int callid)
+{
+    xmlNode *node = create_xml_node(NULL, __func__);
+
+    crm_xml_add(node, F_TYPE, T_ATTRD);
+    crm_xml_add(node, F_ORIG, get_local_node_name());
+    crm_xml_add(node, PCMK__XA_TASK, PCMK__ATTRD_CMD_CONFIRM);
+    crm_xml_add_int(node, XML_LRM_ATTR_CALLID, callid);
+
+    return node;
+}
+
+static void
+attrd_peer_message(crm_node_t *peer, xmlNode *xml)
+{
+    const char *election_op = crm_element_value(xml, F_CRM_TASK);
+
+    if (election_op) {
+        attrd_handle_election_op(peer, xml);
+        return;
+    }
+
+    if (attrd_shutting_down()) {
+        /* If we're shutting down, we want to continue responding to election
+         * ops as long as we're a cluster member (because our vote may be
+         * needed). Ignore all other messages.
+         */
+        return;
+
+    } else {
+        pcmk__request_t request = {
+            .ipc_client     = NULL,
+            .ipc_id         = 0,
+            .ipc_flags      = 0,
+            .peer           = peer->uname,
+            .xml            = xml,
+            .call_options   = 0,
+            .result         = PCMK__UNKNOWN_RESULT,
+        };
+
+        request.op = crm_element_value_copy(request.xml, PCMK__XA_TASK);
+        CRM_CHECK(request.op != NULL, return);
+
+        attrd_handle_request(&request);
+
+        /* Having finished handling the request, check to see if the originating
+         * peer requested confirmation.  If so, send that confirmation back now.
+         */
+        if (pcmk__xe_attr_is_true(xml, PCMK__XA_CONFIRM) &&
+            !pcmk__str_eq(request.op, PCMK__ATTRD_CMD_CONFIRM, pcmk__str_none)) {
+            int callid = 0;
+            xmlNode *reply = NULL;
+
+            /* Add the confirmation ID for the message we are confirming to the
+             * response so the originating peer knows what they're a confirmation
+             * for.
+             */
+            crm_element_value_int(xml, XML_LRM_ATTR_CALLID, &callid);
+            reply = attrd_confirmation(callid);
+
+            /* And then send the confirmation back to the originating peer.  This
+             * ends up right back in this same function (attrd_peer_message) on the
+             * peer where it will have to do something with a PCMK__XA_CONFIRM type
+             * message.
+             */
+            crm_debug("Sending %s a confirmation", peer->uname);
+            attrd_send_message(peer, reply, false);
+            free_xml(reply);
+        }
+
+        pcmk__reset_request(&request);
+    }
+}
+
+static void
+attrd_cpg_dispatch(cpg_handle_t handle,
+                 const struct cpg_name *groupName,
+                 uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
+{
+    uint32_t kind = 0;
+    xmlNode *xml = NULL;
+    const char *from = NULL;
+    char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);
+
+    if(data == NULL) {
+        return;
+    }
+
+    if (kind == crm_class_cluster) {
+        xml = string2xml(data);
+    }
+
+    if (xml == NULL) {
+        crm_err("Bad message of class %d received from %s[%u]: '%.120s'", kind, from, nodeid, data);
+    } else {
+        crm_node_t *peer = crm_get_peer(nodeid, from);
+
+        attrd_peer_message(peer, xml);
+    }
+
+    free_xml(xml);
+    free(data);
+}
+
+static void
+attrd_cpg_destroy(gpointer unused)
+{
+    if (attrd_shutting_down()) {
+        crm_info("Corosync disconnection complete");
+
+    } else {
+        crm_crit("Lost connection to cluster layer, shutting down");
+        attrd_exit_status = CRM_EX_DISCONNECT;
+        attrd_shutdown(0);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Override an attribute sync with a local value
+ *
+ * Broadcast the local node's value for an attribute that's different from the
+ * value provided in a peer's attribute synchronization response. This ensures a
+ * node's values for itself take precedence and all peers are kept in sync.
+ *
+ * \param[in] a          Attribute entry to override
+ *
+ * \return Local instance of attribute value
+ */
+static attribute_value_t *
+broadcast_local_value(const attribute_t *a)
+{
+    attribute_value_t *v = g_hash_table_lookup(a->values, attrd_cluster->uname);
+    xmlNode *sync = create_xml_node(NULL, __func__);
+
+    crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
+    attrd_add_value_xml(sync, a, v, false);
+    attrd_send_message(NULL, sync, false);
+    free_xml(sync);
+    return v;
+}
+
+/*!
+ * \internal
+ * \brief Ensure a Pacemaker Remote node is in the correct peer cache
+ *
+ * \param[in] node_name  Name of Pacemaker Remote node to check
+ */
+static void
+cache_remote_node(const char *node_name)
+{
+    /* If we previously assumed this node was an unseen cluster node,
+     * remove its entry from the cluster peer cache.
+     */
+    crm_node_t *dup = pcmk__search_cluster_node_cache(0, node_name);
+
+    if (dup && (dup->uuid == NULL)) {
+        reap_crm_member(0, node_name);
+    }
+
+    // Ensure node is in the remote peer cache
+    CRM_ASSERT(crm_remote_peer_get(node_name) != NULL);
+}
+
+#define state_text(state) pcmk__s((state), "in unknown state")
+
+/*!
+ * \internal
+ * \brief Return host's hash table entry (creating one if needed)
+ *
+ * \param[in,out] values Hash table of values
+ * \param[in]     host   Name of peer to look up
+ * \param[in]     xml    XML describing the attribute
+ *
+ * \return Pointer to new or existing hash table entry
+ */
+static attribute_value_t *
+attrd_lookup_or_create_value(GHashTable *values, const char *host,
+                             const xmlNode *xml)
+{
+    attribute_value_t *v = g_hash_table_lookup(values, host);
+    int is_remote = 0;
+
+    crm_element_value_int(xml, PCMK__XA_ATTR_IS_REMOTE, &is_remote);
+    if (is_remote) {
+        cache_remote_node(host);
+    }
+
+    if (v == NULL) {
+        v = calloc(1, sizeof(attribute_value_t));
+        CRM_ASSERT(v != NULL);
+
+        pcmk__str_update(&v->nodename, host);
+        v->is_remote = is_remote;
+        g_hash_table_replace(values, v->nodename, v);
+    }
+    return(v);
+}
+
+static void
+attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *data)
+{
+    bool gone = false;
+    bool is_remote = pcmk_is_set(peer->flags, crm_remote_node);
+
+    switch (kind) {
+        case crm_status_uname:
+            crm_debug("%s node %s is now %s",
+                      (is_remote? "Remote" : "Cluster"),
+                      peer->uname, state_text(peer->state));
+            break;
+
+        case crm_status_processes:
+            if (!pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
+                gone = true;
+            }
+            crm_debug("Node %s is %s a peer",
+                      peer->uname, (gone? "no longer" : "now"));
+            break;
+
+        case crm_status_nstate:
+            crm_debug("%s node %s is now %s (was %s)",
+                      (is_remote? "Remote" : "Cluster"),
+                      peer->uname, state_text(peer->state), state_text(data));
+            if (pcmk__str_eq(peer->state, CRM_NODE_MEMBER, pcmk__str_casei)) {
+                /* If we're the writer, send new peers a list of all attributes
+                 * (unless it's a remote node, which doesn't run its own attrd)
+                 */
+                if (attrd_election_won()
+                    && !pcmk_is_set(peer->flags, crm_remote_node)) {
+                    attrd_peer_sync(peer, NULL);
+                }
+            } else {
+                // Remove all attribute values associated with lost nodes
+                attrd_peer_remove(peer->uname, false, "loss");
+                gone = true;
+            }
+            break;
+    }
+
+    // Remove votes from cluster nodes that leave, in case election in progress
+    if (gone && !is_remote) {
+        attrd_remove_voter(peer);
+        attrd_remove_peer_protocol_ver(peer->uname);
+        attrd_do_not_expect_from_peer(peer->uname);
+
+    // Ensure remote nodes that come up are in the remote node cache
+    } else if (!gone && is_remote) {
+        cache_remote_node(peer->uname);
+    }
+}
+
+static void
+record_peer_nodeid(attribute_value_t *v, const char *host)
+{
+    crm_node_t *known_peer = crm_get_peer(v->nodeid, host);
+
+    crm_trace("Learned %s has node id %s", known_peer->uname, known_peer->uuid);
+    if (attrd_election_won()) {
+        attrd_write_attributes(false, false);
+    }
+}
+
+static void
+update_attr_on_host(attribute_t *a, const crm_node_t *peer, const xmlNode *xml,
+                    const char *attr, const char *value, const char *host,
+                    bool filter, int is_force_write)
+{
+    attribute_value_t *v = NULL;
+
+    v = attrd_lookup_or_create_value(a->values, host, xml);
+
+    if (filter && !pcmk__str_eq(v->current, value, pcmk__str_casei)
+        && pcmk__str_eq(host, attrd_cluster->uname, pcmk__str_casei)) {
+
+        crm_notice("%s[%s]: local value '%s' takes priority over '%s' from %s",
+                   attr, host, v->current, value, peer->uname);
+        v = broadcast_local_value(a);
+
+    } else if (!pcmk__str_eq(v->current, value, pcmk__str_casei)) {
+        crm_notice("Setting %s[%s]%s%s: %s -> %s "
+                   CRM_XS " from %s with %s write delay",
+                   attr, host, a->set_type ? " in " : "",
+                   pcmk__s(a->set_type, ""), pcmk__s(v->current, "(unset)"),
+                   pcmk__s(value, "(unset)"), peer->uname,
+                   (a->timeout_ms == 0)? "no" : pcmk__readable_interval(a->timeout_ms));
+        pcmk__str_update(&v->current, value);
+        a->changed = true;
+
+        if (pcmk__str_eq(host, attrd_cluster->uname, pcmk__str_casei)
+            && pcmk__str_eq(attr, XML_CIB_ATTR_SHUTDOWN, pcmk__str_none)) {
+
+            if (!pcmk__str_eq(value, "0", pcmk__str_null_matches)) {
+                attrd_set_requesting_shutdown();
+
+            } else {
+                attrd_clear_requesting_shutdown();
+            }
+        }
+
+        // Write out new value or start dampening timer
+        if (a->timeout_ms && a->timer) {
+            crm_trace("Delayed write out (%dms) for %s", a->timeout_ms, attr);
+            mainloop_timer_start(a->timer);
+        } else {
+            attrd_write_or_elect_attribute(a);
+        }
+
+    } else {
+        if (is_force_write == 1 && a->timeout_ms && a->timer) {
+            /* Save forced writing and set change flag. */
+            /* The actual attribute is written by Writer after election. */
+            crm_trace("Unchanged %s[%s] from %s is %s(Set the forced write flag)",
+                      attr, host, peer->uname, value);
+            a->force_write = TRUE;
+        } else {
+            crm_trace("Unchanged %s[%s] from %s is %s", attr, host, peer->uname, value);
+        }
+    }
+
+    /* Set the seen flag for attribute processing held only in the own node. */
+    v->seen = TRUE;
+
+    /* If this is a cluster node whose node ID we are learning, remember it */
+    if ((v->nodeid == 0) && (v->is_remote == FALSE)
+        && (crm_element_value_int(xml, PCMK__XA_ATTR_NODE_ID,
+                                  (int*)&v->nodeid) == 0) && (v->nodeid > 0)) {
+        record_peer_nodeid(v, host);
+    }
+}
+
+static void
+attrd_peer_update_one(const crm_node_t *peer, xmlNode *xml, bool filter)
+{
+    attribute_t *a = NULL;
+    const char *attr = crm_element_value(xml, PCMK__XA_ATTR_NAME);
+    const char *value = crm_element_value(xml, PCMK__XA_ATTR_VALUE);
+    const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
+    int is_force_write = 0;
+
+    if (attr == NULL) {
+        crm_warn("Could not update attribute: peer did not specify name");
+        return;
+    }
+
+    crm_element_value_int(xml, PCMK__XA_ATTR_FORCE, &is_force_write);
+
+    a = attrd_populate_attribute(xml, attr);
+    if (a == NULL) {
+        return;
+    }
+
+    if (host == NULL) {
+        // If no host was specified, update all hosts
+        GHashTableIter vIter;
+
+        crm_debug("Setting %s for all hosts to %s", attr, value);
+        xml_remove_prop(xml, PCMK__XA_ATTR_NODE_ID);
+        g_hash_table_iter_init(&vIter, a->values);
+
+        while (g_hash_table_iter_next(&vIter, (gpointer *) & host, NULL)) {
+            update_attr_on_host(a, peer, xml, attr, value, host, filter, is_force_write);
+        }
+
+    } else {
+        // Update attribute value for the given host
+        update_attr_on_host(a, peer, xml, attr, value, host, filter, is_force_write);
+    }
+
+    /* If this is a message from some attrd instance broadcasting its protocol
+     * version, check to see if it's a new minimum version.
+     */
+    if (pcmk__str_eq(attr, CRM_ATTR_PROTOCOL, pcmk__str_none)) {
+        attrd_update_minimum_protocol_ver(peer->uname, value);
+    }
+}
+
+static void
+broadcast_unseen_local_values(void)
+{
+    GHashTableIter aIter;
+    GHashTableIter vIter;
+    attribute_t *a = NULL;
+    attribute_value_t *v = NULL;
+    xmlNode *sync = NULL;
+
+    g_hash_table_iter_init(&aIter, attributes);
+    while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
+        g_hash_table_iter_init(&vIter, a->values);
+        while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
+            if (!(v->seen) && pcmk__str_eq(v->nodename, attrd_cluster->uname,
+                                           pcmk__str_casei)) {
+                if (sync == NULL) {
+                    sync = create_xml_node(NULL, __func__);
+                    crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
+                }
+                attrd_add_value_xml(sync, a, v, a->timeout_ms && a->timer);
+            }
+        }
+    }
+
+    if (sync != NULL) {
+        crm_debug("Broadcasting local-only values");
+        attrd_send_message(NULL, sync, false);
+        free_xml(sync);
+    }
+}
+
+int
+attrd_cluster_connect(void)
+{
+    attrd_cluster = pcmk_cluster_new();
+
+    attrd_cluster->destroy = attrd_cpg_destroy;
+    attrd_cluster->cpg.cpg_deliver_fn = attrd_cpg_dispatch;
+    attrd_cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership;
+
+    crm_set_status_callback(&attrd_peer_change_cb);
+
+    if (crm_cluster_connect(attrd_cluster) == FALSE) {
+        crm_err("Cluster connection failed");
+        return -ENOTCONN;
+    }
+    return pcmk_ok;
+}
+
+void
+attrd_peer_clear_failure(pcmk__request_t *request)
+{
+    xmlNode *xml = request->xml;
+    const char *rsc = crm_element_value(xml, PCMK__XA_ATTR_RESOURCE);
+    const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
+    const char *op = crm_element_value(xml, PCMK__XA_ATTR_OPERATION);
+    const char *interval_spec = crm_element_value(xml, PCMK__XA_ATTR_INTERVAL);
+    guint interval_ms = crm_parse_interval_spec(interval_spec);
+    char *attr = NULL;
+    GHashTableIter iter;
+    regex_t regex;
+
+    crm_node_t *peer = crm_get_peer(0, request->peer);
+
+    if (attrd_failure_regex(&regex, rsc, op, interval_ms) != pcmk_ok) {
+        crm_info("Ignoring invalid request to clear failures for %s",
+                 pcmk__s(rsc, "all resources"));
+        return;
+    }
+
+    crm_xml_add(xml, PCMK__XA_TASK, PCMK__ATTRD_CMD_UPDATE);
+
+    /* Make sure value is not set, so we delete */
+    if (crm_element_value(xml, PCMK__XA_ATTR_VALUE)) {
+        crm_xml_replace(xml, PCMK__XA_ATTR_VALUE, NULL);
+    }
+
+    g_hash_table_iter_init(&iter, attributes);
+    while (g_hash_table_iter_next(&iter, (gpointer *) &attr, NULL)) {
+        if (regexec(&regex, attr, 0, NULL, 0) == 0) {
+            crm_trace("Matched %s when clearing %s",
+                      attr, pcmk__s(rsc, "all resources"));
+            crm_xml_add(xml, PCMK__XA_ATTR_NAME, attr);
+            attrd_peer_update(peer, xml, host, false);
+        }
+    }
+    regfree(&regex);
+}
+
+/*!
+ * \internal
+ * \brief Load attributes from a peer sync response
+ *
+ * \param[in]     peer      Peer that sent clear request
+ * \param[in]     peer_won  Whether peer is the attribute writer
+ * \param[in,out] xml       Request XML
+ */
+void
+attrd_peer_sync_response(const crm_node_t *peer, bool peer_won, xmlNode *xml)
+{
+    crm_info("Processing " PCMK__ATTRD_CMD_SYNC_RESPONSE " from %s",
+             peer->uname);
+
+    if (peer_won) {
+        /* Initialize the "seen" flag for all attributes to cleared, so we can
+         * detect attributes that local node has but the writer doesn't.
+         */
+        attrd_clear_value_seen();
+    }
+
+    // Process each attribute update in the sync response
+    for (xmlNode *child = pcmk__xml_first_child(xml); child != NULL;
+         child = pcmk__xml_next(child)) {
+        attrd_peer_update(peer, child,
+                          crm_element_value(child, PCMK__XA_ATTR_NODE_NAME),
+                          true);
+    }
+
+    if (peer_won) {
+        /* If any attributes are still not marked as seen, the writer doesn't
+         * know about them, so send all peers an update with them.
+         */
+        broadcast_unseen_local_values();
+    }
+}
+
+/*!
+ * \internal
+ * \brief Remove all attributes and optionally peer cache entries for a node
+ *
+ * \param[in] host     Name of node to purge
+ * \param[in] uncache  If true, remove node from peer caches
+ * \param[in] source   Who requested removal (only used for logging)
+ */
+void
+attrd_peer_remove(const char *host, bool uncache, const char *source)
+{
+    attribute_t *a = NULL;
+    GHashTableIter aIter;
+
+    CRM_CHECK(host != NULL, return);
+    crm_notice("Removing all %s attributes for peer %s", host, source);
+
+    g_hash_table_iter_init(&aIter, attributes);
+    while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
+        if(g_hash_table_remove(a->values, host)) {
+            crm_debug("Removed %s[%s] for peer %s", a->id, host, source);
+        }
+    }
+
+    if (uncache) {
+        crm_remote_peer_cache_remove(host);
+        reap_crm_member(0, host);
+    }
+}
+
+void
+attrd_peer_sync(crm_node_t *peer, xmlNode *xml)
+{
+    GHashTableIter aIter;
+    GHashTableIter vIter;
+
+    attribute_t *a = NULL;
+    attribute_value_t *v = NULL;
+    xmlNode *sync = create_xml_node(NULL, __func__);
+
+    crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
+
+    g_hash_table_iter_init(&aIter, attributes);
+    while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
+        g_hash_table_iter_init(&vIter, a->values);
+        while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
+            crm_debug("Syncing %s[%s] = %s to %s", a->id, v->nodename, v->current, peer?peer->uname:"everyone");
+            attrd_add_value_xml(sync, a, v, false);
+        }
+    }
+
+    crm_debug("Syncing values to %s", peer?peer->uname:"everyone");
+    attrd_send_message(peer, sync, false);
+    free_xml(sync);
+}
+
+void
+attrd_peer_update(const crm_node_t *peer, xmlNode *xml, const char *host,
+                  bool filter)
+{
+    bool handle_sync_point = false;
+
+    if (xml_has_children(xml)) {
+        for (xmlNode *child = first_named_child(xml, XML_ATTR_OP); child != NULL;
+             child = crm_next_same_xml(child)) {
+            attrd_copy_xml_attributes(xml, child);
+            attrd_peer_update_one(peer, child, filter);
+
+            if (attrd_request_has_sync_point(child)) {
+                handle_sync_point = true;
+            }
+        }
+
+    } else {
+        attrd_peer_update_one(peer, xml, filter);
+
+        if (attrd_request_has_sync_point(xml)) {
+            handle_sync_point = true;
+        }
+    }
+
+    /* If the update XML specified that the client wanted to wait for a sync
+     * point, process that now.
+     */
+    if (handle_sync_point) {
+        crm_trace("Hit local sync point for attribute update");
+        attrd_ack_waitlist_clients(attrd_sync_point_local, xml);
+    }
+}
diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c
new file mode 100644
index 0000000..3b6b55a
--- /dev/null
+++ b/daemons/attrd/attrd_elections.c
@@ -0,0 +1,179 @@
+/*
+ * Copyright 2013-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include <crm/msg_xml.h>
+#include <crm/cluster.h>
+#include <crm/cluster/election_internal.h>
+
+#include "pacemaker-attrd.h"
+
+static char *peer_writer = NULL;
+static election_t *writer = NULL;
+
+static gboolean
+attrd_election_cb(gpointer user_data)
+{
+    attrd_declare_winner();
+
+    /* Update the peers after an election */
+    attrd_peer_sync(NULL, NULL);
+
+    /* Update the CIB after an election */
+    attrd_write_attributes(true, false);
+    return FALSE;
+}
+
+void
+attrd_election_init(void)
+{
+    writer = election_init(T_ATTRD, attrd_cluster->uname, 120000,
+                           attrd_election_cb);
+}
+
+void
+attrd_election_fini(void)
+{
+    election_fini(writer);
+}
+
+void
+attrd_start_election_if_needed(void)
+{
+    if ((peer_writer == NULL)
+        && (election_state(writer) != election_in_progress)
+        && !attrd_shutting_down()) {
+
+        crm_info("Starting an election to determine the writer");
+        election_vote(writer);
+    }
+}
+
+bool
+attrd_election_won(void)
+{
+    return (election_state(writer) == election_won);
+}
+
+void
+attrd_handle_election_op(const crm_node_t *peer, xmlNode *xml)
+{
+    enum election_result rc = 0;
+    enum election_result previous = election_state(writer);
+
+    crm_xml_add(xml, F_CRM_HOST_FROM, peer->uname);
+
+    // Don't become writer if we're shutting down
+    rc = election_count_vote(writer, xml, !attrd_shutting_down());
+
+    switch(rc) {
+        case election_start:
+            crm_debug("Unsetting writer (was %s) and starting new election",
+                      peer_writer? peer_writer : "unset");
+            free(peer_writer);
+            peer_writer = NULL;
+            election_vote(writer);
+            break;
+
+        case election_lost:
+            /* The election API should really distinguish between "we just lost
+             * to this peer" and "we already lost previously, and we are
+             * discarding this vote for some reason", but it doesn't.
+             *
+             * In the first case, we want to tentatively set the peer writer to
+             * this peer, even though another peer may eventually win (which we
+             * will learn via attrd_check_for_new_writer()), so
+             * attrd_start_election_if_needed() doesn't start a new election.
+             *
+             * Approximate a test for that case as best as possible.
+             */
+            if ((peer_writer == NULL) || (previous != election_lost)) {
+                pcmk__str_update(&peer_writer, peer->uname);
+                crm_debug("Election lost, presuming %s is writer for now",
+                          peer_writer);
+            }
+            break;
+
+        case election_in_progress:
+            election_check(writer);
+            break;
+
+        default:
+            crm_info("Ignoring election op from %s due to error", peer->uname);
+            break;
+    }
+}
+
+bool
+attrd_check_for_new_writer(const crm_node_t *peer, const xmlNode *xml)
+{
+    int peer_state = 0;
+
+    crm_element_value_int(xml, PCMK__XA_ATTR_WRITER, &peer_state);
+    if (peer_state == election_won) {
+        if ((election_state(writer) == election_won)
+           && !pcmk__str_eq(peer->uname, attrd_cluster->uname, pcmk__str_casei)) {
+            crm_notice("Detected another attribute writer (%s), starting new election",
+                       peer->uname);
+            election_vote(writer);
+
+        } else if (!pcmk__str_eq(peer->uname, peer_writer, pcmk__str_casei)) {
+            crm_notice("Recorded new attribute writer: %s (was %s)",
+                       peer->uname, (peer_writer? peer_writer : "unset"));
+            pcmk__str_update(&peer_writer, peer->uname);
+        }
+    }
+    return (peer_state == election_won);
+}
+
+void
+attrd_declare_winner(void)
+{
+    crm_notice("Recorded local node as attribute writer (was %s)",
+               (peer_writer? peer_writer : "unset"));
+    pcmk__str_update(&peer_writer, attrd_cluster->uname);
+}
+
+void
+attrd_remove_voter(const crm_node_t *peer)
+{
+    election_remove(writer, peer->uname);
+    if (peer_writer && pcmk__str_eq(peer->uname, peer_writer, pcmk__str_casei)) {
+        free(peer_writer);
+        peer_writer = NULL;
+        crm_notice("Lost attribute writer %s", peer->uname);
+
+        /* Clear any election dampening in effect. Otherwise, if the lost writer
+         * had just won, the election could fizzle out with no new writer.
+         */
+        election_clear_dampening(writer);
+
+        /* If the writer received attribute updates during its shutdown, it will
+         * not have written them to the CIB. Ensure we get a new writer so they
+         * are written out. This means that every node that sees the writer
+         * leave will start a new election, but that's better than losing
+         * attributes.
+         */
+        attrd_start_election_if_needed();
+
+    /* If an election is in progress, we need to call election_check(), in case
+     * this lost peer is the only one that hasn't voted, otherwise the election
+     * would be pending until it's timed out.
+     */
+    } else if (election_state(writer) == election_in_progress) {
+       crm_debug("Checking election status upon loss of voter %s", peer->uname);
+       election_check(writer);
+    }
+}
+
+void
+attrd_xml_add_writer(xmlNode *xml)
+{
+    crm_xml_add_int(xml, PCMK__XA_ATTR_WRITER, election_state(writer));
+}
diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c
new file mode 100644
index 0000000..9d3dfff
--- /dev/null
+++ b/daemons/attrd/attrd_ipc.c
@@ -0,0 +1,628 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <errno.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <sys/types.h>
+
+#include <crm/cluster.h>
+#include <crm/cluster/internal.h>
+#include <crm/msg_xml.h>
+#include <crm/common/acl_internal.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/common/logging.h>
+#include <crm/common/results.h>
+#include <crm/common/strings_internal.h>
+#include <crm/common/util.h>
+
+#include "pacemaker-attrd.h"
+
+static qb_ipcs_service_t *ipcs = NULL;
+
+/*!
+ * \internal
+ * \brief Build the XML reply to a client query
+ *
+ * param[in] attr Name of requested attribute
+ * param[in] host Name of requested host (or NULL for all hosts)
+ *
+ * \return New XML reply
+ * \note Caller is responsible for freeing the resulting XML
+ */
+static xmlNode *build_query_reply(const char *attr, const char *host)
+{
+    xmlNode *reply = create_xml_node(NULL, __func__);
+    attribute_t *a;
+
+    if (reply == NULL) {
+        return NULL;
+    }
+    crm_xml_add(reply, F_TYPE, T_ATTRD);
+    crm_xml_add(reply, F_SUBTYPE, PCMK__ATTRD_CMD_QUERY);
+    crm_xml_add(reply, PCMK__XA_ATTR_VERSION, ATTRD_PROTOCOL_VERSION);
+
+    /* If desired attribute exists, add its value(s) to the reply */
+    a = g_hash_table_lookup(attributes, attr);
+    if (a) {
+        attribute_value_t *v;
+        xmlNode *host_value;
+
+        crm_xml_add(reply, PCMK__XA_ATTR_NAME, attr);
+
+        /* Allow caller to use "localhost" to refer to local node */
+        if (pcmk__str_eq(host, "localhost", pcmk__str_casei)) {
+            host = attrd_cluster->uname;
+            crm_trace("Mapped localhost to %s", host);
+        }
+
+        /* If a specific node was requested, add its value */
+        if (host) {
+            v = g_hash_table_lookup(a->values, host);
+            host_value = create_xml_node(reply, XML_CIB_TAG_NODE);
+            if (host_value == NULL) {
+                free_xml(reply);
+                return NULL;
+            }
+            pcmk__xe_add_node(host_value, host, 0);
+            crm_xml_add(host_value, PCMK__XA_ATTR_VALUE,
+                        (v? v->current : NULL));
+
+        /* Otherwise, add all nodes' values */
+        } else {
+            GHashTableIter iter;
+
+            g_hash_table_iter_init(&iter, a->values);
+            while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &v)) {
+                host_value = create_xml_node(reply, XML_CIB_TAG_NODE);
+                if (host_value == NULL) {
+                    free_xml(reply);
+                    return NULL;
+                }
+                pcmk__xe_add_node(host_value, v->nodename, 0);
+                crm_xml_add(host_value, PCMK__XA_ATTR_VALUE, v->current);
+            }
+        }
+    }
+    return reply;
+}
+
+xmlNode *
+attrd_client_clear_failure(pcmk__request_t *request)
+{
+    xmlNode *xml = request->xml;
+    const char *rsc, *op, *interval_spec;
+
+    if (minimum_protocol_version >= 2) {
+        /* Propagate to all peers (including ourselves).
+         * This ends up at attrd_peer_message().
+         */
+        attrd_send_message(NULL, xml, false);
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+        return NULL;
+    }
+
+    rsc = crm_element_value(xml, PCMK__XA_ATTR_RESOURCE);
+    op = crm_element_value(xml, PCMK__XA_ATTR_OPERATION);
+    interval_spec = crm_element_value(xml, PCMK__XA_ATTR_INTERVAL);
+
+    /* Map this to an update */
+    crm_xml_add(xml, PCMK__XA_TASK, PCMK__ATTRD_CMD_UPDATE);
+
+    /* Add regular expression matching desired attributes */
+
+    if (rsc) {
+        char *pattern;
+
+        if (op == NULL) {
+            pattern = crm_strdup_printf(ATTRD_RE_CLEAR_ONE, rsc);
+
+        } else {
+            guint interval_ms = crm_parse_interval_spec(interval_spec);
+
+            pattern = crm_strdup_printf(ATTRD_RE_CLEAR_OP,
+                                        rsc, op, interval_ms);
+        }
+
+        crm_xml_add(xml, PCMK__XA_ATTR_PATTERN, pattern);
+        free(pattern);
+
+    } else {
+        crm_xml_add(xml, PCMK__XA_ATTR_PATTERN, ATTRD_RE_CLEAR_ALL);
+    }
+
+    /* Make sure attribute and value are not set, so we delete via regex */
+    if (crm_element_value(xml, PCMK__XA_ATTR_NAME)) {
+        crm_xml_replace(xml, PCMK__XA_ATTR_NAME, NULL);
+    }
+    if (crm_element_value(xml, PCMK__XA_ATTR_VALUE)) {
+        crm_xml_replace(xml, PCMK__XA_ATTR_VALUE, NULL);
+    }
+
+    return attrd_client_update(request);
+}
+
+xmlNode *
+attrd_client_peer_remove(pcmk__request_t *request)
+{
+    xmlNode *xml = request->xml;
+
+    // Host and ID are not used in combination, rather host has precedence
+    const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
+    char *host_alloc = NULL;
+
+    attrd_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags);
+
+    if (host == NULL) {
+        int nodeid = 0;
+
+        crm_element_value_int(xml, PCMK__XA_ATTR_NODE_ID, &nodeid);
+        if (nodeid > 0) {
+            crm_node_t *node = pcmk__search_cluster_node_cache(nodeid, NULL);
+            char *host_alloc = NULL;
+
+            if (node && node->uname) {
+                // Use cached name if available
+                host = node->uname;
+            } else {
+                // Otherwise ask cluster layer
+                host_alloc = get_node_name(nodeid);
+                host = host_alloc;
+            }
+            pcmk__xe_add_node(xml, host, 0);
+        }
+    }
+
+    if (host) {
+        crm_info("Client %s is requesting all values for %s be removed",
+                 pcmk__client_name(request->ipc_client), host);
+        attrd_send_message(NULL, xml, false); /* ends up at attrd_peer_message() */
+        free(host_alloc);
+    } else {
+        crm_info("Ignoring request by client %s to remove all peer values without specifying peer",
+                 pcmk__client_name(request->ipc_client));
+    }
+
+    pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    return NULL;
+}
+
+xmlNode *
+attrd_client_query(pcmk__request_t *request)
+{
+    xmlNode *query = request->xml;
+    xmlNode *reply = NULL;
+    const char *attr = NULL;
+
+    crm_debug("Query arrived from %s", pcmk__client_name(request->ipc_client));
+
+    /* Request must specify attribute name to query */
+    attr = crm_element_value(query, PCMK__XA_ATTR_NAME);
+    if (attr == NULL) {
+        pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
+                            "Ignoring malformed query from %s (no attribute name given)",
+                            pcmk__client_name(request->ipc_client));
+        return NULL;
+    }
+
+    /* Build the XML reply */
+    reply = build_query_reply(attr, crm_element_value(query,
+                                                      PCMK__XA_ATTR_NODE_NAME));
+    if (reply == NULL) {
+        pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
+                            "Could not respond to query from %s: could not create XML reply",
+                            pcmk__client_name(request->ipc_client));
+        return NULL;
+    } else {
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    }
+
+    request->ipc_client->request_id = 0;
+    return reply;
+}
+
+xmlNode *
+attrd_client_refresh(pcmk__request_t *request)
+{
+    crm_info("Updating all attributes");
+
+    attrd_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags);
+    attrd_write_attributes(true, true);
+
+    pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    return NULL;
+}
+
+static void
+handle_missing_host(xmlNode *xml)
+{
+    const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
+
+    if (host == NULL) {
+        crm_trace("Inferring host");
+        pcmk__xe_add_node(xml, attrd_cluster->uname, attrd_cluster->nodeid);
+    }
+}
+
+/* Convert a single IPC message with a regex into one with multiple children, one
+ * for each regex match.
+ */
+static int
+expand_regexes(xmlNode *xml, const char *attr, const char *value, const char *regex)
+{
+    if (attr == NULL && regex) {
+        bool matched = false;
+        GHashTableIter aIter;
+        regex_t r_patt;
+
+        crm_debug("Setting %s to %s", regex, value);
+        if (regcomp(&r_patt, regex, REG_EXTENDED|REG_NOSUB)) {
+            return EINVAL;
+        }
+
+        g_hash_table_iter_init(&aIter, attributes);
+        while (g_hash_table_iter_next(&aIter, (gpointer *) & attr, NULL)) {
+            int status = regexec(&r_patt, attr, 0, NULL, 0);
+
+            if (status == 0) {
+                xmlNode *child = create_xml_node(xml, XML_ATTR_OP);
+
+                crm_trace("Matched %s with %s", attr, regex);
+                matched = true;
+
+                /* Copy all the attributes from the parent over, but remove the
+                 * regex and replace it with the name.
+                 */
+                attrd_copy_xml_attributes(xml, child);
+                crm_xml_replace(child, PCMK__XA_ATTR_PATTERN, NULL);
+                crm_xml_add(child, PCMK__XA_ATTR_NAME, attr);
+            }
+        }
+
+        regfree(&r_patt);
+
+        /* Return a code if we never matched anything.  This should not be treated
+         * as an error.  It indicates there was a regex, and it was a valid regex,
+         * but simply did not match anything and the caller should not continue
+         * doing any regex-related processing.
+         */
+        if (!matched) {
+            return pcmk_rc_op_unsatisfied;
+        }
+
+    } else if (attr == NULL) {
+        return pcmk_rc_bad_nvpair;
+    }
+
+    return pcmk_rc_ok;
+}
+
+static int
+handle_regexes(pcmk__request_t *request)
+{
+    xmlNode *xml = request->xml;
+    int rc = pcmk_rc_ok;
+
+    const char *attr = crm_element_value(xml, PCMK__XA_ATTR_NAME);
+    const char *value = crm_element_value(xml, PCMK__XA_ATTR_VALUE);
+    const char *regex = crm_element_value(xml, PCMK__XA_ATTR_PATTERN);
+
+    rc = expand_regexes(xml, attr, value, regex);
+
+    if (rc == EINVAL) {
+        pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
+                            "Bad regex '%s' for update from client %s", regex,
+                            pcmk__client_name(request->ipc_client));
+
+    } else if (rc == pcmk_rc_bad_nvpair) {
+        crm_err("Update request did not specify attribute or regular expression");
+        pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
+                            "Client %s update request did not specify attribute or regular expression",
+                            pcmk__client_name(request->ipc_client));
+    }
+
+    return rc;
+}
+
+static int
+handle_value_expansion(const char **value, xmlNode *xml, const char *op,
+                       const char *attr)
+{
+    attribute_t *a = g_hash_table_lookup(attributes, attr);
+
+    if (a == NULL && pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE_DELAY, pcmk__str_none)) {
+        return EINVAL;
+    }
+
+    if (*value && attrd_value_needs_expansion(*value)) {
+        int int_value;
+        attribute_value_t *v = NULL;
+
+        if (a) {
+            const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
+            v = g_hash_table_lookup(a->values, host);
+        }
+
+        int_value = attrd_expand_value(*value, (v? v->current : NULL));
+
+        crm_info("Expanded %s=%s to %d", attr, *value, int_value);
+        crm_xml_add_int(xml, PCMK__XA_ATTR_VALUE, int_value);
+
+        /* Replacing the value frees the previous memory, so re-query it */
+        *value = crm_element_value(xml, PCMK__XA_ATTR_VALUE);
+    }
+
+    return pcmk_rc_ok;
+}
+
+static void
+send_update_msg_to_cluster(pcmk__request_t *request, xmlNode *xml)
+{
+    if (pcmk__str_eq(attrd_request_sync_point(xml), PCMK__VALUE_CLUSTER, pcmk__str_none)) {
+        /* The client is waiting on the cluster-wide sync point.  In this case,
+         * the response ACK is not sent until this attrd broadcasts the update
+         * and receives its own confirmation back from all peers.
+         */
+        attrd_expect_confirmations(request, attrd_cluster_sync_point_update);
+        attrd_send_message(NULL, xml, true); /* ends up at attrd_peer_message() */
+
+    } else {
+        /* The client is either waiting on the local sync point or was not
+         * waiting on any sync point at all.  For the local sync point, the
+         * response ACK is sent in attrd_peer_update.  For clients not
+         * waiting on any sync point, the response ACK is sent in
+         * handle_update_request immediately before this function was called.
+         */
+        attrd_send_message(NULL, xml, false); /* ends up at attrd_peer_message() */
+    }
+}
+
+static int
+send_child_update(xmlNode *child, void *data)
+{
+    pcmk__request_t *request = (pcmk__request_t *) data;
+
+    /* Calling pcmk__set_result is handled by one of these calls to
+     * attrd_client_update, so no need to do it again here.
+     */
+    request->xml = child;
+    attrd_client_update(request);
+    return pcmk_rc_ok;
+}
+
+xmlNode *
+attrd_client_update(pcmk__request_t *request)
+{
+    xmlNode *xml = request->xml;
+    const char *attr, *value, *regex;
+
+    /* If the message has children, that means it is a message from a newer
+     * client that supports sending multiple operations at a time.  There are
+     * two ways we can handle that.
+     */
+    if (xml_has_children(xml)) {
+        if (ATTRD_SUPPORTS_MULTI_MESSAGE(minimum_protocol_version)) {
+            /* First, if all peers support a certain protocol version, we can
+             * just broadcast the big message and they'll handle it.  However,
+             * we also need to apply all the transformations in this function
+             * to the children since they don't happen anywhere else.
+             */
+            for (xmlNode *child = first_named_child(xml, XML_ATTR_OP); child != NULL;
+                 child = crm_next_same_xml(child)) {
+                attr = crm_element_value(child, PCMK__XA_ATTR_NAME);
+                value = crm_element_value(child, PCMK__XA_ATTR_VALUE);
+
+                handle_missing_host(child);
+
+                if (handle_value_expansion(&value, child, request->op, attr) == EINVAL) {
+                    pcmk__format_result(&request->result, CRM_EX_NOSUCH, PCMK_EXEC_ERROR,
+                                        "Attribute %s does not exist", attr);
+                    return NULL;
+                }
+            }
+
+            send_update_msg_to_cluster(request, xml);
+            pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+
+        } else {
+            /* Save the original xml node pointer so it can be restored after iterating
+             * over all the children.
+             */
+            xmlNode *orig_xml = request->xml;
+
+            /* Second, if they do not support that protocol version, split it
+             * up into individual messages and call attrd_client_update on
+             * each one.
+             */
+            pcmk__xe_foreach_child(xml, XML_ATTR_OP, send_child_update, request);
+            request->xml = orig_xml;
+        }
+
+        return NULL;
+    }
+
+    attr = crm_element_value(xml, PCMK__XA_ATTR_NAME);
+    value = crm_element_value(xml, PCMK__XA_ATTR_VALUE);
+    regex = crm_element_value(xml, PCMK__XA_ATTR_PATTERN);
+
+    if (handle_regexes(request) != pcmk_rc_ok) {
+        /* Error handling was already dealt with in handle_regexes, so just return. */
+        return NULL;
+    } else if (regex) {
+        /* Recursively call attrd_client_update on the new message with regexes
+         * expanded.  If supported by the attribute daemon, this means that all
+         * matches can also be handled atomically.
+         */
+        return attrd_client_update(request);
+    }
+
+    handle_missing_host(xml);
+
+    if (handle_value_expansion(&value, xml, request->op, attr) == EINVAL) {
+        pcmk__format_result(&request->result, CRM_EX_NOSUCH, PCMK_EXEC_ERROR,
+                            "Attribute %s does not exist", attr);
+        return NULL;
+    }
+
+    crm_debug("Broadcasting %s[%s]=%s%s", attr, crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME),
+              value, (attrd_election_won()? " (writer)" : ""));
+
+    send_update_msg_to_cluster(request, xml);
+    pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    return NULL;
+}
+
+/*!
+ * \internal
+ * \brief Accept a new client IPC connection
+ *
+ * \param[in,out] c    New connection
+ * \param[in]     uid  Client user id
+ * \param[in]     gid  Client group id
+ *
+ * \return pcmk_ok on success, -errno otherwise
+ */
+static int32_t
+attrd_ipc_accept(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
+{
+    crm_trace("New client connection %p", c);
+    if (attrd_shutting_down()) {
+        crm_info("Ignoring new connection from pid %d during shutdown",
+                 pcmk__client_pid(c));
+        return -EPERM;
+    }
+
+    if (pcmk__new_client(c, uid, gid) == NULL) {
+        return -EIO;
+    }
+    return pcmk_ok;
+}
+
+/*!
+ * \internal
+ * \brief Destroy a client IPC connection
+ *
+ * \param[in] c  Connection to destroy
+ *
+ * \return FALSE (i.e. do not re-run this callback)
+ */
+static int32_t
+attrd_ipc_closed(qb_ipcs_connection_t *c)
+{
+    pcmk__client_t *client = pcmk__find_client(c);
+
+    if (client == NULL) {
+        crm_trace("Ignoring request to clean up unknown connection %p", c);
+    } else {
+        crm_trace("Cleaning up closed client connection %p", c);
+
+        /* Remove the client from the sync point waitlist if it's present. */
+        attrd_remove_client_from_waitlist(client);
+
+        /* And no longer wait for confirmations from any peers. */
+        attrd_do_not_wait_for_client(client);
+
+        pcmk__free_client(client);
+    }
+
+    return FALSE;
+}
+
+/*!
+ * \internal
+ * \brief Destroy a client IPC connection
+ *
+ * \param[in,out] c  Connection to destroy
+ *
+ * \note We handle a destroyed connection the same as a closed one,
+ *       but we need a separate handler because the return type is different.
+ */
+static void
+attrd_ipc_destroy(qb_ipcs_connection_t *c)
+{
+    crm_trace("Destroying client connection %p", c);
+    attrd_ipc_closed(c);
+}
+
+static int32_t
+attrd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
+{
+    uint32_t id = 0;
+    uint32_t flags = 0;
+    pcmk__client_t *client = pcmk__find_client(c);
+    xmlNode *xml = NULL;
+
+    // Sanity-check, and parse XML from IPC data
+    CRM_CHECK((c != NULL) && (client != NULL), return 0);
+    if (data == NULL) {
+        crm_debug("No IPC data from PID %d", pcmk__client_pid(c));
+        return 0;
+    }
+
+    xml = pcmk__client_data2xml(client, data, &id, &flags);
+
+    if (xml == NULL) {
+        crm_debug("Unrecognizable IPC data from PID %d", pcmk__client_pid(c));
+        pcmk__ipc_send_ack(client, id, flags, "ack", NULL, CRM_EX_PROTOCOL);
+        return 0;
+
+    } else {
+        pcmk__request_t request = {
+            .ipc_client     = client,
+            .ipc_id         = id,
+            .ipc_flags      = flags,
+            .peer           = NULL,
+            .xml            = xml,
+            .call_options   = 0,
+            .result         = PCMK__UNKNOWN_RESULT,
+        };
+
+        CRM_ASSERT(client->user != NULL);
+        pcmk__update_acl_user(xml, PCMK__XA_ATTR_USER, client->user);
+
+        request.op = crm_element_value_copy(request.xml, PCMK__XA_TASK);
+        CRM_CHECK(request.op != NULL, return 0);
+
+        attrd_handle_request(&request);
+        pcmk__reset_request(&request);
+    }
+
+    free_xml(xml);
+    return 0;
+}
+
+static struct qb_ipcs_service_handlers ipc_callbacks = {
+    .connection_accept = attrd_ipc_accept,
+    .connection_created = NULL,
+    .msg_process = attrd_ipc_dispatch,
+    .connection_closed = attrd_ipc_closed,
+    .connection_destroyed = attrd_ipc_destroy
+};
+
+void
+attrd_ipc_fini(void)
+{
+    if (ipcs != NULL) {
+        pcmk__drop_all_clients(ipcs);
+        qb_ipcs_destroy(ipcs);
+        ipcs = NULL;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Set up attrd IPC communication
+ */
+void
+attrd_init_ipc(void)
+{
+    pcmk__serve_attrd_ipc(&ipcs, &ipc_callbacks);
+}
diff --git a/daemons/attrd/attrd_messages.c b/daemons/attrd/attrd_messages.c
new file mode 100644
index 0000000..184176a
--- /dev/null
+++ b/daemons/attrd/attrd_messages.c
@@ -0,0 +1,328 @@
+/*
+ * Copyright 2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <glib.h>
+
+#include <crm/common/messages_internal.h>
+#include <crm/msg_xml.h>
+
+#include "pacemaker-attrd.h"
+
+int minimum_protocol_version = -1;
+
+static GHashTable *attrd_handlers = NULL;
+
+static xmlNode *
+handle_unknown_request(pcmk__request_t *request)
+{
+    crm_err("Unknown IPC request %s from %s %s",
+            request->op, pcmk__request_origin_type(request),
+            pcmk__request_origin(request));
+    pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
+                        "Unknown request type '%s' (bug?)", request->op);
+    return NULL;
+}
+
+static xmlNode *
+handle_clear_failure_request(pcmk__request_t *request)
+{
+    if (request->peer != NULL) {
+        /* It is not currently possible to receive this as a peer command,
+         * but will be, if we one day enable propagating this operation.
+         */
+        attrd_peer_clear_failure(request);
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+        return NULL;
+    } else {
+        if (attrd_request_has_sync_point(request->xml)) {
+            /* If this client supplied a sync point it wants to wait for, add it to
+             * the wait list.  Clients on this list will not receive an ACK until
+             * their sync point is hit which will result in the client stalled there
+             * until it receives a response.
+             *
+             * All other clients will receive the expected response as normal.
+             */
+            attrd_add_client_to_waitlist(request);
+
+        } else {
+            /* If the client doesn't want to wait for a sync point, go ahead and send
+             * the ACK immediately.  Otherwise, we'll send the ACK when the appropriate
+             * sync point is reached.
+             */
+            attrd_send_ack(request->ipc_client, request->ipc_id,
+                           request->ipc_flags);
+        }
+
+        return attrd_client_clear_failure(request);
+    }
+}
+
+static xmlNode *
+handle_confirm_request(pcmk__request_t *request)
+{
+    if (request->peer != NULL) {
+        int callid;
+
+        crm_debug("Received confirmation from %s", request->peer);
+
+        if (crm_element_value_int(request->xml, XML_LRM_ATTR_CALLID, &callid) == -1) {
+            pcmk__set_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
+                             "Could not get callid from XML");
+        } else {
+            attrd_handle_confirmation(callid, request->peer);
+        }
+
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+        return NULL;
+    } else {
+        return handle_unknown_request(request);
+    }
+}
+
+static xmlNode *
+handle_flush_request(pcmk__request_t *request)
+{
+    if (request->peer != NULL) {
+        /* Ignore. The flush command was removed in 2.0.0 but may be
+         * received from peers running older versions.
+         */
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+        return NULL;
+    } else {
+        return handle_unknown_request(request);
+    }
+}
+
+static xmlNode *
+handle_query_request(pcmk__request_t *request)
+{
+    if (request->peer != NULL) {
+        return handle_unknown_request(request);
+    } else {
+        return attrd_client_query(request);
+    }
+}
+
+static xmlNode *
+handle_remove_request(pcmk__request_t *request)
+{
+    if (request->peer != NULL) {
+        const char *host = crm_element_value(request->xml, PCMK__XA_ATTR_NODE_NAME);
+        attrd_peer_remove(host, true, request->peer);
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+        return NULL;
+    } else {
+        return attrd_client_peer_remove(request);
+    }
+}
+
+static xmlNode *
+handle_refresh_request(pcmk__request_t *request)
+{
+    if (request->peer != NULL) {
+        return handle_unknown_request(request);
+    } else {
+        return attrd_client_refresh(request);
+    }
+}
+
+static xmlNode *
+handle_sync_request(pcmk__request_t *request)
+{
+    if (request->peer != NULL) {
+        crm_node_t *peer = crm_get_peer(0, request->peer);
+
+        attrd_peer_sync(peer, request->xml);
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+        return NULL;
+    } else {
+        return handle_unknown_request(request);
+    }
+}
+
+static xmlNode *
+handle_sync_response_request(pcmk__request_t *request)
+{
+    if (request->ipc_client != NULL) {
+        return handle_unknown_request(request);
+    } else {
+        if (request->peer != NULL) {
+            crm_node_t *peer = crm_get_peer(0, request->peer);
+            bool peer_won = attrd_check_for_new_writer(peer, request->xml);
+
+            if (!pcmk__str_eq(peer->uname, attrd_cluster->uname, pcmk__str_casei)) {
+                attrd_peer_sync_response(peer, peer_won, request->xml);
+            }
+        }
+
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+        return NULL;
+    }
+}
+
+static xmlNode *
+handle_update_request(pcmk__request_t *request)
+{
+    if (request->peer != NULL) {
+        const char *host = crm_element_value(request->xml, PCMK__XA_ATTR_NODE_NAME);
+        crm_node_t *peer = crm_get_peer(0, request->peer);
+
+        attrd_peer_update(peer, request->xml, host, false);
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+        return NULL;
+
+    } else {
+        if (attrd_request_has_sync_point(request->xml)) {
+            /* If this client supplied a sync point it wants to wait for, add it to
+             * the wait list.  Clients on this list will not receive an ACK until
+             * their sync point is hit which will result in the client stalled there
+             * until it receives a response.
+             *
+             * All other clients will receive the expected response as normal.
+             */
+            attrd_add_client_to_waitlist(request);
+
+        } else {
+            /* If the client doesn't want to wait for a sync point, go ahead and send
+             * the ACK immediately.  Otherwise, we'll send the ACK when the appropriate
+             * sync point is reached.
+             *
+             * In the normal case, attrd_client_update can be called recursively which
+             * makes where to send the ACK tricky.  Doing it here ensures the client
+             * only ever receives one.
+             */
+            attrd_send_ack(request->ipc_client, request->ipc_id,
+                           request->flags|crm_ipc_client_response);
+        }
+
+        return attrd_client_update(request);
+    }
+}
+
+static void
+attrd_register_handlers(void)
+{
+    pcmk__server_command_t handlers[] = {
+        { PCMK__ATTRD_CMD_CLEAR_FAILURE, handle_clear_failure_request },
+        { PCMK__ATTRD_CMD_CONFIRM, handle_confirm_request },
+        { PCMK__ATTRD_CMD_FLUSH, handle_flush_request },
+        { PCMK__ATTRD_CMD_PEER_REMOVE, handle_remove_request },
+        { PCMK__ATTRD_CMD_QUERY, handle_query_request },
+        { PCMK__ATTRD_CMD_REFRESH, handle_refresh_request },
+        { PCMK__ATTRD_CMD_SYNC, handle_sync_request },
+        { PCMK__ATTRD_CMD_SYNC_RESPONSE, handle_sync_response_request },
+        { PCMK__ATTRD_CMD_UPDATE, handle_update_request },
+        { PCMK__ATTRD_CMD_UPDATE_DELAY, handle_update_request },
+        { PCMK__ATTRD_CMD_UPDATE_BOTH, handle_update_request },
+        { NULL, handle_unknown_request },
+    };
+
+    attrd_handlers = pcmk__register_handlers(handlers);
+}
+
+void
+attrd_unregister_handlers(void)
+{
+    if (attrd_handlers != NULL) {
+        g_hash_table_destroy(attrd_handlers);
+        attrd_handlers = NULL;
+    }
+}
+
+void
+attrd_handle_request(pcmk__request_t *request)
+{
+    xmlNode *reply = NULL;
+    char *log_msg = NULL;
+    const char *reason = NULL;
+
+    if (attrd_handlers == NULL) {
+        attrd_register_handlers();
+    }
+
+    reply = pcmk__process_request(request, attrd_handlers);
+
+    if (reply != NULL) {
+        crm_log_xml_trace(reply, "Reply");
+
+        if (request->ipc_client != NULL) {
+            pcmk__ipc_send_xml(request->ipc_client, request->ipc_id, reply,
+                               request->ipc_flags);
+        } else {
+            crm_err("Not sending CPG reply to client");
+        }
+
+        free_xml(reply);
+    }
+
+    reason = request->result.exit_reason;
+    log_msg = crm_strdup_printf("Processed %s request from %s %s: %s%s%s%s",
+                                request->op, pcmk__request_origin_type(request),
+                                pcmk__request_origin(request),
+                                pcmk_exec_status_str(request->result.execution_status),
+                                (reason == NULL)? "" : " (",
+                                pcmk__s(reason, ""),
+                                (reason == NULL)? "" : ")");
+
+    if (!pcmk__result_ok(&request->result)) {
+        crm_warn("%s", log_msg);
+    } else {
+        crm_debug("%s", log_msg);
+    }
+
+    free(log_msg);
+    pcmk__reset_request(request);
+}
+
+/*!
+    \internal
+    \brief Broadcast private attribute for local node with protocol version
+*/
+void
+attrd_broadcast_protocol(void)
+{
+    xmlNode *attrd_op = create_xml_node(NULL, __func__);
+
+    crm_xml_add(attrd_op, F_TYPE, T_ATTRD);
+    crm_xml_add(attrd_op, F_ORIG, crm_system_name);
+    crm_xml_add(attrd_op, PCMK__XA_TASK, PCMK__ATTRD_CMD_UPDATE);
+    crm_xml_add(attrd_op, PCMK__XA_ATTR_NAME, CRM_ATTR_PROTOCOL);
+    crm_xml_add(attrd_op, PCMK__XA_ATTR_VALUE, ATTRD_PROTOCOL_VERSION);
+    crm_xml_add_int(attrd_op, PCMK__XA_ATTR_IS_PRIVATE, 1);
+    pcmk__xe_add_node(attrd_op, attrd_cluster->uname, attrd_cluster->nodeid);
+
+    crm_debug("Broadcasting attrd protocol version %s for node %s",
+              ATTRD_PROTOCOL_VERSION, attrd_cluster->uname);
+
+    attrd_send_message(NULL, attrd_op, false); /* ends up at attrd_peer_message() */
+
+    free_xml(attrd_op);
+}
+
+gboolean
+attrd_send_message(crm_node_t *node, xmlNode *data, bool confirm)
+{
+    const char *op = crm_element_value(data, PCMK__XA_TASK);
+
+    crm_xml_add(data, F_TYPE, T_ATTRD);
+    crm_xml_add(data, PCMK__XA_ATTR_VERSION, ATTRD_PROTOCOL_VERSION);
+
+    /* Request a confirmation from the destination peer node (which could
+     * be all if node is NULL) that the message has been received and
+     * acted upon.
+     */
+    if (!pcmk__str_eq(op, PCMK__ATTRD_CMD_CONFIRM, pcmk__str_none)) {
+        pcmk__xe_set_bool_attr(data, PCMK__XA_CONFIRM, confirm);
+    }
+
+    attrd_xml_add_writer(data);
+    return send_cluster_message(node, crm_msg_attrd, data, TRUE);
+}
diff --git a/daemons/attrd/attrd_sync.c b/daemons/attrd/attrd_sync.c
new file mode 100644
index 0000000..d59ddd5
--- /dev/null
+++ b/daemons/attrd/attrd_sync.c
@@ -0,0 +1,577 @@
+/*
+ * Copyright 2022-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm/msg_xml.h>
+#include <crm/common/attrd_internal.h>
+
+#include "pacemaker-attrd.h"
+
+/* A hash table storing clients that are waiting on a sync point to be reached.
+ * The key is waitlist_client - just a plain int.  The obvious key would be
+ * the IPC client's ID, but this is not guaranteed to be unique.  A single client
+ * could be waiting on a sync point for multiple attributes at the same time.
+ *
+ * It is not expected that this hash table will ever be especially large.
+ */
+static GHashTable *waitlist = NULL;
+static int waitlist_client = 0;
+
+struct waitlist_node {
+    /* What kind of sync point does this node describe? */
+    enum attrd_sync_point sync_point;
+
+    /* Information required to construct and send a reply to the client. */
+    char *client_id;
+    uint32_t ipc_id;
+    uint32_t flags;
+};
+
+/* A hash table storing information on in-progress IPC requests that are awaiting
+ * confirmations.  These requests are currently being processed by peer attrds and
+ * we are waiting to receive confirmation messages from each peer indicating that
+ * processing is complete.
+ *
+ * Multiple requests could be waiting on confirmations at the same time.
+ *
+ * The key is the unique callid for the IPC request, and the value is a
+ * confirmation_action struct.
+ */
+static GHashTable *expected_confirmations = NULL;
+
+/*!
+ * \internal
+ * \brief A structure describing a single IPC request that is awaiting confirmations
+ */
+struct confirmation_action {
+    /*!
+     * \brief A list of peer attrds that we are waiting to receive confirmation
+     *        messages from
+     *
+     * This list is dynamic - as confirmations arrive from peer attrds, they will
+     * be removed from this list.  When the list is empty, all peers have processed
+     * the request and the associated confirmation action will be taken.
+     */
+    GList *respondents;
+
+    /*!
+     * \brief A timer that will be used to remove the client should it time out
+     *        before receiving all confirmations
+     */
+    mainloop_timer_t *timer;
+
+    /*!
+     * \brief A function to run when all confirmations have been received
+     */
+    attrd_confirmation_action_fn fn;
+
+    /*!
+     * \brief Information required to construct and send a reply to the client
+     */
+    char *client_id;
+    uint32_t ipc_id;
+    uint32_t flags;
+
+    /*!
+     * \brief The XML request containing the callid associated with this action
+     */
+    void *xml;
+};
+
+static void
+next_key(void)
+{
+    do {
+        waitlist_client++;
+        if (waitlist_client < 0) {
+            waitlist_client = 1;
+        }
+    } while (g_hash_table_contains(waitlist, GINT_TO_POINTER(waitlist_client)));
+}
+
+static void
+free_waitlist_node(gpointer data)
+{
+    struct waitlist_node *wl = (struct waitlist_node *) data;
+
+    free(wl->client_id);
+    free(wl);
+}
+
+static const char *
+sync_point_str(enum attrd_sync_point sync_point)
+{
+    if (sync_point == attrd_sync_point_local) {
+        return PCMK__VALUE_LOCAL;
+    } else if  (sync_point == attrd_sync_point_cluster) {
+        return PCMK__VALUE_CLUSTER;
+    } else {
+        return "unknown";
+    }
+}
+
+/*!
+ * \internal
+ * \brief Add a client to the attrd waitlist
+ *
+ * Typically, a client receives an ACK for its XML IPC request immediately.  However,
+ * some clients want to wait until their request has been processed and taken effect.
+ * This is called a sync point.  Any client placed on this waitlist will have its
+ * ACK message delayed until either its requested sync point is hit, or until it
+ * times out.
+ *
+ * The XML IPC request must specify the type of sync point it wants to wait for.
+ *
+ * \param[in,out] request   The request describing the client to place on the waitlist.
+ */
+void
+attrd_add_client_to_waitlist(pcmk__request_t *request)
+{
+    const char *sync_point = attrd_request_sync_point(request->xml);
+    struct waitlist_node *wl = NULL;
+
+    if (sync_point == NULL) {
+        return;
+    }
+
+    if (waitlist == NULL) {
+        waitlist = pcmk__intkey_table(free_waitlist_node);
+    }
+
+    wl = calloc(sizeof(struct waitlist_node), 1);
+
+    CRM_ASSERT(wl != NULL);
+
+    wl->client_id = strdup(request->ipc_client->id);
+
+    CRM_ASSERT(wl->client_id);
+
+    if (pcmk__str_eq(sync_point, PCMK__VALUE_LOCAL, pcmk__str_none)) {
+        wl->sync_point = attrd_sync_point_local;
+    } else if (pcmk__str_eq(sync_point, PCMK__VALUE_CLUSTER, pcmk__str_none)) {
+        wl->sync_point = attrd_sync_point_cluster;
+    } else {
+        free_waitlist_node(wl);
+        return;
+    }
+
+    wl->ipc_id = request->ipc_id;
+    wl->flags = request->flags;
+
+    next_key();
+    pcmk__intkey_table_insert(waitlist, waitlist_client, wl);
+
+    crm_trace("Added client %s to waitlist for %s sync point",
+              wl->client_id, sync_point_str(wl->sync_point));
+    crm_trace("%d clients now on waitlist", g_hash_table_size(waitlist));
+
+    /* And then add the key to the request XML so we can uniquely identify
+     * it when it comes time to issue the ACK.
+     */
+    crm_xml_add_int(request->xml, XML_LRM_ATTR_CALLID, waitlist_client);
+}
+
+/*!
+ * \internal
+ * \brief Free all memory associated with the waitlist.  This is most typically
+ *        used when attrd shuts down.
+ */
+void
+attrd_free_waitlist(void)
+{
+    if (waitlist == NULL) {
+        return;
+    }
+
+    g_hash_table_destroy(waitlist);
+    waitlist = NULL;
+}
+
+/*!
+ * \internal
+ * \brief Unconditionally remove a client from the waitlist, such as when the client
+ *        node disconnects from the cluster
+ *
+ * \param[in] client    The client to remove
+ */
+void
+attrd_remove_client_from_waitlist(pcmk__client_t *client)
+{
+    GHashTableIter iter;
+    gpointer value;
+
+    if (waitlist == NULL) {
+        return;
+    }
+
+    g_hash_table_iter_init(&iter, waitlist);
+
+    while (g_hash_table_iter_next(&iter, NULL, &value)) {
+        struct waitlist_node *wl = (struct waitlist_node *) value;
+
+        if (pcmk__str_eq(wl->client_id, client->id, pcmk__str_none)) {
+            g_hash_table_iter_remove(&iter);
+            crm_trace("%d clients now on waitlist", g_hash_table_size(waitlist));
+        }
+    }
+}
+
+/*!
+ * \internal
+ * \brief Send an IPC ACK message to all awaiting clients
+ *
+ * This function will search the waitlist for all clients that are currently awaiting
+ * an ACK indicating their attrd operation is complete.  Only those clients with a
+ * matching sync point type and callid from their original XML IPC request will be
+ * ACKed.  Once they have received an ACK, they will be removed from the waitlist.
+ *
+ * \param[in] sync_point What kind of sync point have we hit?
+ * \param[in] xml        The original XML IPC request.
+ */
+void
+attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml)
+{
+    int callid;
+    gpointer value;
+
+    if (waitlist == NULL) {
+        return;
+    }
+
+    if (crm_element_value_int(xml, XML_LRM_ATTR_CALLID, &callid) == -1) {
+        crm_warn("Could not get callid from request XML");
+        return;
+    }
+
+    value = pcmk__intkey_table_lookup(waitlist, callid);
+    if (value != NULL) {
+        struct waitlist_node *wl = (struct waitlist_node *) value;
+        pcmk__client_t *client = NULL;
+
+        if (wl->sync_point != sync_point) {
+            return;
+        }
+
+        crm_notice("Alerting client %s for reached %s sync point",
+                   wl->client_id, sync_point_str(wl->sync_point));
+
+        client = pcmk__find_client_by_id(wl->client_id);
+        if (client == NULL) {
+            return;
+        }
+
+        attrd_send_ack(client, wl->ipc_id, wl->flags | crm_ipc_client_response);
+
+        /* And then remove the client so it doesn't get alerted again. */
+        pcmk__intkey_table_remove(waitlist, callid);
+
+        crm_trace("%d clients now on waitlist", g_hash_table_size(waitlist));
+    }
+}
+
+/*!
+ * \internal
+ * \brief Action to take when a cluster sync point is hit for a
+ *        PCMK__ATTRD_CMD_UPDATE* message.
+ *
+ * \param[in] xml  The request that should be passed along to
+ *                 attrd_ack_waitlist_clients.  This should be the original
+ *                 IPC request containing the callid for this update message.
+ */
+int
+attrd_cluster_sync_point_update(xmlNode *xml)
+{
+    crm_trace("Hit cluster sync point for attribute update");
+    attrd_ack_waitlist_clients(attrd_sync_point_cluster, xml);
+    return pcmk_rc_ok;
+}
+
+/*!
+ * \internal
+ * \brief Return the sync point attribute for an IPC request
+ *
+ * This function will check both the top-level element of \p xml for a sync
+ * point attribute, as well as all of its \p op children, if any.  The latter
+ * is useful for newer versions of attrd that can put multiple IPC requests
+ * into a single message.
+ *
+ * \param[in] xml   An XML IPC request
+ *
+ * \note It is assumed that if one child element has a sync point attribute,
+ *       all will have a sync point attribute and they will all be the same
+ *       sync point.  No other configuration is supported.
+ *
+ * \return The sync point attribute of \p xml, or NULL if none.
+ */
+const char *
+attrd_request_sync_point(xmlNode *xml)
+{
+    if (xml_has_children(xml)) {
+        xmlNode *child = pcmk__xe_match(xml, XML_ATTR_OP, PCMK__XA_ATTR_SYNC_POINT, NULL);
+
+        if (child) {
+            return crm_element_value(child, PCMK__XA_ATTR_SYNC_POINT);
+        } else {
+            return NULL;
+        }
+
+    } else {
+        return crm_element_value(xml, PCMK__XA_ATTR_SYNC_POINT);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Does an IPC request contain any sync point attribute?
+ *
+ * \param[in] xml   An XML IPC request
+ *
+ * \return true if there's a sync point attribute, false otherwise
+ */
+bool
+attrd_request_has_sync_point(xmlNode *xml)
+{
+    return attrd_request_sync_point(xml) != NULL;
+}
+
+static void
+free_action(gpointer data)
+{
+    struct confirmation_action *action = (struct confirmation_action *) data;
+    g_list_free_full(action->respondents, free);
+    mainloop_timer_del(action->timer);
+    free_xml(action->xml);
+    free(action->client_id);
+    free(action);
+}
+
+/* Remove an IPC request from the expected_confirmations table if the peer attrds
+ * don't respond before the timeout is hit.  We set the timeout to 15s.  The exact
+ * number isn't critical - we just want to make sure that the table eventually gets
+ * cleared of things that didn't complete.
+ */
+static gboolean
+confirmation_timeout_cb(gpointer data)
+{
+    struct confirmation_action *action = (struct confirmation_action *) data;
+
+    GHashTableIter iter;
+    gpointer value;
+
+    if (expected_confirmations == NULL) {
+        return G_SOURCE_REMOVE;
+    }
+
+    g_hash_table_iter_init(&iter, expected_confirmations);
+
+    while (g_hash_table_iter_next(&iter, NULL, &value)) {
+        if (value == action) {
+            pcmk__client_t *client = pcmk__find_client_by_id(action->client_id);
+            if (client == NULL) {
+                return G_SOURCE_REMOVE;
+            }
+
+            crm_trace("Timed out waiting for confirmations for client %s", client->id);
+            pcmk__ipc_send_ack(client, action->ipc_id, action->flags | crm_ipc_client_response,
+                               "ack", ATTRD_PROTOCOL_VERSION, CRM_EX_TIMEOUT);
+
+            g_hash_table_iter_remove(&iter);
+            crm_trace("%d requests now in expected confirmations table", g_hash_table_size(expected_confirmations));
+            break;
+        }
+    }
+
+    return G_SOURCE_REMOVE;
+}
+
+/*!
+ * \internal
+ * \brief When a peer disconnects from the cluster, no longer wait for its confirmation
+ *        for any IPC action.  If this peer is the last one being waited on, this will
+ *        trigger the confirmation action.
+ *
+ * \param[in] host   The disconnecting peer attrd's uname
+ */
+void
+attrd_do_not_expect_from_peer(const char *host)
+{
+    GList *keys = NULL;
+
+    if (expected_confirmations == NULL) {
+        return;
+    }
+
+    keys = g_hash_table_get_keys(expected_confirmations);
+
+    crm_trace("Removing peer %s from expected confirmations", host);
+
+    for (GList *node = keys; node != NULL; node = node->next) {
+        int callid = *(int *) node->data;
+        attrd_handle_confirmation(callid, host);
+    }
+
+    g_list_free(keys);
+}
+
+/*!
+ * \internal
+ * \brief When a client disconnects from the cluster, no longer wait on confirmations
+ *        for it.  Because the peer attrds may still be processing the original IPC
+ *        message, they may still send us confirmations.  However, we will take no
+ *        action on them.
+ *
+ * \param[in] client    The disconnecting client
+ */
+void
+attrd_do_not_wait_for_client(pcmk__client_t *client)
+{
+    GHashTableIter iter;
+    gpointer value;
+
+    if (expected_confirmations == NULL) {
+        return;
+    }
+
+    g_hash_table_iter_init(&iter, expected_confirmations);
+
+    while (g_hash_table_iter_next(&iter, NULL, &value)) {
+        struct confirmation_action *action = (struct confirmation_action *) value;
+
+        if (pcmk__str_eq(action->client_id, client->id, pcmk__str_none)) {
+            crm_trace("Removing client %s from expected confirmations", client->id);
+            g_hash_table_iter_remove(&iter);
+            crm_trace("%d requests now in expected confirmations table", g_hash_table_size(expected_confirmations));
+            break;
+        }
+    }
+}
+
+/*!
+ * \internal
+ * \brief Register some action to be taken when IPC request confirmations are
+ *        received
+ *
+ * When this function is called, a list of all peer attrds that support confirming
+ * requests is generated.  As confirmations from these peer attrds are received,
+ * they are removed from this list.  When the list is empty, the registered action
+ * will be called.
+ *
+ * \note This function should always be called before attrd_send_message is called
+ *       to broadcast to the peers to ensure that we know what replies we are
+ *       waiting on.  Otherwise, it is possible the peer could finish and confirm
+ *       before we know to expect it.
+ *
+ * \param[in] request The request that is awaiting confirmations
+ * \param[in] fn      A function to be run after all confirmations are received
+ */
+void
+attrd_expect_confirmations(pcmk__request_t *request, attrd_confirmation_action_fn fn)
+{
+    struct confirmation_action *action = NULL;
+    GHashTableIter iter;
+    gpointer host, ver;
+    GList *respondents = NULL;
+    int callid;
+
+    if (expected_confirmations == NULL) {
+        expected_confirmations = pcmk__intkey_table((GDestroyNotify) free_action);
+    }
+
+    if (crm_element_value_int(request->xml, XML_LRM_ATTR_CALLID, &callid) == -1) {
+        crm_err("Could not get callid from xml");
+        return;
+    }
+
+    if (pcmk__intkey_table_lookup(expected_confirmations, callid)) {
+        crm_err("Already waiting on confirmations for call id %d", callid);
+        return;
+    }
+
+    g_hash_table_iter_init(&iter, peer_protocol_vers);
+    while (g_hash_table_iter_next(&iter, &host, &ver)) {
+        if (ATTRD_SUPPORTS_CONFIRMATION(GPOINTER_TO_INT(ver))) {
+            char *s = strdup((char *) host);
+
+            CRM_ASSERT(s != NULL);
+            respondents = g_list_prepend(respondents, s);
+        }
+    }
+
+    action = calloc(1, sizeof(struct confirmation_action));
+    CRM_ASSERT(action != NULL);
+
+    action->respondents = respondents;
+    action->fn = fn;
+    action->xml = copy_xml(request->xml);
+
+    action->client_id = strdup(request->ipc_client->id);
+    CRM_ASSERT(action->client_id != NULL);
+
+    action->ipc_id = request->ipc_id;
+    action->flags = request->flags;
+
+    action->timer = mainloop_timer_add(NULL, 15000, FALSE, confirmation_timeout_cb, action);
+    mainloop_timer_start(action->timer);
+
+    pcmk__intkey_table_insert(expected_confirmations, callid, action);
+    crm_trace("Callid %d now waiting on %d confirmations", callid, g_list_length(respondents));
+    crm_trace("%d requests now in expected confirmations table", g_hash_table_size(expected_confirmations));
+}
+
+void
+attrd_free_confirmations(void)
+{
+    if (expected_confirmations != NULL) {
+        g_hash_table_destroy(expected_confirmations);
+        expected_confirmations = NULL;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Process a confirmation message from a peer attrd
+ *
+ * This function is called every time a PCMK__ATTRD_CMD_CONFIRM message is
+ * received from a peer attrd.  If this is the last confirmation we are waiting
+ * on for a given operation, the registered action will be called.
+ *
+ * \param[in] callid The unique callid for the XML IPC request
+ * \param[in] host   The confirming peer attrd's uname
+ */
+void
+attrd_handle_confirmation(int callid, const char *host)
+{
+    struct confirmation_action *action = NULL;
+    GList *node = NULL;
+
+    if (expected_confirmations == NULL) {
+        return;
+    }
+
+    action = pcmk__intkey_table_lookup(expected_confirmations, callid);
+    if (action == NULL) {
+        return;
+    }
+
+    node = g_list_find_custom(action->respondents, host, (GCompareFunc) strcasecmp);
+
+    if (node == NULL) {
+        return;
+    }
+
+    action->respondents = g_list_remove(action->respondents, node->data);
+    crm_trace("Callid %d now waiting on %d confirmations", callid, g_list_length(action->respondents));
+
+    if (action->respondents == NULL) {
+        action->fn(action->xml);
+        pcmk__intkey_table_remove(expected_confirmations, callid);
+        crm_trace("%d requests now in expected confirmations table", g_hash_table_size(expected_confirmations));
+    }
+}
diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c
new file mode 100644
index 0000000..7de8dd9
--- /dev/null
+++ b/daemons/attrd/attrd_utils.c
@@ -0,0 +1,362 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <glib.h>
+#include <regex.h>
+#include <sys/types.h>
+
+#include <crm/crm.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/common/mainloop.h>
+#include <crm/msg_xml.h>
+
+#include "pacemaker-attrd.h"
+
+cib_t *the_cib = NULL;
+
+static bool requesting_shutdown = false;
+static bool shutting_down = false;
+static GMainLoop *mloop = NULL;
+
+/* A hash table storing information on the protocol version of each peer attrd.
+ * The key is the peer's uname, and the value is the protocol version number.
+ */
+GHashTable *peer_protocol_vers = NULL;
+
+/*!
+ * \internal
+ * \brief  Set requesting_shutdown state
+ */
+void
+attrd_set_requesting_shutdown(void)
+{
+    requesting_shutdown = true;
+}
+
+/*!
+ * \internal
+ * \brief  Clear requesting_shutdown state
+ */
+void
+attrd_clear_requesting_shutdown(void)
+{
+    requesting_shutdown = false;
+}
+
+/*!
+ * \internal
+ * \brief Check whether we're currently requesting shutdown
+ *
+ * \return true if requesting shutdown, false otherwise
+ */
+bool
+attrd_requesting_shutdown(void)
+{
+    return requesting_shutdown;
+}
+
+/*!
+ * \internal
+ * \brief Check whether we're currently shutting down
+ *
+ * \return true if shutting down, false otherwise
+ */
+bool
+attrd_shutting_down(void)
+{
+    return shutting_down;
+}
+
+/*!
+ * \internal
+ * \brief  Exit (using mainloop or not, as appropriate)
+ *
+ * \param[in] nsig  Ignored
+ */
+void
+attrd_shutdown(int nsig)
+{
+    // Tell various functions not to do anthing
+    shutting_down = true;
+
+    // Don't respond to signals while shutting down
+    mainloop_destroy_signal(SIGTERM);
+    mainloop_destroy_signal(SIGCHLD);
+    mainloop_destroy_signal(SIGPIPE);
+    mainloop_destroy_signal(SIGUSR1);
+    mainloop_destroy_signal(SIGUSR2);
+    mainloop_destroy_signal(SIGTRAP);
+
+    attrd_free_waitlist();
+    attrd_free_confirmations();
+
+    if (peer_protocol_vers != NULL) {
+        g_hash_table_destroy(peer_protocol_vers);
+        peer_protocol_vers = NULL;
+    }
+
+    if ((mloop == NULL) || !g_main_loop_is_running(mloop)) {
+        /* If there's no main loop active, just exit. This should be possible
+         * only if we get SIGTERM in brief windows at start-up and shutdown.
+         */
+        crm_exit(CRM_EX_OK);
+    } else {
+        g_main_loop_quit(mloop);
+        g_main_loop_unref(mloop);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Create a main loop for attrd
+ */
+void
+attrd_init_mainloop(void)
+{
+    mloop = g_main_loop_new(NULL, FALSE);
+}
+
+/*!
+ * \internal
+ * \brief Run attrd main loop
+ */
+void
+attrd_run_mainloop(void)
+{
+    g_main_loop_run(mloop);
+}
+
+void
+attrd_cib_disconnect(void)
+{
+    CRM_CHECK(the_cib != NULL, return);
+    the_cib->cmds->del_notify_callback(the_cib, T_CIB_REPLACE_NOTIFY, attrd_cib_replaced_cb);
+    the_cib->cmds->del_notify_callback(the_cib, T_CIB_DIFF_NOTIFY, attrd_cib_updated_cb);
+    cib__clean_up_connection(&the_cib);
+}
+
+void
+attrd_cib_replaced_cb(const char *event, xmlNode * msg)
+{
+    int change_section = cib_change_section_nodes | cib_change_section_status | cib_change_section_alerts;
+
+    if (attrd_requesting_shutdown() || attrd_shutting_down()) {
+        return;
+    }
+
+    crm_element_value_int(msg, F_CIB_CHANGE_SECTION, &change_section);
+
+    if (attrd_election_won()) {
+        if (change_section & (cib_change_section_nodes | cib_change_section_status)) {
+            crm_notice("Updating all attributes after %s event", event);
+            attrd_write_attributes(true, false);
+        }
+    }
+
+    if (change_section & cib_change_section_alerts) {
+        // Check for changes in alerts
+        mainloop_set_trigger(attrd_config_read);
+    }
+}
+
+/* strlen("value") */
+#define plus_plus_len (5)
+
+/*!
+ * \internal
+ * \brief  Check whether an attribute value should be expanded
+ *
+ * \param[in] value  Attribute value to check
+ *
+ * \return true if value needs expansion, false otherwise
+ */
+bool
+attrd_value_needs_expansion(const char *value)
+{
+    return ((strlen(value) >= (plus_plus_len + 2))
+           && (value[plus_plus_len] == '+')
+           && ((value[plus_plus_len + 1] == '+')
+               || (value[plus_plus_len + 1] == '=')));
+}
+
+/*!
+ * \internal
+ * \brief Expand an increment expression into an integer
+ *
+ * \param[in] value      Attribute increment expression to expand
+ * \param[in] old_value  Previous value of attribute
+ *
+ * \return Expanded value
+ */
+int
+attrd_expand_value(const char *value, const char *old_value)
+{
+    int offset = 1;
+    int int_value = char2score(old_value);
+
+    if (value[plus_plus_len + 1] != '+') {
+        const char *offset_s = value + (plus_plus_len + 2);
+
+        offset = char2score(offset_s);
+    }
+    int_value += offset;
+
+    if (int_value > INFINITY) {
+        int_value = INFINITY;
+    }
+    return int_value;
+}
+
+/*!
+ * \internal
+ * \brief Create regular expression matching failure-related attributes
+ *
+ * \param[out] regex  Where to store created regular expression
+ * \param[in]  rsc    Name of resource to clear (or NULL for all)
+ * \param[in]  op     Operation to clear if rsc is specified (or NULL for all)
+ * \param[in]  interval_ms  Interval of operation to clear if op is specified
+ *
+ * \return pcmk_ok on success, -EINVAL if arguments are invalid
+ *
+ * \note The caller is responsible for freeing the result with regfree().
+ */
+int
+attrd_failure_regex(regex_t *regex, const char *rsc, const char *op,
+                    guint interval_ms)
+{
+    char *pattern = NULL;
+    int rc;
+
+    /* Create a pattern that matches desired attributes */
+
+    if (rsc == NULL) {
+        pattern = strdup(ATTRD_RE_CLEAR_ALL);
+    } else if (op == NULL) {
+        pattern = crm_strdup_printf(ATTRD_RE_CLEAR_ONE, rsc);
+    } else {
+        pattern = crm_strdup_printf(ATTRD_RE_CLEAR_OP, rsc, op, interval_ms);
+    }
+
+    /* Compile pattern into regular expression */
+    crm_trace("Clearing attributes matching %s", pattern);
+    rc = regcomp(regex, pattern, REG_EXTENDED|REG_NOSUB);
+    free(pattern);
+
+    return (rc == 0)? pcmk_ok : -EINVAL;
+}
+
+void
+attrd_free_attribute_value(gpointer data)
+{
+    attribute_value_t *v = data;
+
+    free(v->nodename);
+    free(v->current);
+    free(v->requested);
+    free(v);
+}
+
+void
+attrd_free_attribute(gpointer data)
+{
+    attribute_t *a = data;
+    if(a) {
+        free(a->id);
+        free(a->set_id);
+        free(a->set_type);
+        free(a->uuid);
+        free(a->user);
+
+        mainloop_timer_del(a->timer);
+        g_hash_table_destroy(a->values);
+
+        free(a);
+    }
+}
+
+/*!
+ * \internal
+ * \brief When a peer node leaves the cluster, stop tracking its protocol version.
+ *
+ * \param[in] host  The peer node's uname to be removed
+ */
+void
+attrd_remove_peer_protocol_ver(const char *host)
+{
+    if (peer_protocol_vers != NULL) {
+        g_hash_table_remove(peer_protocol_vers, host);
+    }
+}
+
+/*!
+ * \internal
+ * \brief When a peer node broadcasts a message with its protocol version, keep
+ *        track of that information.
+ *
+ * We keep track of each peer's protocol version so we know which peers to
+ * expect confirmation messages from when handling cluster-wide sync points.
+ * We additionally keep track of the lowest protocol version supported by all
+ * peers so we know when we can send IPC messages containing more than one
+ * request.
+ *
+ * \param[in] host  The peer node's uname to be tracked
+ * \param[in] value The peer node's protocol version
+ */
+void
+attrd_update_minimum_protocol_ver(const char *host, const char *value)
+{
+    int ver;
+
+    if (peer_protocol_vers == NULL) {
+        peer_protocol_vers = pcmk__strkey_table(free, NULL);
+    }
+
+    pcmk__scan_min_int(value, &ver, 0);
+
+    if (ver > 0) {
+        char *host_name = strdup(host);
+
+        /* Record the peer attrd's protocol version. */
+        CRM_ASSERT(host_name != NULL);
+        g_hash_table_insert(peer_protocol_vers, host_name, GINT_TO_POINTER(ver));
+
+        /* If the protocol version is a new minimum, record it as such. */
+        if (minimum_protocol_version == -1 || ver < minimum_protocol_version) {
+            minimum_protocol_version = ver;
+            crm_trace("Set minimum attrd protocol version to %d",
+                      minimum_protocol_version);
+        }
+    }
+}
+
+void
+attrd_copy_xml_attributes(xmlNode *src, xmlNode *dest)
+{
+    /* Copy attributes from the wrapper parent node into the child node.
+     * We can't just use copy_in_properties because we want to skip any
+     * attributes that are already set on the child.  For instance, if
+     * we were told to use a specific node, there will already be a node
+     * attribute on the child.  Copying the parent's node attribute over
+     * could result in the wrong value.
+     */
+    for (xmlAttrPtr a = pcmk__xe_first_attr(src); a != NULL; a = a->next) {
+        const char *p_name = (const char *) a->name;
+        const char *p_value = ((a == NULL) || (a->children == NULL)) ? NULL :
+                              (const char *) a->children->content;
+
+        if (crm_element_value(dest, p_name) == NULL) {
+            crm_xml_add(dest, p_name, p_value);
+        }
+    }
+}
diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c
new file mode 100644
index 0000000..037825b
--- /dev/null
+++ b/daemons/attrd/pacemaker-attrd.c
@@ -0,0 +1,358 @@
+/*
+ * Copyright 2013-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+
+#include <crm/crm.h>
+#include <crm/cib/internal.h>
+#include <crm/msg_xml.h>
+#include <crm/pengine/rules.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/iso8601.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/common/output_internal.h>
+#include <crm/common/xml.h>
+#include <crm/cluster/internal.h>
+
+#include <crm/common/attrd_internal.h>
+#include "pacemaker-attrd.h"
+
+#define SUMMARY "daemon for managing Pacemaker node attributes"
+
+gboolean stand_alone = FALSE;
+gchar **log_files = NULL;
+
+static GOptionEntry entries[] = {
+    { "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &stand_alone,
+      "(Advanced use only) Run in stand-alone mode", NULL },
+
+    { "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY,
+      &log_files, "Send logs to the additional named logfile", NULL },
+
+    { NULL }
+};
+
+static pcmk__output_t *out = NULL;
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+lrmd_t *the_lrmd = NULL;
+crm_cluster_t *attrd_cluster = NULL;
+crm_trigger_t *attrd_config_read = NULL;
+crm_exit_t attrd_exit_status = CRM_EX_OK;
+
+static void
+attrd_cib_destroy_cb(gpointer user_data)
+{
+    cib_t *conn = user_data;
+
+    conn->cmds->signoff(conn);  /* Ensure IPC is cleaned up */
+
+    if (attrd_shutting_down()) {
+        crm_info("Connection disconnection complete");
+
+    } else {
+        /* eventually this should trigger a reconnect, not a shutdown */
+        crm_crit("Lost connection to the CIB manager, shutting down");
+        attrd_exit_status = CRM_EX_DISCONNECT;
+        attrd_shutdown(0);
+    }
+
+    return;
+}
+
+static void
+attrd_erase_cb(xmlNode *msg, int call_id, int rc, xmlNode *output,
+               void *user_data)
+{
+    do_crm_log_unlikely((rc? LOG_NOTICE : LOG_DEBUG),
+                        "Cleared transient attributes: %s "
+                        CRM_XS " xpath=%s rc=%d",
+                        pcmk_strerror(rc), (char *) user_data, rc);
+}
+
+#define XPATH_TRANSIENT "//node_state[@uname='%s']/" XML_TAG_TRANSIENT_NODEATTRS
+
+/*!
+ * \internal
+ * \brief Wipe all transient attributes for this node from the CIB
+ *
+ * Clear any previous transient node attributes from the CIB. This is
+ * normally done by the DC's controller when this node leaves the cluster, but
+ * this handles the case where the node restarted so quickly that the
+ * cluster layer didn't notice.
+ *
+ * \todo If pacemaker-attrd respawns after crashing (see PCMK_respawned),
+ *       ideally we'd skip this and sync our attributes from the writer.
+ *       However, currently we reject any values for us that the writer has, in
+ *       attrd_peer_update().
+ */
+static void
+attrd_erase_attrs(void)
+{
+    int call_id;
+    char *xpath = crm_strdup_printf(XPATH_TRANSIENT, attrd_cluster->uname);
+
+    crm_info("Clearing transient attributes from CIB " CRM_XS " xpath=%s",
+             xpath);
+
+    call_id = the_cib->cmds->remove(the_cib, xpath, NULL, cib_xpath);
+    the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE, xpath,
+                                          "attrd_erase_cb", attrd_erase_cb,
+                                          free);
+}
+
+static int
+attrd_cib_connect(int max_retry)
+{
+    static int attempts = 0;
+
+    int rc = -ENOTCONN;
+
+    the_cib = cib_new();
+    if (the_cib == NULL) {
+        return -ENOTCONN;
+    }
+
+    do {
+        if(attempts > 0) {
+            sleep(attempts);
+        }
+
+        attempts++;
+        crm_debug("Connection attempt %d to the CIB manager", attempts);
+        rc = the_cib->cmds->signon(the_cib, T_ATTRD, cib_command);
+
+    } while(rc != pcmk_ok && attempts < max_retry);
+
+    if (rc != pcmk_ok) {
+        crm_err("Connection to the CIB manager failed: %s " CRM_XS " rc=%d",
+                pcmk_strerror(rc), rc);
+        goto cleanup;
+    }
+
+    crm_debug("Connected to the CIB manager after %d attempts", attempts);
+
+    rc = the_cib->cmds->set_connection_dnotify(the_cib, attrd_cib_destroy_cb);
+    if (rc != pcmk_ok) {
+        crm_err("Could not set disconnection callback");
+        goto cleanup;
+    }
+
+    rc = the_cib->cmds->add_notify_callback(the_cib, T_CIB_REPLACE_NOTIFY, attrd_cib_replaced_cb);
+    if(rc != pcmk_ok) {
+        crm_err("Could not set CIB notification callback");
+        goto cleanup;
+    }
+
+    rc = the_cib->cmds->add_notify_callback(the_cib, T_CIB_DIFF_NOTIFY, attrd_cib_updated_cb);
+    if (rc != pcmk_ok) {
+        crm_err("Could not set CIB notification callback (update)");
+        goto cleanup;
+    }
+
+    return pcmk_ok;
+
+  cleanup:
+    cib__clean_up_connection(&the_cib);
+    return -ENOTCONN;
+}
+
+/*!
+ * \internal
+ * \brief Prepare the CIB after cluster is connected
+ */
+static void
+attrd_cib_init(void)
+{
+    // We have no attribute values in memory, wipe the CIB to match
+    attrd_erase_attrs();
+
+    // Set a trigger for reading the CIB (for the alerts section)
+    attrd_config_read = mainloop_add_trigger(G_PRIORITY_HIGH, attrd_read_options, NULL);
+
+    // Always read the CIB at start-up
+    mainloop_set_trigger(attrd_config_read);
+}
+
+static bool
+ipc_already_running(void)
+{
+    pcmk_ipc_api_t *old_instance = NULL;
+    int rc = pcmk_rc_ok;
+
+    rc = pcmk_new_ipc_api(&old_instance, pcmk_ipc_attrd);
+    if (rc != pcmk_rc_ok) {
+        return false;
+    }
+
+    rc = pcmk_connect_ipc(old_instance, pcmk_ipc_dispatch_sync);
+    if (rc != pcmk_rc_ok) {
+        pcmk_free_ipc_api(old_instance);
+        return false;
+    }
+
+    pcmk_disconnect_ipc(old_instance);
+    pcmk_free_ipc_api(old_instance);
+    return true;
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
+    pcmk__add_main_args(context, entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    int rc = pcmk_rc_ok;
+
+    GError *error = NULL;
+    bool initialized = false;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, NULL);
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    attrd_init_mainloop();
+    crm_log_preinit(NULL, argc, argv);
+    mainloop_add_signal(SIGTERM, attrd_shutdown);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        attrd_exit_status = CRM_EX_USAGE;
+        goto done;
+    }
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if ((rc != pcmk_rc_ok) || (out == NULL)) {
+        attrd_exit_status = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, attrd_exit_status,
+                    "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    // Open additional log files
+    pcmk__add_logfiles(log_files, out);
+
+    crm_log_init(T_ATTRD, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
+    crm_notice("Starting Pacemaker node attribute manager%s",
+               stand_alone ? " in standalone mode" : "");
+
+    if (ipc_already_running()) {
+        const char *msg = "pacemaker-attrd is already active, aborting startup";
+
+        attrd_exit_status = CRM_EX_OK;
+        g_set_error(&error, PCMK__EXITC_ERROR, attrd_exit_status, "%s", msg);
+        crm_err(msg);
+        goto done;
+    }
+
+    initialized = true;
+
+    attributes = pcmk__strkey_table(NULL, attrd_free_attribute);
+
+    /* Connect to the CIB before connecting to the cluster or listening for IPC.
+     * This allows us to assume the CIB is connected whenever we process a
+     * cluster or IPC message (which also avoids start-up race conditions).
+     */
+    if (!stand_alone) {
+        if (attrd_cib_connect(30) != pcmk_ok) {
+            attrd_exit_status = CRM_EX_FATAL;
+            g_set_error(&error, PCMK__EXITC_ERROR, attrd_exit_status,
+                        "Could not connect to the CIB");
+            goto done;
+        }
+        crm_info("CIB connection active");
+    }
+
+    if (attrd_cluster_connect() != pcmk_ok) {
+        attrd_exit_status = CRM_EX_FATAL;
+        g_set_error(&error, PCMK__EXITC_ERROR, attrd_exit_status,
+                    "Could not connect to the cluster");
+        goto done;
+    }
+    crm_info("Cluster connection active");
+
+    // Initialization that requires the cluster to be connected
+    attrd_election_init();
+
+    if (!stand_alone) {
+        attrd_cib_init();
+    }
+
+    /* Set a private attribute for ourselves with the protocol version we
+     * support. This lets all nodes determine the minimum supported version
+     * across all nodes. It also ensures that the writer learns our node name,
+     * so it can send our attributes to the CIB.
+     */
+    attrd_broadcast_protocol();
+
+    attrd_init_ipc();
+    crm_notice("Pacemaker node attribute manager successfully started and accepting connections");
+    attrd_run_mainloop();
+
+  done:
+    if (initialized) {
+        crm_info("Shutting down attribute manager");
+
+        attrd_election_fini();
+        attrd_ipc_fini();
+        attrd_lrmd_disconnect();
+
+        if (!stand_alone) {
+            attrd_cib_disconnect();
+        }
+
+        attrd_free_waitlist();
+        pcmk_cluster_free(attrd_cluster);
+        g_hash_table_destroy(attributes);
+    }
+
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    g_strfreev(log_files);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    if (out != NULL) {
+        out->finish(out, attrd_exit_status, true, NULL);
+        pcmk__output_free(out);
+    }
+    pcmk__unregister_formats();
+    crm_exit(attrd_exit_status);
+}
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
new file mode 100644
index 0000000..329fb5a
--- /dev/null
+++ b/daemons/attrd/pacemaker-attrd.h
@@ -0,0 +1,216 @@
+/*
+ * Copyright 2013-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef PACEMAKER_ATTRD__H
+#  define PACEMAKER_ATTRD__H
+
+#include <regex.h>
+#include <glib.h>
+#include <crm/crm.h>
+#include <crm/cluster.h>
+#include <crm/cluster/election_internal.h>
+#include <crm/common/messages_internal.h>
+#include <crm/cib/internal.h>
+
+/*
+ * Legacy attrd (all pre-1.1.11 Pacemaker versions, plus all versions when used
+ * with the no-longer-supported CMAN or corosync-plugin stacks) is unversioned.
+ *
+ * With atomic attrd, each attrd will send ATTRD_PROTOCOL_VERSION with every
+ * peer request and reply. As of Pacemaker 2.0.0, at start-up each attrd will
+ * also set a private attribute for itself with its version, so any attrd can
+ * determine the minimum version supported by all peers.
+ *
+ * Protocol  Pacemaker  Significant changes
+ * --------  ---------  -------------------
+ *     1       1.1.11   PCMK__ATTRD_CMD_UPDATE (PCMK__XA_ATTR_NAME only),
+ *                      PCMK__ATTRD_CMD_PEER_REMOVE, PCMK__ATTRD_CMD_REFRESH,
+ *                      PCMK__ATTRD_CMD_FLUSH, PCMK__ATTRD_CMD_SYNC,
+ *                      PCMK__ATTRD_CMD_SYNC_RESPONSE
+ *     1       1.1.13   PCMK__ATTRD_CMD_UPDATE (with PCMK__XA_ATTR_PATTERN),
+ *                      PCMK__ATTRD_CMD_QUERY
+ *     1       1.1.15   PCMK__ATTRD_CMD_UPDATE_BOTH,
+ *                      PCMK__ATTRD_CMD_UPDATE_DELAY
+ *     2       1.1.17   PCMK__ATTRD_CMD_CLEAR_FAILURE
+ *     3       2.1.1    PCMK__ATTRD_CMD_SYNC_RESPONSE indicates remote nodes
+ *     4       2.1.5    Multiple attributes can be updated in a single IPC
+ *                      message
+ *     5       2.1.5    Peers can request confirmation of a sent message
+ */
+#define ATTRD_PROTOCOL_VERSION "5"
+
+#define ATTRD_SUPPORTS_MULTI_MESSAGE(x) ((x) >= 4)
+#define ATTRD_SUPPORTS_CONFIRMATION(x)  ((x) >= 5)
+
+#define attrd_send_ack(client, id, flags) \
+    pcmk__ipc_send_ack((client), (id), (flags), "ack", ATTRD_PROTOCOL_VERSION, CRM_EX_INDETERMINATE)
+
+void attrd_init_mainloop(void);
+void attrd_run_mainloop(void);
+
+void attrd_set_requesting_shutdown(void);
+void attrd_clear_requesting_shutdown(void);
+void attrd_free_waitlist(void);
+bool attrd_requesting_shutdown(void);
+bool attrd_shutting_down(void);
+void attrd_shutdown(int nsig);
+void attrd_init_ipc(void);
+void attrd_ipc_fini(void);
+
+void attrd_cib_disconnect(void);
+
+bool attrd_value_needs_expansion(const char *value);
+int attrd_expand_value(const char *value, const char *old_value);
+
+/* regular expression to clear failures of all resources */
+#define ATTRD_RE_CLEAR_ALL \
+    "^(" PCMK__FAIL_COUNT_PREFIX "|" PCMK__LAST_FAILURE_PREFIX ")-"
+
+/* regular expression to clear failure of all operations for one resource
+ * (format takes resource name)
+ *
+ * @COMPAT attributes set < 1.1.17:
+ * also match older attributes that do not have the operation part
+ */
+#define ATTRD_RE_CLEAR_ONE ATTRD_RE_CLEAR_ALL "%s(#.+_[0-9]+)?$"
+
+/* regular expression to clear failure of one operation for one resource
+ * (format takes resource name, operation name, and interval)
+ *
+ * @COMPAT attributes set < 1.1.17:
+ * also match older attributes that do not have the operation part
+ */
+#define ATTRD_RE_CLEAR_OP ATTRD_RE_CLEAR_ALL "%s(#%s_%u)?$"
+
+int attrd_failure_regex(regex_t *regex, const char *rsc, const char *op,
+                        guint interval_ms);
+
+extern cib_t *the_cib;
+
+/* Alerts */
+
+extern lrmd_t *the_lrmd;
+extern crm_trigger_t *attrd_config_read;
+
+void attrd_lrmd_disconnect(void);
+gboolean attrd_read_options(gpointer user_data);
+void attrd_cib_replaced_cb(const char *event, xmlNode * msg);
+void attrd_cib_updated_cb(const char *event, xmlNode *msg);
+int attrd_send_attribute_alert(const char *node, int nodeid,
+                               const char *attr, const char *value);
+
+// Elections
+void attrd_election_init(void);
+void attrd_election_fini(void);
+void attrd_start_election_if_needed(void);
+bool attrd_election_won(void);
+void attrd_handle_election_op(const crm_node_t *peer, xmlNode *xml);
+bool attrd_check_for_new_writer(const crm_node_t *peer, const xmlNode *xml);
+void attrd_declare_winner(void);
+void attrd_remove_voter(const crm_node_t *peer);
+void attrd_xml_add_writer(xmlNode *xml);
+
+typedef struct attribute_s {
+    char *uuid; /* TODO: Remove if at all possible */
+    char *id;
+    char *set_id;
+    char *set_type;
+    GHashTable *values;
+    int update;
+    int timeout_ms;
+
+    /* TODO: refactor these three as a bitmask */
+    bool changed; /* whether attribute value has changed since last write */
+    bool unknown_peer_uuids; /* whether we know we're missing a peer uuid */
+    gboolean is_private; /* whether to keep this attribute out of the CIB */
+
+    mainloop_timer_t *timer;
+
+    char *user;
+
+    gboolean force_write; /* Flag for updating attribute by ignoring delay */
+
+} attribute_t;
+
+typedef struct attribute_value_s {
+        uint32_t nodeid;
+        gboolean is_remote;
+        char *nodename;
+        char *current;
+        char *requested;
+        gboolean seen;
+} attribute_value_t;
+
+extern crm_cluster_t *attrd_cluster;
+extern GHashTable *attributes;
+extern GHashTable *peer_protocol_vers;
+
+#define CIB_OP_TIMEOUT_S 120
+
+int attrd_cluster_connect(void);
+void attrd_peer_update(const crm_node_t *peer, xmlNode *xml, const char *host,
+                       bool filter);
+void attrd_peer_sync(crm_node_t *peer, xmlNode *xml);
+void attrd_peer_remove(const char *host, bool uncache, const char *source);
+void attrd_peer_clear_failure(pcmk__request_t *request);
+void attrd_peer_sync_response(const crm_node_t *peer, bool peer_won,
+                              xmlNode *xml);
+
+void attrd_broadcast_protocol(void);
+xmlNode *attrd_client_peer_remove(pcmk__request_t *request);
+xmlNode *attrd_client_clear_failure(pcmk__request_t *request);
+xmlNode *attrd_client_update(pcmk__request_t *request);
+xmlNode *attrd_client_refresh(pcmk__request_t *request);
+xmlNode *attrd_client_query(pcmk__request_t *request);
+gboolean attrd_send_message(crm_node_t *node, xmlNode *data, bool confirm);
+
+xmlNode *attrd_add_value_xml(xmlNode *parent, const attribute_t *a,
+                             const attribute_value_t *v, bool force_write);
+void attrd_clear_value_seen(void);
+void attrd_free_attribute(gpointer data);
+void attrd_free_attribute_value(gpointer data);
+attribute_t *attrd_populate_attribute(xmlNode *xml, const char *attr);
+
+void attrd_write_attribute(attribute_t *a, bool ignore_delay);
+void attrd_write_attributes(bool all, bool ignore_delay);
+void attrd_write_or_elect_attribute(attribute_t *a);
+
+extern int minimum_protocol_version;
+void attrd_remove_peer_protocol_ver(const char *host);
+void attrd_update_minimum_protocol_ver(const char *host, const char *value);
+
+mainloop_timer_t *attrd_add_timer(const char *id, int timeout_ms, attribute_t *attr);
+
+void attrd_unregister_handlers(void);
+void attrd_handle_request(pcmk__request_t *request);
+
+enum attrd_sync_point {
+    attrd_sync_point_local,
+    attrd_sync_point_cluster,
+};
+
+typedef int (*attrd_confirmation_action_fn)(xmlNode *);
+
+void attrd_add_client_to_waitlist(pcmk__request_t *request);
+void attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml);
+int attrd_cluster_sync_point_update(xmlNode *xml);
+void attrd_do_not_expect_from_peer(const char *host);
+void attrd_do_not_wait_for_client(pcmk__client_t *client);
+void attrd_expect_confirmations(pcmk__request_t *request, attrd_confirmation_action_fn fn);
+void attrd_free_confirmations(void);
+void attrd_handle_confirmation(int callid, const char *host);
+void attrd_remove_client_from_waitlist(pcmk__client_t *client);
+const char *attrd_request_sync_point(xmlNode *xml);
+bool attrd_request_has_sync_point(xmlNode *xml);
+
+void attrd_copy_xml_attributes(xmlNode *src, xmlNode *dest);
+
+extern gboolean stand_alone;
+
+#endif /* PACEMAKER_ATTRD__H */
diff --git a/daemons/based/Makefile.am b/daemons/based/Makefile.am
new file mode 100644
index 0000000..053d93c
--- /dev/null
+++ b/daemons/based/Makefile.am
@@ -0,0 +1,47 @@
+#
+# Copyright 2004-2021 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+include $(top_srcdir)/mk/common.mk
+
+EXTRA_DIST	= cib.pam
+
+halibdir	= $(CRM_DAEMON_DIR)
+
+COMMONLIBS	= $(top_builddir)/lib/common/libcrmcommon.la \
+		$(top_builddir)/lib/cib/libcib.la
+
+halib_PROGRAMS	= pacemaker-based
+
+noinst_HEADERS	= pacemaker-based.h
+
+pacemaker_based_CFLAGS	= $(CFLAGS_HARDENED_EXE)
+pacemaker_based_LDFLAGS	= $(LDFLAGS_HARDENED_EXE)
+
+pacemaker_based_LDADD	= $(top_builddir)/lib/cluster/libcrmcluster.la \
+			  $(COMMONLIBS) $(CLUSTERLIBS)
+
+pacemaker_based_SOURCES	= pacemaker-based.c \
+			  based_callbacks.c \
+			  based_common.c \
+			  based_io.c \
+			  based_messages.c \
+			  based_notify.c \
+			  based_remote.c
+
+clean-generic:
+	rm -f *.log *.debug *.xml *~
+
+if BUILD_LEGACY_LINKS
+install-exec-hook:
+	$(MKDIR_P) -- $(DESTDIR)$(CRM_DAEMON_DIR)
+	cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f cib && $(LN_S) pacemaker-based cib
+
+uninstall-hook:
+	cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f cib
+endif
diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c
new file mode 100644
index 0000000..3726caa
--- /dev/null
+++ b/daemons/based/based_callbacks.c
@@ -0,0 +1,1696 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <stdlib.h>
+#include <stdint.h>     // uint32_t, uint64_t, UINT64_C()
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>   // PRIu64
+
+#include <crm/crm.h>
+#include <crm/cib.h>
+#include <crm/msg_xml.h>
+#include <crm/cluster/internal.h>
+
+#include <crm/common/xml.h>
+#include <crm/common/remote_internal.h>
+
+#include <pacemaker-based.h>
+
+#define EXIT_ESCALATION_MS 10000
+#define OUR_NODENAME (stand_alone? "localhost" : crm_cluster->uname)
+
+static unsigned long cib_local_bcast_num = 0;
+
+typedef struct cib_local_notify_s {
+    xmlNode *notify_src;
+    char *client_id;
+    gboolean from_peer;
+    gboolean sync_reply;
+} cib_local_notify_t;
+
+int next_client_id = 0;
+
+gboolean legacy_mode = FALSE;
+
+qb_ipcs_service_t *ipcs_ro = NULL;
+qb_ipcs_service_t *ipcs_rw = NULL;
+qb_ipcs_service_t *ipcs_shm = NULL;
+
+static void cib_process_request(xmlNode *request, gboolean privileged,
+                                const pcmk__client_t *cib_client);
+
+static int cib_process_command(xmlNode *request, xmlNode **reply,
+                               xmlNode **cib_diff, gboolean privileged);
+
+static gboolean cib_common_callback(qb_ipcs_connection_t *c, void *data,
+                                    size_t size, gboolean privileged);
+
+gboolean
+cib_legacy_mode(void)
+{
+    return legacy_mode;
+}
+
+static int32_t
+cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
+{
+    if (cib_shutdown_flag) {
+        crm_info("Ignoring new IPC client [%d] during shutdown",
+                 pcmk__client_pid(c));
+        return -EPERM;
+    }
+
+    if (pcmk__new_client(c, uid, gid) == NULL) {
+        return -EIO;
+    }
+    return 0;
+}
+
+static int32_t
+cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
+{
+    pcmk__client_t *client = pcmk__find_client(c);
+
+    crm_trace("%p message from %s", c, client->id);
+    return cib_common_callback(c, data, size, TRUE);
+}
+
+static int32_t
+cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
+{
+    pcmk__client_t *client = pcmk__find_client(c);
+
+    crm_trace("%p message from %s", c, client->id);
+    return cib_common_callback(c, data, size, FALSE);
+}
+
+/* Error code means? */
+static int32_t
+cib_ipc_closed(qb_ipcs_connection_t * c)
+{
+    pcmk__client_t *client = pcmk__find_client(c);
+
+    if (client == NULL) {
+        return 0;
+    }
+    crm_trace("Connection %p", c);
+    pcmk__free_client(client);
+    return 0;
+}
+
+static void
+cib_ipc_destroy(qb_ipcs_connection_t * c)
+{
+    crm_trace("Connection %p", c);
+    cib_ipc_closed(c);
+    if (cib_shutdown_flag) {
+        cib_shutdown(0);
+    }
+}
+
+struct qb_ipcs_service_handlers ipc_ro_callbacks = {
+    .connection_accept = cib_ipc_accept,
+    .connection_created = NULL,
+    .msg_process = cib_ipc_dispatch_ro,
+    .connection_closed = cib_ipc_closed,
+    .connection_destroyed = cib_ipc_destroy
+};
+
+struct qb_ipcs_service_handlers ipc_rw_callbacks = {
+    .connection_accept = cib_ipc_accept,
+    .connection_created = NULL,
+    .msg_process = cib_ipc_dispatch_rw,
+    .connection_closed = cib_ipc_closed,
+    .connection_destroyed = cib_ipc_destroy
+};
+
+void
+cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
+                           pcmk__client_t *cib_client, gboolean privileged)
+{
+    const char *op = crm_element_value(op_request, F_CIB_OPERATION);
+
+    if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
+        if (flags & crm_ipc_client_response) {
+            xmlNode *ack = create_xml_node(NULL, __func__);
+
+            crm_xml_add(ack, F_CIB_OPERATION, CRM_OP_REGISTER);
+            crm_xml_add(ack, F_CIB_CLIENTID, cib_client->id);
+            pcmk__ipc_send_xml(cib_client, id, ack, flags);
+            cib_client->request_id = 0;
+            free_xml(ack);
+        }
+        return;
+
+    } else if (pcmk__str_eq(op, T_CIB_NOTIFY, pcmk__str_none)) {
+        /* Update the notify filters for this client */
+        int on_off = 0;
+        crm_exit_t status = CRM_EX_OK;
+        uint64_t bit = UINT64_C(0);
+        const char *type = crm_element_value(op_request, F_CIB_NOTIFY_TYPE);
+
+        crm_element_value_int(op_request, F_CIB_NOTIFY_ACTIVATE, &on_off);
+
+        crm_debug("Setting %s callbacks %s for client %s",
+                  type, (on_off? "on" : "off"), pcmk__client_name(cib_client));
+
+        if (pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) {
+            bit = cib_notify_post;
+
+        } else if (pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) {
+            bit = cib_notify_pre;
+
+        } else if (pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) {
+            bit = cib_notify_confirm;
+
+        } else if (pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) {
+            bit = cib_notify_diff;
+
+        } else if (pcmk__str_eq(type, T_CIB_REPLACE_NOTIFY, pcmk__str_casei)) {
+            bit = cib_notify_replace;
+
+        } else {
+            status = CRM_EX_INVALID_PARAM;
+        }
+
+        if (bit != 0) {
+            if (on_off) {
+                pcmk__set_client_flags(cib_client, bit);
+            } else {
+                pcmk__clear_client_flags(cib_client, bit);
+            }
+        }
+
+        pcmk__ipc_send_ack(cib_client, id, flags, "ack", NULL, status);
+        return;
+    }
+
+    cib_process_request(op_request, privileged, cib_client);
+}
+
+int32_t
+cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
+{
+    uint32_t id = 0;
+    uint32_t flags = 0;
+    int call_options = 0;
+    pcmk__client_t *cib_client = pcmk__find_client(c);
+    xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags);
+
+    if (op_request) {
+        crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options);
+    }
+
+    if (op_request == NULL) {
+        crm_trace("Invalid message from %p", c);
+        pcmk__ipc_send_ack(cib_client, id, flags, "nack", NULL, CRM_EX_PROTOCOL);
+        return 0;
+
+    } else if(cib_client == NULL) {
+        crm_trace("Invalid client %p", c);
+        return 0;
+    }
+
+    if (pcmk_is_set(call_options, cib_sync_call)) {
+        CRM_LOG_ASSERT(flags & crm_ipc_client_response);
+        CRM_LOG_ASSERT(cib_client->request_id == 0);    /* This means the client has two synchronous events in-flight */
+        cib_client->request_id = id;    /* Reply only to the last one */
+    }
+
+    if (cib_client->name == NULL) {
+        const char *value = crm_element_value(op_request, F_CIB_CLIENTNAME);
+
+        if (value == NULL) {
+            cib_client->name = pcmk__itoa(cib_client->pid);
+        } else {
+            cib_client->name = strdup(value);
+            if (crm_is_daemon_name(value)) {
+                pcmk__set_client_flags(cib_client, cib_is_daemon);
+            }
+        }
+    }
+
+    /* Allow cluster daemons more leeway before being evicted */
+    if (pcmk_is_set(cib_client->flags, cib_is_daemon)) {
+        const char *qmax = cib_config_lookup("cluster-ipc-limit");
+
+        if (pcmk__set_client_queue_max(cib_client, qmax)) {
+            crm_trace("IPC threshold for client %s[%u] is now %u",
+                      pcmk__client_name(cib_client), cib_client->pid,
+                      cib_client->queue_max);
+        }
+    }
+
+    crm_xml_add(op_request, F_CIB_CLIENTID, cib_client->id);
+    crm_xml_add(op_request, F_CIB_CLIENTNAME, cib_client->name);
+
+    CRM_LOG_ASSERT(cib_client->user != NULL);
+    pcmk__update_acl_user(op_request, F_CIB_USER, cib_client->user);
+
+    cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
+    free_xml(op_request);
+
+    return 0;
+}
+
+static uint64_t ping_seq = 0;
+static char *ping_digest = NULL;
+static bool ping_modified_since = FALSE;
+
+static gboolean
+cib_digester_cb(gpointer data)
+{
+    if (based_is_primary) {
+        char buffer[32];
+        xmlNode *ping = create_xml_node(NULL, "ping");
+
+        ping_seq++;
+        free(ping_digest);
+        ping_digest = NULL;
+        ping_modified_since = FALSE;
+        snprintf(buffer, 32, "%" PRIu64, ping_seq);
+        crm_trace("Requesting peer digests (%s)", buffer);
+
+        crm_xml_add(ping, F_TYPE, "cib");
+        crm_xml_add(ping, F_CIB_OPERATION, CRM_OP_PING);
+        crm_xml_add(ping, F_CIB_PING_ID, buffer);
+
+        crm_xml_add(ping, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
+        send_cluster_message(NULL, crm_msg_cib, ping, TRUE);
+
+        free_xml(ping);
+    }
+    return FALSE;
+}
+
+static void
+process_ping_reply(xmlNode *reply) 
+{
+    uint64_t seq = 0;
+    const char *host = crm_element_value(reply, F_ORIG);
+
+    xmlNode *pong = get_message_xml(reply, F_CIB_CALLDATA);
+    const char *seq_s = crm_element_value(pong, F_CIB_PING_ID);
+    const char *digest = crm_element_value(pong, XML_ATTR_DIGEST);
+
+    if (seq_s == NULL) {
+        crm_debug("Ignoring ping reply with no " F_CIB_PING_ID);
+        return;
+
+    } else {
+        long long seq_ll;
+
+        if (pcmk__scan_ll(seq_s, &seq_ll, 0LL) != pcmk_rc_ok) {
+            return;
+        }
+        seq = (uint64_t) seq_ll;
+    }
+
+    if(digest == NULL) {
+        crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);
+
+    } else if(seq != ping_seq) {
+        crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);
+
+    } else if(ping_modified_since) {
+        crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);
+
+    } else {
+        const char *version = crm_element_value(pong, XML_ATTR_CRM_VERSION);
+
+        if(ping_digest == NULL) {
+            crm_trace("Calculating new digest");
+            ping_digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, version);
+        }
+
+        crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
+        if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) {
+            xmlNode *remote_cib = get_message_xml(pong, F_CIB_CALLDATA);
+
+            crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
+                       crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN),
+                       crm_element_value(the_cib, XML_ATTR_GENERATION),
+                       crm_element_value(the_cib, XML_ATTR_NUMUPDATES),
+                       ping_digest, host,
+                       remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION_ADMIN):"_",
+                       remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION):"_",
+                       remote_cib?crm_element_value(remote_cib, XML_ATTR_NUMUPDATES):"_",
+                       digest, remote_cib);
+
+            if(remote_cib && remote_cib->children) {
+                // Additional debug
+                xml_calculate_changes(the_cib, remote_cib);
+
+                pcmk__output_set_log_level(logger_out, LOG_INFO);
+                pcmk__xml_show_changes(logger_out, remote_cib);
+                crm_trace("End of differences");
+            }
+
+            free_xml(remote_cib);
+            sync_our_cib(reply, FALSE);
+        }
+    }
+}
+
+static void
+do_local_notify(xmlNode * notify_src, const char *client_id,
+                gboolean sync_reply, gboolean from_peer)
+{
+    int rid = 0;
+    int call_id = 0;
+    pcmk__client_t *client_obj = NULL;
+
+    CRM_ASSERT(notify_src && client_id);
+
+    crm_element_value_int(notify_src, F_CIB_CALLID, &call_id);
+
+    client_obj = pcmk__find_client_by_id(client_id);
+    if (client_obj == NULL) {
+        crm_debug("Could not send response %d: client %s not found",
+                  call_id, client_id);
+        return;
+    }
+
+    if (sync_reply) {
+        if (client_obj->ipcs) {
+            CRM_LOG_ASSERT(client_obj->request_id);
+
+            rid = client_obj->request_id;
+            client_obj->request_id = 0;
+
+            crm_trace("Sending response %d to client %s%s",
+                      rid, pcmk__client_name(client_obj),
+                      (from_peer? " (originator of delegated request)" : ""));
+        } else {
+            crm_trace("Sending response (call %d) to client %s%s",
+                      call_id, pcmk__client_name(client_obj),
+                      (from_peer? " (originator of delegated request)" : ""));
+        }
+
+    } else {
+        crm_trace("Sending event %d to client %s%s",
+                  call_id, pcmk__client_name(client_obj),
+                  (from_peer? " (originator of delegated request)" : ""));
+    }
+
+    switch (PCMK__CLIENT_TYPE(client_obj)) {
+        case pcmk__client_ipc:
+            {
+                int rc = pcmk__ipc_send_xml(client_obj, rid, notify_src,
+                                            (sync_reply? crm_ipc_flags_none
+                                             : crm_ipc_server_event));
+
+                if (rc != pcmk_rc_ok) {
+                    crm_warn("%s reply to client %s failed: %s " CRM_XS " rc=%d",
+                             (sync_reply? "Synchronous" : "Asynchronous"),
+                             pcmk__client_name(client_obj), pcmk_rc_str(rc),
+                             rc);
+                }
+            }
+            break;
+#ifdef HAVE_GNUTLS_GNUTLS_H
+        case pcmk__client_tls:
+#endif
+        case pcmk__client_tcp:
+            pcmk__remote_send_xml(client_obj->remote, notify_src);
+            break;
+        default:
+            crm_err("Unknown transport for client %s "
+                    CRM_XS " flags=%#016" PRIx64,
+                    pcmk__client_name(client_obj), client_obj->flags);
+    }
+}
+
+static void
+local_notify_destroy_callback(gpointer data)
+{
+    cib_local_notify_t *notify = data;
+
+    free_xml(notify->notify_src);
+    free(notify->client_id);
+    free(notify);
+}
+
+static void
+check_local_notify(int bcast_id)
+{
+    cib_local_notify_t *notify = NULL;
+
+    if (!local_notify_queue) {
+        return;
+    }
+
+    notify = pcmk__intkey_table_lookup(local_notify_queue, bcast_id);
+
+    if (notify) {
+        do_local_notify(notify->notify_src, notify->client_id, notify->sync_reply,
+                        notify->from_peer);
+        pcmk__intkey_table_remove(local_notify_queue, bcast_id);
+    }
+}
+
+static void
+queue_local_notify(xmlNode * notify_src, const char *client_id, gboolean sync_reply,
+                   gboolean from_peer)
+{
+    cib_local_notify_t *notify = calloc(1, sizeof(cib_local_notify_t));
+
+    notify->notify_src = notify_src;
+    notify->client_id = strdup(client_id);
+    notify->sync_reply = sync_reply;
+    notify->from_peer = from_peer;
+
+    if (!local_notify_queue) {
+        local_notify_queue = pcmk__intkey_table(local_notify_destroy_callback);
+    }
+    pcmk__intkey_table_insert(local_notify_queue, cib_local_bcast_num, notify);
+    // cppcheck doesn't know notify will get freed when hash table is destroyed
+    // cppcheck-suppress memleak
+}
+
+static void
+parse_local_options_v1(const pcmk__client_t *cib_client, int call_type,
+                       int call_options, const char *host, const char *op,
+                       gboolean *local_notify, gboolean *needs_reply,
+                       gboolean *process, gboolean *needs_forward)
+{
+    if (cib_op_modifies(call_type)
+        && !(call_options & cib_inhibit_bcast)) {
+        /* we need to send an update anyway */
+        *needs_reply = TRUE;
+    } else {
+        *needs_reply = FALSE;
+    }
+
+    if (host == NULL && (call_options & cib_scope_local)) {
+        crm_trace("Processing locally scoped %s op from client %s",
+                  op, pcmk__client_name(cib_client));
+        *local_notify = TRUE;
+
+    } else if ((host == NULL) && based_is_primary) {
+        crm_trace("Processing %s op locally from client %s as primary",
+                  op, pcmk__client_name(cib_client));
+        *local_notify = TRUE;
+
+    } else if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
+        crm_trace("Processing locally addressed %s op from client %s",
+                  op, pcmk__client_name(cib_client));
+        *local_notify = TRUE;
+
+    } else if (stand_alone) {
+        *needs_forward = FALSE;
+        *local_notify = TRUE;
+        *process = TRUE;
+
+    } else {
+        crm_trace("%s op from %s needs to be forwarded to client %s",
+                  op, pcmk__client_name(cib_client),
+                  pcmk__s(host, "the primary instance"));
+        *needs_forward = TRUE;
+        *process = FALSE;
+    }
+}
+
+static void
+parse_local_options_v2(const pcmk__client_t *cib_client, int call_type,
+                       int call_options, const char *host, const char *op,
+                       gboolean *local_notify, gboolean *needs_reply,
+                       gboolean *process, gboolean *needs_forward)
+{
+    if (cib_op_modifies(call_type)) {
+        if (pcmk__str_any_of(op, PCMK__CIB_REQUEST_PRIMARY,
+                             PCMK__CIB_REQUEST_SECONDARY, NULL)) {
+            /* Always handle these locally */
+            *process = TRUE;
+            *needs_reply = FALSE;
+            *local_notify = TRUE;
+            *needs_forward = FALSE;
+            return;
+
+        } else {
+            /* Redirect all other updates via CPG */
+            *needs_reply = TRUE;
+            *needs_forward = TRUE;
+            *process = FALSE;
+            crm_trace("%s op from %s needs to be forwarded to client %s",
+                      op, pcmk__client_name(cib_client),
+                      pcmk__s(host, "the primary instance"));
+            return;
+        }
+    }
+
+
+    *process = TRUE;
+    *needs_reply = FALSE;
+    *local_notify = TRUE;
+    *needs_forward = FALSE;
+
+    if (stand_alone) {
+        crm_trace("Processing %s op from client %s (stand-alone)",
+                  op, pcmk__client_name(cib_client));
+
+    } else if (host == NULL) {
+        crm_trace("Processing unaddressed %s op from client %s",
+                  op, pcmk__client_name(cib_client));
+
+    } else if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
+        crm_trace("Processing locally addressed %s op from client %s",
+                  op, pcmk__client_name(cib_client));
+
+    } else {
+        crm_trace("%s op from %s needs to be forwarded to client %s",
+                  op, pcmk__client_name(cib_client), host);
+        *needs_forward = TRUE;
+        *process = FALSE;
+    }
+}
+
+static void
+parse_local_options(const pcmk__client_t *cib_client, int call_type,
+                    int call_options, const char *host, const char *op,
+                    gboolean *local_notify, gboolean *needs_reply,
+                    gboolean *process, gboolean *needs_forward)
+{
+    if(cib_legacy_mode()) {
+        parse_local_options_v1(cib_client, call_type, call_options, host,
+                               op, local_notify, needs_reply, process, needs_forward);
+    } else {
+        parse_local_options_v2(cib_client, call_type, call_options, host,
+                               op, local_notify, needs_reply, process, needs_forward);
+    }
+}
+
+static gboolean
+parse_peer_options_v1(int call_type, xmlNode * request,
+                   gboolean * local_notify, gboolean * needs_reply, gboolean * process,
+                   gboolean * needs_forward)
+{
+    const char *op = NULL;
+    const char *host = NULL;
+    const char *delegated = NULL;
+    const char *originator = crm_element_value(request, F_ORIG);
+    const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
+
+    gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei);
+
+    if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
+        *needs_reply = FALSE;
+        if (is_reply) {
+            *local_notify = TRUE;
+            crm_trace("Processing global/peer update from %s"
+                      " that originated from us", originator);
+        } else {
+            crm_trace("Processing global/peer update from %s", originator);
+        }
+        return TRUE;
+    }
+
+    op = crm_element_value(request, F_CIB_OPERATION);
+    crm_trace("Processing %s request sent by %s", op, originator);
+    if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
+        /* Always process these */
+        *local_notify = FALSE;
+        if (reply_to == NULL || is_reply) {
+            *process = TRUE;
+        }
+        if (is_reply) {
+            *needs_reply = FALSE;
+        }
+        return *process;
+    }
+
+    if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
+        process_ping_reply(request);
+        return FALSE;
+    }
+
+    if (is_reply) {
+        crm_trace("Forward reply sent from %s to local clients", originator);
+        *process = FALSE;
+        *needs_reply = FALSE;
+        *local_notify = TRUE;
+        return TRUE;
+    }
+
+    host = crm_element_value(request, F_CIB_HOST);
+    if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
+        crm_trace("Processing %s request sent to us from %s", op, originator);
+        return TRUE;
+
+    } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
+        crm_trace("Processing %s request sent to %s by %s", op, host?host:"everyone", originator);
+        *needs_reply = TRUE;
+        return TRUE;
+
+    } else if ((host == NULL) && based_is_primary) {
+        crm_trace("Processing %s request sent to primary instance from %s",
+                  op, originator);
+        return TRUE;
+    }
+
+    delegated = crm_element_value(request, F_CIB_DELEGATED);
+    if (delegated != NULL) {
+        crm_trace("Ignoring message for primary instance");
+
+    } else if (host != NULL) {
+        /* this is for a specific instance and we're not it */
+        crm_trace("Ignoring msg for instance on %s", host);
+
+    } else if ((reply_to == NULL) && !based_is_primary) {
+        // This is for the primary instance, and we're not it
+        crm_trace("Ignoring reply for primary instance");
+
+    } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
+        if (reply_to != NULL) {
+            crm_debug("Processing %s from %s", op, originator);
+            *needs_reply = FALSE;
+
+        } else {
+            crm_debug("Processing %s reply from %s", op, originator);
+        }
+        return TRUE;
+
+    } else {
+        crm_err("Nothing for us to do?");
+        crm_log_xml_err(request, "Peer[inbound]");
+    }
+
+    return FALSE;
+}
+
+static gboolean
+parse_peer_options_v2(int call_type, xmlNode * request,
+                   gboolean * local_notify, gboolean * needs_reply, gboolean * process,
+                   gboolean * needs_forward)
+{
+    const char *host = NULL;
+    const char *delegated = crm_element_value(request, F_CIB_DELEGATED);
+    const char *op = crm_element_value(request, F_CIB_OPERATION);
+    const char *originator = crm_element_value(request, F_ORIG);
+    const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
+
+    gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei);
+
+    if (pcmk__str_eq(op, PCMK__CIB_REQUEST_REPLACE, pcmk__str_none)) {
+        /* sync_our_cib() sets F_CIB_ISREPLY */
+        if (reply_to) {
+            delegated = reply_to;
+        }
+        goto skip_is_reply;
+
+    } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SYNC_TO_ALL,
+                            pcmk__str_none)) {
+        // Nothing to do
+
+    } else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
+        process_ping_reply(request);
+        return FALSE;
+
+    } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_UPGRADE, pcmk__str_none)) {
+        /* Only the DC (node with the oldest software) should process
+         * this operation if F_CIB_SCHEMA_MAX is unset
+         *
+         * If the DC is happy it will then send out another
+         * PCMK__CIB_REQUEST_UPGRADE which will tell all nodes to do the actual
+         * upgrade.
+         *
+         * Except this time F_CIB_SCHEMA_MAX will be set which puts a
+         * limit on how far newer nodes will go
+         */
+        const char *max = crm_element_value(request, F_CIB_SCHEMA_MAX);
+        const char *upgrade_rc = crm_element_value(request, F_CIB_UPGRADE_RC);
+
+        crm_trace("Parsing %s operation%s for %s with max=%s and upgrade_rc=%s",
+                  op, (is_reply? " reply" : ""),
+                  (based_is_primary? "primary" : "secondary"),
+                  (max? max : "none"), (upgrade_rc? upgrade_rc : "none"));
+
+        if (upgrade_rc != NULL) {
+            // Our upgrade request was rejected by DC, notify clients of result
+            crm_xml_add(request, F_CIB_RC, upgrade_rc);
+
+        } else if ((max == NULL) && based_is_primary) {
+            /* We are the DC, check if this upgrade is allowed */
+            goto skip_is_reply;
+
+        } else if(max) {
+            /* Ok, go ahead and upgrade to 'max' */
+            goto skip_is_reply;
+
+        } else {
+            // Ignore broadcast client requests when we're not DC
+            return FALSE;
+        }
+
+    } else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
+        crm_info("Detected legacy %s global update from %s", op, originator);
+        send_sync_request(NULL);
+        legacy_mode = TRUE;
+        return FALSE;
+
+    } else if (is_reply && cib_op_modifies(call_type)) {
+        crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
+        return FALSE;
+
+    } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
+        /* Legacy handling */
+        crm_debug("Legacy handling of %s message from %s", op, originator);
+        *local_notify = FALSE;
+        if (reply_to == NULL) {
+            *process = TRUE;
+        }
+        return *process;
+    }
+
+    if(is_reply) {
+        crm_trace("Handling %s reply sent from %s to local clients", op, originator);
+        *process = FALSE;
+        *needs_reply = FALSE;
+        *local_notify = TRUE;
+        return TRUE;
+    }
+
+  skip_is_reply:
+    *process = TRUE;
+    *needs_reply = FALSE;
+
+    *local_notify = pcmk__str_eq(delegated, OUR_NODENAME, pcmk__str_casei);
+
+    host = crm_element_value(request, F_CIB_HOST);
+    if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
+        crm_trace("Processing %s request sent to us from %s", op, originator);
+        *needs_reply = TRUE;
+        return TRUE;
+
+    } else if (host != NULL) {
+        /* this is for a specific instance and we're not it */
+        crm_trace("Ignoring %s operation for instance on %s", op, host);
+        return FALSE;
+
+    } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
+        *needs_reply = TRUE;
+    }
+
+    crm_trace("Processing %s request sent to everyone by %s/%s on %s %s", op,
+              crm_element_value(request, F_CIB_CLIENTNAME),
+              crm_element_value(request, F_CIB_CALLID),
+              originator, (*local_notify)?"(notify)":"");
+    return TRUE;
+}
+
+static gboolean
+parse_peer_options(int call_type, xmlNode * request,
+                   gboolean * local_notify, gboolean * needs_reply, gboolean * process,
+                   gboolean * needs_forward)
+{
+    /* TODO: What happens when an update comes in after node A
+     * requests the CIB from node B, but before it gets the reply (and
+     * sends out the replace operation)
+     */
+    if(cib_legacy_mode()) {
+        return parse_peer_options_v1(
+            call_type, request, local_notify, needs_reply, process, needs_forward);
+    } else {
+        return parse_peer_options_v2(
+            call_type, request, local_notify, needs_reply, process, needs_forward);
+    }
+}
+
+static void
+forward_request(xmlNode *request, int call_options)
+{
+    const char *op = crm_element_value(request, F_CIB_OPERATION);
+    const char *host = crm_element_value(request, F_CIB_HOST);
+
+    crm_xml_add(request, F_CIB_DELEGATED, OUR_NODENAME);
+
+    if (host != NULL) {
+        crm_trace("Forwarding %s op to %s", op, host);
+        send_cluster_message(crm_get_peer(0, host), crm_msg_cib, request, FALSE);
+
+    } else {
+        crm_trace("Forwarding %s op to primary instance", op);
+        send_cluster_message(NULL, crm_msg_cib, request, FALSE);
+    }
+
+    /* Return the request to its original state */
+    xml_remove_prop(request, F_CIB_DELEGATED);
+
+    if (call_options & cib_discard_reply) {
+        crm_trace("Client not interested in reply");
+    }
+}
+
+static gboolean
+send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
+{
+    CRM_ASSERT(msg != NULL);
+
+    if (broadcast) {
+        /* this (successful) call modified the CIB _and_ the
+         * change needs to be broadcast...
+         *   send via HA to other nodes
+         */
+        int diff_add_updates = 0;
+        int diff_add_epoch = 0;
+        int diff_add_admin_epoch = 0;
+
+        int diff_del_updates = 0;
+        int diff_del_epoch = 0;
+        int diff_del_admin_epoch = 0;
+
+        const char *digest = NULL;
+        int format = 1;
+
+        CRM_LOG_ASSERT(result_diff != NULL);
+        digest = crm_element_value(result_diff, XML_ATTR_DIGEST);
+        crm_element_value_int(result_diff, "format", &format);
+
+        cib_diff_version_details(result_diff,
+                                 &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
+                                 &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
+
+        crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s",
+                  diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
+                  diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest);
+
+        crm_xml_add(msg, F_CIB_ISREPLY, originator);
+        pcmk__xe_set_bool_attr(msg, F_CIB_GLOBAL_UPDATE, true);
+        crm_xml_add(msg, F_CIB_OPERATION, PCMK__CIB_REQUEST_APPLY_PATCH);
+        crm_xml_add(msg, F_CIB_USER, CRM_DAEMON_USER);
+
+        if (format == 1) {
+            CRM_ASSERT(digest != NULL);
+        }
+
+        add_message_xml(msg, F_CIB_UPDATE_DIFF, result_diff);
+        crm_log_xml_explicit(msg, "copy");
+        return send_cluster_message(NULL, crm_msg_cib, msg, TRUE);
+
+    } else if (originator != NULL) {
+        /* send reply via HA to originating node */
+        crm_trace("Sending request result to %s only", originator);
+        crm_xml_add(msg, F_CIB_ISREPLY, originator);
+        return send_cluster_message(crm_get_peer(0, originator), crm_msg_cib, msg, FALSE);
+    }
+
+    return FALSE;
+}
+
+/*!
+ * \internal
+ * \brief Handle an IPC or CPG message containing a request
+ *
+ * \param[in,out] request        Request XML
+ * \param[in] privileged         Whether privileged commands may be run
+ *                               (see cib_server_ops[] definition)
+ * \param[in] cib_client         IPC client that sent request (or NULL if CPG)
+ */
+static void
+cib_process_request(xmlNode *request, gboolean privileged,
+                    const pcmk__client_t *cib_client)
+{
+    int call_type = 0;
+    int call_options = 0;
+
+    gboolean process = TRUE;        // Whether to process request locally now
+    gboolean is_update = TRUE;      // Whether request would modify CIB
+    gboolean needs_reply = TRUE;    // Whether to build a reply
+    gboolean local_notify = FALSE;  // Whether to notify (local) requester
+    gboolean needs_forward = FALSE; // Whether to forward request somewhere else
+
+    xmlNode *op_reply = NULL;
+    xmlNode *result_diff = NULL;
+
+    int rc = pcmk_ok;
+    const char *op = crm_element_value(request, F_CIB_OPERATION);
+    const char *originator = crm_element_value(request, F_ORIG);
+    const char *host = crm_element_value(request, F_CIB_HOST);
+    const char *target = NULL;
+    const char *call_id = crm_element_value(request, F_CIB_CALLID);
+    const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
+    const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
+    const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
+
+    crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
+
+    if ((host != NULL) && (*host == '\0')) {
+        host = NULL;
+    }
+
+    if (host) {
+        target = host;
+
+    } else if (call_options & cib_scope_local) {
+        target = "local host";
+
+    } else {
+        target = "primary";
+    }
+
+    if (cib_client == NULL) {
+        crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)",
+                  op, client_name, call_id, originator, target, reply_to);
+    } else {
+        crm_xml_add(request, F_ORIG, OUR_NODENAME);
+        crm_trace("Processing local %s operation from %s/%s intended for %s", op, client_name, call_id, target);
+    }
+
+    rc = cib_get_operation_id(op, &call_type);
+    if (rc != pcmk_ok) {
+        /* TODO: construct error reply? */
+        crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
+        return;
+    }
+
+    if (cib_client != NULL) {
+        parse_local_options(cib_client, call_type, call_options, host, op,
+                            &local_notify, &needs_reply, &process, &needs_forward);
+
+    } else if (parse_peer_options(call_type, request, &local_notify,
+                                  &needs_reply, &process, &needs_forward) == FALSE) {
+        return;
+    }
+
+    is_update = cib_op_modifies(call_type);
+
+    if (call_options & cib_discard_reply) {
+        /* If the request will modify the CIB, and we are in legacy mode, we
+         * need to build a reply so we can broadcast a diff, even if the
+         * requester doesn't want one.
+         */
+        needs_reply = is_update && cib_legacy_mode();
+        local_notify = FALSE;
+    }
+
+    if (needs_forward) {
+        const char *section = crm_element_value(request, F_CIB_SECTION);
+        int log_level = LOG_INFO;
+
+        if (pcmk__str_eq(op, PCMK__CIB_REQUEST_NOOP, pcmk__str_none)) {
+            log_level = LOG_DEBUG;
+        }
+
+        do_crm_log(log_level,
+                   "Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
+                   op,
+                   section ? section : "'all'",
+                   pcmk__s(host, (cib_legacy_mode() ? "primary" : "all")),
+                   originator ? originator : "local",
+                   client_name, call_id);
+
+        forward_request(request, call_options);
+        return;
+    }
+
+    if (cib_status != pcmk_ok) {
+        const char *call = crm_element_value(request, F_CIB_CALLID);
+
+        rc = cib_status;
+        crm_err("Operation ignored, cluster configuration is invalid."
+                " Please repair and restart: %s", pcmk_strerror(cib_status));
+
+        op_reply = create_xml_node(NULL, "cib-reply");
+        crm_xml_add(op_reply, F_TYPE, T_CIB);
+        crm_xml_add(op_reply, F_CIB_OPERATION, op);
+        crm_xml_add(op_reply, F_CIB_CALLID, call);
+        crm_xml_add(op_reply, F_CIB_CLIENTID, client_id);
+        crm_xml_add_int(op_reply, F_CIB_CALLOPTS, call_options);
+        crm_xml_add_int(op_reply, F_CIB_RC, rc);
+
+        crm_trace("Attaching reply output");
+        add_message_xml(op_reply, F_CIB_CALLDATA, the_cib);
+
+        crm_log_xml_explicit(op_reply, "cib:reply");
+
+    } else if (process) {
+        time_t finished = 0;
+        time_t now = time(NULL);
+        int level = LOG_INFO;
+        const char *section = crm_element_value(request, F_CIB_SECTION);
+
+        rc = cib_process_command(request, &op_reply, &result_diff, privileged);
+
+        if (!is_update) {
+            level = LOG_TRACE;
+
+        } else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
+            switch (rc) {
+                case pcmk_ok:
+                    level = LOG_INFO;
+                    break;
+                case -pcmk_err_old_data:
+                case -pcmk_err_diff_resync:
+                case -pcmk_err_diff_failed:
+                    level = LOG_TRACE;
+                    break;
+                default:
+                    level = LOG_ERR;
+            }
+
+        } else if (rc != pcmk_ok) {
+            level = LOG_WARNING;
+        }
+
+        do_crm_log(level,
+                   "Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
+                   op, section ? section : "'all'", pcmk_strerror(rc), rc,
+                   originator ? originator : "local", client_name, call_id,
+                   the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN) : "0",
+                   the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION) : "0",
+                   the_cib ? crm_element_value(the_cib, XML_ATTR_NUMUPDATES) : "0");
+
+        finished = time(NULL);
+        if ((finished - now) > 3) {
+            crm_trace("%s operation took %lds to complete", op, (long)(finished - now));
+            crm_write_blackbox(0, NULL);
+        }
+
+        if (op_reply == NULL && (needs_reply || local_notify)) {
+            crm_err("Unexpected NULL reply to message");
+            crm_log_xml_err(request, "null reply");
+            needs_reply = FALSE;
+            local_notify = FALSE;
+        }
+    }
+
+    if (is_update && !cib_legacy_mode()) {
+        crm_trace("Completed pre-sync update from %s/%s/%s%s",
+                  originator ? originator : "local", client_name, call_id,
+                  local_notify?" with local notification":"");
+
+    } else if (!needs_reply || stand_alone) {
+        // This was a non-originating secondary update
+        crm_trace("Completed update as secondary");
+
+    } else if (cib_legacy_mode() &&
+               rc == pcmk_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) {
+        gboolean broadcast = FALSE;
+
+        cib_local_bcast_num++;
+        crm_xml_add_int(request, F_CIB_LOCAL_NOTIFY_ID, cib_local_bcast_num);
+        broadcast = send_peer_reply(request, result_diff, originator, TRUE);
+
+        if (broadcast && client_id && local_notify && op_reply) {
+
+            /* If we have been asked to sync the reply,
+             * and a bcast msg has gone out, we queue the local notify
+             * until we know the bcast message has been received */
+            local_notify = FALSE;
+            crm_trace("Queuing local %ssync notification for %s",
+                      (call_options & cib_sync_call) ? "" : "a-", client_id);
+
+            queue_local_notify(op_reply, client_id,
+                               pcmk_is_set(call_options, cib_sync_call),
+                               (cib_client == NULL));
+            op_reply = NULL;    /* the reply is queued, so don't free here */
+        }
+
+    } else if (call_options & cib_discard_reply) {
+        crm_trace("Caller isn't interested in reply");
+
+    } else if (cib_client == NULL) {
+        if (is_update == FALSE || result_diff == NULL) {
+            crm_trace("Request not broadcast: R/O call");
+
+        } else if (call_options & cib_inhibit_bcast) {
+            crm_trace("Request not broadcast: inhibited");
+
+        } else if (rc != pcmk_ok) {
+            crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));
+
+        } else {
+            crm_trace("Directing reply to %s", originator);
+        }
+
+        send_peer_reply(op_reply, result_diff, originator, FALSE);
+    }
+
+    if (local_notify && client_id) {
+        crm_trace("Performing local %ssync notification for %s",
+                  (pcmk_is_set(call_options, cib_sync_call)? "" : "a"),
+                  client_id);
+        if (process == FALSE) {
+            do_local_notify(request, client_id,
+                            pcmk_is_set(call_options, cib_sync_call),
+                            (cib_client == NULL));
+        } else {
+            do_local_notify(op_reply, client_id,
+                            pcmk_is_set(call_options, cib_sync_call),
+                            (cib_client == NULL));
+        }
+    }
+
+    free_xml(op_reply);
+    free_xml(result_diff);
+
+    return;
+}
+
+static char *
+calculate_section_digest(const char *xpath, xmlNode * xml_obj)
+{
+    xmlNode *xml_section = NULL;
+
+    if (xml_obj == NULL) {
+        return NULL;
+    }
+
+    xml_section = get_xpath_object(xpath, xml_obj, LOG_TRACE);
+    if (xml_section == NULL) {
+        return NULL;
+    }
+    return calculate_xml_versioned_digest(xml_section, FALSE, TRUE, CRM_FEATURE_SET); 
+
+}
+
+// v1 and v2 patch formats
+#define XPATH_CONFIG_CHANGE             \
+    "//" XML_CIB_TAG_CRMCONFIG " | "    \
+    "//" XML_DIFF_CHANGE                \
+    "[contains(@" XML_DIFF_PATH ",'/" XML_CIB_TAG_CRMCONFIG "/')]"
+
+static bool
+contains_config_change(xmlNode *diff)
+{
+    bool changed = false;
+
+    if (diff) {
+        xmlXPathObject *xpathObj = xpath_search(diff, XPATH_CONFIG_CHANGE);
+
+        if (numXpathResults(xpathObj) > 0) {
+            changed = true;
+        }
+        freeXpathObject(xpathObj);
+    }
+    return changed;
+}
+
+static int
+cib_process_command(xmlNode * request, xmlNode ** reply, xmlNode ** cib_diff, gboolean privileged)
+{
+    xmlNode *input = NULL;
+    xmlNode *output = NULL;
+    xmlNode *result_cib = NULL;
+    xmlNode *current_cib = NULL;
+
+    int call_type = 0;
+    int call_options = 0;
+
+    const char *op = NULL;
+    const char *section = NULL;
+    const char *call_id = crm_element_value(request, F_CIB_CALLID);
+    const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
+    const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
+    const char *origin = crm_element_value(request, F_ORIG);
+
+    int rc = pcmk_ok;
+    int rc2 = pcmk_ok;
+
+    gboolean send_r_notify = FALSE;
+    gboolean config_changed = FALSE;
+    gboolean manage_counters = TRUE;
+
+    static mainloop_timer_t *digest_timer = NULL;
+
+    char *current_nodes_digest = NULL;
+    char *current_alerts_digest = NULL;
+    char *current_status_digest = NULL;
+    uint32_t change_section = cib_change_section_nodes
+                              |cib_change_section_alerts
+                              |cib_change_section_status;
+
+    CRM_ASSERT(cib_status == pcmk_ok);
+
+    if(digest_timer == NULL) {
+        digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
+    }
+
+    *reply = NULL;
+    *cib_diff = NULL;
+    current_cib = the_cib;
+
+    /* Start processing the request... */
+    op = crm_element_value(request, F_CIB_OPERATION);
+    crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
+    rc = cib_get_operation_id(op, &call_type);
+
+    if (rc == pcmk_ok && privileged == FALSE) {
+        rc = cib_op_can_run(call_type, call_options, privileged);
+    }
+
+    rc2 = cib_op_prepare(call_type, request, &input, &section);
+    if (rc == pcmk_ok) {
+        rc = rc2;
+    }
+
+    if (rc != pcmk_ok) {
+        crm_trace("Call setup failed: %s", pcmk_strerror(rc));
+        goto done;
+
+    } else if (cib_op_modifies(call_type) == FALSE) {
+        rc = cib_perform_op(op, call_options, cib_op_func(call_type), TRUE,
+                            section, request, input, FALSE, &config_changed,
+                            current_cib, &result_cib, NULL, &output);
+
+        CRM_CHECK(result_cib == NULL, free_xml(result_cib));
+        goto done;
+    }
+
+    /* Handle a valid write action */
+    if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
+        /* legacy code */
+        manage_counters = FALSE;
+        cib__set_call_options(call_options, "call", cib_force_diff);
+        crm_trace("Global update detected");
+
+        CRM_CHECK(call_type == 3 || call_type == 4, crm_err("Call type: %d", call_type);
+                  crm_log_xml_err(request, "bad op"));
+    }
+
+    ping_modified_since = TRUE;
+    if (pcmk_is_set(call_options, cib_inhibit_bcast)) {
+        crm_trace("Skipping update: inhibit broadcast");
+        manage_counters = FALSE;
+    }
+
+    if (!pcmk_is_set(call_options, cib_dryrun)
+        && pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_casei)) {
+        // Copying large CIBs accounts for a huge percentage of our CIB usage
+        cib__set_call_options(call_options, "call", cib_zero_copy);
+    } else {
+        cib__clear_call_options(call_options, "call", cib_zero_copy);
+    }
+
+#define XPATH_CONFIG    "//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION
+#define XPATH_NODES     XPATH_CONFIG "/" XML_CIB_TAG_NODES
+#define XPATH_ALERTS    XPATH_CONFIG "/" XML_CIB_TAG_ALERTS
+#define XPATH_STATUS    "//" XML_TAG_CIB "/" XML_CIB_TAG_STATUS
+
+    // Calculate the hash value of the section before the change
+    if (pcmk__str_eq(PCMK__CIB_REQUEST_REPLACE, op, pcmk__str_none)) {
+        current_nodes_digest = calculate_section_digest(XPATH_NODES,
+                                                        current_cib);
+        current_alerts_digest = calculate_section_digest(XPATH_ALERTS,
+                                                         current_cib);
+        current_status_digest = calculate_section_digest(XPATH_STATUS,
+                                                         current_cib);
+        crm_trace("current-digest %s:%s:%s", current_nodes_digest,
+                  current_alerts_digest, current_status_digest);
+    }
+
+    // result_cib must not be modified after cib_perform_op() returns
+    rc = cib_perform_op(op, call_options, cib_op_func(call_type), FALSE,
+                        section, request, input, manage_counters,
+                        &config_changed, current_cib, &result_cib, cib_diff,
+                        &output);
+
+    if (!manage_counters) {
+        int format = 1;
+
+        /* Legacy code
+         * If the diff is NULL at this point, it's because nothing changed
+         */
+        if (*cib_diff != NULL) {
+            crm_element_value_int(*cib_diff, "format", &format);
+        }
+
+        if (format == 1) {
+            config_changed = cib__config_changed_v1(NULL, NULL, cib_diff);
+        }
+    }
+
+    /* Always write to disk for successful replace and upgrade ops. This also
+     * negates the need to detect ordering changes.
+     */
+    if ((rc == pcmk_ok)
+        && pcmk__str_any_of(op,
+                            PCMK__CIB_REQUEST_REPLACE,
+                            PCMK__CIB_REQUEST_UPGRADE,
+                            NULL)) {
+        config_changed = TRUE;
+    }
+
+    if (rc == pcmk_ok && !pcmk_is_set(call_options, cib_dryrun)) {
+        crm_trace("Activating %s->%s%s%s",
+                  crm_element_value(current_cib, XML_ATTR_NUMUPDATES),
+                  crm_element_value(result_cib, XML_ATTR_NUMUPDATES),
+                  (pcmk_is_set(call_options, cib_zero_copy)? " zero-copy" : ""),
+                  (config_changed? " changed" : ""));
+        if (!pcmk_is_set(call_options, cib_zero_copy)) {
+            rc = activateCibXml(result_cib, config_changed, op);
+            crm_trace("Activated %s (%d)",
+                      crm_element_value(current_cib, XML_ATTR_NUMUPDATES), rc);
+        }
+
+        if ((rc == pcmk_ok) && contains_config_change(*cib_diff)) {
+            cib_read_config(config_hash, result_cib);
+        }
+
+        if (pcmk__str_eq(PCMK__CIB_REQUEST_REPLACE, op, pcmk__str_none)) {
+            char *result_nodes_digest = NULL;
+            char *result_alerts_digest = NULL;
+            char *result_status_digest = NULL;
+
+            /* Calculate the hash value of the changed section. */
+            result_nodes_digest = calculate_section_digest(XPATH_NODES,
+                                                           result_cib);
+            result_alerts_digest = calculate_section_digest(XPATH_ALERTS,
+                                                            result_cib);
+            result_status_digest = calculate_section_digest(XPATH_STATUS,
+                                                            result_cib);
+            crm_trace("result-digest %s:%s:%s", result_nodes_digest,
+                      result_alerts_digest, result_status_digest);
+
+            if (pcmk__str_eq(current_nodes_digest, result_nodes_digest,
+                             pcmk__str_none)) {
+                change_section =
+                    pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,
+                                         "CIB change section",
+                                         "change_section", change_section,
+                                         cib_change_section_nodes, "nodes");
+            }
+
+            if (pcmk__str_eq(current_alerts_digest, result_alerts_digest,
+                             pcmk__str_none)) {
+                change_section =
+                    pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,
+                                         "CIB change section",
+                                         "change_section", change_section,
+                                         cib_change_section_alerts, "alerts");
+            }
+
+            if (pcmk__str_eq(current_status_digest, result_status_digest,
+                             pcmk__str_none)) {
+                change_section =
+                    pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,
+                                         "CIB change section",
+                                         "change_section", change_section,
+                                         cib_change_section_status, "status");
+            }
+
+            if (change_section != cib_change_section_none) {
+                send_r_notify = TRUE;
+            }
+            
+            free(result_nodes_digest);
+            free(result_alerts_digest);
+            free(result_status_digest);
+
+        } else if (pcmk__str_eq(PCMK__CIB_REQUEST_ERASE, op, pcmk__str_none)) {
+            send_r_notify = TRUE;
+        }
+
+        mainloop_timer_stop(digest_timer);
+        mainloop_timer_start(digest_timer);
+
+    } else if (rc == -pcmk_err_schema_validation) {
+        CRM_ASSERT(!pcmk_is_set(call_options, cib_zero_copy));
+
+        if (output != NULL) {
+            crm_log_xml_info(output, "cib:output");
+            free_xml(output);
+        }
+
+        output = result_cib;
+
+    } else {
+        crm_trace("Not activating %d %d %s", rc,
+                  pcmk_is_set(call_options, cib_dryrun),
+                  crm_element_value(result_cib, XML_ATTR_NUMUPDATES));
+        if (!pcmk_is_set(call_options, cib_zero_copy)) {
+            free_xml(result_cib);
+        }
+    }
+
+    if ((call_options & (cib_inhibit_notify|cib_dryrun)) == 0) {
+        crm_trace("Sending notifications %d",
+                  pcmk_is_set(call_options, cib_dryrun));
+        cib_diff_notify(op, rc, call_id, client_id, client_name, origin, input,
+                        *cib_diff);
+    }
+
+    if (send_r_notify) {
+        cib_replace_notify(op, rc, call_id, client_id, client_name, origin,
+                           the_cib, *cib_diff, change_section);
+    }
+
+    pcmk__output_set_log_level(logger_out, LOG_TRACE);
+    logger_out->message(logger_out, "xml-patchset", *cib_diff);
+
+  done:
+    if (!pcmk_is_set(call_options, cib_discard_reply) || cib_legacy_mode()) {
+        const char *caller = crm_element_value(request, F_CIB_CLIENTID);
+
+        *reply = create_xml_node(NULL, "cib-reply");
+        crm_xml_add(*reply, F_TYPE, T_CIB);
+        crm_xml_add(*reply, F_CIB_OPERATION, op);
+        crm_xml_add(*reply, F_CIB_CALLID, call_id);
+        crm_xml_add(*reply, F_CIB_CLIENTID, caller);
+        crm_xml_add_int(*reply, F_CIB_CALLOPTS, call_options);
+        crm_xml_add_int(*reply, F_CIB_RC, rc);
+
+        if (output != NULL) {
+            crm_trace("Attaching reply output");
+            add_message_xml(*reply, F_CIB_CALLDATA, output);
+        }
+
+        crm_log_xml_explicit(*reply, "cib:reply");
+    }
+
+    crm_trace("cleanup");
+
+    if (cib_op_modifies(call_type) == FALSE && output != current_cib) {
+        free_xml(output);
+        output = NULL;
+    }
+
+    if (call_type >= 0) {
+        cib_op_cleanup(call_type, call_options, &input, &output);
+    }
+
+    free(current_nodes_digest);
+    free(current_alerts_digest);
+    free(current_status_digest);
+
+    crm_trace("done");
+    return rc;
+}
+
+void
+cib_peer_callback(xmlNode * msg, void *private_data)
+{
+    const char *reason = NULL;
+    const char *originator = crm_element_value(msg, F_ORIG);
+
+    if (cib_legacy_mode()
+        && pcmk__str_eq(originator, OUR_NODENAME,
+                        pcmk__str_casei|pcmk__str_null_matches)) {
+        /* message is from ourselves */
+        int bcast_id = 0;
+
+        if (!(crm_element_value_int(msg, F_CIB_LOCAL_NOTIFY_ID, &bcast_id))) {
+            check_local_notify(bcast_id);
+        }
+        return;
+
+    } else if (crm_peer_cache == NULL) {
+        reason = "membership not established";
+        goto bail;
+    }
+
+    if (crm_element_value(msg, F_CIB_CLIENTNAME) == NULL) {
+        crm_xml_add(msg, F_CIB_CLIENTNAME, originator);
+    }
+
+    /* crm_log_xml_trace(msg, "Peer[inbound]"); */
+    cib_process_request(msg, TRUE, NULL);
+    return;
+
+  bail:
+    if (reason) {
+        const char *seq = crm_element_value(msg, F_SEQ);
+        const char *op = crm_element_value(msg, F_CIB_OPERATION);
+
+        crm_warn("Discarding %s message (%s) from %s: %s", op, seq, originator, reason);
+    }
+}
+
+static gboolean
+cib_force_exit(gpointer data)
+{
+    crm_notice("Forcing exit!");
+    terminate_cib(__func__, CRM_EX_ERROR);
+    return FALSE;
+}
+
+static void
+disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
+{
+    pcmk__client_t *a_client = value;
+
+    crm_err("Can't disconnect client %s: Not implemented",
+            pcmk__client_name(a_client));
+}
+
+static void
+initiate_exit(void)
+{
+    int active = 0;
+    xmlNode *leaving = NULL;
+
+    active = crm_active_peers();
+    if (active < 2) {
+        terminate_cib(__func__, 0);
+        return;
+    }
+
+    crm_info("Sending disconnect notification to %d peers...", active);
+
+    leaving = create_xml_node(NULL, "exit-notification");
+    crm_xml_add(leaving, F_TYPE, "cib");
+    crm_xml_add(leaving, F_CIB_OPERATION, PCMK__CIB_REQUEST_SHUTDOWN);
+
+    send_cluster_message(NULL, crm_msg_cib, leaving, TRUE);
+    free_xml(leaving);
+
+    g_timeout_add(EXIT_ESCALATION_MS, cib_force_exit, NULL);
+}
+
+void
+cib_shutdown(int nsig)
+{
+    struct qb_ipcs_stats srv_stats;
+
+    if (cib_shutdown_flag == FALSE) {
+        int disconnects = 0;
+        qb_ipcs_connection_t *c = NULL;
+
+        cib_shutdown_flag = TRUE;
+
+        c = qb_ipcs_connection_first_get(ipcs_rw);
+        while (c != NULL) {
+            qb_ipcs_connection_t *last = c;
+
+            c = qb_ipcs_connection_next_get(ipcs_rw, last);
+
+            crm_debug("Disconnecting r/w client %p...", last);
+            qb_ipcs_disconnect(last);
+            qb_ipcs_connection_unref(last);
+            disconnects++;
+        }
+
+        c = qb_ipcs_connection_first_get(ipcs_ro);
+        while (c != NULL) {
+            qb_ipcs_connection_t *last = c;
+
+            c = qb_ipcs_connection_next_get(ipcs_ro, last);
+
+            crm_debug("Disconnecting r/o client %p...", last);
+            qb_ipcs_disconnect(last);
+            qb_ipcs_connection_unref(last);
+            disconnects++;
+        }
+
+        c = qb_ipcs_connection_first_get(ipcs_shm);
+        while (c != NULL) {
+            qb_ipcs_connection_t *last = c;
+
+            c = qb_ipcs_connection_next_get(ipcs_shm, last);
+
+            crm_debug("Disconnecting non-blocking r/w client %p...", last);
+            qb_ipcs_disconnect(last);
+            qb_ipcs_connection_unref(last);
+            disconnects++;
+        }
+
+        disconnects += pcmk__ipc_client_count();
+
+        crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count());
+        pcmk__foreach_ipc_client(disconnect_remote_client, NULL);
+        crm_info("Disconnected %d clients", disconnects);
+    }
+
+    qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);
+
+    if (pcmk__ipc_client_count() == 0) {
+        crm_info("All clients disconnected (%d)", srv_stats.active_connections);
+        initiate_exit();
+
+    } else {
+        crm_info("Waiting on %d clients to disconnect (%d)",
+                 pcmk__ipc_client_count(), srv_stats.active_connections);
+    }
+}
+
+extern int remote_fd;
+extern int remote_tls_fd;
+
+/*!
+ * \internal
+ * \brief Close remote sockets, free the global CIB and quit
+ *
+ * \param[in] caller           Name of calling function (for log message)
+ * \param[in] fast             If -1, skip disconnect; if positive, exit that
+ */
+void
+terminate_cib(const char *caller, int fast)
+{
+    crm_info("%s: Exiting%s...", caller,
+             (fast > 0)? " fast" : mainloop ? " from mainloop" : "");
+
+    if (remote_fd > 0) {
+        close(remote_fd);
+        remote_fd = 0;
+    }
+    if (remote_tls_fd > 0) {
+        close(remote_tls_fd);
+        remote_tls_fd = 0;
+    }
+
+    uninitializeCib();
+
+    if (logger_out != NULL) {
+        logger_out->finish(logger_out, CRM_EX_OK, true, NULL);
+        pcmk__output_free(logger_out);
+        logger_out = NULL;
+    }
+
+    if (fast > 0) {
+        /* Quit fast on error */
+        pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
+        crm_exit(fast);
+
+    } else if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
+        /* Quit via returning from the main loop. If fast == -1, we skip the
+         * disconnect here, and it will be done when the main loop returns
+         * (this allows the peer status callback to avoid messing with the
+         * peer caches).
+         */
+        if (fast == 0) {
+            crm_cluster_disconnect(crm_cluster);
+        }
+        g_main_loop_quit(mainloop);
+
+    } else {
+        /* Quit via clean exit. Even the peer status callback can disconnect
+         * here, because we're not returning control to the caller. */
+        crm_cluster_disconnect(crm_cluster);
+        pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
+        crm_exit(CRM_EX_OK);
+    }
+}
diff --git a/daemons/based/based_common.c b/daemons/based/based_common.c
new file mode 100644
index 0000000..7e68cf0
--- /dev/null
+++ b/daemons/based/based_common.c
@@ -0,0 +1,352 @@
+/*
+ * Copyright 2008-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+
+#include <crm/crm.h>
+#include <crm/cib.h>
+#include <crm/msg_xml.h>
+#include <crm/common/ipc.h>
+#include <crm/cluster.h>
+
+#include <crm/common/xml.h>
+
+#include <pacemaker-based.h>
+
+gboolean stand_alone = FALSE;
+
+extern int cib_perform_command(xmlNode * request, xmlNode ** reply, xmlNode ** cib_diff,
+                               gboolean privileged);
+
+static xmlNode *
+cib_prepare_common(xmlNode * root, const char *section)
+{
+    xmlNode *data = NULL;
+
+    /* extract the CIB from the fragment */
+    if (root == NULL) {
+        return NULL;
+
+    } else if (pcmk__strcase_any_of(crm_element_name(root), XML_TAG_FRAGMENT,
+                                    F_CRM_DATA, F_CIB_CALLDATA, NULL)) {
+        data = first_named_child(root, XML_TAG_CIB);
+
+    } else {
+        data = root;
+    }
+
+    /* grab the section specified for the command */
+    if (section != NULL && data != NULL && pcmk__str_eq(crm_element_name(data), XML_TAG_CIB, pcmk__str_none)) {
+        data = pcmk_find_cib_element(data, section);
+    }
+
+    /* crm_log_xml_trace(root, "cib:input"); */
+    return data;
+}
+
+static int
+cib_prepare_none(xmlNode * request, xmlNode ** data, const char **section)
+{
+    *data = NULL;
+    *section = crm_element_value(request, F_CIB_SECTION);
+    return pcmk_ok;
+}
+
+static int
+cib_prepare_data(xmlNode * request, xmlNode ** data, const char **section)
+{
+    xmlNode *input_fragment = get_message_xml(request, F_CIB_CALLDATA);
+
+    *section = crm_element_value(request, F_CIB_SECTION);
+    *data = cib_prepare_common(input_fragment, *section);
+    /* crm_log_xml_debug(*data, "data"); */
+    return pcmk_ok;
+}
+
+static int
+cib_prepare_sync(xmlNode * request, xmlNode ** data, const char **section)
+{
+    *data = NULL;
+    *section = crm_element_value(request, F_CIB_SECTION);
+    return pcmk_ok;
+}
+
+static int
+cib_prepare_diff(xmlNode * request, xmlNode ** data, const char **section)
+{
+    xmlNode *input_fragment = NULL;
+
+    *data = NULL;
+    *section = NULL;
+
+    if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
+        input_fragment = get_message_xml(request, F_CIB_UPDATE_DIFF);
+    } else {
+        input_fragment = get_message_xml(request, F_CIB_CALLDATA);
+    }
+
+    CRM_CHECK(input_fragment != NULL, crm_log_xml_warn(request, "no input"));
+    *data = cib_prepare_common(input_fragment, NULL);
+    return pcmk_ok;
+}
+
+static int
+cib_cleanup_query(int options, xmlNode ** data, xmlNode ** output)
+{
+    CRM_LOG_ASSERT(*data == NULL);
+    if ((options & cib_no_children)
+        || pcmk__str_eq(crm_element_name(*output), "xpath-query", pcmk__str_casei)) {
+        free_xml(*output);
+    }
+    return pcmk_ok;
+}
+
+static int
+cib_cleanup_data(int options, xmlNode ** data, xmlNode ** output)
+{
+    free_xml(*output);
+    *data = NULL;
+    return pcmk_ok;
+}
+
+static int
+cib_cleanup_output(int options, xmlNode ** data, xmlNode ** output)
+{
+    free_xml(*output);
+    return pcmk_ok;
+}
+
+static int
+cib_cleanup_none(int options, xmlNode ** data, xmlNode ** output)
+{
+    CRM_LOG_ASSERT(*data == NULL);
+    CRM_LOG_ASSERT(*output == NULL);
+    return pcmk_ok;
+}
+
+static cib_operation_t cib_server_ops[] = {
+    // Booleans are modifies_cib, needs_privileges
+    {
+        NULL, FALSE, FALSE,
+        cib_prepare_none, cib_cleanup_none, cib_process_default
+    },
+    {
+        PCMK__CIB_REQUEST_QUERY, FALSE, FALSE,
+        cib_prepare_none, cib_cleanup_query, cib_process_query
+    },
+    {
+        PCMK__CIB_REQUEST_MODIFY, TRUE, TRUE,
+        cib_prepare_data, cib_cleanup_data, cib_process_modify
+    },
+    {
+        PCMK__CIB_REQUEST_APPLY_PATCH, TRUE, TRUE,
+        cib_prepare_diff, cib_cleanup_data, cib_server_process_diff
+    },
+    {
+        PCMK__CIB_REQUEST_REPLACE, TRUE, TRUE,
+        cib_prepare_data, cib_cleanup_data, cib_process_replace_svr
+    },
+    {
+        PCMK__CIB_REQUEST_CREATE, TRUE, TRUE,
+        cib_prepare_data, cib_cleanup_data, cib_process_create
+    },
+    {
+        PCMK__CIB_REQUEST_DELETE, TRUE, TRUE,
+        cib_prepare_data, cib_cleanup_data, cib_process_delete
+    },
+    {
+        PCMK__CIB_REQUEST_SYNC_TO_ALL, FALSE, TRUE,
+        cib_prepare_sync, cib_cleanup_none, cib_process_sync
+    },
+    {
+        PCMK__CIB_REQUEST_BUMP, TRUE, TRUE,
+        cib_prepare_none, cib_cleanup_output, cib_process_bump
+    },
+    {
+        PCMK__CIB_REQUEST_ERASE, TRUE, TRUE,
+        cib_prepare_none, cib_cleanup_output, cib_process_erase
+    },
+    {
+        PCMK__CIB_REQUEST_NOOP, FALSE, FALSE,
+        cib_prepare_none, cib_cleanup_none, cib_process_default
+    },
+    {
+        PCMK__CIB_REQUEST_ABS_DELETE, TRUE, TRUE,
+        cib_prepare_data, cib_cleanup_data, cib_process_delete_absolute
+    },
+    {
+        PCMK__CIB_REQUEST_UPGRADE, TRUE, TRUE,
+        cib_prepare_none, cib_cleanup_output, cib_process_upgrade_server
+    },
+    {
+        PCMK__CIB_REQUEST_SECONDARY, FALSE, TRUE,
+        cib_prepare_none, cib_cleanup_none, cib_process_readwrite
+    },
+    {
+        PCMK__CIB_REQUEST_ALL_SECONDARY, FALSE, TRUE,
+        cib_prepare_none, cib_cleanup_none, cib_process_readwrite
+    },
+    {
+        PCMK__CIB_REQUEST_SYNC_TO_ONE, FALSE, TRUE,
+        cib_prepare_sync, cib_cleanup_none, cib_process_sync_one
+    },
+    {
+        PCMK__CIB_REQUEST_PRIMARY, TRUE, TRUE,
+        cib_prepare_data, cib_cleanup_data, cib_process_readwrite
+    },
+    {
+        PCMK__CIB_REQUEST_IS_PRIMARY, FALSE, TRUE,
+        cib_prepare_none, cib_cleanup_none, cib_process_readwrite
+    },
+    {
+        PCMK__CIB_REQUEST_SHUTDOWN, FALSE, TRUE,
+        cib_prepare_sync, cib_cleanup_none, cib_process_shutdown_req
+    },
+    {
+        CRM_OP_PING, FALSE, FALSE,
+        cib_prepare_none, cib_cleanup_output, cib_process_ping
+    },
+};
+
+int
+cib_get_operation_id(const char *op, int *operation)
+{
+    static GHashTable *operation_hash = NULL;
+
+    if (operation_hash == NULL) {
+        int lpc = 0;
+        int max_msg_types = PCMK__NELEM(cib_server_ops);
+
+        operation_hash = pcmk__strkey_table(NULL, free);
+        for (lpc = 1; lpc < max_msg_types; lpc++) {
+            int *value = malloc(sizeof(int));
+
+            if(value) {
+                *value = lpc;
+                g_hash_table_insert(operation_hash, (gpointer) cib_server_ops[lpc].operation, value);
+            }
+        }
+    }
+
+    if (op != NULL) {
+        int *value = g_hash_table_lookup(operation_hash, op);
+
+        if (value) {
+            *operation = *value;
+            return pcmk_ok;
+        }
+    }
+    crm_err("Operation %s is not valid", op);
+    *operation = -1;
+    return -EINVAL;
+}
+
+xmlNode *
+cib_msg_copy(xmlNode * msg, gboolean with_data)
+{
+    int lpc = 0;
+    const char *field = NULL;
+    const char *value = NULL;
+    xmlNode *value_struct = NULL;
+
+    static const char *field_list[] = {
+        F_XML_TAGNAME,
+        F_TYPE,
+        F_CIB_CLIENTID,
+        F_CIB_CALLOPTS,
+        F_CIB_CALLID,
+        F_CIB_OPERATION,
+        F_CIB_ISREPLY,
+        F_CIB_SECTION,
+        F_CIB_HOST,
+        F_CIB_RC,
+        F_CIB_DELEGATED,
+        F_CIB_OBJID,
+        F_CIB_OBJTYPE,
+        F_CIB_EXISTING,
+        F_CIB_SEENCOUNT,
+        F_CIB_TIMEOUT,
+        F_CIB_GLOBAL_UPDATE,
+        F_CIB_CLIENTNAME,
+        F_CIB_USER,
+        F_CIB_NOTIFY_TYPE,
+        F_CIB_NOTIFY_ACTIVATE
+    };
+
+    static const char *data_list[] = {
+        F_CIB_CALLDATA,
+        F_CIB_UPDATE,
+        F_CIB_UPDATE_RESULT
+    };
+
+    xmlNode *copy = create_xml_node(NULL, "copy");
+
+    CRM_ASSERT(copy != NULL);
+
+    for (lpc = 0; lpc < PCMK__NELEM(field_list); lpc++) {
+        field = field_list[lpc];
+        value = crm_element_value(msg, field);
+        if (value != NULL) {
+            crm_xml_add(copy, field, value);
+        }
+    }
+    for (lpc = 0; with_data && lpc < PCMK__NELEM(data_list); lpc++) {
+        field = data_list[lpc];
+        value_struct = get_message_xml(msg, field);
+        if (value_struct != NULL) {
+            add_message_xml(copy, field, value_struct);
+        }
+    }
+
+    return copy;
+}
+
+cib_op_t *
+cib_op_func(int call_type)
+{
+    return &(cib_server_ops[call_type].fn);
+}
+
+gboolean
+cib_op_modifies(int call_type)
+{
+    return cib_server_ops[call_type].modifies_cib;
+}
+
+int
+cib_op_can_run(int call_type, int call_options, bool privileged)
+{
+    if (!privileged && cib_server_ops[call_type].needs_privileges) {
+        return -EACCES;
+    }
+    return pcmk_ok;
+}
+
+int
+cib_op_prepare(int call_type, xmlNode * request, xmlNode ** input, const char **section)
+{
+    crm_trace("Prepare %d", call_type);
+    return cib_server_ops[call_type].prepare(request, input, section);
+}
+
+int
+cib_op_cleanup(int call_type, int options, xmlNode ** input, xmlNode ** output)
+{
+    crm_trace("Cleanup %d", call_type);
+    return cib_server_ops[call_type].cleanup(options, input, output);
+}
diff --git a/daemons/based/based_io.c b/daemons/based/based_io.c
new file mode 100644
index 0000000..fc34f39
--- /dev/null
+++ b/daemons/based/based_io.c
@@ -0,0 +1,473 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <dirent.h>
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/stat.h>
+
+#include <crm/crm.h>
+
+#include <crm/cib.h>
+#include <crm/common/util.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/cib/internal.h>
+#include <crm/cluster.h>
+
+#include <pacemaker-based.h>
+
+crm_trigger_t *cib_writer = NULL;
+
+int write_cib_contents(gpointer p);
+
+static void
+cib_rename(const char *old)
+{
+    int new_fd;
+    char *new = crm_strdup_printf("%s/cib.auto.XXXXXX", cib_root);
+
+    umask(S_IWGRP | S_IWOTH | S_IROTH);
+    new_fd = mkstemp(new);
+    crm_err("Archiving unusable file %s as %s", old, new);
+    if ((new_fd < 0) || (rename(old, new) < 0)) {
+        crm_perror(LOG_ERR, "Couldn't rename %s as %s", old, new);
+        crm_err("Disabling disk writes and continuing");
+        cib_writes_enabled = FALSE;
+    }
+    if (new_fd > 0) {
+        close(new_fd);
+    }
+    free(new);
+}
+
+/*
+ * It is the callers responsibility to free the output of this function
+ */
+
+static xmlNode *
+retrieveCib(const char *filename, const char *sigfile)
+{
+    xmlNode *root = NULL;
+
+    crm_info("Reading cluster configuration file %s (digest: %s)",
+             filename, sigfile);
+    switch (cib_file_read_and_verify(filename, sigfile, &root)) {
+        case -pcmk_err_cib_corrupt:
+            crm_warn("Continuing but %s will NOT be used.", filename);
+            break;
+
+        case -pcmk_err_cib_modified:
+            /* Archive the original files so the contents are not lost */
+            crm_warn("Continuing but %s will NOT be used.", filename);
+            cib_rename(filename);
+            cib_rename(sigfile);
+            break;
+    }
+    return root;
+}
+
+/*
+ * for OSs without support for direntry->d_type, like Solaris
+ */
+#ifndef DT_UNKNOWN
+# define DT_UNKNOWN     0
+# define DT_FIFO        1
+# define DT_CHR         2
+# define DT_DIR         4
+# define DT_BLK         6
+# define DT_REG         8
+# define DT_LNK         10
+# define DT_SOCK        12
+# define DT_WHT         14
+#endif /*DT_UNKNOWN*/
+
+static int cib_archive_filter(const struct dirent * a)
+{
+    int rc = 0;
+    /* Looking for regular files (d_type = 8) starting with 'cib-' and not ending in .sig */
+    struct stat s;
+    char *a_path = crm_strdup_printf("%s/%s", cib_root, a->d_name);
+
+    if(stat(a_path, &s) != 0) {
+        rc = errno;
+        crm_trace("%s - stat failed: %s (%d)", a->d_name, pcmk_strerror(rc), rc);
+        rc = 0;
+
+    } else if ((s.st_mode & S_IFREG) != S_IFREG) {
+        unsigned char dtype;
+#ifdef HAVE_STRUCT_DIRENT_D_TYPE
+        dtype = a->d_type;
+#else
+        switch (s.st_mode & S_IFMT) {
+            case S_IFREG:  dtype = DT_REG;      break;
+            case S_IFDIR:  dtype = DT_DIR;      break;
+            case S_IFCHR:  dtype = DT_CHR;      break;
+            case S_IFBLK:  dtype = DT_BLK;      break;
+            case S_IFLNK:  dtype = DT_LNK;      break;
+            case S_IFIFO:  dtype = DT_FIFO;     break;
+            case S_IFSOCK: dtype = DT_SOCK;     break;
+            default:       dtype = DT_UNKNOWN;  break;
+        }
+#endif
+         crm_trace("%s - wrong type (%d)", a->d_name, dtype);
+
+    } else if(strstr(a->d_name, "cib-") != a->d_name) {
+        crm_trace("%s - wrong prefix", a->d_name);
+
+    } else if (pcmk__ends_with_ext(a->d_name, ".sig")) {
+        crm_trace("%s - wrong suffix", a->d_name);
+
+    } else {
+        crm_debug("%s - candidate", a->d_name);
+        rc = 1;
+    }
+
+    free(a_path);
+    return rc;
+}
+
+static int cib_archive_sort(const struct dirent ** a, const struct dirent **b)
+{
+    /* Order by creation date - most recently created file first */
+    int rc = 0;
+    struct stat buf;
+
+    time_t a_age = 0;
+    time_t b_age = 0;
+
+    char *a_path = crm_strdup_printf("%s/%s", cib_root, a[0]->d_name);
+    char *b_path = crm_strdup_printf("%s/%s", cib_root, b[0]->d_name);
+
+    if(stat(a_path, &buf) == 0) {
+        a_age = buf.st_ctime;
+    }
+    if(stat(b_path, &buf) == 0) {
+        b_age = buf.st_ctime;
+    }
+
+    free(a_path);
+    free(b_path);
+
+    if(a_age > b_age) {
+        rc = 1;
+    } else if(a_age < b_age) {
+        rc = -1;
+    }
+
+    crm_trace("%s (%lu) vs. %s (%lu) : %d",
+	a[0]->d_name, (unsigned long)a_age,
+	b[0]->d_name, (unsigned long)b_age, rc);
+    return rc;
+}
+
+xmlNode *
+readCibXmlFile(const char *dir, const char *file, gboolean discard_status)
+{
+    struct dirent **namelist = NULL;
+
+    int lpc = 0;
+    char *sigfile = NULL;
+    char *sigfilepath = NULL;
+    char *filename = NULL;
+    const char *name = NULL;
+    const char *value = NULL;
+    const char *validation = NULL;
+    const char *use_valgrind = getenv("PCMK_valgrind_enabled");
+
+    xmlNode *root = NULL;
+    xmlNode *status = NULL;
+
+    sigfile = crm_strdup_printf("%s.sig", file);
+    if (pcmk__daemon_can_write(dir, file) == FALSE
+            || pcmk__daemon_can_write(dir, sigfile) == FALSE) {
+        cib_status = -EACCES;
+        return NULL;
+    }
+
+    filename = crm_strdup_printf("%s/%s", dir, file);
+    sigfilepath = crm_strdup_printf("%s/%s", dir, sigfile);
+    free(sigfile);
+
+    cib_status = pcmk_ok;
+    root = retrieveCib(filename, sigfilepath);
+    free(filename);
+    free(sigfilepath);
+
+    if (root == NULL) {
+        crm_warn("Primary configuration corrupt or unusable, trying backups in %s", cib_root);
+        lpc = scandir(cib_root, &namelist, cib_archive_filter, cib_archive_sort);
+        if (lpc < 0) {
+            crm_perror(LOG_NOTICE, "scandir(%s) failed", cib_root);
+        }
+    }
+
+    while (root == NULL && lpc > 1) {
+        crm_debug("Testing %d candidates", lpc);
+
+        lpc--;
+
+        filename = crm_strdup_printf("%s/%s", cib_root, namelist[lpc]->d_name);
+        sigfile = crm_strdup_printf("%s.sig", filename);
+
+        crm_info("Reading cluster configuration file %s (digest: %s)",
+                 filename, sigfile);
+        if (cib_file_read_and_verify(filename, sigfile, &root) < 0) {
+            crm_warn("Continuing but %s will NOT be used.", filename);
+        } else {
+            crm_notice("Continuing with last valid configuration archive: %s", filename);
+        }
+
+        free(namelist[lpc]);
+        free(filename);
+        free(sigfile);
+    }
+    free(namelist);
+
+    if (root == NULL) {
+        root = createEmptyCib(0);
+        crm_warn("Continuing with an empty configuration.");
+    }
+
+    if (cib_writes_enabled && use_valgrind &&
+        (crm_is_true(use_valgrind) || strstr(use_valgrind, "pacemaker-based"))) {
+
+        cib_writes_enabled = FALSE;
+        crm_err("*** Disabling disk writes to avoid confusing Valgrind ***");
+    }
+
+    status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE);
+    if (discard_status && status != NULL) {
+        /* strip out the status section if there is one */
+        free_xml(status);
+        status = NULL;
+    }
+    if (status == NULL) {
+        create_xml_node(root, XML_CIB_TAG_STATUS);
+    }
+
+    /* Do this before schema validation happens */
+
+    /* fill in some defaults */
+    name = XML_ATTR_GENERATION_ADMIN;
+    value = crm_element_value(root, name);
+    if (value == NULL) {
+        crm_warn("No value for %s was specified in the configuration.", name);
+        crm_warn("The recommended course of action is to shutdown,"
+                 " run crm_verify and fix any errors it reports.");
+        crm_warn("We will default to zero and continue but may get"
+                 " confused about which configuration to use if"
+                 " multiple nodes are powered up at the same time.");
+        crm_xml_add_int(root, name, 0);
+    }
+
+    name = XML_ATTR_GENERATION;
+    value = crm_element_value(root, name);
+    if (value == NULL) {
+        crm_xml_add_int(root, name, 0);
+    }
+
+    name = XML_ATTR_NUMUPDATES;
+    value = crm_element_value(root, name);
+    if (value == NULL) {
+        crm_xml_add_int(root, name, 0);
+    }
+
+    // Unset (DC should set appropriate value)
+    xml_remove_prop(root, XML_ATTR_DC_UUID);
+
+    if (discard_status) {
+        crm_log_xml_trace(root, "[on-disk]");
+    }
+
+    validation = crm_element_value(root, XML_ATTR_VALIDATION);
+    if (validate_xml(root, NULL, TRUE) == FALSE) {
+        crm_err("CIB does not validate with %s",
+                pcmk__s(validation, "no schema specified"));
+        cib_status = -pcmk_err_schema_validation;
+
+    } else if (validation == NULL) {
+        int version = 0;
+
+        update_validation(&root, &version, 0, FALSE, FALSE);
+        if (version > 0) {
+            crm_notice("Enabling %s validation on"
+                       " the existing (sane) configuration", get_schema_name(version));
+        } else {
+            crm_err("CIB does not validate with any known schema");
+            cib_status = -pcmk_err_schema_validation;
+        }
+    }
+
+    return root;
+}
+
+gboolean
+uninitializeCib(void)
+{
+    xmlNode *tmp_cib = the_cib;
+
+    if (tmp_cib == NULL) {
+        crm_debug("The CIB has already been deallocated.");
+        return FALSE;
+    }
+
+    the_cib = NULL;
+
+    crm_debug("Deallocating the CIB.");
+
+    free_xml(tmp_cib);
+
+    crm_debug("The CIB has been deallocated.");
+
+    return TRUE;
+}
+
+/*
+ * This method will free the old CIB pointer on success and the new one
+ * on failure.
+ */
+int
+activateCibXml(xmlNode * new_cib, gboolean to_disk, const char *op)
+{
+    if (new_cib) {
+        xmlNode *saved_cib = the_cib;
+
+        CRM_ASSERT(new_cib != saved_cib);
+        the_cib = new_cib;
+        free_xml(saved_cib);
+        if (cib_writes_enabled && cib_status == pcmk_ok && to_disk) {
+            crm_debug("Triggering CIB write for %s op", op);
+            mainloop_set_trigger(cib_writer);
+        }
+        return pcmk_ok;
+    }
+
+    crm_err("Ignoring invalid CIB");
+    if (the_cib) {
+        crm_warn("Reverting to last known CIB");
+    } else {
+        crm_crit("Could not write out new CIB and no saved version to revert to");
+    }
+    return -ENODATA;
+}
+
+static void
+cib_diskwrite_complete(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
+{
+    const char *errmsg = "Could not write CIB to disk";
+
+    if ((exitcode != 0) && cib_writes_enabled) {
+        cib_writes_enabled = FALSE;
+        errmsg = "Disabling CIB disk writes after failure";
+    }
+
+    if ((signo == 0) && (exitcode == 0)) {
+        crm_trace("Disk write [%d] succeeded", (int) pid);
+
+    } else if (signo == 0) {
+        crm_err("%s: process %d exited %d", errmsg, (int) pid, exitcode);
+
+    } else {
+        crm_err("%s: process %d terminated with signal %d (%s)%s",
+                errmsg, (int) pid, signo, strsignal(signo),
+                (core? " and dumped core" : ""));
+    }
+
+    mainloop_trigger_complete(cib_writer);
+}
+
+int
+write_cib_contents(gpointer p)
+{
+    int exit_rc = pcmk_ok;
+    xmlNode *cib_local = NULL;
+
+    /* Make a copy of the CIB to write (possibly in a forked child) */
+    if (p) {
+        /* Synchronous write out */
+        cib_local = copy_xml(p);
+
+    } else {
+        int pid = 0;
+        int bb_state = qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_STATE_GET, 0);
+
+        /* Turn it off before the fork() to avoid:
+         * - 2 processes writing to the same shared mem
+         * - the child needing to disable it
+         *   (which would close it from underneath the parent)
+         * This way, the shared mem files are already closed
+         */
+        qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);
+
+        pid = fork();
+        if (pid < 0) {
+            crm_perror(LOG_ERR, "Disabling disk writes after fork failure");
+            cib_writes_enabled = FALSE;
+            return FALSE;
+        }
+
+        if (pid) {
+            /* Parent */
+            mainloop_child_add(pid, 0, "disk-writer", NULL, cib_diskwrite_complete);
+            if (bb_state == QB_LOG_STATE_ENABLED) {
+                /* Re-enable now that it it safe */
+                qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_TRUE);
+            }
+
+            return -1;          /* -1 means 'still work to do' */
+        }
+
+        /* Asynchronous write-out after a fork() */
+
+        /* In theory, we can scribble on the_cib here and not affect the parent,
+         * but let's be safe anyway.
+         */
+        cib_local = copy_xml(the_cib);
+    }
+
+    /* Write the CIB */
+    exit_rc = cib_file_write_with_digest(cib_local, cib_root, "cib.xml");
+
+    /* A nonzero exit code will cause further writes to be disabled */
+    free_xml(cib_local);
+    if (p == NULL) {
+        crm_exit_t exit_code = CRM_EX_OK;
+
+        switch (exit_rc) {
+            case pcmk_ok:
+                exit_code = CRM_EX_OK;
+                break;
+            case pcmk_err_cib_modified:
+                exit_code = CRM_EX_DIGEST; // Existing CIB doesn't match digest
+                break;
+            case pcmk_err_cib_backup: // Existing CIB couldn't be backed up
+            case pcmk_err_cib_save:   // New CIB couldn't be saved
+                exit_code = CRM_EX_CANTCREAT;
+                break;
+            default:
+                exit_code = CRM_EX_ERROR;
+                break;
+        }
+
+        /* Use _exit() because exit() could affect the parent adversely */
+        _exit(exit_code);
+    }
+    return exit_rc;
+}
diff --git a/daemons/based/based_messages.c b/daemons/based/based_messages.c
new file mode 100644
index 0000000..d46456c
--- /dev/null
+++ b/daemons/based/based_messages.c
@@ -0,0 +1,427 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <time.h>
+
+#include <sys/param.h>
+#include <sys/types.h>
+
+#include <crm/crm.h>
+#include <crm/cib/internal.h>
+#include <crm/msg_xml.h>
+
+#include <crm/common/xml.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/common/xml_internal.h>
+#include <crm/cluster/internal.h>
+
+#include <pacemaker-based.h>
+
+/* Maximum number of diffs to ignore while waiting for a resync */
+#define MAX_DIFF_RETRY 5
+
+bool based_is_primary = false;
+
+xmlNode *the_cib = NULL;
+
+int
+cib_process_shutdown_req(const char *op, int options, const char *section, xmlNode * req,
+                         xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
+                         xmlNode ** answer)
+{
+    const char *host = crm_element_value(req, F_ORIG);
+
+    *answer = NULL;
+
+    if (crm_element_value(req, F_CIB_ISREPLY) == NULL) {
+        crm_info("Peer %s is requesting to shut down", host);
+        return pcmk_ok;
+    }
+
+    if (cib_shutdown_flag == FALSE) {
+        crm_err("Peer %s mistakenly thinks we wanted to shut down", host);
+        return -EINVAL;
+    }
+
+    crm_info("Peer %s has acknowledged our shutdown request", host);
+    terminate_cib(__func__, 0);
+    return pcmk_ok;
+}
+
+int
+cib_process_default(const char *op, int options, const char *section, xmlNode * req,
+                    xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
+                    xmlNode ** answer)
+{
+    int result = pcmk_ok;
+
+    crm_trace("Processing \"%s\" event", op);
+    *answer = NULL;
+
+    if (op == NULL) {
+        result = -EINVAL;
+        crm_err("No operation specified");
+
+    } else if (strcmp(PCMK__CIB_REQUEST_NOOP, op) != 0) {
+        result = -EPROTONOSUPPORT;
+        crm_err("Action [%s] is not supported by the CIB manager", op);
+    }
+    return result;
+}
+
+int
+cib_process_readwrite(const char *op, int options, const char *section, xmlNode * req,
+                      xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
+                      xmlNode ** answer)
+{
+    int result = pcmk_ok;
+
+    crm_trace("Processing \"%s\" event", op);
+
+    if (pcmk__str_eq(op, PCMK__CIB_REQUEST_IS_PRIMARY, pcmk__str_none)) {
+        if (based_is_primary) {
+            result = pcmk_ok;
+        } else {
+            result = -EPERM;
+        }
+        return result;
+    }
+
+    if (pcmk__str_eq(op, PCMK__CIB_REQUEST_PRIMARY, pcmk__str_none)) {
+        if (!based_is_primary) {
+            crm_info("We are now in R/W mode");
+            based_is_primary = true;
+        } else {
+            crm_debug("We are still in R/W mode");
+        }
+
+    } else if (based_is_primary) {
+        crm_info("We are now in R/O mode");
+        based_is_primary = false;
+    }
+
+    return result;
+}
+
+/* Set to 1 when a sync is requested, incremented when a diff is ignored,
+ * reset to 0 when a sync is received
+ */
+static int sync_in_progress = 0;
+
+void
+send_sync_request(const char *host)
+{
+    xmlNode *sync_me = create_xml_node(NULL, "sync-me");
+
+    crm_info("Requesting re-sync from %s", (host? host : "all peers"));
+    sync_in_progress = 1;
+
+    crm_xml_add(sync_me, F_TYPE, "cib");
+    crm_xml_add(sync_me, F_CIB_OPERATION, PCMK__CIB_REQUEST_SYNC_TO_ONE);
+    crm_xml_add(sync_me, F_CIB_DELEGATED,
+                stand_alone? "localhost" : crm_cluster->uname);
+
+    send_cluster_message(host ? crm_get_peer(0, host) : NULL, crm_msg_cib, sync_me, FALSE);
+    free_xml(sync_me);
+}
+
+int
+cib_process_ping(const char *op, int options, const char *section, xmlNode * req, xmlNode * input,
+                 xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer)
+{
+    const char *host = crm_element_value(req, F_ORIG);
+    const char *seq = crm_element_value(req, F_CIB_PING_ID);
+    char *digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, CRM_FEATURE_SET);
+
+    crm_trace("Processing \"%s\" event %s from %s", op, seq, host);
+    *answer = create_xml_node(NULL, XML_CRM_TAG_PING);
+
+    crm_xml_add(*answer, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
+    crm_xml_add(*answer, XML_ATTR_DIGEST, digest);
+    crm_xml_add(*answer, F_CIB_PING_ID, seq);
+
+    pcmk__if_tracing(
+        {
+            // Append additional detail so the receiver can log the differences
+            add_message_xml(*answer, F_CIB_CALLDATA, the_cib);
+        },
+        {
+            // Always include at least the version details
+            const char *tag = TYPE(the_cib);
+            xmlNode *shallow = create_xml_node(NULL, tag);
+
+            copy_in_properties(shallow, the_cib);
+            add_message_xml(*answer, F_CIB_CALLDATA, shallow);
+            free_xml(shallow);
+        }
+    );
+
+    crm_info("Reporting our current digest to %s: %s for %s.%s.%s",
+             host, digest,
+             crm_element_value(existing_cib, XML_ATTR_GENERATION_ADMIN),
+             crm_element_value(existing_cib, XML_ATTR_GENERATION),
+             crm_element_value(existing_cib, XML_ATTR_NUMUPDATES));
+
+    free(digest);
+
+    return pcmk_ok;
+}
+
+int
+cib_process_sync(const char *op, int options, const char *section, xmlNode * req, xmlNode * input,
+                 xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer)
+{
+    return sync_our_cib(req, TRUE);
+}
+
+int
+cib_process_upgrade_server(const char *op, int options, const char *section, xmlNode * req, xmlNode * input,
+                           xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer)
+{
+    int rc = pcmk_ok;
+
+    *answer = NULL;
+
+    if(crm_element_value(req, F_CIB_SCHEMA_MAX)) {
+        /* The originator of an upgrade request sends it to the DC, without
+         * F_CIB_SCHEMA_MAX. If an upgrade is needed, the DC re-broadcasts the
+         * request with F_CIB_SCHEMA_MAX, and each node performs the upgrade
+         * (and notifies its local clients) here.
+         */
+        return cib_process_upgrade(
+            op, options, section, req, input, existing_cib, result_cib, answer);
+
+    } else {
+        int new_version = 0;
+        int current_version = 0;
+        xmlNode *scratch = copy_xml(existing_cib);
+        const char *host = crm_element_value(req, F_ORIG);
+        const char *value = crm_element_value(existing_cib, XML_ATTR_VALIDATION);
+        const char *client_id = crm_element_value(req, F_CIB_CLIENTID);
+        const char *call_opts = crm_element_value(req, F_CIB_CALLOPTS);
+        const char *call_id = crm_element_value(req, F_CIB_CALLID);
+
+        crm_trace("Processing \"%s\" event", op);
+        if (value != NULL) {
+            current_version = get_schema_version(value);
+        }
+
+        rc = update_validation(&scratch, &new_version, 0, TRUE, TRUE);
+        if (new_version > current_version) {
+            xmlNode *up = create_xml_node(NULL, __func__);
+
+            rc = pcmk_ok;
+            crm_notice("Upgrade request from %s verified", host);
+
+            crm_xml_add(up, F_TYPE, "cib");
+            crm_xml_add(up, F_CIB_OPERATION, PCMK__CIB_REQUEST_UPGRADE);
+            crm_xml_add(up, F_CIB_SCHEMA_MAX, get_schema_name(new_version));
+            crm_xml_add(up, F_CIB_DELEGATED, host);
+            crm_xml_add(up, F_CIB_CLIENTID, client_id);
+            crm_xml_add(up, F_CIB_CALLOPTS, call_opts);
+            crm_xml_add(up, F_CIB_CALLID, call_id);
+
+            if (cib_legacy_mode() && based_is_primary) {
+                rc = cib_process_upgrade(
+                    op, options, section, up, input, existing_cib, result_cib, answer);
+
+            } else {
+                send_cluster_message(NULL, crm_msg_cib, up, FALSE);
+            }
+
+            free_xml(up);
+
+        } else if(rc == pcmk_ok) {
+            rc = -pcmk_err_schema_unchanged;
+        }
+
+        if (rc != pcmk_ok) {
+            // Notify originating peer so it can notify its local clients
+            crm_node_t *origin = pcmk__search_cluster_node_cache(0, host);
+
+            crm_info("Rejecting upgrade request from %s: %s "
+                     CRM_XS " rc=%d peer=%s", host, pcmk_strerror(rc), rc,
+                     (origin? origin->uname : "lost"));
+
+            if (origin) {
+                xmlNode *up = create_xml_node(NULL, __func__);
+
+                crm_xml_add(up, F_TYPE, "cib");
+                crm_xml_add(up, F_CIB_OPERATION, PCMK__CIB_REQUEST_UPGRADE);
+                crm_xml_add(up, F_CIB_DELEGATED, host);
+                crm_xml_add(up, F_CIB_ISREPLY, host);
+                crm_xml_add(up, F_CIB_CLIENTID, client_id);
+                crm_xml_add(up, F_CIB_CALLOPTS, call_opts);
+                crm_xml_add(up, F_CIB_CALLID, call_id);
+                crm_xml_add_int(up, F_CIB_UPGRADE_RC, rc);
+                if (send_cluster_message(origin, crm_msg_cib, up, TRUE)
+                    == FALSE) {
+                    crm_warn("Could not send CIB upgrade result to %s", host);
+                }
+                free_xml(up);
+            }
+        }
+        free_xml(scratch);
+    }
+    return rc;
+}
+
+int
+cib_process_sync_one(const char *op, int options, const char *section, xmlNode * req,
+                     xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
+                     xmlNode ** answer)
+{
+    return sync_our_cib(req, FALSE);
+}
+
+int
+cib_server_process_diff(const char *op, int options, const char *section, xmlNode * req,
+                        xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
+                        xmlNode ** answer)
+{
+    int rc = pcmk_ok;
+
+    if (sync_in_progress > MAX_DIFF_RETRY) {
+        /* Don't ignore diffs forever; the last request may have been lost.
+         * If the diff fails, we'll ask for another full resync.
+         */
+        sync_in_progress = 0;
+    }
+
+    // The primary instance should never ignore a diff
+    if (sync_in_progress && !based_is_primary) {
+        int diff_add_updates = 0;
+        int diff_add_epoch = 0;
+        int diff_add_admin_epoch = 0;
+
+        int diff_del_updates = 0;
+        int diff_del_epoch = 0;
+        int diff_del_admin_epoch = 0;
+
+        cib_diff_version_details(input,
+                                 &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
+                                 &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
+
+        sync_in_progress++;
+        crm_notice("Not applying diff %d.%d.%d -> %d.%d.%d (sync in progress)",
+                   diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
+                   diff_add_admin_epoch, diff_add_epoch, diff_add_updates);
+        return -pcmk_err_diff_resync;
+    }
+
+    rc = cib_process_diff(op, options, section, req, input, existing_cib, result_cib, answer);
+    crm_trace("result: %s (%d), %s", pcmk_strerror(rc), rc,
+              (based_is_primary? "primary": "secondary"));
+
+    if ((rc == -pcmk_err_diff_resync) && !based_is_primary) {
+        free_xml(*result_cib);
+        *result_cib = NULL;
+        send_sync_request(NULL);
+
+    } else if (rc == -pcmk_err_diff_resync) {
+        rc = -pcmk_err_diff_failed;
+        if (options & cib_force_diff) {
+            crm_warn("Not requesting full refresh in R/W mode");
+        }
+
+    } else if ((rc != pcmk_ok) && !based_is_primary && cib_legacy_mode()) {
+        crm_warn("Requesting full CIB refresh because update failed: %s"
+                 CRM_XS " rc=%d", pcmk_strerror(rc), rc);
+
+        pcmk__output_set_log_level(logger_out, LOG_INFO);
+        logger_out->message(logger_out, "xml-patchset", input);
+        free_xml(*result_cib);
+        *result_cib = NULL;
+        send_sync_request(NULL);
+    }
+
+    return rc;
+}
+
+int
+cib_process_replace_svr(const char *op, int options, const char *section, xmlNode * req,
+                        xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
+                        xmlNode ** answer)
+{
+    const char *tag = crm_element_name(input);
+    int rc =
+        cib_process_replace(op, options, section, req, input, existing_cib, result_cib, answer);
+    if (rc == pcmk_ok && pcmk__str_eq(tag, XML_TAG_CIB, pcmk__str_casei)) {
+        sync_in_progress = 0;
+    }
+    return rc;
+}
+
+int
+cib_process_delete_absolute(const char *op, int options, const char *section, xmlNode * req,
+                            xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
+                            xmlNode ** answer)
+{
+    return -EINVAL;
+}
+
+int
+sync_our_cib(xmlNode * request, gboolean all)
+{
+    int result = pcmk_ok;
+    char *digest = NULL;
+    const char *host = crm_element_value(request, F_ORIG);
+    const char *op = crm_element_value(request, F_CIB_OPERATION);
+
+    xmlNode *replace_request = NULL;
+
+    CRM_CHECK(the_cib != NULL, return -EINVAL);
+
+    replace_request = cib_msg_copy(request, FALSE);
+    CRM_CHECK(replace_request != NULL, return -EINVAL);
+
+    crm_debug("Syncing CIB to %s", all ? "all peers" : host);
+    if (all == FALSE && host == NULL) {
+        crm_log_xml_err(request, "bad sync");
+    }
+
+    /* remove the "all == FALSE" condition
+     *
+     * sync_from was failing, the local client wasn't being notified
+     *    because it didn't know it was a reply
+     * setting this does not prevent the other nodes from applying it
+     *    if all == TRUE
+     */
+    if (host != NULL) {
+        crm_xml_add(replace_request, F_CIB_ISREPLY, host);
+    }
+    if (all) {
+        xml_remove_prop(replace_request, F_CIB_HOST);
+    }
+
+    crm_xml_add(replace_request, F_CIB_OPERATION, PCMK__CIB_REQUEST_REPLACE);
+    crm_xml_add(replace_request, "original_" F_CIB_OPERATION, op);
+    pcmk__xe_set_bool_attr(replace_request, F_CIB_GLOBAL_UPDATE, true);
+
+    crm_xml_add(replace_request, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
+    digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, CRM_FEATURE_SET);
+    crm_xml_add(replace_request, XML_ATTR_DIGEST, digest);
+
+    add_message_xml(replace_request, F_CIB_CALLDATA, the_cib);
+
+    if (send_cluster_message
+        (all ? NULL : crm_get_peer(0, host), crm_msg_cib, replace_request, FALSE) == FALSE) {
+        result = -ENOTCONN;
+    }
+    free_xml(replace_request);
+    free(digest);
+    return result;
+}
diff --git a/daemons/based/based_notify.c b/daemons/based/based_notify.c
new file mode 100644
index 0000000..5881f6d
--- /dev/null
+++ b/daemons/based/based_notify.c
@@ -0,0 +1,305 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <inttypes.h>           // PRIx64
+
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+
+#include <time.h>
+
+#include <crm/crm.h>
+#include <crm/cib/internal.h>
+#include <crm/msg_xml.h>
+
+#include <crm/common/xml.h>
+#include <crm/common/remote_internal.h>
+#include <pacemaker-based.h>
+
+struct cib_notification_s {
+    xmlNode *msg;
+    struct iovec *iov;
+    int32_t iov_size;
+};
+
+static void
+cib_notify_send_one(gpointer key, gpointer value, gpointer user_data)
+{
+    const char *type = NULL;
+    gboolean do_send = FALSE;
+    int rc = pcmk_rc_ok;
+
+    pcmk__client_t *client = value;
+    struct cib_notification_s *update = user_data;
+
+    if (client->ipcs == NULL && client->remote == NULL) {
+        crm_warn("Skipping client with NULL channel");
+        return;
+    }
+
+    type = crm_element_value(update->msg, F_SUBTYPE);
+    CRM_LOG_ASSERT(type != NULL);
+
+    if (pcmk_is_set(client->flags, cib_notify_diff)
+        && pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) {
+
+        do_send = TRUE;
+
+    } else if (pcmk_is_set(client->flags, cib_notify_replace)
+               && pcmk__str_eq(type, T_CIB_REPLACE_NOTIFY, pcmk__str_casei)) {
+        do_send = TRUE;
+
+    } else if (pcmk_is_set(client->flags, cib_notify_confirm)
+               && pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) {
+        do_send = TRUE;
+
+    } else if (pcmk_is_set(client->flags, cib_notify_pre)
+               && pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) {
+        do_send = TRUE;
+
+    } else if (pcmk_is_set(client->flags, cib_notify_post)
+               && pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) {
+
+        do_send = TRUE;
+    }
+
+    if (do_send) {
+        switch (PCMK__CLIENT_TYPE(client)) {
+            case pcmk__client_ipc:
+                rc = pcmk__ipc_send_iov(client, update->iov,
+                                        crm_ipc_server_event);
+                if (rc != pcmk_rc_ok) {
+                    crm_warn("Could not notify client %s: %s " CRM_XS " id=%s",
+                             pcmk__client_name(client), pcmk_rc_str(rc),
+                             client->id);
+                }
+                break;
+#ifdef HAVE_GNUTLS_GNUTLS_H
+            case pcmk__client_tls:
+#endif
+            case pcmk__client_tcp:
+                crm_debug("Sent %s notification to client %s (id %s)",
+                          type, pcmk__client_name(client), client->id);
+                pcmk__remote_send_xml(client->remote, update->msg);
+                break;
+            default:
+                crm_err("Unknown transport for client %s "
+                        CRM_XS " flags=%#016" PRIx64,
+                        pcmk__client_name(client), client->flags);
+        }
+    }
+}
+
+static void
+cib_notify_send(xmlNode * xml)
+{
+    struct iovec *iov;
+    struct cib_notification_s update;
+
+    ssize_t bytes = 0;
+    int rc = pcmk__ipc_prepare_iov(0, xml, 0, &iov, &bytes);
+
+    if (rc == pcmk_rc_ok) {
+        update.msg = xml;
+        update.iov = iov;
+        update.iov_size = bytes;
+        pcmk__foreach_ipc_client(cib_notify_send_one, &update);
+
+    } else {
+        crm_notice("Could not notify clients: %s " CRM_XS " rc=%d",
+                   pcmk_rc_str(rc), rc);
+    }
+    pcmk_free_ipc_event(iov);
+}
+
+static void
+attach_cib_generation(xmlNode *msg, const char *field, xmlNode *a_cib)
+{
+    xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);
+
+    if (a_cib != NULL) {
+        copy_in_properties(generation, a_cib);
+    }
+    add_message_xml(msg, field, generation);
+    free_xml(generation);
+}
+
+void
+cib_diff_notify(const char *op, int result, const char *call_id,
+                const char *client_id, const char *client_name,
+                const char *origin, xmlNode *update, xmlNode *diff)
+{
+    int add_updates = 0;
+    int add_epoch = 0;
+    int add_admin_epoch = 0;
+
+    int del_updates = 0;
+    int del_epoch = 0;
+    int del_admin_epoch = 0;
+
+    uint8_t log_level = LOG_TRACE;
+
+    xmlNode *update_msg = NULL;
+    const char *type = NULL;
+
+    if (diff == NULL) {
+        return;
+    }
+
+    if (result != pcmk_ok) {
+        log_level = LOG_WARNING;
+    }
+
+    cib_diff_version_details(diff, &add_admin_epoch, &add_epoch, &add_updates,
+                             &del_admin_epoch, &del_epoch, &del_updates);
+
+    if ((add_admin_epoch != del_admin_epoch)
+        || (add_epoch != del_epoch)
+        || (add_updates != del_updates)) {
+
+        do_crm_log(log_level,
+                   "Updated CIB generation %d.%d.%d to %d.%d.%d from client "
+                   "%s%s%s (%s) (%s)",
+                   del_admin_epoch, del_epoch, del_updates,
+                   add_admin_epoch, add_epoch, add_updates,
+                   client_name,
+                   ((call_id != NULL)? " call " : ""), pcmk__s(call_id, ""),
+                   pcmk__s(origin, "unspecified peer"), pcmk_strerror(result));
+
+    } else if ((add_admin_epoch != 0)
+               || (add_epoch != 0)
+               || (add_updates != 0)) {
+
+        do_crm_log(log_level,
+                   "Local-only change to CIB generation %d.%d.%d from client "
+                   "%s%s%s (%s) (%s)",
+                   add_admin_epoch, add_epoch, add_updates,
+                   client_name,
+                   ((call_id != NULL)? " call " : ""), pcmk__s(call_id, ""),
+                   pcmk__s(origin, "unspecified peer"), pcmk_strerror(result));
+    }
+
+    update_msg = create_xml_node(NULL, "notify");
+
+    crm_xml_add(update_msg, F_TYPE, T_CIB_NOTIFY);
+    crm_xml_add(update_msg, F_SUBTYPE, T_CIB_DIFF_NOTIFY);
+    crm_xml_add(update_msg, F_CIB_OPERATION, op);
+    crm_xml_add(update_msg, F_CIB_CLIENTID, client_id);
+    crm_xml_add(update_msg, F_CIB_CALLID, call_id);
+    crm_xml_add(update_msg, F_ORIG, origin);
+    crm_xml_add_int(update_msg, F_CIB_RC, result);
+
+    if (update != NULL) {
+        type = crm_element_name(update);
+        crm_trace("Setting type to update->name: %s", type);
+    } else {
+        type = crm_element_name(diff);
+        crm_trace("Setting type to new_obj->name: %s", type);
+    }
+    crm_xml_add(update_msg, F_CIB_OBJID, ID(diff));
+    crm_xml_add(update_msg, F_CIB_OBJTYPE, type);
+    attach_cib_generation(update_msg, "cib_generation", the_cib);
+
+    if (update != NULL) {
+        add_message_xml(update_msg, F_CIB_UPDATE, update);
+    }
+    add_message_xml(update_msg, F_CIB_UPDATE_RESULT, diff);
+
+    cib_notify_send(update_msg);
+    free_xml(update_msg);
+}
+
+void
+cib_replace_notify(const char *op, int result, const char *call_id,
+                   const char *client_id, const char *client_name,
+                   const char *origin, xmlNode *update, xmlNode *diff,
+                   uint32_t change_section)
+{
+    xmlNode *replace_msg = NULL;
+
+    int add_updates = 0;
+    int add_epoch = 0;
+    int add_admin_epoch = 0;
+
+    int del_updates = 0;
+    int del_epoch = 0;
+    int del_admin_epoch = 0;
+
+    uint8_t log_level = LOG_INFO;
+
+    if (diff == NULL) {
+        return;
+    }
+
+    if (result != pcmk_ok) {
+        log_level = LOG_WARNING;
+    }
+
+    cib_diff_version_details(diff, &add_admin_epoch, &add_epoch, &add_updates,
+                             &del_admin_epoch, &del_epoch, &del_updates);
+
+    if (del_updates < 0) {
+        crm_log_xml_debug(diff, "Bad replace diff");
+    }
+
+    if ((add_admin_epoch != del_admin_epoch)
+        || (add_epoch != del_epoch)
+        || (add_updates != del_updates)) {
+
+        do_crm_log(log_level,
+                   "Replaced CIB generation %d.%d.%d with %d.%d.%d from client "
+                   "%s%s%s (%s) (%s)",
+                   del_admin_epoch, del_epoch, del_updates,
+                   add_admin_epoch, add_epoch, add_updates,
+                   client_name,
+                   ((call_id != NULL)? " call " : ""), pcmk__s(call_id, ""),
+                   pcmk__s(origin, "unspecified peer"), pcmk_strerror(result));
+
+    } else if ((add_admin_epoch != 0)
+               || (add_epoch != 0)
+               || (add_updates != 0)) {
+
+        do_crm_log(log_level,
+                   "Local-only replace of CIB generation %d.%d.%d from client "
+                   "%s%s%s (%s) (%s)",
+                   add_admin_epoch, add_epoch, add_updates,
+                   client_name,
+                   ((call_id != NULL)? " call " : ""), pcmk__s(call_id, ""),
+                   pcmk__s(origin, "unspecified peer"), pcmk_strerror(result));
+    }
+
+    replace_msg = create_xml_node(NULL, "notify-replace");
+
+    crm_xml_add(replace_msg, F_TYPE, T_CIB_NOTIFY);
+    crm_xml_add(replace_msg, F_SUBTYPE, T_CIB_REPLACE_NOTIFY);
+    crm_xml_add(replace_msg, F_CIB_OPERATION, op);
+    crm_xml_add(replace_msg, F_CIB_CLIENTID, client_id);
+    crm_xml_add(replace_msg, F_CIB_CALLID, call_id);
+    crm_xml_add(replace_msg, F_ORIG, origin);
+    crm_xml_add_int(replace_msg, F_CIB_RC, result);
+    crm_xml_add_ll(replace_msg, F_CIB_CHANGE_SECTION,
+                   (long long) change_section);
+    attach_cib_generation(replace_msg, "cib-replace-generation", update);
+
+    /* We can include update and diff if a replace callback needs them. Until
+     * then, avoid the overhead.
+     */
+
+    crm_log_xml_trace(replace_msg, "CIB replaced");
+
+    cib_notify_send(replace_msg);
+    free_xml(replace_msg);
+}
diff --git a/daemons/based/based_remote.c b/daemons/based/based_remote.c
new file mode 100644
index 0000000..38136d2
--- /dev/null
+++ b/daemons/based/based_remote.c
@@ -0,0 +1,680 @@
+/*
+ * Copyright 2004-2021 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include <crm/crm.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <inttypes.h>           // PRIx64
+#include <sys/socket.h>
+#include <arpa/inet.h>
+
+#include <netinet/ip.h>
+
+#include <stdlib.h>
+#include <errno.h>
+#include <glib.h>
+
+#include <crm/msg_xml.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/common/xml.h>
+#include <crm/common/remote_internal.h>
+#include <crm/cib/internal.h>
+
+#include "pacemaker-based.h"
+
+/* #undef HAVE_PAM_PAM_APPL_H */
+/* #undef HAVE_GNUTLS_GNUTLS_H */
+
+#ifdef HAVE_GNUTLS_GNUTLS_H
+#  include <gnutls/gnutls.h>
+#endif
+
+#include <pwd.h>
+#include <grp.h>
+#if HAVE_SECURITY_PAM_APPL_H
+#  include <security/pam_appl.h>
+#  define HAVE_PAM 1
+#else
+#  if HAVE_PAM_PAM_APPL_H
+#    include <pam/pam_appl.h>
+#    define HAVE_PAM 1
+#  endif
+#endif
+
+extern int remote_tls_fd;
+extern gboolean cib_shutdown_flag;
+
+int init_remote_listener(int port, gboolean encrypted);
+void cib_remote_connection_destroy(gpointer user_data);
+
+#ifdef HAVE_GNUTLS_GNUTLS_H
+gnutls_dh_params_t dh_params;
+gnutls_anon_server_credentials_t anon_cred_s;
+static void
+debug_log(int level, const char *str)
+{
+    fputs(str, stderr);
+}
+#endif
+
+#define REMOTE_AUTH_TIMEOUT 10000
+
+int num_clients;
+int authenticate_user(const char *user, const char *passwd);
+static int cib_remote_listen(gpointer data);
+static int cib_remote_msg(gpointer data);
+
+static void
+remote_connection_destroy(gpointer user_data)
+{
+    crm_info("No longer listening for remote connections");
+    return;
+}
+
+int
+init_remote_listener(int port, gboolean encrypted)
+{
+    int rc;
+    int *ssock = NULL;
+    struct sockaddr_in saddr;
+    int optval;
+
+    static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
+        .dispatch = cib_remote_listen,
+        .destroy = remote_connection_destroy,
+    };
+
+    if (port <= 0) {
+        /* don't start it */
+        return 0;
+    }
+
+    if (encrypted) {
+#ifndef HAVE_GNUTLS_GNUTLS_H
+        crm_warn("TLS support is not available");
+        return 0;
+#else
+        crm_notice("Starting TLS listener on port %d", port);
+        crm_gnutls_global_init();
+        /* gnutls_global_set_log_level (10); */
+        gnutls_global_set_log_function(debug_log);
+        if (pcmk__init_tls_dh(&dh_params) != pcmk_rc_ok) {
+            return -1;
+        }
+        gnutls_anon_allocate_server_credentials(&anon_cred_s);
+        gnutls_anon_set_server_dh_params(anon_cred_s, dh_params);
+#endif
+    } else {
+        crm_warn("Starting plain-text listener on port %d", port);
+    }
+#ifndef HAVE_PAM
+    crm_warn("PAM is _not_ enabled!");
+#endif
+
+    /* create server socket */
+    ssock = malloc(sizeof(int));
+    if(ssock == NULL) {
+        crm_perror(LOG_ERR, "Listener socket allocation failed");
+        return -1;
+    }
+
+    *ssock = socket(AF_INET, SOCK_STREAM, 0);
+    if (*ssock == -1) {
+        crm_perror(LOG_ERR, "Listener socket creation failed");
+        free(ssock);
+        return -1;
+    }
+
+    /* reuse address */
+    optval = 1;
+    rc = setsockopt(*ssock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
+    if (rc < 0) {
+        crm_perror(LOG_WARNING,
+                   "Local address reuse not allowed on listener socket");
+    }
+
+    /* bind server socket */
+    memset(&saddr, '\0', sizeof(saddr));
+    saddr.sin_family = AF_INET;
+    saddr.sin_addr.s_addr = INADDR_ANY;
+    saddr.sin_port = htons(port);
+    if (bind(*ssock, (struct sockaddr *)&saddr, sizeof(saddr)) == -1) {
+        crm_perror(LOG_ERR, "Cannot bind to listener socket");
+        close(*ssock);
+        free(ssock);
+        return -2;
+    }
+    if (listen(*ssock, 10) == -1) {
+        crm_perror(LOG_ERR, "Cannot listen on socket");
+        close(*ssock);
+        free(ssock);
+        return -3;
+    }
+
+    mainloop_add_fd("cib-remote", G_PRIORITY_DEFAULT, *ssock, ssock, &remote_listen_fd_callbacks);
+    crm_debug("Started listener on port %d", port);
+
+    return *ssock;
+}
+
+static int
+check_group_membership(const char *usr, const char *grp)
+{
+    int index = 0;
+    struct passwd *pwd = NULL;
+    struct group *group = NULL;
+
+    CRM_CHECK(usr != NULL, return FALSE);
+    CRM_CHECK(grp != NULL, return FALSE);
+
+    pwd = getpwnam(usr);
+    if (pwd == NULL) {
+        crm_err("No user named '%s' exists!", usr);
+        return FALSE;
+    }
+
+    group = getgrgid(pwd->pw_gid);
+    if (group != NULL && pcmk__str_eq(grp, group->gr_name, pcmk__str_none)) {
+        return TRUE;
+    }
+
+    group = getgrnam(grp);
+    if (group == NULL) {
+        crm_err("No group named '%s' exists!", grp);
+        return FALSE;
+    }
+
+    while (TRUE) {
+        char *member = group->gr_mem[index++];
+
+        if (member == NULL) {
+            break;
+
+        } else if (pcmk__str_eq(usr, member, pcmk__str_none)) {
+            return TRUE;
+        }
+    };
+
+    return FALSE;
+}
+
+static gboolean
+cib_remote_auth(xmlNode * login)
+{
+    const char *user = NULL;
+    const char *pass = NULL;
+    const char *tmp = NULL;
+
+    crm_log_xml_info(login, "Login: ");
+    if (login == NULL) {
+        return FALSE;
+    }
+
+    tmp = crm_element_name(login);
+    if (!pcmk__str_eq(tmp, "cib_command", pcmk__str_casei)) {
+        crm_err("Wrong tag: %s", tmp);
+        return FALSE;
+    }
+
+    tmp = crm_element_value(login, "op");
+    if (!pcmk__str_eq(tmp, "authenticate", pcmk__str_casei)) {
+        crm_err("Wrong operation: %s", tmp);
+        return FALSE;
+    }
+
+    user = crm_element_value(login, "user");
+    pass = crm_element_value(login, "password");
+
+    if (!user || !pass) {
+        crm_err("missing auth credentials");
+        return FALSE;
+    }
+
+    /* Non-root daemons can only validate the password of the
+     * user they're running as
+     */
+    if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
+        crm_err("User is not a member of the required group");
+        return FALSE;
+
+    } else if (authenticate_user(user, pass) == FALSE) {
+        crm_err("PAM auth failed");
+        return FALSE;
+    }
+
+    return TRUE;
+}
+
+static gboolean
+remote_auth_timeout_cb(gpointer data)
+{
+    pcmk__client_t *client = data;
+
+    client->remote->auth_timeout = 0;
+
+    if (pcmk_is_set(client->flags, pcmk__client_authenticated)) {
+        return FALSE;
+    }
+
+    mainloop_del_fd(client->remote->source);
+    crm_err("Remote client authentication timed out");
+
+    return FALSE;
+}
+
+static int
+cib_remote_listen(gpointer data)
+{
+    int csock = 0;
+    unsigned laddr;
+    struct sockaddr_storage addr;
+    char ipstr[INET6_ADDRSTRLEN];
+    int ssock = *(int *)data;
+    int rc;
+
+    pcmk__client_t *new_client = NULL;
+
+    static struct mainloop_fd_callbacks remote_client_fd_callbacks = {
+        .dispatch = cib_remote_msg,
+        .destroy = cib_remote_connection_destroy,
+    };
+
+    /* accept the connection */
+    laddr = sizeof(addr);
+    memset(&addr, 0, sizeof(addr));
+    csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
+    if (csock == -1) {
+        crm_perror(LOG_ERR, "Could not accept socket connection");
+        return TRUE;
+    }
+
+    pcmk__sockaddr2str(&addr, ipstr);
+    crm_debug("New %s connection from %s",
+              ((ssock == remote_tls_fd)? "secure" : "clear-text"), ipstr);
+
+    rc = pcmk__set_nonblocking(csock);
+    if (rc != pcmk_rc_ok) {
+        crm_err("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
+                pcmk_rc_str(rc), rc);
+        close(csock);
+        return TRUE;
+    }
+
+    num_clients++;
+
+    new_client = pcmk__new_unauth_client(NULL);
+    new_client->remote = calloc(1, sizeof(pcmk__remote_t));
+
+    if (ssock == remote_tls_fd) {
+#ifdef HAVE_GNUTLS_GNUTLS_H
+        pcmk__set_client_flags(new_client, pcmk__client_tls);
+
+        /* create gnutls session for the server socket */
+        new_client->remote->tls_session = pcmk__new_tls_session(csock,
+                                                                GNUTLS_SERVER,
+                                                                GNUTLS_CRD_ANON,
+                                                                anon_cred_s);
+        if (new_client->remote->tls_session == NULL) {
+            close(csock);
+            return TRUE;
+        }
+#endif
+    } else {
+        pcmk__set_client_flags(new_client, pcmk__client_tcp);
+        new_client->remote->tcp_socket = csock;
+    }
+
+    // Require the client to authenticate within this time
+    new_client->remote->auth_timeout = g_timeout_add(REMOTE_AUTH_TIMEOUT,
+                                                     remote_auth_timeout_cb,
+                                                     new_client);
+    crm_info("Remote CIB client pending authentication "
+             CRM_XS " %p id: %s", new_client, new_client->id);
+
+    new_client->remote->source =
+        mainloop_add_fd("cib-remote-client", G_PRIORITY_DEFAULT, csock, new_client,
+                        &remote_client_fd_callbacks);
+
+    return TRUE;
+}
+
+void
+cib_remote_connection_destroy(gpointer user_data)
+{
+    pcmk__client_t *client = user_data;
+    int csock = 0;
+
+    if (client == NULL) {
+        return;
+    }
+
+    crm_trace("Cleaning up after client %s disconnect",
+              pcmk__client_name(client));
+
+    num_clients--;
+    crm_trace("Num unfree'd clients: %d", num_clients);
+
+    switch (PCMK__CLIENT_TYPE(client)) {
+        case pcmk__client_tcp:
+            csock = client->remote->tcp_socket;
+            break;
+#ifdef HAVE_GNUTLS_GNUTLS_H
+        case pcmk__client_tls:
+            if (client->remote->tls_session) {
+                void *sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
+
+                csock = GPOINTER_TO_INT(sock_ptr);
+                if (pcmk_is_set(client->flags,
+                                pcmk__client_tls_handshake_complete)) {
+                    gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_WR);
+                }
+                gnutls_deinit(*client->remote->tls_session);
+                gnutls_free(client->remote->tls_session);
+                client->remote->tls_session = NULL;
+            }
+            break;
+#endif
+        default:
+            crm_warn("Unknown transport for client %s "
+                     CRM_XS " flags=%#016" PRIx64,
+                     pcmk__client_name(client), client->flags);
+    }
+
+    if (csock > 0) {
+        close(csock);
+    }
+
+    pcmk__free_client(client);
+
+    crm_trace("Freed the cib client");
+
+    if (cib_shutdown_flag) {
+        cib_shutdown(0);
+    }
+    return;
+}
+
+static void
+cib_handle_remote_msg(pcmk__client_t *client, xmlNode *command)
+{
+    const char *value = NULL;
+
+    value = crm_element_name(command);
+    if (!pcmk__str_eq(value, "cib_command", pcmk__str_casei)) {
+        crm_log_xml_trace(command, "Bad command: ");
+        return;
+    }
+
+    if (client->name == NULL) {
+        value = crm_element_value(command, F_CLIENTNAME);
+        if (value == NULL) {
+            client->name = strdup(client->id);
+        } else {
+            client->name = strdup(value);
+        }
+    }
+
+    /* unset dangerous options */
+    xml_remove_prop(command, F_ORIG);
+    xml_remove_prop(command, F_CIB_HOST);
+    xml_remove_prop(command, F_CIB_GLOBAL_UPDATE);
+
+    crm_xml_add(command, F_TYPE, T_CIB);
+    crm_xml_add(command, F_CIB_CLIENTID, client->id);
+    crm_xml_add(command, F_CIB_CLIENTNAME, client->name);
+    crm_xml_add(command, F_CIB_USER, client->user);
+
+    if (crm_element_value(command, F_CIB_CALLID) == NULL) {
+        char *call_uuid = crm_generate_uuid();
+
+        /* fix the command */
+        crm_xml_add(command, F_CIB_CALLID, call_uuid);
+        free(call_uuid);
+    }
+
+    if (crm_element_value(command, F_CIB_CALLOPTS) == NULL) {
+        crm_xml_add_int(command, F_CIB_CALLOPTS, 0);
+    }
+
+    crm_log_xml_trace(command, "Remote command: ");
+    cib_common_callback_worker(0, 0, command, client, TRUE);
+}
+
+static int
+cib_remote_msg(gpointer data)
+{
+    xmlNode *command = NULL;
+    pcmk__client_t *client = data;
+    int rc;
+    int timeout = 1000;
+
+    if (pcmk_is_set(client->flags, pcmk__client_authenticated)) {
+        timeout = -1;
+    }
+
+    crm_trace("Remote %s message received for client %s",
+              pcmk__client_type_str(PCMK__CLIENT_TYPE(client)),
+              pcmk__client_name(client));
+
+#ifdef HAVE_GNUTLS_GNUTLS_H
+    if ((PCMK__CLIENT_TYPE(client) == pcmk__client_tls)
+        && !pcmk_is_set(client->flags, pcmk__client_tls_handshake_complete)) {
+
+        int rc = pcmk__read_handshake_data(client);
+
+        if (rc == EAGAIN) {
+            /* No more data is available at the moment. Just return for now;
+             * we'll get invoked again once the client sends more.
+             */
+            return 0;
+        } else if (rc != pcmk_rc_ok) {
+            return -1;
+        }
+
+        crm_debug("TLS handshake with remote CIB client completed");
+        pcmk__set_client_flags(client, pcmk__client_tls_handshake_complete);
+        if (client->remote->auth_timeout) {
+            g_source_remove(client->remote->auth_timeout);
+        }
+
+        // Require the client to authenticate within this time
+        client->remote->auth_timeout = g_timeout_add(REMOTE_AUTH_TIMEOUT,
+                                                     remote_auth_timeout_cb,
+                                                     client);
+        return 0;
+    }
+#endif
+
+    rc = pcmk__read_remote_message(client->remote, timeout);
+
+    /* must pass auth before we will process anything else */
+    if (!pcmk_is_set(client->flags, pcmk__client_authenticated)) {
+        xmlNode *reg;
+        const char *user = NULL;
+
+        command = pcmk__remote_message_xml(client->remote);
+        if (cib_remote_auth(command) == FALSE) {
+            free_xml(command);
+            return -1;
+        }
+
+        crm_notice("Remote CIB client connection accepted");
+        pcmk__set_client_flags(client, pcmk__client_authenticated);
+        g_source_remove(client->remote->auth_timeout);
+        client->remote->auth_timeout = 0;
+        client->name = crm_element_value_copy(command, "name");
+
+        user = crm_element_value(command, "user");
+        if (user) {
+            client->user = strdup(user);
+        }
+
+        /* send ACK */
+        reg = create_xml_node(NULL, "cib_result");
+        crm_xml_add(reg, F_CIB_OPERATION, CRM_OP_REGISTER);
+        crm_xml_add(reg, F_CIB_CLIENTID, client->id);
+        pcmk__remote_send_xml(client->remote, reg);
+        free_xml(reg);
+        free_xml(command);
+    }
+
+    command = pcmk__remote_message_xml(client->remote);
+    while (command) {
+        crm_trace("Remote client message received");
+        cib_handle_remote_msg(client, command);
+        free_xml(command);
+        command = pcmk__remote_message_xml(client->remote);
+    }
+
+    if (rc == ENOTCONN) {
+        crm_trace("Remote CIB client disconnected while reading from it");
+        return -1;
+    }
+
+    return 0;
+}
+
+#ifdef HAVE_PAM
+static int
+construct_pam_passwd(int num_msg, const struct pam_message **msg,
+                     struct pam_response **response, void *data)
+{
+    int count = 0;
+    struct pam_response *reply;
+    char *string = (char *)data;
+
+    CRM_CHECK(data, return PAM_CONV_ERR);
+    CRM_CHECK(num_msg == 1, return PAM_CONV_ERR);       /* We only want to handle one message */
+
+    reply = calloc(1, sizeof(struct pam_response));
+    CRM_ASSERT(reply != NULL);
+
+    for (count = 0; count < num_msg; ++count) {
+        switch (msg[count]->msg_style) {
+            case PAM_TEXT_INFO:
+                crm_info("PAM: %s", msg[count]->msg);
+                break;
+            case PAM_PROMPT_ECHO_OFF:
+            case PAM_PROMPT_ECHO_ON:
+                reply[count].resp_retcode = 0;
+                reply[count].resp = string;     /* We already made a copy */
+                break;
+            case PAM_ERROR_MSG:
+                /* In theory we'd want to print this, but then
+                 * we see the password prompt in the logs
+                 */
+                /* crm_err("PAM error: %s", msg[count]->msg); */
+                break;
+            default:
+                crm_err("Unhandled conversation type: %d", msg[count]->msg_style);
+                goto bail;
+        }
+    }
+
+    *response = reply;
+    reply = NULL;
+
+    return PAM_SUCCESS;
+
+  bail:
+    for (count = 0; count < num_msg; ++count) {
+        if (reply[count].resp != NULL) {
+            switch (msg[count]->msg_style) {
+                case PAM_PROMPT_ECHO_ON:
+                case PAM_PROMPT_ECHO_OFF:
+                    /* Erase the data - it contained a password */
+                    while (*(reply[count].resp)) {
+                        *(reply[count].resp)++ = '\0';
+                    }
+                    free(reply[count].resp);
+                    break;
+            }
+            reply[count].resp = NULL;
+        }
+    }
+    free(reply);
+    reply = NULL;
+
+    return PAM_CONV_ERR;
+}
+#endif
+
+int
+authenticate_user(const char *user, const char *passwd)
+{
+#ifndef HAVE_PAM
+    gboolean pass = TRUE;
+#else
+    int rc = 0;
+    gboolean pass = FALSE;
+    const void *p_user = NULL;
+
+    struct pam_conv p_conv;
+    struct pam_handle *pam_h = NULL;
+    static const char *pam_name = NULL;
+
+    if (pam_name == NULL) {
+        pam_name = getenv("CIB_pam_service");
+    }
+    if (pam_name == NULL) {
+        pam_name = "login";
+    }
+
+    p_conv.conv = construct_pam_passwd;
+    p_conv.appdata_ptr = strdup(passwd);
+
+    rc = pam_start(pam_name, user, &p_conv, &pam_h);
+    if (rc != PAM_SUCCESS) {
+        crm_err("Could not initialize PAM: %s (%d)", pam_strerror(pam_h, rc), rc);
+        goto bail;
+    }
+
+    rc = pam_authenticate(pam_h, 0);
+    if (rc != PAM_SUCCESS) {
+        crm_err("Authentication failed for %s: %s (%d)", user, pam_strerror(pam_h, rc), rc);
+        goto bail;
+    }
+
+    /* Make sure we authenticated the user we wanted to authenticate.
+     * Since we also run as non-root, it might be worth pre-checking
+     * the user has the same EID as us, since that the only user we
+     * can authenticate.
+     */
+    rc = pam_get_item(pam_h, PAM_USER, &p_user);
+    if (rc != PAM_SUCCESS) {
+        crm_err("Internal PAM error: %s (%d)", pam_strerror(pam_h, rc), rc);
+        goto bail;
+
+    } else if (p_user == NULL) {
+        crm_err("Unknown user authenticated.");
+        goto bail;
+
+    } else if (!pcmk__str_eq(p_user, user, pcmk__str_casei)) {
+        crm_err("User mismatch: %s vs. %s.", (const char *)p_user, (const char *)user);
+        goto bail;
+    }
+
+    rc = pam_acct_mgmt(pam_h, 0);
+    if (rc != PAM_SUCCESS) {
+        crm_err("Access denied: %s (%d)", pam_strerror(pam_h, rc), rc);
+        goto bail;
+    }
+    pass = TRUE;
+
+  bail:
+    pam_end(pam_h, rc);
+#endif
+    return pass;
+}
diff --git a/daemons/based/cib.pam b/daemons/based/cib.pam
new file mode 100644
index 0000000..5d0f655
--- /dev/null
+++ b/daemons/based/cib.pam
@@ -0,0 +1,6 @@
+# login: auth account password session
+# may require permission to read /etc/shadow 
+auth     include        common-auth
+account  include        common-account
+password include        common-password
+session  include        common-session
diff --git a/daemons/based/pacemaker-based.c b/daemons/based/pacemaker-based.c
new file mode 100644
index 0000000..129997e
--- /dev/null
+++ b/daemons/based/pacemaker-based.c
@@ -0,0 +1,442 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <pwd.h>
+#include <grp.h>
+#include <bzlib.h>
+#include <sys/types.h>
+
+#include <libxml/parser.h>
+
+#include <crm/crm.h>
+#include <crm/cib/internal.h>
+#include <crm/msg_xml.h>
+#include <crm/cluster/internal.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/mainloop.h>
+#include <crm/common/output_internal.h>
+#include <crm/common/xml.h>
+
+#include <pacemaker-based.h>
+
+#define SUMMARY "daemon for managing the configuration of a Pacemaker cluster"
+
+extern int init_remote_listener(int port, gboolean encrypted);
+gboolean cib_shutdown_flag = FALSE;
+int cib_status = pcmk_ok;
+
+crm_cluster_t *crm_cluster = NULL;
+
+GMainLoop *mainloop = NULL;
+gchar *cib_root = NULL;
+static gboolean preserve_status = FALSE;
+
+gboolean cib_writes_enabled = TRUE;
+
+int remote_fd = 0;
+int remote_tls_fd = 0;
+
+GHashTable *config_hash = NULL;
+GHashTable *local_notify_queue = NULL;
+
+pcmk__output_t *logger_out = NULL;
+
+static void cib_init(void);
+void cib_shutdown(int nsig);
+static bool startCib(const char *filename);
+extern int write_cib_contents(gpointer p);
+
+static crm_exit_t exit_code = CRM_EX_OK;
+
+static void
+cib_enable_writes(int nsig)
+{
+    crm_info("(Re)enabling disk writes");
+    cib_writes_enabled = TRUE;
+}
+
+/*!
+ * \internal
+ * \brief Set up options, users, and groups for stand-alone mode
+ *
+ * \param[out] error  GLib error object
+ *
+ * \return Standard Pacemaker return code
+ */
+static int
+setup_stand_alone(GError **error)
+{
+    int rc = 0;
+    struct passwd *pwentry = NULL;
+
+    preserve_status = TRUE;
+    cib_writes_enabled = FALSE;
+
+    errno = 0;
+    pwentry = getpwnam(CRM_DAEMON_USER);
+    if (pwentry == NULL) {
+        exit_code = CRM_EX_FATAL;
+        if (errno != 0) {
+            g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                        "Error getting password DB entry for %s: %s",
+                        CRM_DAEMON_USER, strerror(errno));
+            return errno;
+        }
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Password DB entry for '%s' not found", CRM_DAEMON_USER);
+        return ENXIO;
+    }
+
+    rc = setgid(pwentry->pw_gid);
+    if (rc < 0) {
+        exit_code = CRM_EX_FATAL;
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not set group to %d: %s",
+                    pwentry->pw_gid, strerror(errno));
+        return errno;
+    }
+
+    rc = initgroups(CRM_DAEMON_USER, pwentry->pw_gid);
+    if (rc < 0) {
+        exit_code = CRM_EX_FATAL;
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not setup groups for user %d: %s",
+                    pwentry->pw_uid, strerror(errno));
+        return errno;
+    }
+
+    rc = setuid(pwentry->pw_uid);
+    if (rc < 0) {
+        exit_code = CRM_EX_FATAL;
+        g_set_error(error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not set user to %d: %s",
+                    pwentry->pw_uid, strerror(errno));
+        return errno;
+    }
+    return pcmk_rc_ok;
+}
+
+static GOptionEntry entries[] = {
+    { "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &stand_alone,
+      "(Advanced use only) Run in stand-alone mode", NULL },
+
+    { "disk-writes", 'w', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
+      &cib_writes_enabled,
+      "(Advanced use only) Enable disk writes (enabled by default unless in "
+      "stand-alone mode)", NULL },
+
+    { "cib-root", 'r', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME, &cib_root,
+      "(Advanced use only) Directory where the CIB XML file should be located "
+      "(default: " CRM_CONFIG_DIR ")", NULL },
+
+    { NULL }
+};
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group)
+{
+    GOptionContext *context = NULL;
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group,
+                                      "[metadata]");
+    pcmk__add_main_args(context, entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    int rc = pcmk_rc_ok;
+    crm_ipc_t *old_instance = NULL;
+
+    pcmk__output_t *out = NULL;
+
+    GError *error = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "r");
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    crm_log_preinit(NULL, argc, argv);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    rc = pcmk__log_output_new(&logger_out);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Error creating output format log: %s", pcmk_rc_str(rc));
+        goto done;
+    }
+    pcmk__output_set_log_level(logger_out, LOG_TRACE);
+
+    mainloop_add_signal(SIGTERM, cib_shutdown);
+    mainloop_add_signal(SIGPIPE, cib_enable_writes);
+
+    cib_writer = mainloop_add_trigger(G_PRIORITY_LOW, write_cib_contents, NULL);
+
+    if ((g_strv_length(processed_args) >= 2)
+        && pcmk__str_eq(processed_args[1], "metadata", pcmk__str_none)) {
+        cib_metadata();
+        goto done;
+    }
+
+    pcmk__cli_init_logging("pacemaker-based", args->verbosity);
+    crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
+    crm_notice("Starting Pacemaker CIB manager");
+
+    old_instance = crm_ipc_new(PCMK__SERVER_BASED_RO, 0);
+    if (old_instance == NULL) {
+        /* crm_ipc_new() will have already logged an error message with
+         * crm_err()
+         */
+        exit_code = CRM_EX_FATAL;
+        goto done;
+    }
+
+    if (crm_ipc_connect(old_instance)) {
+        /* IPC end-point already up */
+        crm_ipc_close(old_instance);
+        crm_ipc_destroy(old_instance);
+        crm_err("pacemaker-based is already active, aborting startup");
+        goto done;
+    } else {
+        /* not up or not authentic, we'll proceed either way */
+        crm_ipc_destroy(old_instance);
+        old_instance = NULL;
+    }
+
+    if (stand_alone) {
+        rc = setup_stand_alone(&error);
+        if (rc != pcmk_rc_ok) {
+            goto done;
+        }
+    }
+
+    if (cib_root == NULL) {
+        cib_root = g_strdup(CRM_CONFIG_DIR);
+    } else {
+        crm_notice("Using custom config location: %s", cib_root);
+    }
+
+    if (!pcmk__daemon_can_write(cib_root, NULL)) {
+        exit_code = CRM_EX_FATAL;
+        crm_err("Terminating due to bad permissions on %s", cib_root);
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Bad permissions on %s (see logs for details)", cib_root);
+        goto done;
+    }
+
+    crm_peer_init();
+
+    // Read initial CIB, connect to cluster, and start IPC servers
+    cib_init();
+
+    // Run the main loop
+    mainloop = g_main_loop_new(NULL, FALSE);
+    crm_notice("Pacemaker CIB manager successfully started and accepting connections");
+    g_main_loop_run(mainloop);
+
+    /* If main loop returned, clean up and exit. We disconnect in case
+     * terminate_cib() was called with fast=-1.
+     */
+    crm_cluster_disconnect(crm_cluster);
+    pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    crm_peer_destroy();
+
+    if (local_notify_queue != NULL) {
+        g_hash_table_destroy(local_notify_queue);
+    }
+
+    if (config_hash != NULL) {
+        g_hash_table_destroy(config_hash);
+    }
+    pcmk__client_cleanup();
+    pcmk_cluster_free(crm_cluster);
+    g_free(cib_root);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+    pcmk__unregister_formats();
+    crm_exit(exit_code);
+}
+
+#if SUPPORT_COROSYNC
+static void
+cib_cs_dispatch(cpg_handle_t handle,
+                 const struct cpg_name *groupName,
+                 uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
+{
+    uint32_t kind = 0;
+    xmlNode *xml = NULL;
+    const char *from = NULL;
+    char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);
+
+    if(data == NULL) {
+        return;
+    }
+    if (kind == crm_class_cluster) {
+        xml = string2xml(data);
+        if (xml == NULL) {
+            crm_err("Invalid XML: '%.120s'", data);
+            free(data);
+            return;
+        }
+        crm_xml_add(xml, F_ORIG, from);
+        /* crm_xml_add_int(xml, F_SEQ, wrapper->id); */
+        cib_peer_callback(xml, NULL);
+    }
+
+    free_xml(xml);
+    free(data);
+}
+
+static void
+cib_cs_destroy(gpointer user_data)
+{
+    if (cib_shutdown_flag) {
+        crm_info("Corosync disconnection complete");
+    } else {
+        crm_crit("Lost connection to cluster layer, shutting down");
+        terminate_cib(__func__, CRM_EX_DISCONNECT);
+    }
+}
+#endif
+
+static void
+cib_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
+{
+    switch (type) {
+        case crm_status_processes:
+            if (cib_legacy_mode()
+                && !pcmk_is_set(node->processes, crm_get_cluster_proc())) {
+
+                uint32_t old = data? *(const uint32_t *)data : 0;
+
+                if ((node->processes ^ old) & crm_proc_cpg) {
+                    crm_info("Attempting to disable legacy mode after %s left the cluster",
+                             node->uname);
+                    legacy_mode = FALSE;
+                }
+            }
+            break;
+
+        case crm_status_uname:
+        case crm_status_nstate:
+            if (cib_shutdown_flag && (crm_active_peers() < 2)
+                && (pcmk__ipc_client_count() == 0)) {
+
+                crm_info("No more peers");
+                terminate_cib(__func__, -1);
+            }
+            break;
+    }
+}
+
+static void
+cib_init(void)
+{
+    crm_cluster = pcmk_cluster_new();
+
+#if SUPPORT_COROSYNC
+    if (is_corosync_cluster()) {
+        crm_cluster->destroy = cib_cs_destroy;
+        crm_cluster->cpg.cpg_deliver_fn = cib_cs_dispatch;
+        crm_cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership;
+    }
+#endif // SUPPORT_COROSYNC
+
+    config_hash = pcmk__strkey_table(free, free);
+
+    if (startCib("cib.xml") == FALSE) {
+        crm_crit("Cannot start CIB... terminating");
+        crm_exit(CRM_EX_NOINPUT);
+    }
+
+    if (!stand_alone) {
+        crm_set_status_callback(&cib_peer_update_callback);
+
+        if (!crm_cluster_connect(crm_cluster)) {
+            crm_crit("Cannot sign in to the cluster... terminating");
+            crm_exit(CRM_EX_FATAL);
+        }
+    }
+
+    pcmk__serve_based_ipc(&ipcs_ro, &ipcs_rw, &ipcs_shm, &ipc_ro_callbacks,
+                          &ipc_rw_callbacks);
+
+    if (stand_alone) {
+        based_is_primary = true;
+    }
+}
+
+static bool
+startCib(const char *filename)
+{
+    gboolean active = FALSE;
+    xmlNode *cib = readCibXmlFile(cib_root, filename, !preserve_status);
+
+    if (activateCibXml(cib, TRUE, "start") == 0) {
+        int port = 0;
+
+        active = TRUE;
+
+        cib_read_config(config_hash, cib);
+
+        pcmk__scan_port(crm_element_value(cib, "remote-tls-port"), &port);
+        if (port >= 0) {
+            remote_tls_fd = init_remote_listener(port, TRUE);
+        }
+
+        pcmk__scan_port(crm_element_value(cib, "remote-clear-port"), &port);
+        if (port >= 0) {
+            remote_fd = init_remote_listener(port, FALSE);
+        }
+    }
+    return active;
+}
diff --git a/daemons/based/pacemaker-based.h b/daemons/based/pacemaker-based.h
new file mode 100644
index 0000000..05e49b3
--- /dev/null
+++ b/daemons/based/pacemaker-based.h
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef PACEMAKER_BASED__H
+#  define PACEMAKER_BASED__H
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <glib.h>
+#include <errno.h>
+#include <fcntl.h>
+
+#include <crm/crm.h>
+#include <crm/cib.h>
+#include <crm/common/xml.h>
+#include <crm/cluster.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/common/mainloop.h>
+#include <crm/cib/internal.h>
+
+#ifdef HAVE_GNUTLS_GNUTLS_H
+#  include <gnutls/gnutls.h>
+#endif
+
+// CIB-specific client flags
+enum cib_client_flags {
+    // Notifications
+    cib_notify_pre     = (UINT64_C(1) << 0),
+    cib_notify_post    = (UINT64_C(1) << 1),
+    cib_notify_replace = (UINT64_C(1) << 2),
+    cib_notify_confirm = (UINT64_C(1) << 3),
+    cib_notify_diff    = (UINT64_C(1) << 4),
+
+    // Whether client is another cluster daemon
+    cib_is_daemon      = (UINT64_C(1) << 12),
+};
+
+typedef struct cib_operation_s {
+    const char *operation;
+    gboolean modifies_cib;
+    gboolean needs_privileges;
+    int (*prepare) (xmlNode *, xmlNode **, const char **);
+    int (*cleanup) (int, xmlNode **, xmlNode **);
+    int (*fn) (const char *, int, const char *, xmlNode *,
+               xmlNode *, xmlNode *, xmlNode **, xmlNode **);
+} cib_operation_t;
+
+extern bool based_is_primary;
+extern GHashTable *config_hash;
+extern xmlNode *the_cib;
+extern crm_trigger_t *cib_writer;
+extern gboolean cib_writes_enabled;
+
+extern GMainLoop *mainloop;
+extern crm_cluster_t *crm_cluster;
+extern GHashTable *local_notify_queue;
+extern gboolean legacy_mode;
+extern gboolean stand_alone;
+extern gboolean cib_shutdown_flag;
+extern gchar *cib_root;
+extern int cib_status;
+extern pcmk__output_t *logger_out;
+
+extern struct qb_ipcs_service_handlers ipc_ro_callbacks;
+extern struct qb_ipcs_service_handlers ipc_rw_callbacks;
+extern qb_ipcs_service_t *ipcs_ro;
+extern qb_ipcs_service_t *ipcs_rw;
+extern qb_ipcs_service_t *ipcs_shm;
+
+void cib_peer_callback(xmlNode *msg, void *private_data);
+void cib_common_callback_worker(uint32_t id, uint32_t flags,
+                                xmlNode *op_request, pcmk__client_t *cib_client,
+                                gboolean privileged);
+void cib_shutdown(int nsig);
+void terminate_cib(const char *caller, int fast);
+gboolean cib_legacy_mode(void);
+
+gboolean uninitializeCib(void);
+xmlNode *readCibXmlFile(const char *dir, const char *file,
+                        gboolean discard_status);
+int activateCibXml(xmlNode *doc, gboolean to_disk, const char *op);
+
+int cib_process_shutdown_req(const char *op, int options, const char *section,
+                             xmlNode *req, xmlNode *input,
+                             xmlNode *existing_cib, xmlNode **result_cib,
+                             xmlNode **answer);
+int cib_process_default(const char *op, int options, const char *section,
+                        xmlNode *req, xmlNode *input, xmlNode *existing_cib,
+                        xmlNode **result_cib, xmlNode **answer);
+int cib_process_ping(const char *op, int options, const char *section,
+                     xmlNode *req, xmlNode *input, xmlNode *existing_cib,
+                     xmlNode **result_cib, xmlNode **answer);
+int cib_process_readwrite(const char *op, int options, const char *section,
+                          xmlNode *req, xmlNode *input, xmlNode *existing_cib,
+                          xmlNode **result_cib, xmlNode **answer);
+int cib_process_replace_svr(const char *op, int options, const char *section,
+                            xmlNode *req, xmlNode *input, xmlNode *existing_cib,
+                            xmlNode **result_cib, xmlNode **answer);
+int cib_server_process_diff(const char *op, int options, const char *section,
+                            xmlNode *req, xmlNode *input, xmlNode *existing_cib,
+                            xmlNode **result_cib, xmlNode **answer);
+int cib_process_sync(const char *op, int options, const char *section,
+                     xmlNode *req, xmlNode *input, xmlNode *existing_cib,
+                     xmlNode **result_cib, xmlNode **answer);
+int cib_process_sync_one(const char *op, int options, const char *section,
+                         xmlNode *req, xmlNode *input, xmlNode *existing_cib,
+                         xmlNode **result_cib, xmlNode **answer);
+int cib_process_delete_absolute(const char *op, int options,
+                                const char *section, xmlNode *req,
+                                xmlNode *input, xmlNode *existing_cib,
+                                xmlNode **result_cib, xmlNode **answer);
+int cib_process_upgrade_server(const char *op, int options, const char *section,
+                               xmlNode *req, xmlNode *input,
+                               xmlNode *existing_cib, xmlNode **result_cib,
+                               xmlNode **answer);
+void send_sync_request(const char *host);
+int sync_our_cib(xmlNode *request, gboolean all);
+
+xmlNode *cib_msg_copy(xmlNode *msg, gboolean with_data);
+int cib_get_operation_id(const char *op, int *operation);
+cib_op_t *cib_op_func(int call_type);
+gboolean cib_op_modifies(int call_type);
+int cib_op_prepare(int call_type, xmlNode *request, xmlNode **input,
+                   const char **section);
+int cib_op_cleanup(int call_type, int options, xmlNode **input,
+                   xmlNode **output);
+int cib_op_can_run(int call_type, int call_options, bool privileged);
+void cib_diff_notify(const char *op, int result, const char *call_id,
+                     const char *client_id, const char *client_name,
+                     const char *origin, xmlNode *update, xmlNode *diff);
+void cib_replace_notify(const char *op, int result, const char *call_id,
+                        const char *client_id, const char *client_name,
+                        const char *origin, xmlNode *update, xmlNode *diff,
+                        uint32_t change_section);
+
+static inline const char *
+cib_config_lookup(const char *opt)
+{
+    return g_hash_table_lookup(config_hash, opt);
+}
+
+#endif // PACEMAKER_BASED__H
diff --git a/daemons/controld/Makefile.am b/daemons/controld/Makefile.am
new file mode 100644
index 0000000..08be1ff
--- /dev/null
+++ b/daemons/controld/Makefile.am
@@ -0,0 +1,87 @@
+#
+# Copyright 2018-2023 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+include $(top_srcdir)/mk/common.mk
+include $(top_srcdir)/mk/man.mk
+
+halibdir	= $(CRM_DAEMON_DIR)
+
+halib_PROGRAMS	= pacemaker-controld
+
+noinst_HEADERS	= controld_alerts.h		\
+		  controld_callbacks.h		\
+		  controld_cib.h		\
+		  controld_fencing.h		\
+		  controld_fsa.h		\
+		  controld_globals.h		\
+		  controld_lrm.h		\
+		  controld_membership.h		\
+		  controld_messages.h		\
+		  controld_metadata.h		\
+		  controld_throttle.h		\
+		  controld_timers.h		\
+		  controld_transition.h		\
+		  controld_utils.h		\
+		  pacemaker-controld.h
+
+pacemaker_controld_CFLAGS	= $(CFLAGS_HARDENED_EXE)
+pacemaker_controld_LDFLAGS	= $(LDFLAGS_HARDENED_EXE)
+
+pacemaker_controld_LDADD = $(top_builddir)/lib/fencing/libstonithd.la		\
+			   $(top_builddir)/lib/pacemaker/libpacemaker.la \
+			   $(top_builddir)/lib/pengine/libpe_rules.la		\
+			   $(top_builddir)/lib/cib/libcib.la			\
+			   $(top_builddir)/lib/cluster/libcrmcluster.la		\
+			   $(top_builddir)/lib/common/libcrmcommon.la		\
+			   $(top_builddir)/lib/services/libcrmservice.la	\
+			   $(top_builddir)/lib/lrmd/liblrmd.la			\
+			   $(CLUSTERLIBS)
+
+pacemaker_controld_SOURCES = pacemaker-controld.c	\
+			     controld_alerts.c		\
+			     controld_attrd.c		\
+			     controld_callbacks.c	\
+			     controld_cib.c		\
+			     controld_control.c		\
+			     controld_corosync.c	\
+			     controld_election.c	\
+			     controld_execd.c		\
+			     controld_execd_state.c	\
+			     controld_fencing.c		\
+			     controld_fsa.c		\
+			     controld_join_client.c	\
+			     controld_join_dc.c		\
+			     controld_matrix.c		\
+			     controld_membership.c	\
+			     controld_messages.c	\
+			     controld_metadata.c	\
+			     controld_remote_ra.c	\
+			     controld_schedulerd.c	\
+			     controld_te_actions.c	\
+			     controld_te_callbacks.c	\
+			     controld_te_events.c	\
+			     controld_te_utils.c	\
+			     controld_throttle.c	\
+			     controld_timers.c		\
+			     controld_transition.c	\
+			     controld_utils.c
+
+if BUILD_XML_HELP
+man7_MANS 	= pacemaker-controld.7
+endif
+
+CLEANFILES = $(man7_MANS)
+
+if BUILD_LEGACY_LINKS
+install-exec-hook:
+	cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f crmd && $(LN_S) pacemaker-controld crmd
+
+uninstall-hook:
+	cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f crmd
+endif
diff --git a/daemons/controld/controld_alerts.c b/daemons/controld/controld_alerts.c
new file mode 100644
index 0000000..27a5ce2
--- /dev/null
+++ b/daemons/controld/controld_alerts.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2012-2021 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <glib.h>
+#include <libxml/tree.h>
+
+#include <crm/fencing/internal.h>
+#include <crm/lrmd.h>
+#include <crm/lrmd_internal.h>
+#include <crm/pengine/rules_internal.h>
+#include <crm/pengine/status.h>
+#include <crm/stonith-ng.h>
+
+#include <pacemaker-controld.h>
+
+static GList *crmd_alert_list = NULL;
+
+void
+crmd_unpack_alerts(xmlNode *alerts)
+{
+    pe_free_alert_list(crmd_alert_list);
+    crmd_alert_list = pe_unpack_alerts(alerts);
+}
+
+void
+crmd_alert_node_event(crm_node_t *node)
+{
+    lrm_state_t *lrm_state;
+
+    if (crmd_alert_list == NULL) {
+        return;
+    }
+
+    lrm_state = lrm_state_find(controld_globals.our_nodename);
+    if (lrm_state == NULL) {
+        return;
+    }
+
+    lrmd_send_node_alert((lrmd_t *) lrm_state->conn, crmd_alert_list,
+                         node->uname, node->id, node->state);
+}
+
+void
+crmd_alert_fencing_op(stonith_event_t * e)
+{
+    char *desc;
+    lrm_state_t *lrm_state;
+
+    if (crmd_alert_list == NULL) {
+        return;
+    }
+
+    lrm_state = lrm_state_find(controld_globals.our_nodename);
+    if (lrm_state == NULL) {
+        return;
+    }
+
+    desc = stonith__event_description(e);
+    lrmd_send_fencing_alert((lrmd_t *) lrm_state->conn, crmd_alert_list,
+                            e->target, e->operation, desc, e->result);
+    free(desc);
+}
+
+void
+crmd_alert_resource_op(const char *node, lrmd_event_data_t * op)
+{
+    lrm_state_t *lrm_state;
+
+    if (crmd_alert_list == NULL) {
+        return;
+    }
+
+    lrm_state = lrm_state_find(controld_globals.our_nodename);
+    if (lrm_state == NULL) {
+        return;
+    }
+
+    lrmd_send_resource_alert((lrmd_t *) lrm_state->conn, crmd_alert_list, node,
+                             op);
+}
diff --git a/daemons/controld/controld_alerts.h b/daemons/controld/controld_alerts.h
new file mode 100644
index 0000000..ec5852a
--- /dev/null
+++ b/daemons/controld/controld_alerts.h
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2015-2021 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef CONTROLD_ALERTS__H
+#  define CONTROLD_ALERTS__H
+
+#  include <crm/crm.h>
+#  include <crm/cluster.h>
+#  include <crm/stonith-ng.h>
+
+void crmd_unpack_alerts(xmlNode *alerts);
+void crmd_alert_node_event(crm_node_t *node);
+void crmd_alert_fencing_op(stonith_event_t *e);
+void crmd_alert_resource_op(const char *node, lrmd_event_data_t *op);
+
+#endif
diff --git a/daemons/controld/controld_attrd.c b/daemons/controld/controld_attrd.c
new file mode 100644
index 0000000..923abb9
--- /dev/null
+++ b/daemons/controld/controld_attrd.c
@@ -0,0 +1,160 @@
+/*
+ * Copyright 2006-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm/crm.h>
+#include <crm/common/attrd_internal.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_attrd_internal.h>
+#include <crm/msg_xml.h>
+
+#include <pacemaker-controld.h>
+
+static pcmk_ipc_api_t *attrd_api = NULL;
+
+void
+controld_close_attrd_ipc(void)
+{
+    if (attrd_api != NULL) {
+        crm_trace("Closing connection to pacemaker-attrd");
+        pcmk_disconnect_ipc(attrd_api);
+        pcmk_free_ipc_api(attrd_api);
+        attrd_api = NULL;
+    }
+}
+
+static inline const char *
+node_type(bool is_remote)
+{
+    return is_remote? "Pacemaker Remote" : "cluster";
+}
+
+static inline const char *
+when(void)
+{
+    return pcmk_is_set(controld_globals.fsa_input_register,
+                       R_SHUTDOWN)? " at shutdown" : "";
+}
+
+static void
+handle_attr_error(void)
+{
+    if (AM_I_DC) {
+        /* We are unable to provide accurate information to the
+         * scheduler, so allow another node to take over DC.
+         * @TODO Should we do this unconditionally on any failure?
+         */
+        crmd_exit(CRM_EX_FATAL);
+
+    } else if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+        // Fast-track shutdown since unable to request via attribute
+        register_fsa_input(C_FSA_INTERNAL, I_FAIL, NULL);
+    }
+}
+
+void
+update_attrd(const char *host, const char *name, const char *value,
+             const char *user_name, gboolean is_remote_node)
+{
+    int rc = pcmk_rc_ok;
+
+    if (attrd_api == NULL) {
+        rc = pcmk_new_ipc_api(&attrd_api, pcmk_ipc_attrd);
+    }
+    if (rc == pcmk_rc_ok) {
+        uint32_t attrd_opts = pcmk__node_attr_value;
+
+        if (is_remote_node) {
+            pcmk__set_node_attr_flags(attrd_opts, pcmk__node_attr_remote);
+        }
+        rc = pcmk__attrd_api_update(attrd_api, host, name, value,
+                                    NULL, NULL, user_name, attrd_opts);
+    }
+    if (rc != pcmk_rc_ok) {
+        do_crm_log(AM_I_DC? LOG_CRIT : LOG_ERR,
+                   "Could not update attribute %s=%s for %s node %s%s: %s "
+                   CRM_XS " rc=%d", name, value, node_type(is_remote_node),
+                   host, when(), pcmk_rc_str(rc), rc);
+        handle_attr_error();
+    }
+}
+
+void
+update_attrd_list(GList *attrs, uint32_t opts)
+{
+    int rc = pcmk_rc_ok;
+
+    if (attrd_api == NULL) {
+        rc = pcmk_new_ipc_api(&attrd_api, pcmk_ipc_attrd);
+    }
+    if (rc == pcmk_rc_ok) {
+        rc = pcmk__attrd_api_update_list(attrd_api, attrs, NULL, NULL, NULL,
+                                         opts | pcmk__node_attr_value);
+    }
+    if (rc != pcmk_rc_ok) {
+        do_crm_log(AM_I_DC? LOG_CRIT : LOG_ERR,
+                   "Could not update multiple node attributes: %s "
+                   CRM_XS " rc=%d", pcmk_rc_str(rc), rc);
+        handle_attr_error();
+    }
+}
+
+void
+update_attrd_remote_node_removed(const char *host, const char *user_name)
+{
+    int rc = pcmk_rc_ok;
+
+    if (attrd_api == NULL) {
+        rc = pcmk_new_ipc_api(&attrd_api, pcmk_ipc_attrd);
+    }
+    if (rc == pcmk_rc_ok) {
+        crm_trace("Asking attribute manager to purge Pacemaker Remote node %s",
+                  host);
+        rc = pcmk__attrd_api_purge(attrd_api, host);
+    }
+    if (rc != pcmk_rc_ok) {
+        crm_err("Could not purge Pacemaker Remote node %s "
+                "in attribute manager%s: %s " CRM_XS " rc=%d",
+                host, when(), pcmk_rc_str(rc), rc);
+    }
+}
+
+void
+update_attrd_clear_failures(const char *host, const char *rsc, const char *op,
+                            const char *interval_spec, gboolean is_remote_node)
+{
+    int rc = pcmk_rc_ok;
+
+    if (attrd_api == NULL) {
+        rc = pcmk_new_ipc_api(&attrd_api, pcmk_ipc_attrd);
+    }
+    if (rc == pcmk_rc_ok) {
+        const char *op_desc = pcmk__s(op, "operations");
+        const char *interval_desc = "all";
+        uint32_t attrd_opts = pcmk__node_attr_none;
+
+        if (op != NULL) {
+            interval_desc = pcmk__s(interval_spec, "nonrecurring");
+        }
+        if (is_remote_node) {
+            pcmk__set_node_attr_flags(attrd_opts, pcmk__node_attr_remote);
+        }
+        crm_info("Asking attribute manager to clear failure of %s %s for %s "
+                 "on %s node %s", interval_desc, op_desc, rsc,
+                 node_type(is_remote_node), host);
+        rc = pcmk__attrd_api_clear_failures(attrd_api, host, rsc, op,
+                                            interval_spec, NULL, attrd_opts);
+    }
+    if (rc != pcmk_rc_ok) {
+        crm_err("Could not clear failure attributes for %s on %s node %s%s: %s "
+                CRM_XS " rc=%d", pcmk__s(rsc, "all resources"),
+                node_type(is_remote_node), host, when(), pcmk_rc_str(rc), rc);
+    }
+}
diff --git a/daemons/controld/controld_callbacks.c b/daemons/controld/controld_callbacks.c
new file mode 100644
index 0000000..d578adc
--- /dev/null
+++ b/daemons/controld/controld_callbacks.c
@@ -0,0 +1,367 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <string.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/cluster.h>
+#include <crm/cib.h>
+
+#include <pacemaker-controld.h>
+
+/* From join_dc... */
+extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
+
+void
+crmd_ha_msg_filter(xmlNode * msg)
+{
+    if (AM_I_DC) {
+        const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
+
+        if (pcmk__str_eq(sys_from, CRM_SYSTEM_DC, pcmk__str_casei)) {
+            const char *from = crm_element_value(msg, F_ORIG);
+
+            if (!pcmk__str_eq(from, controld_globals.our_nodename,
+                              pcmk__str_casei)) {
+                int level = LOG_INFO;
+                const char *op = crm_element_value(msg, F_CRM_TASK);
+
+                /* make sure the election happens NOW */
+                if (controld_globals.fsa_state != S_ELECTION) {
+                    ha_msg_input_t new_input;
+
+                    level = LOG_WARNING;
+                    new_input.msg = msg;
+                    register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, &new_input,
+                                           __func__);
+                }
+
+                do_crm_log(level, "Another DC detected: %s (op=%s)", from, op);
+                goto done;
+            }
+        }
+
+    } else {
+        const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
+
+        if (pcmk__str_eq(sys_to, CRM_SYSTEM_DC, pcmk__str_casei)) {
+            return;
+        }
+    }
+
+    /* crm_log_xml_trace(msg, "HA[inbound]"); */
+    route_message(C_HA_MESSAGE, msg);
+
+  done:
+    controld_trigger_fsa();
+}
+
+/*!
+ * \internal
+ * \brief Check whether a node is online
+ *
+ * \param[in] node  Node to check
+ *
+ * \retval -1 if completely dead
+ * \retval  0 if partially alive
+ * \retval  1 if completely alive
+ */
+static int
+node_alive(const crm_node_t *node)
+{
+    if (pcmk_is_set(node->flags, crm_remote_node)) {
+        // Pacemaker Remote nodes can't be partially alive
+        return pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_casei) ? 1: -1;
+
+    } else if (crm_is_peer_active(node)) {
+        // Completely up cluster node: both cluster member and peer
+        return 1;
+
+    } else if (!pcmk_is_set(node->processes, crm_get_cluster_proc())
+               && !pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_casei)) {
+        // Completely down cluster node: neither cluster member nor peer
+        return -1;
+    }
+
+    // Partially up cluster node: only cluster member or only peer
+    return 0;
+}
+
+#define state_text(state) ((state)? (const char *)(state) : "in unknown state")
+
+void
+peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
+{
+    uint32_t old = 0;
+    bool appeared = FALSE;
+    bool is_remote = pcmk_is_set(node->flags, crm_remote_node);
+
+    /* The controller waits to receive some information from the membership
+     * layer before declaring itself operational. If this is being called for a
+     * cluster node, indicate that we have it.
+     */
+    if (!is_remote) {
+        controld_set_fsa_input_flags(R_PEER_DATA);
+    }
+
+    if (type == crm_status_processes
+        && pcmk_is_set(node->processes, crm_get_cluster_proc())
+        && !AM_I_DC
+        && !is_remote) {
+        /*
+         * This is a hack until we can send to a nodeid and/or we fix node name lookups
+         * These messages are ignored in crmd_ha_msg_filter()
+         */
+        xmlNode *query = create_request(CRM_OP_HELLO, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
+
+        crm_debug("Sending hello to node %u so that it learns our node name", node->id);
+        send_cluster_message(node, crm_msg_crmd, query, FALSE);
+
+        free_xml(query);
+    }
+
+    if (node->uname == NULL) {
+        return;
+    }
+
+    switch (type) {
+        case crm_status_uname:
+            /* If we've never seen the node, then it also won't be in the status section */
+            crm_info("%s node %s is now %s",
+                     (is_remote? "Remote" : "Cluster"),
+                     node->uname, state_text(node->state));
+            return;
+
+        case crm_status_nstate:
+            /* This callback should not be called unless the state actually
+             * changed, but here's a failsafe just in case.
+             */
+            CRM_CHECK(!pcmk__str_eq(data, node->state, pcmk__str_casei),
+                      return);
+
+            crm_info("%s node %s is now %s (was %s)",
+                     (is_remote? "Remote" : "Cluster"),
+                     node->uname, state_text(node->state), state_text(data));
+
+            if (pcmk__str_eq(CRM_NODE_MEMBER, node->state, pcmk__str_casei)) {
+                appeared = TRUE;
+                if (!is_remote) {
+                    remove_stonith_cleanup(node->uname);
+                }
+            } else {
+                controld_remove_failed_sync_node(node->uname);
+                controld_remove_voter(node->uname);
+            }
+
+            crmd_alert_node_event(node);
+            break;
+
+        case crm_status_processes:
+            CRM_CHECK(data != NULL, return);
+            old = *(const uint32_t *)data;
+            appeared = pcmk_is_set(node->processes, crm_get_cluster_proc());
+
+            {
+                const char *dc_s = controld_globals.dc_name;
+
+                if ((dc_s == NULL) && AM_I_DC) {
+                    dc_s = "true";
+                }
+
+                crm_info("Node %s is %s a peer " CRM_XS
+                         " DC=%s old=%#07x new=%#07x",
+                         node->uname, (appeared? "now" : "no longer"),
+                         pcmk__s(dc_s, "<none>"), old, node->processes);
+            }
+
+            if (!pcmk_is_set((node->processes ^ old), crm_get_cluster_proc())) {
+                /* Peer status did not change. This should not be possible,
+                 * since we don't track process flags other than peer status.
+                 */
+                crm_trace("Process flag %#7x did not change from %#7x to %#7x",
+                          crm_get_cluster_proc(), old, node->processes);
+                return;
+
+            }
+
+            if (!appeared) {
+                node->peer_lost = time(NULL);
+                controld_remove_failed_sync_node(node->uname);
+                controld_remove_voter(node->uname);
+            }
+
+            if (!pcmk_is_set(controld_globals.fsa_input_register,
+                             R_CIB_CONNECTED)) {
+                crm_trace("Ignoring peer status change because not connected to CIB");
+                return;
+
+            } else if (controld_globals.fsa_state == S_STOPPING) {
+                crm_trace("Ignoring peer status change because stopping");
+                return;
+            }
+
+            if (!appeared
+                && pcmk__str_eq(node->uname, controld_globals.our_nodename,
+                                pcmk__str_casei)) {
+                /* Did we get evicted? */
+                crm_notice("Our peer connection failed");
+                register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);
+
+            } else if (pcmk__str_eq(node->uname, controld_globals.dc_name,
+                                    pcmk__str_casei)
+                       && !crm_is_peer_active(node)) {
+                /* Did the DC leave us? */
+                crm_notice("Our peer on the DC (%s) is dead",
+                           controld_globals.dc_name);
+                register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);
+
+                /* @COMPAT DC < 1.1.13: If a DC shuts down normally, we don't
+                 * want to fence it. Newer DCs will send their shutdown request
+                 * to all peers, who will update the DC's expected state to
+                 * down, thus avoiding fencing. We can safely erase the DC's
+                 * transient attributes when it leaves in that case. However,
+                 * the only way to avoid fencing older DCs is to leave the
+                 * transient attributes intact until it rejoins.
+                 */
+                if (compare_version(controld_globals.dc_version, "3.0.9") > 0) {
+                    controld_delete_node_state(node->uname,
+                                               controld_section_attrs,
+                                               cib_scope_local);
+                }
+
+            } else if (AM_I_DC
+                       || pcmk_is_set(controld_globals.flags, controld_dc_left)
+                       || (controld_globals.dc_name == NULL)) {
+                /* This only needs to be done once, so normally the DC should do
+                 * it. However if there is no DC, every node must do it, since
+                 * there is no other way to ensure some one node does it.
+                 */
+                if (appeared) {
+                    te_trigger_stonith_history_sync(FALSE);
+                } else {
+                    controld_delete_node_state(node->uname,
+                                               controld_section_attrs,
+                                               cib_scope_local);
+                }
+            }
+            break;
+    }
+
+    if (AM_I_DC) {
+        xmlNode *update = NULL;
+        int flags = node_update_peer;
+        int alive = node_alive(node);
+        pcmk__graph_action_t *down = match_down_event(node->uuid);
+
+        crm_trace("Alive=%d, appeared=%d, down=%d",
+                  alive, appeared, (down? down->id : -1));
+
+        if (appeared && (alive > 0) && !is_remote) {
+            register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
+        }
+
+        if (down) {
+            const char *task = crm_element_value(down->xml, XML_LRM_ATTR_TASK);
+
+            if (pcmk__str_eq(task, CRM_OP_FENCE, pcmk__str_casei)) {
+
+                /* tengine_stonith_callback() confirms fence actions */
+                crm_trace("Updating CIB %s fencer reported fencing of %s complete",
+                          (pcmk_is_set(down->flags, pcmk__graph_action_confirmed)? "after" : "before"), node->uname);
+
+            } else if (!appeared && pcmk__str_eq(task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
+
+                // Shutdown actions are immediately confirmed (i.e. no_wait)
+                if (!is_remote) {
+                    flags |= node_update_join | node_update_expected;
+                    crmd_peer_down(node, FALSE);
+                    check_join_state(controld_globals.fsa_state, __func__);
+                }
+                if (alive >= 0) {
+                    crm_info("%s of peer %s is in progress " CRM_XS " action=%d",
+                             task, node->uname, down->id);
+                } else {
+                    crm_notice("%s of peer %s is complete " CRM_XS " action=%d",
+                               task, node->uname, down->id);
+                    pcmk__update_graph(controld_globals.transition_graph, down);
+                    trigger_graph();
+                }
+
+            } else {
+                crm_trace("Node %s is %s, was expected to %s (op %d)",
+                          node->uname,
+                          ((alive > 0)? "alive" :
+                           ((alive < 0)? "dead" : "partially alive")),
+                          task, down->id);
+            }
+
+        } else if (appeared == FALSE) {
+            if ((controld_globals.transition_graph == NULL)
+                || (controld_globals.transition_graph->id == -1)) {
+                crm_info("Stonith/shutdown of node %s is unknown to the "
+                         "current DC", node->uname);
+            } else {
+                crm_warn("Stonith/shutdown of node %s was not expected",
+                         node->uname);
+            }
+            if (!is_remote) {
+                crm_update_peer_join(__func__, node, crm_join_none);
+                check_join_state(controld_globals.fsa_state, __func__);
+            }
+            abort_transition(INFINITY, pcmk__graph_restart, "Node failure",
+                             NULL);
+            fail_incompletable_actions(controld_globals.transition_graph,
+                                       node->uuid);
+
+        } else {
+            crm_trace("Node %s came up, was not expected to be down",
+                      node->uname);
+        }
+
+        if (is_remote) {
+            /* A pacemaker_remote node won't have its cluster status updated
+             * in the CIB by membership-layer callbacks, so do it here.
+             */
+            flags |= node_update_cluster;
+
+            /* Trigger resource placement on newly integrated nodes */
+            if (appeared) {
+                abort_transition(INFINITY, pcmk__graph_restart,
+                                 "Pacemaker Remote node integrated", NULL);
+            }
+        }
+
+        /* Update the CIB node state */
+        update = create_node_state_update(node, flags, NULL, __func__);
+        if (update == NULL) {
+            crm_debug("Node state update not yet possible for %s", node->uname);
+        } else {
+            fsa_cib_anon_update(XML_CIB_TAG_STATUS, update);
+        }
+        free_xml(update);
+    }
+
+    controld_trigger_fsa();
+}
+
+gboolean
+crm_fsa_trigger(gpointer user_data)
+{
+    crm_trace("Invoked (queue len: %d)",
+              g_list_length(controld_globals.fsa_message_queue));
+    s_crmd_fsa(C_FSA_INTERNAL);
+    crm_trace("Exited  (queue len: %d)",
+              g_list_length(controld_globals.fsa_message_queue));
+    return TRUE;
+}
diff --git a/daemons/controld/controld_callbacks.h b/daemons/controld/controld_callbacks.h
new file mode 100644
index 0000000..a69d515
--- /dev/null
+++ b/daemons/controld/controld_callbacks.h
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef CONTROLD_CALLBACKS__H
+#  define CONTROLD_CALLBACKS__H
+
+#include <crm/cluster.h>
+
+extern void crmd_ha_msg_filter(xmlNode * msg);
+
+extern gboolean crm_fsa_trigger(gpointer user_data);
+
+extern void peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data);
+
+#endif
diff --git a/daemons/controld/controld_cib.c b/daemons/controld/controld_cib.c
new file mode 100644
index 0000000..94b99dd
--- /dev/null
+++ b/daemons/controld/controld_cib.c
@@ -0,0 +1,1138 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <unistd.h>  /* sleep */
+
+#include <crm/common/alerts_internal.h>
+#include <crm/common/xml.h>
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/lrmd_internal.h>
+
+#include <pacemaker-controld.h>
+
+// Call ID of the most recent in-progress CIB resource update (or 0 if none)
+static int pending_rsc_update = 0;
+
+// Call IDs of requested CIB replacements that won't trigger a new election
+// (used as a set of gint values)
+static GHashTable *cib_replacements = NULL;
+
+/*!
+ * \internal
+ * \brief Store the call ID of a CIB replacement that the controller requested
+ *
+ * The \p do_cib_replaced() callback function will avoid triggering a new
+ * election when we're notified of one of these expected replacements.
+ *
+ * \param[in] call_id  CIB call ID (or 0 for a synchronous call)
+ *
+ * \note This function should be called after making any asynchronous CIB
+ *       request (or before making any synchronous CIB request) that may replace
+ *       part of the nodes or status section. This may include CIB sync calls.
+ */
+void
+controld_record_cib_replace_call(int call_id)
+{
+    CRM_CHECK(call_id >= 0, return);
+
+    if (cib_replacements == NULL) {
+        cib_replacements = g_hash_table_new(NULL, NULL);
+    }
+
+    /* If the call ID is already present in the table, then it's old. We may not
+     * be removing them properly, and we could improperly ignore replacement
+     * notifications if cib_t:call_id wraps around.
+     */
+    CRM_LOG_ASSERT(g_hash_table_add(cib_replacements,
+                                    GINT_TO_POINTER((gint) call_id)));
+}
+
+/*!
+ * \internal
+ * \brief Remove the call ID of a CIB replacement from the replacements table
+ *
+ * \param[in] call_id  CIB call ID (or 0 for a synchronous call)
+ *
+ * \return \p true if \p call_id was found in the table, or \p false otherwise
+ *
+ * \note CIB notifications run before CIB callbacks. If this function is called
+ *       from within a callback, \p do_cib_replaced() will have removed
+ *       \p call_id from the table first if relevant changes triggered a
+ *       notification.
+ */
+bool
+controld_forget_cib_replace_call(int call_id)
+{
+    CRM_CHECK(call_id >= 0, return false);
+
+    if (cib_replacements == NULL) {
+        return false;
+    }
+    return g_hash_table_remove(cib_replacements,
+                               GINT_TO_POINTER((gint) call_id));
+}
+
+/*!
+ * \internal
+ * \brief Empty the hash table containing call IDs of CIB replacement requests
+ */
+void
+controld_forget_all_cib_replace_calls(void)
+{
+    if (cib_replacements != NULL) {
+        g_hash_table_remove_all(cib_replacements);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Free the hash table containing call IDs of CIB replacement requests
+ */
+void
+controld_destroy_cib_replacements_table(void)
+{
+    if (cib_replacements != NULL) {
+        g_hash_table_destroy(cib_replacements);
+        cib_replacements = NULL;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Respond to a dropped CIB connection
+ *
+ * \param[in] user_data  CIB connection that dropped
+ */
+static void
+handle_cib_disconnect(gpointer user_data)
+{
+    CRM_LOG_ASSERT(user_data == controld_globals.cib_conn);
+
+    controld_trigger_fsa();
+    controld_globals.cib_conn->state = cib_disconnected;
+
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
+        // @TODO This should trigger a reconnect, not a shutdown
+        crm_crit("Lost connection to the CIB manager, shutting down");
+        register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
+        controld_clear_fsa_input_flags(R_CIB_CONNECTED);
+
+    } else { // Expected
+        crm_info("Connection to the CIB manager terminated");
+    }
+}
+
+static void
+do_cib_updated(const char *event, xmlNode * msg)
+{
+    if (pcmk__alert_in_patchset(msg, TRUE)) {
+        controld_trigger_config();
+    }
+}
+
+static void
+do_cib_replaced(const char *event, xmlNode * msg)
+{
+    int call_id = 0;
+    const char *client_id = crm_element_value(msg, F_CIB_CLIENTID);
+    uint32_t change_section = cib_change_section_nodes
+                              |cib_change_section_status;
+    long long value = 0;
+
+    crm_debug("Updating the CIB after a replace: DC=%s", pcmk__btoa(AM_I_DC));
+    if (!AM_I_DC) {
+        return;
+    }
+
+    if ((crm_element_value_int(msg, F_CIB_CALLID, &call_id) == 0)
+        && pcmk__str_eq(client_id, controld_globals.cib_client_id,
+                        pcmk__str_none)
+        && controld_forget_cib_replace_call(call_id)) {
+        // We requested this replace op. No need to restart the join.
+        return;
+    }
+
+    if ((crm_element_value_ll(msg, F_CIB_CHANGE_SECTION, &value) < 0)
+        || (value < 0) || (value > UINT32_MAX)) {
+
+        crm_trace("Couldn't parse '%s' from message", F_CIB_CHANGE_SECTION);
+    } else {
+        change_section = (uint32_t) value;
+    }
+
+    if (pcmk_any_flags_set(change_section, cib_change_section_nodes
+                                           |cib_change_section_status)) {
+
+        /* start the join process again so we get everyone's LRM status */
+        populate_cib_nodes(node_update_quick|node_update_all, __func__);
+
+        register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
+    }
+}
+
+void
+controld_disconnect_cib_manager(void)
+{
+    cib_t *cib_conn = controld_globals.cib_conn;
+
+    CRM_ASSERT(cib_conn != NULL);
+
+    crm_info("Disconnecting from the CIB manager");
+
+    controld_clear_fsa_input_flags(R_CIB_CONNECTED);
+
+    cib_conn->cmds->del_notify_callback(cib_conn, T_CIB_REPLACE_NOTIFY,
+                                        do_cib_replaced);
+    cib_conn->cmds->del_notify_callback(cib_conn, T_CIB_DIFF_NOTIFY,
+                                        do_cib_updated);
+    cib_free_callbacks(cib_conn);
+
+    if (cib_conn->state != cib_disconnected) {
+        cib_conn->cmds->set_secondary(cib_conn,
+                                      cib_scope_local|cib_discard_reply);
+        cib_conn->cmds->signoff(cib_conn);
+    }
+
+    crm_notice("Disconnected from the CIB manager");
+}
+
+/* A_CIB_STOP, A_CIB_START, O_CIB_RESTART */
+void
+do_cib_control(long long action,
+               enum crmd_fsa_cause cause,
+               enum crmd_fsa_state cur_state,
+               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    static int cib_retries = 0;
+
+    cib_t *cib_conn = controld_globals.cib_conn;
+
+    void (*dnotify_fn) (gpointer user_data) = handle_cib_disconnect;
+    void (*replace_cb) (const char *event, xmlNodePtr msg) = do_cib_replaced;
+    void (*update_cb) (const char *event, xmlNodePtr msg) = do_cib_updated;
+
+    int rc = pcmk_ok;
+
+    CRM_ASSERT(cib_conn != NULL);
+
+    if (pcmk_is_set(action, A_CIB_STOP)) {
+        if ((cib_conn->state != cib_disconnected)
+            && (pending_rsc_update != 0)) {
+
+            crm_info("Waiting for resource update %d to complete",
+                     pending_rsc_update);
+            crmd_fsa_stall(FALSE);
+            return;
+        }
+        controld_disconnect_cib_manager();
+    }
+
+    if (!pcmk_is_set(action, A_CIB_START)) {
+        return;
+    }
+
+    if (cur_state == S_STOPPING) {
+        crm_err("Ignoring request to connect to the CIB manager after "
+                "shutdown");
+        return;
+    }
+
+    rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
+                                cib_command_nonblocking);
+
+    if (rc != pcmk_ok) {
+        // A short wait that usually avoids stalling the FSA
+        sleep(1);
+        rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
+                                    cib_command_nonblocking);
+    }
+
+    if (rc != pcmk_ok) {
+        crm_info("Could not connect to the CIB manager: %s", pcmk_strerror(rc));
+
+    } else if (cib_conn->cmds->set_connection_dnotify(cib_conn,
+                                                      dnotify_fn) != pcmk_ok) {
+        crm_err("Could not set dnotify callback");
+
+    } else if (cib_conn->cmds->add_notify_callback(cib_conn,
+                                                   T_CIB_REPLACE_NOTIFY,
+                                                   replace_cb) != pcmk_ok) {
+        crm_err("Could not set CIB notification callback (replace)");
+
+    } else if (cib_conn->cmds->add_notify_callback(cib_conn,
+                                                   T_CIB_DIFF_NOTIFY,
+                                                   update_cb) != pcmk_ok) {
+        crm_err("Could not set CIB notification callback (update)");
+
+    } else {
+        controld_set_fsa_input_flags(R_CIB_CONNECTED);
+        cib_retries = 0;
+        cib_conn->cmds->client_id(cib_conn, &controld_globals.cib_client_id,
+                                  NULL);
+    }
+
+    if (!pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
+        cib_retries++;
+
+        if (cib_retries < 30) {
+            crm_warn("Couldn't complete CIB registration %d times... "
+                     "pause and retry", cib_retries);
+            controld_start_wait_timer();
+            crmd_fsa_stall(FALSE);
+
+        } else {
+            crm_err("Could not complete CIB registration %d times... "
+                    "hard error", cib_retries);
+            register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+        }
+    }
+}
+
+#define MIN_CIB_OP_TIMEOUT (30)
+
+/*!
+ * \internal
+ * \brief Get the timeout (in seconds) that should be used with CIB operations
+ *
+ * \return The maximum of 30 seconds, the value of the PCMK_cib_timeout
+ *         environment variable, or 10 seconds times one more than the number of
+ *         nodes in the cluster.
+ */
+unsigned int
+cib_op_timeout(void)
+{
+    static int env_timeout = -1;
+    unsigned int calculated_timeout = 0;
+
+    if (env_timeout == -1) {
+        const char *env = getenv("PCMK_cib_timeout");
+
+        pcmk__scan_min_int(env, &env_timeout, MIN_CIB_OP_TIMEOUT);
+        crm_trace("Minimum CIB op timeout: %ds (environment: %s)",
+                  env_timeout, (env? env : "none"));
+    }
+
+    calculated_timeout = 1 + crm_active_peers();
+    if (crm_remote_peer_cache) {
+        calculated_timeout += g_hash_table_size(crm_remote_peer_cache);
+    }
+    calculated_timeout *= 10;
+
+    calculated_timeout = QB_MAX(calculated_timeout, env_timeout);
+    crm_trace("Calculated timeout: %us", calculated_timeout);
+
+    if (controld_globals.cib_conn) {
+        controld_globals.cib_conn->call_timeout = calculated_timeout;
+    }
+    return calculated_timeout;
+}
+
+/*!
+ * \internal
+ * \brief Get CIB call options to use local scope if primary is unavailable
+ *
+ * \return CIB call options
+ */
+int
+crmd_cib_smart_opt(void)
+{
+    int call_opt = cib_none;
+
+    if ((controld_globals.fsa_state == S_ELECTION)
+        || (controld_globals.fsa_state == S_PENDING)) {
+        crm_info("Sending update to local CIB in state: %s",
+                 fsa_state2string(controld_globals.fsa_state));
+        cib__set_call_options(call_opt, "update", cib_scope_local);
+    }
+    return call_opt;
+}
+
+static void
+cib_delete_callback(xmlNode *msg, int call_id, int rc, xmlNode *output,
+                    void *user_data)
+{
+    char *desc = user_data;
+
+    if (rc == 0) {
+        crm_debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id);
+    } else {
+        crm_warn("Deletion of %s (via CIB call %d) failed: %s " CRM_XS " rc=%d",
+                 desc, call_id, pcmk_strerror(rc), rc);
+    }
+}
+
+// Searches for various portions of node_state to delete
+
+// Match a particular node's node_state (takes node name 1x)
+#define XPATH_NODE_STATE        "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"
+
+// Node's lrm section (name 1x)
+#define XPATH_NODE_LRM          XPATH_NODE_STATE "/" XML_CIB_TAG_LRM
+
+/* Node's lrm_rsc_op entries and lrm_resource entries without unexpired lock
+ * (name 2x, (seconds_since_epoch - XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT) 1x)
+ */
+#define XPATH_NODE_LRM_UNLOCKED XPATH_NODE_STATE "//" XML_LRM_TAG_RSC_OP    \
+                                "|" XPATH_NODE_STATE                        \
+                                "//" XML_LRM_TAG_RESOURCE                   \
+                                "[not(@" XML_CONFIG_ATTR_SHUTDOWN_LOCK ") " \
+                                "or " XML_CONFIG_ATTR_SHUTDOWN_LOCK "<%lld]"
+
+// Node's transient_attributes section (name 1x)
+#define XPATH_NODE_ATTRS        XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS
+
+// Everything under node_state (name 1x)
+#define XPATH_NODE_ALL          XPATH_NODE_STATE "/*"
+
+/* Unlocked history + transient attributes
+ * (name 2x, (seconds_since_epoch - XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT) 1x,
+ * name 1x)
+ */
+#define XPATH_NODE_ALL_UNLOCKED XPATH_NODE_LRM_UNLOCKED "|" XPATH_NODE_ATTRS
+
+/*!
+ * \internal
+ * \brief Delete subsection of a node's CIB node_state
+ *
+ * \param[in] uname    Desired node
+ * \param[in] section  Subsection of node_state to delete
+ * \param[in] options  CIB call options to use
+ */
+void
+controld_delete_node_state(const char *uname, enum controld_section_e section,
+                           int options)
+{
+    cib_t *cib_conn = controld_globals.cib_conn;
+
+    char *xpath = NULL;
+    char *desc = NULL;
+
+    // Shutdown locks that started before this time are expired
+    long long expire = (long long) time(NULL)
+                       - controld_globals.shutdown_lock_limit;
+
+    CRM_CHECK(uname != NULL, return);
+    switch (section) {
+        case controld_section_lrm:
+            xpath = crm_strdup_printf(XPATH_NODE_LRM, uname);
+            desc = crm_strdup_printf("resource history for node %s", uname);
+            break;
+        case controld_section_lrm_unlocked:
+            xpath = crm_strdup_printf(XPATH_NODE_LRM_UNLOCKED,
+                                      uname, uname, expire);
+            desc = crm_strdup_printf("resource history (other than shutdown "
+                                     "locks) for node %s", uname);
+            break;
+        case controld_section_attrs:
+            xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname);
+            desc = crm_strdup_printf("transient attributes for node %s", uname);
+            break;
+        case controld_section_all:
+            xpath = crm_strdup_printf(XPATH_NODE_ALL, uname);
+            desc = crm_strdup_printf("all state for node %s", uname);
+            break;
+        case controld_section_all_unlocked:
+            xpath = crm_strdup_printf(XPATH_NODE_ALL_UNLOCKED,
+                                      uname, uname, expire, uname);
+            desc = crm_strdup_printf("all state (other than shutdown locks) "
+                                     "for node %s", uname);
+            break;
+    }
+
+    if (cib_conn == NULL) {
+        crm_warn("Unable to delete %s: no CIB connection", desc);
+        free(desc);
+    } else {
+        int call_id;
+
+        cib__set_call_options(options, "node state deletion",
+                              cib_xpath|cib_multiple);
+        call_id = cib_conn->cmds->remove(cib_conn, xpath, NULL, options);
+        crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s",
+                 desc, call_id, xpath);
+        fsa_register_cib_callback(call_id, desc, cib_delete_callback);
+        // CIB library handles freeing desc
+    }
+    free(xpath);
+}
+
+// Takes node name and resource ID
+#define XPATH_RESOURCE_HISTORY "//" XML_CIB_TAG_STATE                       \
+                               "[@" XML_ATTR_UNAME "='%s']/"                \
+                               XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES    \
+                               "/" XML_LRM_TAG_RESOURCE                     \
+                               "[@" XML_ATTR_ID "='%s']"
+// @TODO could add "and @XML_CONFIG_ATTR_SHUTDOWN_LOCK" to limit to locks
+
+/*!
+ * \internal
+ * \brief Clear resource history from CIB for a given resource and node
+ *
+ * \param[in]  rsc_id        ID of resource to be cleared
+ * \param[in]  node          Node whose resource history should be cleared
+ * \param[in]  user_name     ACL user name to use
+ * \param[in]  call_options  CIB call options
+ *
+ * \return Standard Pacemaker return code
+ */
+int
+controld_delete_resource_history(const char *rsc_id, const char *node,
+                                 const char *user_name, int call_options)
+{
+    char *desc = NULL;
+    char *xpath = NULL;
+    int rc = pcmk_rc_ok;
+
+    CRM_CHECK((rsc_id != NULL) && (node != NULL), return EINVAL);
+
+    desc = crm_strdup_printf("resource history for %s on %s", rsc_id, node);
+    if (controld_globals.cib_conn == NULL) {
+        crm_err("Unable to clear %s: no CIB connection", desc);
+        free(desc);
+        return ENOTCONN;
+    }
+
+    // Ask CIB to delete the entry
+    xpath = crm_strdup_printf(XPATH_RESOURCE_HISTORY, node, rsc_id);
+    rc = cib_internal_op(controld_globals.cib_conn, PCMK__CIB_REQUEST_DELETE,
+                         NULL, xpath, NULL, NULL, call_options|cib_xpath,
+                         user_name);
+
+    if (rc < 0) {
+        rc = pcmk_legacy2rc(rc);
+        crm_err("Could not delete resource status of %s on %s%s%s: %s "
+                CRM_XS " rc=%d", rsc_id, node,
+                (user_name? " for user " : ""), (user_name? user_name : ""),
+                pcmk_rc_str(rc), rc);
+        free(desc);
+        free(xpath);
+        return rc;
+    }
+
+    if (pcmk_is_set(call_options, cib_sync_call)) {
+        if (pcmk_is_set(call_options, cib_dryrun)) {
+            crm_debug("Deletion of %s would succeed", desc);
+        } else {
+            crm_debug("Deletion of %s succeeded", desc);
+        }
+        free(desc);
+
+    } else {
+        crm_info("Clearing %s (via CIB call %d) " CRM_XS " xpath=%s",
+                 desc, rc, xpath);
+        fsa_register_cib_callback(rc, desc, cib_delete_callback);
+        // CIB library handles freeing desc
+    }
+
+    free(xpath);
+    return pcmk_rc_ok;
+}
+
+/*!
+ * \internal
+ * \brief Build XML and string of parameters meeting some criteria, for digest
+ *
+ * \param[in]  op          Executor event with parameter table to use
+ * \param[in]  metadata    Parsed meta-data for executed resource agent
+ * \param[in]  param_type  Flag used for selection criteria
+ * \param[out] result      Will be set to newly created XML with selected
+ *                         parameters as attributes
+ *
+ * \return Newly allocated space-separated string of parameter names
+ * \note Selection criteria varies by param_type: for the restart digest, we
+ *       want parameters that are *not* marked reloadable (OCF 1.1) or that
+ *       *are* marked unique (pre-1.1), for both string and XML results; for the
+ *       secure digest, we want parameters that *are* marked private for the
+ *       string, but parameters that are *not* marked private for the XML.
+ * \note It is the caller's responsibility to free the string return value with
+ *       \p g_string_free() and the XML result with \p free_xml().
+ */
+static GString *
+build_parameter_list(const lrmd_event_data_t *op,
+                     const struct ra_metadata_s *metadata,
+                     enum ra_param_flags_e param_type, xmlNode **result)
+{
+    GString *list = NULL;
+
+    *result = create_xml_node(NULL, XML_TAG_PARAMS);
+
+    /* Consider all parameters only except private ones to be consistent with
+     * what scheduler does with calculate_secure_digest().
+     */
+    if (param_type == ra_param_private
+        && compare_version(controld_globals.dc_version, "3.16.0") >= 0) {
+        g_hash_table_foreach(op->params, hash2field, *result);
+        pcmk__filter_op_for_digest(*result);
+    }
+
+    for (GList *iter = metadata->ra_params; iter != NULL; iter = iter->next) {
+        struct ra_param_s *param = (struct ra_param_s *) iter->data;
+
+        bool accept_for_list = false;
+        bool accept_for_xml = false;
+
+        switch (param_type) {
+            case ra_param_reloadable:
+                accept_for_list = !pcmk_is_set(param->rap_flags, param_type);
+                accept_for_xml = accept_for_list;
+                break;
+
+            case ra_param_unique:
+                accept_for_list = pcmk_is_set(param->rap_flags, param_type);
+                accept_for_xml = accept_for_list;
+                break;
+
+            case ra_param_private:
+                accept_for_list = pcmk_is_set(param->rap_flags, param_type);
+                accept_for_xml = !accept_for_list;
+                break;
+        }
+
+        if (accept_for_list) {
+            crm_trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type));
+
+            if (list == NULL) {
+                // We will later search for " WORD ", so start list with a space
+                pcmk__add_word(&list, 256, " ");
+            }
+            pcmk__add_word(&list, 0, param->rap_name);
+
+        } else {
+            crm_trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type));
+        }
+
+        if (accept_for_xml) {
+            const char *v = g_hash_table_lookup(op->params, param->rap_name);
+
+            if (v != NULL) {
+                crm_trace("Adding attr %s=%s to the xml result", param->rap_name, v);
+                crm_xml_add(*result, param->rap_name, v);
+            }
+
+        } else {
+            crm_trace("Removing attr %s from the xml result", param->rap_name);
+            xml_remove_prop(*result, param->rap_name);
+        }
+    }
+
+    if (list != NULL) {
+        // We will later search for " WORD ", so end list with a space
+        pcmk__add_word(&list, 0, " ");
+    }
+    return list;
+}
+
+static void
+append_restart_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
+                    xmlNode *update, const char *version)
+{
+    GString *list = NULL;
+    char *digest = NULL;
+    xmlNode *restart = NULL;
+
+    CRM_LOG_ASSERT(op->params != NULL);
+
+    if (op->interval_ms > 0) {
+        /* monitors are not reloadable */
+        return;
+    }
+
+    if (pcmk_is_set(metadata->ra_flags, ra_supports_reload_agent)) {
+        // Add parameters not marked reloadable to the "op-force-restart" list
+        list = build_parameter_list(op, metadata, ra_param_reloadable,
+                                    &restart);
+
+    } else if (pcmk_is_set(metadata->ra_flags, ra_supports_legacy_reload)) {
+        /* @COMPAT pre-OCF-1.1 resource agents
+         *
+         * Before OCF 1.1, Pacemaker abused "unique=0" to indicate
+         * reloadability. Add any parameters with unique="1" to the
+         * "op-force-restart" list.
+         */
+        list = build_parameter_list(op, metadata, ra_param_unique, &restart);
+
+    } else {
+        // Resource does not support agent reloads
+        return;
+    }
+
+    digest = calculate_operation_digest(restart, version);
+    /* Add "op-force-restart" and "op-restart-digest" to indicate the resource supports reload,
+     * no matter if it actually supports any parameters with unique="1"). */
+    crm_xml_add(update, XML_LRM_ATTR_OP_RESTART,
+                (list == NULL)? "" : (const char *) list->str);
+    crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);
+
+    if ((list != NULL) && (list->len > 0)) {
+        crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
+    } else {
+        crm_trace("%s: %s", op->rsc_id, digest);
+    }
+
+    if (list != NULL) {
+        g_string_free(list, TRUE);
+    }
+    free_xml(restart);
+    free(digest);
+}
+
+static void
+append_secure_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
+                   xmlNode *update, const char *version)
+{
+    GString *list = NULL;
+    char *digest = NULL;
+    xmlNode *secure = NULL;
+
+    CRM_LOG_ASSERT(op->params != NULL);
+
+    /*
+     * To keep XML_LRM_ATTR_OP_SECURE short, we want it to contain the
+     * secure parameters but XML_LRM_ATTR_SECURE_DIGEST to be based on
+     * the insecure ones
+     */
+    list = build_parameter_list(op, metadata, ra_param_private, &secure);
+
+    if (list != NULL) {
+        digest = calculate_operation_digest(secure, version);
+        crm_xml_add(update, XML_LRM_ATTR_OP_SECURE, (const char *) list->str);
+        crm_xml_add(update, XML_LRM_ATTR_SECURE_DIGEST, digest);
+
+        crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
+        g_string_free(list, TRUE);
+    } else {
+        crm_trace("%s: no secure parameters", op->rsc_id);
+    }
+
+    free_xml(secure);
+    free(digest);
+}
+
+/*!
+ * \internal
+ * \brief Create XML for a resource history entry
+ *
+ * \param[in]     func       Function name of caller
+ * \param[in,out] parent     XML to add entry to
+ * \param[in]     rsc        Affected resource
+ * \param[in,out] op         Action to add an entry for (or NULL to do nothing)
+ * \param[in]     node_name  Node where action occurred
+ */
+void
+controld_add_resource_history_xml_as(const char *func, xmlNode *parent,
+                                     const lrmd_rsc_info_t *rsc,
+                                     lrmd_event_data_t *op,
+                                     const char *node_name)
+{
+    int target_rc = 0;
+    xmlNode *xml_op = NULL;
+    struct ra_metadata_s *metadata = NULL;
+    const char *caller_version = NULL;
+    lrm_state_t *lrm_state = NULL;
+
+    if (op == NULL) {
+        return;
+    }
+
+    target_rc = rsc_op_expected_rc(op);
+
+    caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
+    CRM_CHECK(caller_version != NULL, caller_version = CRM_FEATURE_SET);
+
+    xml_op = pcmk__create_history_xml(parent, op, caller_version, target_rc,
+                                      controld_globals.our_nodename, func);
+    if (xml_op == NULL) {
+        return;
+    }
+
+    if ((rsc == NULL) || (op->params == NULL)
+        || !crm_op_needs_metadata(rsc->standard, op->op_type)) {
+
+        crm_trace("No digests needed for %s action on %s (params=%p rsc=%p)",
+                  op->op_type, op->rsc_id, op->params, rsc);
+        return;
+    }
+
+    lrm_state = lrm_state_find(node_name);
+    if (lrm_state == NULL) {
+        crm_warn("Cannot calculate digests for operation " PCMK__OP_FMT
+                 " because we have no connection to executor for %s",
+                 op->rsc_id, op->op_type, op->interval_ms, node_name);
+        return;
+    }
+
+    /* Ideally the metadata is cached, and the agent is just a fallback.
+     *
+     * @TODO Go through all callers and ensure they get metadata asynchronously
+     * first.
+     */
+    metadata = controld_get_rsc_metadata(lrm_state, rsc,
+                                         controld_metadata_from_agent
+                                         |controld_metadata_from_cache);
+    if (metadata == NULL) {
+        return;
+    }
+
+    crm_trace("Including additional digests for %s:%s:%s",
+              rsc->standard, rsc->provider, rsc->type);
+    append_restart_list(op, metadata, xml_op, caller_version);
+    append_secure_list(op, metadata, xml_op, caller_version);
+
+    return;
+}
+
+/*!
+ * \internal
+ * \brief Record an action as pending in the CIB, if appropriate
+ *
+ * \param[in]     node_name  Node where the action is pending
+ * \param[in]     rsc        Resource that action is for
+ * \param[in,out] op         Pending action
+ *
+ * \return true if action was recorded in CIB, otherwise false
+ */
+bool
+controld_record_pending_op(const char *node_name, const lrmd_rsc_info_t *rsc,
+                           lrmd_event_data_t *op)
+{
+    const char *record_pending = NULL;
+
+    CRM_CHECK((node_name != NULL) && (rsc != NULL) && (op != NULL),
+              return false);
+
+    // Never record certain operation types as pending
+    if ((op->op_type == NULL) || (op->params == NULL)
+        || !controld_action_is_recordable(op->op_type)) {
+        return false;
+    }
+
+    // Check action's record-pending meta-attribute (defaults to true)
+    record_pending = crm_meta_value(op->params, XML_OP_ATTR_PENDING);
+    if ((record_pending != NULL) && !crm_is_true(record_pending)) {
+        return false;
+    }
+
+    op->call_id = -1;
+    op->t_run = time(NULL);
+    op->t_rcchange = op->t_run;
+
+    lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
+
+    crm_debug("Recording pending %s-interval %s for %s on %s in the CIB",
+              pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
+              node_name);
+    controld_update_resource_history(node_name, rsc, op, 0);
+    return true;
+}
+
+static void
+cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    switch (rc) {
+        case pcmk_ok:
+        case -pcmk_err_diff_failed:
+        case -pcmk_err_diff_resync:
+            crm_trace("Resource update %d complete: rc=%d", call_id, rc);
+            break;
+        default:
+            crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc));
+    }
+
+    if (call_id == pending_rsc_update) {
+        pending_rsc_update = 0;
+        controld_trigger_fsa();
+    }
+}
+
+/* Only successful stops, and probes that found the resource inactive, get locks
+ * recorded in the history. This ensures the resource stays locked to the node
+ * until it is active there again after the node comes back up.
+ */
+static bool
+should_preserve_lock(lrmd_event_data_t *op)
+{
+    if (!pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
+        return false;
+    }
+    if (!strcmp(op->op_type, RSC_STOP) && (op->rc == PCMK_OCF_OK)) {
+        return true;
+    }
+    if (!strcmp(op->op_type, RSC_STATUS) && (op->rc == PCMK_OCF_NOT_RUNNING)) {
+        return true;
+    }
+    return false;
+}
+
+/*!
+ * \internal
+ * \brief Request a CIB update
+ *
+ * \param[in]     section   Section of CIB to update
+ * \param[in,out] data      New XML of CIB section to update
+ * \param[in]     options   CIB call options
+ * \param[in]     callback  If not NULL, set this as the operation callback
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note If \p callback is \p cib_rsc_callback(), the CIB update's call ID is
+ *       stored in \p pending_rsc_update on success.
+ */
+int
+controld_update_cib(const char *section, xmlNode *data, int options,
+                    void (*callback)(xmlNode *, int, int, xmlNode *, void *))
+{
+    int cib_rc = -ENOTCONN;
+
+    CRM_ASSERT(data != NULL);
+
+    if (controld_globals.cib_conn != NULL) {
+        cib_rc = cib_internal_op(controld_globals.cib_conn,
+                                 PCMK__CIB_REQUEST_MODIFY, NULL, section,
+                                 data, NULL, options, NULL);
+        if (cib_rc >= 0) {
+            crm_debug("Submitted CIB update %d for %s section",
+                      cib_rc, section);
+        }
+    }
+
+    if (callback == NULL) {
+        if (cib_rc < 0) {
+            crm_err("Failed to update CIB %s section: %s",
+                    section, pcmk_rc_str(pcmk_legacy2rc(cib_rc)));
+        }
+
+    } else {
+        if ((cib_rc >= 0) && (callback == cib_rsc_callback)) {
+            /* Checking for a particular callback is a little hacky, but it
+             * didn't seem worth adding an output argument for cib_rc for just
+             * one use case.
+             */
+            pending_rsc_update = cib_rc;
+        }
+        fsa_register_cib_callback(cib_rc, NULL, callback);
+    }
+
+    return (cib_rc >= 0)? pcmk_rc_ok : pcmk_legacy2rc(cib_rc);
+}
+
+/*!
+ * \internal
+ * \brief Update resource history entry in CIB
+ *
+ * \param[in]     node_name  Node where action occurred
+ * \param[in]     rsc        Resource that action is for
+ * \param[in,out] op         Action to record
+ * \param[in]     lock_time  If nonzero, when resource was locked to node
+ *
+ * \note On success, the CIB update's call ID will be stored in
+ *       pending_rsc_update.
+ */
+void
+controld_update_resource_history(const char *node_name,
+                                 const lrmd_rsc_info_t *rsc,
+                                 lrmd_event_data_t *op, time_t lock_time)
+{
+    xmlNode *update = NULL;
+    xmlNode *xml = NULL;
+    int call_opt = crmd_cib_smart_opt();
+    const char *node_id = NULL;
+    const char *container = NULL;
+
+    CRM_CHECK((node_name != NULL) && (op != NULL), return);
+
+    if (rsc == NULL) {
+        crm_warn("Resource %s no longer exists in the executor", op->rsc_id);
+        controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id);
+        return;
+    }
+
+    // <status>
+    update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
+
+    //   <node_state ...>
+    xml = create_xml_node(update, XML_CIB_TAG_STATE);
+    if (pcmk__str_eq(node_name, controld_globals.our_nodename,
+                     pcmk__str_casei)) {
+        node_id = controld_globals.our_uuid;
+    } else {
+        node_id = node_name;
+        pcmk__xe_set_bool_attr(xml, XML_NODE_IS_REMOTE, true);
+    }
+    crm_xml_add(xml, XML_ATTR_ID, node_id);
+    crm_xml_add(xml, XML_ATTR_UNAME, node_name);
+    crm_xml_add(xml, XML_ATTR_ORIGIN, __func__);
+
+    //     <lrm ...>
+    xml = create_xml_node(xml, XML_CIB_TAG_LRM);
+    crm_xml_add(xml, XML_ATTR_ID, node_id);
+
+    //       <lrm_resources>
+    xml = create_xml_node(xml, XML_LRM_TAG_RESOURCES);
+
+    //         <lrm_resource ...>
+    xml = create_xml_node(xml, XML_LRM_TAG_RESOURCE);
+    crm_xml_add(xml, XML_ATTR_ID, op->rsc_id);
+    crm_xml_add(xml, XML_AGENT_ATTR_CLASS, rsc->standard);
+    crm_xml_add(xml, XML_AGENT_ATTR_PROVIDER, rsc->provider);
+    crm_xml_add(xml, XML_ATTR_TYPE, rsc->type);
+    if (lock_time != 0) {
+        /* Actions on a locked resource should either preserve the lock by
+         * recording it with the action result, or clear it.
+         */
+        if (!should_preserve_lock(op)) {
+            lock_time = 0;
+        }
+        crm_xml_add_ll(xml, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
+                       (long long) lock_time);
+    }
+    if (op->params != NULL) {
+        container = g_hash_table_lookup(op->params,
+                                        CRM_META "_" XML_RSC_ATTR_CONTAINER);
+        if (container != NULL) {
+            crm_trace("Resource %s is a part of container resource %s",
+                      op->rsc_id, container);
+            crm_xml_add(xml, XML_RSC_ATTR_CONTAINER, container);
+        }
+    }
+
+    //           <lrm_resource_op ...> (possibly more than one)
+    controld_add_resource_history_xml(xml, rsc, op, node_name);
+
+    /* Update CIB asynchronously. Even if it fails, the resource state should be
+     * discovered during the next election. Worst case, the node is wrongly
+     * fenced for running a resource it isn't.
+     */
+    crm_log_xml_trace(update, __func__);
+    controld_update_cib(XML_CIB_TAG_STATUS, update, call_opt, cib_rsc_callback);
+    free_xml(update);
+}
+
+/*!
+ * \internal
+ * \brief Erase an LRM history entry from the CIB, given the operation data
+ *
+ * \param[in] op         Operation whose history should be deleted
+ */
+void
+controld_delete_action_history(const lrmd_event_data_t *op)
+{
+    xmlNode *xml_top = NULL;
+
+    CRM_CHECK(op != NULL, return);
+
+    xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
+    crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
+    crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);
+
+    if (op->interval_ms > 0) {
+        char *op_id = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
+
+        /* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
+        crm_xml_add(xml_top, XML_ATTR_ID, op_id);
+        free(op_id);
+    }
+
+    crm_debug("Erasing resource operation history for " PCMK__OP_FMT " (call=%d)",
+              op->rsc_id, op->op_type, op->interval_ms, op->call_id);
+
+    controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn,
+                                            XML_CIB_TAG_STATUS, xml_top,
+                                            cib_none);
+
+    crm_log_xml_trace(xml_top, "op:cancel");
+    free_xml(xml_top);
+}
+
+/* Define xpath to find LRM resource history entry by node and resource */
+#define XPATH_HISTORY                                   \
+    "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS              \
+    "/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"  \
+    "/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES       \
+    "/" XML_LRM_TAG_RESOURCE "[@" XML_ATTR_ID "='%s']"  \
+    "/" XML_LRM_TAG_RSC_OP
+
+/* ... and also by operation key */
+#define XPATH_HISTORY_ID XPATH_HISTORY \
+    "[@" XML_ATTR_ID "='%s']"
+
+/* ... and also by operation key and operation call ID */
+#define XPATH_HISTORY_CALL XPATH_HISTORY \
+    "[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_CALLID "='%d']"
+
+/* ... and also by operation key and original operation key */
+#define XPATH_HISTORY_ORIG XPATH_HISTORY \
+    "[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_TASK_KEY "='%s']"
+
+/*!
+ * \internal
+ * \brief Delete a last_failure resource history entry from the CIB
+ *
+ * \param[in] rsc_id       Name of resource to clear history for
+ * \param[in] node         Name of node to clear history for
+ * \param[in] action       If specified, delete only if this was failed action
+ * \param[in] interval_ms  If \p action is specified, it has this interval
+ */
+void
+controld_cib_delete_last_failure(const char *rsc_id, const char *node,
+                                 const char *action, guint interval_ms)
+{
+    char *xpath = NULL;
+    char *last_failure_key = NULL;
+
+    CRM_CHECK((rsc_id != NULL) && (node != NULL), return);
+
+    // Generate XPath to match desired entry
+    last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);
+    if (action == NULL) {
+        xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id,
+                                  last_failure_key);
+    } else {
+        char *action_key = pcmk__op_key(rsc_id, action, interval_ms);
+
+        xpath = crm_strdup_printf(XPATH_HISTORY_ORIG, node, rsc_id,
+                                  last_failure_key, action_key);
+        free(action_key);
+    }
+    free(last_failure_key);
+
+    controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
+                                            NULL, cib_xpath);
+    free(xpath);
+}
+
+/*!
+ * \internal
+ * \brief Delete resource history entry from the CIB, given operation key
+ *
+ * \param[in] rsc_id     Name of resource to clear history for
+ * \param[in] node       Name of node to clear history for
+ * \param[in] key        Operation key of operation to clear history for
+ * \param[in] call_id    If specified, delete entry only if it has this call ID
+ */
+void
+controld_delete_action_history_by_key(const char *rsc_id, const char *node,
+                                      const char *key, int call_id)
+{
+    char *xpath = NULL;
+
+    CRM_CHECK((rsc_id != NULL) && (node != NULL) && (key != NULL), return);
+
+    if (call_id > 0) {
+        xpath = crm_strdup_printf(XPATH_HISTORY_CALL, node, rsc_id, key,
+                                  call_id);
+    } else {
+        xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id, key);
+    }
+    controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
+                                            NULL, cib_xpath);
+    free(xpath);
+}
diff --git a/daemons/controld/controld_cib.h b/daemons/controld/controld_cib.h
new file mode 100644
index 0000000..bd9492a
--- /dev/null
+++ b/daemons/controld/controld_cib.h
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef PCMK__CONTROLD_CIB__H
+#define PCMK__CONTROLD_CIB__H
+
+#include <crm_internal.h>
+
+#include <glib.h>
+
+#include <crm/crm.h>
+#include <crm/common/xml.h>
+#include <crm/cib/internal.h>   // PCMK__CIB_REQUEST_MODIFY
+#include "controld_globals.h"   // controld_globals.cib_conn
+
+static inline void
+fsa_cib_anon_update(const char *section, xmlNode *data) {
+    if (controld_globals.cib_conn == NULL) {
+        crm_err("No CIB connection available");
+    } else {
+        controld_globals.cib_conn->cmds->modify(controld_globals.cib_conn,
+                                                section, data,
+                                                cib_scope_local|cib_can_create);
+    }
+}
+
+static inline void
+fsa_cib_anon_update_discard_reply(const char *section, xmlNode *data) {
+    if (controld_globals.cib_conn == NULL) {
+        crm_err("No CIB connection available");
+    } else {
+        controld_globals.cib_conn->cmds->modify(controld_globals.cib_conn,
+                                                section, data,
+                                                cib_scope_local
+                                                |cib_can_create
+                                                |cib_discard_reply);
+    }
+}
+
+void controld_record_cib_replace_call(int call_id);
+bool controld_forget_cib_replace_call(int call_id);
+void controld_forget_all_cib_replace_calls(void);
+void controld_destroy_cib_replacements_table(void);
+
+int controld_update_cib(const char *section, xmlNode *data, int options,
+                        void (*callback)(xmlNode *, int, int, xmlNode *,
+                                         void *));
+unsigned int cib_op_timeout(void);
+
+// Subsections of node_state
+enum controld_section_e {
+    controld_section_lrm,
+    controld_section_lrm_unlocked,
+    controld_section_attrs,
+    controld_section_all,
+    controld_section_all_unlocked
+};
+
+void controld_delete_node_state(const char *uname,
+                                enum controld_section_e section, int options);
+int controld_delete_resource_history(const char *rsc_id, const char *node,
+                                     const char *user_name, int call_options);
+
+/* Convenience macro for registering a CIB callback
+ * (assumes that data can be freed with free())
+ */
+#  define fsa_register_cib_callback(id, data, fn) do {                      \
+    cib_t *cib_conn = controld_globals.cib_conn;                            \
+                                                                            \
+    CRM_ASSERT(cib_conn != NULL);                                           \
+    cib_conn->cmds->register_callback_full(cib_conn, id, cib_op_timeout(),  \
+                                           FALSE, data, #fn, fn, free);     \
+    } while(0)
+
+void controld_add_resource_history_xml_as(const char *func, xmlNode *parent,
+                                          const lrmd_rsc_info_t *rsc,
+                                          lrmd_event_data_t *op,
+                                          const char *node_name);
+
+#define controld_add_resource_history_xml(parent, rsc, op, node_name)   \
+    controld_add_resource_history_xml_as(__func__, (parent), (rsc),     \
+                                         (op), (node_name))
+
+bool controld_record_pending_op(const char *node_name,
+                                const lrmd_rsc_info_t *rsc,
+                                lrmd_event_data_t *op);
+
+void controld_update_resource_history(const char *node_name,
+                                      const lrmd_rsc_info_t *rsc,
+                                      lrmd_event_data_t *op, time_t lock_time);
+
+void controld_delete_action_history(const lrmd_event_data_t *op);
+
+void controld_cib_delete_last_failure(const char *rsc_id, const char *node,
+                                      const char *action, guint interval_ms);
+
+void controld_delete_action_history_by_key(const char *rsc_id, const char *node,
+                                           const char *key, int call_id);
+
+void controld_disconnect_cib_manager(void);
+
+int crmd_cib_smart_opt(void);
+
+/*!
+ * \internal
+ * \brief Check whether an action type should be recorded in the CIB
+ *
+ * \param[in] action  Action type
+ *
+ * \return true if action should be recorded, false otherwise
+ */
+static inline bool
+controld_action_is_recordable(const char *action)
+{
+    return !pcmk__str_any_of(action, CRMD_ACTION_CANCEL, CRMD_ACTION_DELETE,
+                             CRMD_ACTION_NOTIFY, CRMD_ACTION_METADATA, NULL);
+}
+
+#endif // PCMK__CONTROLD_CIB__H
diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c
new file mode 100644
index 0000000..ffc62a0
--- /dev/null
+++ b/daemons/controld/controld_control.c
@@ -0,0 +1,857 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/pengine/rules.h>
+#include <crm/cluster/internal.h>
+#include <crm/cluster/election_internal.h>
+#include <crm/common/ipc_internal.h>
+
+#include <pacemaker-controld.h>
+
+static qb_ipcs_service_t *ipcs = NULL;
+
+static crm_trigger_t *config_read_trigger = NULL;
+
+#if SUPPORT_COROSYNC
+extern gboolean crm_connect_corosync(crm_cluster_t * cluster);
+#endif
+
+void crm_shutdown(int nsig);
+static gboolean crm_read_options(gpointer user_data);
+
+/*	 A_HA_CONNECT	*/
+void
+do_ha_control(long long action,
+              enum crmd_fsa_cause cause,
+              enum crmd_fsa_state cur_state,
+              enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    gboolean registered = FALSE;
+    static crm_cluster_t *cluster = NULL;
+
+    if (cluster == NULL) {
+        cluster = pcmk_cluster_new();
+    }
+
+    if (action & A_HA_DISCONNECT) {
+        crm_cluster_disconnect(cluster);
+        crm_info("Disconnected from the cluster");
+
+        controld_set_fsa_input_flags(R_HA_DISCONNECTED);
+    }
+
+    if (action & A_HA_CONNECT) {
+        crm_set_status_callback(&peer_update_callback);
+        crm_set_autoreap(FALSE);
+
+#if SUPPORT_COROSYNC
+        if (is_corosync_cluster()) {
+            registered = crm_connect_corosync(cluster);
+        }
+#endif // SUPPORT_COROSYNC
+
+        if (registered) {
+            controld_election_init(cluster->uname);
+            controld_globals.our_nodename = cluster->uname;
+            controld_globals.our_uuid = cluster->uuid;
+            if(cluster->uuid == NULL) {
+                crm_err("Could not obtain local uuid");
+                registered = FALSE;
+            }
+        }
+
+        if (!registered) {
+            controld_set_fsa_input_flags(R_HA_DISCONNECTED);
+            register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+            return;
+        }
+
+        populate_cib_nodes(node_update_none, __func__);
+        controld_clear_fsa_input_flags(R_HA_DISCONNECTED);
+        crm_info("Connected to the cluster");
+    }
+
+    if (action & ~(A_HA_CONNECT | A_HA_DISCONNECT)) {
+        crm_err("Unexpected action %s in %s", fsa_action2string(action),
+                __func__);
+    }
+}
+
+/*	 A_SHUTDOWN	*/
+void
+do_shutdown(long long action,
+            enum crmd_fsa_cause cause,
+            enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    /* just in case */
+    controld_set_fsa_input_flags(R_SHUTDOWN);
+    controld_disconnect_fencer(FALSE);
+}
+
+/*	 A_SHUTDOWN_REQ	*/
+void
+do_shutdown_req(long long action,
+                enum crmd_fsa_cause cause,
+                enum crmd_fsa_state cur_state,
+                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    xmlNode *msg = NULL;
+
+    controld_set_fsa_input_flags(R_SHUTDOWN);
+    //controld_set_fsa_input_flags(R_STAYDOWN);
+    crm_info("Sending shutdown request to all peers (DC is %s)",
+             pcmk__s(controld_globals.dc_name, "not set"));
+    msg = create_request(CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
+
+    if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+    }
+    free_xml(msg);
+}
+
+void
+crmd_fast_exit(crm_exit_t exit_code)
+{
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_STAYDOWN)) {
+        crm_warn("Inhibiting respawn "CRM_XS" remapping exit code %d to %d",
+                 exit_code, CRM_EX_FATAL);
+        exit_code = CRM_EX_FATAL;
+
+    } else if ((exit_code == CRM_EX_OK)
+               && pcmk_is_set(controld_globals.fsa_input_register,
+                              R_IN_RECOVERY)) {
+        crm_err("Could not recover from internal error");
+        exit_code = CRM_EX_ERROR;
+    }
+
+    if (controld_globals.logger_out != NULL) {
+        controld_globals.logger_out->finish(controld_globals.logger_out,
+                                            exit_code, true, NULL);
+        pcmk__output_free(controld_globals.logger_out);
+        controld_globals.logger_out = NULL;
+    }
+
+    crm_exit(exit_code);
+}
+
+crm_exit_t
+crmd_exit(crm_exit_t exit_code)
+{
+    GMainLoop *mloop = controld_globals.mainloop;
+
+    static bool in_progress = FALSE;
+
+    if (in_progress && (exit_code == CRM_EX_OK)) {
+        crm_debug("Exit is already in progress");
+        return exit_code;
+
+    } else if(in_progress) {
+        crm_notice("Error during shutdown process, exiting now with status %d (%s)",
+                   exit_code, crm_exit_str(exit_code));
+        crm_write_blackbox(SIGTRAP, NULL);
+        crmd_fast_exit(exit_code);
+    }
+
+    in_progress = TRUE;
+    crm_trace("Preparing to exit with status %d (%s)",
+              exit_code, crm_exit_str(exit_code));
+
+    /* Suppress secondary errors resulting from us disconnecting everything */
+    controld_set_fsa_input_flags(R_HA_DISCONNECTED);
+
+/* Close all IPC servers and clients to ensure any and all shared memory files are cleaned up */
+
+    if(ipcs) {
+        crm_trace("Closing IPC server");
+        mainloop_del_ipc_server(ipcs);
+        ipcs = NULL;
+    }
+
+    controld_close_attrd_ipc();
+    controld_shutdown_schedulerd_ipc();
+    controld_disconnect_fencer(TRUE);
+
+    if ((exit_code == CRM_EX_OK) && (controld_globals.mainloop == NULL)) {
+        crm_debug("No mainloop detected");
+        exit_code = CRM_EX_ERROR;
+    }
+
+    /* On an error, just get out.
+     *
+     * Otherwise, make the effort to have mainloop exit gracefully so
+     * that it (mostly) cleans up after itself and valgrind has less
+     * to report on - allowing real errors stand out
+     */
+    if (exit_code != CRM_EX_OK) {
+        crm_notice("Forcing immediate exit with status %d (%s)",
+                   exit_code, crm_exit_str(exit_code));
+        crm_write_blackbox(SIGTRAP, NULL);
+        crmd_fast_exit(exit_code);
+    }
+
+/* Clean up as much memory as possible for valgrind */
+
+    for (GList *iter = controld_globals.fsa_message_queue; iter != NULL;
+         iter = iter->next) {
+        fsa_data_t *fsa_data = (fsa_data_t *) iter->data;
+
+        crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]",
+                 fsa_input2string(fsa_data->fsa_input),
+                 fsa_state2string(controld_globals.fsa_state),
+                 fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
+        delete_fsa_input(fsa_data);
+    }
+
+    controld_clear_fsa_input_flags(R_MEMBERSHIP);
+
+    g_list_free(controld_globals.fsa_message_queue);
+    controld_globals.fsa_message_queue = NULL;
+
+    controld_election_fini();
+
+    /* Tear down the CIB manager connection, but don't free it yet -- it could
+     * be used when we drain the mainloop later.
+     */
+
+    controld_disconnect_cib_manager();
+
+    verify_stopped(controld_globals.fsa_state, LOG_WARNING);
+    controld_clear_fsa_input_flags(R_LRM_CONNECTED);
+    lrm_state_destroy_all();
+
+    mainloop_destroy_trigger(config_read_trigger);
+    config_read_trigger = NULL;
+
+    controld_destroy_fsa_trigger();
+    controld_destroy_transition_trigger();
+
+    pcmk__client_cleanup();
+    crm_peer_destroy();
+
+    controld_free_fsa_timers();
+    te_cleanup_stonith_history_sync(NULL, TRUE);
+    controld_free_sched_timer();
+
+    free(controld_globals.our_nodename);
+    controld_globals.our_nodename = NULL;
+
+    free(controld_globals.our_uuid);
+    controld_globals.our_uuid = NULL;
+
+    free(controld_globals.dc_name);
+    controld_globals.dc_name = NULL;
+
+    free(controld_globals.dc_version);
+    controld_globals.dc_version = NULL;
+
+    free(controld_globals.cluster_name);
+    controld_globals.cluster_name = NULL;
+
+    free(controld_globals.te_uuid);
+    controld_globals.te_uuid = NULL;
+
+    free_max_generation();
+    controld_destroy_cib_replacements_table();
+    controld_destroy_failed_sync_table();
+    controld_destroy_outside_events_table();
+
+    mainloop_destroy_signal(SIGPIPE);
+    mainloop_destroy_signal(SIGUSR1);
+    mainloop_destroy_signal(SIGTERM);
+    mainloop_destroy_signal(SIGTRAP);
+    /* leave SIGCHLD engaged as we might still want to drain some service-actions */
+
+    if (mloop) {
+        GMainContext *ctx = g_main_loop_get_context(controld_globals.mainloop);
+
+        /* Don't re-enter this block */
+        controld_globals.mainloop = NULL;
+
+        /* no signals on final draining anymore */
+        mainloop_destroy_signal(SIGCHLD);
+
+        crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
+
+        {
+            int lpc = 0;
+
+            while((g_main_context_pending(ctx) && lpc < 10)) {
+                lpc++;
+                crm_trace("Iteration %d", lpc);
+                g_main_context_dispatch(ctx);
+            }
+        }
+
+        crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
+        g_main_loop_quit(mloop);
+
+        /* Won't do anything yet, since we're inside it now */
+        g_main_loop_unref(mloop);
+    } else {
+        mainloop_destroy_signal(SIGCHLD);
+    }
+
+    cib_delete(controld_globals.cib_conn);
+    controld_globals.cib_conn = NULL;
+
+    throttle_fini();
+
+    /* Graceful */
+    crm_trace("Done preparing for exit with status %d (%s)",
+              exit_code, crm_exit_str(exit_code));
+    return exit_code;
+}
+
+/*	 A_EXIT_0, A_EXIT_1	*/
+void
+do_exit(long long action,
+        enum crmd_fsa_cause cause,
+        enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    crm_exit_t exit_code = CRM_EX_OK;
+    int log_level = LOG_INFO;
+    const char *exit_type = "gracefully";
+
+    if (action & A_EXIT_1) {
+        log_level = LOG_ERR;
+        exit_type = "forcefully";
+        exit_code = CRM_EX_ERROR;
+    }
+
+    verify_stopped(cur_state, LOG_ERR);
+    do_crm_log(log_level, "Performing %s - %s exiting the controller",
+               fsa_action2string(action), exit_type);
+
+    crm_info("[%s] stopped (%d)", crm_system_name, exit_code);
+    crmd_exit(exit_code);
+}
+
+static void sigpipe_ignore(int nsig) { return; }
+
+/*	 A_STARTUP	*/
+void
+do_startup(long long action,
+           enum crmd_fsa_cause cause,
+           enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    crm_debug("Registering Signal Handlers");
+    mainloop_add_signal(SIGTERM, crm_shutdown);
+    mainloop_add_signal(SIGPIPE, sigpipe_ignore);
+
+    config_read_trigger = mainloop_add_trigger(G_PRIORITY_HIGH,
+                                               crm_read_options, NULL);
+
+    controld_init_fsa_trigger();
+    controld_init_transition_trigger();
+
+    crm_debug("Creating CIB manager and executor objects");
+    controld_globals.cib_conn = cib_new();
+
+    lrm_state_init_local();
+    if (controld_init_fsa_timers() == FALSE) {
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+    }
+}
+
+// \return libqb error code (0 on success, -errno on error)
+static int32_t
+accept_controller_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
+{
+    crm_trace("Accepting new IPC client connection");
+    if (pcmk__new_client(c, uid, gid) == NULL) {
+        return -EIO;
+    }
+    return 0;
+}
+
+// \return libqb error code (0 on success, -errno on error)
+static int32_t
+dispatch_controller_ipc(qb_ipcs_connection_t * c, void *data, size_t size)
+{
+    uint32_t id = 0;
+    uint32_t flags = 0;
+    pcmk__client_t *client = pcmk__find_client(c);
+
+    xmlNode *msg = pcmk__client_data2xml(client, data, &id, &flags);
+
+    if (msg == NULL) {
+        pcmk__ipc_send_ack(client, id, flags, "ack", NULL, CRM_EX_PROTOCOL);
+        return 0;
+    }
+    pcmk__ipc_send_ack(client, id, flags, "ack", NULL, CRM_EX_INDETERMINATE);
+
+    CRM_ASSERT(client->user != NULL);
+    pcmk__update_acl_user(msg, F_CRM_USER, client->user);
+
+    crm_xml_add(msg, F_CRM_SYS_FROM, client->id);
+    if (controld_authorize_ipc_message(msg, client, NULL)) {
+        crm_trace("Processing IPC message from client %s",
+                  pcmk__client_name(client));
+        route_message(C_IPC_MESSAGE, msg);
+    }
+
+    controld_trigger_fsa();
+    free_xml(msg);
+    return 0;
+}
+
+static int32_t
+ipc_client_disconnected(qb_ipcs_connection_t *c)
+{
+    pcmk__client_t *client = pcmk__find_client(c);
+
+    if (client) {
+        crm_trace("Disconnecting %sregistered client %s (%p/%p)",
+                  (client->userdata? "" : "un"), pcmk__client_name(client),
+                  c, client);
+        free(client->userdata);
+        pcmk__free_client(client);
+        controld_trigger_fsa();
+    }
+    return 0;
+}
+
+static void
+ipc_connection_destroyed(qb_ipcs_connection_t *c)
+{
+    crm_trace("Connection %p", c);
+    ipc_client_disconnected(c);
+}
+
+/*	 A_STOP	*/
+void
+do_stop(long long action,
+        enum crmd_fsa_cause cause,
+        enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    crm_trace("Closing IPC server");
+    mainloop_del_ipc_server(ipcs); ipcs = NULL;
+    register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
+}
+
+/*	 A_STARTED	*/
+void
+do_started(long long action,
+           enum crmd_fsa_cause cause,
+           enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    static struct qb_ipcs_service_handlers crmd_callbacks = {
+        .connection_accept = accept_controller_client,
+        .connection_created = NULL,
+        .msg_process = dispatch_controller_ipc,
+        .connection_closed = ipc_client_disconnected,
+        .connection_destroyed = ipc_connection_destroyed
+    };
+
+    if (cur_state != S_STARTING) {
+        crm_err("Start cancelled... %s", fsa_state2string(cur_state));
+        return;
+
+    } else if (!pcmk_is_set(controld_globals.fsa_input_register,
+                            R_MEMBERSHIP)) {
+        crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP);
+
+        crmd_fsa_stall(TRUE);
+        return;
+
+    } else if (!pcmk_is_set(controld_globals.fsa_input_register,
+                            R_LRM_CONNECTED)) {
+        crm_info("Delaying start, not connected to executor (%.16llx)", R_LRM_CONNECTED);
+
+        crmd_fsa_stall(TRUE);
+        return;
+
+    } else if (!pcmk_is_set(controld_globals.fsa_input_register,
+                            R_CIB_CONNECTED)) {
+        crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED);
+
+        crmd_fsa_stall(TRUE);
+        return;
+
+    } else if (!pcmk_is_set(controld_globals.fsa_input_register,
+                            R_READ_CONFIG)) {
+        crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG);
+
+        crmd_fsa_stall(TRUE);
+        return;
+
+    } else if (!pcmk_is_set(controld_globals.fsa_input_register, R_PEER_DATA)) {
+
+        crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA);
+        crmd_fsa_stall(TRUE);
+        return;
+    }
+
+    crm_debug("Init server comms");
+    ipcs = pcmk__serve_controld_ipc(&crmd_callbacks);
+    if (ipcs == NULL) {
+        crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+    } else {
+        crm_notice("Pacemaker controller successfully started and accepting connections");
+    }
+    controld_trigger_fencer_connect();
+
+    controld_clear_fsa_input_flags(R_STARTING);
+    register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL);
+}
+
+/*	 A_RECOVER	*/
+void
+do_recover(long long action,
+           enum crmd_fsa_cause cause,
+           enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    controld_set_fsa_input_flags(R_IN_RECOVERY);
+    crm_warn("Fast-tracking shutdown in response to errors");
+
+    register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
+}
+
+static pcmk__cluster_option_t controller_options[] = {
+    /* name, old name, type, allowed values,
+     * default value, validator,
+     * short description,
+     * long description
+     */
+    {
+        "dc-version", NULL, "string", NULL, PCMK__VALUE_NONE, NULL,
+        N_("Pacemaker version on cluster node elected Designated Controller (DC)"),
+        N_("Includes a hash which identifies the exact changeset the code was "
+            "built from. Used for diagnostic purposes.")
+    },
+    {
+        "cluster-infrastructure", NULL, "string", NULL, "corosync", NULL,
+        N_("The messaging stack on which Pacemaker is currently running"),
+        N_("Used for informational and diagnostic purposes.")
+    },
+    {
+        "cluster-name", NULL, "string", NULL, NULL, NULL,
+        N_("An arbitrary name for the cluster"),
+        N_("This optional value is mostly for users' convenience as desired "
+            "in administration, but may also be used in Pacemaker "
+            "configuration rules via the #cluster-name node attribute, and "
+            "by higher-level tools and resource agents.")
+    },
+    {
+        XML_CONFIG_ATTR_DC_DEADTIME, NULL, "time",
+        NULL, "20s", pcmk__valid_interval_spec,
+        N_("How long to wait for a response from other nodes during start-up"),
+        N_("The optimal value will depend on the speed and load of your network "
+            "and the type of switches used.")
+    },
+    {
+        XML_CONFIG_ATTR_RECHECK, NULL, "time",
+        N_("Zero disables polling, while positive values are an interval in seconds"
+            "(unless other units are specified, for example \"5min\")"),
+        "15min", pcmk__valid_interval_spec,
+        N_("Polling interval to recheck cluster state and evaluate rules "
+            "with date specifications"),
+        N_("Pacemaker is primarily event-driven, and looks ahead to know when to "
+            "recheck cluster state for failure timeouts and most time-based "
+            "rules. However, it will also recheck the cluster after this "
+            "amount of inactivity, to evaluate rules with date specifications "
+            "and serve as a fail-safe for certain types of scheduler bugs.")
+    },
+    {
+        "load-threshold", NULL, "percentage", NULL,
+        "80%", pcmk__valid_percentage,
+        N_("Maximum amount of system load that should be used by cluster nodes"),
+        N_("The cluster will slow down its recovery process when the amount of "
+            "system resources used (currently CPU) approaches this limit"),
+    },
+    {
+        "node-action-limit", NULL, "integer", NULL,
+        "0", pcmk__valid_number,
+        N_("Maximum number of jobs that can be scheduled per node "
+            "(defaults to 2x cores)")
+    },
+    { XML_CONFIG_ATTR_FENCE_REACTION, NULL, "string", NULL, "stop", NULL,
+        N_("How a cluster node should react if notified of its own fencing"),
+        N_("A cluster node may receive notification of its own fencing if fencing "
+        "is misconfigured, or if fabric fencing is in use that doesn't cut "
+        "cluster communication. Allowed values are \"stop\" to attempt to "
+        "immediately stop Pacemaker and stay stopped, or \"panic\" to attempt "
+        "to immediately reboot the local node, falling back to stop on failure.")
+    },
+    {
+        XML_CONFIG_ATTR_ELECTION_FAIL, NULL, "time", NULL,
+        "2min", pcmk__valid_interval_spec,
+        "*** Advanced Use Only ***",
+        N_("Declare an election failed if it is not decided within this much "
+            "time. If you need to adjust this value, it probably indicates "
+            "the presence of a bug.")
+    },
+    {
+        XML_CONFIG_ATTR_FORCE_QUIT, NULL, "time", NULL,
+        "20min", pcmk__valid_interval_spec,
+        "*** Advanced Use Only ***",
+        N_("Exit immediately if shutdown does not complete within this much "
+            "time. If you need to adjust this value, it probably indicates "
+            "the presence of a bug.")
+    },
+    {
+        "join-integration-timeout", "crmd-integration-timeout", "time", NULL,
+        "3min", pcmk__valid_interval_spec,
+        "*** Advanced Use Only ***",
+        N_("If you need to adjust this value, it probably indicates "
+            "the presence of a bug.")
+    },
+    {
+        "join-finalization-timeout", "crmd-finalization-timeout", "time", NULL,
+        "30min", pcmk__valid_interval_spec,
+        "*** Advanced Use Only ***",
+        N_("If you need to adjust this value, it probably indicates "
+            "the presence of a bug.")
+    },
+    {
+        "transition-delay", "crmd-transition-delay", "time", NULL,
+        "0s", pcmk__valid_interval_spec,
+        N_("*** Advanced Use Only *** Enabling this option will slow down "
+            "cluster recovery under all conditions"),
+        N_("Delay cluster recovery for this much time to allow for additional "
+            "events to occur. Useful if your configuration is sensitive to "
+            "the order in which ping updates arrive.")
+    },
+    {
+        "stonith-watchdog-timeout", NULL, "time", NULL,
+        "0", controld_verify_stonith_watchdog_timeout,
+        N_("How long before nodes can be assumed to be safely down when "
+           "watchdog-based self-fencing via SBD is in use"),
+        N_("If this is set to a positive value, lost nodes are assumed to "
+           "self-fence using watchdog-based SBD within this much time. This "
+           "does not require a fencing resource to be explicitly configured, "
+           "though a fence_watchdog resource can be configured, to limit use "
+           "to specific nodes. If this is set to 0 (the default), the cluster "
+           "will never assume watchdog-based self-fencing. If this is set to a "
+           "negative value, the cluster will use twice the local value of the "
+           "`SBD_WATCHDOG_TIMEOUT` environment variable if that is positive, "
+           "or otherwise treat this as 0. WARNING: When used, this timeout "
+           "must be larger than `SBD_WATCHDOG_TIMEOUT` on all nodes that use "
+           "watchdog-based SBD, and Pacemaker will refuse to start on any of "
+           "those nodes where this is not true for the local value or SBD is "
+           "not active. When this is set to a negative value, "
+           "`SBD_WATCHDOG_TIMEOUT` must be set to the same value on all nodes "
+           "that use SBD, otherwise data corruption or loss could occur.")
+    },
+    {
+        "stonith-max-attempts", NULL, "integer", NULL,
+        "10", pcmk__valid_positive_number,
+        N_("How many times fencing can fail before it will no longer be "
+            "immediately re-attempted on a target")
+    },
+
+    // Already documented in libpe_status (other values must be kept identical)
+    {
+        "no-quorum-policy", NULL, "select",
+        "stop, freeze, ignore, demote, suicide", "stop", pcmk__valid_quorum,
+        N_("What to do when the cluster does not have quorum"), NULL
+    },
+    {
+        XML_CONFIG_ATTR_SHUTDOWN_LOCK, NULL, "boolean", NULL,
+        "false", pcmk__valid_boolean,
+        N_("Whether to lock resources to a cleanly shut down node"),
+        N_("When true, resources active on a node when it is cleanly shut down "
+            "are kept \"locked\" to that node (not allowed to run elsewhere) "
+            "until they start again on that node after it rejoins (or for at "
+            "most shutdown-lock-limit, if set). Stonith resources and "
+            "Pacemaker Remote connections are never locked. Clone and bundle "
+            "instances and the promoted role of promotable clones are "
+            "currently never locked, though support could be added in a future "
+            "release.")
+    },
+    {
+        XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL,
+        "0", pcmk__valid_interval_spec,
+        N_("Do not lock resources to a cleanly shut down node longer than "
+           "this"),
+        N_("If shutdown-lock is true and this is set to a nonzero time "
+            "duration, shutdown locks will expire after this much time has "
+            "passed since the shutdown was initiated, even if the node has not "
+            "rejoined.")
+    },
+};
+
+void
+crmd_metadata(void)
+{
+    const char *desc_short = "Pacemaker controller options";
+    const char *desc_long = "Cluster options used by Pacemaker's controller";
+
+    gchar *s = pcmk__format_option_metadata("pacemaker-controld", desc_short,
+                                            desc_long, controller_options,
+                                            PCMK__NELEM(controller_options));
+    printf("%s", s);
+    g_free(s);
+}
+
+static void
+config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    const char *value = NULL;
+    GHashTable *config_hash = NULL;
+    crm_time_t *now = crm_time_new(NULL);
+    xmlNode *crmconfig = NULL;
+    xmlNode *alerts = NULL;
+
+    if (rc != pcmk_ok) {
+        fsa_data_t *msg_data = NULL;
+
+        crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc));
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+
+        if (rc == -EACCES || rc == -pcmk_err_schema_validation) {
+            crm_err("The cluster is mis-configured - shutting down and staying down");
+            controld_set_fsa_input_flags(R_STAYDOWN);
+        }
+        goto bail;
+    }
+
+    crmconfig = output;
+    if ((crmconfig) &&
+        (crm_element_name(crmconfig)) &&
+        (strcmp(crm_element_name(crmconfig), XML_CIB_TAG_CRMCONFIG) != 0)) {
+        crmconfig = first_named_child(crmconfig, XML_CIB_TAG_CRMCONFIG);
+    }
+    if (!crmconfig) {
+        fsa_data_t *msg_data = NULL;
+
+        crm_err("Local CIB query for " XML_CIB_TAG_CRMCONFIG " section failed");
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+        goto bail;
+    }
+
+    crm_debug("Call %d : Parsing CIB options", call_id);
+    config_hash = pcmk__strkey_table(free, free);
+    pe_unpack_nvpairs(crmconfig, crmconfig, XML_CIB_TAG_PROPSET, NULL,
+                      config_hash, CIB_OPTIONS_FIRST, FALSE, now, NULL);
+
+    // Validate all options, and use defaults if not already present in hash
+    pcmk__validate_cluster_options(config_hash, controller_options,
+                                   PCMK__NELEM(controller_options));
+
+    value = g_hash_table_lookup(config_hash, "no-quorum-policy");
+    if (pcmk__str_eq(value, "suicide", pcmk__str_casei) && pcmk__locate_sbd()) {
+        controld_set_global_flags(controld_no_quorum_suicide);
+    }
+
+    value = g_hash_table_lookup(config_hash, XML_CONFIG_ATTR_SHUTDOWN_LOCK);
+    if (crm_is_true(value)) {
+        controld_set_global_flags(controld_shutdown_lock_enabled);
+    } else {
+        controld_clear_global_flags(controld_shutdown_lock_enabled);
+    }
+
+    value = g_hash_table_lookup(config_hash,
+                                XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT);
+    controld_globals.shutdown_lock_limit = crm_parse_interval_spec(value)
+                                           / 1000;
+
+    value = g_hash_table_lookup(config_hash, "cluster-name");
+    pcmk__str_update(&(controld_globals.cluster_name), value);
+
+    // Let subcomponents initialize their own static variables
+    controld_configure_election(config_hash);
+    controld_configure_fencing(config_hash);
+    controld_configure_fsa_timers(config_hash);
+    controld_configure_throttle(config_hash);
+
+    alerts = first_named_child(output, XML_CIB_TAG_ALERTS);
+    crmd_unpack_alerts(alerts);
+
+    controld_set_fsa_input_flags(R_READ_CONFIG);
+    controld_trigger_fsa();
+
+    g_hash_table_destroy(config_hash);
+  bail:
+    crm_time_free(now);
+}
+
+/*!
+ * \internal
+ * \brief Trigger read and processing of the configuration
+ *
+ * \param[in] fn    Calling function name
+ * \param[in] line  Line number where call occurred
+ */
+void
+controld_trigger_config_as(const char *fn, int line)
+{
+    if (config_read_trigger != NULL) {
+        crm_trace("%s:%d - Triggered config processing", fn, line);
+        mainloop_set_trigger(config_read_trigger);
+    }
+}
+
+gboolean
+crm_read_options(gpointer user_data)
+{
+    cib_t *cib_conn = controld_globals.cib_conn;
+    int call_id = cib_conn->cmds->query(cib_conn,
+                                        "//" XML_CIB_TAG_CRMCONFIG
+                                        " | //" XML_CIB_TAG_ALERTS,
+                                        NULL, cib_xpath|cib_scope_local);
+
+    fsa_register_cib_callback(call_id, NULL, config_query_callback);
+    crm_trace("Querying the CIB... call %d", call_id);
+    return TRUE;
+}
+
+/*	 A_READCONFIG	*/
+void
+do_read_config(long long action,
+               enum crmd_fsa_cause cause,
+               enum crmd_fsa_state cur_state,
+               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    throttle_init();
+    controld_trigger_config();
+}
+
+void
+crm_shutdown(int nsig)
+{
+    const char *value = NULL;
+    guint default_period_ms = 0;
+
+    if ((controld_globals.mainloop == NULL)
+        || !g_main_loop_is_running(controld_globals.mainloop)) {
+        crmd_exit(CRM_EX_OK);
+        return;
+    }
+
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+        crm_err("Escalating shutdown");
+        register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL);
+        return;
+    }
+
+    controld_set_fsa_input_flags(R_SHUTDOWN);
+    register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
+
+    /* If shutdown timer doesn't have a period set, use the default
+     *
+     * @TODO: Evaluate whether this is still necessary. As long as
+     * config_query_callback() has been run at least once, it doesn't look like
+     * anything could have changed the timer period since then.
+     */
+    value = pcmk__cluster_option(NULL, controller_options,
+                                 PCMK__NELEM(controller_options),
+                                 XML_CONFIG_ATTR_FORCE_QUIT);
+    default_period_ms = crm_parse_interval_spec(value);
+    controld_shutdown_start_countdown(default_period_ms);
+}
diff --git a/daemons/controld/controld_corosync.c b/daemons/controld/controld_corosync.c
new file mode 100644
index 0000000..4378b30
--- /dev/null
+++ b/daemons/controld/controld_corosync.c
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include <crm/crm.h>
+#include <crm/cluster/internal.h>
+#include <crm/common/xml.h>
+
+#include <pacemaker-controld.h>
+
+#if SUPPORT_COROSYNC
+
+extern void post_cache_update(int seq);
+
+/*	 A_HA_CONNECT	*/
+
+static void
+crmd_cs_dispatch(cpg_handle_t handle, const struct cpg_name *groupName,
+                 uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
+{
+    uint32_t kind = 0;
+    const char *from = NULL;
+    char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);
+
+    if(data == NULL) {
+        return;
+    }
+    if (kind == crm_class_cluster) {
+        crm_node_t *peer = NULL;
+        xmlNode *xml = string2xml(data);
+
+        if (xml == NULL) {
+            crm_err("Could not parse message content (%d): %.100s", kind, data);
+            free(data);
+            return;
+        }
+
+        crm_xml_add(xml, F_ORIG, from);
+        /* crm_xml_add_int(xml, F_SEQ, wrapper->id); Fake? */
+
+        peer = crm_get_peer(0, from);
+        if (!pcmk_is_set(peer->processes, crm_proc_cpg)) {
+            /* If we can still talk to our peer process on that node,
+             * then it must be part of the corosync membership
+             */
+            crm_warn("Receiving messages from a node we think is dead: %s[%d]",
+                     peer->uname, peer->id);
+            crm_update_peer_proc(__func__, peer, crm_proc_cpg,
+                                 ONLINESTATUS);
+        }
+        crmd_ha_msg_filter(xml);
+        free_xml(xml);
+    } else {
+        crm_err("Invalid message class (%d): %.100s", kind, data);
+    }
+    free(data);
+}
+
+static gboolean
+crmd_quorum_callback(unsigned long long seq, gboolean quorate)
+{
+    crm_update_quorum(quorate, FALSE);
+    post_cache_update(seq);
+    return TRUE;
+}
+
+static void
+crmd_cs_destroy(gpointer user_data)
+{
+    if (!pcmk_is_set(controld_globals.fsa_input_register, R_HA_DISCONNECTED)) {
+        crm_crit("Lost connection to cluster layer, shutting down");
+        crmd_exit(CRM_EX_DISCONNECT);
+
+    } else {
+        crm_info("Corosync connection closed");
+    }
+}
+
+/*!
+ * \brief Handle a Corosync notification of a CPG configuration change
+ *
+ * \param[in] handle               CPG connection
+ * \param[in] cpg_name             CPG group name
+ * \param[in] member_list          List of current CPG members
+ * \param[in] member_list_entries  Number of entries in \p member_list
+ * \param[in] left_list            List of CPG members that left
+ * \param[in] left_list_entries    Number of entries in \p left_list
+ * \param[in] joined_list          List of CPG members that joined
+ * \param[in] joined_list_entries  Number of entries in \p joined_list
+ */
+static void
+cpg_membership_callback(cpg_handle_t handle, const struct cpg_name *cpg_name,
+                        const struct cpg_address *member_list,
+                        size_t member_list_entries,
+                        const struct cpg_address *left_list,
+                        size_t left_list_entries,
+                        const struct cpg_address *joined_list,
+                        size_t joined_list_entries)
+{
+    /* When nodes leave CPG, the DC clears their transient node attributes.
+     *
+     * However if there is no DC, or the DC is among the nodes that left, each
+     * remaining node needs to do the clearing, to ensure it gets done.
+     * Otherwise, the attributes would persist when the nodes rejoin, which
+     * could have serious consequences for unfencing, agents that use attributes
+     * for internal logic, etc.
+     *
+     * Here, we set a global boolean if the DC is among the nodes that left, for
+     * use by the peer callback.
+     */
+    if (controld_globals.dc_name != NULL) {
+        crm_node_t *peer = NULL;
+
+        peer = pcmk__search_cluster_node_cache(0, controld_globals.dc_name);
+        if (peer != NULL) {
+            for (int i = 0; i < left_list_entries; ++i) {
+                if (left_list[i].nodeid == peer->id) {
+                    controld_set_global_flags(controld_dc_left);
+                    break;
+                }
+            }
+        }
+    }
+
+    // Process the change normally, which will call the peer callback as needed
+    pcmk_cpg_membership(handle, cpg_name, member_list, member_list_entries,
+                        left_list, left_list_entries,
+                        joined_list, joined_list_entries);
+
+    controld_clear_global_flags(controld_dc_left);
+}
+
+extern gboolean crm_connect_corosync(crm_cluster_t * cluster);
+
+gboolean
+crm_connect_corosync(crm_cluster_t * cluster)
+{
+    if (is_corosync_cluster()) {
+        crm_set_status_callback(&peer_update_callback);
+        cluster->cpg.cpg_deliver_fn = crmd_cs_dispatch;
+        cluster->cpg.cpg_confchg_fn = cpg_membership_callback;
+        cluster->destroy = crmd_cs_destroy;
+
+        if (crm_cluster_connect(cluster)) {
+            pcmk__corosync_quorum_connect(crmd_quorum_callback,
+                                          crmd_cs_destroy);
+            return TRUE;
+        }
+    }
+    return FALSE;
+}
+
+#endif
diff --git a/daemons/controld/controld_election.c b/daemons/controld/controld_election.c
new file mode 100644
index 0000000..5f33d5b
--- /dev/null
+++ b/daemons/controld/controld_election.c
@@ -0,0 +1,292 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/time.h>
+#include <sys/resource.h>
+
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/cluster/internal.h>
+#include <crm/cluster/election_internal.h>
+#include <crm/crm.h>
+
+#include <pacemaker-controld.h>
+
+static election_t *fsa_election = NULL;
+
+static gboolean
+election_win_cb(gpointer data)
+{
+    register_fsa_input(C_FSA_INTERNAL, I_ELECTION_DC, NULL);
+    return FALSE;
+}
+
+void
+controld_election_init(const char *uname)
+{
+    fsa_election = election_init("DC", uname, 60000 /*60s*/, election_win_cb);
+}
+
+/*!
+ * \internal
+ * \brief Configure election options based on the CIB
+ *
+ * \param[in,out] options  Name/value pairs for configured options
+ */
+void
+controld_configure_election(GHashTable *options)
+{
+    const char *value = NULL;
+
+    value = g_hash_table_lookup(options, XML_CONFIG_ATTR_ELECTION_FAIL);
+    election_timeout_set_period(fsa_election, crm_parse_interval_spec(value));
+}
+
+void
+controld_remove_voter(const char *uname)
+{
+    election_remove(fsa_election, uname);
+
+    if (pcmk__str_eq(uname, controld_globals.dc_name, pcmk__str_casei)) {
+        /* Clear any election dampening in effect. Otherwise, if the lost DC had
+         * just won, an immediate new election could fizzle out with no new DC.
+         */
+        election_clear_dampening(fsa_election);
+    }
+}
+
+void
+controld_election_fini(void)
+{
+    election_fini(fsa_election);
+    fsa_election = NULL;
+}
+
+void
+controld_stop_current_election_timeout(void)
+{
+    election_timeout_stop(fsa_election);
+}
+
+/*	A_ELECTION_VOTE	*/
+void
+do_election_vote(long long action,
+                 enum crmd_fsa_cause cause,
+                 enum crmd_fsa_state cur_state,
+                 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    gboolean not_voting = FALSE;
+
+    /* don't vote if we're in one of these states or wanting to shut down */
+    switch (cur_state) {
+        case S_STARTING:
+        case S_RECOVERY:
+        case S_STOPPING:
+        case S_TERMINATE:
+            crm_warn("Not voting in election, we're in state %s", fsa_state2string(cur_state));
+            not_voting = TRUE;
+            break;
+        case S_ELECTION:
+        case S_INTEGRATION:
+        case S_RELEASE_DC:
+            break;
+        default:
+            crm_err("Broken? Voting in state %s", fsa_state2string(cur_state));
+            break;
+    }
+
+    if (not_voting == FALSE) {
+        if (pcmk_is_set(controld_globals.fsa_input_register, R_STARTING)) {
+            not_voting = TRUE;
+        }
+    }
+
+    if (not_voting) {
+        if (AM_I_DC) {
+            register_fsa_input(C_FSA_INTERNAL, I_RELEASE_DC, NULL);
+
+        } else {
+            register_fsa_input(C_FSA_INTERNAL, I_PENDING, NULL);
+        }
+        return;
+    }
+
+    election_vote(fsa_election);
+    return;
+}
+
+void
+do_election_check(long long action,
+                  enum crmd_fsa_cause cause,
+                  enum crmd_fsa_state cur_state,
+                  enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    if (controld_globals.fsa_state == S_ELECTION) {
+        election_check(fsa_election);
+    } else {
+        crm_debug("Ignoring election check because we are not in an election");
+    }
+}
+
+/*	A_ELECTION_COUNT	*/
+void
+do_election_count_vote(long long action,
+                       enum crmd_fsa_cause cause,
+                       enum crmd_fsa_state cur_state,
+                       enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    enum election_result rc = 0;
+    ha_msg_input_t *vote = fsa_typed_data(fsa_dt_ha_msg);
+
+    if(crm_peer_cache == NULL) {
+        if (!pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+            crm_err("Internal error, no peer cache");
+        }
+        return;
+    }
+
+    rc = election_count_vote(fsa_election, vote->msg, cur_state != S_STARTING);
+    switch(rc) {
+        case election_start:
+            election_reset(fsa_election);
+            register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
+            break;
+
+        case election_lost:
+            update_dc(NULL);
+
+            if (pcmk_is_set(controld_globals.fsa_input_register, R_THE_DC)) {
+                cib_t *cib_conn = controld_globals.cib_conn;
+
+                register_fsa_input(C_FSA_INTERNAL, I_RELEASE_DC, NULL);
+                cib_conn->cmds->set_secondary(cib_conn, cib_scope_local);
+
+            } else if (cur_state != S_STARTING) {
+                register_fsa_input(C_FSA_INTERNAL, I_PENDING, NULL);
+            }
+            break;
+
+        default:
+            crm_trace("Election message resulted in state %d", rc);
+    }
+}
+
+static void
+feature_update_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    if (rc != pcmk_ok) {
+        fsa_data_t *msg_data = NULL;
+
+        crm_notice("Feature update failed: %s "CRM_XS" rc=%d",
+                   pcmk_strerror(rc), rc);
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Update a node attribute in the CIB during a DC takeover
+ *
+ * \param[in] name   Name of attribute to update
+ * \param[in] value  New attribute value
+ */
+#define dc_takeover_update_attr(name, value) do {                           \
+       cib__update_node_attr(controld_globals.logger_out,                   \
+                             controld_globals.cib_conn, cib_none,           \
+                             XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL, \
+                             name, value, NULL, NULL);                      \
+    } while (0)
+
+/*	 A_DC_TAKEOVER	*/
+void
+do_dc_takeover(long long action,
+               enum crmd_fsa_cause cause,
+               enum crmd_fsa_state cur_state,
+               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    xmlNode *cib = NULL;
+    const char *cluster_type = name_for_cluster_type(get_cluster_type());
+    pid_t watchdog = pcmk__locate_sbd();
+
+    crm_info("Taking over DC status for this partition");
+    controld_set_fsa_input_flags(R_THE_DC);
+    execute_stonith_cleanup();
+
+    election_reset(fsa_election);
+    controld_set_fsa_input_flags(R_JOIN_OK|R_INVOKE_PE);
+
+    controld_globals.cib_conn->cmds->set_primary(controld_globals.cib_conn,
+                                                 cib_scope_local);
+
+    cib = create_xml_node(NULL, XML_TAG_CIB);
+    crm_xml_add(cib, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
+    controld_update_cib(XML_TAG_CIB, cib, cib_none, feature_update_callback);
+
+    dc_takeover_update_attr(XML_ATTR_HAVE_WATCHDOG, pcmk__btoa(watchdog));
+    dc_takeover_update_attr("dc-version", PACEMAKER_VERSION "-" BUILD_VERSION);
+    dc_takeover_update_attr("cluster-infrastructure", cluster_type);
+
+#if SUPPORT_COROSYNC
+    if ((controld_globals.cluster_name == NULL) && is_corosync_cluster()) {
+        char *cluster_name = pcmk__corosync_cluster_name();
+
+        if (cluster_name != NULL) {
+            dc_takeover_update_attr("cluster-name", cluster_name);
+        }
+        free(cluster_name);
+    }
+#endif
+
+    controld_trigger_config();
+    free_xml(cib);
+}
+
+/*	 A_DC_RELEASE	*/
+void
+do_dc_release(long long action,
+              enum crmd_fsa_cause cause,
+              enum crmd_fsa_state cur_state,
+              enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    if (action & A_DC_RELEASE) {
+        crm_debug("Releasing the role of DC");
+        controld_clear_fsa_input_flags(R_THE_DC);
+        controld_expect_sched_reply(NULL);
+
+    } else if (action & A_DC_RELEASED) {
+        crm_info("DC role released");
+#if 0
+        if (are there errors) {
+            /* we can't stay up if not healthy */
+            /* or perhaps I_ERROR and go to S_RECOVER? */
+            result = I_SHUTDOWN;
+        }
+#endif
+        if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+            xmlNode *update = NULL;
+            crm_node_t *node = crm_get_peer(0, controld_globals.our_nodename);
+
+            pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN);
+            update = create_node_state_update(node, node_update_expected, NULL,
+                                              __func__);
+            /* Don't need a based response because controld will stop. */
+            fsa_cib_anon_update_discard_reply(XML_CIB_TAG_STATUS, update);
+            free_xml(update);
+        }
+        register_fsa_input(C_FSA_INTERNAL, I_RELEASE_SUCCESS, NULL);
+
+    } else {
+        crm_err("Unknown DC action %s", fsa_action2string(action));
+    }
+
+    crm_trace("Am I still the DC? %s", AM_I_DC ? XML_BOOLEAN_YES : XML_BOOLEAN_NO);
+
+}
diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c
new file mode 100644
index 0000000..0de399c
--- /dev/null
+++ b/daemons/controld/controld_execd.c
@@ -0,0 +1,2433 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <regex.h>
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include <crm/crm.h>
+#include <crm/lrmd.h>           // lrmd_event_data_t, lrmd_rsc_info_t, etc.
+#include <crm/services.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/pengine/rules.h>
+#include <crm/lrmd_internal.h>
+
+#include <pacemaker-internal.h>
+#include <pacemaker-controld.h>
+
+#define START_DELAY_THRESHOLD 5 * 60 * 1000
+#define MAX_LRM_REG_FAILS 30
+
+struct delete_event_s {
+    int rc;
+    const char *rsc;
+    lrm_state_t *lrm_state;
+};
+
+static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
+static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
+static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
+
+static lrmd_event_data_t *construct_op(const lrm_state_t *lrm_state,
+                                       const xmlNode *rsc_op,
+                                       const char *rsc_id,
+                                       const char *operation);
+static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
+                          xmlNode *msg, struct ra_metadata_s *md);
+
+static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
+                                         int log_level);
+
+static void
+lrm_connection_destroy(void)
+{
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_LRM_CONNECTED)) {
+        crm_crit("Connection to executor failed");
+        register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
+        controld_clear_fsa_input_flags(R_LRM_CONNECTED);
+
+    } else {
+        crm_info("Disconnected from executor");
+    }
+
+}
+
+static char *
+make_stop_id(const char *rsc, int call_id)
+{
+    return crm_strdup_printf("%s:%d", rsc, call_id);
+}
+
+static void
+copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
+{
+    if (strstr(key, CRM_META "_") == NULL) {
+        g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
+    }
+}
+
+static void
+copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
+{
+    if (strstr(key, CRM_META "_") != NULL) {
+        g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
+    }
+}
+
+/*!
+ * \internal
+ * \brief Remove a recurring operation from a resource's history
+ *
+ * \param[in,out] history  Resource history to modify
+ * \param[in]     op       Operation to remove
+ *
+ * \return TRUE if the operation was found and removed, FALSE otherwise
+ */
+static gboolean
+history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op)
+{
+    GList *iter;
+
+    for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
+        lrmd_event_data_t *existing = iter->data;
+
+        if ((op->interval_ms == existing->interval_ms)
+            && pcmk__str_eq(op->rsc_id, existing->rsc_id, pcmk__str_none)
+            && pcmk__str_eq(op->op_type, existing->op_type, pcmk__str_casei)) {
+
+            history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
+            lrmd_free_event(existing);
+            return TRUE;
+        }
+    }
+    return FALSE;
+}
+
+/*!
+ * \internal
+ * \brief Free all recurring operations in resource history
+ *
+ * \param[in,out] history  Resource history to modify
+ */
+static void
+history_free_recurring_ops(rsc_history_t *history)
+{
+    GList *iter;
+
+    for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
+        lrmd_free_event(iter->data);
+    }
+    g_list_free(history->recurring_op_list);
+    history->recurring_op_list = NULL;
+}
+
+/*!
+ * \internal
+ * \brief Free resource history
+ *
+ * \param[in,out] history  Resource history to free
+ */
+void
+history_free(gpointer data)
+{
+    rsc_history_t *history = (rsc_history_t*)data;
+
+    if (history->stop_params) {
+        g_hash_table_destroy(history->stop_params);
+    }
+
+    /* Don't need to free history->rsc.id because it's set to history->id */
+    free(history->rsc.type);
+    free(history->rsc.standard);
+    free(history->rsc.provider);
+
+    lrmd_free_event(history->failed);
+    lrmd_free_event(history->last);
+    free(history->id);
+    history_free_recurring_ops(history);
+    free(history);
+}
+
+static void
+update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
+{
+    int target_rc = 0;
+    rsc_history_t *entry = NULL;
+
+    if (op->rsc_deleted) {
+        crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
+        controld_delete_resource_history(op->rsc_id, lrm_state->node_name,
+                                         NULL, crmd_cib_smart_opt());
+        return;
+    }
+
+    if (pcmk__str_eq(op->op_type, RSC_NOTIFY, pcmk__str_casei)) {
+        return;
+    }
+
+    crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);
+
+    entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
+    if (entry == NULL && rsc) {
+        entry = calloc(1, sizeof(rsc_history_t));
+        entry->id = strdup(op->rsc_id);
+        g_hash_table_insert(lrm_state->resource_history, entry->id, entry);
+
+        entry->rsc.id = entry->id;
+        entry->rsc.type = strdup(rsc->type);
+        entry->rsc.standard = strdup(rsc->standard);
+        pcmk__str_update(&entry->rsc.provider, rsc->provider);
+
+    } else if (entry == NULL) {
+        crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
+        return;
+    }
+
+    entry->last_callid = op->call_id;
+    target_rc = rsc_op_expected_rc(op);
+    if (op->op_status == PCMK_EXEC_CANCELLED) {
+        if (op->interval_ms > 0) {
+            crm_trace("Removing cancelled recurring op: " PCMK__OP_FMT,
+                      op->rsc_id, op->op_type, op->interval_ms);
+            history_remove_recurring_op(entry, op);
+            return;
+        } else {
+            crm_trace("Skipping " PCMK__OP_FMT " rc=%d, status=%d",
+                      op->rsc_id, op->op_type, op->interval_ms, op->rc,
+                      op->op_status);
+        }
+
+    } else if (did_rsc_op_fail(op, target_rc)) {
+        /* Store failed monitors here, otherwise the block below will cause them
+         * to be forgotten when a stop happens.
+         */
+        if (entry->failed) {
+            lrmd_free_event(entry->failed);
+        }
+        entry->failed = lrmd_copy_event(op);
+
+    } else if (op->interval_ms == 0) {
+        if (entry->last) {
+            lrmd_free_event(entry->last);
+        }
+        entry->last = lrmd_copy_event(op);
+
+        if (op->params && pcmk__strcase_any_of(op->op_type, CRMD_ACTION_START,
+                                               CRMD_ACTION_RELOAD,
+                                               CRMD_ACTION_RELOAD_AGENT,
+                                               CRMD_ACTION_STATUS, NULL)) {
+            if (entry->stop_params) {
+                g_hash_table_destroy(entry->stop_params);
+            }
+            entry->stop_params = pcmk__strkey_table(free, free);
+
+            g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
+        }
+    }
+
+    if (op->interval_ms > 0) {
+        /* Ensure there are no duplicates */
+        history_remove_recurring_op(entry, op);
+
+        crm_trace("Adding recurring op: " PCMK__OP_FMT,
+                  op->rsc_id, op->op_type, op->interval_ms);
+        entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
+
+    } else if (entry->recurring_op_list && !pcmk__str_eq(op->op_type, RSC_STATUS, pcmk__str_casei)) {
+        crm_trace("Dropping %d recurring ops because of: " PCMK__OP_FMT,
+                  g_list_length(entry->recurring_op_list), op->rsc_id,
+                  op->op_type, op->interval_ms);
+        history_free_recurring_ops(entry);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Send a direct OK ack for a resource task
+ *
+ * \param[in] lrm_state  LRM connection
+ * \param[in] input      Input message being ack'ed
+ * \param[in] rsc_id     ID of affected resource
+ * \param[in] rsc        Affected resource (if available)
+ * \param[in] task       Operation task being ack'ed
+ * \param[in] ack_host   Name of host to send ack to
+ * \param[in] ack_sys    IPC system name to ack
+ */
+static void
+send_task_ok_ack(const lrm_state_t *lrm_state, const ha_msg_input_t *input,
+                 const char *rsc_id, const lrmd_rsc_info_t *rsc,
+                 const char *task, const char *ack_host, const char *ack_sys)
+{
+    lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task);
+
+    lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
+    controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id);
+    lrmd_free_event(op);
+}
+
+static inline const char *
+op_node_name(lrmd_event_data_t *op)
+{
+    return pcmk__s(op->remote_nodename, controld_globals.our_nodename);
+}
+
+void
+lrm_op_callback(lrmd_event_data_t * op)
+{
+    CRM_CHECK(op != NULL, return);
+    switch (op->type) {
+        case lrmd_event_disconnect:
+            if (op->remote_nodename == NULL) {
+                /* If this is the local executor IPC connection, set the right
+                 * bits in the controller when the connection goes down.
+                 */
+                lrm_connection_destroy();
+            }
+            break;
+
+        case lrmd_event_exec_complete:
+            {
+                lrm_state_t *lrm_state = lrm_state_find(op_node_name(op));
+
+                CRM_ASSERT(lrm_state != NULL);
+                process_lrm_event(lrm_state, op, NULL, NULL);
+            }
+            break;
+
+        default:
+            break;
+    }
+}
+
+static void
+try_local_executor_connect(long long action, fsa_data_t *msg_data,
+                           lrm_state_t *lrm_state)
+{
+    int rc = pcmk_rc_ok;
+
+    crm_debug("Connecting to the local executor");
+
+    // If we can connect, great
+    rc = controld_connect_local_executor(lrm_state);
+    if (rc == pcmk_rc_ok) {
+        controld_set_fsa_input_flags(R_LRM_CONNECTED);
+        crm_info("Connection to the local executor established");
+        return;
+    }
+
+    // Otherwise, if we can try again, set a timer to do so
+    if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
+        crm_warn("Failed to connect to the local executor %d time%s "
+                 "(%d max): %s", lrm_state->num_lrm_register_fails,
+                 pcmk__plural_s(lrm_state->num_lrm_register_fails),
+                 MAX_LRM_REG_FAILS, pcmk_rc_str(rc));
+        controld_start_wait_timer();
+        crmd_fsa_stall(FALSE);
+        return;
+    }
+
+    // Otherwise give up
+    crm_err("Failed to connect to the executor the max allowed "
+            "%d time%s: %s", lrm_state->num_lrm_register_fails,
+            pcmk__plural_s(lrm_state->num_lrm_register_fails),
+            pcmk_rc_str(rc));
+    register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+}
+
+/*	 A_LRM_CONNECT	*/
+void
+do_lrm_control(long long action,
+               enum crmd_fsa_cause cause,
+               enum crmd_fsa_state cur_state,
+               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    /* This only pertains to local executor connections. Remote connections are
+     * handled as resources within the scheduler. Connecting and disconnecting
+     * from remote executor instances is handled differently.
+     */
+
+    lrm_state_t *lrm_state = NULL;
+
+    if (controld_globals.our_nodename == NULL) {
+        return; /* Nothing to do */
+    }
+    lrm_state = lrm_state_find_or_create(controld_globals.our_nodename);
+    if (lrm_state == NULL) {
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+        return;
+    }
+
+    if (action & A_LRM_DISCONNECT) {
+        if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
+            if (action == A_LRM_DISCONNECT) {
+                crmd_fsa_stall(FALSE);
+                return;
+            }
+        }
+
+        controld_clear_fsa_input_flags(R_LRM_CONNECTED);
+        crm_info("Disconnecting from the executor");
+        lrm_state_disconnect(lrm_state);
+        lrm_state_reset_tables(lrm_state, FALSE);
+        crm_notice("Disconnected from the executor");
+    }
+
+    if (action & A_LRM_CONNECT) {
+        try_local_executor_connect(action, msg_data, lrm_state);
+    }
+
+    if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) {
+        crm_err("Unexpected action %s in %s", fsa_action2string(action),
+                __func__);
+    }
+}
+
+static gboolean
+lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
+{
+    int counter = 0;
+    gboolean rc = TRUE;
+    const char *when = "lrm disconnect";
+
+    GHashTableIter gIter;
+    const char *key = NULL;
+    rsc_history_t *entry = NULL;
+    active_op_t *pending = NULL;
+
+    crm_debug("Checking for active resources before exit");
+
+    if (cur_state == S_TERMINATE) {
+        log_level = LOG_ERR;
+        when = "shutdown";
+
+    } else if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+        when = "shutdown... waiting";
+    }
+
+    if ((lrm_state->active_ops != NULL) && lrm_state_is_connected(lrm_state)) {
+        guint removed = g_hash_table_foreach_remove(lrm_state->active_ops,
+                                                    stop_recurring_actions,
+                                                    lrm_state);
+        guint nremaining = g_hash_table_size(lrm_state->active_ops);
+
+        if (removed || nremaining) {
+            crm_notice("Stopped %u recurring operation%s at %s (%u remaining)",
+                       removed, pcmk__plural_s(removed), when, nremaining);
+        }
+    }
+
+    if (lrm_state->active_ops != NULL) {
+        g_hash_table_iter_init(&gIter, lrm_state->active_ops);
+        while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
+            /* Ignore recurring actions in the shutdown calculations */
+            if (pending->interval_ms == 0) {
+                counter++;
+            }
+        }
+    }
+
+    if (counter > 0) {
+        do_crm_log(log_level, "%d pending executor operation%s at %s",
+                   counter, pcmk__plural_s(counter), when);
+
+        if ((cur_state == S_TERMINATE)
+            || !pcmk_is_set(controld_globals.fsa_input_register,
+                            R_SENT_RSC_STOP)) {
+            g_hash_table_iter_init(&gIter, lrm_state->active_ops);
+            while (g_hash_table_iter_next(&gIter, (gpointer*)&key, (gpointer*)&pending)) {
+                do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
+            }
+
+        } else {
+            rc = FALSE;
+        }
+        return rc;
+    }
+
+    if (lrm_state->resource_history == NULL) {
+        return rc;
+    }
+
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+        /* At this point we're not waiting, we're just shutting down */
+        when = "shutdown";
+    }
+
+    counter = 0;
+    g_hash_table_iter_init(&gIter, lrm_state->resource_history);
+    while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
+        if (is_rsc_active(lrm_state, entry->id) == FALSE) {
+            continue;
+        }
+
+        counter++;
+        if (log_level == LOG_ERR) {
+            crm_info("Found %s active at %s", entry->id, when);
+        } else {
+            crm_trace("Found %s active at %s", entry->id, when);
+        }
+        if (lrm_state->active_ops != NULL) {
+            GHashTableIter hIter;
+
+            g_hash_table_iter_init(&hIter, lrm_state->active_ops);
+            while (g_hash_table_iter_next(&hIter, (gpointer*)&key, (gpointer*)&pending)) {
+                if (pcmk__str_eq(entry->id, pending->rsc_id, pcmk__str_none)) {
+                    crm_notice("%sction %s (%s) incomplete at %s",
+                               pending->interval_ms == 0 ? "A" : "Recurring a",
+                               key, pending->op_key, when);
+                }
+            }
+        }
+    }
+
+    if (counter) {
+        crm_err("%d resource%s active at %s",
+                counter, (counter == 1)? " was" : "s were", when);
+    }
+
+    return rc;
+}
+
+static gboolean
+is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
+{
+    rsc_history_t *entry = NULL;
+
+    entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
+    if (entry == NULL || entry->last == NULL) {
+        return FALSE;
+    }
+
+    crm_trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type,
+              entry->last->interval_ms, entry->last->rc);
+    if (entry->last->rc == PCMK_OCF_OK && pcmk__str_eq(entry->last->op_type, CRMD_ACTION_STOP, pcmk__str_casei)) {
+        return FALSE;
+
+    } else if (entry->last->rc == PCMK_OCF_OK
+               && pcmk__str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE, pcmk__str_casei)) {
+        // A stricter check is too complex ... leave that to the scheduler
+        return FALSE;
+
+    } else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
+        return FALSE;
+
+    } else if ((entry->last->interval_ms == 0)
+               && (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) {
+        /* Badly configured resources can't be reliably stopped */
+        return FALSE;
+    }
+
+    return TRUE;
+}
+
+static gboolean
+build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
+{
+    GHashTableIter iter;
+    rsc_history_t *entry = NULL;
+
+    g_hash_table_iter_init(&iter, lrm_state->resource_history);
+    while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
+
+        GList *gIter = NULL;
+        xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);
+
+        crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id);
+        crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type);
+        crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.standard);
+        crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);
+
+        if (entry->last && entry->last->params) {
+            const char *container = g_hash_table_lookup(entry->last->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
+            if (container) {
+                crm_trace("Resource %s is a part of container resource %s", entry->id, container);
+                crm_xml_add(xml_rsc, XML_RSC_ATTR_CONTAINER, container);
+            }
+        }
+        controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->failed,
+                                          lrm_state->node_name);
+        controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->last,
+                                          lrm_state->node_name);
+        for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
+            controld_add_resource_history_xml(xml_rsc, &(entry->rsc), gIter->data,
+                                              lrm_state->node_name);
+        }
+    }
+
+    return FALSE;
+}
+
+xmlNode *
+controld_query_executor_state(void)
+{
+    xmlNode *xml_state = NULL;
+    xmlNode *xml_data = NULL;
+    xmlNode *rsc_list = NULL;
+    crm_node_t *peer = NULL;
+    lrm_state_t *lrm_state = lrm_state_find(controld_globals.our_nodename);
+
+    if (!lrm_state) {
+        crm_err("Could not find executor state for node %s",
+                controld_globals.our_nodename);
+        return NULL;
+    }
+
+    peer = crm_get_peer_full(0, lrm_state->node_name, CRM_GET_PEER_ANY);
+    CRM_CHECK(peer != NULL, return NULL);
+
+    xml_state = create_node_state_update(peer,
+                                         node_update_cluster|node_update_peer,
+                                         NULL, __func__);
+    if (xml_state == NULL) {
+        return NULL;
+    }
+
+    xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
+    crm_xml_add(xml_data, XML_ATTR_ID, peer->uuid);
+    rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);
+
+    /* Build a list of active (not always running) resources */
+    build_active_RAs(lrm_state, rsc_list);
+
+    crm_log_xml_trace(xml_state, "Current executor state");
+
+    return xml_state;
+}
+
+/*!
+ * \internal
+ * \brief Map standard Pacemaker return code to operation status and OCF code
+ *
+ * \param[out] event  Executor event whose status and return code should be set
+ * \param[in]  rc     Standard Pacemaker return code
+ */
+void
+controld_rc2event(lrmd_event_data_t *event, int rc)
+{
+    /* This is called for cleanup requests from controller peers/clients, not
+     * for resource actions, so no exit reason is needed.
+     */
+    switch (rc) {
+        case pcmk_rc_ok:
+            lrmd__set_result(event, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
+            break;
+        case EACCES:
+            lrmd__set_result(event, PCMK_OCF_INSUFFICIENT_PRIV,
+                             PCMK_EXEC_ERROR, NULL);
+            break;
+        default:
+            lrmd__set_result(event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
+                             NULL);
+            break;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Trigger a new transition after CIB status was deleted
+ *
+ * If a CIB status delete was not expected (as part of the transition graph),
+ * trigger a new transition by updating the (arbitrary) "last-lrm-refresh"
+ * cluster property.
+ *
+ * \param[in] from_sys  IPC name that requested the delete
+ * \param[in] rsc_id    Resource whose status was deleted (for logging only)
+ */
+void
+controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id)
+{
+    if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
+        char *now_s = crm_strdup_printf("%lld", (long long) time(NULL));
+
+        crm_debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id);
+        cib__update_node_attr(controld_globals.logger_out,
+                              controld_globals.cib_conn, cib_none,
+                              XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
+                              "last-lrm-refresh", now_s, NULL, NULL);
+        free(now_s);
+    }
+}
+
+static void
+notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
+{
+    lrmd_event_data_t *op = NULL;
+    const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
+    const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
+
+    crm_info("Notifying %s on %s that %s was%s deleted",
+             from_sys, (from_host? from_host : "localhost"), rsc_id,
+             ((rc == pcmk_ok)? "" : " not"));
+    op = construct_op(lrm_state, input->xml, rsc_id, CRMD_ACTION_DELETE);
+    controld_rc2event(op, pcmk_legacy2rc(rc));
+    controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
+    lrmd_free_event(op);
+    controld_trigger_delete_refresh(from_sys, rsc_id);
+}
+
+static gboolean
+lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
+{
+    struct delete_event_s *event = user_data;
+    struct pending_deletion_op_s *op = value;
+
+    if (pcmk__str_eq(event->rsc, op->rsc, pcmk__str_none)) {
+        notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
+        return TRUE;
+    }
+    return FALSE;
+}
+
+static gboolean
+lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
+{
+    const char *rsc = user_data;
+    active_op_t *pending = value;
+
+    if (pcmk__str_eq(rsc, pending->rsc_id, pcmk__str_none)) {
+        crm_info("Removing op %s:%d for deleted resource %s",
+                 pending->op_key, pending->call_id, rsc);
+        return TRUE;
+    }
+    return FALSE;
+}
+
+static void
+delete_rsc_entry(lrm_state_t *lrm_state, ha_msg_input_t *input,
+                 const char *rsc_id, GHashTableIter *rsc_iter, int rc,
+                 const char *user_name, bool from_cib)
+{
+    struct delete_event_s event;
+
+    CRM_CHECK(rsc_id != NULL, return);
+
+    if (rc == pcmk_ok) {
+        char *rsc_id_copy = strdup(rsc_id);
+
+        if (rsc_iter) {
+            g_hash_table_iter_remove(rsc_iter);
+        } else {
+            g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
+        }
+
+        if (from_cib) {
+            controld_delete_resource_history(rsc_id_copy, lrm_state->node_name,
+                                             user_name, crmd_cib_smart_opt());
+        }
+        g_hash_table_foreach_remove(lrm_state->active_ops,
+                                    lrm_remove_deleted_op, rsc_id_copy);
+        free(rsc_id_copy);
+    }
+
+    if (input) {
+        notify_deleted(lrm_state, input, rsc_id, rc);
+    }
+
+    event.rc = rc;
+    event.rsc = rsc_id;
+    event.lrm_state = lrm_state;
+    g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
+}
+
+static inline gboolean
+last_failed_matches_op(rsc_history_t *entry, const char *op, guint interval_ms)
+{
+    if (entry == NULL) {
+        return FALSE;
+    }
+    if (op == NULL) {
+        return TRUE;
+    }
+    return (pcmk__str_eq(op, entry->failed->op_type, pcmk__str_casei)
+            && (interval_ms == entry->failed->interval_ms));
+}
+
+/*!
+ * \internal
+ * \brief Clear a resource's last failure
+ *
+ * Erase a resource's last failure on a particular node from both the
+ * LRM resource history in the CIB, and the resource history remembered
+ * for the LRM state.
+ *
+ * \param[in] rsc_id      Resource name
+ * \param[in] node_name   Node name
+ * \param[in] operation   If specified, only clear if matching this operation
+ * \param[in] interval_ms If operation is specified, it has this interval
+ */
+void
+lrm_clear_last_failure(const char *rsc_id, const char *node_name,
+                       const char *operation, guint interval_ms)
+{
+    lrm_state_t *lrm_state = lrm_state_find(node_name);
+
+    if (lrm_state == NULL) {
+        return;
+    }
+    if (lrm_state->resource_history != NULL) {
+        rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history,
+                                                   rsc_id);
+
+        if (last_failed_matches_op(entry, operation, interval_ms)) {
+            lrmd_free_event(entry->failed);
+            entry->failed = NULL;
+        }
+    }
+}
+
+/* Returns: gboolean - cancellation is in progress */
+static gboolean
+cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, gboolean remove)
+{
+    int rc = pcmk_ok;
+    char *local_key = NULL;
+    active_op_t *pending = NULL;
+
+    CRM_CHECK(op != 0, return FALSE);
+    CRM_CHECK(rsc_id != NULL, return FALSE);
+    if (key == NULL) {
+        local_key = make_stop_id(rsc_id, op);
+        key = local_key;
+    }
+    pending = g_hash_table_lookup(lrm_state->active_ops, key);
+
+    if (pending) {
+        if (remove && !pcmk_is_set(pending->flags, active_op_remove)) {
+            controld_set_active_op_flags(pending, active_op_remove);
+            crm_debug("Scheduling %s for removal", key);
+        }
+
+        if (pcmk_is_set(pending->flags, active_op_cancelled)) {
+            crm_debug("Operation %s already cancelled", key);
+            free(local_key);
+            return FALSE;
+        }
+        controld_set_active_op_flags(pending, active_op_cancelled);
+
+    } else {
+        crm_info("No pending op found for %s", key);
+        free(local_key);
+        return FALSE;
+    }
+
+    crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
+    rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type,
+                          pending->interval_ms);
+    if (rc == pcmk_ok) {
+        crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
+        free(local_key);
+        return TRUE;
+    }
+
+    crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
+    /* The caller needs to make sure the entry is
+     * removed from the active operations list
+     *
+     * Usually by returning TRUE inside the worker function
+     * supplied to g_hash_table_foreach_remove()
+     *
+     * Not removing the entry from active operations will block
+     * the node from shutting down
+     */
+    free(local_key);
+    return FALSE;
+}
+
+struct cancel_data {
+    gboolean done;
+    gboolean remove;
+    const char *key;
+    lrmd_rsc_info_t *rsc;
+    lrm_state_t *lrm_state;
+};
+
+static gboolean
+cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
+{
+    gboolean remove = FALSE;
+    struct cancel_data *data = user_data;
+    active_op_t *op = value;
+
+    if (pcmk__str_eq(op->op_key, data->key, pcmk__str_none)) {
+        data->done = TRUE;
+        remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
+    }
+    return remove;
+}
+
+static gboolean
+cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
+{
+    guint removed = 0;
+    struct cancel_data data;
+
+    CRM_CHECK(rsc != NULL, return FALSE);
+    CRM_CHECK(key != NULL, return FALSE);
+
+    data.key = key;
+    data.rsc = rsc;
+    data.done = FALSE;
+    data.remove = remove;
+    data.lrm_state = lrm_state;
+
+    removed = g_hash_table_foreach_remove(lrm_state->active_ops,
+                                          cancel_action_by_key, &data);
+    crm_trace("Removed %u op cache entries, new size: %u",
+              removed, g_hash_table_size(lrm_state->active_ops));
+    return data.done;
+}
+
+/*!
+ * \internal
+ * \brief Retrieve resource information from LRM
+ *
+ * \param[in,out]  lrm_state  Executor connection state to use
+ * \param[in]      rsc_xml    XML containing resource configuration
+ * \param[in]      do_create  If true, register resource if not already
+ * \param[out]     rsc_info   Where to store information obtained from executor
+ *
+ * \retval pcmk_ok   Success (and rsc_info holds newly allocated result)
+ * \retval -EINVAL   Required information is missing from arguments
+ * \retval -ENOTCONN No active connection to LRM
+ * \retval -ENODEV   Resource not found
+ * \retval -errno    Error communicating with executor when registering resource
+ *
+ * \note Caller is responsible for freeing result on success.
+ */
+static int
+get_lrm_resource(lrm_state_t *lrm_state, const xmlNode *rsc_xml,
+                 gboolean do_create, lrmd_rsc_info_t **rsc_info)
+{
+    const char *id = ID(rsc_xml);
+
+    CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL);
+    CRM_CHECK(id, return -EINVAL);
+
+    if (lrm_state_is_connected(lrm_state) == FALSE) {
+        return -ENOTCONN;
+    }
+
+    crm_trace("Retrieving resource information for %s from the executor", id);
+    *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
+
+    // If resource isn't known by ID, try clone name, if provided
+    if (!*rsc_info) {
+        const char *long_id = crm_element_value(rsc_xml, XML_ATTR_ID_LONG);
+
+        if (long_id) {
+            *rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0);
+        }
+    }
+
+    if ((*rsc_info == NULL) && do_create) {
+        const char *class = crm_element_value(rsc_xml, XML_AGENT_ATTR_CLASS);
+        const char *provider = crm_element_value(rsc_xml, XML_AGENT_ATTR_PROVIDER);
+        const char *type = crm_element_value(rsc_xml, XML_ATTR_TYPE);
+        int rc;
+
+        crm_trace("Registering resource %s with the executor", id);
+        rc = lrm_state_register_rsc(lrm_state, id, class, provider, type,
+                                    lrmd_opt_drop_recurring);
+        if (rc != pcmk_ok) {
+            fsa_data_t *msg_data = NULL;
+
+            crm_err("Could not register resource %s with the executor on %s: %s "
+                    CRM_XS " rc=%d",
+                    id, lrm_state->node_name, pcmk_strerror(rc), rc);
+
+            /* Register this as an internal error if this involves the local
+             * executor. Otherwise, we're likely dealing with an unresponsive
+             * remote node, which is not an FSA failure.
+             */
+            if (lrm_state_is_local(lrm_state) == TRUE) {
+                register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
+            }
+            return rc;
+        }
+
+        *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
+    }
+    return *rsc_info? pcmk_ok : -ENODEV;
+}
+
+static void
+delete_resource(lrm_state_t *lrm_state, const char *id, lrmd_rsc_info_t *rsc,
+                GHashTableIter *iter, const char *sys, const char *user,
+                ha_msg_input_t *request, bool unregister, bool from_cib)
+{
+    int rc = pcmk_ok;
+
+    crm_info("Removing resource %s from executor for %s%s%s",
+             id, sys, (user? " as " : ""), (user? user : ""));
+
+    if (rsc && unregister) {
+        rc = lrm_state_unregister_rsc(lrm_state, id, 0);
+    }
+
+    if (rc == pcmk_ok) {
+        crm_trace("Resource %s deleted from executor", id);
+    } else if (rc == -EINPROGRESS) {
+        crm_info("Deletion of resource '%s' from executor is pending", id);
+        if (request) {
+            struct pending_deletion_op_s *op = NULL;
+            char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);
+
+            op = calloc(1, sizeof(struct pending_deletion_op_s));
+            op->rsc = strdup(rsc->id);
+            op->input = copy_ha_msg_input(request);
+            g_hash_table_insert(lrm_state->deletion_ops, ref, op);
+        }
+        return;
+    } else {
+        crm_warn("Could not delete '%s' from executor for %s%s%s: %s "
+                 CRM_XS " rc=%d", id, sys, (user? " as " : ""),
+                 (user? user : ""), pcmk_strerror(rc), rc);
+    }
+
+    delete_rsc_entry(lrm_state, request, id, iter, rc, user, from_cib);
+}
+
+static int
+get_fake_call_id(lrm_state_t *lrm_state, const char *rsc_id)
+{
+    int call_id = 999999999;
+    rsc_history_t *entry = NULL;
+
+    if(lrm_state) {
+        entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
+    }
+
+    /* Make sure the call id is greater than the last successful operation,
+     * otherwise the failure will not result in a possible recovery of the resource
+     * as it could appear the failure occurred before the successful start */
+    if (entry) {
+        call_id = entry->last_callid + 1;
+    }
+
+    if (call_id < 0) {
+        call_id = 1;
+    }
+    return call_id;
+}
+
+static void
+fake_op_status(lrm_state_t *lrm_state, lrmd_event_data_t *op, int op_status,
+               enum ocf_exitcode op_exitcode, const char *exit_reason)
+{
+    op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
+    op->t_run = time(NULL);
+    op->t_rcchange = op->t_run;
+    lrmd__set_result(op, op_exitcode, op_status, exit_reason);
+}
+
+static void
+force_reprobe(lrm_state_t *lrm_state, const char *from_sys,
+              const char *from_host, const char *user_name,
+              gboolean is_remote_node, bool reprobe_all_nodes)
+{
+    GHashTableIter gIter;
+    rsc_history_t *entry = NULL;
+
+    crm_info("Clearing resource history on node %s", lrm_state->node_name);
+    g_hash_table_iter_init(&gIter, lrm_state->resource_history);
+    while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
+        /* only unregister the resource during a reprobe if it is not a remote connection
+         * resource. otherwise unregistering the connection will terminate remote-node
+         * membership */
+        bool unregister = true;
+
+        if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
+            unregister = false;
+
+            if (reprobe_all_nodes) {
+                lrm_state_t *remote_lrm_state = lrm_state_find(entry->id);
+
+                if (remote_lrm_state != NULL) {
+                    /* If reprobing all nodes, be sure to reprobe the remote
+                     * node before clearing its connection resource
+                     */
+                    force_reprobe(remote_lrm_state, from_sys, from_host,
+                                  user_name, TRUE, reprobe_all_nodes);
+                }
+            }
+        }
+
+        /* Don't delete from the CIB, since we'll delete the whole node's LRM
+         * state from the CIB soon
+         */
+        delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys,
+                        user_name, NULL, unregister, false);
+    }
+
+    /* Now delete the copy in the CIB */
+    controld_delete_node_state(lrm_state->node_name, controld_section_lrm,
+                               cib_scope_local);
+
+    // @COMPAT DCs < 1.1.14 need this deleted (in case it was explicitly false)
+    update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
+}
+
+/*!
+ * \internal
+ * \brief Fail a requested action without actually executing it
+ *
+ * For an action that can't be executed, process it similarly to an actual
+ * execution result, with specified error status (except for notify actions,
+ * which will always be treated as successful).
+ *
+ * \param[in,out] lrm_state    Executor connection that action is for
+ * \param[in]     action       Action XML from request
+ * \param[in]     rc           Desired return code to use
+ * \param[in]     op_status    Desired operation status to use
+ * \param[in]     exit_reason  Human-friendly detail, if error
+ */
+static void
+synthesize_lrmd_failure(lrm_state_t *lrm_state, const xmlNode *action,
+                        int op_status, enum ocf_exitcode rc,
+                        const char *exit_reason)
+{
+    lrmd_event_data_t *op = NULL;
+    const char *operation = crm_element_value(action, XML_LRM_ATTR_TASK);
+    const char *target_node = crm_element_value(action, XML_LRM_ATTR_TARGET);
+    xmlNode *xml_rsc = find_xml_node(action, XML_CIB_TAG_RESOURCE, TRUE);
+
+    if ((xml_rsc == NULL) || (ID(xml_rsc) == NULL)) {
+        /* @TODO Should we do something else, like direct ack? */
+        crm_info("Can't fake %s failure (%d) on %s without resource configuration",
+                 crm_element_value(action, XML_LRM_ATTR_TASK_KEY), rc,
+                 target_node);
+        return;
+
+    } else if(operation == NULL) {
+        /* This probably came from crm_resource -C, nothing to do */
+        crm_info("Can't fake %s failure (%d) on %s without operation",
+                 ID(xml_rsc), rc, target_node);
+        return;
+    }
+
+    op = construct_op(lrm_state, action, ID(xml_rsc), operation);
+
+    if (pcmk__str_eq(operation, RSC_NOTIFY, pcmk__str_casei)) { // Notifications can't fail
+        fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_OK, NULL);
+    } else {
+        fake_op_status(lrm_state, op, op_status, rc, exit_reason);
+    }
+
+    crm_info("Faking " PCMK__OP_FMT " result (%d) on %s",
+             op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node);
+
+    // Process the result as if it came from the LRM
+    process_lrm_event(lrm_state, op, NULL, action);
+    lrmd_free_event(op);
+}
+
+/*!
+ * \internal
+ * \brief Get target of an LRM operation (replacing \p NULL with local node
+ *        name)
+ *
+ * \param[in] xml  LRM operation data XML
+ *
+ * \return LRM operation target node name (local node or Pacemaker Remote node)
+ */
+static const char *
+lrm_op_target(const xmlNode *xml)
+{
+    const char *target = NULL;
+
+    if (xml) {
+        target = crm_element_value(xml, XML_LRM_ATTR_TARGET);
+    }
+    if (target == NULL) {
+        target = controld_globals.our_nodename;
+    }
+    return target;
+}
+
+static void
+fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name,
+                  const char *from_host, const char *from_sys)
+{
+    lrmd_event_data_t *op = NULL;
+    lrmd_rsc_info_t *rsc = NULL;
+    xmlNode *xml_rsc = find_xml_node(xml, XML_CIB_TAG_RESOURCE, TRUE);
+
+    CRM_CHECK(xml_rsc != NULL, return);
+
+    /* The executor simply executes operations and reports the results, without
+     * any concept of success or failure, so to fail a resource, we must fake
+     * what a failure looks like.
+     *
+     * To do this, we create a fake executor operation event for the resource,
+     * and pass that event to the executor client callback so it will be
+     * processed as if it came from the executor.
+     */
+    op = construct_op(lrm_state, xml, ID(xml_rsc), "asyncmon");
+
+    free((char*) op->user_data);
+    op->user_data = NULL;
+    op->interval_ms = 0;
+
+    if (user_name && !pcmk__is_privileged(user_name)) {
+        crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
+        fake_op_status(lrm_state, op, PCMK_EXEC_ERROR,
+                       PCMK_OCF_INSUFFICIENT_PRIV,
+                       "Unprivileged user cannot fail resources");
+        controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
+        lrmd_free_event(op);
+        return;
+    }
+
+
+    if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
+        crm_info("Failing resource %s...", rsc->id);
+        fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_UNKNOWN_ERROR,
+                       "Simulated failure");
+        process_lrm_event(lrm_state, op, NULL, xml);
+        op->rc = PCMK_OCF_OK; // The request to fail the resource succeeded
+        lrmd_free_rsc_info(rsc);
+
+    } else {
+        crm_info("Cannot find/create resource in order to fail it...");
+        crm_log_xml_warn(xml, "bad input");
+        fake_op_status(lrm_state, op, PCMK_EXEC_ERROR, PCMK_OCF_UNKNOWN_ERROR,
+                       "Cannot fail unknown resource");
+    }
+
+    controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
+    lrmd_free_event(op);
+}
+
+static void
+handle_reprobe_op(lrm_state_t *lrm_state, const char *from_sys,
+                  const char *from_host, const char *user_name,
+                  gboolean is_remote_node, bool reprobe_all_nodes)
+{
+    crm_notice("Forcing the status of all resources to be redetected");
+    force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node,
+                  reprobe_all_nodes);
+
+    if (!pcmk__strcase_any_of(from_sys, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) {
+
+        xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, NULL, from_host,
+                                        from_sys, CRM_SYSTEM_LRMD,
+                                        controld_globals.our_uuid);
+
+        crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);
+
+        if (relay_message(reply, TRUE) == FALSE) {
+            crm_log_xml_err(reply, "Unable to route reply");
+        }
+        free_xml(reply);
+    }
+}
+
+static bool do_lrm_cancel(ha_msg_input_t *input, lrm_state_t *lrm_state,
+              lrmd_rsc_info_t *rsc, const char *from_host, const char *from_sys)
+{
+    char *op_key = NULL;
+    char *meta_key = NULL;
+    int call = 0;
+    const char *call_id = NULL;
+    const char *op_task = NULL;
+    guint interval_ms = 0;
+    gboolean in_progress = FALSE;
+    xmlNode *params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);
+
+    CRM_CHECK(params != NULL, return FALSE);
+
+    meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
+    op_task = crm_element_value(params, meta_key);
+    free(meta_key);
+    CRM_CHECK(op_task != NULL, return FALSE);
+
+    meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL_MS);
+    if (crm_element_value_ms(params, meta_key, &interval_ms) != pcmk_ok) {
+        free(meta_key);
+        return FALSE;
+    }
+    free(meta_key);
+
+    op_key = pcmk__op_key(rsc->id, op_task, interval_ms);
+
+    meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
+    call_id = crm_element_value(params, meta_key);
+    free(meta_key);
+
+    crm_debug("Scheduler requested op %s (call=%s) be cancelled",
+              op_key, (call_id? call_id : "NA"));
+    pcmk__scan_min_int(call_id, &call, 0);
+    if (call == 0) {
+        // Normal case when the scheduler cancels a recurring op
+        in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);
+
+    } else {
+        // Normal case when the scheduler cancels an orphan op
+        in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
+    }
+
+    // Acknowledge cancellation operation if for a remote connection resource
+    if (!in_progress || is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
+        char *op_id = make_stop_id(rsc->id, call);
+
+        if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) {
+            crm_info("Nothing known about operation %d for %s", call, op_key);
+        }
+        controld_delete_action_history_by_key(rsc->id, lrm_state->node_name,
+                                              op_key, call);
+        send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
+                         from_host, from_sys);
+
+        /* needed at least for cancellation of a remote operation */
+        if (lrm_state->active_ops != NULL) {
+            g_hash_table_remove(lrm_state->active_ops, op_id);
+        }
+        free(op_id);
+
+    } else {
+        /* No ack is needed since abcdaa8, but peers with older versions
+         * in a rolling upgrade need one. We didn't bump the feature set
+         * at that commit, so we can only compare against the previous
+         * CRM version (3.0.8). If any peers have feature set 3.0.9 but
+         * not abcdaa8, they will time out waiting for the ack (no
+         * released versions of Pacemaker are affected).
+         */
+        const char *peer_version = crm_element_value(params, XML_ATTR_CRM_VERSION);
+
+        if (compare_version(peer_version, "3.0.8") <= 0) {
+            crm_info("Sending compatibility ack for %s cancellation to %s (CRM version %s)",
+                     op_key, from_host, peer_version);
+            send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
+                             from_host, from_sys);
+        }
+    }
+
+    free(op_key);
+    return TRUE;
+}
+
+static void
+do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state,
+              lrmd_rsc_info_t *rsc, const char *from_sys, const char *from_host,
+              bool crm_rsc_delete, const char *user_name)
+{
+    bool unregister = true;
+    int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name,
+                                                  user_name,
+                                                  cib_dryrun|cib_sync_call);
+
+    if (cib_rc != pcmk_rc_ok) {
+        lrmd_event_data_t *op = NULL;
+
+        op = construct_op(lrm_state, input->xml, rsc->id, CRMD_ACTION_DELETE);
+
+        /* These are resource clean-ups, not actions, so no exit reason is
+         * needed.
+         */
+        lrmd__set_result(op, pcmk_rc2ocf(cib_rc), PCMK_EXEC_ERROR, NULL);
+        controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id);
+        lrmd_free_event(op);
+        return;
+    }
+
+    if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
+        unregister = false;
+    }
+
+    delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys,
+                    user_name, input, unregister, true);
+}
+
+// User data for asynchronous metadata execution
+struct metadata_cb_data {
+    lrmd_rsc_info_t *rsc;   // Copy of resource information
+    xmlNode *input_xml;     // Copy of FSA input XML
+};
+
+static struct metadata_cb_data *
+new_metadata_cb_data(lrmd_rsc_info_t *rsc, xmlNode *input_xml)
+{
+    struct metadata_cb_data *data = NULL;
+
+    data = calloc(1, sizeof(struct metadata_cb_data));
+    CRM_ASSERT(data != NULL);
+    data->input_xml = copy_xml(input_xml);
+    data->rsc = lrmd_copy_rsc_info(rsc);
+    return data;
+}
+
+static void
+free_metadata_cb_data(struct metadata_cb_data *data)
+{
+    lrmd_free_rsc_info(data->rsc);
+    free_xml(data->input_xml);
+    free(data);
+}
+
+/*!
+ * \internal
+ * \brief Execute an action after metadata has been retrieved
+ *
+ * \param[in] pid        Ignored
+ * \param[in] result     Result of metadata action
+ * \param[in] user_data  Metadata callback data
+ */
+static void
+metadata_complete(int pid, const pcmk__action_result_t *result, void *user_data)
+{
+    struct metadata_cb_data *data = (struct metadata_cb_data *) user_data;
+
+    struct ra_metadata_s *md = NULL;
+    lrm_state_t *lrm_state = lrm_state_find(lrm_op_target(data->input_xml));
+
+    if ((lrm_state != NULL) && pcmk__result_ok(result)) {
+        md = controld_cache_metadata(lrm_state->metadata_cache, data->rsc,
+                                     result->action_stdout);
+    }
+    do_lrm_rsc_op(lrm_state, data->rsc, data->input_xml, md);
+    free_metadata_cb_data(data);
+}
+
+/*	 A_LRM_INVOKE	*/
+void
+do_lrm_invoke(long long action,
+              enum crmd_fsa_cause cause,
+              enum crmd_fsa_state cur_state,
+              enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    lrm_state_t *lrm_state = NULL;
+    const char *crm_op = NULL;
+    const char *from_sys = NULL;
+    const char *from_host = NULL;
+    const char *operation = NULL;
+    ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
+    const char *user_name = NULL;
+    const char *target_node = lrm_op_target(input->xml);
+    gboolean is_remote_node = FALSE;
+    bool crm_rsc_delete = FALSE;
+
+    // Message routed to the local node is targeting a specific, non-local node
+    is_remote_node = !pcmk__str_eq(target_node, controld_globals.our_nodename,
+                                   pcmk__str_casei);
+
+    lrm_state = lrm_state_find(target_node);
+    if ((lrm_state == NULL) && is_remote_node) {
+        crm_err("Failing action because local node has never had connection to remote node %s",
+                target_node);
+        synthesize_lrmd_failure(NULL, input->xml, PCMK_EXEC_NOT_CONNECTED,
+                                PCMK_OCF_UNKNOWN_ERROR,
+                                "Local node has no connection to remote");
+        return;
+    }
+    CRM_ASSERT(lrm_state != NULL);
+
+    user_name = pcmk__update_acl_user(input->msg, F_CRM_USER, NULL);
+    crm_op = crm_element_value(input->msg, F_CRM_TASK);
+    from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
+    if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
+        from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
+    }
+
+    if (pcmk__str_eq(crm_op, CRM_OP_LRM_DELETE, pcmk__str_none)) {
+        if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
+            crm_rsc_delete = TRUE; // from crm_resource
+        }
+        operation = CRMD_ACTION_DELETE;
+
+    } else if (input->xml != NULL) {
+        operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
+    }
+
+    CRM_CHECK(!pcmk__str_empty(crm_op) || !pcmk__str_empty(operation), return);
+
+    crm_trace("'%s' execution request from %s as %s user",
+              pcmk__s(crm_op, operation),
+              pcmk__s(from_sys, "unknown subsystem"),
+              pcmk__s(user_name, "current"));
+
+    if (pcmk__str_eq(crm_op, CRM_OP_LRM_FAIL, pcmk__str_none)) {
+        fail_lrm_resource(input->xml, lrm_state, user_name, from_host,
+                          from_sys);
+
+    } else if (pcmk__str_eq(crm_op, CRM_OP_LRM_REFRESH, pcmk__str_none)) {
+        /* @COMPAT This can only be sent by crm_resource --refresh on a
+         * Pacemaker Remote node running Pacemaker 1.1.9, which is extremely
+         * unlikely. It previously would cause the controller to re-write its
+         * resource history to the CIB. Just ignore it.
+         */
+        crm_notice("Ignoring refresh request from Pacemaker Remote 1.1.9 node");
+
+    // @COMPAT DCs <1.1.14 in a rolling upgrade might schedule this op
+    } else if (pcmk__str_eq(operation, CRM_OP_PROBED, pcmk__str_none)) {
+        update_attrd(lrm_state->node_name, CRM_OP_PROBED, XML_BOOLEAN_TRUE,
+                     user_name, is_remote_node);
+
+    } else if (pcmk__str_eq(crm_op, CRM_OP_REPROBE, pcmk__str_none)
+               || pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
+        const char *raw_target = NULL;
+
+        if (input->xml != NULL) {
+            // For CRM_OP_REPROBE, a NULL target means we're targeting all nodes
+            raw_target = crm_element_value(input->xml, XML_LRM_ATTR_TARGET);
+        }
+        handle_reprobe_op(lrm_state, from_sys, from_host, user_name,
+                          is_remote_node, (raw_target == NULL));
+
+    } else if (operation != NULL) {
+        lrmd_rsc_info_t *rsc = NULL;
+        xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);
+        gboolean create_rsc = !pcmk__str_eq(operation, CRMD_ACTION_DELETE,
+                                            pcmk__str_none);
+        int rc;
+
+        // We can't return anything meaningful without a resource ID
+        CRM_CHECK(xml_rsc && ID(xml_rsc), return);
+
+        rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc);
+        if (rc == -ENOTCONN) {
+            synthesize_lrmd_failure(lrm_state, input->xml,
+                                    PCMK_EXEC_NOT_CONNECTED,
+                                    PCMK_OCF_UNKNOWN_ERROR,
+                                    "Not connected to remote executor");
+            return;
+
+        } else if ((rc < 0) && !create_rsc) {
+            /* Delete of malformed or nonexistent resource
+             * (deleting something that does not exist is a success)
+             */
+            crm_notice("Not registering resource '%s' for a %s event "
+                       CRM_XS " get-rc=%d (%s) transition-key=%s",
+                       ID(xml_rsc), operation,
+                       rc, pcmk_strerror(rc), ID(input->xml));
+            delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok,
+                             user_name, true);
+            return;
+
+        } else if (rc == -EINVAL) {
+            // Resource operation on malformed resource
+            crm_err("Invalid resource definition for %s", ID(xml_rsc));
+            crm_log_xml_warn(input->msg, "invalid resource");
+            synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
+                                    PCMK_OCF_NOT_CONFIGURED, // fatal error
+                                    "Invalid resource definition");
+            return;
+
+        } else if (rc < 0) {
+            // Error communicating with the executor
+            crm_err("Could not register resource '%s' with executor: %s "
+                    CRM_XS " rc=%d",
+                    ID(xml_rsc), pcmk_strerror(rc), rc);
+            crm_log_xml_warn(input->msg, "failed registration");
+            synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
+                                    PCMK_OCF_INVALID_PARAM, // hard error
+                                    "Could not register resource with executor");
+            return;
+        }
+
+        if (pcmk__str_eq(operation, CRMD_ACTION_CANCEL, pcmk__str_none)) {
+            if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) {
+                crm_log_xml_warn(input->xml, "Bad command");
+            }
+
+        } else if (pcmk__str_eq(operation, CRMD_ACTION_DELETE, pcmk__str_none)) {
+            do_lrm_delete(input, lrm_state, rsc, from_sys, from_host,
+                          crm_rsc_delete, user_name);
+
+        } else {
+            struct ra_metadata_s *md = NULL;
+
+            /* Getting metadata from cache is OK except for start actions --
+             * always refresh from the agent for those, in case the resource
+             * agent was updated.
+             *
+             * @TODO Only refresh metadata for starts if the agent actually
+             * changed (using something like inotify, or a hash or modification
+             * time of the agent executable).
+             */
+            if (strcmp(operation, CRMD_ACTION_START) != 0) {
+                md = controld_get_rsc_metadata(lrm_state, rsc,
+                                               controld_metadata_from_cache);
+            }
+
+            if ((md == NULL) && crm_op_needs_metadata(rsc->standard,
+                                                      operation)) {
+                /* Most likely, we'll need the agent metadata to record the
+                 * pending operation and the operation result. Get it now rather
+                 * than wait until then, so the metadata action doesn't eat into
+                 * the real action's timeout.
+                 *
+                 * @TODO Metadata is retrieved via direct execution of the
+                 * agent, which has a couple of related issues: the executor
+                 * should execute agents, not the controller; and metadata for
+                 * Pacemaker Remote nodes should be collected on those nodes,
+                 * not locally.
+                 */
+                struct metadata_cb_data *data = NULL;
+
+                data = new_metadata_cb_data(rsc, input->xml);
+                crm_info("Retrieving metadata for %s (%s%s%s:%s) asynchronously",
+                         rsc->id, rsc->standard,
+                         ((rsc->provider == NULL)? "" : ":"),
+                         ((rsc->provider == NULL)? "" : rsc->provider),
+                         rsc->type);
+                (void) lrmd__metadata_async(rsc, metadata_complete,
+                                            (void *) data);
+            } else {
+                do_lrm_rsc_op(lrm_state, rsc, input->xml, md);
+            }
+        }
+
+        lrmd_free_rsc_info(rsc);
+
+    } else {
+        crm_err("Invalid execution request: unknown command '%s' (bug?)",
+                crm_op);
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+    }
+}
+
+static lrmd_event_data_t *
+construct_op(const lrm_state_t *lrm_state, const xmlNode *rsc_op,
+             const char *rsc_id, const char *operation)
+{
+    lrmd_event_data_t *op = NULL;
+    const char *op_delay = NULL;
+    const char *op_timeout = NULL;
+    GHashTable *params = NULL;
+
+    xmlNode *primitive = NULL;
+    const char *class = NULL;
+
+    const char *transition = NULL;
+
+    CRM_ASSERT(rsc_id && operation);
+
+    op = lrmd_new_event(rsc_id, operation, 0);
+    op->type = lrmd_event_exec_complete;
+    op->timeout = 0;
+    op->start_delay = 0;
+    lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
+
+    if (rsc_op == NULL) {
+        CRM_LOG_ASSERT(pcmk__str_eq(CRMD_ACTION_STOP, operation, pcmk__str_casei));
+        op->user_data = NULL;
+        /* the stop_all_resources() case
+         * by definition there is no DC (or they'd be shutting
+         *   us down).
+         * So we should put our version here.
+         */
+        op->params = pcmk__strkey_table(free, free);
+
+        g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));
+
+        crm_trace("Constructed %s op for %s", operation, rsc_id);
+        return op;
+    }
+
+    params = xml2list(rsc_op);
+    g_hash_table_remove(params, CRM_META "_op_target_rc");
+
+    op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
+    pcmk__scan_min_int(op_delay, &op->start_delay, 0);
+
+    op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
+    pcmk__scan_min_int(op_timeout, &op->timeout, 0);
+
+    if (pcmk__guint_from_hash(params, CRM_META "_" XML_LRM_ATTR_INTERVAL_MS, 0,
+                              &(op->interval_ms)) != pcmk_rc_ok) {
+        op->interval_ms = 0;
+    }
+
+    /* Use pcmk_monitor_timeout instead of meta timeout for stonith
+       recurring monitor, if set */
+    primitive = find_xml_node(rsc_op, XML_CIB_TAG_RESOURCE, FALSE);
+    class = crm_element_value(primitive, XML_AGENT_ATTR_CLASS);
+
+    if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_fence_params)
+            && pcmk__str_eq(operation, CRMD_ACTION_STATUS, pcmk__str_casei)
+            && (op->interval_ms > 0)) {
+
+        op_timeout = g_hash_table_lookup(params, "pcmk_monitor_timeout");
+        if (op_timeout != NULL) {
+            op->timeout = crm_get_msec(op_timeout);
+        }
+    }
+
+    if (!pcmk__str_eq(operation, RSC_STOP, pcmk__str_casei)) {
+        op->params = params;
+
+    } else {
+        rsc_history_t *entry = NULL;
+
+        if (lrm_state) {
+            entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
+        }
+
+        /* If we do not have stop parameters cached, use
+         * whatever we are given */
+        if (!entry || !entry->stop_params) {
+            op->params = params;
+        } else {
+            /* Copy the cached parameter list so that we stop the resource
+             * with the old attributes, not the new ones */
+            op->params = pcmk__strkey_table(free, free);
+
+            g_hash_table_foreach(params, copy_meta_keys, op->params);
+            g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
+            g_hash_table_destroy(params);
+            params = NULL;
+        }
+    }
+
+    /* sanity */
+    if (op->timeout <= 0) {
+        op->timeout = op->interval_ms;
+    }
+    if (op->start_delay < 0) {
+        op->start_delay = 0;
+    }
+
+    transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
+    CRM_CHECK(transition != NULL, return op);
+
+    op->user_data = strdup(transition);
+
+    if (op->interval_ms != 0) {
+        if (pcmk__strcase_any_of(operation, CRMD_ACTION_START, CRMD_ACTION_STOP, NULL)) {
+            crm_err("Start and Stop actions cannot have an interval: %u",
+                    op->interval_ms);
+            op->interval_ms = 0;
+        }
+    }
+
+    crm_trace("Constructed %s op for %s: interval=%u",
+              operation, rsc_id, op->interval_ms);
+
+    return op;
+}
+
+/*!
+ * \internal
+ * \brief Send a (synthesized) event result
+ *
+ * Reply with a synthesized event result directly, as opposed to going through
+ * the executor.
+ *
+ * \param[in]     to_host  Host to send result to
+ * \param[in]     to_sys   IPC name to send result (NULL for transition engine)
+ * \param[in]     rsc      Type information about resource the result is for
+ * \param[in,out] op       Event with result to send
+ * \param[in]     rsc_id   ID of resource the result is for
+ */
+void
+controld_ack_event_directly(const char *to_host, const char *to_sys,
+                            const lrmd_rsc_info_t *rsc, lrmd_event_data_t *op,
+                            const char *rsc_id)
+{
+    xmlNode *reply = NULL;
+    xmlNode *update, *iter;
+    crm_node_t *peer = NULL;
+
+    CRM_CHECK(op != NULL, return);
+    if (op->rsc_id == NULL) {
+        CRM_ASSERT(rsc_id != NULL);
+        op->rsc_id = strdup(rsc_id);
+    }
+    if (to_sys == NULL) {
+        to_sys = CRM_SYSTEM_TENGINE;
+    }
+
+    peer = crm_get_peer(0, controld_globals.our_nodename);
+    update = create_node_state_update(peer, node_update_none, NULL,
+                                      __func__);
+
+    iter = create_xml_node(update, XML_CIB_TAG_LRM);
+    crm_xml_add(iter, XML_ATTR_ID, controld_globals.our_uuid);
+    iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
+    iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
+
+    crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
+
+    controld_add_resource_history_xml(iter, rsc, op,
+                                      controld_globals.our_nodename);
+    reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);
+
+    crm_log_xml_trace(update, "[direct ACK]");
+
+    crm_debug("ACK'ing resource op " PCMK__OP_FMT " from %s: %s",
+              op->rsc_id, op->op_type, op->interval_ms, op->user_data,
+              crm_element_value(reply, XML_ATTR_REFERENCE));
+
+    if (relay_message(reply, TRUE) == FALSE) {
+        crm_log_xml_err(reply, "Unable to route reply");
+    }
+
+    free_xml(update);
+    free_xml(reply);
+}
+
+gboolean
+verify_stopped(enum crmd_fsa_state cur_state, int log_level)
+{
+    gboolean res = TRUE;
+    GList *lrm_state_list = lrm_state_get_list();
+    GList *state_entry;
+
+    for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
+        lrm_state_t *lrm_state = state_entry->data;
+
+        if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
+            /* keep iterating through all even when false is returned */
+            res = FALSE;
+        }
+    }
+
+    controld_set_fsa_input_flags(R_SENT_RSC_STOP);
+    g_list_free(lrm_state_list); lrm_state_list = NULL;
+    return res;
+}
+
+struct stop_recurring_action_s {
+    lrmd_rsc_info_t *rsc;
+    lrm_state_t *lrm_state;
+};
+
+static gboolean
+stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
+{
+    gboolean remove = FALSE;
+    struct stop_recurring_action_s *event = user_data;
+    active_op_t *op = value;
+
+    if ((op->interval_ms != 0)
+        && pcmk__str_eq(op->rsc_id, event->rsc->id, pcmk__str_none)) {
+
+        crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key);
+        remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
+    }
+
+    return remove;
+}
+
+static gboolean
+stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
+{
+    gboolean remove = FALSE;
+    lrm_state_t *lrm_state = user_data;
+    active_op_t *op = value;
+
+    if (op->interval_ms != 0) {
+        crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id,
+                 (const char *) key);
+        remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
+    }
+
+    return remove;
+}
+
+/*!
+ * \internal
+ * \brief Check whether recurring actions should be cancelled before an action
+ *
+ * \param[in] rsc_id       Resource that action is for
+ * \param[in] action       Action being performed
+ * \param[in] interval_ms  Operation interval of \p action (in milliseconds)
+ *
+ * \return true if recurring actions should be cancelled, otherwise false
+ */
+static bool
+should_cancel_recurring(const char *rsc_id, const char *action, guint interval_ms)
+{
+    if (is_remote_lrmd_ra(NULL, NULL, rsc_id) && (interval_ms == 0)
+        && (strcmp(action, CRMD_ACTION_MIGRATE) == 0)) {
+        /* Don't stop monitoring a migrating Pacemaker Remote connection
+         * resource until the entire migration has completed. We must detect if
+         * the connection is unexpectedly severed, even during a migration.
+         */
+        return false;
+    }
+
+    // Cancel recurring actions before changing resource state
+    return (interval_ms == 0)
+            && !pcmk__str_any_of(action, CRMD_ACTION_STATUS, CRMD_ACTION_NOTIFY,
+                                 NULL);
+}
+
+/*!
+ * \internal
+ * \brief Check whether an action should not be performed at this time
+ *
+ * \param[in] operation  Action to be performed
+ *
+ * \return Readable description of why action should not be performed,
+ *         or NULL if it should be performed
+ */
+static const char *
+should_nack_action(const char *action)
+{
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)
+        && pcmk__str_eq(action, RSC_START, pcmk__str_none)) {
+
+        register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
+        return "Not attempting start due to shutdown in progress";
+    }
+
+    switch (controld_globals.fsa_state) {
+        case S_NOT_DC:
+        case S_POLICY_ENGINE:   // Recalculating
+        case S_TRANSITION_ENGINE:
+            break;
+        default:
+            if (!pcmk__str_eq(action, CRMD_ACTION_STOP, pcmk__str_none)) {
+                return "Controller cannot attempt actions at this time";
+            }
+            break;
+    }
+    return NULL;
+}
+
+static void
+do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, xmlNode *msg,
+              struct ra_metadata_s *md)
+{
+    int rc;
+    int call_id = 0;
+    char *op_id = NULL;
+    lrmd_event_data_t *op = NULL;
+    fsa_data_t *msg_data = NULL;
+    const char *transition = NULL;
+    const char *operation = NULL;
+    const char *nack_reason = NULL;
+
+    CRM_CHECK((rsc != NULL) && (msg != NULL), return);
+
+    operation = crm_element_value(msg, XML_LRM_ATTR_TASK);
+    CRM_CHECK(!pcmk__str_empty(operation), return);
+
+    transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
+    if (pcmk__str_empty(transition)) {
+        crm_log_xml_err(msg, "Missing transition number");
+    }
+
+    if (lrm_state == NULL) {
+        // This shouldn't be possible, but provide a failsafe just in case
+        crm_err("Cannot execute %s of %s: No executor connection "
+                CRM_XS " transition_key=%s",
+                operation, rsc->id, pcmk__s(transition, ""));
+        synthesize_lrmd_failure(NULL, msg, PCMK_EXEC_INVALID,
+                                PCMK_OCF_UNKNOWN_ERROR,
+                                "No executor connection");
+        return;
+    }
+
+    if (pcmk__str_any_of(operation, CRMD_ACTION_RELOAD,
+                         CRMD_ACTION_RELOAD_AGENT, NULL)) {
+        /* Pre-2.1.0 DCs will schedule reload actions only, and 2.1.0+ DCs
+         * will schedule reload-agent actions only. In either case, we need
+         * to map that to whatever the resource agent actually supports.
+         * Default to the OCF 1.1 name.
+         */
+        if ((md != NULL)
+            && pcmk_is_set(md->ra_flags, ra_supports_legacy_reload)) {
+            operation = CRMD_ACTION_RELOAD;
+        } else {
+            operation = CRMD_ACTION_RELOAD_AGENT;
+        }
+    }
+
+    op = construct_op(lrm_state, msg, rsc->id, operation);
+    CRM_CHECK(op != NULL, return);
+
+    if (should_cancel_recurring(rsc->id, operation, op->interval_ms)) {
+        guint removed = 0;
+        struct stop_recurring_action_s data;
+
+        data.rsc = rsc;
+        data.lrm_state = lrm_state;
+        removed = g_hash_table_foreach_remove(lrm_state->active_ops,
+                                              stop_recurring_action_by_rsc,
+                                              &data);
+
+        if (removed) {
+            crm_debug("Stopped %u recurring operation%s in preparation for "
+                      PCMK__OP_FMT, removed, pcmk__plural_s(removed),
+                      rsc->id, operation, op->interval_ms);
+        }
+    }
+
+    /* now do the op */
+    crm_notice("Requesting local execution of %s operation for %s on %s "
+               CRM_XS " transition_key=%s op_key=" PCMK__OP_FMT,
+               crm_action_str(op->op_type, op->interval_ms), rsc->id, lrm_state->node_name,
+               pcmk__s(transition, ""), rsc->id, operation, op->interval_ms);
+
+    nack_reason = should_nack_action(operation);
+    if (nack_reason != NULL) {
+        crm_notice("Discarding attempt to perform action %s on %s in state %s "
+                   "(shutdown=%s)", operation, rsc->id,
+                   fsa_state2string(controld_globals.fsa_state),
+                   pcmk__btoa(pcmk_is_set(controld_globals.fsa_input_register,
+                                          R_SHUTDOWN)));
+
+        lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_INVALID,
+                         nack_reason);
+        controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
+        lrmd_free_event(op);
+        free(op_id);
+        return;
+    }
+
+    controld_record_pending_op(lrm_state->node_name, rsc, op);
+
+    op_id = pcmk__op_key(rsc->id, op->op_type, op->interval_ms);
+
+    if (op->interval_ms > 0) {
+        /* cancel it so we can then restart it without conflict */
+        cancel_op_key(lrm_state, rsc, op_id, FALSE);
+    }
+
+    rc = controld_execute_resource_agent(lrm_state, rsc->id, op->op_type,
+                                         op->user_data, op->interval_ms,
+                                         op->timeout, op->start_delay,
+                                         op->params, &call_id);
+    if (rc == pcmk_rc_ok) {
+        /* record all operations so we can wait
+         * for them to complete during shutdown
+         */
+        char *call_id_s = make_stop_id(rsc->id, call_id);
+        active_op_t *pending = NULL;
+
+        pending = calloc(1, sizeof(active_op_t));
+        crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);
+
+        pending->call_id = call_id;
+        pending->interval_ms = op->interval_ms;
+        pending->op_type = strdup(operation);
+        pending->op_key = strdup(op_id);
+        pending->rsc_id = strdup(rsc->id);
+        pending->start_time = time(NULL);
+        pcmk__str_update(&pending->user_data, op->user_data);
+        if (crm_element_value_epoch(msg, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
+                                    &(pending->lock_time)) != pcmk_ok) {
+            pending->lock_time = 0;
+        }
+        g_hash_table_replace(lrm_state->active_ops, call_id_s, pending);
+
+        if ((op->interval_ms > 0)
+            && (op->start_delay > START_DELAY_THRESHOLD)) {
+            int target_rc = PCMK_OCF_OK;
+
+            crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);
+            decode_transition_key(op->user_data, NULL, NULL, NULL, &target_rc);
+            lrmd__set_result(op, target_rc, PCMK_EXEC_DONE, NULL);
+            controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
+        }
+
+        pending->params = op->params;
+        op->params = NULL;
+
+    } else if (lrm_state_is_local(lrm_state)) {
+        crm_err("Could not initiate %s action for resource %s locally: %s "
+                CRM_XS " rc=%d", operation, rsc->id, pcmk_rc_str(rc), rc);
+        fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
+                       PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
+        process_lrm_event(lrm_state, op, NULL, NULL);
+        register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
+
+    } else {
+        crm_err("Could not initiate %s action for resource %s remotely on %s: "
+                "%s " CRM_XS " rc=%d",
+                operation, rsc->id, lrm_state->node_name, pcmk_rc_str(rc), rc);
+        fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
+                       PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
+        process_lrm_event(lrm_state, op, NULL, NULL);
+    }
+
+    free(op_id);
+    lrmd_free_event(op);
+}
+
+void
+do_lrm_event(long long action,
+             enum crmd_fsa_cause cause,
+             enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
+{
+    CRM_CHECK(FALSE, return);
+}
+
+static char *
+unescape_newlines(const char *string)
+{
+    char *pch = NULL;
+    char *ret = NULL;
+    static const char *escaped_newline = "\\n";
+
+    if (!string) {
+        return NULL;
+    }
+
+    ret = strdup(string);
+    pch = strstr(ret, escaped_newline);
+    while (pch != NULL) {
+        /* Replace newline escape pattern with actual newline (and a space so we
+         * don't have to shuffle the rest of the buffer)
+         */
+        pch[0] = '\n';
+        pch[1] = ' ';
+        pch = strstr(pch, escaped_newline);
+    }
+
+    return ret;
+}
+
+static bool
+did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id,
+                    const char * op_type, guint interval_ms)
+{
+    rsc_history_t *entry = NULL;
+
+    CRM_CHECK(lrm_state != NULL, return FALSE);
+    CRM_CHECK(rsc_id != NULL, return FALSE);
+    CRM_CHECK(op_type != NULL, return FALSE);
+
+    entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
+    if (entry == NULL || entry->failed == NULL) {
+        return FALSE;
+    }
+
+    if (pcmk__str_eq(entry->failed->rsc_id, rsc_id, pcmk__str_none)
+        && pcmk__str_eq(entry->failed->op_type, op_type, pcmk__str_casei)
+        && entry->failed->interval_ms == interval_ms) {
+        return TRUE;
+    }
+
+    return FALSE;
+}
+
+/*!
+ * \internal
+ * \brief Log the result of an executor action (actual or synthesized)
+ *
+ * \param[in] op         Executor action to log result for
+ * \param[in] op_key     Operation key for action
+ * \param[in] node_name  Name of node action was performed on, if known
+ * \param[in] confirmed  Whether to log that graph action was confirmed
+ */
+static void
+log_executor_event(const lrmd_event_data_t *op, const char *op_key,
+                   const char *node_name, gboolean confirmed)
+{
+    int log_level = LOG_ERR;
+    GString *str = g_string_sized_new(100); // reasonable starting size
+
+    pcmk__g_strcat(str,
+                   "Result of ", crm_action_str(op->op_type, op->interval_ms),
+                   " operation for ", op->rsc_id, NULL);
+
+    if (node_name != NULL) {
+        pcmk__g_strcat(str, " on ", node_name, NULL);
+    }
+
+    switch (op->op_status) {
+        case PCMK_EXEC_DONE:
+            log_level = LOG_NOTICE;
+            pcmk__g_strcat(str, ": ", services_ocf_exitcode_str(op->rc), NULL);
+            break;
+
+        case PCMK_EXEC_TIMEOUT:
+            pcmk__g_strcat(str,
+                           ": ", pcmk_exec_status_str(op->op_status), " after ",
+                           pcmk__readable_interval(op->timeout), NULL);
+            break;
+
+        case PCMK_EXEC_CANCELLED:
+            log_level = LOG_INFO;
+	    /* order of __attribute__ and Fall through comment is IMPORTANT!
+	     * do not change it without proper testing with both clang and gcc
+	     * in multiple versions.
+	     * the clang check allows to build with all versions of clang.
+	     * the has_c_attribute check is to workaround a bug in clang version
+	     * in rhel7. has_attribute would happily return "YES SIR WE GOT IT"
+	     * and fail the build the next line.
+	     */
+#ifdef __clang__
+#ifdef __has_c_attribute
+#if __has_attribute(fallthrough)
+	    __attribute__((fallthrough));
+#endif
+#endif
+#endif
+            // Fall through
+        default:
+            pcmk__g_strcat(str, ": ", pcmk_exec_status_str(op->op_status),
+                           NULL);
+    }
+
+    if ((op->exit_reason != NULL)
+        && ((op->op_status != PCMK_EXEC_DONE) || (op->rc != PCMK_OCF_OK))) {
+
+        pcmk__g_strcat(str, " (", op->exit_reason, ")", NULL);
+    }
+
+    g_string_append(str, " " CRM_XS);
+    g_string_append_printf(str, " graph action %sconfirmed; call=%d key=%s",
+                           (confirmed? "" : "un"), op->call_id, op_key);
+    if (op->op_status == PCMK_EXEC_DONE) {
+        g_string_append_printf(str, " rc=%d", op->rc);
+    }
+
+    do_crm_log(log_level, "%s", str->str);
+    g_string_free(str, TRUE);
+
+    /* The services library has already logged the output at info or debug
+     * level, so just raise to notice if it looks like a failure.
+     */
+    if ((op->output != NULL) && (op->rc != PCMK_OCF_OK)) {
+        char *prefix = crm_strdup_printf(PCMK__OP_FMT "@%s output",
+                                         op->rsc_id, op->op_type,
+                                         op->interval_ms, node_name);
+
+        crm_log_output(LOG_NOTICE, prefix, op->output);
+        free(prefix);
+    }
+}
+
+void
+process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
+                  active_op_t *pending, const xmlNode *action_xml)
+{
+    char *op_id = NULL;
+    char *op_key = NULL;
+
+    gboolean remove = FALSE;
+    gboolean removed = FALSE;
+    bool need_direct_ack = FALSE;
+    lrmd_rsc_info_t *rsc = NULL;
+    const char *node_name = NULL;
+
+    CRM_CHECK(op != NULL, return);
+    CRM_CHECK(op->rsc_id != NULL, return);
+
+    // Remap new status codes for older DCs
+    if (compare_version(controld_globals.dc_version, "3.2.0") < 0) {
+        switch (op->op_status) {
+            case PCMK_EXEC_NOT_CONNECTED:
+                lrmd__set_result(op, PCMK_OCF_CONNECTION_DIED,
+                                 PCMK_EXEC_ERROR, op->exit_reason);
+                break;
+            case PCMK_EXEC_INVALID:
+                lrmd__set_result(op, CRM_DIRECT_NACK_RC, PCMK_EXEC_ERROR,
+                                 op->exit_reason);
+                break;
+            default:
+                break;
+        }
+    }
+
+    op_id = make_stop_id(op->rsc_id, op->call_id);
+    op_key = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
+
+    // Get resource info if available (from executor state or action XML)
+    if (lrm_state) {
+        rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
+    }
+    if ((rsc == NULL) && action_xml) {
+        xmlNode *xml = find_xml_node(action_xml, XML_CIB_TAG_RESOURCE, TRUE);
+
+        const char *standard = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
+        const char *provider = crm_element_value(xml, XML_AGENT_ATTR_PROVIDER);
+        const char *type = crm_element_value(xml, XML_ATTR_TYPE);
+
+        if (standard && type) {
+            crm_info("%s agent information not cached, using %s%s%s:%s from action XML",
+                     op->rsc_id, standard,
+                     (provider? ":" : ""), (provider? provider : ""), type);
+            rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
+        } else {
+            crm_err("Can't process %s result because %s agent information not cached or in XML",
+                    op_key, op->rsc_id);
+        }
+    }
+
+    // Get node name if available (from executor state or action XML)
+    if (lrm_state) {
+        node_name = lrm_state->node_name;
+    } else if (action_xml) {
+        node_name = crm_element_value(action_xml, XML_LRM_ATTR_TARGET);
+    }
+
+    if(pending == NULL) {
+        remove = TRUE;
+        if (lrm_state) {
+            pending = g_hash_table_lookup(lrm_state->active_ops, op_id);
+        }
+    }
+
+    if (op->op_status == PCMK_EXEC_ERROR) {
+        switch(op->rc) {
+            case PCMK_OCF_NOT_RUNNING:
+            case PCMK_OCF_RUNNING_PROMOTED:
+            case PCMK_OCF_DEGRADED:
+            case PCMK_OCF_DEGRADED_PROMOTED:
+                // Leave it to the TE/scheduler to decide if this is an error
+                op->op_status = PCMK_EXEC_DONE;
+                break;
+            default:
+                /* Nothing to do */
+                break;
+        }
+    }
+
+    if (op->op_status != PCMK_EXEC_CANCELLED) {
+        /* We might not record the result, so directly acknowledge it to the
+         * originator instead, so it doesn't time out waiting for the result
+         * (especially important if part of a transition).
+         */
+        need_direct_ack = TRUE;
+
+        if (controld_action_is_recordable(op->op_type)) {
+            if (node_name && rsc) {
+                // We should record the result, and happily, we can
+                time_t lock_time = (pending == NULL)? 0 : pending->lock_time;
+
+                controld_update_resource_history(node_name, rsc, op, lock_time);
+                need_direct_ack = FALSE;
+
+            } else if (op->rsc_deleted) {
+                /* We shouldn't record the result (likely the resource was
+                 * refreshed, cleaned, or removed while this operation was
+                 * in flight).
+                 */
+                crm_notice("Not recording %s result in CIB because "
+                           "resource information was removed since it was initiated",
+                           op_key);
+            } else {
+                /* This shouldn't be possible; the executor didn't consider the
+                 * resource deleted, but we couldn't find resource or node
+                 * information.
+                 */
+                crm_err("Unable to record %s result in CIB: %s", op_key,
+                        (node_name? "No resource information" : "No node name"));
+            }
+        }
+
+    } else if (op->interval_ms == 0) {
+        /* A non-recurring operation was cancelled. Most likely, the
+         * never-initiated action was removed from the executor's pending
+         * operations list upon resource removal.
+         */
+        need_direct_ack = TRUE;
+
+    } else if (pending == NULL) {
+        /* This recurring operation was cancelled, but was not pending. No
+         * transition actions are waiting on it, nothing needs to be done.
+         */
+
+    } else if (op->user_data == NULL) {
+        /* This recurring operation was cancelled and pending, but we don't
+         * have a transition key. This should never happen.
+         */
+        crm_err("Recurring operation %s was cancelled without transition information",
+                op_key);
+
+    } else if (pcmk_is_set(pending->flags, active_op_remove)) {
+        /* This recurring operation was cancelled (by us) and pending, and we
+         * have been waiting for it to finish.
+         */
+        if (lrm_state) {
+            controld_delete_action_history(op);
+        }
+
+        /* Directly acknowledge failed recurring actions here. The above call to
+         * controld_delete_action_history() will not erase any corresponding
+         * last_failure entry, which means that the DC won't confirm the
+         * cancellation via process_op_deletion(), and the transition would
+         * otherwise wait for the action timer to pop.
+         */
+        if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id,
+                                pending->op_type, pending->interval_ms)) {
+            need_direct_ack = TRUE;
+        }
+
+    } else if (op->rsc_deleted) {
+        /* This recurring operation was cancelled (but not by us, and the
+         * executor does not have resource information, likely due to resource
+         * cleanup, refresh, or removal) and pending.
+         */
+        crm_debug("Recurring op %s was cancelled due to resource deletion",
+                  op_key);
+        need_direct_ack = TRUE;
+
+    } else {
+        /* This recurring operation was cancelled (but not by us, likely by the
+         * executor before stopping the resource) and pending. We don't need to
+         * do anything special.
+         */
+    }
+
+    if (need_direct_ack) {
+        controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id);
+    }
+
+    if(remove == FALSE) {
+        /* The caller will do this afterwards, but keep the logging consistent */
+        removed = TRUE;
+
+    } else if (lrm_state && ((op->interval_ms == 0)
+                             || (op->op_status == PCMK_EXEC_CANCELLED))) {
+
+        gboolean found = g_hash_table_remove(lrm_state->active_ops, op_id);
+
+        if (op->interval_ms != 0) {
+            removed = TRUE;
+        } else if (found) {
+            removed = TRUE;
+            crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
+                      op_key, op->call_id, op_id,
+                      g_hash_table_size(lrm_state->active_ops));
+        }
+    }
+
+    log_executor_event(op, op_key, node_name, removed);
+
+    if (lrm_state) {
+        if (!pcmk__str_eq(op->op_type, RSC_METADATA, pcmk__str_casei)) {
+            crmd_alert_resource_op(lrm_state->node_name, op);
+        } else if (rsc && (op->rc == PCMK_OCF_OK)) {
+            char *metadata = unescape_newlines(op->output);
+
+            controld_cache_metadata(lrm_state->metadata_cache, rsc, metadata);
+            free(metadata);
+        }
+    }
+
+    if (op->rsc_deleted) {
+        crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
+        if (lrm_state) {
+            delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL,
+                             true);
+        }
+    }
+
+    /* If a shutdown was escalated while operations were pending,
+     * then the FSA will be stalled right now... allow it to continue
+     */
+    controld_trigger_fsa();
+    if (lrm_state && rsc) {
+        update_history_cache(lrm_state, rsc, op);
+    }
+
+    lrmd_free_rsc_info(rsc);
+    free(op_key);
+    free(op_id);
+}
diff --git a/daemons/controld/controld_execd_state.c b/daemons/controld/controld_execd_state.c
new file mode 100644
index 0000000..8c68bfc
--- /dev/null
+++ b/daemons/controld/controld_execd_state.c
@@ -0,0 +1,814 @@
+/*
+ * Copyright 2012-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <errno.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/iso8601.h>
+#include <crm/pengine/rules.h>
+#include <crm/pengine/rules_internal.h>
+#include <crm/lrmd_internal.h>
+
+#include <pacemaker-internal.h>
+#include <pacemaker-controld.h>
+
+static GHashTable *lrm_state_table = NULL;
+extern GHashTable *proxy_table;
+int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
+void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
+
+static void
+free_rsc_info(gpointer value)
+{
+    lrmd_rsc_info_t *rsc_info = value;
+
+    lrmd_free_rsc_info(rsc_info);
+}
+
+static void
+free_deletion_op(gpointer value)
+{
+    struct pending_deletion_op_s *op = value;
+
+    free(op->rsc);
+    delete_ha_msg_input(op->input);
+    free(op);
+}
+
+static void
+free_recurring_op(gpointer value)
+{
+    active_op_t *op = value;
+
+    free(op->user_data);
+    free(op->rsc_id);
+    free(op->op_type);
+    free(op->op_key);
+    if (op->params) {
+        g_hash_table_destroy(op->params);
+    }
+    free(op);
+}
+
+static gboolean
+fail_pending_op(gpointer key, gpointer value, gpointer user_data)
+{
+    lrmd_event_data_t event = { 0, };
+    lrm_state_t *lrm_state = user_data;
+    active_op_t *op = value;
+
+    crm_trace("Pre-emptively failing " PCMK__OP_FMT " on %s (call=%s, %s)",
+              op->rsc_id, op->op_type, op->interval_ms,
+              lrm_state->node_name, (char*)key, op->user_data);
+
+    event.type = lrmd_event_exec_complete;
+    event.rsc_id = op->rsc_id;
+    event.op_type = op->op_type;
+    event.user_data = op->user_data;
+    event.timeout = 0;
+    event.interval_ms = op->interval_ms;
+    lrmd__set_result(&event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_NOT_CONNECTED,
+                     "Action was pending when executor connection was dropped");
+    event.t_run = (unsigned int) op->start_time;
+    event.t_rcchange = (unsigned int) op->start_time;
+
+    event.call_id = op->call_id;
+    event.remote_nodename = lrm_state->node_name;
+    event.params = op->params;
+
+    process_lrm_event(lrm_state, &event, op, NULL);
+    lrmd__reset_result(&event);
+    return TRUE;
+}
+
+gboolean
+lrm_state_is_local(lrm_state_t *lrm_state)
+{
+    return (lrm_state != NULL)
+           && pcmk__str_eq(lrm_state->node_name, controld_globals.our_nodename,
+                           pcmk__str_casei);
+}
+
+/*!
+ * \internal
+ * \brief Create executor state entry for a node and add it to the state table
+ *
+ * \param[in]  node_name  Node to create entry for
+ *
+ * \return Newly allocated executor state object initialized for \p node_name
+ */
+static lrm_state_t *
+lrm_state_create(const char *node_name)
+{
+    lrm_state_t *state = NULL;
+
+    if (!node_name) {
+        crm_err("No node name given for lrm state object");
+        return NULL;
+    }
+
+    state = calloc(1, sizeof(lrm_state_t));
+    if (!state) {
+        return NULL;
+    }
+
+    state->node_name = strdup(node_name);
+    state->rsc_info_cache = pcmk__strkey_table(NULL, free_rsc_info);
+    state->deletion_ops = pcmk__strkey_table(free, free_deletion_op);
+    state->active_ops = pcmk__strkey_table(free, free_recurring_op);
+    state->resource_history = pcmk__strkey_table(NULL, history_free);
+    state->metadata_cache = metadata_cache_new();
+
+    g_hash_table_insert(lrm_state_table, (char *)state->node_name, state);
+    return state;
+}
+
+void
+lrm_state_destroy(const char *node_name)
+{
+    g_hash_table_remove(lrm_state_table, node_name);
+}
+
+static gboolean
+remote_proxy_remove_by_node(gpointer key, gpointer value, gpointer user_data)
+{
+    remote_proxy_t *proxy = value;
+    const char *node_name = user_data;
+
+    if (pcmk__str_eq(node_name, proxy->node_name, pcmk__str_casei)) {
+        return TRUE;
+    }
+
+    return FALSE;
+}
+
+static remote_proxy_t *
+find_connected_proxy_by_node(const char * node_name)
+{
+    GHashTableIter gIter;
+    remote_proxy_t *proxy = NULL;
+
+    CRM_CHECK(proxy_table != NULL, return NULL);
+
+    g_hash_table_iter_init(&gIter, proxy_table);
+
+    while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) &proxy)) {
+        if (proxy->source
+            && pcmk__str_eq(node_name, proxy->node_name, pcmk__str_casei)) {
+            return proxy;
+        }
+    }
+
+    return NULL;
+}
+
+static void
+remote_proxy_disconnect_by_node(const char * node_name)
+{
+    remote_proxy_t *proxy = NULL;
+
+    CRM_CHECK(proxy_table != NULL, return);
+
+    while ((proxy = find_connected_proxy_by_node(node_name)) != NULL) {
+        /* mainloop_del_ipc_client() eventually calls remote_proxy_disconnected()
+         * , which removes the entry from proxy_table.
+         * Do not do this in a g_hash_table_iter_next() loop. */
+        if (proxy->source) {
+            mainloop_del_ipc_client(proxy->source);
+        }
+    }
+
+    return;
+}
+
+static void
+internal_lrm_state_destroy(gpointer data)
+{
+    lrm_state_t *lrm_state = data;
+
+    if (!lrm_state) {
+        return;
+    }
+
+    /* Rather than directly remove the recorded proxy entries from proxy_table,
+     * make sure any connected proxies get disconnected. So that
+     * remote_proxy_disconnected() will be called and as well remove the
+     * entries from proxy_table.
+     */
+    remote_proxy_disconnect_by_node(lrm_state->node_name);
+
+    crm_trace("Destroying proxy table %s with %u members",
+              lrm_state->node_name, g_hash_table_size(proxy_table));
+    // Just in case there's still any leftovers in proxy_table
+    g_hash_table_foreach_remove(proxy_table, remote_proxy_remove_by_node, (char *) lrm_state->node_name);
+    remote_ra_cleanup(lrm_state);
+    lrmd_api_delete(lrm_state->conn);
+
+    if (lrm_state->rsc_info_cache) {
+        crm_trace("Destroying rsc info cache with %u members",
+                  g_hash_table_size(lrm_state->rsc_info_cache));
+        g_hash_table_destroy(lrm_state->rsc_info_cache);
+    }
+    if (lrm_state->resource_history) {
+        crm_trace("Destroying history op cache with %u members",
+                  g_hash_table_size(lrm_state->resource_history));
+        g_hash_table_destroy(lrm_state->resource_history);
+    }
+    if (lrm_state->deletion_ops) {
+        crm_trace("Destroying deletion op cache with %u members",
+                  g_hash_table_size(lrm_state->deletion_ops));
+        g_hash_table_destroy(lrm_state->deletion_ops);
+    }
+    if (lrm_state->active_ops != NULL) {
+        crm_trace("Destroying pending op cache with %u members",
+                  g_hash_table_size(lrm_state->active_ops));
+        g_hash_table_destroy(lrm_state->active_ops);
+    }
+    metadata_cache_free(lrm_state->metadata_cache);
+
+    free((char *)lrm_state->node_name);
+    free(lrm_state);
+}
+
+void
+lrm_state_reset_tables(lrm_state_t * lrm_state, gboolean reset_metadata)
+{
+    if (lrm_state->resource_history) {
+        crm_trace("Resetting resource history cache with %u members",
+                  g_hash_table_size(lrm_state->resource_history));
+        g_hash_table_remove_all(lrm_state->resource_history);
+    }
+    if (lrm_state->deletion_ops) {
+        crm_trace("Resetting deletion operations cache with %u members",
+                  g_hash_table_size(lrm_state->deletion_ops));
+        g_hash_table_remove_all(lrm_state->deletion_ops);
+    }
+    if (lrm_state->active_ops != NULL) {
+        crm_trace("Resetting active operations cache with %u members",
+                  g_hash_table_size(lrm_state->active_ops));
+        g_hash_table_remove_all(lrm_state->active_ops);
+    }
+    if (lrm_state->rsc_info_cache) {
+        crm_trace("Resetting resource information cache with %u members",
+                  g_hash_table_size(lrm_state->rsc_info_cache));
+        g_hash_table_remove_all(lrm_state->rsc_info_cache);
+    }
+    if (reset_metadata) {
+        metadata_cache_reset(lrm_state->metadata_cache);
+    }
+}
+
+gboolean
+lrm_state_init_local(void)
+{
+    if (lrm_state_table) {
+        return TRUE;
+    }
+
+    lrm_state_table = pcmk__strikey_table(NULL, internal_lrm_state_destroy);
+    if (!lrm_state_table) {
+        return FALSE;
+    }
+
+    proxy_table = pcmk__strikey_table(NULL, remote_proxy_free);
+    if (!proxy_table) {
+        g_hash_table_destroy(lrm_state_table);
+        lrm_state_table = NULL;
+        return FALSE;
+    }
+
+    return TRUE;
+}
+
+void
+lrm_state_destroy_all(void)
+{
+    if (lrm_state_table) {
+        crm_trace("Destroying state table with %u members",
+                  g_hash_table_size(lrm_state_table));
+        g_hash_table_destroy(lrm_state_table); lrm_state_table = NULL;
+    }
+    if(proxy_table) {
+        crm_trace("Destroying proxy table with %u members",
+                  g_hash_table_size(proxy_table));
+        g_hash_table_destroy(proxy_table); proxy_table = NULL;
+    }
+}
+
+lrm_state_t *
+lrm_state_find(const char *node_name)
+{
+    if (!node_name) {
+        return NULL;
+    }
+    return g_hash_table_lookup(lrm_state_table, node_name);
+}
+
+lrm_state_t *
+lrm_state_find_or_create(const char *node_name)
+{
+    lrm_state_t *lrm_state;
+
+    lrm_state = g_hash_table_lookup(lrm_state_table, node_name);
+    if (!lrm_state) {
+        lrm_state = lrm_state_create(node_name);
+    }
+
+    return lrm_state;
+}
+
+GList *
+lrm_state_get_list(void)
+{
+    return g_hash_table_get_values(lrm_state_table);
+}
+
+void
+lrm_state_disconnect_only(lrm_state_t * lrm_state)
+{
+    int removed = 0;
+
+    if (!lrm_state->conn) {
+        return;
+    }
+    crm_trace("Disconnecting %s", lrm_state->node_name);
+
+    remote_proxy_disconnect_by_node(lrm_state->node_name);
+
+    ((lrmd_t *) lrm_state->conn)->cmds->disconnect(lrm_state->conn);
+
+    if (!pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+        removed = g_hash_table_foreach_remove(lrm_state->active_ops,
+                                              fail_pending_op, lrm_state);
+        crm_trace("Synthesized %d operation failures for %s", removed, lrm_state->node_name);
+    }
+}
+
+void
+lrm_state_disconnect(lrm_state_t * lrm_state)
+{
+    if (!lrm_state->conn) {
+        return;
+    }
+
+    lrm_state_disconnect_only(lrm_state);
+
+    lrmd_api_delete(lrm_state->conn);
+    lrm_state->conn = NULL;
+}
+
+int
+lrm_state_is_connected(lrm_state_t * lrm_state)
+{
+    if (!lrm_state->conn) {
+        return FALSE;
+    }
+    return ((lrmd_t *) lrm_state->conn)->cmds->is_connected(lrm_state->conn);
+}
+
+int
+lrm_state_poke_connection(lrm_state_t * lrm_state)
+{
+
+    if (!lrm_state->conn) {
+        return -ENOTCONN;
+    }
+    return ((lrmd_t *) lrm_state->conn)->cmds->poke_connection(lrm_state->conn);
+}
+
+// \return Standard Pacemaker return code
+int
+controld_connect_local_executor(lrm_state_t *lrm_state)
+{
+    int rc = pcmk_rc_ok;
+
+    if (lrm_state->conn == NULL) {
+        lrmd_t *api = NULL;
+
+        rc = lrmd__new(&api, NULL, NULL, 0);
+        if (rc != pcmk_rc_ok) {
+            return rc;
+        }
+        api->cmds->set_callback(api, lrm_op_callback);
+        lrm_state->conn = api;
+    }
+
+    rc = ((lrmd_t *) lrm_state->conn)->cmds->connect(lrm_state->conn,
+                                                     CRM_SYSTEM_CRMD, NULL);
+    rc = pcmk_legacy2rc(rc);
+
+    if (rc == pcmk_rc_ok) {
+        lrm_state->num_lrm_register_fails = 0;
+    } else {
+        lrm_state->num_lrm_register_fails++;
+    }
+    return rc;
+}
+
+static remote_proxy_t *
+crmd_remote_proxy_new(lrmd_t *lrmd, const char *node_name, const char *session_id, const char *channel)
+{
+    struct ipc_client_callbacks proxy_callbacks = {
+        .dispatch = remote_proxy_dispatch,
+        .destroy = remote_proxy_disconnected
+    };
+    remote_proxy_t *proxy = remote_proxy_new(lrmd, &proxy_callbacks, node_name,
+                                             session_id, channel);
+    return proxy;
+}
+
+gboolean
+crmd_is_proxy_session(const char *session)
+{
+    return g_hash_table_lookup(proxy_table, session) ? TRUE : FALSE;
+}
+
+void
+crmd_proxy_send(const char *session, xmlNode *msg)
+{
+    remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
+    lrm_state_t *lrm_state = NULL;
+
+    if (!proxy) {
+        return;
+    }
+    crm_log_xml_trace(msg, "to-proxy");
+    lrm_state = lrm_state_find(proxy->node_name);
+    if (lrm_state) {
+        crm_trace("Sending event to %.8s on %s", proxy->session_id, proxy->node_name);
+        remote_proxy_relay_event(proxy, msg);
+    }
+}
+
+static void
+crmd_proxy_dispatch(const char *session, xmlNode *msg)
+{
+    crm_trace("Processing proxied IPC message from session %s", session);
+    crm_log_xml_trace(msg, "controller[inbound]");
+    crm_xml_add(msg, F_CRM_SYS_FROM, session);
+    if (controld_authorize_ipc_message(msg, NULL, session)) {
+        route_message(C_IPC_MESSAGE, msg);
+    }
+    controld_trigger_fsa();
+}
+
+static void
+remote_config_check(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    if (rc != pcmk_ok) {
+        crm_err("Query resulted in an error: %s", pcmk_strerror(rc));
+
+        if (rc == -EACCES || rc == -pcmk_err_schema_validation) {
+            crm_err("The cluster is mis-configured - shutting down and staying down");
+        }
+
+    } else {
+        lrmd_t * lrmd = (lrmd_t *)user_data;
+        crm_time_t *now = crm_time_new(NULL);
+        GHashTable *config_hash = pcmk__strkey_table(free, free);
+
+        crm_debug("Call %d : Parsing CIB options", call_id);
+
+        pe_unpack_nvpairs(output, output, XML_CIB_TAG_PROPSET, NULL,
+                          config_hash, CIB_OPTIONS_FIRST, FALSE, now, NULL);
+
+        /* Now send it to the remote peer */
+        lrmd__validate_remote_settings(lrmd, config_hash);
+
+        g_hash_table_destroy(config_hash);
+        crm_time_free(now);
+    }
+}
+
+static void
+crmd_remote_proxy_cb(lrmd_t *lrmd, void *userdata, xmlNode *msg)
+{
+    lrm_state_t *lrm_state = userdata;
+    const char *session = crm_element_value(msg, F_LRMD_IPC_SESSION);
+    remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
+
+    const char *op = crm_element_value(msg, F_LRMD_IPC_OP);
+    if (pcmk__str_eq(op, LRMD_IPC_OP_NEW, pcmk__str_casei)) {
+        const char *channel = crm_element_value(msg, F_LRMD_IPC_IPC_SERVER);
+
+        proxy = crmd_remote_proxy_new(lrmd, lrm_state->node_name, session, channel);
+        if (!remote_ra_controlling_guest(lrm_state)) {
+            if (proxy != NULL) {
+                cib_t *cib_conn = controld_globals.cib_conn;
+
+                /* Look up stonith-watchdog-timeout and send to the remote peer for validation */
+                int rc = cib_conn->cmds->query(cib_conn, XML_CIB_TAG_CRMCONFIG,
+                                               NULL, cib_scope_local);
+                cib_conn->cmds->register_callback_full(cib_conn, rc, 10, FALSE,
+                                                       lrmd,
+                                                       "remote_config_check",
+                                                       remote_config_check,
+                                                       NULL);
+            }
+        } else {
+            crm_debug("Skipping remote_config_check for guest-nodes");
+        }
+
+    } else if (pcmk__str_eq(op, LRMD_IPC_OP_SHUTDOWN_REQ, pcmk__str_casei)) {
+        char *now_s = NULL;
+
+        crm_notice("%s requested shutdown of its remote connection",
+                   lrm_state->node_name);
+
+        if (!remote_ra_is_in_maintenance(lrm_state)) {
+            now_s = pcmk__ttoa(time(NULL));
+            update_attrd(lrm_state->node_name, XML_CIB_ATTR_SHUTDOWN, now_s, NULL, TRUE);
+            free(now_s);
+
+            remote_proxy_ack_shutdown(lrmd);
+
+            crm_warn("Reconnection attempts to %s may result in failures that must be cleared",
+                    lrm_state->node_name);
+        } else {
+            remote_proxy_nack_shutdown(lrmd);
+
+            crm_notice("Remote resource for %s is not managed so no ordered shutdown happening",
+                    lrm_state->node_name);
+        }
+        return;
+
+    } else if (pcmk__str_eq(op, LRMD_IPC_OP_REQUEST, pcmk__str_casei) && proxy && proxy->is_local) {
+        /* This is for the controller, which we are, so don't try
+         * to send to ourselves over IPC -- do it directly.
+         */
+        int flags = 0;
+        xmlNode *request = get_message_xml(msg, F_LRMD_IPC_MSG);
+
+        CRM_CHECK(request != NULL, return);
+        CRM_CHECK(lrm_state->node_name, return);
+        crm_xml_add(request, XML_ACL_TAG_ROLE, "pacemaker-remote");
+        pcmk__update_acl_user(request, F_LRMD_IPC_USER, lrm_state->node_name);
+
+        /* Pacemaker Remote nodes don't know their own names (as known to the
+         * cluster). When getting a node info request with no name or ID, add
+         * the name, so we don't return info for ourselves instead of the
+         * Pacemaker Remote node.
+         */
+        if (pcmk__str_eq(crm_element_value(request, F_CRM_TASK), CRM_OP_NODE_INFO, pcmk__str_casei)) {
+            int node_id = 0;
+
+            crm_element_value_int(request, XML_ATTR_ID, &node_id);
+            if ((node_id <= 0)
+                && (crm_element_value(request, XML_ATTR_UNAME) == NULL)) {
+                crm_xml_add(request, XML_ATTR_UNAME, lrm_state->node_name);
+            }
+        }
+
+        crmd_proxy_dispatch(session, request);
+
+        crm_element_value_int(msg, F_LRMD_IPC_MSG_FLAGS, &flags);
+        if (flags & crm_ipc_client_response) {
+            int msg_id = 0;
+            xmlNode *op_reply = create_xml_node(NULL, "ack");
+
+            crm_xml_add(op_reply, "function", __func__);
+            crm_xml_add_int(op_reply, "line", __LINE__);
+
+            crm_element_value_int(msg, F_LRMD_IPC_MSG_ID, &msg_id);
+            remote_proxy_relay_response(proxy, op_reply, msg_id);
+
+            free_xml(op_reply);
+        }
+
+    } else {
+        remote_proxy_cb(lrmd, lrm_state->node_name, msg);
+    }
+}
+
+
+// \return Standard Pacemaker return code
+int
+controld_connect_remote_executor(lrm_state_t *lrm_state, const char *server,
+                                 int port, int timeout_ms)
+{
+    int rc = pcmk_rc_ok;
+
+    if (lrm_state->conn == NULL) {
+        lrmd_t *api = NULL;
+
+        rc = lrmd__new(&api, lrm_state->node_name, server, port);
+        if (rc != pcmk_rc_ok) {
+            crm_warn("Pacemaker Remote connection to %s:%s failed: %s "
+                     CRM_XS " rc=%d", server, port, pcmk_rc_str(rc), rc);
+
+            return rc;
+        }
+        lrm_state->conn = api;
+        api->cmds->set_callback(api, remote_lrm_op_callback);
+        lrmd_internal_set_proxy_callback(api, lrm_state, crmd_remote_proxy_cb);
+    }
+
+    crm_trace("Initiating remote connection to %s:%d with timeout %dms",
+              server, port, timeout_ms);
+    rc = ((lrmd_t *) lrm_state->conn)->cmds->connect_async(lrm_state->conn,
+                                                           lrm_state->node_name,
+                                                           timeout_ms);
+    if (rc == pcmk_ok) {
+        lrm_state->num_lrm_register_fails = 0;
+    } else {
+        lrm_state->num_lrm_register_fails++; // Ignored for remote connections
+    }
+    return pcmk_legacy2rc(rc);
+}
+
+int
+lrm_state_get_metadata(lrm_state_t * lrm_state,
+                       const char *class,
+                       const char *provider,
+                       const char *agent, char **output, enum lrmd_call_options options)
+{
+    lrmd_key_value_t *params = NULL;
+
+    if (!lrm_state->conn) {
+        return -ENOTCONN;
+    }
+
+    /* Add the node name to the environment, as is done with normal resource
+     * action calls. Meta-data calls shouldn't need it, but some agents are
+     * written with an ocf_local_nodename call at the beginning regardless of
+     * action. Without the environment variable, the agent would try to contact
+     * the controller to get the node name -- but the controller would be
+     * blocking on the synchronous meta-data call.
+     *
+     * At this point, we have to assume that agents are unlikely to make other
+     * calls that require the controller, such as crm_node --quorum or
+     * --cluster-id.
+     *
+     * @TODO Make meta-data calls asynchronous. (This will be part of a larger
+     * project to make meta-data calls via the executor rather than directly.)
+     */
+    params = lrmd_key_value_add(params, CRM_META "_" XML_LRM_ATTR_TARGET,
+                                lrm_state->node_name);
+
+    return ((lrmd_t *) lrm_state->conn)->cmds->get_metadata_params(lrm_state->conn,
+            class, provider, agent, output, options, params);
+}
+
+int
+lrm_state_cancel(lrm_state_t *lrm_state, const char *rsc_id, const char *action,
+                 guint interval_ms)
+{
+    if (!lrm_state->conn) {
+        return -ENOTCONN;
+    }
+
+    /* Figure out a way to make this async?
+     * NOTICE: Currently it's synced and directly acknowledged in do_lrm_invoke(). */
+    if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
+        return remote_ra_cancel(lrm_state, rsc_id, action, interval_ms);
+    }
+    return ((lrmd_t *) lrm_state->conn)->cmds->cancel(lrm_state->conn, rsc_id,
+                                                      action, interval_ms);
+}
+
+lrmd_rsc_info_t *
+lrm_state_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id, enum lrmd_call_options options)
+{
+    lrmd_rsc_info_t *rsc = NULL;
+
+    if (!lrm_state->conn) {
+        return NULL;
+    }
+    if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
+        return remote_ra_get_rsc_info(lrm_state, rsc_id);
+    }
+
+    rsc = g_hash_table_lookup(lrm_state->rsc_info_cache, rsc_id);
+    if (rsc == NULL) {
+        /* only contact the lrmd if we don't already have a cached rsc info */
+        rsc = ((lrmd_t *) lrm_state->conn)->cmds->get_rsc_info(lrm_state->conn, rsc_id, options);
+        if (rsc == NULL) {
+		    return NULL;
+        }
+        /* cache the result */
+        g_hash_table_insert(lrm_state->rsc_info_cache, rsc->id, rsc);
+    }
+
+    return lrmd_copy_rsc_info(rsc);
+
+}
+
+/*!
+ * \internal
+ * \brief Initiate a resource agent action
+ *
+ * \param[in,out] lrm_state       Executor state object
+ * \param[in]     rsc_id          ID of resource for action
+ * \param[in]     action          Action to execute
+ * \param[in]     userdata        String to copy and pass to execution callback
+ * \param[in]     interval_ms     Action interval (in milliseconds)
+ * \param[in]     timeout_ms      Action timeout (in milliseconds)
+ * \param[in]     start_delay_ms  Delay (in ms) before initiating action
+ * \param[in]     parameters      Hash table of resource parameters
+ * \param[out]    call_id         Where to store call ID on success
+ *
+ * \return Standard Pacemaker return code
+ */
+int
+controld_execute_resource_agent(lrm_state_t *lrm_state, const char *rsc_id,
+                                const char *action, const char *userdata,
+                                guint interval_ms, int timeout_ms,
+                                int start_delay_ms, GHashTable *parameters,
+                                int *call_id)
+{
+    int rc = pcmk_rc_ok;
+    lrmd_key_value_t *params = NULL;
+
+    if (lrm_state->conn == NULL) {
+        return ENOTCONN;
+    }
+
+    // Convert parameters from hash table to list
+    if (parameters != NULL) {
+        const char *key = NULL;
+        const char *value = NULL;
+        GHashTableIter iter;
+
+        g_hash_table_iter_init(&iter, parameters);
+        while (g_hash_table_iter_next(&iter, (gpointer *) &key,
+                                      (gpointer *) &value)) {
+            params = lrmd_key_value_add(params, key, value);
+        }
+    }
+
+    if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
+        rc = controld_execute_remote_agent(lrm_state, rsc_id, action,
+                                           userdata, interval_ms, timeout_ms,
+                                           start_delay_ms, params, call_id);
+
+    } else {
+        rc = ((lrmd_t *) lrm_state->conn)->cmds->exec(lrm_state->conn, rsc_id,
+                                                      action, userdata,
+                                                      interval_ms, timeout_ms,
+                                                      start_delay_ms,
+                                                      lrmd_opt_notify_changes_only,
+                                                      params);
+        if (rc < 0) {
+            rc = pcmk_legacy2rc(rc);
+        } else {
+            *call_id = rc;
+            rc = pcmk_rc_ok;
+        }
+    }
+    return rc;
+}
+
+int
+lrm_state_register_rsc(lrm_state_t * lrm_state,
+                       const char *rsc_id,
+                       const char *class,
+                       const char *provider, const char *agent, enum lrmd_call_options options)
+{
+    lrmd_t *conn = (lrmd_t *) lrm_state->conn;
+
+    if (conn == NULL) {
+        return -ENOTCONN;
+    }
+
+    if (is_remote_lrmd_ra(agent, provider, NULL)) {
+        return lrm_state_find_or_create(rsc_id)? pcmk_ok : -EINVAL;
+    }
+
+    /* @TODO Implement an asynchronous version of this (currently a blocking
+     * call to the lrmd).
+     */
+    return conn->cmds->register_rsc(lrm_state->conn, rsc_id, class, provider,
+                                    agent, options);
+}
+
+int
+lrm_state_unregister_rsc(lrm_state_t * lrm_state,
+                         const char *rsc_id, enum lrmd_call_options options)
+{
+    if (!lrm_state->conn) {
+        return -ENOTCONN;
+    }
+
+    if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
+        lrm_state_destroy(rsc_id);
+        return pcmk_ok;
+    }
+
+    g_hash_table_remove(lrm_state->rsc_info_cache, rsc_id);
+
+    /* @TODO Optimize this ... this function is a blocking round trip from
+     * client to daemon. The controld_execd_state.c code path that uses this
+     * function should always treat it as an async operation. The executor API
+     * should make an async version available.
+     */
+    return ((lrmd_t *) lrm_state->conn)->cmds->unregister_rsc(lrm_state->conn, rsc_id, options);
+}
diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c
new file mode 100644
index 0000000..89cb61f
--- /dev/null
+++ b/daemons/controld/controld_fencing.c
@@ -0,0 +1,1108 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/stonith-ng.h>
+#include <crm/fencing/internal.h>
+
+#include <pacemaker-controld.h>
+
+static void
+tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event);
+
+/*
+ * stonith failure counting
+ *
+ * We don't want to get stuck in a permanent fencing loop. Keep track of the
+ * number of fencing failures for each target node, and the most we'll restart a
+ * transition for.
+ */
+
+struct st_fail_rec {
+    int count;
+};
+
+static bool fence_reaction_panic = false;
+static unsigned long int stonith_max_attempts = 10;
+static GHashTable *stonith_failures = NULL;
+
+/*!
+ * \internal
+ * \brief Update max fencing attempts before giving up
+ *
+ * \param[in] value  New max fencing attempts
+ */
+static void
+update_stonith_max_attempts(const char *value)
+{
+    stonith_max_attempts = char2score(value);
+    if (stonith_max_attempts < 1UL) {
+        stonith_max_attempts = 10UL;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Configure reaction to notification of local node being fenced
+ *
+ * \param[in] reaction_s  Reaction type
+ */
+static void
+set_fence_reaction(const char *reaction_s)
+{
+    if (pcmk__str_eq(reaction_s, "panic", pcmk__str_casei)) {
+        fence_reaction_panic = true;
+
+    } else {
+        if (!pcmk__str_eq(reaction_s, "stop", pcmk__str_casei)) {
+            crm_warn("Invalid value '%s' for %s, using 'stop'",
+                     reaction_s, XML_CONFIG_ATTR_FENCE_REACTION);
+        }
+        fence_reaction_panic = false;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Configure fencing options based on the CIB
+ *
+ * \param[in,out] options  Name/value pairs for configured options
+ */
+void
+controld_configure_fencing(GHashTable *options)
+{
+    const char *value = NULL;
+
+    value = g_hash_table_lookup(options, XML_CONFIG_ATTR_FENCE_REACTION);
+    set_fence_reaction(value);
+
+    value = g_hash_table_lookup(options, "stonith-max-attempts");
+    update_stonith_max_attempts(value);
+}
+
+static gboolean
+too_many_st_failures(const char *target)
+{
+    GHashTableIter iter;
+    const char *key = NULL;
+    struct st_fail_rec *value = NULL;
+
+    if (stonith_failures == NULL) {
+        return FALSE;
+    }
+
+    if (target == NULL) {
+        g_hash_table_iter_init(&iter, stonith_failures);
+        while (g_hash_table_iter_next(&iter, (gpointer *) &key,
+               (gpointer *) &value)) {
+
+            if (value->count >= stonith_max_attempts) {
+                target = (const char*)key;
+                goto too_many;
+            }
+        }
+    } else {
+        value = g_hash_table_lookup(stonith_failures, target);
+        if ((value != NULL) && (value->count >= stonith_max_attempts)) {
+            goto too_many;
+        }
+    }
+    return FALSE;
+
+too_many:
+    crm_warn("Too many failures (%d) to fence %s, giving up",
+             value->count, target);
+    return TRUE;
+}
+
+/*!
+ * \internal
+ * \brief Reset a stonith fail count
+ *
+ * \param[in] target  Name of node to reset, or NULL for all
+ */
+void
+st_fail_count_reset(const char *target)
+{
+    if (stonith_failures == NULL) {
+        return;
+    }
+
+    if (target) {
+        struct st_fail_rec *rec = NULL;
+
+        rec = g_hash_table_lookup(stonith_failures, target);
+        if (rec) {
+            rec->count = 0;
+        }
+    } else {
+        GHashTableIter iter;
+        const char *key = NULL;
+        struct st_fail_rec *rec = NULL;
+
+        g_hash_table_iter_init(&iter, stonith_failures);
+        while (g_hash_table_iter_next(&iter, (gpointer *) &key,
+                                      (gpointer *) &rec)) {
+            rec->count = 0;
+        }
+    }
+}
+
+static void
+st_fail_count_increment(const char *target)
+{
+    struct st_fail_rec *rec = NULL;
+
+    if (stonith_failures == NULL) {
+        stonith_failures = pcmk__strkey_table(free, free);
+    }
+
+    rec = g_hash_table_lookup(stonith_failures, target);
+    if (rec) {
+        rec->count++;
+    } else {
+        rec = malloc(sizeof(struct st_fail_rec));
+        if(rec == NULL) {
+            return;
+        }
+
+        rec->count = 1;
+        g_hash_table_insert(stonith_failures, strdup(target), rec);
+    }
+}
+
+/* end stonith fail count functions */
+
+
+static void
+cib_fencing_updated(xmlNode *msg, int call_id, int rc, xmlNode *output,
+                    void *user_data)
+{
+    if (rc < pcmk_ok) {
+        crm_err("Fencing update %d for %s: failed - %s (%d)",
+                call_id, (char *)user_data, pcmk_strerror(rc), rc);
+        crm_log_xml_warn(msg, "Failed update");
+        abort_transition(INFINITY, pcmk__graph_shutdown, "CIB update failed",
+                         NULL);
+
+    } else {
+        crm_info("Fencing update %d for %s: complete", call_id, (char *)user_data);
+    }
+}
+
+static void
+send_stonith_update(pcmk__graph_action_t *action, const char *target,
+                    const char *uuid)
+{
+    int rc = pcmk_ok;
+    crm_node_t *peer = NULL;
+
+    /* We (usually) rely on the membership layer to do node_update_cluster,
+     * and the peer status callback to do node_update_peer, because the node
+     * might have already rejoined before we get the stonith result here.
+     */
+    int flags = node_update_join | node_update_expected;
+
+    /* zero out the node-status & remove all LRM status info */
+    xmlNode *node_state = NULL;
+
+    CRM_CHECK(target != NULL, return);
+    CRM_CHECK(uuid != NULL, return);
+
+    /* Make sure the membership and join caches are accurate */
+    peer = crm_get_peer_full(0, target, CRM_GET_PEER_ANY);
+
+    CRM_CHECK(peer != NULL, return);
+
+    if (peer->state == NULL) {
+        /* Usually, we rely on the membership layer to update the cluster state
+         * in the CIB. However, if the node has never been seen, do it here, so
+         * the node is not considered unclean.
+         */
+        flags |= node_update_cluster;
+    }
+
+    if (peer->uuid == NULL) {
+        crm_info("Recording uuid '%s' for node '%s'", uuid, target);
+        peer->uuid = strdup(uuid);
+    }
+
+    crmd_peer_down(peer, TRUE);
+
+    /* Generate a node state update for the CIB */
+    node_state = create_node_state_update(peer, flags, NULL, __func__);
+
+    /* we have to mark whether or not remote nodes have already been fenced */
+    if (peer->flags & crm_remote_node) {
+        char *now_s = pcmk__ttoa(time(NULL));
+
+        crm_xml_add(node_state, XML_NODE_IS_FENCED, now_s);
+        free(now_s);
+    }
+
+    /* Force our known ID */
+    crm_xml_add(node_state, XML_ATTR_ID, uuid);
+
+    rc = controld_globals.cib_conn->cmds->modify(controld_globals.cib_conn,
+                                                 XML_CIB_TAG_STATUS, node_state,
+                                                 cib_scope_local
+                                                 |cib_can_create);
+
+    /* Delay processing the trigger until the update completes */
+    crm_debug("Sending fencing update %d for %s", rc, target);
+    fsa_register_cib_callback(rc, strdup(target), cib_fencing_updated);
+
+    // Make sure it sticks
+    /* controld_globals.cib_conn->cmds->bump_epoch(controld_globals.cib_conn,
+     *                                             cib_scope_local);
+     */
+
+    controld_delete_node_state(peer->uname, controld_section_all,
+                               cib_scope_local);
+    free_xml(node_state);
+    return;
+}
+
+/*!
+ * \internal
+ * \brief Abort transition due to stonith failure
+ *
+ * \param[in] abort_action  Whether to restart or stop transition
+ * \param[in] target  Don't restart if this (NULL for any) has too many failures
+ * \param[in] reason  Log this stonith action XML as abort reason (or NULL)
+ */
+static void
+abort_for_stonith_failure(enum pcmk__graph_next abort_action,
+                          const char *target, const xmlNode *reason)
+{
+    /* If stonith repeatedly fails, we eventually give up on starting a new
+     * transition for that reason.
+     */
+    if ((abort_action != pcmk__graph_wait) && too_many_st_failures(target)) {
+        abort_action = pcmk__graph_wait;
+    }
+    abort_transition(INFINITY, abort_action, "Stonith failed", reason);
+}
+
+
+/*
+ * stonith cleanup list
+ *
+ * If the DC is shot, proper notifications might not go out.
+ * The stonith cleanup list allows the cluster to (re-)send
+ * notifications once a new DC is elected.
+ */
+
+static GList *stonith_cleanup_list = NULL;
+
+/*!
+ * \internal
+ * \brief Add a node to the stonith cleanup list
+ *
+ * \param[in] target  Name of node to add
+ */
+void
+add_stonith_cleanup(const char *target) {
+    stonith_cleanup_list = g_list_append(stonith_cleanup_list, strdup(target));
+}
+
+/*!
+ * \internal
+ * \brief Remove a node from the stonith cleanup list
+ *
+ * \param[in] Name of node to remove
+ */
+void
+remove_stonith_cleanup(const char *target)
+{
+    GList *iter = stonith_cleanup_list;
+
+    while (iter != NULL) {
+        GList *tmp = iter;
+        char *iter_name = tmp->data;
+
+        iter = iter->next;
+        if (pcmk__str_eq(target, iter_name, pcmk__str_casei)) {
+            crm_trace("Removing %s from the cleanup list", iter_name);
+            stonith_cleanup_list = g_list_delete_link(stonith_cleanup_list, tmp);
+            free(iter_name);
+        }
+    }
+}
+
+/*!
+ * \internal
+ * \brief Purge all entries from the stonith cleanup list
+ */
+void
+purge_stonith_cleanup(void)
+{
+    if (stonith_cleanup_list) {
+        GList *iter = NULL;
+
+        for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) {
+            char *target = iter->data;
+
+            crm_info("Purging %s from stonith cleanup list", target);
+            free(target);
+        }
+        g_list_free(stonith_cleanup_list);
+        stonith_cleanup_list = NULL;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Send stonith updates for all entries in cleanup list, then purge it
+ */
+void
+execute_stonith_cleanup(void)
+{
+    GList *iter;
+
+    for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) {
+        char *target = iter->data;
+        crm_node_t *target_node = crm_get_peer(0, target);
+        const char *uuid = crm_peer_uuid(target_node);
+
+        crm_notice("Marking %s, target of a previous stonith action, as clean", target);
+        send_stonith_update(NULL, target, uuid);
+        free(target);
+    }
+    g_list_free(stonith_cleanup_list);
+    stonith_cleanup_list = NULL;
+}
+
+/* end stonith cleanup list functions */
+
+
+/* stonith API client
+ *
+ * Functions that need to interact directly with the fencer via its API
+ */
+
+static stonith_t *stonith_api = NULL;
+static crm_trigger_t *stonith_reconnect = NULL;
+static char *te_client_id = NULL;
+
+static gboolean
+fail_incompletable_stonith(pcmk__graph_t *graph)
+{
+    GList *lpc = NULL;
+    const char *task = NULL;
+    xmlNode *last_action = NULL;
+
+    if (graph == NULL) {
+        return FALSE;
+    }
+
+    for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
+        GList *lpc2 = NULL;
+        pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) lpc->data;
+
+        if (pcmk_is_set(synapse->flags, pcmk__synapse_confirmed)) {
+            continue;
+        }
+
+        for (lpc2 = synapse->actions; lpc2 != NULL; lpc2 = lpc2->next) {
+            pcmk__graph_action_t *action = (pcmk__graph_action_t *) lpc2->data;
+
+            if ((action->type != pcmk__cluster_graph_action)
+                || pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
+                continue;
+            }
+
+            task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
+            if (task && pcmk__str_eq(task, CRM_OP_FENCE, pcmk__str_casei)) {
+                pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
+                last_action = action->xml;
+                pcmk__update_graph(graph, action);
+                crm_notice("Failing action %d (%s): fencer terminated",
+                           action->id, ID(action->xml));
+            }
+        }
+    }
+
+    if (last_action != NULL) {
+        crm_warn("Fencer failure resulted in unrunnable actions");
+        abort_for_stonith_failure(pcmk__graph_restart, NULL, last_action);
+        return TRUE;
+    }
+
+    return FALSE;
+}
+
+static void
+tengine_stonith_connection_destroy(stonith_t *st, stonith_event_t *e)
+{
+    te_cleanup_stonith_history_sync(st, FALSE);
+
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_ST_REQUIRED)) {
+        crm_crit("Fencing daemon connection failed");
+        mainloop_set_trigger(stonith_reconnect);
+
+    } else {
+        crm_info("Fencing daemon disconnected");
+    }
+
+    if (stonith_api) {
+        /* the client API won't properly reconnect notifications
+         * if they are still in the table - so remove them
+         */
+        if (stonith_api->state != stonith_disconnected) {
+            stonith_api->cmds->disconnect(st);
+        }
+        stonith_api->cmds->remove_notification(stonith_api, NULL);
+    }
+
+    if (AM_I_DC) {
+        fail_incompletable_stonith(controld_globals.transition_graph);
+        trigger_graph();
+    }
+}
+
+/*!
+ * \internal
+ * \brief Handle an event notification from the fencing API
+ *
+ * \param[in] st     Fencing API connection (ignored)
+ * \param[in] event  Fencing API event notification
+ */
+static void
+handle_fence_notification(stonith_t *st, stonith_event_t *event)
+{
+    bool succeeded = true;
+    const char *executioner = "the cluster";
+    const char *client = "a client";
+    const char *reason = NULL;
+    int exec_status;
+
+    if (te_client_id == NULL) {
+        te_client_id = crm_strdup_printf("%s.%lu", crm_system_name,
+                                         (unsigned long) getpid());
+    }
+
+    if (event == NULL) {
+        crm_err("Notify data not found");
+        return;
+    }
+
+    if (event->executioner != NULL) {
+        executioner = event->executioner;
+    }
+    if (event->client_origin != NULL) {
+        client = event->client_origin;
+    }
+
+    exec_status = stonith__event_execution_status(event);
+    if ((stonith__event_exit_status(event) != CRM_EX_OK)
+        || (exec_status != PCMK_EXEC_DONE)) {
+        succeeded = false;
+        if (exec_status == PCMK_EXEC_DONE) {
+            exec_status = PCMK_EXEC_ERROR;
+        }
+    }
+    reason = stonith__event_exit_reason(event);
+
+    crmd_alert_fencing_op(event);
+
+    if (pcmk__str_eq("on", event->action, pcmk__str_none)) {
+        // Unfencing doesn't need special handling, just a log message
+        if (succeeded) {
+            crm_notice("%s was unfenced by %s at the request of %s@%s",
+                       event->target, executioner, client, event->origin);
+        } else {
+            crm_err("Unfencing of %s by %s failed (%s%s%s) with exit status %d",
+                    event->target, executioner,
+                    pcmk_exec_status_str(exec_status),
+                    ((reason == NULL)? "" : ": "),
+                    ((reason == NULL)? "" : reason),
+                    stonith__event_exit_status(event));
+        }
+        return;
+    }
+
+    if (succeeded
+        && pcmk__str_eq(event->target, controld_globals.our_nodename,
+                        pcmk__str_casei)) {
+        /* We were notified of our own fencing. Most likely, either fencing was
+         * misconfigured, or fabric fencing that doesn't cut cluster
+         * communication is in use.
+         *
+         * Either way, shutting down the local host is a good idea, to require
+         * administrator intervention. Also, other nodes would otherwise likely
+         * set our status to lost because of the fencing callback and discard
+         * our subsequent election votes as "not part of our cluster".
+         */
+        crm_crit("We were allegedly just fenced by %s for %s!",
+                 executioner, event->origin); // Dumps blackbox if enabled
+        if (fence_reaction_panic) {
+            pcmk__panic(__func__);
+        } else {
+            crm_exit(CRM_EX_FATAL);
+        }
+        return; // Should never get here
+    }
+
+    /* Update the count of fencing failures for this target, in case we become
+     * DC later. The current DC has already updated its fail count in
+     * tengine_stonith_callback().
+     */
+    if (!AM_I_DC) {
+        if (succeeded) {
+            st_fail_count_reset(event->target);
+        } else {
+            st_fail_count_increment(event->target);
+        }
+    }
+
+    crm_notice("Peer %s was%s terminated (%s) by %s on behalf of %s@%s: "
+               "%s%s%s%s " CRM_XS " event=%s",
+               event->target, (succeeded? "" : " not"),
+               event->action, executioner, client, event->origin,
+               (succeeded? "OK" : pcmk_exec_status_str(exec_status)),
+               ((reason == NULL)? "" : " ("),
+               ((reason == NULL)? "" : reason),
+               ((reason == NULL)? "" : ")"),
+               event->id);
+
+    if (succeeded) {
+        crm_node_t *peer = pcmk__search_known_node_cache(0, event->target,
+                                                         CRM_GET_PEER_ANY);
+        const char *uuid = NULL;
+
+        if (peer == NULL) {
+            return;
+        }
+
+        uuid = crm_peer_uuid(peer);
+
+        if (AM_I_DC) {
+            /* The DC always sends updates */
+            send_stonith_update(NULL, event->target, uuid);
+
+            /* @TODO Ideally, at this point, we'd check whether the fenced node
+             * hosted any guest nodes, and call remote_node_down() for them.
+             * Unfortunately, the controller doesn't have a simple, reliable way
+             * to map hosts to guests. It might be possible to track this in the
+             * peer cache via crm_remote_peer_cache_refresh(). For now, we rely
+             * on the scheduler creating fence pseudo-events for the guests.
+             */
+
+            if (!pcmk__str_eq(client, te_client_id, pcmk__str_casei)) {
+                /* Abort the current transition if it wasn't the cluster that
+                 * initiated fencing.
+                 */
+                crm_info("External fencing operation from %s fenced %s",
+                         client, event->target);
+                abort_transition(INFINITY, pcmk__graph_restart,
+                                 "External Fencing Operation", NULL);
+            }
+
+        } else if (pcmk__str_eq(controld_globals.dc_name, event->target,
+                                pcmk__str_null_matches|pcmk__str_casei)
+                   && !pcmk_is_set(peer->flags, crm_remote_node)) {
+            // Assume the target was our DC if we don't currently have one
+
+            if (controld_globals.dc_name != NULL) {
+                crm_notice("Fencing target %s was our DC", event->target);
+            } else {
+                crm_notice("Fencing target %s may have been our DC",
+                           event->target);
+            }
+
+            /* Given the CIB resyncing that occurs around elections,
+             * have one node update the CIB now and, if the new DC is different,
+             * have them do so too after the election
+             */
+            if (pcmk__str_eq(event->executioner, controld_globals.our_nodename,
+                             pcmk__str_casei)) {
+                send_stonith_update(NULL, event->target, uuid);
+            }
+            add_stonith_cleanup(event->target);
+        }
+
+        /* If the target is a remote node, and we host its connection,
+         * immediately fail all monitors so it can be recovered quickly.
+         * The connection won't necessarily drop when a remote node is fenced,
+         * so the failure might not otherwise be detected until the next poke.
+         */
+        if (pcmk_is_set(peer->flags, crm_remote_node)) {
+            remote_ra_fail(event->target);
+        }
+
+        crmd_peer_down(peer, TRUE);
+     }
+}
+
+/*!
+ * \brief Connect to fencer
+ *
+ * \param[in] user_data  If NULL, retry failures now, otherwise retry in main loop
+ *
+ * \return TRUE
+ * \note If user_data is NULL, this will wait 2s between attempts, for up to
+ *       30 attempts, meaning the controller could be blocked as long as 58s.
+ */
+static gboolean
+te_connect_stonith(gpointer user_data)
+{
+    int rc = pcmk_ok;
+
+    if (stonith_api == NULL) {
+        stonith_api = stonith_api_new();
+        if (stonith_api == NULL) {
+            crm_err("Could not connect to fencer: API memory allocation failed");
+            return TRUE;
+        }
+    }
+
+    if (stonith_api->state != stonith_disconnected) {
+        crm_trace("Already connected to fencer, no need to retry");
+        return TRUE;
+    }
+
+    if (user_data == NULL) {
+        // Blocking (retry failures now until successful)
+        rc = stonith_api_connect_retry(stonith_api, crm_system_name, 30);
+        if (rc != pcmk_ok) {
+            crm_err("Could not connect to fencer in 30 attempts: %s "
+                    CRM_XS " rc=%d", pcmk_strerror(rc), rc);
+        }
+    } else {
+        // Non-blocking (retry failures later in main loop)
+        rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL);
+        if (rc != pcmk_ok) {
+            if (pcmk_is_set(controld_globals.fsa_input_register,
+                            R_ST_REQUIRED)) {
+                crm_notice("Fencer connection failed (will retry): %s "
+                           CRM_XS " rc=%d", pcmk_strerror(rc), rc);
+                mainloop_set_trigger(stonith_reconnect);
+            } else {
+                crm_info("Fencer connection failed (ignoring because no longer required): %s "
+                         CRM_XS " rc=%d", pcmk_strerror(rc), rc);
+            }
+            return TRUE;
+        }
+    }
+
+    if (rc == pcmk_ok) {
+        stonith_api->cmds->register_notification(stonith_api,
+                                                 T_STONITH_NOTIFY_DISCONNECT,
+                                                 tengine_stonith_connection_destroy);
+        stonith_api->cmds->register_notification(stonith_api,
+                                                 T_STONITH_NOTIFY_FENCE,
+                                                 handle_fence_notification);
+        stonith_api->cmds->register_notification(stonith_api,
+                                                 T_STONITH_NOTIFY_HISTORY_SYNCED,
+                                                 tengine_stonith_history_synced);
+        te_trigger_stonith_history_sync(TRUE);
+        crm_notice("Fencer successfully connected");
+    }
+
+    return TRUE;
+}
+
+/*!
+    \internal
+    \brief Schedule fencer connection attempt in main loop
+*/
+void
+controld_trigger_fencer_connect(void)
+{
+    if (stonith_reconnect == NULL) {
+        stonith_reconnect = mainloop_add_trigger(G_PRIORITY_LOW,
+                                                 te_connect_stonith,
+                                                 GINT_TO_POINTER(TRUE));
+    }
+    controld_set_fsa_input_flags(R_ST_REQUIRED);
+    mainloop_set_trigger(stonith_reconnect);
+}
+
+void
+controld_disconnect_fencer(bool destroy)
+{
+    if (stonith_api) {
+        // Prevent fencer connection from coming up again
+        controld_clear_fsa_input_flags(R_ST_REQUIRED);
+
+        if (stonith_api->state != stonith_disconnected) {
+            stonith_api->cmds->disconnect(stonith_api);
+        }
+        stonith_api->cmds->remove_notification(stonith_api, NULL);
+    }
+    if (destroy) {
+        if (stonith_api) {
+            stonith_api->cmds->free(stonith_api);
+            stonith_api = NULL;
+        }
+        if (stonith_reconnect) {
+            mainloop_destroy_trigger(stonith_reconnect);
+            stonith_reconnect = NULL;
+        }
+        if (te_client_id) {
+            free(te_client_id);
+            te_client_id = NULL;
+        }
+    }
+}
+
+static gboolean
+do_stonith_history_sync(gpointer user_data)
+{
+    if (stonith_api && (stonith_api->state != stonith_disconnected)) {
+        stonith_history_t *history = NULL;
+
+        te_cleanup_stonith_history_sync(stonith_api, FALSE);
+        stonith_api->cmds->history(stonith_api,
+                                   st_opt_sync_call | st_opt_broadcast,
+                                   NULL, &history, 5);
+        stonith_history_free(history);
+        return TRUE;
+    } else {
+        crm_info("Skip triggering stonith history-sync as stonith is disconnected");
+        return FALSE;
+    }
+}
+
+static void
+tengine_stonith_callback(stonith_t *stonith, stonith_callback_data_t *data)
+{
+    char *uuid = NULL;
+    int stonith_id = -1;
+    int transition_id = -1;
+    pcmk__graph_action_t *action = NULL;
+    const char *target = NULL;
+
+    if ((data == NULL) || (data->userdata == NULL)) {
+        crm_err("Ignoring fence operation %d result: "
+                "No transition key given (bug?)",
+                ((data == NULL)? -1 : data->call_id));
+        return;
+    }
+
+    if (!AM_I_DC) {
+        const char *reason = stonith__exit_reason(data);
+
+        if (reason == NULL) {
+           reason = pcmk_exec_status_str(stonith__execution_status(data));
+        }
+        crm_notice("Result of fence operation %d: %d (%s) " CRM_XS " key=%s",
+                   data->call_id, stonith__exit_status(data), reason,
+                   (const char *) data->userdata);
+        return;
+    }
+
+    CRM_CHECK(decode_transition_key(data->userdata, &uuid, &transition_id,
+                                    &stonith_id, NULL),
+              goto bail);
+
+    if (controld_globals.transition_graph->complete || (stonith_id < 0)
+        || !pcmk__str_eq(uuid, controld_globals.te_uuid, pcmk__str_none)
+        || (controld_globals.transition_graph->id != transition_id)) {
+        crm_info("Ignoring fence operation %d result: "
+                 "Not from current transition " CRM_XS
+                 " complete=%s action=%d uuid=%s (vs %s) transition=%d (vs %d)",
+                 data->call_id,
+                 pcmk__btoa(controld_globals.transition_graph->complete),
+                 stonith_id, uuid, controld_globals.te_uuid, transition_id,
+                 controld_globals.transition_graph->id);
+        goto bail;
+    }
+
+    action = controld_get_action(stonith_id);
+    if (action == NULL) {
+        crm_err("Ignoring fence operation %d result: "
+                "Action %d not found in transition graph (bug?) "
+                CRM_XS " uuid=%s transition=%d",
+                data->call_id, stonith_id, uuid, transition_id);
+        goto bail;
+    }
+
+    target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
+    if (target == NULL) {
+        crm_err("Ignoring fence operation %d result: No target given (bug?)",
+                data->call_id);
+        goto bail;
+    }
+
+    stop_te_timer(action);
+    if (stonith__exit_status(data) == CRM_EX_OK) {
+        const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
+        const char *op = crm_meta_value(action->params, "stonith_action");
+
+        crm_info("Fence operation %d for %s succeeded", data->call_id, target);
+        if (!(pcmk_is_set(action->flags, pcmk__graph_action_confirmed))) {
+            te_action_confirmed(action, NULL);
+            if (pcmk__str_eq("on", op, pcmk__str_casei)) {
+                const char *value = NULL;
+                char *now = pcmk__ttoa(time(NULL));
+                gboolean is_remote_node = FALSE;
+
+                /* This check is not 100% reliable, since this node is not
+                 * guaranteed to have the remote node cached. However, it
+                 * doesn't have to be reliable, since the attribute manager can
+                 * learn a node's "remoteness" by other means sooner or later.
+                 * This allows it to learn more quickly if this node does have
+                 * the information.
+                 */
+                if (g_hash_table_lookup(crm_remote_peer_cache, uuid) != NULL) {
+                    is_remote_node = TRUE;
+                }
+
+                update_attrd(target, CRM_ATTR_UNFENCED, now, NULL,
+                             is_remote_node);
+                free(now);
+
+                value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_ALL);
+                update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL,
+                             is_remote_node);
+
+                value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_SECURE);
+                update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL,
+                             is_remote_node);
+
+            } else if (!(pcmk_is_set(action->flags, pcmk__graph_action_sent_update))) {
+                send_stonith_update(action, target, uuid);
+                pcmk__set_graph_action_flags(action,
+                                             pcmk__graph_action_sent_update);
+            }
+        }
+        st_fail_count_reset(target);
+
+    } else {
+        enum pcmk__graph_next abort_action = pcmk__graph_restart;
+        int status = stonith__execution_status(data);
+        const char *reason = stonith__exit_reason(data);
+
+        if (reason == NULL) {
+            if (status == PCMK_EXEC_DONE) {
+                reason = "Agent returned error";
+            } else {
+                reason = pcmk_exec_status_str(status);
+            }
+        }
+        pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
+
+        /* If no fence devices were available, there's no use in immediately
+         * checking again, so don't start a new transition in that case.
+         */
+        if (status == PCMK_EXEC_NO_FENCE_DEVICE) {
+            crm_warn("Fence operation %d for %s failed: %s "
+                     "(aborting transition and giving up for now)",
+                     data->call_id, target, reason);
+            abort_action = pcmk__graph_wait;
+        } else {
+            crm_notice("Fence operation %d for %s failed: %s "
+                       "(aborting transition)", data->call_id, target, reason);
+        }
+
+        /* Increment the fail count now, so abort_for_stonith_failure() can
+         * check it. Non-DC nodes will increment it in
+         * handle_fence_notification().
+         */
+        st_fail_count_increment(target);
+        abort_for_stonith_failure(abort_action, target, NULL);
+    }
+
+    pcmk__update_graph(controld_globals.transition_graph, action);
+    trigger_graph();
+
+  bail:
+    free(data->userdata);
+    free(uuid);
+    return;
+}
+
+static int
+fence_with_delay(const char *target, const char *type, int delay)
+{
+    uint32_t options = st_opt_none; // Group of enum stonith_call_options
+    int timeout_sec = (int) (controld_globals.transition_graph->stonith_timeout
+                             / 1000);
+
+    if (crmd_join_phase_count(crm_join_confirmed) == 1) {
+        stonith__set_call_options(options, target, st_opt_allow_suicide);
+    }
+    return stonith_api->cmds->fence_with_delay(stonith_api, options, target,
+                                               type, timeout_sec, 0, delay);
+}
+
+/*!
+ * \internal
+ * \brief Execute a fencing action from a transition graph
+ *
+ * \param[in] graph   Transition graph being executed (ignored)
+ * \param[in] action  Fencing action to execute
+ *
+ * \return Standard Pacemaker return code
+ */
+int
+controld_execute_fence_action(pcmk__graph_t *graph,
+                              pcmk__graph_action_t *action)
+{
+    int rc = 0;
+    const char *id = ID(action->xml);
+    const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
+    const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
+    const char *type = crm_meta_value(action->params, "stonith_action");
+    char *transition_key = NULL;
+    const char *priority_delay = NULL;
+    int delay_i = 0;
+    gboolean invalid_action = FALSE;
+    int stonith_timeout = (int) (controld_globals.transition_graph->stonith_timeout
+                                 / 1000);
+
+    CRM_CHECK(id != NULL, invalid_action = TRUE);
+    CRM_CHECK(uuid != NULL, invalid_action = TRUE);
+    CRM_CHECK(type != NULL, invalid_action = TRUE);
+    CRM_CHECK(target != NULL, invalid_action = TRUE);
+
+    if (invalid_action) {
+        crm_log_xml_warn(action->xml, "BadAction");
+        return EPROTO;
+    }
+
+    priority_delay = crm_meta_value(action->params, XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);
+
+    crm_notice("Requesting fencing (%s) targeting node %s "
+               CRM_XS " action=%s timeout=%i%s%s",
+               type, target, id, stonith_timeout,
+               priority_delay ? " priority_delay=" : "",
+               priority_delay ? priority_delay : "");
+
+    /* Passing NULL means block until we can connect... */
+    te_connect_stonith(NULL);
+
+    pcmk__scan_min_int(priority_delay, &delay_i, 0);
+    rc = fence_with_delay(target, type, delay_i);
+    transition_key = pcmk__transition_key(controld_globals.transition_graph->id,
+                                          action->id, 0,
+                                          controld_globals.te_uuid),
+    stonith_api->cmds->register_callback(stonith_api, rc,
+                                         (stonith_timeout
+                                          + (delay_i > 0 ? delay_i : 0)),
+                                         st_opt_timeout_updates, transition_key,
+                                         "tengine_stonith_callback",
+                                         tengine_stonith_callback);
+    return pcmk_rc_ok;
+}
+
+bool
+controld_verify_stonith_watchdog_timeout(const char *value)
+{
+    const char *our_nodename = controld_globals.our_nodename;
+    gboolean rv = TRUE;
+
+    if (stonith_api && (stonith_api->state != stonith_disconnected) &&
+        stonith__watchdog_fencing_enabled_for_node_api(stonith_api,
+                                                       our_nodename)) {
+        rv = pcmk__valid_sbd_timeout(value);
+    }
+    return rv;
+}
+
+/* end stonith API client functions */
+
+
+/*
+ * stonith history synchronization
+ *
+ * Each node's fencer keeps track of a cluster-wide fencing history. When a node
+ * joins or leaves, we need to synchronize the history across all nodes.
+ */
+
+static crm_trigger_t *stonith_history_sync_trigger = NULL;
+static mainloop_timer_t *stonith_history_sync_timer_short = NULL;
+static mainloop_timer_t *stonith_history_sync_timer_long = NULL;
+
+void
+te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers)
+{
+    if (free_timers) {
+        mainloop_timer_del(stonith_history_sync_timer_short);
+        stonith_history_sync_timer_short = NULL;
+        mainloop_timer_del(stonith_history_sync_timer_long);
+        stonith_history_sync_timer_long = NULL;
+    } else {
+        mainloop_timer_stop(stonith_history_sync_timer_short);
+        mainloop_timer_stop(stonith_history_sync_timer_long);
+    }
+
+    if (st) {
+        st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY_SYNCED);
+    }
+}
+
+static void
+tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event)
+{
+    te_cleanup_stonith_history_sync(st, FALSE);
+    crm_debug("Fence-history synced - cancel all timers");
+}
+
+static gboolean
+stonith_history_sync_set_trigger(gpointer user_data)
+{
+    mainloop_set_trigger(stonith_history_sync_trigger);
+    return FALSE;
+}
+
+void
+te_trigger_stonith_history_sync(bool long_timeout)
+{
+    /* trigger a sync in 5s to give more nodes the
+     * chance to show up so that we don't create
+     * unnecessary stonith-history-sync traffic
+     *
+     * the long timeout of 30s is there as a fallback
+     * so that after a successful connection to fenced
+     * we will wait for 30s for the DC to trigger a
+     * history-sync
+     * if this doesn't happen we trigger a sync locally
+     * (e.g. fenced segfaults and is restarted by pacemakerd)
+     */
+
+    /* as we are finally checking the stonith-connection
+     * in do_stonith_history_sync we should be fine
+     * leaving stonith_history_sync_time & stonith_history_sync_trigger
+     * around
+     */
+    if (stonith_history_sync_trigger == NULL) {
+        stonith_history_sync_trigger =
+            mainloop_add_trigger(G_PRIORITY_LOW,
+                                 do_stonith_history_sync, NULL);
+    }
+
+    if (long_timeout) {
+        if(stonith_history_sync_timer_long == NULL) {
+            stonith_history_sync_timer_long =
+                mainloop_timer_add("history_sync_long", 30000,
+                                   FALSE, stonith_history_sync_set_trigger,
+                                   NULL);
+        }
+        crm_info("Fence history will be synchronized cluster-wide within 30 seconds");
+        mainloop_timer_start(stonith_history_sync_timer_long);
+    } else {
+        if(stonith_history_sync_timer_short == NULL) {
+            stonith_history_sync_timer_short =
+                mainloop_timer_add("history_sync_short", 5000,
+                                   FALSE, stonith_history_sync_set_trigger,
+                                   NULL);
+        }
+        crm_info("Fence history will be synchronized cluster-wide within 5 seconds");
+        mainloop_timer_start(stonith_history_sync_timer_short);
+    }
+
+}
+
+/* end stonith history synchronization functions */
diff --git a/daemons/controld/controld_fencing.h b/daemons/controld/controld_fencing.h
new file mode 100644
index 0000000..86a5050
--- /dev/null
+++ b/daemons/controld/controld_fencing.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef CONTROLD_FENCING__H
+#  define CONTROLD_FENCING__H
+
+#include <stdbool.h>                // bool
+#include <pacemaker-internal.h>     // pcmk__graph_t, pcmk__graph_action_t
+
+void controld_configure_fencing(GHashTable *options);
+
+// stonith fail counts
+void st_fail_count_reset(const char * target);
+
+// stonith API client
+void controld_trigger_fencer_connect(void);
+void controld_disconnect_fencer(bool destroy);
+int controld_execute_fence_action(pcmk__graph_t *graph,
+                                  pcmk__graph_action_t *action);
+bool controld_verify_stonith_watchdog_timeout(const char *value);
+
+// stonith cleanup list
+void add_stonith_cleanup(const char *target);
+void remove_stonith_cleanup(const char *target);
+void purge_stonith_cleanup(void);
+void execute_stonith_cleanup(void);
+
+// stonith history synchronization
+void te_trigger_stonith_history_sync(bool long_timeout);
+void te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers);
+
+#endif
diff --git a/daemons/controld/controld_fsa.c b/daemons/controld/controld_fsa.c
new file mode 100644
index 0000000..622d1c8
--- /dev/null
+++ b/daemons/controld/controld_fsa.c
@@ -0,0 +1,741 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <stdint.h>                 // uint64_t
+#include <string.h>
+#include <time.h>
+
+#include <crm/crm.h>
+#include <crm/lrmd.h>
+#include <crm/cib.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/cluster/election_internal.h>
+#include <crm/cluster.h>
+
+#include <pacemaker-controld.h>
+
+//! Triggers an FSA invocation
+static crm_trigger_t *fsa_trigger = NULL;
+
+#define DOT_PREFIX "actions:trace: "
+#define do_dot_log(fmt, args...)     crm_trace( fmt, ##args)
+
+static void do_state_transition(enum crmd_fsa_state cur_state,
+                                enum crmd_fsa_state next_state,
+                                fsa_data_t *msg_data);
+
+void s_crmd_fsa_actions(fsa_data_t * fsa_data);
+void log_fsa_input(fsa_data_t * stored_msg);
+void init_dotfile(void);
+
+void
+init_dotfile(void)
+{
+    do_dot_log(DOT_PREFIX "digraph \"g\" {");
+    do_dot_log(DOT_PREFIX "	size = \"30,30\"");
+    do_dot_log(DOT_PREFIX "	graph [");
+    do_dot_log(DOT_PREFIX "		fontsize = \"12\"");
+    do_dot_log(DOT_PREFIX "		fontname = \"Times-Roman\"");
+    do_dot_log(DOT_PREFIX "		fontcolor = \"black\"");
+    do_dot_log(DOT_PREFIX "		bb = \"0,0,398.922306,478.927856\"");
+    do_dot_log(DOT_PREFIX "		color = \"black\"");
+    do_dot_log(DOT_PREFIX "	]");
+    do_dot_log(DOT_PREFIX "	node [");
+    do_dot_log(DOT_PREFIX "		fontsize = \"12\"");
+    do_dot_log(DOT_PREFIX "		fontname = \"Times-Roman\"");
+    do_dot_log(DOT_PREFIX "		fontcolor = \"black\"");
+    do_dot_log(DOT_PREFIX "		shape = \"ellipse\"");
+    do_dot_log(DOT_PREFIX "		color = \"black\"");
+    do_dot_log(DOT_PREFIX "	]");
+    do_dot_log(DOT_PREFIX "	edge [");
+    do_dot_log(DOT_PREFIX "		fontsize = \"12\"");
+    do_dot_log(DOT_PREFIX "		fontname = \"Times-Roman\"");
+    do_dot_log(DOT_PREFIX "		fontcolor = \"black\"");
+    do_dot_log(DOT_PREFIX "		color = \"black\"");
+    do_dot_log(DOT_PREFIX "	]");
+    do_dot_log(DOT_PREFIX "// special nodes");
+    do_dot_log(DOT_PREFIX "	\"S_PENDING\" ");
+    do_dot_log(DOT_PREFIX "	[");
+    do_dot_log(DOT_PREFIX "	 color = \"blue\"");
+    do_dot_log(DOT_PREFIX "	 fontcolor = \"blue\"");
+    do_dot_log(DOT_PREFIX "	 ]");
+    do_dot_log(DOT_PREFIX "	\"S_TERMINATE\" ");
+    do_dot_log(DOT_PREFIX "	[");
+    do_dot_log(DOT_PREFIX "	 color = \"red\"");
+    do_dot_log(DOT_PREFIX "	 fontcolor = \"red\"");
+    do_dot_log(DOT_PREFIX "	 ]");
+    do_dot_log(DOT_PREFIX "// DC only nodes");
+    do_dot_log(DOT_PREFIX "	\"S_INTEGRATION\" [ fontcolor = \"green\" ]");
+    do_dot_log(DOT_PREFIX "	\"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]");
+    do_dot_log(DOT_PREFIX "	\"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]");
+    do_dot_log(DOT_PREFIX "	\"S_RELEASE_DC\" [ fontcolor = \"green\" ]");
+    do_dot_log(DOT_PREFIX "	\"S_IDLE\" [ fontcolor = \"green\" ]");
+}
+
+static void
+do_fsa_action(fsa_data_t * fsa_data, long long an_action,
+              void (*function) (long long action,
+                                enum crmd_fsa_cause cause,
+                                enum crmd_fsa_state cur_state,
+                                enum crmd_fsa_input cur_input, fsa_data_t * msg_data))
+{
+    controld_clear_fsa_action_flags(an_action);
+    crm_trace(DOT_PREFIX "\t// %s", fsa_action2string(an_action));
+    function(an_action, fsa_data->fsa_cause, controld_globals.fsa_state,
+             fsa_data->fsa_input, fsa_data);
+}
+
+static const uint64_t startup_actions =
+    A_STARTUP | A_CIB_START | A_LRM_CONNECT | A_HA_CONNECT | A_READCONFIG |
+    A_STARTED | A_CL_JOIN_QUERY;
+
+// A_LOG, A_WARN, A_ERROR
+void
+do_log(long long action, enum crmd_fsa_cause cause,
+       enum crmd_fsa_state cur_state,
+       enum crmd_fsa_input current_input, fsa_data_t *msg_data)
+{
+    unsigned log_type = LOG_TRACE;
+
+    if (action & A_LOG) {
+        log_type = LOG_INFO;
+    } else if (action & A_WARN) {
+        log_type = LOG_WARNING;
+    } else if (action & A_ERROR) {
+        log_type = LOG_ERR;
+    }
+
+    do_crm_log(log_type, "Input %s received in state %s from %s",
+               fsa_input2string(msg_data->fsa_input),
+               fsa_state2string(cur_state), msg_data->origin);
+
+    if (msg_data->data_type == fsa_dt_ha_msg) {
+        ha_msg_input_t *input = fsa_typed_data(msg_data->data_type);
+
+        crm_log_xml_debug(input->msg, __func__);
+
+    } else if (msg_data->data_type == fsa_dt_xml) {
+        xmlNode *input = fsa_typed_data(msg_data->data_type);
+
+        crm_log_xml_debug(input, __func__);
+
+    } else if (msg_data->data_type == fsa_dt_lrm) {
+        lrmd_event_data_t *input = fsa_typed_data(msg_data->data_type);
+
+        do_crm_log(log_type,
+                   "Resource %s: Call ID %d returned %d (%d)."
+                   "  New status if rc=0: %s",
+                   input->rsc_id, input->call_id, input->rc,
+                   input->op_status, (char *)input->user_data);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Initialize the FSA trigger
+ */
+void
+controld_init_fsa_trigger(void)
+{
+    fsa_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL);
+}
+
+/*!
+ * \internal
+ * \brief Destroy the FSA trigger
+ */
+void
+controld_destroy_fsa_trigger(void)
+{
+    // This basically will not work, since mainloop has a reference to it
+    mainloop_destroy_trigger(fsa_trigger);
+    fsa_trigger = NULL;
+}
+
+/*!
+ * \internal
+ * \brief Trigger an FSA invocation
+ *
+ * \param[in] fn    Calling function name
+ * \param[in] line  Line number where call occurred
+ */
+void
+controld_trigger_fsa_as(const char *fn, int line)
+{
+    if (fsa_trigger != NULL) {
+        crm_trace("%s:%d - Triggered FSA invocation", fn, line);
+        mainloop_set_trigger(fsa_trigger);
+    }
+}
+
+enum crmd_fsa_state
+s_crmd_fsa(enum crmd_fsa_cause cause)
+{
+    controld_globals_t *globals = &controld_globals;
+    fsa_data_t *fsa_data = NULL;
+    uint64_t register_copy = controld_globals.fsa_input_register;
+    uint64_t new_actions = A_NOTHING;
+    enum crmd_fsa_state last_state;
+
+    crm_trace("FSA invoked with Cause: %s\tState: %s",
+              fsa_cause2string(cause),
+              fsa_state2string(globals->fsa_state));
+
+    fsa_dump_actions(controld_globals.fsa_actions, "Initial");
+
+    controld_clear_global_flags(controld_fsa_is_stalled);
+    if ((controld_globals.fsa_message_queue == NULL)
+        && (controld_globals.fsa_actions != A_NOTHING)) {
+        /* fake the first message so we can get into the loop */
+        fsa_data = calloc(1, sizeof(fsa_data_t));
+        fsa_data->fsa_input = I_NULL;
+        fsa_data->fsa_cause = C_FSA_INTERNAL;
+        fsa_data->origin = __func__;
+        fsa_data->data_type = fsa_dt_none;
+        controld_globals.fsa_message_queue
+            = g_list_append(controld_globals.fsa_message_queue, fsa_data);
+        fsa_data = NULL;
+    }
+    while ((controld_globals.fsa_message_queue != NULL)
+           && !pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
+        crm_trace("Checking messages (%d remaining)",
+                  g_list_length(controld_globals.fsa_message_queue));
+
+        fsa_data = get_message();
+        if(fsa_data == NULL) {
+            continue;
+        }
+
+        log_fsa_input(fsa_data);
+
+        /* add any actions back to the queue */
+        controld_set_fsa_action_flags(fsa_data->actions);
+        fsa_dump_actions(fsa_data->actions, "Restored actions");
+
+        /* get the next batch of actions */
+        new_actions = controld_fsa_get_action(fsa_data->fsa_input);
+        controld_set_fsa_action_flags(new_actions);
+        fsa_dump_actions(new_actions, "New actions");
+
+        if (fsa_data->fsa_input != I_NULL && fsa_data->fsa_input != I_ROUTER) {
+            crm_debug("Processing %s: [ state=%s cause=%s origin=%s ]",
+                      fsa_input2string(fsa_data->fsa_input),
+                      fsa_state2string(globals->fsa_state),
+                      fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
+        }
+
+        /* logging : *before* the state is changed */
+        if (pcmk_is_set(controld_globals.fsa_actions, A_ERROR)) {
+            do_fsa_action(fsa_data, A_ERROR, do_log);
+        }
+        if (pcmk_is_set(controld_globals.fsa_actions, A_WARN)) {
+            do_fsa_action(fsa_data, A_WARN, do_log);
+        }
+        if (pcmk_is_set(controld_globals.fsa_actions, A_LOG)) {
+            do_fsa_action(fsa_data, A_LOG, do_log);
+        }
+
+        /* update state variables */
+        last_state = globals->fsa_state;
+        globals->fsa_state = controld_fsa_get_next_state(fsa_data->fsa_input);
+
+        /*
+         * Remove certain actions during shutdown
+         */
+        if ((globals->fsa_state == S_STOPPING)
+            || pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+            controld_clear_fsa_action_flags(startup_actions);
+        }
+
+        /*
+         * Hook for change of state.
+         * Allows actions to be added or removed when entering a state
+         */
+        if (last_state != globals->fsa_state) {
+            do_state_transition(last_state, globals->fsa_state, fsa_data);
+        } else {
+            do_dot_log(DOT_PREFIX "\t// FSA input: State=%s \tCause=%s"
+                       " \tInput=%s \tOrigin=%s() \tid=%d",
+                       fsa_state2string(globals->fsa_state),
+                       fsa_cause2string(fsa_data->fsa_cause),
+                       fsa_input2string(fsa_data->fsa_input), fsa_data->origin, fsa_data->id);
+        }
+
+        /* start doing things... */
+        s_crmd_fsa_actions(fsa_data);
+        delete_fsa_input(fsa_data);
+        fsa_data = NULL;
+    }
+
+    if ((controld_globals.fsa_message_queue != NULL)
+        || (controld_globals.fsa_actions != A_NOTHING)
+        || pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
+
+        crm_debug("Exiting the FSA: queue=%d, fsa_actions=%#llx, stalled=%s",
+                  g_list_length(controld_globals.fsa_message_queue),
+                  (unsigned long long) controld_globals.fsa_actions,
+                  pcmk__btoa(pcmk_is_set(controld_globals.flags,
+                                         controld_fsa_is_stalled)));
+    } else {
+        crm_trace("Exiting the FSA");
+    }
+
+    /* cleanup inputs? */
+    if (register_copy != controld_globals.fsa_input_register) {
+        uint64_t same = register_copy & controld_globals.fsa_input_register;
+
+        fsa_dump_inputs(LOG_DEBUG, "Added",
+                        controld_globals.fsa_input_register ^ same);
+        fsa_dump_inputs(LOG_DEBUG, "Removed", register_copy ^ same);
+    }
+
+    fsa_dump_actions(controld_globals.fsa_actions, "Remaining");
+    fsa_dump_queue(LOG_DEBUG);
+
+    return globals->fsa_state;
+}
+
+void
+s_crmd_fsa_actions(fsa_data_t * fsa_data)
+{
+    /*
+     * Process actions in order of priority but do only one
+     * action at a time to avoid complicating the ordering.
+     */
+    CRM_CHECK(fsa_data != NULL, return);
+    while ((controld_globals.fsa_actions != A_NOTHING)
+           && !pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
+
+        /* regular action processing in order of action priority
+         *
+         * Make sure all actions that connect to required systems
+         * are performed first
+         */
+        if (pcmk_is_set(controld_globals.fsa_actions, A_ERROR)) {
+            do_fsa_action(fsa_data, A_ERROR, do_log);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_WARN)) {
+            do_fsa_action(fsa_data, A_WARN, do_log);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_LOG)) {
+            do_fsa_action(fsa_data, A_LOG, do_log);
+
+            /* get out of here NOW! before anything worse happens */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_EXIT_1)) {
+            do_fsa_action(fsa_data, A_EXIT_1, do_exit);
+
+            /* sub-system restart */
+        } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
+                                      O_LRM_RECONNECT)) {
+            do_fsa_action(fsa_data, O_LRM_RECONNECT, do_lrm_control);
+
+        } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
+                                      O_CIB_RESTART)) {
+            do_fsa_action(fsa_data, O_CIB_RESTART, do_cib_control);
+
+        } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
+                                      O_PE_RESTART)) {
+            do_fsa_action(fsa_data, O_PE_RESTART, do_pe_control);
+
+        } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
+                                      O_TE_RESTART)) {
+            do_fsa_action(fsa_data, O_TE_RESTART, do_te_control);
+
+            /* essential start tasks */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_STARTUP)) {
+            do_fsa_action(fsa_data, A_STARTUP, do_startup);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_CIB_START)) {
+            do_fsa_action(fsa_data, A_CIB_START, do_cib_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_HA_CONNECT)) {
+            do_fsa_action(fsa_data, A_HA_CONNECT, do_ha_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_READCONFIG)) {
+            do_fsa_action(fsa_data, A_READCONFIG, do_read_config);
+
+            /* sub-system start/connect */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_CONNECT)) {
+            do_fsa_action(fsa_data, A_LRM_CONNECT, do_lrm_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_START)) {
+            do_fsa_action(fsa_data, A_TE_START, do_te_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_START)) {
+            do_fsa_action(fsa_data, A_PE_START, do_pe_control);
+
+            /* Timers */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_TIMER_STOP)) {
+            do_fsa_action(fsa_data, A_DC_TIMER_STOP, do_timer_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_INTEGRATE_TIMER_STOP)) {
+            do_fsa_action(fsa_data, A_INTEGRATE_TIMER_STOP, do_timer_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_INTEGRATE_TIMER_START)) {
+            do_fsa_action(fsa_data, A_INTEGRATE_TIMER_START, do_timer_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_FINALIZE_TIMER_STOP)) {
+            do_fsa_action(fsa_data, A_FINALIZE_TIMER_STOP, do_timer_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_FINALIZE_TIMER_START)) {
+            do_fsa_action(fsa_data, A_FINALIZE_TIMER_START, do_timer_control);
+
+            /*
+             * Highest priority actions
+             */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_MSG_ROUTE)) {
+            do_fsa_action(fsa_data, A_MSG_ROUTE, do_msg_route);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_RECOVER)) {
+            do_fsa_action(fsa_data, A_RECOVER, do_recover);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_CL_JOIN_RESULT)) {
+            do_fsa_action(fsa_data, A_CL_JOIN_RESULT,
+                          do_cl_join_finalize_respond);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_CL_JOIN_REQUEST)) {
+            do_fsa_action(fsa_data, A_CL_JOIN_REQUEST,
+                          do_cl_join_offer_respond);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_SHUTDOWN_REQ)) {
+            do_fsa_action(fsa_data, A_SHUTDOWN_REQ, do_shutdown_req);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_ELECTION_VOTE)) {
+            do_fsa_action(fsa_data, A_ELECTION_VOTE, do_election_vote);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_ELECTION_COUNT)) {
+            do_fsa_action(fsa_data, A_ELECTION_COUNT, do_election_count_vote);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_EVENT)) {
+            do_fsa_action(fsa_data, A_LRM_EVENT, do_lrm_event);
+
+            /*
+             * High priority actions
+             */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_STARTED)) {
+            do_fsa_action(fsa_data, A_STARTED, do_started);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_CL_JOIN_QUERY)) {
+            do_fsa_action(fsa_data, A_CL_JOIN_QUERY, do_cl_join_query);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_DC_TIMER_START)) {
+            do_fsa_action(fsa_data, A_DC_TIMER_START, do_timer_control);
+
+            /*
+             * Medium priority actions
+             * - Membership
+             */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_TAKEOVER)) {
+            do_fsa_action(fsa_data, A_DC_TAKEOVER, do_dc_takeover);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_RELEASE)) {
+            do_fsa_action(fsa_data, A_DC_RELEASE, do_dc_release);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_JOIN_FINAL)) {
+            do_fsa_action(fsa_data, A_DC_JOIN_FINAL, do_dc_join_final);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_ELECTION_CHECK)) {
+            do_fsa_action(fsa_data, A_ELECTION_CHECK, do_election_check);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_ELECTION_START)) {
+            do_fsa_action(fsa_data, A_ELECTION_START, do_election_vote);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_DC_JOIN_OFFER_ALL)) {
+            do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_DC_JOIN_OFFER_ONE)) {
+            do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ONE, do_dc_join_offer_one);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_DC_JOIN_PROCESS_REQ)) {
+            do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_REQ,
+                          do_dc_join_filter_offer);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_DC_JOIN_PROCESS_ACK)) {
+            do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_ACK, do_dc_join_ack);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_DC_JOIN_FINALIZE)) {
+            do_fsa_action(fsa_data, A_DC_JOIN_FINALIZE, do_dc_join_finalize);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_CL_JOIN_ANNOUNCE)) {
+            do_fsa_action(fsa_data, A_CL_JOIN_ANNOUNCE, do_cl_join_announce);
+
+            /*
+             * Low(er) priority actions
+             * Make sure the CIB is always updated before invoking the
+             * scheduler, and the scheduler before the transition engine.
+             */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_HALT)) {
+            do_fsa_action(fsa_data, A_TE_HALT, do_te_invoke);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_CANCEL)) {
+            do_fsa_action(fsa_data, A_TE_CANCEL, do_te_invoke);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_INVOKE)) {
+            do_fsa_action(fsa_data, A_LRM_INVOKE, do_lrm_invoke);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_INVOKE)) {
+            do_fsa_action(fsa_data, A_PE_INVOKE, do_pe_invoke);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_INVOKE)) {
+            do_fsa_action(fsa_data, A_TE_INVOKE, do_te_invoke);
+
+            /* Shutdown actions */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_RELEASED)) {
+            do_fsa_action(fsa_data, A_DC_RELEASED, do_dc_release);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_STOP)) {
+            do_fsa_action(fsa_data, A_PE_STOP, do_pe_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_STOP)) {
+            do_fsa_action(fsa_data, A_TE_STOP, do_te_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_SHUTDOWN)) {
+            do_fsa_action(fsa_data, A_SHUTDOWN, do_shutdown);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_LRM_DISCONNECT)) {
+            do_fsa_action(fsa_data, A_LRM_DISCONNECT, do_lrm_control);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_HA_DISCONNECT)) {
+            do_fsa_action(fsa_data, A_HA_DISCONNECT, do_ha_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_CIB_STOP)) {
+            do_fsa_action(fsa_data, A_CIB_STOP, do_cib_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_STOP)) {
+            do_fsa_action(fsa_data, A_STOP, do_stop);
+
+            /* exit gracefully */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_EXIT_0)) {
+            do_fsa_action(fsa_data, A_EXIT_0, do_exit);
+
+            /* Error checking and reporting */
+        } else {
+            crm_err("Action %s not supported "CRM_XS" %#llx",
+                    fsa_action2string(controld_globals.fsa_actions),
+                    (unsigned long long) controld_globals.fsa_actions);
+            register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, fsa_data, NULL,
+                                   __func__);
+        }
+    }
+}
+
+void
+log_fsa_input(fsa_data_t * stored_msg)
+{
+    CRM_ASSERT(stored_msg);
+    crm_trace("Processing queued input %d", stored_msg->id);
+    if (stored_msg->fsa_cause == C_LRM_OP_CALLBACK) {
+        crm_trace("FSA processing LRM callback from %s", stored_msg->origin);
+
+    } else if (stored_msg->data == NULL) {
+        crm_trace("FSA processing input from %s", stored_msg->origin);
+
+    } else {
+        ha_msg_input_t *ha_input = fsa_typed_data_adv(stored_msg, fsa_dt_ha_msg,
+                                                      __func__);
+
+        crm_trace("FSA processing XML message from %s", stored_msg->origin);
+        crm_log_xml_trace(ha_input->xml, "FSA message data");
+    }
+}
+
+static void
+check_join_counts(fsa_data_t *msg_data)
+{
+    int count;
+    guint npeers;
+
+    count = crmd_join_phase_count(crm_join_finalized);
+    if (count > 0) {
+        crm_err("%d cluster node%s failed to confirm join",
+                count, pcmk__plural_s(count));
+        crmd_join_phase_log(LOG_NOTICE);
+        return;
+    }
+
+    npeers = crm_active_peers();
+    count = crmd_join_phase_count(crm_join_confirmed);
+    if (count == npeers) {
+        if (npeers == 1) {
+            crm_debug("Sole active cluster node is fully joined");
+        } else {
+            crm_debug("All %d active cluster nodes are fully joined", count);
+        }
+
+    } else if (count > npeers) {
+        crm_err("New election needed because more nodes confirmed join "
+                "than are in membership (%d > %u)", count, npeers);
+        register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
+
+    } else if (controld_globals.membership_id != crm_peer_seq) {
+        crm_info("New join needed because membership changed (%llu -> %llu)",
+                 controld_globals.membership_id, crm_peer_seq);
+        register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
+
+    } else {
+        crm_warn("Only %d of %u active cluster nodes fully joined "
+                 "(%d did not respond to offer)",
+                 count, npeers, crmd_join_phase_count(crm_join_welcomed));
+    }
+}
+
+static void
+do_state_transition(enum crmd_fsa_state cur_state,
+                    enum crmd_fsa_state next_state, fsa_data_t *msg_data)
+{
+    int level = LOG_INFO;
+    int count = 0;
+    gboolean clear_recovery_bit = TRUE;
+#if 0
+    uint64_t original_fsa_actions = controld_globals.fsa_actions;
+#endif
+
+    enum crmd_fsa_cause cause = msg_data->fsa_cause;
+    enum crmd_fsa_input current_input = msg_data->fsa_input;
+
+    const char *state_from = fsa_state2string(cur_state);
+    const char *state_to = fsa_state2string(next_state);
+    const char *input = fsa_input2string(current_input);
+
+    CRM_LOG_ASSERT(cur_state != next_state);
+
+    do_dot_log(DOT_PREFIX "\t%s -> %s [ label=%s cause=%s origin=%s ]",
+               state_from, state_to, input, fsa_cause2string(cause), msg_data->origin);
+
+    if (cur_state == S_IDLE || next_state == S_IDLE) {
+        level = LOG_NOTICE;
+    } else if (cur_state == S_NOT_DC || next_state == S_NOT_DC) {
+        level = LOG_NOTICE;
+    } else if (cur_state == S_ELECTION) {
+        level = LOG_NOTICE;
+    } else if (cur_state == S_STARTING) {
+        level = LOG_NOTICE;
+    } else if (next_state == S_RECOVERY) {
+        level = LOG_WARNING;
+    }
+
+    do_crm_log(level, "State transition %s -> %s "
+               CRM_XS " input=%s cause=%s origin=%s",
+               state_from, state_to, input, fsa_cause2string(cause),
+               msg_data->origin);
+
+    if (next_state != S_ELECTION && cur_state != S_RELEASE_DC) {
+        controld_stop_current_election_timeout();
+    }
+#if 0
+    if ((controld_globals.fsa_input_register & R_SHUTDOWN)) {
+        controld_set_fsa_action_flags(A_DC_TIMER_STOP);
+    }
+#endif
+    if (next_state == S_INTEGRATION) {
+        controld_set_fsa_action_flags(A_INTEGRATE_TIMER_START);
+    } else {
+        controld_set_fsa_action_flags(A_INTEGRATE_TIMER_STOP);
+    }
+
+    if (next_state == S_FINALIZE_JOIN) {
+        controld_set_fsa_action_flags(A_FINALIZE_TIMER_START);
+    } else {
+        controld_set_fsa_action_flags(A_FINALIZE_TIMER_STOP);
+    }
+
+    if (next_state != S_PENDING) {
+        controld_set_fsa_action_flags(A_DC_TIMER_STOP);
+    }
+    if (next_state != S_IDLE) {
+        controld_stop_recheck_timer();
+    }
+
+    if (cur_state == S_FINALIZE_JOIN && next_state == S_POLICY_ENGINE) {
+        populate_cib_nodes(node_update_quick|node_update_all, __func__);
+    }
+
+    switch (next_state) {
+        case S_PENDING:
+            {
+                cib_t *cib_conn = controld_globals.cib_conn;
+                cib_conn->cmds->set_secondary(cib_conn, cib_scope_local);
+            }
+            update_dc(NULL);
+            break;
+
+        case S_ELECTION:
+            update_dc(NULL);
+            break;
+
+        case S_NOT_DC:
+            controld_reset_counter_election_timer();
+            purge_stonith_cleanup();
+
+            if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+                crm_info("(Re)Issuing shutdown request now" " that we have a new DC");
+                controld_set_fsa_action_flags(A_SHUTDOWN_REQ);
+            }
+            CRM_LOG_ASSERT(controld_globals.dc_name != NULL);
+            if (controld_globals.dc_name == NULL) {
+                crm_err("Reached S_NOT_DC without a DC" " being recorded");
+            }
+            break;
+
+        case S_RECOVERY:
+            clear_recovery_bit = FALSE;
+            break;
+
+        case S_FINALIZE_JOIN:
+            CRM_LOG_ASSERT(AM_I_DC);
+            if (cause == C_TIMER_POPPED) {
+                crm_warn("Progressed to state %s after %s",
+                         fsa_state2string(next_state), fsa_cause2string(cause));
+            }
+            count = crmd_join_phase_count(crm_join_welcomed);
+            if (count > 0) {
+                crm_warn("%d cluster node%s failed to respond to join offer",
+                         count, pcmk__plural_s(count));
+                crmd_join_phase_log(LOG_NOTICE);
+
+            } else {
+                crm_debug("All cluster nodes (%d) responded to join offer",
+                          crmd_join_phase_count(crm_join_integrated));
+            }
+            break;
+
+        case S_POLICY_ENGINE:
+            controld_reset_counter_election_timer();
+            CRM_LOG_ASSERT(AM_I_DC);
+            if (cause == C_TIMER_POPPED) {
+                crm_info("Progressed to state %s after %s",
+                         fsa_state2string(next_state), fsa_cause2string(cause));
+            }
+            check_join_counts(msg_data);
+            break;
+
+        case S_STOPPING:
+        case S_TERMINATE:
+            /* possibly redundant */
+            controld_set_fsa_input_flags(R_SHUTDOWN);
+            break;
+
+        case S_IDLE:
+            CRM_LOG_ASSERT(AM_I_DC);
+            if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+                crm_info("(Re)Issuing shutdown request now" " that we are the DC");
+                controld_set_fsa_action_flags(A_SHUTDOWN_REQ);
+            }
+            controld_start_recheck_timer();
+            break;
+
+        default:
+            break;
+    }
+
+    if (clear_recovery_bit && next_state != S_PENDING) {
+        controld_clear_fsa_action_flags(A_RECOVER);
+    } else if (clear_recovery_bit == FALSE) {
+        controld_set_fsa_action_flags(A_RECOVER);
+    }
+
+#if 0
+    if (original_fsa_actions != controld_globals.fsa_actions) {
+        fsa_dump_actions(original_fsa_actions ^ controld_globals.fsa_actions,
+                         "New actions");
+    }
+#endif
+}
diff --git a/daemons/controld/controld_fsa.h b/daemons/controld/controld_fsa.h
new file mode 100644
index 0000000..2b79f07
--- /dev/null
+++ b/daemons/controld/controld_fsa.h
@@ -0,0 +1,694 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef CRMD_FSA__H
+#  define CRMD_FSA__H
+
+#  include <crm/crm.h>
+#  include <crm/cib.h>
+#  include <crm/common/xml.h>
+#  include <crm/common/mainloop.h>
+#  include <crm/cluster.h>
+#  include <crm/cluster/election_internal.h>
+#  include <crm/common/ipc_internal.h>
+
+/*! States the controller can be in */
+enum crmd_fsa_state {
+    S_IDLE = 0,                 /* Nothing happening */
+
+    S_ELECTION,                 /* Take part in the election algorithm as
+                                 * described below
+                                 */
+    S_INTEGRATION,              /* integrate that status of new nodes (which is
+                                 * all of them if we have just been elected DC)
+                                 * to form a complete and up-to-date picture of
+                                 * the CIB
+                                 */
+    S_FINALIZE_JOIN,            /* integrate that status of new nodes (which is
+                                 * all of them if we have just been elected DC)
+                                 * to form a complete and up-to-date picture of
+                                 * the CIB
+                                 */
+    S_NOT_DC,                   /* we are in non-DC mode */
+    S_POLICY_ENGINE,            /* Determine next stable state of the cluster */
+    S_RECOVERY,                 /* Something bad happened, check everything is ok
+                                 * before continuing and attempt to recover if
+                                 * required
+                                 */
+    S_RELEASE_DC,               /* we were the DC, but now we arent anymore,
+                                 * possibly by our own request, and we should
+                                 * release all unnecessary sub-systems, finish
+                                 * any pending actions, do general cleanup and
+                                 * unset anything that makes us think we are
+                                 * special :)
+                                 */
+    S_STARTING,                 /* we are just starting out */
+    S_PENDING,                  /* we are not a full/active member yet */
+    S_STOPPING,                 /* We are in the final stages of shutting down */
+    S_TERMINATE,                /* We are going to shutdown, this is the equiv of
+                                 * "Sending TERM signal to all processes" in Linux
+                                 * and in worst case scenarios could be considered
+                                 * a self STONITH
+                                 */
+    S_TRANSITION_ENGINE,        /* Attempt to make the calculated next stable
+                                 * state of the cluster a reality
+                                 */
+
+    S_HALT,                     /* Freeze - don't do anything
+                                 * Something bad happened that needs the admin to fix
+                                 * Wait for I_ELECTION
+                                 */
+
+    /*  ----------- Last input found in table is above ---------- */
+    S_ILLEGAL                   /* This is an illegal FSA state */
+        /* (must be last) */
+};
+
+#  define MAXSTATE S_ILLEGAL
+
+/*
+      Once we start and do some basic sanity checks, we go into the
+      S_NOT_DC state and await instructions from the DC or input from
+      the cluster layer which indicates the election algorithm needs to run.
+
+      If the election algorithm is triggered, we enter the S_ELECTION state
+      from where we can either go back to the S_NOT_DC state or progress
+      to the S_INTEGRATION state (or S_RELEASE_DC if we used to be the DC
+      but aren't anymore). See the libcrmcluster API documentation for more
+      information about the election algorithm.
+
+      Once the election is complete, if we are the DC, we enter the
+      S_INTEGRATION state which is a DC-in-waiting style state.  We are
+      the DC, but we shouldn't do anything yet because we may not have an
+      up-to-date picture of the cluster.  There may of course be times
+      when this fails, so we should go back to the S_RECOVERY stage and
+      check everything is ok.  We may also end up here if a new node came
+      online, since each node is authoritative about itself, and we would want
+      to incorporate its information into the CIB.
+
+      Once we have the latest CIB, we then enter the S_POLICY_ENGINE state
+      where invoke the scheduler. It is possible that between
+      invoking the scheduler and receiving an answer, that we receive
+      more input. In this case, we would discard the orginal result and
+      invoke it again.
+
+      Once we are satisfied with the output from the scheduler, we
+      enter S_TRANSITION_ENGINE and feed the scheduler's output to the
+      Transition Engine who attempts to make the scheduler's
+      calculation a reality. If the transition completes successfully,
+      we enter S_IDLE, otherwise we go back to S_POLICY_ENGINE with the
+      current unstable state and try again.
+
+      Of course, we may be asked to shutdown at any time, however we must
+      progress to S_NOT_DC before doing so.  Once we have handed over DC
+      duties to another node, we can then shut down like everyone else,
+      that is, by asking the DC for permission and waiting for it to take all
+      our resources away.
+
+      The case where we are the DC and the only node in the cluster is a
+      special case and handled as an escalation which takes us to
+      S_SHUTDOWN. Similarly, if any other point in the shutdown
+      fails or stalls, this is escalated and we end up in S_TERMINATE.
+
+      At any point, the controller can relay messages for its subsystems,
+      but outbound messages (from subsystems) should probably be blocked
+      until S_INTEGRATION (for the DC) or the join protocol has
+      completed (for non-DC controllers).
+*/
+
+/*======================================
+ *
+ *  Inputs/Events/Stimuli to be given to the finite state machine
+ *
+ *  Some of these a true events, and others are synthesised based on
+ *  the "register" (see below) and the contents or source of messages.
+ *
+ *  The machine keeps processing until receiving I_NULL
+ *
+ *======================================*/
+enum crmd_fsa_input {
+/* 0 */
+    I_NULL,                     /* Nothing happened */
+/* 1 */
+
+    I_CIB_OP,                   /* An update to the CIB occurred */
+    I_CIB_UPDATE,               /* An update to the CIB occurred */
+    I_DC_TIMEOUT,               /* We have lost communication with the DC */
+    I_ELECTION,                 /* Someone started an election */
+    I_PE_CALC,                  /* The scheduler needs to be invoked */
+    I_RELEASE_DC,               /* The election completed and we were not
+                                 * elected, but we were the DC beforehand
+                                 */
+    I_ELECTION_DC,              /* The election completed and we were (re-)elected
+                                 * DC
+                                 */
+    I_ERROR,                    /* Something bad happened (more serious than
+                                 * I_FAIL) and may not have been due to the action
+                                 * being performed.  For example, we may have lost
+                                 * our connection to the CIB.
+                                 */
+/* 9 */
+    I_FAIL,                     /* The action failed to complete successfully */
+    I_INTEGRATED,
+    I_FINALIZED,
+    I_NODE_JOIN,                /* A node has entered the cluster */
+    I_NOT_DC,                   /* We are not and were not the DC before or after
+                                 * the current operation or state
+                                 */
+    I_RECOVERED,                /* The recovery process completed successfully */
+    I_RELEASE_FAIL,             /* We could not give up DC status for some reason
+                                 */
+    I_RELEASE_SUCCESS,          /* We are no longer the DC */
+    I_RESTART,                  /* The current set of actions needs to be
+                                 * restarted
+                                 */
+    I_TE_SUCCESS,               /* Some non-resource, non-cluster-layer action
+                                 * is required of us, e.g. ping
+                                 */
+/* 20 */
+    I_ROUTER,                   /* Do our job as router and forward this to the
+                                 * right place
+                                 */
+    I_SHUTDOWN,                 /* We are asking to shutdown */
+    I_STOP,                     /* We have been told to shutdown */
+    I_TERMINATE,                /* Actually exit */
+    I_STARTUP,
+    I_PE_SUCCESS,               /* The action completed successfully */
+
+    I_JOIN_OFFER,               /* The DC is offering membership */
+    I_JOIN_REQUEST,             /* The client is requesting membership */
+    I_JOIN_RESULT,              /* If not the DC: The result of a join request
+                                 * Else: A client is responding with its local state info
+                                 */
+
+    I_WAIT_FOR_EVENT,           /* we may be waiting for an async task to "happen"
+                                 * and until it does, we can't do anything else
+                                 */
+
+    I_DC_HEARTBEAT,             /* The DC is telling us that it is alive and well */
+
+    I_LRM_EVENT,
+
+/* 30 */
+    I_PENDING,
+    I_HALT,
+
+    /*  ------------ Last input found in table is above ----------- */
+    I_ILLEGAL                   /* This is an illegal value for an FSA input */
+        /* (must be last) */
+};
+
+#  define MAXINPUT  I_ILLEGAL
+
+#  define I_MESSAGE I_ROUTER
+
+/*======================================
+ *
+ * actions
+ *
+ * Some of the actions below will always occur together for now, but this may
+ * not always be the case, so they are split up so that they can easily be
+ * called independently in the future, if necessary.
+ *
+ * For example, separating A_LRM_CONNECT from A_STARTUP might be useful
+ * if we ever try to recover from a faulty or disconnected executor.
+ *
+ *======================================*/
+
+         /* Don't do anything */
+#  define A_NOTHING                 0x0000000000000000ULL
+
+/* -- Startup actions -- */
+        /* Hook to perform any actions (other than connecting to other daemons)
+         * that might be needed as part of the startup.
+         */
+#  define A_STARTUP                 0x0000000000000001ULL
+        /* Hook to perform any actions that might be needed as part
+         * after startup is successful.
+         */
+#  define A_STARTED                 0x0000000000000002ULL
+        /* Connect to cluster layer */
+#  define A_HA_CONNECT              0x0000000000000004ULL
+#  define A_HA_DISCONNECT           0x0000000000000008ULL
+
+#  define A_INTEGRATE_TIMER_START   0x0000000000000010ULL
+#  define A_INTEGRATE_TIMER_STOP    0x0000000000000020ULL
+#  define A_FINALIZE_TIMER_START    0x0000000000000040ULL
+#  define A_FINALIZE_TIMER_STOP     0x0000000000000080ULL
+
+/* -- Election actions -- */
+#  define A_DC_TIMER_START          0x0000000000000100ULL
+#  define A_DC_TIMER_STOP           0x0000000000000200ULL
+#  define A_ELECTION_COUNT          0x0000000000000400ULL
+#  define A_ELECTION_VOTE           0x0000000000000800ULL
+
+#  define A_ELECTION_START          0x0000000000001000ULL
+
+/* -- Message processing -- */
+        /* Process the queue of requests */
+#  define A_MSG_PROCESS             0x0000000000002000ULL
+        /* Send the message to the correct recipient */
+#  define A_MSG_ROUTE               0x0000000000004000ULL
+
+        /* Send a welcome message to new node(s) */
+#  define A_DC_JOIN_OFFER_ONE       0x0000000000008000ULL
+
+/* -- Server Join protocol actions -- */
+        /* Send a welcome message to all nodes */
+#  define A_DC_JOIN_OFFER_ALL       0x0000000000010000ULL
+        /* Process the remote node's ack of our join message */
+#  define A_DC_JOIN_PROCESS_REQ     0x0000000000020000ULL
+        /* Send out the results of the Join phase */
+#  define A_DC_JOIN_FINALIZE        0x0000000000040000ULL
+        /* Send out the results of the Join phase */
+#  define A_DC_JOIN_PROCESS_ACK     0x0000000000080000ULL
+
+/* -- Client Join protocol actions -- */
+#  define A_CL_JOIN_QUERY           0x0000000000100000ULL
+#  define A_CL_JOIN_ANNOUNCE        0x0000000000200000ULL
+        /* Request membership to the DC list */
+#  define A_CL_JOIN_REQUEST         0x0000000000400000ULL
+        /* Did the DC accept or reject the request */
+#  define A_CL_JOIN_RESULT          0x0000000000800000ULL
+
+/* -- Recovery, DC start/stop -- */
+        /* Something bad happened, try to recover */
+#  define A_RECOVER                 0x0000000001000000ULL
+        /* Hook to perform any actions (apart from starting, the TE, scheduler,
+         * and gathering the latest CIB) that might be necessary before
+         * giving up the responsibilities of being the DC.
+         */
+#  define A_DC_RELEASE              0x0000000002000000ULL
+        /* */
+#  define A_DC_RELEASED             0x0000000004000000ULL
+        /* Hook to perform any actions (apart from starting, the TE, scheduler,
+         * and gathering the latest CIB) that might be necessary before
+         * taking over the responsibilities of being the DC.
+         */
+#  define A_DC_TAKEOVER             0x0000000008000000ULL
+
+/* -- Shutdown actions -- */
+#  define A_SHUTDOWN                0x0000000010000000ULL
+#  define A_STOP                    0x0000000020000000ULL
+#  define A_EXIT_0                  0x0000000040000000ULL
+#  define A_EXIT_1                  0x0000000080000000ULL
+
+#  define A_SHUTDOWN_REQ            0x0000000100000000ULL
+#  define A_ELECTION_CHECK          0x0000000200000000ULL
+#  define A_DC_JOIN_FINAL           0x0000000400000000ULL
+
+/* -- CIB actions -- */
+#  define A_CIB_START               0x0000020000000000ULL
+#  define A_CIB_STOP                0x0000040000000000ULL
+
+/* -- Transition Engine actions -- */
+        /* Attempt to reach the newly calculated cluster state. This is
+         * only called once per transition (except if it is asked to
+         * stop the transition or start a new one).
+         * Once given a cluster state to reach, the TE will determine
+         * tasks that can be performed in parallel, execute them, wait
+         * for replies and then determine the next set until the new
+         * state is reached or no further tasks can be taken.
+         */
+#  define A_TE_INVOKE               0x0000100000000000ULL
+#  define A_TE_START                0x0000200000000000ULL
+#  define A_TE_STOP                 0x0000400000000000ULL
+#  define A_TE_CANCEL               0x0000800000000000ULL
+#  define A_TE_HALT                 0x0001000000000000ULL
+
+/* -- Scheduler actions -- */
+        /* Calculate the next state for the cluster.  This is only
+         * invoked once per needed calculation.
+         */
+#  define A_PE_INVOKE               0x0002000000000000ULL
+#  define A_PE_START                0x0004000000000000ULL
+#  define A_PE_STOP                 0x0008000000000000ULL
+/* -- Misc actions -- */
+        /* Add a system generate "block" so that resources arent moved
+         * to or are activly moved away from the affected node.  This
+         * way we can return quickly even if busy with other things.
+         */
+#  define A_NODE_BLOCK              0x0010000000000000ULL
+        /* Update our information in the local CIB */
+#  define A_UPDATE_NODESTATUS       0x0020000000000000ULL
+#  define A_READCONFIG              0x0080000000000000ULL
+
+/* -- LRM Actions -- */
+        /* Connect to pacemaker-execd */
+#  define A_LRM_CONNECT             0x0100000000000000ULL
+        /* Disconnect from pacemaker-execd */
+#  define A_LRM_DISCONNECT          0x0200000000000000ULL
+#  define A_LRM_INVOKE              0x0400000000000000ULL
+#  define A_LRM_EVENT               0x0800000000000000ULL
+
+/* -- Logging actions -- */
+#  define A_LOG                     0x1000000000000000ULL
+#  define A_ERROR                   0x2000000000000000ULL
+#  define A_WARN                    0x4000000000000000ULL
+
+#  define O_EXIT                (A_SHUTDOWN|A_STOP|A_LRM_DISCONNECT|A_HA_DISCONNECT|A_EXIT_0|A_CIB_STOP)
+#  define O_RELEASE             (A_DC_TIMER_STOP|A_DC_RELEASE|A_PE_STOP|A_TE_STOP|A_DC_RELEASED)
+#  define O_PE_RESTART          (A_PE_START|A_PE_STOP)
+#  define O_TE_RESTART          (A_TE_START|A_TE_STOP)
+#  define O_CIB_RESTART         (A_CIB_START|A_CIB_STOP)
+#  define O_LRM_RECONNECT       (A_LRM_CONNECT|A_LRM_DISCONNECT)
+#  define O_DC_TIMER_RESTART    (A_DC_TIMER_STOP|A_DC_TIMER_START)
+/*======================================
+ *
+ * "register" contents
+ *
+ * Things we may want to remember regardless of which state we are in.
+ *
+ * These also count as inputs for synthesizing I_*
+ *
+ *======================================*/
+#  define R_THE_DC          0x00000001ULL
+                                        /* Are we the DC? */
+#  define R_STARTING        0x00000002ULL
+                                        /* Are we starting up? */
+#  define R_SHUTDOWN        0x00000004ULL
+                                        /* Are we trying to shut down? */
+#  define R_STAYDOWN        0x00000008ULL
+                                        /* Should we restart? */
+
+#  define R_JOIN_OK         0x00000010ULL   /* Have we completed the join process */
+#  define R_READ_CONFIG     0x00000040ULL
+#  define R_INVOKE_PE       0x00000080ULL   // Should the scheduler be invoked?
+
+#  define R_CIB_CONNECTED   0x00000100ULL
+                                        /* Is the CIB connected? */
+#  define R_PE_CONNECTED    0x00000200ULL   // Is the scheduler connected?
+#  define R_TE_CONNECTED    0x00000400ULL
+                                        /* Is the Transition Engine connected? */
+#  define R_LRM_CONNECTED   0x00000800ULL   // Is pacemaker-execd connected?
+
+#  define R_CIB_REQUIRED    0x00001000ULL
+                                        /* Is the CIB required? */
+#  define R_PE_REQUIRED     0x00002000ULL   // Is the scheduler required?
+#  define R_TE_REQUIRED     0x00004000ULL
+                                        /* Is the Transition Engine required? */
+#  define R_ST_REQUIRED     0x00008000ULL
+                                        /* Is the Stonith daemon required? */
+
+#  define R_CIB_DONE        0x00010000ULL
+                                        /* Have we calculated the CIB? */
+#  define R_HAVE_CIB        0x00020000ULL   /* Do we have an up-to-date CIB */
+
+#  define R_MEMBERSHIP      0x00100000ULL   /* Have we got cluster layer data yet */
+#  define R_PEER_DATA       0x00200000ULL   /* Have we got T_CL_STATUS data yet */
+
+#  define R_HA_DISCONNECTED 0x00400000ULL      /* did we sign out of our own accord */
+
+#  define R_REQ_PEND        0x01000000ULL
+                                        /* Are there Requests waiting for
+                                           processing? */
+#  define R_PE_PEND         0x02000000ULL   // Are we awaiting reply from scheduler?
+#  define R_TE_PEND         0x04000000ULL
+                                        /* Has the TE been invoked and we're
+                                           awaiting completion? */
+#  define R_RESP_PEND       0x08000000ULL
+                                        /* Do we have clients waiting on a
+                                           response? if so perhaps we shouldn't
+                                           stop yet */
+
+#  define R_SENT_RSC_STOP   0x20000000ULL /* Have we sent a stop action to all
+                                         * resources in preparation for
+                                         * shutting down */
+
+#  define R_IN_RECOVERY     0x80000000ULL
+
+#define CRM_DIRECT_NACK_RC (99) // Deprecated (see PCMK_EXEC_INVALID)
+
+enum crmd_fsa_cause {
+    C_UNKNOWN = 0,
+    C_STARTUP,
+    C_IPC_MESSAGE,
+    C_HA_MESSAGE,
+    C_CRMD_STATUS_CALLBACK,
+    C_LRM_OP_CALLBACK,
+    C_TIMER_POPPED,
+    C_SHUTDOWN,
+    C_FSA_INTERNAL,
+};
+
+enum fsa_data_type {
+    fsa_dt_none,
+    fsa_dt_ha_msg,
+    fsa_dt_xml,
+    fsa_dt_lrm,
+};
+
+typedef struct fsa_data_s fsa_data_t;
+struct fsa_data_s {
+    int id;
+    enum crmd_fsa_input fsa_input;
+    enum crmd_fsa_cause fsa_cause;
+    uint64_t actions;
+    const char *origin;
+    void *data;
+    enum fsa_data_type data_type;
+};
+
+#define controld_set_fsa_input_flags(flags_to_set) do {                 \
+        controld_globals.fsa_input_register                             \
+            = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,         \
+                                 "FSA input", "controller",             \
+                                 controld_globals.fsa_input_register,   \
+                                 (flags_to_set), #flags_to_set);        \
+    } while (0)
+
+#define controld_clear_fsa_input_flags(flags_to_clear) do {             \
+        controld_globals.fsa_input_register                             \
+            = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,       \
+                                   "FSA input", "controller",           \
+                                   controld_globals.fsa_input_register, \
+                                   (flags_to_clear),                    \
+                                   #flags_to_clear);                    \
+    } while (0)
+
+#define controld_set_fsa_action_flags(flags_to_set) do {            \
+        controld_globals.fsa_actions                                \
+            = pcmk__set_flags_as(__func__, __LINE__, LOG_DEBUG,     \
+                                 "FSA action", "controller",        \
+                                 controld_globals.fsa_actions,      \
+                                 (flags_to_set), #flags_to_set);    \
+    } while (0)
+
+#define controld_clear_fsa_action_flags(flags_to_clear) do {            \
+        controld_globals.fsa_actions                                    \
+            = pcmk__clear_flags_as(__func__, __LINE__, LOG_DEBUG,       \
+                                   "FSA action", "controller",          \
+                                   controld_globals.fsa_actions,        \
+                                   (flags_to_clear), #flags_to_clear);  \
+    } while (0)
+
+// This should be moved elsewhere
+xmlNode *controld_query_executor_state(void);
+
+const char *fsa_input2string(enum crmd_fsa_input input);
+const char *fsa_state2string(enum crmd_fsa_state state);
+const char *fsa_cause2string(enum crmd_fsa_cause cause);
+const char *fsa_action2string(long long action);
+
+enum crmd_fsa_state s_crmd_fsa(enum crmd_fsa_cause cause);
+
+enum crmd_fsa_state controld_fsa_get_next_state(enum crmd_fsa_input input);
+
+uint64_t controld_fsa_get_action(enum crmd_fsa_input input);
+
+void controld_init_fsa_trigger(void);
+void controld_destroy_fsa_trigger(void);
+
+void free_max_generation(void);
+
+#  define AM_I_DC pcmk_is_set(controld_globals.fsa_input_register, R_THE_DC)
+#  define controld_trigger_fsa() controld_trigger_fsa_as(__func__, __LINE__)
+
+void controld_trigger_fsa_as(const char *fn, int line);
+
+/* A_READCONFIG */
+void do_read_config(long long action, enum crmd_fsa_cause cause,
+                    enum crmd_fsa_state cur_state,
+                    enum crmd_fsa_input current_input, fsa_data_t *msg_data);
+
+/* A_PE_INVOKE */
+void do_pe_invoke(long long action, enum crmd_fsa_cause cause,
+                  enum crmd_fsa_state cur_state,
+                  enum crmd_fsa_input current_input, fsa_data_t *msg_data);
+
+/* A_LOG */
+void do_log(long long action, enum crmd_fsa_cause cause,
+            enum crmd_fsa_state cur_state,
+            enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_STARTUP */
+void do_startup(long long action, enum crmd_fsa_cause cause,
+                enum crmd_fsa_state cur_state,
+                enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_CIB_START, STOP, RESTART */
+void do_cib_control(long long action, enum crmd_fsa_cause cause,
+                    enum crmd_fsa_state cur_state,
+                    enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_HA_CONNECT */
+void do_ha_control(long long action, enum crmd_fsa_cause cause,
+                   enum crmd_fsa_state cur_state,
+                   enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_LRM_CONNECT */
+void do_lrm_control(long long action, enum crmd_fsa_cause cause,
+                    enum crmd_fsa_state cur_state,
+                    enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_PE_START, STOP, RESTART */
+void do_pe_control(long long action, enum crmd_fsa_cause cause,
+                   enum crmd_fsa_state cur_state,
+                   enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_TE_START, STOP, RESTART */
+void do_te_control(long long action, enum crmd_fsa_cause cause,
+                   enum crmd_fsa_state cur_state,
+                   enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_STARTED */
+void do_started(long long action, enum crmd_fsa_cause cause,
+                enum crmd_fsa_state cur_state,
+                enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_MSG_ROUTE */
+void do_msg_route(long long action, enum crmd_fsa_cause cause,
+                  enum crmd_fsa_state cur_state,
+                  enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_RECOVER */
+void do_recover(long long action, enum crmd_fsa_cause cause,
+                enum crmd_fsa_state cur_state,
+                enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_ELECTION_VOTE */
+void do_election_vote(long long action, enum crmd_fsa_cause cause,
+                      enum crmd_fsa_state cur_state,
+                      enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_ELECTION_COUNT */
+void do_election_count_vote(long long action, enum crmd_fsa_cause cause,
+                            enum crmd_fsa_state cur_state,
+                            enum crmd_fsa_input cur_input,
+                            fsa_data_t *msg_data);
+
+/* A_ELECTION_CHECK */
+void do_election_check(long long action, enum crmd_fsa_cause cause,
+                       enum crmd_fsa_state cur_state,
+                       enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_DC_TIMER_STOP */
+void do_timer_control(long long action, enum crmd_fsa_cause cause,
+                      enum crmd_fsa_state cur_state,
+                      enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_DC_TAKEOVER */
+void do_dc_takeover(long long action, enum crmd_fsa_cause cause,
+                    enum crmd_fsa_state cur_state,
+                    enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_DC_RELEASE */
+void do_dc_release(long long action, enum crmd_fsa_cause cause,
+                   enum crmd_fsa_state cur_state,
+                   enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_DC_JOIN_OFFER_ALL */
+void do_dc_join_offer_all(long long action, enum crmd_fsa_cause cause,
+                          enum crmd_fsa_state cur_state,
+                          enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_DC_JOIN_OFFER_ONE */
+void do_dc_join_offer_one(long long action, enum crmd_fsa_cause cause,
+                          enum crmd_fsa_state cur_state,
+                          enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_DC_JOIN_ACK */
+void do_dc_join_ack(long long action, enum crmd_fsa_cause cause,
+                    enum crmd_fsa_state cur_state,
+                    enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_DC_JOIN_REQ */
+void do_dc_join_filter_offer(long long action, enum crmd_fsa_cause cause,
+                             enum crmd_fsa_state cur_state,
+                             enum crmd_fsa_input cur_input,
+                             fsa_data_t *msg_data);
+
+/* A_DC_JOIN_FINALIZE */
+void do_dc_join_finalize(long long action, enum crmd_fsa_cause cause,
+                         enum crmd_fsa_state cur_state,
+                         enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_CL_JOIN_QUERY */
+/* is there a DC out there? */
+void do_cl_join_query(long long action, enum crmd_fsa_cause cause,
+                      enum crmd_fsa_state cur_state,
+                      enum crmd_fsa_input current_input, fsa_data_t *msg_data);
+
+/* A_CL_JOIN_ANNOUNCE */
+void do_cl_join_announce(long long action, enum crmd_fsa_cause cause,
+                         enum crmd_fsa_state cur_state,
+                         enum crmd_fsa_input current_input, fsa_data_t *msg_data);
+
+/* A_CL_JOIN_REQUEST */
+void do_cl_join_offer_respond(long long action, enum crmd_fsa_cause cause,
+                              enum crmd_fsa_state cur_state,
+                              enum crmd_fsa_input current_input,
+                              fsa_data_t *msg_data);
+
+/* A_CL_JOIN_RESULT */
+void do_cl_join_finalize_respond(long long action, enum crmd_fsa_cause cause,
+                                 enum crmd_fsa_state cur_state,
+                                 enum crmd_fsa_input current_input,
+                                 fsa_data_t *msg_data);
+
+/* A_LRM_INVOKE */
+void do_lrm_invoke(long long action, enum crmd_fsa_cause cause,
+                   enum crmd_fsa_state cur_state,
+                   enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_LRM_EVENT */
+void do_lrm_event(long long action, enum crmd_fsa_cause cause,
+                  enum crmd_fsa_state cur_state,
+                  enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_TE_INVOKE, A_TE_CANCEL */
+void do_te_invoke(long long action, enum crmd_fsa_cause cause,
+                  enum crmd_fsa_state cur_state,
+                  enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_SHUTDOWN_REQ */
+void do_shutdown_req(long long action, enum crmd_fsa_cause cause,
+                     enum crmd_fsa_state cur_state,
+                     enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_SHUTDOWN */
+void do_shutdown(long long action, enum crmd_fsa_cause cause,
+                 enum crmd_fsa_state cur_state,
+                 enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_STOP */
+void do_stop(long long action, enum crmd_fsa_cause cause,
+             enum crmd_fsa_state cur_state,
+             enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_EXIT_0, A_EXIT_1 */
+void do_exit(long long action, enum crmd_fsa_cause cause,
+             enum crmd_fsa_state cur_state,
+             enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
+
+/* A_DC_JOIN_FINAL */
+void do_dc_join_final(long long action, enum crmd_fsa_cause cause,
+                      enum crmd_fsa_state cur_state,
+                      enum crmd_fsa_input current_input, fsa_data_t *msg_data);
+#endif
diff --git a/daemons/controld/controld_globals.h b/daemons/controld/controld_globals.h
new file mode 100644
index 0000000..eff1607
--- /dev/null
+++ b/daemons/controld/controld_globals.h
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2022-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef CONTROLD_GLOBALS__H
+#  define CONTROLD_GLOBALS__H
+
+#include <crm_internal.h>       // pcmk__output_t, etc.
+
+#include <stdint.h>             // uint32_t, uint64_t
+#include <glib.h>               // GList, GMainLoop
+#include <crm/cib.h>            // cib_t
+#include <pacemaker-internal.h> // pcmk__graph_t
+#include <controld_fsa.h>       // enum crmd_fsa_state
+
+typedef struct {
+    // Booleans
+
+    //! Group of \p controld_flags values
+    uint32_t flags;
+
+
+    // Controller FSA
+
+    //! FSA state
+    enum crmd_fsa_state fsa_state;
+
+    //! FSA actions (group of \p A_* flags)
+    uint64_t fsa_actions;
+
+    //! FSA input register contents (group of \p R_* flags)
+    uint64_t fsa_input_register;
+
+    //! FSA message queue
+    GList *fsa_message_queue;
+
+
+    // CIB
+
+    //! Connection to the CIB
+    cib_t *cib_conn;
+
+    //! CIB connection's client ID
+    const char *cib_client_id;
+
+
+    // Scheduler
+
+    //! Reference of the scheduler request being waited on
+    char *fsa_pe_ref;
+
+
+    // Transitioner
+
+    //! Transitioner UUID
+    char *te_uuid;
+
+    //! Graph of transition currently being processed
+    pcmk__graph_t *transition_graph;
+
+
+    // Logging
+
+    //! Output object for controller log messages
+    pcmk__output_t *logger_out;
+
+
+    // Other
+
+    //! Cluster name
+    char *cluster_name;
+
+    //! Designated controller name
+    char *dc_name;
+
+    //! Designated controller's Pacemaker version
+    char *dc_version;
+
+    //! Local node's node name
+    char *our_nodename;
+
+    //! Local node's UUID
+    char *our_uuid;
+
+    //! Last saved cluster communication layer membership ID
+    unsigned long long membership_id;
+
+    //! Max lifetime (in seconds) of a resource's shutdown lock to a node
+    guint shutdown_lock_limit;
+
+    //! Main event loop
+    GMainLoop *mainloop;
+} controld_globals_t;
+
+extern controld_globals_t controld_globals;
+
+/*!
+ * \internal
+ * \enum controld_flags
+ * \brief Bit flags to store various controller state and configuration info
+ */
+enum controld_flags {
+    //! The DC left in a membership change that is being processed
+    controld_dc_left                = (1 << 0),
+
+    //! The FSA is stalled waiting for further input
+    controld_fsa_is_stalled         = (1 << 1),
+
+    //! The local node has been in a quorate partition at some point
+    controld_ever_had_quorum        = (1 << 2),
+
+    //! The local node is currently in a quorate partition
+    controld_has_quorum             = (1 << 3),
+
+    //! Panic the local node if it loses quorum
+    controld_no_quorum_suicide      = (1 << 4),
+
+    //! Lock resources to the local node when it shuts down cleanly
+    controld_shutdown_lock_enabled  = (1 << 5),
+};
+
+#  define controld_set_global_flags(flags_to_set) do {                      \
+        controld_globals.flags = pcmk__set_flags_as(__func__, __LINE__,     \
+                                                    LOG_TRACE,              \
+                                                    "Global", "controller", \
+                                                    controld_globals.flags, \
+                                                    (flags_to_set),         \
+                                                    #flags_to_set);         \
+    } while (0)
+
+#  define controld_clear_global_flags(flags_to_clear) do {                  \
+        controld_globals.flags                                              \
+            = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, "Global", \
+                                   "controller", controld_globals.flags,    \
+                                   (flags_to_clear), #flags_to_clear);      \
+    } while (0)
+
+#endif  // ifndef CONTROLD_GLOBALS__H
diff --git a/daemons/controld/controld_join_client.c b/daemons/controld/controld_join_client.c
new file mode 100644
index 0000000..da6a9d6
--- /dev/null
+++ b/daemons/controld/controld_join_client.c
@@ -0,0 +1,366 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm/crm.h>
+#include <crm/cib.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+
+#include <pacemaker-controld.h>
+
+void join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
+
+extern ha_msg_input_t *copy_ha_msg_input(ha_msg_input_t * orig);
+
+/*!
+ * \internal
+ * \brief Remember if DC is shutting down as we join
+ *
+ * If we're joining while the current DC is shutting down, update its expected
+ * state, so we don't fence it if we become the new DC. (We weren't a peer
+ * when it broadcast its shutdown request.)
+ *
+ * \param[in] msg  A join message from the DC
+ */
+static void
+update_dc_expected(const xmlNode *msg)
+{
+    if ((controld_globals.dc_name != NULL)
+        && pcmk__xe_attr_is_true(msg, F_CRM_DC_LEAVING)) {
+        crm_node_t *dc_node = crm_get_peer(0, controld_globals.dc_name);
+
+        pcmk__update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_DOWN);
+    }
+}
+
+/*	A_CL_JOIN_QUERY		*/
+/* is there a DC out there? */
+void
+do_cl_join_query(long long action,
+                 enum crmd_fsa_cause cause,
+                 enum crmd_fsa_state cur_state,
+                 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
+                                  CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
+
+    sleep(1);                   // Give the cluster layer time to propagate to the DC
+    update_dc(NULL);            /* Unset any existing value so that the result is not discarded */
+    crm_debug("Querying for a DC");
+    send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
+    free_xml(req);
+}
+
+/*	 A_CL_JOIN_ANNOUNCE	*/
+
+/* this is kind of a workaround for the fact that we may not be around or
+ * are otherwise unable to reply when the DC sends out A_DC_JOIN_OFFER_ALL
+ */
+void
+do_cl_join_announce(long long action,
+                    enum crmd_fsa_cause cause,
+                    enum crmd_fsa_state cur_state,
+                    enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    /* don't announce if we're in one of these states */
+    if (cur_state != S_PENDING) {
+        crm_warn("Not announcing cluster join because in state %s",
+                 fsa_state2string(cur_state));
+        return;
+    }
+
+    if (!pcmk_is_set(controld_globals.fsa_input_register, R_STARTING)) {
+        /* send as a broadcast */
+        xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
+                                      CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
+
+        crm_debug("Announcing availability");
+        update_dc(NULL);
+        send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
+        free_xml(req);
+
+    } else {
+        /* Delay announce until we have finished local startup */
+        crm_warn("Delaying announce of cluster join until local startup is complete");
+        return;
+    }
+}
+
+static int query_call_id = 0;
+
+/*	 A_CL_JOIN_REQUEST	*/
+/* aka. accept the welcome offer */
+void
+do_cl_join_offer_respond(long long action,
+                         enum crmd_fsa_cause cause,
+                         enum crmd_fsa_state cur_state,
+                         enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    cib_t *cib_conn = controld_globals.cib_conn;
+
+    ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
+    const char *welcome_from;
+    const char *join_id;
+
+    CRM_CHECK(input != NULL, return);
+
+#if 0
+    if (we are sick) {
+        log error;
+
+        /* save the request for later? */
+        return;
+    }
+#endif
+
+    welcome_from = crm_element_value(input->msg, F_CRM_HOST_FROM);
+    join_id = crm_element_value(input->msg, F_CRM_JOIN_ID);
+    crm_trace("Accepting cluster join offer from node %s "CRM_XS" join-%s",
+              welcome_from, crm_element_value(input->msg, F_CRM_JOIN_ID));
+
+    /* we only ever want the last one */
+    if (query_call_id > 0) {
+        crm_trace("Cancelling previous join query: %d", query_call_id);
+        remove_cib_op_callback(query_call_id, FALSE);
+        query_call_id = 0;
+    }
+
+    if (update_dc(input->msg) == FALSE) {
+        crm_warn("Discarding cluster join offer from node %s (expected %s)",
+                 welcome_from, controld_globals.dc_name);
+        return;
+    }
+
+    update_dc_expected(input->msg);
+
+    query_call_id = cib_conn->cmds->query(cib_conn, NULL, NULL,
+                                          cib_scope_local|cib_no_children);
+    fsa_register_cib_callback(query_call_id, strdup(join_id),
+                              join_query_callback);
+    crm_trace("Registered join query callback: %d", query_call_id);
+
+    controld_set_fsa_action_flags(A_DC_TIMER_STOP);
+    controld_trigger_fsa();
+}
+
+void
+join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    char *join_id = user_data;
+    xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);
+
+    CRM_LOG_ASSERT(join_id != NULL);
+
+    if (query_call_id != call_id) {
+        crm_trace("Query %d superseded", call_id);
+        goto done;
+    }
+
+    query_call_id = 0;
+    if(rc != pcmk_ok || output == NULL) {
+        crm_err("Could not retrieve version details for join-%s: %s (%d)",
+                join_id, pcmk_strerror(rc), rc);
+        register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);
+
+    } else if (controld_globals.dc_name == NULL) {
+        crm_debug("Membership is in flux, not continuing join-%s", join_id);
+
+    } else {
+        xmlNode *reply = NULL;
+
+        crm_debug("Respond to join offer join-%s from %s",
+                  join_id, controld_globals.dc_name);
+        copy_in_properties(generation, output);
+
+        reply = create_request(CRM_OP_JOIN_REQUEST, generation,
+                               controld_globals.dc_name, CRM_SYSTEM_DC,
+                               CRM_SYSTEM_CRMD, NULL);
+
+        crm_xml_add(reply, F_CRM_JOIN_ID, join_id);
+        crm_xml_add(reply, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
+        send_cluster_message(crm_get_peer(0, controld_globals.dc_name),
+                             crm_msg_crmd, reply, TRUE);
+        free_xml(reply);
+    }
+
+  done:
+    free_xml(generation);
+}
+
+static void
+set_join_state(const char * start_state)
+{
+    if (pcmk__str_eq(start_state, "standby", pcmk__str_casei)) {
+        crm_notice("Forcing node %s to join in %s state per configured "
+                   "environment", controld_globals.our_nodename, start_state);
+        cib__update_node_attr(controld_globals.logger_out,
+                              controld_globals.cib_conn, cib_sync_call,
+                              XML_CIB_TAG_NODES, controld_globals.our_uuid,
+                              NULL, NULL, NULL, "standby", "on", NULL, NULL);
+
+    } else if (pcmk__str_eq(start_state, "online", pcmk__str_casei)) {
+        crm_notice("Forcing node %s to join in %s state per configured "
+                   "environment", controld_globals.our_nodename, start_state);
+        cib__update_node_attr(controld_globals.logger_out,
+                              controld_globals.cib_conn, cib_sync_call,
+                              XML_CIB_TAG_NODES, controld_globals.our_uuid,
+                              NULL, NULL, NULL, "standby", "off", NULL, NULL);
+
+    } else if (pcmk__str_eq(start_state, "default", pcmk__str_casei)) {
+        crm_debug("Not forcing a starting state on node %s",
+                  controld_globals.our_nodename);
+
+    } else {
+        crm_warn("Unrecognized start state '%s', using 'default' (%s)",
+                 start_state, controld_globals.our_nodename);
+    }
+}
+
+static int
+update_conn_host_cache(xmlNode *node, void *userdata)
+{
+    const char *remote = crm_element_value(node, XML_ATTR_ID);
+    const char *conn_host = crm_element_value(node, PCMK__XA_CONN_HOST);
+    const char *state = crm_element_value(node, XML_CIB_TAG_STATE);
+
+    crm_node_t *remote_peer = crm_remote_peer_get(remote);
+
+    if (remote_peer == NULL) {
+        return pcmk_rc_ok;
+    }
+
+    if (conn_host != NULL) {
+        pcmk__str_update(&remote_peer->conn_host, conn_host);
+    }
+
+    if (state != NULL) {
+        pcmk__update_peer_state(__func__, remote_peer, state, 0);
+    }
+
+    return pcmk_rc_ok;
+}
+
+/*	A_CL_JOIN_RESULT	*/
+/* aka. this is notification that we have (or have not) been accepted */
+void
+do_cl_join_finalize_respond(long long action,
+                            enum crmd_fsa_cause cause,
+                            enum crmd_fsa_state cur_state,
+                            enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    xmlNode *tmp1 = NULL;
+    gboolean was_nack = TRUE;
+    static gboolean first_join = TRUE;
+    ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
+    const char *start_state = pcmk__env_option(PCMK__ENV_NODE_START_STATE);
+
+    int join_id = -1;
+    const char *op = crm_element_value(input->msg, F_CRM_TASK);
+    const char *welcome_from = crm_element_value(input->msg, F_CRM_HOST_FROM);
+
+    if (!pcmk__str_eq(op, CRM_OP_JOIN_ACKNAK, pcmk__str_casei)) {
+        crm_trace("Ignoring op=%s message", op);
+        return;
+    }
+
+    /* calculate if it was an ack or a nack */
+    if (pcmk__xe_attr_is_true(input->msg, CRM_OP_JOIN_ACKNAK)) {
+        was_nack = FALSE;
+    }
+
+    crm_element_value_int(input->msg, F_CRM_JOIN_ID, &join_id);
+
+    if (was_nack) {
+        crm_err("Shutting down because cluster join with leader %s failed "
+                CRM_XS" join-%d NACK'd", welcome_from, join_id);
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+        controld_set_fsa_input_flags(R_STAYDOWN);
+        return;
+    }
+
+    if (!AM_I_DC
+        && pcmk__str_eq(welcome_from, controld_globals.our_nodename,
+                        pcmk__str_casei)) {
+        crm_warn("Discarding our own welcome - we're no longer the DC");
+        return;
+    }
+
+    if (update_dc(input->msg) == FALSE) {
+        crm_warn("Discarding %s from node %s (expected from %s)",
+                 op, welcome_from, controld_globals.dc_name);
+        return;
+    }
+
+    update_dc_expected(input->msg);
+
+    /* record the node's feature set as a transient attribute */
+    update_attrd(controld_globals.our_nodename, CRM_ATTR_FEATURE_SET,
+                 CRM_FEATURE_SET, NULL, FALSE);
+
+    /* send our status section to the DC */
+    tmp1 = controld_query_executor_state();
+    if (tmp1 != NULL) {
+        xmlNode *remotes = NULL;
+        xmlNode *reply = create_request(CRM_OP_JOIN_CONFIRM, tmp1,
+                                        controld_globals.dc_name, CRM_SYSTEM_DC,
+                                        CRM_SYSTEM_CRMD, NULL);
+
+        crm_xml_add_int(reply, F_CRM_JOIN_ID, join_id);
+
+        crm_debug("Confirming join-%d: sending local operation history to %s",
+                  join_id, controld_globals.dc_name);
+
+        /*
+         * If this is the node's first join since the controller started on it,
+         * set its initial state (standby or member) according to the user's
+         * preference.
+         *
+         * We do not clear the LRM history here. Even if the DC failed to do it
+         * when we last left, removing them here creates a race condition if the
+         * controller is being recovered. Instead of a list of active resources
+         * from the executor, we may end up with a blank status section. If we
+         * are _NOT_ lucky, we will probe for the "wrong" instance of anonymous
+         * clones and end up with multiple active instances on the machine.
+         */
+        if (first_join
+            && !pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+
+            first_join = FALSE;
+            if (start_state) {
+                set_join_state(start_state);
+            }
+        }
+
+        send_cluster_message(crm_get_peer(0, controld_globals.dc_name),
+                             crm_msg_crmd, reply, TRUE);
+        free_xml(reply);
+
+        if (AM_I_DC == FALSE) {
+            register_fsa_input_adv(cause, I_NOT_DC, NULL, A_NOTHING, TRUE,
+                                   __func__);
+        }
+
+        free_xml(tmp1);
+
+        /* Update the remote node cache with information about which node
+         * is hosting the connection.
+         */
+        remotes = pcmk__xe_match(input->msg, XML_CIB_TAG_NODES, NULL, NULL);
+        if (remotes != NULL) {
+            pcmk__xe_foreach_child(remotes, XML_CIB_TAG_NODE, update_conn_host_cache, NULL);
+        }
+
+    } else {
+        crm_err("Could not confirm join-%d with %s: Local operation history "
+                "failed", join_id, controld_globals.dc_name);
+        register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
+    }
+}
diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c
new file mode 100644
index 0000000..f82b132
--- /dev/null
+++ b/daemons/controld/controld_join_dc.c
@@ -0,0 +1,987 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm/crm.h>
+
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/cluster.h>
+
+#include <pacemaker-controld.h>
+
+static char *max_generation_from = NULL;
+static xmlNodePtr max_generation_xml = NULL;
+
+/*!
+ * \internal
+ * \brief Nodes from which a CIB sync has failed since the peer joined
+ *
+ * This table is of the form (<tt>node_name -> join_id</tt>). \p node_name is
+ * the name of a client node from which a CIB \p sync_from() call has failed in
+ * \p do_dc_join_finalize() since the client joined the cluster as a peer.
+ * \p join_id is the ID of the join round in which the \p sync_from() failed,
+ * and is intended for use in nack log messages.
+ */
+static GHashTable *failed_sync_nodes = NULL;
+
+void finalize_join_for(gpointer key, gpointer value, gpointer user_data);
+void finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
+gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
+
+/* Numeric counter used to identify join rounds (an unsigned int would be
+ * appropriate, except we get and set it in XML as int)
+ */
+static int current_join_id = 0;
+
+/*!
+ * \internal
+ * \brief Destroy the hash table containing failed sync nodes
+ */
+void
+controld_destroy_failed_sync_table(void)
+{
+    if (failed_sync_nodes != NULL) {
+        g_hash_table_destroy(failed_sync_nodes);
+        failed_sync_nodes = NULL;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Remove a node from the failed sync nodes table if present
+ *
+ * \param[in] node_name  Node name to remove
+ */
+void
+controld_remove_failed_sync_node(const char *node_name)
+{
+    if (failed_sync_nodes != NULL) {
+        g_hash_table_remove(failed_sync_nodes, (gchar *) node_name);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Add to a hash table a node whose CIB failed to sync
+ *
+ * \param[in] node_name  Name of node whose CIB failed to sync
+ * \param[in] join_id    Join round when the failure occurred
+ */
+static void
+record_failed_sync_node(const char *node_name, gint join_id)
+{
+    if (failed_sync_nodes == NULL) {
+        failed_sync_nodes = pcmk__strikey_table(g_free, NULL);
+    }
+
+    /* If the node is already in the table then we failed to nack it during the
+     * filter offer step
+     */
+    CRM_LOG_ASSERT(g_hash_table_insert(failed_sync_nodes, g_strdup(node_name),
+                                       GINT_TO_POINTER(join_id)));
+}
+
+/*!
+ * \internal
+ * \brief Look up a node name in the failed sync table
+ *
+ * \param[in]  node_name  Name of node to look up
+ * \param[out] join_id    Where to store the join ID of when the sync failed
+ *
+ * \return Standard Pacemaker return code. Specifically, \p pcmk_rc_ok if the
+ *         node name was found, or \p pcmk_rc_node_unknown otherwise.
+ * \note \p *join_id is set to -1 if the node is not found.
+ */
+static int
+lookup_failed_sync_node(const char *node_name, gint *join_id)
+{
+    *join_id = -1;
+
+    if (failed_sync_nodes != NULL) {
+        gpointer result = g_hash_table_lookup(failed_sync_nodes,
+                                              (gchar *) node_name);
+        if (result != NULL) {
+            *join_id = GPOINTER_TO_INT(result);
+            return pcmk_rc_ok;
+        }
+    }
+    return pcmk_rc_node_unknown;
+}
+
+void
+crm_update_peer_join(const char *source, crm_node_t * node, enum crm_join_phase phase)
+{
+    enum crm_join_phase last = 0;
+
+    CRM_CHECK(node != NULL, return);
+
+    /* Remote nodes do not participate in joins */
+    if (pcmk_is_set(node->flags, crm_remote_node)) {
+        return;
+    }
+
+    last = node->join;
+
+    if(phase == last) {
+        crm_trace("Node %s join-%d phase is still %s "
+                  CRM_XS " nodeid=%u source=%s",
+                  node->uname, current_join_id, crm_join_phase_str(last),
+                  node->id, source);
+
+    } else if ((phase <= crm_join_none) || (phase == (last + 1))) {
+        node->join = phase;
+        crm_trace("Node %s join-%d phase is now %s (was %s) "
+                  CRM_XS " nodeid=%u source=%s",
+                 node->uname, current_join_id, crm_join_phase_str(phase),
+                 crm_join_phase_str(last), node->id, source);
+
+    } else {
+        crm_warn("Rejecting join-%d phase update for node %s because "
+                 "can't go from %s to %s " CRM_XS " nodeid=%u source=%s",
+                 current_join_id, node->uname, crm_join_phase_str(last),
+                 crm_join_phase_str(phase), node->id, source);
+    }
+}
+
+static void
+start_join_round(void)
+{
+    GHashTableIter iter;
+    crm_node_t *peer = NULL;
+
+    crm_debug("Starting new join round join-%d", current_join_id);
+
+    g_hash_table_iter_init(&iter, crm_peer_cache);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
+        crm_update_peer_join(__func__, peer, crm_join_none);
+    }
+    if (max_generation_from != NULL) {
+        free(max_generation_from);
+        max_generation_from = NULL;
+    }
+    if (max_generation_xml != NULL) {
+        free_xml(max_generation_xml);
+        max_generation_xml = NULL;
+    }
+    controld_clear_fsa_input_flags(R_HAVE_CIB);
+    controld_forget_all_cib_replace_calls();
+}
+
+/*!
+ * \internal
+ * \brief Create a join message from the DC
+ *
+ * \param[in] join_op  Join operation name
+ * \param[in] host_to  Recipient of message
+ */
+static xmlNode *
+create_dc_message(const char *join_op, const char *host_to)
+{
+    xmlNode *msg = create_request(join_op, NULL, host_to, CRM_SYSTEM_CRMD,
+                                  CRM_SYSTEM_DC, NULL);
+
+    /* Identify which election this is a part of */
+    crm_xml_add_int(msg, F_CRM_JOIN_ID, current_join_id);
+
+    /* Add a field specifying whether the DC is shutting down. This keeps the
+     * joining node from fencing the old DC if it becomes the new DC.
+     */
+    pcmk__xe_set_bool_attr(msg, F_CRM_DC_LEAVING,
+                           pcmk_is_set(controld_globals.fsa_input_register,
+                                       R_SHUTDOWN));
+    return msg;
+}
+
+static void
+join_make_offer(gpointer key, gpointer value, gpointer user_data)
+{
+    xmlNode *offer = NULL;
+    crm_node_t *member = (crm_node_t *)value;
+
+    CRM_ASSERT(member != NULL);
+    if (crm_is_peer_active(member) == FALSE) {
+        crm_info("Not making join-%d offer to inactive node %s",
+                 current_join_id,
+                 (member->uname? member->uname : "with unknown name"));
+        if(member->expected == NULL && pcmk__str_eq(member->state, CRM_NODE_LOST, pcmk__str_casei)) {
+            /* You would think this unsafe, but in fact this plus an
+             * active resource is what causes it to be fenced.
+             *
+             * Yes, this does mean that any node that dies at the same
+             * time as the old DC and is not running resource (still)
+             * won't be fenced.
+             *
+             * I'm not happy about this either.
+             */
+            pcmk__update_peer_expected(__func__, member, CRMD_JOINSTATE_DOWN);
+        }
+        return;
+    }
+
+    if (member->uname == NULL) {
+        crm_info("Not making join-%d offer to node uuid %s with unknown name",
+                 current_join_id, member->uuid);
+        return;
+    }
+
+    if (controld_globals.membership_id != crm_peer_seq) {
+        controld_globals.membership_id = crm_peer_seq;
+        crm_info("Making join-%d offers based on membership event %llu",
+                 current_join_id, crm_peer_seq);
+    }
+
+    if(user_data && member->join > crm_join_none) {
+        crm_info("Not making join-%d offer to already known node %s (%s)",
+                 current_join_id, member->uname,
+                 crm_join_phase_str(member->join));
+        return;
+    }
+
+    crm_update_peer_join(__func__, (crm_node_t*)member, crm_join_none);
+
+    offer = create_dc_message(CRM_OP_JOIN_OFFER, member->uname);
+
+    // Advertise our feature set so the joining node can bail if not compatible
+    crm_xml_add(offer, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
+
+    crm_info("Sending join-%d offer to %s", current_join_id, member->uname);
+    send_cluster_message(member, crm_msg_crmd, offer, TRUE);
+    free_xml(offer);
+
+    crm_update_peer_join(__func__, member, crm_join_welcomed);
+}
+
+/*	 A_DC_JOIN_OFFER_ALL	*/
+void
+do_dc_join_offer_all(long long action,
+                     enum crmd_fsa_cause cause,
+                     enum crmd_fsa_state cur_state,
+                     enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    int count;
+
+    /* Reset everyone's status back to down or in_ccm in the CIB.
+     * Any nodes that are active in the CIB but not in the cluster membership
+     * will be seen as offline by the scheduler anyway.
+     */
+    current_join_id++;
+    start_join_round();
+
+    update_dc(NULL);
+    if (cause == C_HA_MESSAGE && current_input == I_NODE_JOIN) {
+        crm_info("A new node joined the cluster");
+    }
+    g_hash_table_foreach(crm_peer_cache, join_make_offer, NULL);
+
+    count = crmd_join_phase_count(crm_join_welcomed);
+    crm_info("Waiting on join-%d requests from %d outstanding node%s",
+             current_join_id, count, pcmk__plural_s(count));
+
+    // Don't waste time by invoking the scheduler yet
+}
+
+/*	 A_DC_JOIN_OFFER_ONE	*/
+void
+do_dc_join_offer_one(long long action,
+                     enum crmd_fsa_cause cause,
+                     enum crmd_fsa_state cur_state,
+                     enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    crm_node_t *member;
+    ha_msg_input_t *welcome = NULL;
+    int count;
+    const char *join_to = NULL;
+
+    if (msg_data->data == NULL) {
+        crm_info("Making join-%d offers to any unconfirmed nodes "
+                 "because an unknown node joined", current_join_id);
+        g_hash_table_foreach(crm_peer_cache, join_make_offer, &member);
+        check_join_state(cur_state, __func__);
+        return;
+    }
+
+    welcome = fsa_typed_data(fsa_dt_ha_msg);
+    if (welcome == NULL) {
+        // fsa_typed_data() already logged an error
+        return;
+    }
+
+    join_to = crm_element_value(welcome->msg, F_CRM_HOST_FROM);
+    if (join_to == NULL) {
+        crm_err("Can't make join-%d offer to unknown node", current_join_id);
+        return;
+    }
+    member = crm_get_peer(0, join_to);
+
+    /* It is possible that a node will have been sick or starting up when the
+     * original offer was made. However, it will either re-announce itself in
+     * due course, or we can re-store the original offer on the client.
+     */
+
+    crm_update_peer_join(__func__, member, crm_join_none);
+    join_make_offer(NULL, member, NULL);
+
+    /* If the offer isn't to the local node, make an offer to the local node as
+     * well, to ensure the correct value for max_generation_from.
+     */
+    if (strcasecmp(join_to, controld_globals.our_nodename) != 0) {
+        member = crm_get_peer(0, controld_globals.our_nodename);
+        join_make_offer(NULL, member, NULL);
+    }
+
+    /* This was a genuine join request; cancel any existing transition and
+     * invoke the scheduler.
+     */
+    abort_transition(INFINITY, pcmk__graph_restart, "Node join", NULL);
+
+    count = crmd_join_phase_count(crm_join_welcomed);
+    crm_info("Waiting on join-%d requests from %d outstanding node%s",
+             current_join_id, count, pcmk__plural_s(count));
+
+    // Don't waste time by invoking the scheduler yet
+}
+
+static int
+compare_int_fields(xmlNode * left, xmlNode * right, const char *field)
+{
+    const char *elem_l = crm_element_value(left, field);
+    const char *elem_r = crm_element_value(right, field);
+
+    long long int_elem_l;
+    long long int_elem_r;
+
+    pcmk__scan_ll(elem_l, &int_elem_l, -1LL);
+    pcmk__scan_ll(elem_r, &int_elem_r, -1LL);
+
+    if (int_elem_l < int_elem_r) {
+        return -1;
+
+    } else if (int_elem_l > int_elem_r) {
+        return 1;
+    }
+
+    return 0;
+}
+
+/*	 A_DC_JOIN_PROCESS_REQ	*/
+void
+do_dc_join_filter_offer(long long action,
+                        enum crmd_fsa_cause cause,
+                        enum crmd_fsa_state cur_state,
+                        enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    xmlNode *generation = NULL;
+
+    int cmp = 0;
+    int join_id = -1;
+    int count = 0;
+    gint value = 0;
+    gboolean ack_nack_bool = TRUE;
+    ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);
+
+    const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
+    const char *ref = crm_element_value(join_ack->msg, F_CRM_REFERENCE);
+    const char *join_version = crm_element_value(join_ack->msg,
+                                                 XML_ATTR_CRM_VERSION);
+    crm_node_t *join_node = NULL;
+
+    if (join_from == NULL) {
+        crm_err("Ignoring invalid join request without node name");
+        return;
+    }
+    join_node = crm_get_peer(0, join_from);
+
+    crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id);
+    if (join_id != current_join_id) {
+        crm_debug("Ignoring join-%d request from %s because we are on join-%d",
+                  join_id, join_from, current_join_id);
+        check_join_state(cur_state, __func__);
+        return;
+    }
+
+    generation = join_ack->xml;
+    if (max_generation_xml != NULL && generation != NULL) {
+        int lpc = 0;
+
+        const char *attributes[] = {
+            XML_ATTR_GENERATION_ADMIN,
+            XML_ATTR_GENERATION,
+            XML_ATTR_NUMUPDATES,
+        };
+
+        for (lpc = 0; cmp == 0 && lpc < PCMK__NELEM(attributes); lpc++) {
+            cmp = compare_int_fields(max_generation_xml, generation, attributes[lpc]);
+        }
+    }
+
+    if (ref == NULL) {
+        ref = "none"; // for logging only
+    }
+
+    if (lookup_failed_sync_node(join_from, &value) == pcmk_rc_ok) {
+        crm_err("Rejecting join-%d request from node %s because we failed to "
+                "sync its CIB in join-%d " CRM_XS " ref=%s",
+                join_id, join_from, value, ref);
+        ack_nack_bool = FALSE;
+
+    } else if (!crm_is_peer_active(join_node)) {
+        if (match_down_event(join_from) != NULL) {
+            /* The join request was received after the node was fenced or
+             * otherwise shutdown in a way that we're aware of. No need to log
+             * an error in this rare occurrence; we know the client was recently
+             * shut down, and receiving a lingering in-flight request is not
+             * cause for alarm.
+             */
+            crm_debug("Rejecting join-%d request from inactive node %s "
+                      CRM_XS " ref=%s", join_id, join_from, ref);
+        } else {
+            crm_err("Rejecting join-%d request from inactive node %s "
+                    CRM_XS " ref=%s", join_id, join_from, ref);
+        }
+        ack_nack_bool = FALSE;
+
+    } else if (generation == NULL) {
+        crm_err("Rejecting invalid join-%d request from node %s "
+                "missing CIB generation " CRM_XS " ref=%s",
+                join_id, join_from, ref);
+        ack_nack_bool = FALSE;
+
+    } else if ((join_version == NULL)
+               || !feature_set_compatible(CRM_FEATURE_SET, join_version)) {
+        crm_err("Rejecting join-%d request from node %s because feature set %s"
+                " is incompatible with ours (%s) " CRM_XS " ref=%s",
+                join_id, join_from, (join_version? join_version : "pre-3.1.0"),
+                CRM_FEATURE_SET, ref);
+        ack_nack_bool = FALSE;
+
+    } else if (max_generation_xml == NULL) {
+        const char *validation = crm_element_value(generation,
+                                                   XML_ATTR_VALIDATION);
+
+        if (get_schema_version(validation) < 0) {
+            crm_err("Rejecting join-%d request from %s (with first CIB "
+                    "generation) due to unknown schema version %s "
+                    CRM_XS " ref=%s",
+                    join_id, join_from, validation, ref);
+            ack_nack_bool = FALSE;
+
+        } else {
+            crm_debug("Accepting join-%d request from %s (with first CIB "
+                      "generation) " CRM_XS " ref=%s",
+                      join_id, join_from, ref);
+            max_generation_xml = copy_xml(generation);
+            pcmk__str_update(&max_generation_from, join_from);
+        }
+
+    } else if ((cmp < 0)
+               || ((cmp == 0)
+                   && pcmk__str_eq(join_from, controld_globals.our_nodename,
+                                   pcmk__str_casei))) {
+        const char *validation = crm_element_value(generation,
+                                                   XML_ATTR_VALIDATION);
+
+        if (get_schema_version(validation) < 0) {
+            crm_err("Rejecting join-%d request from %s (with better CIB "
+                    "generation than current best from %s) due to unknown "
+                    "schema version %s " CRM_XS " ref=%s",
+                    join_id, join_from, max_generation_from, validation, ref);
+            ack_nack_bool = FALSE;
+
+        } else {
+            crm_debug("Accepting join-%d request from %s (with better CIB "
+                      "generation than current best from %s) " CRM_XS " ref=%s",
+                      join_id, join_from, max_generation_from, ref);
+            crm_log_xml_debug(max_generation_xml, "Old max generation");
+            crm_log_xml_debug(generation, "New max generation");
+
+            free_xml(max_generation_xml);
+            max_generation_xml = copy_xml(join_ack->xml);
+            pcmk__str_update(&max_generation_from, join_from);
+        }
+
+    } else {
+        crm_debug("Accepting join-%d request from %s " CRM_XS " ref=%s",
+                  join_id, join_from, ref);
+    }
+
+    if (!ack_nack_bool) {
+        if (compare_version(join_version, "3.17.0") < 0) {
+            /* Clients with CRM_FEATURE_SET < 3.17.0 may respawn infinitely
+             * after a nack message, don't send one
+             */
+            crm_update_peer_join(__func__, join_node, crm_join_nack_quiet);
+        } else {
+            crm_update_peer_join(__func__, join_node, crm_join_nack);
+        }
+        pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_NACK);
+
+    } else {
+        crm_update_peer_join(__func__, join_node, crm_join_integrated);
+        pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
+    }
+
+    count = crmd_join_phase_count(crm_join_integrated);
+    crm_debug("%d node%s currently integrated in join-%d",
+              count, pcmk__plural_s(count), join_id);
+
+    if (check_join_state(cur_state, __func__) == FALSE) {
+        // Don't waste time by invoking the scheduler yet
+        count = crmd_join_phase_count(crm_join_welcomed);
+        crm_debug("Waiting on join-%d requests from %d outstanding node%s",
+                  join_id, count, pcmk__plural_s(count));
+    }
+}
+
+/*	A_DC_JOIN_FINALIZE	*/
+void
+do_dc_join_finalize(long long action,
+                    enum crmd_fsa_cause cause,
+                    enum crmd_fsa_state cur_state,
+                    enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    char *sync_from = NULL;
+    int rc = pcmk_ok;
+    int count_welcomed = crmd_join_phase_count(crm_join_welcomed);
+    int count_finalizable = crmd_join_phase_count(crm_join_integrated)
+                            + crmd_join_phase_count(crm_join_nack)
+                            + crmd_join_phase_count(crm_join_nack_quiet);
+
+    /* This we can do straight away and avoid clients timing us out
+     *  while we compute the latest CIB
+     */
+    if (count_welcomed != 0) {
+        crm_debug("Waiting on join-%d requests from %d outstanding node%s "
+                  "before finalizing join", current_join_id, count_welcomed,
+                  pcmk__plural_s(count_welcomed));
+        crmd_join_phase_log(LOG_DEBUG);
+        /* crmd_fsa_stall(FALSE); Needed? */
+        return;
+
+    } else if (count_finalizable == 0) {
+        crm_debug("Finalization not needed for join-%d at the current time",
+                  current_join_id);
+        crmd_join_phase_log(LOG_DEBUG);
+        check_join_state(controld_globals.fsa_state, __func__);
+        return;
+    }
+
+    controld_clear_fsa_input_flags(R_HAVE_CIB);
+    if (pcmk__str_eq(max_generation_from, controld_globals.our_nodename,
+                     pcmk__str_null_matches|pcmk__str_casei)) {
+        controld_set_fsa_input_flags(R_HAVE_CIB);
+    }
+
+    if (!controld_globals.transition_graph->complete) {
+        crm_warn("Delaying join-%d finalization while transition in progress",
+                 current_join_id);
+        crmd_join_phase_log(LOG_DEBUG);
+        crmd_fsa_stall(FALSE);
+        return;
+    }
+
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
+        // Send our CIB out to everyone
+        pcmk__str_update(&sync_from, controld_globals.our_nodename);
+        crm_debug("Finalizing join-%d for %d node%s (sync'ing from local CIB)",
+                  current_join_id, count_finalizable,
+                  pcmk__plural_s(count_finalizable));
+        crm_log_xml_debug(max_generation_xml, "Requested CIB version");
+
+    } else {
+        // Ask for the agreed best CIB
+        pcmk__str_update(&sync_from, max_generation_from);
+        crm_notice("Finalizing join-%d for %d node%s (sync'ing CIB from %s)",
+                   current_join_id, count_finalizable,
+                   pcmk__plural_s(count_finalizable), sync_from);
+        crm_log_xml_notice(max_generation_xml, "Requested CIB version");
+    }
+    crmd_join_phase_log(LOG_DEBUG);
+
+    rc = controld_globals.cib_conn->cmds->sync_from(controld_globals.cib_conn,
+                                                    sync_from, NULL, cib_none);
+
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
+        controld_record_cib_replace_call(rc);
+    }
+    fsa_register_cib_callback(rc, sync_from, finalize_sync_callback);
+}
+
+void
+free_max_generation(void)
+{
+    free(max_generation_from);
+    max_generation_from = NULL;
+
+    free_xml(max_generation_xml);
+    max_generation_xml = NULL;
+}
+
+void
+finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    CRM_LOG_ASSERT(-EPERM != rc);
+
+    controld_forget_cib_replace_call(call_id);
+
+    if (rc != pcmk_ok) {
+        const char *sync_from = (const char *) user_data;
+
+        do_crm_log(((rc == -pcmk_err_old_data)? LOG_WARNING : LOG_ERR),
+                   "Could not sync CIB from %s in join-%d: %s",
+                   sync_from, current_join_id, pcmk_strerror(rc));
+
+        if (rc != -pcmk_err_old_data) {
+            record_failed_sync_node(sync_from, current_join_id);
+        }
+
+        /* restart the whole join process */
+        register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION_DC, NULL, NULL,
+                               __func__);
+
+    } else if (!AM_I_DC) {
+        crm_debug("Sync'ed CIB for join-%d but no longer DC", current_join_id);
+
+    } else if (controld_globals.fsa_state != S_FINALIZE_JOIN) {
+        crm_debug("Sync'ed CIB for join-%d but no longer in S_FINALIZE_JOIN "
+                  "(%s)", current_join_id,
+                  fsa_state2string(controld_globals.fsa_state));
+
+    } else {
+        controld_set_fsa_input_flags(R_HAVE_CIB);
+
+        /* make sure dc_uuid is re-set to us */
+        if (!check_join_state(controld_globals.fsa_state, __func__)) {
+            int count_finalizable = 0;
+
+            count_finalizable = crmd_join_phase_count(crm_join_integrated)
+                                + crmd_join_phase_count(crm_join_nack)
+                                + crmd_join_phase_count(crm_join_nack_quiet);
+
+            crm_debug("Notifying %d node%s of join-%d results",
+                      count_finalizable, pcmk__plural_s(count_finalizable),
+                      current_join_id);
+            g_hash_table_foreach(crm_peer_cache, finalize_join_for, NULL);
+        }
+    }
+}
+
+static void
+join_update_complete_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    fsa_data_t *msg_data = NULL;
+
+    if (rc == pcmk_ok) {
+        crm_debug("join-%d node history update (via CIB call %d) complete",
+                  current_join_id, call_id);
+        check_join_state(controld_globals.fsa_state, __func__);
+
+    } else {
+        crm_err("join-%d node history update (via CIB call %d) failed: %s "
+                "(next transition may determine resource status incorrectly)",
+                current_join_id, call_id, pcmk_strerror(rc));
+        crm_log_xml_debug(msg, "failed");
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+    }
+}
+
+/*	A_DC_JOIN_PROCESS_ACK	*/
+void
+do_dc_join_ack(long long action,
+               enum crmd_fsa_cause cause,
+               enum crmd_fsa_state cur_state,
+               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    int join_id = -1;
+    ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);
+    enum controld_section_e section = controld_section_lrm;
+    const int cib_opts = cib_scope_local|cib_can_create;
+
+    const char *op = crm_element_value(join_ack->msg, F_CRM_TASK);
+    const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
+    crm_node_t *peer = NULL;
+
+    // Sanity checks
+    if (join_from == NULL) {
+        crm_warn("Ignoring message received without node identification");
+        return;
+    }
+    if (op == NULL) {
+        crm_warn("Ignoring message received from %s without task", join_from);
+        return;
+    }
+
+    if (strcmp(op, CRM_OP_JOIN_CONFIRM)) {
+        crm_debug("Ignoring '%s' message from %s while waiting for '%s'",
+                  op, join_from, CRM_OP_JOIN_CONFIRM);
+        return;
+    }
+
+    if (crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id) != 0) {
+        crm_warn("Ignoring join confirmation from %s without valid join ID",
+                 join_from);
+        return;
+    }
+
+    peer = crm_get_peer(0, join_from);
+    if (peer->join != crm_join_finalized) {
+        crm_info("Ignoring out-of-sequence join-%d confirmation from %s "
+                 "(currently %s not %s)",
+                 join_id, join_from, crm_join_phase_str(peer->join),
+                 crm_join_phase_str(crm_join_finalized));
+        return;
+    }
+
+    if (join_id != current_join_id) {
+        crm_err("Rejecting join-%d confirmation from %s "
+                "because currently on join-%d",
+                join_id, join_from, current_join_id);
+        crm_update_peer_join(__func__, peer, crm_join_nack);
+        return;
+    }
+
+    crm_update_peer_join(__func__, peer, crm_join_confirmed);
+
+    /* Update CIB with node's current executor state. A new transition will be
+     * triggered later, when the CIB notifies us of the change.
+     */
+    if (pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
+        section = controld_section_lrm_unlocked;
+    }
+    controld_delete_node_state(join_from, section, cib_scope_local);
+    if (pcmk__str_eq(join_from, controld_globals.our_nodename,
+                     pcmk__str_casei)) {
+        xmlNode *now_dc_lrmd_state = controld_query_executor_state();
+
+        if (now_dc_lrmd_state != NULL) {
+            crm_debug("Updating local node history for join-%d "
+                      "from query result", join_id);
+            controld_update_cib(XML_CIB_TAG_STATUS, now_dc_lrmd_state, cib_opts,
+                                join_update_complete_callback);
+            free_xml(now_dc_lrmd_state);
+        } else {
+            crm_warn("Updating local node history from join-%d confirmation "
+                     "because query failed", join_id);
+            controld_update_cib(XML_CIB_TAG_STATUS, join_ack->xml, cib_opts,
+                                join_update_complete_callback);
+        }
+    } else {
+        crm_debug("Updating node history for %s from join-%d confirmation",
+                  join_from, join_id);
+        controld_update_cib(XML_CIB_TAG_STATUS, join_ack->xml, cib_opts,
+                            join_update_complete_callback);
+    }
+}
+
+void
+finalize_join_for(gpointer key, gpointer value, gpointer user_data)
+{
+    xmlNode *acknak = NULL;
+    xmlNode *tmp1 = NULL;
+    crm_node_t *join_node = value;
+    const char *join_to = join_node->uname;
+    bool integrated = false;
+
+    switch (join_node->join) {
+        case crm_join_integrated:
+            integrated = true;
+            break;
+        case crm_join_nack:
+        case crm_join_nack_quiet:
+            break;
+        default:
+            crm_trace("Not updating non-integrated and non-nacked node %s (%s) "
+                      "for join-%d", join_to,
+                      crm_join_phase_str(join_node->join), current_join_id);
+            return;
+    }
+
+    /* Update the <node> element with the node's name and UUID, in case they
+     * weren't known before
+     */
+    crm_trace("Updating node name and UUID in CIB for %s", join_to);
+    tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE);
+    set_uuid(tmp1, XML_ATTR_ID, join_node);
+    crm_xml_add(tmp1, XML_ATTR_UNAME, join_to);
+    fsa_cib_anon_update(XML_CIB_TAG_NODES, tmp1);
+    free_xml(tmp1);
+
+    if (join_node->join == crm_join_nack_quiet) {
+        crm_trace("Not sending nack message to node %s with feature set older "
+                  "than 3.17.0", join_to);
+        return;
+    }
+
+    join_node = crm_get_peer(0, join_to);
+    if (!crm_is_peer_active(join_node)) {
+        /*
+         * NACK'ing nodes that the membership layer doesn't know about yet
+         * simply creates more churn
+         *
+         * Better to leave them waiting and let the join restart when
+         * the new membership event comes in
+         *
+         * All other NACKs (due to versions etc) should still be processed
+         */
+        pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_PENDING);
+        return;
+    }
+
+    // Acknowledge or nack node's join request
+    crm_debug("%sing join-%d request from %s",
+              integrated? "Acknowledg" : "Nack", current_join_id, join_to);
+    acknak = create_dc_message(CRM_OP_JOIN_ACKNAK, join_to);
+    pcmk__xe_set_bool_attr(acknak, CRM_OP_JOIN_ACKNAK, integrated);
+
+    if (integrated) {
+        // No change needed for a nacked node
+        crm_update_peer_join(__func__, join_node, crm_join_finalized);
+        pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
+
+        /* Iterate through the remote peer cache and add information on which
+         * node hosts each to the ACK message.  This keeps new controllers in
+         * sync with what has already happened.
+         */
+        if (crm_remote_peer_cache_size() != 0) {
+            GHashTableIter iter;
+            crm_node_t *node = NULL;
+            xmlNode *remotes = create_xml_node(acknak, XML_CIB_TAG_NODES);
+
+            g_hash_table_iter_init(&iter, crm_remote_peer_cache);
+            while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
+                xmlNode *remote = NULL;
+
+                if (!node->conn_host) {
+                    continue;
+                }
+
+                remote = create_xml_node(remotes, XML_CIB_TAG_NODE);
+                pcmk__xe_set_props(remote,
+                                   XML_ATTR_ID, node->uname,
+                                   XML_CIB_TAG_STATE, node->state,
+                                   PCMK__XA_CONN_HOST, node->conn_host,
+                                   NULL);
+            }
+        }
+    }
+    send_cluster_message(join_node, crm_msg_crmd, acknak, TRUE);
+    free_xml(acknak);
+    return;
+}
+
+gboolean
+check_join_state(enum crmd_fsa_state cur_state, const char *source)
+{
+    static unsigned long long highest_seq = 0;
+
+    if (controld_globals.membership_id != crm_peer_seq) {
+        crm_debug("join-%d: Membership changed from %llu to %llu "
+                  CRM_XS " highest=%llu state=%s for=%s",
+                  current_join_id, controld_globals.membership_id, crm_peer_seq,
+                  highest_seq, fsa_state2string(cur_state), source);
+        if(highest_seq < crm_peer_seq) {
+            /* Don't spam the FSA with duplicates */
+            highest_seq = crm_peer_seq;
+            register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
+        }
+
+    } else if (cur_state == S_INTEGRATION) {
+        if (crmd_join_phase_count(crm_join_welcomed) == 0) {
+            int count = crmd_join_phase_count(crm_join_integrated);
+
+            crm_debug("join-%d: Integration of %d peer%s complete "
+                      CRM_XS " state=%s for=%s",
+                      current_join_id, count, pcmk__plural_s(count),
+                      fsa_state2string(cur_state), source);
+            register_fsa_input_before(C_FSA_INTERNAL, I_INTEGRATED, NULL);
+            return TRUE;
+        }
+
+    } else if (cur_state == S_FINALIZE_JOIN) {
+        if (!pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
+            crm_debug("join-%d: Delaying finalization until we have CIB "
+                      CRM_XS " state=%s for=%s",
+                      current_join_id, fsa_state2string(cur_state), source);
+            return TRUE;
+
+        } else if (crmd_join_phase_count(crm_join_welcomed) != 0) {
+            int count = crmd_join_phase_count(crm_join_welcomed);
+
+            crm_debug("join-%d: Still waiting on %d welcomed node%s "
+                      CRM_XS " state=%s for=%s",
+                      current_join_id, count, pcmk__plural_s(count),
+                      fsa_state2string(cur_state), source);
+            crmd_join_phase_log(LOG_DEBUG);
+
+        } else if (crmd_join_phase_count(crm_join_integrated) != 0) {
+            int count = crmd_join_phase_count(crm_join_integrated);
+
+            crm_debug("join-%d: Still waiting on %d integrated node%s "
+                      CRM_XS " state=%s for=%s",
+                      current_join_id, count, pcmk__plural_s(count),
+                      fsa_state2string(cur_state), source);
+            crmd_join_phase_log(LOG_DEBUG);
+
+        } else if (crmd_join_phase_count(crm_join_finalized) != 0) {
+            int count = crmd_join_phase_count(crm_join_finalized);
+
+            crm_debug("join-%d: Still waiting on %d finalized node%s "
+                      CRM_XS " state=%s for=%s",
+                      current_join_id, count, pcmk__plural_s(count),
+                      fsa_state2string(cur_state), source);
+            crmd_join_phase_log(LOG_DEBUG);
+
+        } else {
+            crm_debug("join-%d: Complete " CRM_XS " state=%s for=%s",
+                      current_join_id, fsa_state2string(cur_state), source);
+            register_fsa_input_later(C_FSA_INTERNAL, I_FINALIZED, NULL);
+            return TRUE;
+        }
+    }
+
+    return FALSE;
+}
+
+void
+do_dc_join_final(long long action,
+                 enum crmd_fsa_cause cause,
+                 enum crmd_fsa_state cur_state,
+                 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    crm_debug("Ensuring DC, quorum and node attributes are up-to-date");
+    crm_update_quorum(crm_have_quorum, TRUE);
+}
+
+int crmd_join_phase_count(enum crm_join_phase phase)
+{
+    int count = 0;
+    crm_node_t *peer;
+    GHashTableIter iter;
+
+    g_hash_table_iter_init(&iter, crm_peer_cache);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
+        if(peer->join == phase) {
+            count++;
+        }
+    }
+    return count;
+}
+
+void crmd_join_phase_log(int level)
+{
+    crm_node_t *peer;
+    GHashTableIter iter;
+
+    g_hash_table_iter_init(&iter, crm_peer_cache);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
+        do_crm_log(level, "join-%d: %s=%s", current_join_id, peer->uname,
+                   crm_join_phase_str(peer->join));
+    }
+}
diff --git a/daemons/controld/controld_lrm.h b/daemons/controld/controld_lrm.h
new file mode 100644
index 0000000..25f3db3
--- /dev/null
+++ b/daemons/controld/controld_lrm.h
@@ -0,0 +1,188 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+#ifndef CONTROLD_LRM__H
+#  define CONTROLD_LRM__H
+
+#include <controld_messages.h>
+
+extern gboolean verify_stopped(enum crmd_fsa_state cur_state, int log_level);
+void lrm_clear_last_failure(const char *rsc_id, const char *node_name,
+                            const char *operation, guint interval_ms);
+void lrm_op_callback(lrmd_event_data_t * op);
+lrmd_t *crmd_local_lrmd_conn(void);
+
+typedef struct resource_history_s {
+    char *id;
+    uint32_t last_callid;
+    lrmd_rsc_info_t rsc;
+    lrmd_event_data_t *last;
+    lrmd_event_data_t *failed;
+    GList *recurring_op_list;
+
+    /* Resources must be stopped using the same
+     * parameters they were started with.  This hashtable
+     * holds the parameters that should be used for the next stop
+     * cmd on this resource. */
+    GHashTable *stop_params;
+} rsc_history_t;
+
+void history_free(gpointer data);
+
+enum active_op_e {
+    active_op_remove    = (1 << 0),
+    active_op_cancelled = (1 << 1),
+};
+
+// In-flight action (recurring or pending)
+typedef struct active_op_s {
+    guint interval_ms;
+    int call_id;
+    uint32_t flags; // bitmask of active_op_e
+    time_t start_time;
+    time_t lock_time;
+    char *rsc_id;
+    char *op_type;
+    char *op_key;
+    char *user_data;
+    GHashTable *params;
+} active_op_t;
+
+#define controld_set_active_op_flags(active_op, flags_to_set) do {          \
+        (active_op)->flags = pcmk__set_flags_as(__func__, __LINE__,         \
+            LOG_TRACE, "Active operation", (active_op)->op_key,             \
+            (active_op)->flags, (flags_to_set), #flags_to_set);             \
+    } while (0)
+
+#define controld_clear_active_op_flags(active_op, flags_to_clear) do {      \
+        (active_op)->flags = pcmk__clear_flags_as(__func__, __LINE__,       \
+            LOG_TRACE, "Active operation", (active_op)->op_key,             \
+            (active_op)->flags, (flags_to_clear), #flags_to_clear);         \
+    } while (0)
+
+typedef struct lrm_state_s {
+    const char *node_name;
+    void *conn;                 // Reserved for controld_execd_state.c usage
+    void *remote_ra_data;       // Reserved for controld_remote_ra.c usage
+
+    GHashTable *resource_history;
+    GHashTable *active_ops;     // Pending and recurring actions
+    GHashTable *deletion_ops;
+    GHashTable *rsc_info_cache;
+    GHashTable *metadata_cache; // key = class[:provider]:agent, value = ra_metadata_s
+
+    int num_lrm_register_fails;
+} lrm_state_t;
+
+struct pending_deletion_op_s {
+    char *rsc;
+    ha_msg_input_t *input;
+};
+
+/*!
+ * \brief Check whether this the local IPC connection to the executor
+ */
+gboolean
+lrm_state_is_local(lrm_state_t *lrm_state);
+
+/*!
+ * \brief Clear all state information from a single state entry.
+ * \note It sometimes useful to save metadata cache when it won't go stale.
+ * \note This does not close the executor connection
+ */
+void lrm_state_reset_tables(lrm_state_t * lrm_state, gboolean reset_metadata);
+GList *lrm_state_get_list(void);
+
+/*!
+ * \brief Initiate internal state tables
+ */
+gboolean lrm_state_init_local(void);
+
+/*!
+ * \brief Destroy all state entries and internal state tables
+ */
+void lrm_state_destroy_all(void);
+
+/*!
+ * \brief Destroy executor connection by node name
+ */
+void lrm_state_destroy(const char *node_name);
+
+/*!
+ * \brief Find lrm_state data by node name
+ */
+lrm_state_t *lrm_state_find(const char *node_name);
+
+/*!
+ * \brief Either find or create a new entry
+ */
+lrm_state_t *lrm_state_find_or_create(const char *node_name);
+
+/*!
+ * The functions below are wrappers for the executor API the the controller
+ * uses. These wrapper functions allow us to treat the controller's remote
+ * executor connection resources the same as regular resources. Internally,
+ * regular resources go to the executor, and remote connection resources are
+ * handled locally in the controller.
+ */
+void lrm_state_disconnect_only(lrm_state_t * lrm_state);
+void lrm_state_disconnect(lrm_state_t * lrm_state);
+int controld_connect_local_executor(lrm_state_t *lrm_state);
+int controld_connect_remote_executor(lrm_state_t *lrm_state, const char *server,
+                                     int port, int timeout);
+int lrm_state_is_connected(lrm_state_t * lrm_state);
+int lrm_state_poke_connection(lrm_state_t * lrm_state);
+
+int lrm_state_get_metadata(lrm_state_t * lrm_state,
+                           const char *class,
+                           const char *provider,
+                           const char *agent, char **output, enum lrmd_call_options options);
+int lrm_state_cancel(lrm_state_t *lrm_state, const char *rsc_id,
+                     const char *action, guint interval_ms);
+int controld_execute_resource_agent(lrm_state_t *lrm_state, const char *rsc_id,
+                                    const char *action, const char *userdata,
+                                    guint interval_ms, int timeout_ms,
+                                    int start_delay_ms,
+                                    GHashTable *parameters, int *call_id);
+lrmd_rsc_info_t *lrm_state_get_rsc_info(lrm_state_t * lrm_state,
+                                        const char *rsc_id, enum lrmd_call_options options);
+int lrm_state_register_rsc(lrm_state_t * lrm_state,
+                           const char *rsc_id,
+                           const char *class,
+                           const char *provider, const char *agent, enum lrmd_call_options options);
+int lrm_state_unregister_rsc(lrm_state_t * lrm_state,
+                             const char *rsc_id, enum lrmd_call_options options);
+
+// Functions used to manage remote executor connection resources
+void remote_lrm_op_callback(lrmd_event_data_t * op);
+gboolean is_remote_lrmd_ra(const char *agent, const char *provider, const char *id);
+lrmd_rsc_info_t *remote_ra_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id);
+int remote_ra_cancel(lrm_state_t *lrm_state, const char *rsc_id,
+                     const char *action, guint interval_ms);
+int controld_execute_remote_agent(const lrm_state_t *lrm_state,
+                                  const char *rsc_id, const char *action,
+                                  const char *userdata,
+                                  guint interval_ms, int timeout_ms,
+                                  int start_delay_ms, lrmd_key_value_t *params,
+                                  int *call_id);
+void remote_ra_cleanup(lrm_state_t * lrm_state);
+void remote_ra_fail(const char *node_name);
+void remote_ra_process_pseudo(xmlNode *xml);
+gboolean remote_ra_is_in_maintenance(lrm_state_t * lrm_state);
+void remote_ra_process_maintenance_nodes(xmlNode *xml);
+gboolean remote_ra_controlling_guest(lrm_state_t * lrm_state);
+
+void process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
+                       active_op_t *pending, const xmlNode *action_xml);
+void controld_ack_event_directly(const char *to_host, const char *to_sys,
+                                 const lrmd_rsc_info_t *rsc,
+                                 lrmd_event_data_t *op, const char *rsc_id);
+void controld_rc2event(lrmd_event_data_t *event, int rc);
+void controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id);
+
+#endif
diff --git a/daemons/controld/controld_matrix.c b/daemons/controld/controld_matrix.c
new file mode 100644
index 0000000..a404f0a
--- /dev/null
+++ b/daemons/controld/controld_matrix.c
@@ -0,0 +1,1250 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdint.h>                 // uint64_t
+
+#include <pacemaker-controld.h>
+
+/*
+ *	The state transition table.  The rows are inputs, and
+ *	the columns are states.
+ */
+static const enum crmd_fsa_state fsa_next_states[MAXINPUT][MAXSTATE] = {
+/* Got an I_NULL */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_CIB_OP */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_CIB_UPDATE */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_RECOVERY,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_RECOVERY,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_DC_TIMEOUT */
+    {
+     /* S_IDLE               ==> */ S_RECOVERY,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_RECOVERY,
+     /* S_FINALIZE_JOIN      ==> */ S_RECOVERY,
+     /* S_NOT_DC             ==> */ S_ELECTION,
+     /* S_POLICY_ENGINE      ==> */ S_RECOVERY,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RECOVERY,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_ELECTION,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_RECOVERY,
+     /* S_HALT               ==> */ S_ELECTION,
+     },
+
+/* Got an I_ELECTION */
+    {
+     /* S_IDLE               ==> */ S_ELECTION,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_ELECTION,
+     /* S_FINALIZE_JOIN      ==> */ S_ELECTION,
+     /* S_NOT_DC             ==> */ S_ELECTION,
+     /* S_POLICY_ENGINE      ==> */ S_ELECTION,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_ELECTION,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_ELECTION,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_PE_CALC */
+    {
+     /* S_IDLE               ==> */ S_POLICY_ENGINE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_POLICY_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_RELEASE_DC */
+    {
+     /* S_IDLE               ==> */ S_RELEASE_DC,
+     /* S_ELECTION           ==> */ S_RELEASE_DC,
+     /* S_INTEGRATION        ==> */ S_RELEASE_DC,
+     /* S_FINALIZE_JOIN      ==> */ S_RELEASE_DC,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_RELEASE_DC,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_RELEASE_DC,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_ELECTION_DC */
+    {
+     /* S_IDLE               ==> */ S_INTEGRATION,
+     /* S_ELECTION           ==> */ S_INTEGRATION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_INTEGRATION,
+     /* S_NOT_DC             ==> */ S_INTEGRATION,
+     /* S_POLICY_ENGINE      ==> */ S_INTEGRATION,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_INTEGRATION,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_ERROR */
+    {
+     /* S_IDLE               ==> */ S_RECOVERY,
+     /* S_ELECTION           ==> */ S_RECOVERY,
+     /* S_INTEGRATION        ==> */ S_RECOVERY,
+     /* S_FINALIZE_JOIN      ==> */ S_RECOVERY,
+     /* S_NOT_DC             ==> */ S_RECOVERY,
+     /* S_POLICY_ENGINE      ==> */ S_RECOVERY,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RECOVERY,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_RECOVERY,
+     /* S_STOPPING           ==> */ S_TERMINATE,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_RECOVERY,
+     /* S_HALT               ==> */ S_RECOVERY,
+     },
+
+/* Got an I_FAIL */
+    {
+     /* S_IDLE               ==> */ S_RECOVERY,
+     /* S_ELECTION           ==> */ S_RELEASE_DC,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_INTEGRATION,
+     /* S_NOT_DC             ==> */ S_RECOVERY,
+     /* S_POLICY_ENGINE      ==> */ S_INTEGRATION,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STOPPING,
+     /* S_PENDING            ==> */ S_STOPPING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_POLICY_ENGINE,
+     /* S_HALT               ==> */ S_RELEASE_DC,
+     },
+
+/* Got an I_INTEGRATED */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_FINALIZE_JOIN,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_RECOVERY,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_FINALIZED */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_POLICY_ENGINE,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_RECOVERY,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_NODE_JOIN */
+    {
+     /* S_IDLE               ==> */ S_INTEGRATION,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_INTEGRATION,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_INTEGRATION,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_INTEGRATION,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_NOT_DC */
+    {
+     /* S_IDLE               ==> */ S_RECOVERY,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_RECOVERY,
+     /* S_FINALIZE_JOIN      ==> */ S_RECOVERY,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_RECOVERY,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_NOT_DC,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_RECOVERY,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_RECOVERED */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_INTEGRATION,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_PENDING,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_RELEASE_FAIL */
+    {
+     /* S_IDLE               ==> */ S_STOPPING,
+     /* S_ELECTION           ==> */ S_STOPPING,
+     /* S_INTEGRATION        ==> */ S_STOPPING,
+     /* S_FINALIZE_JOIN      ==> */ S_STOPPING,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_STOPPING,
+     /* S_RECOVERY           ==> */ S_STOPPING,
+     /* S_RELEASE_DC         ==> */ S_STOPPING,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_STOPPING,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_RELEASE_SUCCESS */
+    {
+     /* S_IDLE               ==> */ S_RECOVERY,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_RECOVERY,
+     /* S_FINALIZE_JOIN      ==> */ S_RECOVERY,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_RECOVERY,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_PENDING,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_RECOVERY,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_RESTART */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_TE_SUCCESS */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_IDLE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_ROUTER */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_SHUTDOWN */
+    {
+     /* S_IDLE               ==> */ S_POLICY_ENGINE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_STOPPING,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STOPPING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_POLICY_ENGINE,
+     /* S_HALT               ==> */ S_ELECTION,
+     },
+
+/* Got an I_STOP */
+    {
+     /* S_IDLE               ==> */ S_STOPPING,
+     /* S_ELECTION           ==> */ S_STOPPING,
+     /* S_INTEGRATION        ==> */ S_STOPPING,
+     /* S_FINALIZE_JOIN      ==> */ S_STOPPING,
+     /* S_NOT_DC             ==> */ S_STOPPING,
+     /* S_POLICY_ENGINE      ==> */ S_STOPPING,
+     /* S_RECOVERY           ==> */ S_STOPPING,
+     /* S_RELEASE_DC         ==> */ S_STOPPING,
+     /* S_STARTING           ==> */ S_STOPPING,
+     /* S_PENDING            ==> */ S_STOPPING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_STOPPING,
+     /* S_HALT               ==> */ S_STOPPING,
+     },
+
+/* Got an I_TERMINATE */
+    {
+     /* S_IDLE               ==> */ S_TERMINATE,
+     /* S_ELECTION           ==> */ S_TERMINATE,
+     /* S_INTEGRATION        ==> */ S_TERMINATE,
+     /* S_FINALIZE_JOIN      ==> */ S_TERMINATE,
+     /* S_NOT_DC             ==> */ S_TERMINATE,
+     /* S_POLICY_ENGINE      ==> */ S_TERMINATE,
+     /* S_RECOVERY           ==> */ S_TERMINATE,
+     /* S_RELEASE_DC         ==> */ S_TERMINATE,
+     /* S_STARTING           ==> */ S_TERMINATE,
+     /* S_PENDING            ==> */ S_TERMINATE,
+     /* S_STOPPING           ==> */ S_TERMINATE,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TERMINATE,
+     /* S_HALT               ==> */ S_TERMINATE,
+     },
+
+/* Got an I_STARTUP */
+    {
+     /* S_IDLE               ==> */ S_RECOVERY,
+     /* S_ELECTION           ==> */ S_RECOVERY,
+     /* S_INTEGRATION        ==> */ S_RECOVERY,
+     /* S_FINALIZE_JOIN      ==> */ S_RECOVERY,
+     /* S_NOT_DC             ==> */ S_RECOVERY,
+     /* S_POLICY_ENGINE      ==> */ S_RECOVERY,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_RECOVERY,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_PE_SUCCESS */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_TRANSITION_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_JOIN_OFFER */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_PENDING,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_JOIN_REQUEST */
+    {
+     /* S_IDLE               ==> */ S_INTEGRATION,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_INTEGRATION,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_INTEGRATION,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_INTEGRATION,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_JOIN_RESULT */
+    {
+     /* S_IDLE               ==> */ S_INTEGRATION,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_PENDING,
+     /* S_POLICY_ENGINE      ==> */ S_INTEGRATION,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_RECOVERY,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_INTEGRATION,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_WAIT_FOR_EVENT */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_DC_HEARTBEAT */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_LRM_EVENT */
+    {
+     /* S_IDLE               ==> */ S_IDLE,
+     /* S_ELECTION           ==> */ S_ELECTION,
+     /* S_INTEGRATION        ==> */ S_INTEGRATION,
+     /* S_FINALIZE_JOIN      ==> */ S_FINALIZE_JOIN,
+     /* S_NOT_DC             ==> */ S_NOT_DC,
+     /* S_POLICY_ENGINE      ==> */ S_POLICY_ENGINE,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_TRANSITION_ENGINE,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_PENDING */
+    {
+     /* S_IDLE               ==> */ S_PENDING,
+     /* S_ELECTION           ==> */ S_PENDING,
+     /* S_INTEGRATION        ==> */ S_PENDING,
+     /* S_FINALIZE_JOIN      ==> */ S_PENDING,
+     /* S_NOT_DC             ==> */ S_PENDING,
+     /* S_POLICY_ENGINE      ==> */ S_PENDING,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_PENDING,
+     /* S_PENDING            ==> */ S_PENDING,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_PENDING,
+     /* S_HALT               ==> */ S_HALT,
+     },
+
+/* Got an I_HALT */
+    {
+     /* S_IDLE               ==> */ S_HALT,
+     /* S_ELECTION           ==> */ S_HALT,
+     /* S_INTEGRATION        ==> */ S_HALT,
+     /* S_FINALIZE_JOIN      ==> */ S_HALT,
+     /* S_NOT_DC             ==> */ S_HALT,
+     /* S_POLICY_ENGINE      ==> */ S_HALT,
+     /* S_RECOVERY           ==> */ S_RECOVERY,
+     /* S_RELEASE_DC         ==> */ S_RELEASE_DC,
+     /* S_STARTING           ==> */ S_STARTING,
+     /* S_PENDING            ==> */ S_HALT,
+     /* S_STOPPING           ==> */ S_STOPPING,
+     /* S_TERMINATE          ==> */ S_TERMINATE,
+     /* S_TRANSITION_ENGINE  ==> */ S_HALT,
+     /* S_HALT               ==> */ S_HALT,
+     },
+};
+
+/*
+ *	The action table.  Each entry is a set of actions to take or-ed
+ *	together.  Like the state table, the rows are inputs, and
+ *	the columns are states.
+ */
+
+/* NOTE: In the fsa, the actions are extracted then state is updated. */
+
+static const uint64_t fsa_actions[MAXINPUT][MAXSTATE] = {
+
+/* Got an I_NULL */
+    {
+     /* S_IDLE               ==> */ A_NOTHING,
+     /* S_ELECTION           ==> */ A_NOTHING,
+     /* S_INTEGRATION        ==> */ A_NOTHING,
+     /* S_FINALIZE_JOIN      ==> */ A_NOTHING,
+     /* S_NOT_DC             ==> */ A_NOTHING,
+     /* S_POLICY_ENGINE      ==> */ A_NOTHING,
+     /* S_RECOVERY           ==> */ A_NOTHING,
+     /* S_RELEASE_DC         ==> */ A_NOTHING,
+     /* S_STARTING           ==> */ A_NOTHING,
+     /* S_PENDING            ==> */ A_NOTHING,
+     /* S_STOPPING           ==> */ A_NOTHING,
+     /* S_TERMINATE          ==> */ A_NOTHING,
+     /* S_TRANSITION_ENGINE  ==> */ A_NOTHING,
+     /* S_HALT               ==> */ A_NOTHING,
+     },
+
+/* Got an I_CIB_OP */
+    {
+     /* S_IDLE               ==> */ A_ERROR,
+     /* S_ELECTION           ==> */ A_ERROR,
+     /* S_INTEGRATION        ==> */ A_ERROR,
+     /* S_FINALIZE_JOIN      ==> */ A_ERROR,
+     /* S_NOT_DC             ==> */ A_ERROR,
+     /* S_POLICY_ENGINE      ==> */ A_ERROR,
+     /* S_RECOVERY           ==> */ A_ERROR,
+     /* S_RELEASE_DC         ==> */ A_ERROR,
+     /* S_STARTING           ==> */ A_ERROR,
+     /* S_PENDING            ==> */ A_ERROR,
+     /* S_STOPPING           ==> */ A_ERROR,
+     /* S_TERMINATE          ==> */ A_ERROR,
+     /* S_TRANSITION_ENGINE  ==> */ A_ERROR,
+     /* S_HALT               ==> */ A_ERROR,
+     },
+
+/* Got an I_CIB_UPDATE */
+    {
+     /* S_IDLE               ==> */ A_LOG,
+     /* S_ELECTION           ==> */ A_LOG,
+     /* S_INTEGRATION        ==> */ A_WARN,
+     /* S_FINALIZE_JOIN      ==> */ A_WARN,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_LOG,
+     /* S_RECOVERY           ==> */ A_WARN,
+     /* S_RELEASE_DC         ==> */ A_WARN,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_WARN,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_LOG,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_DC_TIMEOUT */
+    {
+     /* S_IDLE               ==> */ A_WARN,
+     /* S_ELECTION           ==> */ A_ELECTION_VOTE,
+     /* S_INTEGRATION        ==> */ A_WARN,
+     /* S_FINALIZE_JOIN      ==> */ A_WARN,
+     /* S_NOT_DC             ==> */ A_ELECTION_VOTE | A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_WARN,
+     /* S_RECOVERY           ==> */ A_NOTHING,
+     /* S_RELEASE_DC         ==> */ A_WARN,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_ELECTION_VOTE | A_WARN,
+     /* S_STOPPING           ==> */ A_NOTHING,
+     /* S_TERMINATE          ==> */ A_NOTHING,
+     /* S_TRANSITION_ENGINE  ==> */ A_TE_CANCEL | A_WARN,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_ELECTION */
+    {
+     /* S_IDLE               ==> */ A_ELECTION_VOTE,
+     /* S_ELECTION           ==> */ A_ELECTION_VOTE,
+     /* S_INTEGRATION        ==> */ A_ELECTION_VOTE,
+     /* S_FINALIZE_JOIN      ==> */ A_ELECTION_VOTE,
+     /* S_NOT_DC             ==> */ A_ELECTION_VOTE,
+     /* S_POLICY_ENGINE      ==> */ A_ELECTION_VOTE,
+     /* S_RECOVERY           ==> */ A_LOG,
+     /* S_RELEASE_DC         ==> */ A_LOG,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_ELECTION_VOTE,
+     /* S_STOPPING           ==> */ A_LOG,
+     /* S_TERMINATE          ==> */ A_LOG,
+     /* S_TRANSITION_ENGINE  ==> */ A_ELECTION_VOTE,
+     /* S_HALT               ==> */ A_ELECTION_VOTE,
+     },
+
+/* Got an I_PE_CALC */
+    {
+     /* S_IDLE               ==> */ A_PE_INVOKE,
+     /* S_ELECTION           ==> */ A_NOTHING,
+     /* S_INTEGRATION        ==> */ A_NOTHING,
+     /* S_FINALIZE_JOIN      ==> */ A_NOTHING,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_PE_INVOKE,
+     /* S_RECOVERY           ==> */ A_NOTHING,
+     /* S_RELEASE_DC         ==> */ A_NOTHING,
+     /* S_STARTING           ==> */ A_ERROR,
+     /* S_PENDING            ==> */ A_WARN,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_ERROR,
+     /* S_TRANSITION_ENGINE  ==> */ A_PE_INVOKE,
+     /* S_HALT               ==> */ A_ERROR,
+     },
+
+/* Got an I_RELEASE_DC */
+    {
+     /* S_IDLE               ==> */ O_RELEASE,
+     /* S_ELECTION           ==> */ O_RELEASE,
+     /* S_INTEGRATION        ==> */ O_RELEASE | A_WARN,
+     /* S_FINALIZE_JOIN      ==> */ O_RELEASE | A_WARN,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ O_RELEASE | A_WARN,
+     /* S_RECOVERY           ==> */ O_RELEASE,
+     /* S_RELEASE_DC         ==> */ O_RELEASE | A_WARN,
+     /* S_STARTING           ==> */ A_ERROR,
+     /* S_PENDING            ==> */ A_WARN,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ O_RELEASE | A_WARN,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_ELECTION_DC */
+    {
+     /* S_IDLE               ==> */ A_WARN | A_ELECTION_VOTE,
+     /* S_ELECTION           ==> */
+     A_LOG | A_DC_TAKEOVER | A_PE_START | A_TE_START | A_DC_JOIN_OFFER_ALL | A_DC_TIMER_STOP,
+     /* S_INTEGRATION        ==> */ A_WARN | A_ELECTION_VOTE | A_DC_JOIN_OFFER_ALL,
+     /* S_FINALIZE_JOIN      ==> */ A_WARN | A_ELECTION_VOTE | A_DC_JOIN_OFFER_ALL,
+     /* S_NOT_DC             ==> */ A_LOG | A_ELECTION_VOTE,
+     /* S_POLICY_ENGINE      ==> */ A_WARN | A_ELECTION_VOTE,
+     /* S_RECOVERY           ==> */ A_WARN,
+     /* S_RELEASE_DC         ==> */ A_WARN | A_ELECTION_VOTE,
+     /* S_STARTING           ==> */ A_LOG | A_WARN,
+     /* S_PENDING            ==> */ A_LOG | A_WARN,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_WARN | A_ELECTION_VOTE,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_ERROR */
+    {
+     /* S_IDLE               ==> */ A_ERROR | A_RECOVER | O_RELEASE | A_ELECTION_START,
+     /* S_ELECTION           ==> */ A_ERROR | A_RECOVER | O_RELEASE,
+     /* S_INTEGRATION        ==> */ A_ERROR | A_RECOVER | O_RELEASE | A_ELECTION_START,
+     /* S_FINALIZE_JOIN      ==> */ A_ERROR | A_RECOVER | O_RELEASE | A_ELECTION_START,
+     /* S_NOT_DC             ==> */ A_ERROR | A_RECOVER,
+     /* S_POLICY_ENGINE      ==> */ A_ERROR | A_RECOVER | O_RELEASE | A_ELECTION_START,
+     /* S_RECOVERY           ==> */ A_ERROR | O_RELEASE,
+     /* S_RELEASE_DC         ==> */ A_ERROR | A_RECOVER,
+     /* S_STARTING           ==> */ A_ERROR | A_RECOVER,
+     /* S_PENDING            ==> */ A_ERROR | A_RECOVER,
+     /* S_STOPPING           ==> */ A_ERROR | A_EXIT_1,
+     /* S_TERMINATE          ==> */ A_ERROR | A_EXIT_1,
+     /* S_TRANSITION_ENGINE  ==> */ A_ERROR | A_RECOVER | O_RELEASE | A_ELECTION_START,
+     /* S_HALT               ==> */ A_ERROR | A_RECOVER | O_RELEASE | A_ELECTION_START,
+     },
+
+/* Got an I_FAIL */
+    {
+     /* S_IDLE               ==> */ A_WARN,
+     /* S_ELECTION           ==> */ A_WARN,
+     /* S_INTEGRATION        ==> */ A_WARN | A_DC_JOIN_OFFER_ALL,
+     /* S_FINALIZE_JOIN      ==> */ A_WARN | A_DC_JOIN_OFFER_ALL,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_WARN | A_DC_JOIN_OFFER_ALL | A_TE_CANCEL,
+     /* S_RECOVERY           ==> */ A_WARN | O_RELEASE,
+     /* S_RELEASE_DC         ==> */ A_WARN,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_WARN,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN | A_EXIT_1,
+     /* S_TRANSITION_ENGINE  ==> */ A_WARN | O_LRM_RECONNECT | A_PE_INVOKE | A_TE_CANCEL,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_INTEGRATED */
+    {
+     /* S_IDLE               ==> */ A_NOTHING,
+     /* S_ELECTION           ==> */ A_WARN,
+     /* S_INTEGRATION        ==> */ A_DC_JOIN_FINALIZE,
+     /* S_FINALIZE_JOIN      ==> */ A_WARN,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_NOTHING,
+     /* S_RECOVERY           ==> */ A_WARN,
+     /* S_RELEASE_DC         ==> */ A_WARN,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_WARN,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_NOTHING,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_FINALIZED */
+    {
+     /* S_IDLE               ==> */ A_NOTHING,
+     /* S_ELECTION           ==> */ A_WARN,
+     /* S_INTEGRATION        ==> */ A_WARN,
+     /* S_FINALIZE_JOIN      ==> */ A_DC_JOIN_FINAL | A_TE_CANCEL,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_NOTHING,
+     /* S_RECOVERY           ==> */ A_WARN,
+     /* S_RELEASE_DC         ==> */ A_WARN,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_WARN,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_NOTHING,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_NODE_JOIN */
+    {
+     /* S_IDLE               ==> */ A_TE_HALT | A_DC_JOIN_OFFER_ONE,
+     /* S_ELECTION           ==> */ A_WARN,
+     /* S_INTEGRATION        ==> */ A_DC_JOIN_OFFER_ONE,
+     /* S_FINALIZE_JOIN      ==> */ A_DC_JOIN_OFFER_ONE,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_DC_JOIN_OFFER_ONE,
+     /* S_RECOVERY           ==> */ A_WARN,
+     /* S_RELEASE_DC         ==> */ A_WARN,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_WARN,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_TE_HALT | A_DC_JOIN_OFFER_ONE,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_NOT_DC */
+    {
+     /* S_IDLE               ==> */ A_WARN | O_RELEASE,
+     /* S_ELECTION           ==> */ A_ERROR | A_ELECTION_START | A_DC_TIMER_STOP,
+     /* S_INTEGRATION        ==> */ A_ERROR | O_RELEASE,
+     /* S_FINALIZE_JOIN      ==> */ A_ERROR | O_RELEASE,
+     /* S_NOT_DC             ==> */ A_LOG,
+     /* S_POLICY_ENGINE      ==> */ A_ERROR | O_RELEASE,
+     /* S_RECOVERY           ==> */ A_ERROR | O_RELEASE,
+     /* S_RELEASE_DC         ==> */ A_ERROR | O_RELEASE,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_LOG | A_DC_TIMER_STOP,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_ERROR | O_RELEASE,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_RECOVERED */
+    {
+     /* S_IDLE               ==> */ A_WARN,
+     /* S_ELECTION           ==> */ A_ELECTION_VOTE,
+     /* S_INTEGRATION        ==> */ A_WARN,
+     /* S_FINALIZE_JOIN      ==> */ A_WARN,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_WARN,
+     /* S_RECOVERY           ==> */ A_LOG,
+     /* S_RELEASE_DC         ==> */ A_WARN,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_WARN,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_WARN,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_RELEASE_FAIL */
+    {
+     /* S_IDLE               ==> */ A_WARN,
+     /* S_ELECTION           ==> */ A_WARN,
+     /* S_INTEGRATION        ==> */ A_WARN,
+     /* S_FINALIZE_JOIN      ==> */ A_WARN,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_NOTHING,
+     /* S_RECOVERY           ==> */ A_WARN | A_SHUTDOWN_REQ,
+     /* S_RELEASE_DC         ==> */ A_NOTHING,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_WARN,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_WARN,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_RELEASE_SUCCESS */
+    {
+     /* S_IDLE               ==> */ A_WARN,
+     /* S_ELECTION           ==> */ A_WARN,
+     /* S_INTEGRATION        ==> */ A_WARN,
+     /* S_FINALIZE_JOIN      ==> */ A_WARN,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_WARN,
+     /* S_RECOVERY           ==> */ A_WARN,
+     /* S_RELEASE_DC         ==> */ A_LOG,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_LOG,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_WARN,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_RESTART */
+    {
+     /* S_IDLE               ==> */ A_NOTHING,
+     /* S_ELECTION           ==> */ A_LOG | A_ELECTION_VOTE,
+     /* S_INTEGRATION        ==> */ A_LOG | A_DC_JOIN_OFFER_ALL,
+     /* S_FINALIZE_JOIN      ==> */ A_LOG | A_DC_JOIN_FINALIZE,
+     /* S_NOT_DC             ==> */ A_LOG | A_NOTHING,
+     /* S_POLICY_ENGINE      ==> */ A_LOG | A_PE_INVOKE,
+     /* S_RECOVERY           ==> */ A_LOG | A_RECOVER | O_RELEASE,
+     /* S_RELEASE_DC         ==> */ A_LOG | O_RELEASE,
+     /* S_STARTING           ==> */ A_LOG,
+     /* S_PENDING            ==> */ A_LOG,
+     /* S_STOPPING           ==> */ A_LOG,
+     /* S_TERMINATE          ==> */ A_LOG,
+     /* S_TRANSITION_ENGINE  ==> */ A_LOG | A_TE_INVOKE,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_TE_SUCCESS */
+    {
+     /* S_IDLE               ==> */ A_LOG,
+     /* S_ELECTION           ==> */ A_WARN,
+     /* S_INTEGRATION        ==> */ A_WARN,
+     /* S_FINALIZE_JOIN      ==> */ A_WARN,
+     /* S_NOT_DC             ==> */ A_ERROR,
+     /* S_POLICY_ENGINE      ==> */ A_WARN,
+     /* S_RECOVERY           ==> */ A_RECOVER | A_WARN,
+     /* S_RELEASE_DC         ==> */ A_WARN,
+     /* S_STARTING           ==> */ A_ERROR,
+     /* S_PENDING            ==> */ A_ERROR,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_LOG,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_ROUTER */
+    {
+     /* S_IDLE               ==> */ A_MSG_ROUTE,
+     /* S_ELECTION           ==> */ A_MSG_ROUTE,
+     /* S_INTEGRATION        ==> */ A_MSG_ROUTE,
+     /* S_FINALIZE_JOIN      ==> */ A_MSG_ROUTE,
+     /* S_NOT_DC             ==> */ A_MSG_ROUTE,
+     /* S_POLICY_ENGINE      ==> */ A_MSG_ROUTE,
+     /* S_RECOVERY           ==> */ A_MSG_ROUTE,
+     /* S_RELEASE_DC         ==> */ A_MSG_ROUTE,
+     /* S_STARTING           ==> */ A_MSG_ROUTE,
+     /* S_PENDING            ==> */ A_MSG_ROUTE,
+     /* S_STOPPING           ==> */ A_MSG_ROUTE,
+     /* S_TERMINATE          ==> */ A_MSG_ROUTE,
+     /* S_TRANSITION_ENGINE  ==> */ A_MSG_ROUTE,
+     /* S_HALT               ==> */ A_WARN | A_MSG_ROUTE,
+     },
+
+/* Got an I_SHUTDOWN */
+    {
+     /* S_IDLE               ==> */ A_LOG | A_SHUTDOWN_REQ,
+     /* S_ELECTION           ==> */ A_LOG | A_SHUTDOWN_REQ | A_ELECTION_VOTE,
+     /* S_INTEGRATION        ==> */ A_LOG | A_SHUTDOWN_REQ,
+     /* S_FINALIZE_JOIN      ==> */ A_LOG | A_SHUTDOWN_REQ,
+     /* S_NOT_DC             ==> */ A_SHUTDOWN_REQ,
+     /* S_POLICY_ENGINE      ==> */ A_LOG | A_SHUTDOWN_REQ,
+     /* S_RECOVERY           ==> */ A_WARN | O_EXIT | O_RELEASE,
+     /* S_RELEASE_DC         ==> */ A_WARN | A_SHUTDOWN_REQ,
+     /* S_STARTING           ==> */ A_WARN | O_EXIT,
+     /* S_PENDING            ==> */ A_SHUTDOWN_REQ,
+     /* S_STOPPING           ==> */ A_LOG,
+     /* S_TERMINATE          ==> */ A_LOG,
+     /* S_TRANSITION_ENGINE  ==> */ A_WARN | A_SHUTDOWN_REQ,
+     /* S_HALT               ==> */ A_WARN | A_ELECTION_START | A_SHUTDOWN_REQ,
+     },
+
+/* Got an I_STOP */
+    {
+     /* S_IDLE               ==> */ A_ERROR | O_RELEASE | O_EXIT,
+     /* S_ELECTION           ==> */ O_RELEASE | O_EXIT,
+     /* S_INTEGRATION        ==> */ A_WARN | O_RELEASE | O_EXIT,
+     /* S_FINALIZE_JOIN      ==> */ A_ERROR | O_RELEASE | O_EXIT,
+     /* S_NOT_DC             ==> */ O_EXIT,
+     /* S_POLICY_ENGINE      ==> */ A_WARN | O_RELEASE | O_EXIT,
+     /* S_RECOVERY           ==> */ A_ERROR | O_RELEASE | O_EXIT,
+     /* S_RELEASE_DC         ==> */ A_ERROR | O_RELEASE | O_EXIT,
+     /* S_STARTING           ==> */ O_EXIT,
+     /* S_PENDING            ==> */ O_EXIT,
+     /* S_STOPPING           ==> */ O_EXIT,
+     /* S_TERMINATE          ==> */ A_ERROR | A_EXIT_1,
+     /* S_TRANSITION_ENGINE  ==> */ A_LOG | O_RELEASE | O_EXIT,
+     /* S_HALT               ==> */ O_RELEASE | O_EXIT | A_WARN,
+     },
+
+/* Got an I_TERMINATE */
+    {
+     /* S_IDLE               ==> */ A_ERROR | O_EXIT,
+     /* S_ELECTION           ==> */ A_ERROR | O_EXIT,
+     /* S_INTEGRATION        ==> */ A_ERROR | O_EXIT,
+     /* S_FINALIZE_JOIN      ==> */ A_ERROR | O_EXIT,
+     /* S_NOT_DC             ==> */ A_ERROR | O_EXIT,
+     /* S_POLICY_ENGINE      ==> */ A_ERROR | O_EXIT,
+     /* S_RECOVERY           ==> */ A_ERROR | O_EXIT,
+     /* S_RELEASE_DC         ==> */ A_ERROR | O_EXIT,
+     /* S_STARTING           ==> */ O_EXIT,
+     /* S_PENDING            ==> */ A_ERROR | O_EXIT,
+     /* S_STOPPING           ==> */ O_EXIT,
+     /* S_TERMINATE          ==> */ O_EXIT,
+     /* S_TRANSITION_ENGINE  ==> */ A_ERROR | O_EXIT,
+     /* S_HALT               ==> */ A_ERROR | O_EXIT,
+     },
+
+/* Got an I_STARTUP */
+    {
+     /* S_IDLE               ==> */ A_WARN,
+     /* S_ELECTION           ==> */ A_WARN,
+     /* S_INTEGRATION        ==> */ A_WARN,
+     /* S_FINALIZE_JOIN      ==> */ A_WARN,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_WARN,
+     /* S_RECOVERY           ==> */ A_WARN,
+     /* S_RELEASE_DC         ==> */ A_WARN,
+     /* S_STARTING           ==> */
+     A_LOG | A_STARTUP | A_CIB_START | A_LRM_CONNECT | A_HA_CONNECT | A_READCONFIG | A_STARTED,
+     /* S_PENDING            ==> */ A_LOG,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_WARN,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_PE_SUCCESS */
+    {
+     /* S_IDLE               ==> */ A_LOG,
+     /* S_ELECTION           ==> */ A_WARN,
+     /* S_INTEGRATION        ==> */ A_WARN,
+     /* S_FINALIZE_JOIN      ==> */ A_WARN,
+     /* S_NOT_DC             ==> */ A_NOTHING,
+     /* S_POLICY_ENGINE      ==> */ A_TE_INVOKE,
+     /* S_RECOVERY           ==> */ A_RECOVER | A_LOG,
+     /* S_RELEASE_DC         ==> */ A_LOG,
+     /* S_STARTING           ==> */ A_ERROR,
+     /* S_PENDING            ==> */ A_LOG,
+     /* S_STOPPING           ==> */ A_ERROR,
+     /* S_TERMINATE          ==> */ A_ERROR,
+     /* S_TRANSITION_ENGINE  ==> */ A_LOG,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_JOIN_OFFER */
+    {
+     /* S_IDLE               ==> */ A_WARN | A_CL_JOIN_REQUEST,
+     /* S_ELECTION           ==> */ A_WARN | A_ELECTION_VOTE,
+     /* S_INTEGRATION        ==> */ A_CL_JOIN_REQUEST,
+     /* S_FINALIZE_JOIN      ==> */ A_CL_JOIN_REQUEST,
+     /* S_NOT_DC             ==> */ A_CL_JOIN_REQUEST | A_DC_TIMER_STOP,
+     /* S_POLICY_ENGINE      ==> */ A_WARN | A_CL_JOIN_REQUEST,
+     /* S_RECOVERY           ==> */ A_WARN | A_CL_JOIN_REQUEST | A_DC_TIMER_STOP,
+     /* S_RELEASE_DC         ==> */ A_WARN | A_CL_JOIN_REQUEST | A_DC_TIMER_STOP,
+     /* S_STARTING           ==> */ A_LOG,
+     /* S_PENDING            ==> */ A_CL_JOIN_REQUEST | A_DC_TIMER_STOP,
+     /* S_STOPPING           ==> */ A_LOG,
+     /* S_TERMINATE          ==> */ A_LOG,
+     /* S_TRANSITION_ENGINE  ==> */ A_WARN | A_CL_JOIN_REQUEST,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_JOIN_REQUEST */
+    {
+     /* S_IDLE               ==> */ A_DC_JOIN_OFFER_ONE,
+     /* S_ELECTION           ==> */ A_WARN,
+     /* S_INTEGRATION        ==> */ A_DC_JOIN_PROCESS_REQ,
+     /* S_FINALIZE_JOIN      ==> */ A_DC_JOIN_OFFER_ONE,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_DC_JOIN_OFFER_ONE,
+     /* S_RECOVERY           ==> */ A_WARN,
+     /* S_RELEASE_DC         ==> */ A_WARN,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_WARN,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_DC_JOIN_OFFER_ONE,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_JOIN_RESULT */
+    {
+     /* S_IDLE               ==> */ A_ERROR | A_TE_HALT | A_DC_JOIN_OFFER_ALL,
+     /* S_ELECTION           ==> */ A_LOG,
+     /* S_INTEGRATION        ==> */ A_LOG | A_CL_JOIN_RESULT  | A_DC_JOIN_PROCESS_ACK,
+     /* S_FINALIZE_JOIN      ==> */ A_CL_JOIN_RESULT | A_DC_JOIN_PROCESS_ACK,
+     /* S_NOT_DC             ==> */ A_ERROR | A_CL_JOIN_ANNOUNCE,
+     /* S_POLICY_ENGINE      ==> */ A_ERROR | A_TE_HALT | A_DC_JOIN_OFFER_ALL,
+     /* S_RECOVERY           ==> */ A_LOG,
+     /* S_RELEASE_DC         ==> */ A_LOG,
+     /* S_STARTING           ==> */ A_ERROR,
+     /* S_PENDING            ==> */ A_CL_JOIN_RESULT,
+     /* S_STOPPING           ==> */ A_ERROR,
+     /* S_TERMINATE          ==> */ A_ERROR,
+     /* S_TRANSITION_ENGINE  ==> */ A_ERROR | A_TE_HALT | A_DC_JOIN_OFFER_ALL,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_WAIT_FOR_EVENT */
+    {
+     /* S_IDLE               ==> */ A_LOG,
+     /* S_ELECTION           ==> */ A_LOG,
+     /* S_INTEGRATION        ==> */ A_LOG,
+     /* S_FINALIZE_JOIN      ==> */ A_LOG,
+     /* S_NOT_DC             ==> */ A_LOG,
+     /* S_POLICY_ENGINE      ==> */ A_LOG,
+     /* S_RECOVERY           ==> */ A_LOG,
+     /* S_RELEASE_DC         ==> */ A_LOG,
+     /* S_STARTING           ==> */ A_LOG,
+     /* S_PENDING            ==> */ A_LOG,
+     /* S_STOPPING           ==> */ A_LOG,
+     /* S_TERMINATE          ==> */ A_LOG,
+     /* S_TRANSITION_ENGINE  ==> */ A_LOG,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_DC_HEARTBEAT */
+    {
+     /* S_IDLE               ==> */ A_ERROR,
+     /* S_ELECTION           ==> */ A_WARN | A_ELECTION_VOTE,
+     /* S_INTEGRATION        ==> */ A_ERROR,
+     /* S_FINALIZE_JOIN      ==> */ A_ERROR,
+     /* S_NOT_DC             ==> */ A_NOTHING,
+     /* S_POLICY_ENGINE      ==> */ A_ERROR,
+     /* S_RECOVERY           ==> */ A_NOTHING,
+     /* S_RELEASE_DC         ==> */ A_LOG,
+     /* S_STARTING           ==> */ A_LOG,
+     /* S_PENDING            ==> */ A_LOG | A_CL_JOIN_ANNOUNCE,
+     /* S_STOPPING           ==> */ A_NOTHING,
+     /* S_TERMINATE          ==> */ A_NOTHING,
+     /* S_TRANSITION_ENGINE  ==> */ A_ERROR,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_LRM_EVENT */
+    {
+     /* S_IDLE               ==> */ A_LRM_EVENT,
+     /* S_ELECTION           ==> */ A_LRM_EVENT,
+     /* S_INTEGRATION        ==> */ A_LRM_EVENT,
+     /* S_FINALIZE_JOIN      ==> */ A_LRM_EVENT,
+     /* S_NOT_DC             ==> */ A_LRM_EVENT,
+     /* S_POLICY_ENGINE      ==> */ A_LRM_EVENT,
+     /* S_RECOVERY           ==> */ A_LRM_EVENT,
+     /* S_RELEASE_DC         ==> */ A_LRM_EVENT,
+     /* S_STARTING           ==> */ A_LRM_EVENT,
+     /* S_PENDING            ==> */ A_LRM_EVENT,
+     /* S_STOPPING           ==> */ A_LRM_EVENT,
+     /* S_TERMINATE          ==> */ A_LRM_EVENT,
+     /* S_TRANSITION_ENGINE  ==> */ A_LRM_EVENT,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* For everyone ending up in S_PENDING, (re)start the DC timer and wait for I_JOIN_OFFER or I_NOT_DC */
+/* Got an I_PENDING */
+    {
+     /* S_IDLE               ==> */ O_RELEASE | O_DC_TIMER_RESTART,
+     /* S_ELECTION           ==> */ O_RELEASE | O_DC_TIMER_RESTART,
+     /* S_INTEGRATION        ==> */ O_RELEASE | O_DC_TIMER_RESTART,
+     /* S_FINALIZE_JOIN      ==> */ O_RELEASE | O_DC_TIMER_RESTART,
+     /* S_NOT_DC             ==> */ A_LOG | O_DC_TIMER_RESTART,
+     /* S_POLICY_ENGINE      ==> */ O_RELEASE | O_DC_TIMER_RESTART,
+     /* S_RECOVERY           ==> */ A_WARN,
+     /* S_RELEASE_DC         ==> */ A_WARN | O_DC_TIMER_RESTART,
+     /* S_STARTING           ==> */ A_LOG | A_DC_TIMER_START | A_CL_JOIN_QUERY,
+     /* S_PENDING            ==> */ A_LOG | O_DC_TIMER_RESTART,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ O_RELEASE | O_DC_TIMER_RESTART,
+     /* S_HALT               ==> */ A_WARN,
+     },
+
+/* Got an I_HALT */
+    {
+     /* S_IDLE               ==> */ A_WARN,
+     /* S_ELECTION           ==> */ A_WARN,
+     /* S_INTEGRATION        ==> */ A_WARN,
+     /* S_FINALIZE_JOIN      ==> */ A_WARN,
+     /* S_NOT_DC             ==> */ A_WARN,
+     /* S_POLICY_ENGINE      ==> */ A_WARN,
+     /* S_RECOVERY           ==> */ A_WARN,
+     /* S_RELEASE_DC         ==> */ A_WARN,
+     /* S_STARTING           ==> */ A_WARN,
+     /* S_PENDING            ==> */ A_WARN,
+     /* S_STOPPING           ==> */ A_WARN,
+     /* S_TERMINATE          ==> */ A_WARN,
+     /* S_TRANSITION_ENGINE  ==> */ A_WARN,
+     /* S_HALT               ==> */ A_WARN,
+     },
+};
+
+/*!
+ * \internal
+ * \brief Get the next FSA state given an input and the current state
+ *
+ * \param[in] input  FSA input
+ *
+ * \return The next FSA state
+ */
+enum crmd_fsa_state
+controld_fsa_get_next_state(enum crmd_fsa_input input)
+{
+    return fsa_next_states[input][controld_globals.fsa_state];
+}
+
+/*!
+ * \internal
+ * \brief Get the appropriate FSA action given an input and the current state
+ *
+ * \param[in] input  FSA input
+ *
+ * \return The appropriate FSA action
+ */
+uint64_t
+controld_fsa_get_action(enum crmd_fsa_input input)
+{
+    return fsa_actions[input][controld_globals.fsa_state];
+}
diff --git a/daemons/controld/controld_membership.c b/daemons/controld/controld_membership.c
new file mode 100644
index 0000000..1f7e4c0
--- /dev/null
+++ b/daemons/controld/controld_membership.c
@@ -0,0 +1,457 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+/* put these first so that uuid_t is defined without conflicts */
+#include <crm_internal.h>
+
+#include <string.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/common/xml_internal.h>
+#include <crm/cluster/internal.h>
+
+#include <pacemaker-controld.h>
+
+void post_cache_update(int instance);
+
+extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
+
+static void
+reap_dead_nodes(gpointer key, gpointer value, gpointer user_data)
+{
+    crm_node_t *node = value;
+
+    if (crm_is_peer_active(node) == FALSE) {
+        crm_update_peer_join(__func__, node, crm_join_none);
+
+        if(node && node->uname) {
+            if (pcmk__str_eq(controld_globals.our_nodename, node->uname,
+                             pcmk__str_casei)) {
+                crm_err("We're not part of the cluster anymore");
+                register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
+
+            } else if (!AM_I_DC
+                       && pcmk__str_eq(node->uname, controld_globals.dc_name,
+                                       pcmk__str_casei)) {
+                crm_warn("Our DC node (%s) left the cluster", node->uname);
+                register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
+            }
+        }
+
+        if ((controld_globals.fsa_state == S_INTEGRATION)
+            || (controld_globals.fsa_state == S_FINALIZE_JOIN)) {
+            check_join_state(controld_globals.fsa_state, __func__);
+        }
+        if ((node != NULL) && (node->uuid != NULL)) {
+            fail_incompletable_actions(controld_globals.transition_graph,
+                                       node->uuid);
+        }
+    }
+}
+
+void
+post_cache_update(int instance)
+{
+    xmlNode *no_op = NULL;
+
+    crm_peer_seq = instance;
+    crm_debug("Updated cache after membership event %d.", instance);
+
+    g_hash_table_foreach(crm_peer_cache, reap_dead_nodes, NULL);
+    controld_set_fsa_input_flags(R_MEMBERSHIP);
+
+    if (AM_I_DC) {
+        populate_cib_nodes(node_update_quick | node_update_cluster | node_update_peer |
+                           node_update_expected, __func__);
+    }
+
+    /*
+     * If we lost nodes, we should re-check the election status
+     * Safe to call outside of an election
+     */
+    controld_set_fsa_action_flags(A_ELECTION_CHECK);
+    controld_trigger_fsa();
+
+    /* Membership changed, remind everyone we're here.
+     * This will aid detection of duplicate DCs
+     */
+    no_op = create_request(CRM_OP_NOOP, NULL, NULL, CRM_SYSTEM_CRMD,
+                           AM_I_DC ? CRM_SYSTEM_DC : CRM_SYSTEM_CRMD, NULL);
+    send_cluster_message(NULL, crm_msg_crmd, no_op, FALSE);
+    free_xml(no_op);
+}
+
+static void
+crmd_node_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    fsa_data_t *msg_data = NULL;
+
+    if (rc == pcmk_ok) {
+        crm_trace("Node update %d complete", call_id);
+
+    } else if(call_id < pcmk_ok) {
+        crm_err("Node update failed: %s (%d)", pcmk_strerror(call_id), call_id);
+        crm_log_xml_debug(msg, "failed");
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+
+    } else {
+        crm_err("Node update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
+        crm_log_xml_debug(msg, "failed");
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Create an XML node state tag with updates
+ *
+ * \param[in,out] node    Node whose state will be used for update
+ * \param[in]     flags   Bitmask of node_update_flags indicating what to update
+ * \param[in,out] parent  XML node to contain update (or NULL)
+ * \param[in]     source  Who requested the update (only used for logging)
+ *
+ * \return Pointer to created node state tag
+ */
+xmlNode *
+create_node_state_update(crm_node_t *node, int flags, xmlNode *parent,
+                         const char *source)
+{
+    const char *value = NULL;
+    xmlNode *node_state;
+
+    if (!node->state) {
+        crm_info("Node update for %s cancelled: no state, not seen yet", node->uname);
+       return NULL;
+    }
+
+    node_state = create_xml_node(parent, XML_CIB_TAG_STATE);
+
+    if (pcmk_is_set(node->flags, crm_remote_node)) {
+        pcmk__xe_set_bool_attr(node_state, XML_NODE_IS_REMOTE, true);
+    }
+
+    set_uuid(node_state, XML_ATTR_ID, node);
+
+    if (crm_element_value(node_state, XML_ATTR_ID) == NULL) {
+        crm_info("Node update for %s cancelled: no id", node->uname);
+        free_xml(node_state);
+        return NULL;
+    }
+
+    crm_xml_add(node_state, XML_ATTR_UNAME, node->uname);
+
+    if ((flags & node_update_cluster) && node->state) {
+        pcmk__xe_set_bool_attr(node_state, XML_NODE_IN_CLUSTER,
+                               pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_casei));
+    }
+
+    if (!pcmk_is_set(node->flags, crm_remote_node)) {
+        if (flags & node_update_peer) {
+            value = OFFLINESTATUS;
+            if (pcmk_is_set(node->processes, crm_get_cluster_proc())) {
+                value = ONLINESTATUS;
+            }
+            crm_xml_add(node_state, XML_NODE_IS_PEER, value);
+        }
+
+        if (flags & node_update_join) {
+            if (node->join <= crm_join_none) {
+                value = CRMD_JOINSTATE_DOWN;
+            } else {
+                value = CRMD_JOINSTATE_MEMBER;
+            }
+            crm_xml_add(node_state, XML_NODE_JOIN_STATE, value);
+        }
+
+        if (flags & node_update_expected) {
+            crm_xml_add(node_state, XML_NODE_EXPECTED, node->expected);
+        }
+    }
+
+    crm_xml_add(node_state, XML_ATTR_ORIGIN, source);
+
+    return node_state;
+}
+
+static void
+remove_conflicting_node_callback(xmlNode * msg, int call_id, int rc,
+                                 xmlNode * output, void *user_data)
+{
+    char *node_uuid = user_data;
+
+    do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
+                        "Deletion of the unknown conflicting node \"%s\": %s (rc=%d)",
+                        node_uuid, pcmk_strerror(rc), rc);
+}
+
+static void
+search_conflicting_node_callback(xmlNode * msg, int call_id, int rc,
+                                 xmlNode * output, void *user_data)
+{
+    char *new_node_uuid = user_data;
+    xmlNode *node_xml = NULL;
+
+    if (rc != pcmk_ok) {
+        if (rc != -ENXIO) {
+            crm_notice("Searching conflicting nodes for %s failed: %s (%d)",
+                       new_node_uuid, pcmk_strerror(rc), rc);
+        }
+        return;
+
+    } else if (output == NULL) {
+        return;
+    }
+
+    if (pcmk__str_eq(crm_element_name(output), XML_CIB_TAG_NODE, pcmk__str_casei)) {
+        node_xml = output;
+
+    } else {
+        node_xml = pcmk__xml_first_child(output);
+    }
+
+    for (; node_xml != NULL; node_xml = pcmk__xml_next(node_xml)) {
+        const char *node_uuid = NULL;
+        const char *node_uname = NULL;
+        GHashTableIter iter;
+        crm_node_t *node = NULL;
+        gboolean known = FALSE;
+
+        if (!pcmk__str_eq(crm_element_name(node_xml), XML_CIB_TAG_NODE, pcmk__str_casei)) {
+            continue;
+        }
+
+        node_uuid = crm_element_value(node_xml, XML_ATTR_ID);
+        node_uname = crm_element_value(node_xml, XML_ATTR_UNAME);
+
+        if (node_uuid == NULL || node_uname == NULL) {
+            continue;
+        }
+
+        g_hash_table_iter_init(&iter, crm_peer_cache);
+        while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
+            if (node->uuid
+                && pcmk__str_eq(node->uuid, node_uuid, pcmk__str_casei)
+                && node->uname
+                && pcmk__str_eq(node->uname, node_uname, pcmk__str_casei)) {
+
+                known = TRUE;
+                break;
+            }
+        }
+
+        if (known == FALSE) {
+            cib_t *cib_conn = controld_globals.cib_conn;
+            int delete_call_id = 0;
+            xmlNode *node_state_xml = NULL;
+
+            crm_notice("Deleting unknown node %s/%s which has conflicting uname with %s",
+                       node_uuid, node_uname, new_node_uuid);
+
+            delete_call_id = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_NODES,
+                                                    node_xml, cib_scope_local);
+            fsa_register_cib_callback(delete_call_id, strdup(node_uuid),
+                                      remove_conflicting_node_callback);
+
+            node_state_xml = create_xml_node(NULL, XML_CIB_TAG_STATE);
+            crm_xml_add(node_state_xml, XML_ATTR_ID, node_uuid);
+            crm_xml_add(node_state_xml, XML_ATTR_UNAME, node_uname);
+
+            delete_call_id = cib_conn->cmds->remove(cib_conn,
+                                                    XML_CIB_TAG_STATUS,
+                                                    node_state_xml,
+                                                    cib_scope_local);
+            fsa_register_cib_callback(delete_call_id, strdup(node_uuid),
+                                      remove_conflicting_node_callback);
+            free_xml(node_state_xml);
+        }
+    }
+}
+
+static void
+node_list_update_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    fsa_data_t *msg_data = NULL;
+
+    if(call_id < pcmk_ok) {
+        crm_err("Node list update failed: %s (%d)", pcmk_strerror(call_id), call_id);
+        crm_log_xml_debug(msg, "update:failed");
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+
+    } else if(rc < pcmk_ok) {
+        crm_err("Node update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
+        crm_log_xml_debug(msg, "update:failed");
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+    }
+}
+
+void
+populate_cib_nodes(enum node_update_flags flags, const char *source)
+{
+    cib_t *cib_conn = controld_globals.cib_conn;
+
+    int call_id = 0;
+    gboolean from_hashtable = TRUE;
+    xmlNode *node_list = create_xml_node(NULL, XML_CIB_TAG_NODES);
+
+#if SUPPORT_COROSYNC
+    if (!pcmk_is_set(flags, node_update_quick) && is_corosync_cluster()) {
+        from_hashtable = pcmk__corosync_add_nodes(node_list);
+    }
+#endif
+
+    if (from_hashtable) {
+        GHashTableIter iter;
+        crm_node_t *node = NULL;
+        GString *xpath = NULL;
+
+        g_hash_table_iter_init(&iter, crm_peer_cache);
+        while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
+            xmlNode *new_node = NULL;
+
+            if ((node->uuid != NULL) && (node->uname != NULL)) {
+                crm_trace("Creating node entry for %s/%s", node->uname, node->uuid);
+                if (xpath == NULL) {
+                    xpath = g_string_sized_new(512);
+                } else {
+                    g_string_truncate(xpath, 0);
+                }
+
+                /* We need both to be valid */
+                new_node = create_xml_node(node_list, XML_CIB_TAG_NODE);
+                crm_xml_add(new_node, XML_ATTR_ID, node->uuid);
+                crm_xml_add(new_node, XML_ATTR_UNAME, node->uname);
+
+                /* Search and remove unknown nodes with the conflicting uname from CIB */
+                pcmk__g_strcat(xpath,
+                               "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION
+                               "/" XML_CIB_TAG_NODES "/" XML_CIB_TAG_NODE
+                               "[@" XML_ATTR_UNAME "='", node->uname, "']"
+                               "[@" XML_ATTR_ID "!='", node->uuid, "']", NULL);
+
+                call_id = cib_conn->cmds->query(cib_conn,
+                                                (const char *) xpath->str,
+                                                NULL,
+                                                cib_scope_local|cib_xpath);
+                fsa_register_cib_callback(call_id, strdup(node->uuid),
+                                          search_conflicting_node_callback);
+            }
+        }
+
+        if (xpath != NULL) {
+            g_string_free(xpath, TRUE);
+        }
+    }
+
+    crm_trace("Populating <nodes> section from %s", from_hashtable ? "hashtable" : "cluster");
+
+    if ((controld_update_cib(XML_CIB_TAG_NODES, node_list, cib_scope_local,
+                             node_list_update_callback) == pcmk_rc_ok)
+         && (crm_peer_cache != NULL) && AM_I_DC) {
+        /*
+         * There is no need to update the local CIB with our values if
+         * we've not seen valid membership data
+         */
+        GHashTableIter iter;
+        crm_node_t *node = NULL;
+
+        free_xml(node_list);
+        node_list = create_xml_node(NULL, XML_CIB_TAG_STATUS);
+
+        g_hash_table_iter_init(&iter, crm_peer_cache);
+        while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
+            create_node_state_update(node, flags, node_list, source);
+        }
+
+        if (crm_remote_peer_cache) {
+            g_hash_table_iter_init(&iter, crm_remote_peer_cache);
+            while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
+                create_node_state_update(node, flags, node_list, source);
+            }
+        }
+
+        controld_update_cib(XML_CIB_TAG_STATUS, node_list, cib_scope_local,
+                            crmd_node_update_complete);
+    }
+    free_xml(node_list);
+}
+
+static void
+cib_quorum_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    fsa_data_t *msg_data = NULL;
+
+    if (rc == pcmk_ok) {
+        crm_trace("Quorum update %d complete", call_id);
+
+    } else {
+        crm_err("Quorum update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
+        crm_log_xml_debug(msg, "failed");
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+    }
+}
+
+void
+crm_update_quorum(gboolean quorum, gboolean force_update)
+{
+    bool has_quorum = pcmk_is_set(controld_globals.flags, controld_has_quorum);
+
+    if (quorum) {
+        controld_set_global_flags(controld_ever_had_quorum);
+
+    } else if (pcmk_all_flags_set(controld_globals.flags,
+                                  controld_ever_had_quorum
+                                  |controld_no_quorum_suicide)) {
+        pcmk__panic(__func__);
+    }
+
+    if (AM_I_DC
+        && ((has_quorum && !quorum) || (!has_quorum && quorum)
+            || force_update)) {
+        xmlNode *update = NULL;
+
+        update = create_xml_node(NULL, XML_TAG_CIB);
+        crm_xml_add_int(update, XML_ATTR_HAVE_QUORUM, quorum);
+        crm_xml_add(update, XML_ATTR_DC_UUID, controld_globals.our_uuid);
+
+        crm_debug("Updating quorum status to %s", pcmk__btoa(quorum));
+        controld_update_cib(XML_TAG_CIB, update, cib_scope_local,
+                            cib_quorum_update_complete);
+        free_xml(update);
+
+        /* Quorum changes usually cause a new transition via other activity:
+         * quorum gained via a node joining will abort via the node join,
+         * and quorum lost via a node leaving will usually abort via resource
+         * activity and/or fencing.
+         *
+         * However, it is possible that nothing else causes a transition (e.g.
+         * someone forces quorum via corosync-cmaptcl, or quorum is lost due to
+         * a node in standby shutting down cleanly), so here ensure a new
+         * transition is triggered.
+         */
+        if (quorum) {
+            /* If quorum was gained, abort after a short delay, in case multiple
+             * nodes are joining around the same time, so the one that brings us
+             * to quorum doesn't cause all the remaining ones to be fenced.
+             */
+            abort_after_delay(INFINITY, pcmk__graph_restart, "Quorum gained",
+                              5000);
+        } else {
+            abort_transition(INFINITY, pcmk__graph_restart, "Quorum lost",
+                             NULL);
+        }
+    }
+
+    if (quorum) {
+        controld_set_global_flags(controld_has_quorum);
+    } else {
+        controld_clear_global_flags(controld_has_quorum);
+    }
+}
diff --git a/daemons/controld/controld_membership.h b/daemons/controld/controld_membership.h
new file mode 100644
index 0000000..cfe8cee
--- /dev/null
+++ b/daemons/controld/controld_membership.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2012-2021 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+#ifndef MEMBERSHIP__H
+#  define MEMBERSHIP__H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <crm/cluster/internal.h>
+
+void post_cache_update(int instance);
+
+extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
+
+void controld_destroy_failed_sync_table(void);
+void controld_remove_failed_sync_node(const char *node_name);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c
new file mode 100644
index 0000000..54b27ec
--- /dev/null
+++ b/daemons/controld/controld_messages.c
@@ -0,0 +1,1307 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <string.h>
+#include <time.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/cluster/internal.h>
+#include <crm/cib.h>
+#include <crm/common/ipc_internal.h>
+
+#include <pacemaker-controld.h>
+
+extern void crm_shutdown(int nsig);
+
+static enum crmd_fsa_input handle_message(xmlNode *msg,
+                                          enum crmd_fsa_cause cause);
+static void handle_response(xmlNode *stored_msg);
+static enum crmd_fsa_input handle_request(xmlNode *stored_msg,
+                                          enum crmd_fsa_cause cause);
+static enum crmd_fsa_input handle_shutdown_request(xmlNode *stored_msg);
+static void send_msg_via_ipc(xmlNode * msg, const char *sys);
+
+/* debug only, can wrap all it likes */
+static int last_data_id = 0;
+
+void
+register_fsa_error_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
+                       fsa_data_t * cur_data, void *new_data, const char *raised_from)
+{
+    /* save the current actions if any */
+    if (controld_globals.fsa_actions != A_NOTHING) {
+        register_fsa_input_adv(cur_data ? cur_data->fsa_cause : C_FSA_INTERNAL,
+                               I_NULL, cur_data ? cur_data->data : NULL,
+                               controld_globals.fsa_actions, TRUE, __func__);
+    }
+
+    /* reset the action list */
+    crm_info("Resetting the current action list");
+    fsa_dump_actions(controld_globals.fsa_actions, "Drop");
+    controld_globals.fsa_actions = A_NOTHING;
+
+    /* register the error */
+    register_fsa_input_adv(cause, input, new_data, A_NOTHING, TRUE, raised_from);
+}
+
+void
+register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
+                       void *data, uint64_t with_actions,
+                       gboolean prepend, const char *raised_from)
+{
+    unsigned old_len = g_list_length(controld_globals.fsa_message_queue);
+    fsa_data_t *fsa_data = NULL;
+
+    if (raised_from == NULL) {
+        raised_from = "<unknown>";
+    }
+
+    if (input == I_NULL && with_actions == A_NOTHING /* && data == NULL */ ) {
+        /* no point doing anything */
+        crm_err("Cannot add entry to queue: no input and no action");
+        return;
+    }
+
+    if (input == I_WAIT_FOR_EVENT) {
+        controld_set_global_flags(controld_fsa_is_stalled);
+        crm_debug("Stalling the FSA pending further input: source=%s cause=%s data=%p queue=%d",
+                  raised_from, fsa_cause2string(cause), data, old_len);
+
+        if (old_len > 0) {
+            fsa_dump_queue(LOG_TRACE);
+            prepend = FALSE;
+        }
+
+        if (data == NULL) {
+            controld_set_fsa_action_flags(with_actions);
+            fsa_dump_actions(with_actions, "Restored");
+            return;
+        }
+
+        /* Store everything in the new event and reset
+         * controld_globals.fsa_actions
+         */
+        with_actions |= controld_globals.fsa_actions;
+        controld_globals.fsa_actions = A_NOTHING;
+    }
+
+    last_data_id++;
+    crm_trace("%s %s FSA input %d (%s) due to %s, %s data",
+              raised_from, (prepend? "prepended" : "appended"), last_data_id,
+              fsa_input2string(input), fsa_cause2string(cause),
+              (data? "with" : "without"));
+
+    fsa_data = calloc(1, sizeof(fsa_data_t));
+    fsa_data->id = last_data_id;
+    fsa_data->fsa_input = input;
+    fsa_data->fsa_cause = cause;
+    fsa_data->origin = raised_from;
+    fsa_data->data = NULL;
+    fsa_data->data_type = fsa_dt_none;
+    fsa_data->actions = with_actions;
+
+    if (with_actions != A_NOTHING) {
+        crm_trace("Adding actions %.16llx to input",
+                  (unsigned long long) with_actions);
+    }
+
+    if (data != NULL) {
+        switch (cause) {
+            case C_FSA_INTERNAL:
+            case C_CRMD_STATUS_CALLBACK:
+            case C_IPC_MESSAGE:
+            case C_HA_MESSAGE:
+                CRM_CHECK(((ha_msg_input_t *) data)->msg != NULL,
+                          crm_err("Bogus data from %s", raised_from));
+                crm_trace("Copying %s data from %s as cluster message data",
+                          fsa_cause2string(cause), raised_from);
+                fsa_data->data = copy_ha_msg_input(data);
+                fsa_data->data_type = fsa_dt_ha_msg;
+                break;
+
+            case C_LRM_OP_CALLBACK:
+                crm_trace("Copying %s data from %s as lrmd_event_data_t",
+                          fsa_cause2string(cause), raised_from);
+                fsa_data->data = lrmd_copy_event((lrmd_event_data_t *) data);
+                fsa_data->data_type = fsa_dt_lrm;
+                break;
+
+            case C_TIMER_POPPED:
+            case C_SHUTDOWN:
+            case C_UNKNOWN:
+            case C_STARTUP:
+                crm_crit("Copying %s data (from %s) is not yet implemented",
+                         fsa_cause2string(cause), raised_from);
+                crmd_exit(CRM_EX_SOFTWARE);
+                break;
+        }
+    }
+
+    /* make sure to free it properly later */
+    if (prepend) {
+        controld_globals.fsa_message_queue
+            = g_list_prepend(controld_globals.fsa_message_queue, fsa_data);
+    } else {
+        controld_globals.fsa_message_queue
+            = g_list_append(controld_globals.fsa_message_queue, fsa_data);
+    }
+
+    crm_trace("FSA message queue length is %d",
+              g_list_length(controld_globals.fsa_message_queue));
+
+    /* fsa_dump_queue(LOG_TRACE); */
+
+    if (old_len == g_list_length(controld_globals.fsa_message_queue)) {
+        crm_err("Couldn't add message to the queue");
+    }
+
+    if (input != I_WAIT_FOR_EVENT) {
+        controld_trigger_fsa();
+    }
+}
+
+void
+fsa_dump_queue(int log_level)
+{
+    int offset = 0;
+
+    for (GList *iter = controld_globals.fsa_message_queue; iter != NULL;
+         iter = iter->next) {
+        fsa_data_t *data = (fsa_data_t *) iter->data;
+
+        do_crm_log_unlikely(log_level,
+                            "queue[%d.%d]: input %s raised by %s(%p.%d)\t(cause=%s)",
+                            offset++, data->id, fsa_input2string(data->fsa_input),
+                            data->origin, data->data, data->data_type,
+                            fsa_cause2string(data->fsa_cause));
+    }
+}
+
+ha_msg_input_t *
+copy_ha_msg_input(ha_msg_input_t * orig)
+{
+    ha_msg_input_t *copy = calloc(1, sizeof(ha_msg_input_t));
+
+    CRM_ASSERT(copy != NULL);
+    copy->msg = (orig && orig->msg)? copy_xml(orig->msg) : NULL;
+    copy->xml = get_message_xml(copy->msg, F_CRM_DATA);
+    return copy;
+}
+
+void
+delete_fsa_input(fsa_data_t * fsa_data)
+{
+    lrmd_event_data_t *op = NULL;
+    xmlNode *foo = NULL;
+
+    if (fsa_data == NULL) {
+        return;
+    }
+    crm_trace("About to free %s data", fsa_cause2string(fsa_data->fsa_cause));
+
+    if (fsa_data->data != NULL) {
+        switch (fsa_data->data_type) {
+            case fsa_dt_ha_msg:
+                delete_ha_msg_input(fsa_data->data);
+                break;
+
+            case fsa_dt_xml:
+                foo = fsa_data->data;
+                free_xml(foo);
+                break;
+
+            case fsa_dt_lrm:
+                op = (lrmd_event_data_t *) fsa_data->data;
+                lrmd_free_event(op);
+                break;
+
+            case fsa_dt_none:
+                if (fsa_data->data != NULL) {
+                    crm_err("Don't know how to free %s data from %s",
+                            fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
+                    crmd_exit(CRM_EX_SOFTWARE);
+                }
+                break;
+        }
+        crm_trace("%s data freed", fsa_cause2string(fsa_data->fsa_cause));
+    }
+
+    free(fsa_data);
+}
+
+/* returns the next message */
+fsa_data_t *
+get_message(void)
+{
+    fsa_data_t *message
+        = (fsa_data_t *) controld_globals.fsa_message_queue->data;
+
+    controld_globals.fsa_message_queue
+        = g_list_remove(controld_globals.fsa_message_queue, message);
+    crm_trace("Processing input %d", message->id);
+    return message;
+}
+
+void *
+fsa_typed_data_adv(fsa_data_t * fsa_data, enum fsa_data_type a_type, const char *caller)
+{
+    void *ret_val = NULL;
+
+    if (fsa_data == NULL) {
+        crm_err("%s: No FSA data available", caller);
+
+    } else if (fsa_data->data == NULL) {
+        crm_err("%s: No message data available. Origin: %s", caller, fsa_data->origin);
+
+    } else if (fsa_data->data_type != a_type) {
+        crm_crit("%s: Message data was the wrong type! %d vs. requested=%d.  Origin: %s",
+                 caller, fsa_data->data_type, a_type, fsa_data->origin);
+        CRM_ASSERT(fsa_data->data_type == a_type);
+    } else {
+        ret_val = fsa_data->data;
+    }
+
+    return ret_val;
+}
+
+/*	A_MSG_ROUTE	*/
+void
+do_msg_route(long long action,
+             enum crmd_fsa_cause cause,
+             enum crmd_fsa_state cur_state,
+             enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
+
+    route_message(msg_data->fsa_cause, input->msg);
+}
+
+void
+route_message(enum crmd_fsa_cause cause, xmlNode * input)
+{
+    ha_msg_input_t fsa_input;
+    enum crmd_fsa_input result = I_NULL;
+
+    fsa_input.msg = input;
+    CRM_CHECK(cause == C_IPC_MESSAGE || cause == C_HA_MESSAGE, return);
+
+    /* try passing the buck first */
+    if (relay_message(input, cause == C_IPC_MESSAGE)) {
+        return;
+    }
+
+    /* handle locally */
+    result = handle_message(input, cause);
+
+    /* done or process later? */
+    switch (result) {
+        case I_NULL:
+        case I_CIB_OP:
+        case I_ROUTER:
+        case I_NODE_JOIN:
+        case I_JOIN_REQUEST:
+        case I_JOIN_RESULT:
+            break;
+        default:
+            /* Defering local processing of message */
+            register_fsa_input_later(cause, result, &fsa_input);
+            return;
+    }
+
+    if (result != I_NULL) {
+        /* add to the front of the queue */
+        register_fsa_input(cause, result, &fsa_input);
+    }
+}
+
+gboolean
+relay_message(xmlNode * msg, gboolean originated_locally)
+{
+    int dest = 1;
+    bool is_for_dc = false;
+    bool is_for_dcib = false;
+    bool is_for_te = false;
+    bool is_for_crm = false;
+    bool is_for_cib = false;
+    bool is_local = false;
+    const char *host_to = crm_element_value(msg, F_CRM_HOST_TO);
+    const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
+    const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
+    const char *type = crm_element_value(msg, F_TYPE);
+    const char *task = crm_element_value(msg, F_CRM_TASK);
+    const char *ref = crm_element_value(msg, XML_ATTR_REFERENCE);
+
+    if (ref == NULL) {
+        ref = "without reference ID";
+    }
+
+    if (msg == NULL) {
+        crm_warn("Cannot route empty message");
+        return TRUE;
+
+    } else if (pcmk__str_eq(task, CRM_OP_HELLO, pcmk__str_casei)) {
+        crm_trace("No routing needed for hello message %s", ref);
+        return TRUE;
+
+    } else if (!pcmk__str_eq(type, T_CRM, pcmk__str_casei)) {
+        crm_warn("Received invalid message %s: type '%s' not '" T_CRM "'",
+                 ref, pcmk__s(type, ""));
+        crm_log_xml_warn(msg, "[bad message type]");
+        return TRUE;
+
+    } else if (sys_to == NULL) {
+        crm_warn("Received invalid message %s: no subsystem", ref);
+        crm_log_xml_warn(msg, "[no subsystem]");
+        return TRUE;
+    }
+
+    is_for_dc = (strcasecmp(CRM_SYSTEM_DC, sys_to) == 0);
+    is_for_dcib = (strcasecmp(CRM_SYSTEM_DCIB, sys_to) == 0);
+    is_for_te = (strcasecmp(CRM_SYSTEM_TENGINE, sys_to) == 0);
+    is_for_cib = (strcasecmp(CRM_SYSTEM_CIB, sys_to) == 0);
+    is_for_crm = (strcasecmp(CRM_SYSTEM_CRMD, sys_to) == 0);
+
+    is_local = false;
+    if (pcmk__str_empty(host_to)) {
+        if (is_for_dc || is_for_te) {
+            is_local = false;
+
+        } else if (is_for_crm) {
+            if (pcmk__strcase_any_of(task, CRM_OP_NODE_INFO,
+                                     PCMK__CONTROLD_CMD_NODES, NULL)) {
+                /* Node info requests do not specify a host, which is normally
+                 * treated as "all hosts", because the whole point is that the
+                 * client may not know the local node name. Always handle these
+                 * requests locally.
+                 */
+                is_local = true;
+            } else {
+                is_local = !originated_locally;
+            }
+
+        } else {
+            is_local = true;
+        }
+
+    } else if (pcmk__str_eq(controld_globals.our_nodename, host_to,
+                            pcmk__str_casei)) {
+        is_local = true;
+    } else if (is_for_crm && pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
+        xmlNode *msg_data = get_message_xml(msg, F_CRM_DATA);
+        const char *mode = crm_element_value(msg_data, PCMK__XA_MODE);
+
+        if (pcmk__str_eq(mode, XML_TAG_CIB, pcmk__str_casei)) {
+            // Local delete of an offline node's resource history
+            is_local = true;
+        }
+    }
+
+    if (is_for_dc || is_for_dcib || is_for_te) {
+        if (AM_I_DC && is_for_te) {
+            crm_trace("Route message %s locally as transition request", ref);
+            send_msg_via_ipc(msg, sys_to);
+
+        } else if (AM_I_DC) {
+            crm_trace("Route message %s locally as DC request", ref);
+            return FALSE; // More to be done by caller
+
+        } else if (originated_locally && !pcmk__strcase_any_of(sys_from, CRM_SYSTEM_PENGINE,
+                                                               CRM_SYSTEM_TENGINE, NULL)) {
+
+            if (is_corosync_cluster()) {
+                dest = text2msg_type(sys_to);
+            }
+            crm_trace("Relay message %s to DC", ref);
+            send_cluster_message(host_to ? crm_get_peer(0, host_to) : NULL, dest, msg, TRUE);
+
+        } else {
+            /* Neither the TE nor the scheduler should be sending messages
+             * to DCs on other nodes. By definition, if we are no longer the DC,
+             * then the scheduler's or TE's data should be discarded.
+             */
+            crm_trace("Discard message %s because we are not DC", ref);
+        }
+
+    } else if (is_local && (is_for_crm || is_for_cib)) {
+        crm_trace("Route message %s locally as controller request", ref);
+        return FALSE; // More to be done by caller
+
+    } else if (is_local) {
+        crm_trace("Relay message %s locally to %s",
+                  ref, (sys_to? sys_to : "unknown client"));
+        crm_log_xml_trace(msg, "[IPC relay]");
+        send_msg_via_ipc(msg, sys_to);
+
+    } else {
+        crm_node_t *node_to = NULL;
+
+        if (is_corosync_cluster()) {
+            dest = text2msg_type(sys_to);
+
+            if (dest == crm_msg_none || dest > crm_msg_stonith_ng) {
+                dest = crm_msg_crmd;
+            }
+        }
+
+        if (host_to) {
+            node_to = pcmk__search_cluster_node_cache(0, host_to);
+            if (node_to == NULL) {
+                crm_warn("Cannot route message %s: Unknown node %s",
+                         ref, host_to);
+                return TRUE;
+            }
+            crm_trace("Relay message %s to %s",
+                      ref, (node_to->uname? node_to->uname : "peer"));
+        } else {
+            crm_trace("Broadcast message %s to all peers", ref);
+        }
+        send_cluster_message(host_to ? node_to : NULL, dest, msg, TRUE);
+    }
+
+    return TRUE; // No further processing of message is needed
+}
+
+// Return true if field contains a positive integer
+static bool
+authorize_version(xmlNode *message_data, const char *field,
+                  const char *client_name, const char *ref, const char *uuid)
+{
+    const char *version = crm_element_value(message_data, field);
+    long long version_num;
+
+    if ((pcmk__scan_ll(version, &version_num, -1LL) != pcmk_rc_ok)
+        || (version_num < 0LL)) {
+
+        crm_warn("Rejected IPC hello from %s: '%s' is not a valid protocol %s "
+                 CRM_XS " ref=%s uuid=%s",
+                 client_name, ((version == NULL)? "" : version),
+                 field, (ref? ref : "none"), uuid);
+        return false;
+    }
+    return true;
+}
+
+/*!
+ * \internal
+ * \brief Check whether a client IPC message is acceptable
+ *
+ * If a given client IPC message is a hello, "authorize" it by ensuring it has
+ * valid information such as a protocol version, and return false indicating
+ * that nothing further needs to be done with the message. If the message is not
+ * a hello, just return true to indicate it needs further processing.
+ *
+ * \param[in]     client_msg     XML of IPC message
+ * \param[in,out] curr_client    If IPC is not proxied, client that sent message
+ * \param[in]     proxy_session  If IPC is proxied, the session ID
+ *
+ * \return true if message needs further processing, false if it doesn't
+ */
+bool
+controld_authorize_ipc_message(const xmlNode *client_msg, pcmk__client_t *curr_client,
+                               const char *proxy_session)
+{
+    xmlNode *message_data = NULL;
+    const char *client_name = NULL;
+    const char *op = crm_element_value(client_msg, F_CRM_TASK);
+    const char *ref = crm_element_value(client_msg, XML_ATTR_REFERENCE);
+    const char *uuid = (curr_client? curr_client->id : proxy_session);
+
+    if (uuid == NULL) {
+        crm_warn("IPC message from client rejected: No client identifier "
+                 CRM_XS " ref=%s", (ref? ref : "none"));
+        goto rejected;
+    }
+
+    if (!pcmk__str_eq(CRM_OP_HELLO, op, pcmk__str_casei)) {
+        // Only hello messages need to be authorized
+        return true;
+    }
+
+    message_data = get_message_xml(client_msg, F_CRM_DATA);
+
+    client_name = crm_element_value(message_data, "client_name");
+    if (pcmk__str_empty(client_name)) {
+        crm_warn("IPC hello from client rejected: No client name",
+                 CRM_XS " ref=%s uuid=%s", (ref? ref : "none"), uuid);
+        goto rejected;
+    }
+    if (!authorize_version(message_data, "major_version", client_name, ref,
+                           uuid)) {
+        goto rejected;
+    }
+    if (!authorize_version(message_data, "minor_version", client_name, ref,
+                           uuid)) {
+        goto rejected;
+    }
+
+    crm_trace("Validated IPC hello from client %s", client_name);
+    if (curr_client) {
+        curr_client->userdata = strdup(client_name);
+    }
+    controld_trigger_fsa();
+    return false;
+
+rejected:
+    if (curr_client) {
+        qb_ipcs_disconnect(curr_client->ipcs);
+    }
+    return false;
+}
+
+static enum crmd_fsa_input
+handle_message(xmlNode *msg, enum crmd_fsa_cause cause)
+{
+    const char *type = NULL;
+
+    CRM_CHECK(msg != NULL, return I_NULL);
+
+    type = crm_element_value(msg, F_CRM_MSG_TYPE);
+    if (pcmk__str_eq(type, XML_ATTR_REQUEST, pcmk__str_none)) {
+        return handle_request(msg, cause);
+
+    } else if (pcmk__str_eq(type, XML_ATTR_RESPONSE, pcmk__str_none)) {
+        handle_response(msg);
+        return I_NULL;
+    }
+
+    crm_err("Unknown message type: %s", type);
+    return I_NULL;
+}
+
+static enum crmd_fsa_input
+handle_failcount_op(xmlNode * stored_msg)
+{
+    const char *rsc = NULL;
+    const char *uname = NULL;
+    const char *op = NULL;
+    char *interval_spec = NULL;
+    guint interval_ms = 0;
+    gboolean is_remote_node = FALSE;
+    xmlNode *xml_op = get_message_xml(stored_msg, F_CRM_DATA);
+
+    if (xml_op) {
+        xmlNode *xml_rsc = first_named_child(xml_op, XML_CIB_TAG_RESOURCE);
+        xmlNode *xml_attrs = first_named_child(xml_op, XML_TAG_ATTRS);
+
+        if (xml_rsc) {
+            rsc = ID(xml_rsc);
+        }
+        if (xml_attrs) {
+            op = crm_element_value(xml_attrs,
+                                   CRM_META "_" XML_RSC_ATTR_CLEAR_OP);
+            crm_element_value_ms(xml_attrs,
+                                 CRM_META "_" XML_RSC_ATTR_CLEAR_INTERVAL,
+                                 &interval_ms);
+        }
+    }
+    uname = crm_element_value(xml_op, XML_LRM_ATTR_TARGET);
+
+    if ((rsc == NULL) || (uname == NULL)) {
+        crm_log_xml_warn(stored_msg, "invalid failcount op");
+        return I_NULL;
+    }
+
+    if (crm_element_value(xml_op, XML_LRM_ATTR_ROUTER_NODE)) {
+        is_remote_node = TRUE;
+    }
+
+    crm_debug("Clearing failures for %s-interval %s on %s "
+              "from attribute manager, CIB, and executor state",
+              pcmk__readable_interval(interval_ms), rsc, uname);
+
+    if (interval_ms) {
+        interval_spec = crm_strdup_printf("%ums", interval_ms);
+    }
+    update_attrd_clear_failures(uname, rsc, op, interval_spec, is_remote_node);
+    free(interval_spec);
+
+    controld_cib_delete_last_failure(rsc, uname, op, interval_ms);
+
+    lrm_clear_last_failure(rsc, uname, op, interval_ms);
+
+    return I_NULL;
+}
+
+static enum crmd_fsa_input
+handle_lrm_delete(xmlNode *stored_msg)
+{
+    const char *mode = NULL;
+    xmlNode *msg_data = get_message_xml(stored_msg, F_CRM_DATA);
+
+    CRM_CHECK(msg_data != NULL, return I_NULL);
+
+    /* CRM_OP_LRM_DELETE has two distinct modes. The default behavior is to
+     * relay the operation to the affected node, which will unregister the
+     * resource from the local executor, clear the resource's history from the
+     * CIB, and do some bookkeeping in the controller.
+     *
+     * However, if the affected node is offline, the client will specify
+     * mode="cib" which means the controller receiving the operation should
+     * clear the resource's history from the CIB and nothing else. This is used
+     * to clear shutdown locks.
+     */
+    mode = crm_element_value(msg_data, PCMK__XA_MODE);
+    if ((mode == NULL) || strcmp(mode, XML_TAG_CIB)) {
+        // Relay to affected node
+        crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD);
+        return I_ROUTER;
+
+    } else {
+        // Delete CIB history locally (compare with do_lrm_delete())
+        const char *from_sys = NULL;
+        const char *user_name = NULL;
+        const char *rsc_id = NULL;
+        const char *node = NULL;
+        xmlNode *rsc_xml = NULL;
+        int rc = pcmk_rc_ok;
+
+        rsc_xml = first_named_child(msg_data, XML_CIB_TAG_RESOURCE);
+        CRM_CHECK(rsc_xml != NULL, return I_NULL);
+
+        rsc_id = ID(rsc_xml);
+        from_sys = crm_element_value(stored_msg, F_CRM_SYS_FROM);
+        node = crm_element_value(msg_data, XML_LRM_ATTR_TARGET);
+        user_name = pcmk__update_acl_user(stored_msg, F_CRM_USER, NULL);
+        crm_debug("Handling " CRM_OP_LRM_DELETE " for %s on %s locally%s%s "
+                  "(clearing CIB resource history only)", rsc_id, node,
+                  (user_name? " for user " : ""), (user_name? user_name : ""));
+        rc = controld_delete_resource_history(rsc_id, node, user_name,
+                                              cib_dryrun|cib_sync_call);
+        if (rc == pcmk_rc_ok) {
+            rc = controld_delete_resource_history(rsc_id, node, user_name,
+                                                  crmd_cib_smart_opt());
+        }
+
+        //Notify client and tengine.(Only notify tengine if mode = "cib" and CRM_OP_LRM_DELETE.)
+        if (from_sys) {
+            lrmd_event_data_t *op = NULL;
+            const char *from_host = crm_element_value(stored_msg,
+                                                      F_CRM_HOST_FROM);
+            const char *transition;
+
+            if (strcmp(from_sys, CRM_SYSTEM_TENGINE)) {
+                transition = crm_element_value(msg_data,
+                                                       XML_ATTR_TRANSITION_KEY);
+            } else {
+                transition = crm_element_value(stored_msg,
+                                                       XML_ATTR_TRANSITION_KEY);
+            }
+
+            crm_info("Notifying %s on %s that %s was%s deleted",
+                     from_sys, (from_host? from_host : "local node"), rsc_id,
+                     ((rc == pcmk_rc_ok)? "" : " not"));
+            op = lrmd_new_event(rsc_id, CRMD_ACTION_DELETE, 0);
+            op->type = lrmd_event_exec_complete;
+            op->user_data = strdup(transition? transition : FAKE_TE_ID);
+            op->params = pcmk__strkey_table(free, free);
+            g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION),
+                                strdup(CRM_FEATURE_SET));
+            controld_rc2event(op, rc);
+            controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
+            lrmd_free_event(op);
+            controld_trigger_delete_refresh(from_sys, rsc_id);
+        }
+        return I_NULL;
+    }
+}
+
+/*!
+ * \brief Handle a CRM_OP_REMOTE_STATE message by updating remote peer cache
+ *
+ * \param[in] msg  Message XML
+ *
+ * \return Next FSA input
+ */
+static enum crmd_fsa_input
+handle_remote_state(const xmlNode *msg)
+{
+    const char *conn_host = NULL;
+    const char *remote_uname = ID(msg);
+    crm_node_t *remote_peer;
+    bool remote_is_up = false;
+    int rc = pcmk_rc_ok;
+
+    rc = pcmk__xe_get_bool_attr(msg, XML_NODE_IN_CLUSTER, &remote_is_up);
+
+    CRM_CHECK(remote_uname && rc == pcmk_rc_ok, return I_NULL);
+
+    remote_peer = crm_remote_peer_get(remote_uname);
+    CRM_CHECK(remote_peer, return I_NULL);
+
+    pcmk__update_peer_state(__func__, remote_peer,
+                            remote_is_up ? CRM_NODE_MEMBER : CRM_NODE_LOST,
+                            0);
+
+    conn_host = crm_element_value(msg, PCMK__XA_CONN_HOST);
+    if (conn_host) {
+        pcmk__str_update(&remote_peer->conn_host, conn_host);
+    } else if (remote_peer->conn_host) {
+        free(remote_peer->conn_host);
+        remote_peer->conn_host = NULL;
+    }
+
+    return I_NULL;
+}
+
+/*!
+ * \brief Handle a CRM_OP_PING message
+ *
+ * \param[in] msg  Message XML
+ *
+ * \return Next FSA input
+ */
+static enum crmd_fsa_input
+handle_ping(const xmlNode *msg)
+{
+    const char *value = NULL;
+    xmlNode *ping = NULL;
+    xmlNode *reply = NULL;
+
+    // Build reply
+
+    ping = create_xml_node(NULL, XML_CRM_TAG_PING);
+    value = crm_element_value(msg, F_CRM_SYS_TO);
+    crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value);
+
+    // Add controller state
+    value = fsa_state2string(controld_globals.fsa_state);
+    crm_xml_add(ping, XML_PING_ATTR_CRMDSTATE, value);
+    crm_notice("Current ping state: %s", value); // CTS needs this
+
+    // Add controller health
+    // @TODO maybe do some checks to determine meaningful status
+    crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");
+
+    // Send reply
+    reply = create_reply(msg, ping);
+    free_xml(ping);
+    if (reply != NULL) {
+        (void) relay_message(reply, TRUE);
+        free_xml(reply);
+    }
+
+    // Nothing further to do
+    return I_NULL;
+}
+
+/*!
+ * \brief Handle a PCMK__CONTROLD_CMD_NODES message
+ *
+ * \param[in] request  Message XML
+ *
+ * \return Next FSA input
+ */
+static enum crmd_fsa_input
+handle_node_list(const xmlNode *request)
+{
+    GHashTableIter iter;
+    crm_node_t *node = NULL;
+    xmlNode *reply = NULL;
+    xmlNode *reply_data = NULL;
+
+    // Create message data for reply
+    reply_data = create_xml_node(NULL, XML_CIB_TAG_NODES);
+    g_hash_table_iter_init(&iter, crm_peer_cache);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
+        xmlNode *xml = create_xml_node(reply_data, XML_CIB_TAG_NODE);
+
+        crm_xml_add_ll(xml, XML_ATTR_ID, (long long) node->id); // uint32_t
+        crm_xml_add(xml, XML_ATTR_UNAME, node->uname);
+        crm_xml_add(xml, XML_NODE_IN_CLUSTER, node->state);
+    }
+
+    // Create and send reply
+    reply = create_reply(request, reply_data);
+    free_xml(reply_data);
+    if (reply) {
+        (void) relay_message(reply, TRUE);
+        free_xml(reply);
+    }
+
+    // Nothing further to do
+    return I_NULL;
+}
+
+/*!
+ * \brief Handle a CRM_OP_NODE_INFO request
+ *
+ * \param[in] msg  Message XML
+ *
+ * \return Next FSA input
+ */
+static enum crmd_fsa_input
+handle_node_info_request(const xmlNode *msg)
+{
+    const char *value = NULL;
+    crm_node_t *node = NULL;
+    int node_id = 0;
+    xmlNode *reply = NULL;
+    xmlNode *reply_data = NULL;
+
+    // Build reply
+
+    reply_data = create_xml_node(NULL, XML_CIB_TAG_NODE);
+    crm_xml_add(reply_data, XML_PING_ATTR_SYSFROM, CRM_SYSTEM_CRMD);
+
+    // Add whether current partition has quorum
+    pcmk__xe_set_bool_attr(reply_data, XML_ATTR_HAVE_QUORUM,
+                           pcmk_is_set(controld_globals.flags,
+                                       controld_has_quorum));
+
+    // Check whether client requested node info by ID and/or name
+    crm_element_value_int(msg, XML_ATTR_ID, &node_id);
+    if (node_id < 0) {
+        node_id = 0;
+    }
+    value = crm_element_value(msg, XML_ATTR_UNAME);
+
+    // Default to local node if none given
+    if ((node_id == 0) && (value == NULL)) {
+        value = controld_globals.our_nodename;
+    }
+
+    node = pcmk__search_node_caches(node_id, value, CRM_GET_PEER_ANY);
+    if (node) {
+        crm_xml_add(reply_data, XML_ATTR_ID, node->uuid);
+        crm_xml_add(reply_data, XML_ATTR_UNAME, node->uname);
+        crm_xml_add(reply_data, XML_NODE_IS_PEER, node->state);
+        pcmk__xe_set_bool_attr(reply_data, XML_NODE_IS_REMOTE,
+                               pcmk_is_set(node->flags, crm_remote_node));
+    }
+
+    // Send reply
+    reply = create_reply(msg, reply_data);
+    free_xml(reply_data);
+    if (reply != NULL) {
+        (void) relay_message(reply, TRUE);
+        free_xml(reply);
+    }
+
+    // Nothing further to do
+    return I_NULL;
+}
+
+static void
+verify_feature_set(xmlNode *msg)
+{
+    const char *dc_version = crm_element_value(msg, XML_ATTR_CRM_VERSION);
+
+    if (dc_version == NULL) {
+        /* All we really know is that the DC feature set is older than 3.1.0,
+         * but that's also all that really matters.
+         */
+        dc_version = "3.0.14";
+    }
+
+    if (feature_set_compatible(dc_version, CRM_FEATURE_SET)) {
+        crm_trace("Local feature set (%s) is compatible with DC's (%s)",
+                  CRM_FEATURE_SET, dc_version);
+    } else {
+        crm_err("Local feature set (%s) is incompatible with DC's (%s)",
+                CRM_FEATURE_SET, dc_version);
+
+        // Nothing is likely to improve without administrator involvement
+        controld_set_fsa_input_flags(R_STAYDOWN);
+        crmd_exit(CRM_EX_FATAL);
+    }
+}
+
+// DC gets own shutdown all-clear
+static enum crmd_fsa_input
+handle_shutdown_self_ack(xmlNode *stored_msg)
+{
+    const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
+
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+        // The expected case -- we initiated own shutdown sequence
+        crm_info("Shutting down controller");
+        return I_STOP;
+    }
+
+    if (pcmk__str_eq(host_from, controld_globals.dc_name, pcmk__str_casei)) {
+        // Must be logic error -- DC confirming its own unrequested shutdown
+        crm_err("Shutting down controller immediately due to "
+                "unexpected shutdown confirmation");
+        return I_TERMINATE;
+    }
+
+    if (controld_globals.fsa_state != S_STOPPING) {
+        // Shouldn't happen -- non-DC confirming unrequested shutdown
+        crm_err("Starting new DC election because %s is "
+                "confirming shutdown we did not request",
+                (host_from? host_from : "another node"));
+        return I_ELECTION;
+    }
+
+    // Shouldn't happen, but we are already stopping anyway
+    crm_debug("Ignoring unexpected shutdown confirmation from %s",
+              (host_from? host_from : "another node"));
+    return I_NULL;
+}
+
+// Non-DC gets shutdown all-clear from DC
+static enum crmd_fsa_input
+handle_shutdown_ack(xmlNode *stored_msg)
+{
+    const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
+
+    if (host_from == NULL) {
+        crm_warn("Ignoring shutdown request without origin specified");
+        return I_NULL;
+    }
+
+    if (pcmk__str_eq(host_from, controld_globals.dc_name,
+                     pcmk__str_null_matches|pcmk__str_casei)) {
+
+        if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+            crm_info("Shutting down controller after confirmation from %s",
+                     host_from);
+        } else {
+            crm_err("Shutting down controller after unexpected "
+                    "shutdown request from %s", host_from);
+            controld_set_fsa_input_flags(R_STAYDOWN);
+        }
+        return I_STOP;
+    }
+
+    crm_warn("Ignoring shutdown request from %s because DC is %s",
+             host_from, controld_globals.dc_name);
+    return I_NULL;
+}
+
+static enum crmd_fsa_input
+handle_request(xmlNode *stored_msg, enum crmd_fsa_cause cause)
+{
+    xmlNode *msg = NULL;
+    const char *op = crm_element_value(stored_msg, F_CRM_TASK);
+
+    /* Optimize this for the DC - it has the most to do */
+
+    if (op == NULL) {
+        crm_log_xml_warn(stored_msg, "[request without " F_CRM_TASK "]");
+        return I_NULL;
+    }
+
+    if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
+        const char *from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
+        crm_node_t *node = pcmk__search_cluster_node_cache(0, from);
+
+        pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN);
+        if(AM_I_DC == FALSE) {
+            return I_NULL; /* Done */
+        }
+    }
+
+    /*========== DC-Only Actions ==========*/
+    if (AM_I_DC) {
+        if (strcmp(op, CRM_OP_JOIN_ANNOUNCE) == 0) {
+            return I_NODE_JOIN;
+
+        } else if (strcmp(op, CRM_OP_JOIN_REQUEST) == 0) {
+            return I_JOIN_REQUEST;
+
+        } else if (strcmp(op, CRM_OP_JOIN_CONFIRM) == 0) {
+            return I_JOIN_RESULT;
+
+        } else if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
+            return handle_shutdown_self_ack(stored_msg);
+
+        } else if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
+            // Another controller wants to shut down its node
+            return handle_shutdown_request(stored_msg);
+        }
+    }
+
+    /*========== common actions ==========*/
+    if (strcmp(op, CRM_OP_NOVOTE) == 0) {
+        ha_msg_input_t fsa_input;
+
+        fsa_input.msg = stored_msg;
+        register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
+                               A_ELECTION_COUNT | A_ELECTION_CHECK, FALSE,
+                               __func__);
+
+    } else if (strcmp(op, CRM_OP_REMOTE_STATE) == 0) {
+        /* a remote connection host is letting us know the node state */
+        return handle_remote_state(stored_msg);
+
+    } else if (strcmp(op, CRM_OP_THROTTLE) == 0) {
+        throttle_update(stored_msg);
+        if (AM_I_DC && (controld_globals.transition_graph != NULL)
+            && !controld_globals.transition_graph->complete) {
+
+            crm_debug("The throttle changed. Trigger a graph.");
+            trigger_graph();
+        }
+        return I_NULL;
+
+    } else if (strcmp(op, CRM_OP_CLEAR_FAILCOUNT) == 0) {
+        return handle_failcount_op(stored_msg);
+
+    } else if (strcmp(op, CRM_OP_VOTE) == 0) {
+        /* count the vote and decide what to do after that */
+        ha_msg_input_t fsa_input;
+
+        fsa_input.msg = stored_msg;
+        register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
+                               A_ELECTION_COUNT | A_ELECTION_CHECK, FALSE,
+                               __func__);
+
+        /* Sometimes we _must_ go into S_ELECTION */
+        if (controld_globals.fsa_state == S_HALT) {
+            crm_debug("Forcing an election from S_HALT");
+            return I_ELECTION;
+#if 0
+        } else if (AM_I_DC) {
+            /* This is the old way of doing things but what is gained? */
+            return I_ELECTION;
+#endif
+        }
+
+    } else if (strcmp(op, CRM_OP_JOIN_OFFER) == 0) {
+        verify_feature_set(stored_msg);
+        crm_debug("Raising I_JOIN_OFFER: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
+        return I_JOIN_OFFER;
+
+    } else if (strcmp(op, CRM_OP_JOIN_ACKNAK) == 0) {
+        crm_debug("Raising I_JOIN_RESULT: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
+        return I_JOIN_RESULT;
+
+    } else if (strcmp(op, CRM_OP_LRM_DELETE) == 0) {
+        return handle_lrm_delete(stored_msg);
+
+    } else if ((strcmp(op, CRM_OP_LRM_FAIL) == 0)
+               || (strcmp(op, CRM_OP_LRM_REFRESH) == 0) // @COMPAT
+               || (strcmp(op, CRM_OP_REPROBE) == 0)) {
+
+        crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD);
+        return I_ROUTER;
+
+    } else if (strcmp(op, CRM_OP_NOOP) == 0) {
+        return I_NULL;
+
+    } else if (strcmp(op, CRM_OP_LOCAL_SHUTDOWN) == 0) {
+
+        crm_shutdown(SIGTERM);
+        /*return I_SHUTDOWN; */
+        return I_NULL;
+
+    } else if (strcmp(op, CRM_OP_PING) == 0) {
+        return handle_ping(stored_msg);
+
+    } else if (strcmp(op, CRM_OP_NODE_INFO) == 0) {
+        return handle_node_info_request(stored_msg);
+
+    } else if (strcmp(op, CRM_OP_RM_NODE_CACHE) == 0) {
+        int id = 0;
+        const char *name = NULL;
+
+        crm_element_value_int(stored_msg, XML_ATTR_ID, &id);
+        name = crm_element_value(stored_msg, XML_ATTR_UNAME);
+
+        if(cause == C_IPC_MESSAGE) {
+            msg = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
+            if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
+                crm_err("Could not instruct peers to remove references to node %s/%u", name, id);
+            } else {
+                crm_notice("Instructing peers to remove references to node %s/%u", name, id);
+            }
+            free_xml(msg);
+
+        } else {
+            reap_crm_member(id, name);
+
+            /* If we're forgetting this node, also forget any failures to fence
+             * it, so we don't carry that over to any node added later with the
+             * same name.
+             */
+            st_fail_count_reset(name);
+        }
+
+    } else if (strcmp(op, CRM_OP_MAINTENANCE_NODES) == 0) {
+        xmlNode *xml = get_message_xml(stored_msg, F_CRM_DATA);
+
+        remote_ra_process_maintenance_nodes(xml);
+
+    } else if (strcmp(op, PCMK__CONTROLD_CMD_NODES) == 0) {
+        return handle_node_list(stored_msg);
+
+        /*========== (NOT_DC)-Only Actions ==========*/
+    } else if (!AM_I_DC) {
+
+        if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
+            return handle_shutdown_ack(stored_msg);
+        }
+
+    } else {
+        crm_err("Unexpected request (%s) sent to %s", op, AM_I_DC ? "the DC" : "non-DC node");
+        crm_log_xml_err(stored_msg, "Unexpected");
+    }
+
+    return I_NULL;
+}
+
+static void
+handle_response(xmlNode *stored_msg)
+{
+    const char *op = crm_element_value(stored_msg, F_CRM_TASK);
+
+    if (op == NULL) {
+        crm_log_xml_err(stored_msg, "Bad message");
+
+    } else if (AM_I_DC && strcmp(op, CRM_OP_PECALC) == 0) {
+        // Check whether scheduler answer been superseded by subsequent request
+        const char *msg_ref = crm_element_value(stored_msg, XML_ATTR_REFERENCE);
+
+        if (msg_ref == NULL) {
+            crm_err("%s - Ignoring calculation with no reference", op);
+
+        } else if (pcmk__str_eq(msg_ref, controld_globals.fsa_pe_ref,
+                                pcmk__str_none)) {
+            ha_msg_input_t fsa_input;
+
+            controld_stop_sched_timer();
+            fsa_input.msg = stored_msg;
+            register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input);
+
+        } else {
+            crm_info("%s calculation %s is obsolete", op, msg_ref);
+        }
+
+    } else if (strcmp(op, CRM_OP_VOTE) == 0
+               || strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0 || strcmp(op, CRM_OP_SHUTDOWN) == 0) {
+
+    } else {
+        const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
+
+        crm_err("Unexpected response (op=%s, src=%s) sent to the %s",
+                op, host_from, AM_I_DC ? "DC" : "controller");
+    }
+}
+
+static enum crmd_fsa_input
+handle_shutdown_request(xmlNode * stored_msg)
+{
+    /* handle here to avoid potential version issues
+     *   where the shutdown message/procedure may have
+     *   been changed in later versions.
+     *
+     * This way the DC is always in control of the shutdown
+     */
+
+    char *now_s = NULL;
+    const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
+
+    if (host_from == NULL) {
+        /* we're shutting down and the DC */
+        host_from = controld_globals.our_nodename;
+    }
+
+    crm_info("Creating shutdown request for %s (state=%s)", host_from,
+             fsa_state2string(controld_globals.fsa_state));
+    crm_log_xml_trace(stored_msg, "message");
+
+    now_s = pcmk__ttoa(time(NULL));
+    update_attrd(host_from, XML_CIB_ATTR_SHUTDOWN, now_s, NULL, FALSE);
+    free(now_s);
+
+    /* will be picked up by the TE as long as its running */
+    return I_NULL;
+}
+
+static void
+send_msg_via_ipc(xmlNode * msg, const char *sys)
+{
+    pcmk__client_t *client_channel = NULL;
+
+    CRM_CHECK(sys != NULL, return);
+
+    client_channel = pcmk__find_client_by_id(sys);
+
+    if (crm_element_value(msg, F_CRM_HOST_FROM) == NULL) {
+        crm_xml_add(msg, F_CRM_HOST_FROM, controld_globals.our_nodename);
+    }
+
+    if (client_channel != NULL) {
+        /* Transient clients such as crmadmin */
+        pcmk__ipc_send_xml(client_channel, 0, msg, crm_ipc_server_event);
+
+    } else if (pcmk__str_eq(sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
+        xmlNode *data = get_message_xml(msg, F_CRM_DATA);
+
+        process_te_message(msg, data);
+
+    } else if (pcmk__str_eq(sys, CRM_SYSTEM_LRMD, pcmk__str_none)) {
+        fsa_data_t fsa_data;
+        ha_msg_input_t fsa_input;
+
+        fsa_input.msg = msg;
+        fsa_input.xml = get_message_xml(msg, F_CRM_DATA);
+
+        fsa_data.id = 0;
+        fsa_data.actions = 0;
+        fsa_data.data = &fsa_input;
+        fsa_data.fsa_input = I_MESSAGE;
+        fsa_data.fsa_cause = C_IPC_MESSAGE;
+        fsa_data.origin = __func__;
+        fsa_data.data_type = fsa_dt_ha_msg;
+
+        do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE, controld_globals.fsa_state,
+                      I_MESSAGE, &fsa_data);
+
+    } else if (crmd_is_proxy_session(sys)) {
+        crmd_proxy_send(sys, msg);
+
+    } else {
+        crm_info("Received invalid request: unknown subsystem '%s'", sys);
+    }
+}
+
+void
+delete_ha_msg_input(ha_msg_input_t * orig)
+{
+    if (orig == NULL) {
+        return;
+    }
+    free_xml(orig->msg);
+    free(orig);
+}
+
+/*!
+ * \internal
+ * \brief Notify the cluster of a remote node state change
+ *
+ * \param[in] node_name  Node's name
+ * \param[in] node_up    true if node is up, false if down
+ */
+void
+broadcast_remote_state_message(const char *node_name, bool node_up)
+{
+    xmlNode *msg = create_request(CRM_OP_REMOTE_STATE, NULL, NULL,
+                                  CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
+
+    crm_info("Notifying cluster of Pacemaker Remote node %s %s",
+             node_name, node_up? "coming up" : "going down");
+
+    crm_xml_add(msg, XML_ATTR_ID, node_name);
+    pcmk__xe_set_bool_attr(msg, XML_NODE_IN_CLUSTER, node_up);
+
+    if (node_up) {
+        crm_xml_add(msg, PCMK__XA_CONN_HOST, controld_globals.our_nodename);
+    }
+
+    send_cluster_message(NULL, crm_msg_crmd, msg, TRUE);
+    free_xml(msg);
+}
+
diff --git a/daemons/controld/controld_messages.h b/daemons/controld/controld_messages.h
new file mode 100644
index 0000000..4108961
--- /dev/null
+++ b/daemons/controld/controld_messages.h
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef XML_CRM_MESSAGES__H
+#  define XML_CRM_MESSAGES__H
+
+#  include <crm/crm.h>
+#  include <crm/common/ipc_internal.h>
+#  include <crm/common/xml.h>
+#  include <crm/cluster/internal.h>
+#  include <controld_fsa.h>
+
+typedef struct ha_msg_input_s {
+    xmlNode *msg;
+    xmlNode *xml;
+
+} ha_msg_input_t;
+
+extern void delete_ha_msg_input(ha_msg_input_t * orig);
+
+extern void *fsa_typed_data_adv(fsa_data_t * fsa_data, enum fsa_data_type a_type,
+                                const char *caller);
+
+#  define fsa_typed_data(x) fsa_typed_data_adv(msg_data, x, __func__)
+
+extern void register_fsa_error_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
+                                   fsa_data_t * cur_data, void *new_data, const char *raised_from);
+
+#define register_fsa_error(cause, input, new_data)  \
+    register_fsa_error_adv(cause, input, msg_data, new_data, __func__)
+
+void register_fsa_input_adv(enum crmd_fsa_cause cause,
+                            enum crmd_fsa_input input, void *data,
+                            uint64_t with_actions, gboolean prepend,
+                            const char *raised_from);
+
+extern void fsa_dump_queue(int log_level);
+extern void route_message(enum crmd_fsa_cause cause, xmlNode * input);
+
+#  define crmd_fsa_stall(suppress) do {                                 \
+    if(suppress == FALSE && msg_data != NULL) {                         \
+        register_fsa_input_adv(                                         \
+            ((fsa_data_t*)msg_data)->fsa_cause, I_WAIT_FOR_EVENT,       \
+            ((fsa_data_t*)msg_data)->data, action, TRUE, __func__);     \
+    } else {                                                            \
+        register_fsa_input_adv(                                         \
+            C_FSA_INTERNAL, I_WAIT_FOR_EVENT,                           \
+            NULL, action, TRUE, __func__);                              \
+    }                                                                   \
+    } while(0)
+
+#define register_fsa_input(cause, input, data)          \
+    register_fsa_input_adv(cause, input, data, A_NOTHING, FALSE, __func__)
+
+#define register_fsa_input_before(cause, input, data)   \
+    register_fsa_input_adv(cause, input, data, A_NOTHING, TRUE, __func__)
+
+#define register_fsa_input_later(cause, input, data)    \
+    register_fsa_input_adv(cause, input, data, A_NOTHING, FALSE, __func__)
+
+void delete_fsa_input(fsa_data_t * fsa_data);
+
+fsa_data_t *get_message(void);
+
+extern gboolean relay_message(xmlNode * relay_message, gboolean originated_locally);
+
+gboolean crmd_is_proxy_session(const char *session);
+void crmd_proxy_send(const char *session, xmlNode *msg);
+
+bool controld_authorize_ipc_message(const xmlNode *client_msg,
+                                    pcmk__client_t *curr_client,
+                                    const char *proxy_session);
+
+extern gboolean send_request(xmlNode * msg, char **msg_reference);
+
+extern ha_msg_input_t *copy_ha_msg_input(ha_msg_input_t * orig);
+
+void broadcast_remote_state_message(const char *node_name, bool node_up);
+
+#endif
diff --git a/daemons/controld/controld_metadata.c b/daemons/controld/controld_metadata.c
new file mode 100644
index 0000000..240a978
--- /dev/null
+++ b/daemons/controld/controld_metadata.c
@@ -0,0 +1,320 @@
+/*
+ * Copyright 2017-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <glib.h>
+#include <regex.h>
+
+#include <crm/crm.h>
+#include <crm/lrmd.h>
+
+#include <pacemaker-controld.h>
+
+static void
+ra_param_free(void *param)
+{
+    if (param) {
+        struct ra_param_s *p = (struct ra_param_s *) param;
+
+        if (p->rap_name) {
+            free(p->rap_name);
+        }
+        free(param);
+    }
+}
+
+static void
+metadata_free(void *metadata)
+{
+    if (metadata) {
+        struct ra_metadata_s *md = (struct ra_metadata_s *) metadata;
+
+        g_list_free_full(md->ra_params, ra_param_free);
+        free(metadata);
+    }
+}
+
+GHashTable *
+metadata_cache_new(void)
+{
+    return pcmk__strkey_table(free, metadata_free);
+}
+
+void
+metadata_cache_free(GHashTable *mdc)
+{
+    if (mdc) {
+        crm_trace("Destroying metadata cache with %d members", g_hash_table_size(mdc));
+        g_hash_table_destroy(mdc);
+    }
+}
+
+void
+metadata_cache_reset(GHashTable *mdc)
+{
+    if (mdc) {
+        crm_trace("Resetting metadata cache with %d members",
+                  g_hash_table_size(mdc));
+        g_hash_table_remove_all(mdc);
+    }
+}
+
+static struct ra_param_s *
+ra_param_from_xml(xmlNode *param_xml)
+{
+    const char *param_name = crm_element_value(param_xml, "name");
+    struct ra_param_s *p;
+
+    p = calloc(1, sizeof(struct ra_param_s));
+    if (p == NULL) {
+        return NULL;
+    }
+
+    p->rap_name = strdup(param_name);
+    if (p->rap_name == NULL) {
+        free(p);
+        return NULL;
+    }
+
+    if (pcmk__xe_attr_is_true(param_xml, "reloadable")) {
+        controld_set_ra_param_flags(p, ra_param_reloadable);
+    }
+
+    if (pcmk__xe_attr_is_true(param_xml, "unique")) {
+        controld_set_ra_param_flags(p, ra_param_unique);
+    }
+
+    if (pcmk__xe_attr_is_true(param_xml, "private")) {
+        controld_set_ra_param_flags(p, ra_param_private);
+    }
+    return p;
+}
+
+static void
+log_ra_ocf_version(const char *ra_key, const char *ra_ocf_version)
+{
+    if (pcmk__str_empty(ra_ocf_version)) {
+        crm_warn("%s does not advertise OCF version supported", ra_key);
+
+    } else if (compare_version(ra_ocf_version, "2") >= 0) {
+        crm_warn("%s supports OCF version %s (this Pacemaker version supports "
+                 PCMK_OCF_VERSION " and might not work properly with agent)",
+                 ra_key, ra_ocf_version);
+
+    } else if (compare_version(ra_ocf_version, PCMK_OCF_VERSION) > 0) {
+        crm_info("%s supports OCF version %s (this Pacemaker version supports "
+                 PCMK_OCF_VERSION " and might not use all agent features)",
+                 ra_key, ra_ocf_version);
+
+    } else {
+        crm_debug("%s supports OCF version %s", ra_key, ra_ocf_version);
+    }
+}
+
+struct ra_metadata_s *
+controld_cache_metadata(GHashTable *mdc, const lrmd_rsc_info_t *rsc,
+                        const char *metadata_str)
+{
+    char *key = NULL;
+    const char *reason = NULL;
+    xmlNode *metadata = NULL;
+    xmlNode *match = NULL;
+    struct ra_metadata_s *md = NULL;
+    bool any_private_params = false;
+    bool ocf1_1 = false;
+
+    CRM_CHECK(mdc && rsc && metadata_str, return NULL);
+
+    key = crm_generate_ra_key(rsc->standard, rsc->provider, rsc->type);
+    if (!key) {
+        reason = "Invalid resource agent standard or type";
+        goto err;
+    }
+
+    metadata = string2xml(metadata_str);
+    if (!metadata) {
+        reason = "Metadata is not valid XML";
+        goto err;
+    }
+
+    md = calloc(1, sizeof(struct ra_metadata_s));
+    if (md == NULL) {
+        reason = "Could not allocate memory";
+        goto err;
+    }
+
+    if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_OCF) == 0) {
+        xmlChar *content = NULL;
+        xmlNode *version_element = first_named_child(metadata, "version");
+
+        if (version_element != NULL) {
+            content = xmlNodeGetContent(version_element);
+        }
+        log_ra_ocf_version(key, (const char *) content);
+        if (content != NULL) {
+            ocf1_1 = (compare_version((const char *) content, "1.1") >= 0);
+            xmlFree(content);
+        }
+    }
+
+    // Check supported actions
+    match = first_named_child(metadata, "actions");
+    for (match = first_named_child(match, "action"); match != NULL;
+         match = crm_next_same_xml(match)) {
+
+        const char *action_name = crm_element_value(match, "name");
+
+        if (pcmk__str_eq(action_name, CRMD_ACTION_RELOAD_AGENT,
+                         pcmk__str_none)) {
+            if (ocf1_1) {
+                controld_set_ra_flags(md, key, ra_supports_reload_agent);
+            } else {
+                crm_notice("reload-agent action will not be used with %s "
+                           "because it does not support OCF 1.1 or later", key);
+            }
+
+        } else if (!ocf1_1 && pcmk__str_eq(action_name, CRMD_ACTION_RELOAD,
+                                           pcmk__str_casei)) {
+            controld_set_ra_flags(md, key, ra_supports_legacy_reload);
+        }
+    }
+
+    // Build a parameter list
+    match = first_named_child(metadata, "parameters");
+    for (match = first_named_child(match, "parameter"); match != NULL;
+         match = crm_next_same_xml(match)) {
+
+        const char *param_name = crm_element_value(match, "name");
+
+        if (param_name == NULL) {
+            crm_warn("Metadata for %s:%s:%s has parameter without a name",
+                     rsc->standard, rsc->provider, rsc->type);
+        } else {
+            struct ra_param_s *p = ra_param_from_xml(match);
+
+            if (p == NULL) {
+                reason = "Could not allocate memory";
+                goto err;
+            }
+            if (pcmk_is_set(p->rap_flags, ra_param_private)) {
+                any_private_params = true;
+            }
+            md->ra_params = g_list_prepend(md->ra_params, p);
+        }
+    }
+
+    /* Newer resource agents support the "private" parameter attribute to
+     * indicate sensitive parameters. For backward compatibility with older
+     * agents, implicitly treat a few common names as private when the agent
+     * doesn't specify any explicitly.
+     */
+    if (!any_private_params) {
+        for (GList *iter = md->ra_params; iter != NULL; iter = iter->next) {
+            struct ra_param_s *p = iter->data;
+
+            if (pcmk__str_any_of(p->rap_name, "password", "passwd", "user",
+                                 NULL)) {
+                controld_set_ra_param_flags(p, ra_param_private);
+            }
+        }
+    }
+
+    g_hash_table_replace(mdc, key, md);
+    free_xml(metadata);
+    return md;
+
+err:
+    crm_warn("Unable to update metadata for %s (%s%s%s:%s): %s",
+             rsc->id, rsc->standard, ((rsc->provider == NULL)? "" : ":"),
+             pcmk__s(rsc->provider, ""), rsc->type, reason);
+    free(key);
+    free_xml(metadata);
+    metadata_free(md);
+    return NULL;
+}
+
+/*!
+ * \internal
+ * \brief Get meta-data for a resource
+ *
+ * \param[in,out] lrm_state  Use meta-data cache from this executor connection
+ * \param[in]     rsc        Resource to get meta-data for
+ * \param[in]     source     Allowed meta-data sources (bitmask of
+ *                           enum controld_metadata_source_e values)
+ *
+ * \return Meta-data cache entry for given resource, or NULL if not available
+ */
+struct ra_metadata_s *
+controld_get_rsc_metadata(lrm_state_t *lrm_state, const lrmd_rsc_info_t *rsc,
+                          uint32_t source)
+{
+    struct ra_metadata_s *metadata = NULL;
+    char *metadata_str = NULL;
+    char *key = NULL;
+    int rc = pcmk_ok;
+
+    CRM_CHECK((lrm_state != NULL) && (rsc != NULL), return NULL);
+
+    if (pcmk_is_set(source, controld_metadata_from_cache)) {
+        key = crm_generate_ra_key(rsc->standard, rsc->provider, rsc->type);
+        if (key != NULL) {
+            metadata = g_hash_table_lookup(lrm_state->metadata_cache, key);
+            free(key);
+        }
+        if (metadata != NULL) {
+            crm_debug("Retrieved metadata for %s (%s%s%s:%s) from cache",
+                      rsc->id, rsc->standard,
+                      ((rsc->provider == NULL)? "" : ":"),
+                      ((rsc->provider == NULL)? "" : rsc->provider),
+                      rsc->type);
+            return metadata;
+        }
+    }
+
+    if (!pcmk_is_set(source, controld_metadata_from_agent)) {
+        return NULL;
+    }
+
+    /* For most actions, metadata was cached asynchronously before action
+     * execution (via metadata_complete()).
+     *
+     * However if that failed, and for other actions, retrieve the metadata now
+     * via a local, synchronous, direct execution of the agent.
+     *
+     * This has multiple issues, which is why this is just a fallback: the
+     * executor should execute agents, not the controller; metadata for
+     * Pacemaker Remote nodes should be collected on those nodes, not locally;
+     * the metadata call shouldn't eat into the timeout of the real action being
+     * performed; and the synchronous call blocks the controller (which also
+     * means that if the metadata action tries to contact the controller,
+     * everything will hang until the timeout).
+     */
+    crm_debug("Retrieving metadata for %s (%s%s%s:%s) synchronously",
+              rsc->id, rsc->standard,
+              ((rsc->provider == NULL)? "" : ":"),
+              ((rsc->provider == NULL)? "" : rsc->provider),
+              rsc->type);
+    rc = lrm_state_get_metadata(lrm_state, rsc->standard, rsc->provider,
+                                rsc->type, &metadata_str, 0);
+    if (rc != pcmk_ok) {
+        crm_warn("Failed to get metadata for %s (%s%s%s:%s): %s",
+                 rsc->id, rsc->standard,
+                 ((rsc->provider == NULL)? "" : ":"),
+                 ((rsc->provider == NULL)? "" : rsc->provider),
+                 rsc->type, pcmk_strerror(rc));
+        return NULL;
+    }
+
+    metadata = controld_cache_metadata(lrm_state->metadata_cache, rsc,
+                                       metadata_str);
+    free(metadata_str);
+    return metadata;
+}
diff --git a/daemons/controld/controld_metadata.h b/daemons/controld/controld_metadata.h
new file mode 100644
index 0000000..12ea327
--- /dev/null
+++ b/daemons/controld/controld_metadata.h
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2017-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef CRMD_METADATA_H
+#define CRMD_METADATA_H
+
+#include <stdint.h>             // uint32_t
+#include <glib.h>               // GList, GHashTable
+#include "controld_lrm.h"       // lrm_state_t, lrm_rsc_info_t
+
+/*
+ * @COMPAT pre-OCF-1.1 resource agents
+ *
+ * Pacemaker previously used the "reload" action to reload agent parameters,
+ * but most agents used it to reload the service configuration. Pacemaker also
+ * misused the OCF 1.0 "unique" parameter attribute to indicate reloadability.
+ *
+ * OCF 1.1 created the "reload-agent" action and "reloadable" parameter
+ * attribute for the Pacemaker usage.
+ *
+ * Pacemaker now supports the OCF 1.1 usage. The old usage is now deprecated,
+ * but will be supported if the agent does not claim OCF 1.1 or later
+ * compliance and does not advertise the reload-agent action.
+ */
+enum ra_flags_e {
+    ra_supports_legacy_reload   = (1 << 0),
+    ra_supports_reload_agent    = (1 << 1),
+};
+
+enum ra_param_flags_e {
+    ra_param_unique             = (1 << 0),
+    ra_param_private            = (1 << 1),
+    ra_param_reloadable         = (1 << 2),
+};
+
+// Allowed sources of resource agent meta-data when requesting it
+enum controld_metadata_source_e {
+    controld_metadata_from_cache    = (1 << 0),
+    controld_metadata_from_agent    = (1 << 1),
+};
+
+struct ra_param_s {
+    char *rap_name;
+    uint32_t rap_flags; // bitmask of ra_param_flags_s
+};
+
+struct ra_metadata_s {
+    GList *ra_params;   // ra_param_s
+    uint32_t ra_flags;  // bitmask of ra_flags_e
+};
+
+#define controld_set_ra_flags(ra_md, ra_key, flags_to_set) do {             \
+        (ra_md)->ra_flags = pcmk__set_flags_as(__func__, __LINE__,          \
+            LOG_TRACE, "Resource agent", ra_key,                            \
+            (ra_md)->ra_flags, (flags_to_set), #flags_to_set);              \
+    } while (0)
+
+#define controld_set_ra_param_flags(ra_param, flags_to_set) do {            \
+        (ra_param)->rap_flags = pcmk__set_flags_as(__func__, __LINE__,      \
+            LOG_TRACE, "Resource agent parameter", (ra_param)->rap_name,    \
+            (ra_param)->rap_flags, (flags_to_set), #flags_to_set);          \
+    } while (0)
+
+GHashTable *metadata_cache_new(void);
+void metadata_cache_free(GHashTable *mdc);
+void metadata_cache_reset(GHashTable *mdc);
+
+struct ra_metadata_s *controld_cache_metadata(GHashTable *mdc,
+                                              const lrmd_rsc_info_t *rsc,
+                                              const char *metadata_str);
+struct ra_metadata_s *controld_get_rsc_metadata(lrm_state_t *lrm_state,
+                                                const lrmd_rsc_info_t *rsc,
+                                                uint32_t source);
+
+static inline const char *
+ra_param_flag2text(enum ra_param_flags_e flag)
+{
+    switch (flag) {
+        case ra_param_reloadable:
+            return "reloadable";
+        case ra_param_unique:
+            return "unique";
+        case ra_param_private:
+            return "private";
+        default:
+            return "unknown";
+    }
+}
+
+#endif
diff --git a/daemons/controld/controld_remote_ra.c b/daemons/controld/controld_remote_ra.c
new file mode 100644
index 0000000..f24b755
--- /dev/null
+++ b/daemons/controld/controld_remote_ra.c
@@ -0,0 +1,1440 @@
+/*
+ * Copyright 2013-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml_internal.h>
+#include <crm/lrmd.h>
+#include <crm/lrmd_internal.h>
+#include <crm/services.h>
+
+#include <pacemaker-controld.h>
+
+#define REMOTE_LRMD_RA "remote"
+
+/* The max start timeout before cmd retry */
+#define MAX_START_TIMEOUT_MS 10000
+
+#define cmd_set_flags(cmd, flags_to_set) do { \
+    (cmd)->status = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
+                                       "Remote command", (cmd)->rsc_id, (cmd)->status, \
+                                       (flags_to_set), #flags_to_set); \
+        } while (0)
+
+#define cmd_clear_flags(cmd, flags_to_clear) do { \
+    (cmd)->status = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
+                                         "Remote command", (cmd)->rsc_id, (cmd)->status, \
+                                         (flags_to_clear), #flags_to_clear); \
+        } while (0)
+
+enum remote_cmd_status {
+    cmd_reported_success    = (1 << 0),
+    cmd_cancel              = (1 << 1),
+};
+
+typedef struct remote_ra_cmd_s {
+    /*! the local node the cmd is issued from */
+    char *owner;
+    /*! the remote node the cmd is executed on */
+    char *rsc_id;
+    /*! the action to execute */
+    char *action;
+    /*! some string the client wants us to give it back */
+    char *userdata;
+    /*! start delay in ms */
+    int start_delay;
+    /*! timer id used for start delay. */
+    int delay_id;
+    /*! timeout in ms for cmd */
+    int timeout;
+    int remaining_timeout;
+    /*! recurring interval in ms */
+    guint interval_ms;
+    /*! interval timer id */
+    int interval_id;
+    int monitor_timeout_id;
+    int takeover_timeout_id;
+    /*! action parameters */
+    lrmd_key_value_t *params;
+    pcmk__action_result_t result;
+    int call_id;
+    time_t start_time;
+    uint32_t status;
+} remote_ra_cmd_t;
+
+#define lrm_remote_set_flags(lrm_state, flags_to_set) do { \
+    lrm_state_t *lrm = (lrm_state); \
+    remote_ra_data_t *ra = lrm->remote_ra_data; \
+    ra->status = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, "Remote", \
+                                    lrm->node_name, ra->status, \
+                                    (flags_to_set), #flags_to_set); \
+        } while (0)
+
+#define lrm_remote_clear_flags(lrm_state, flags_to_clear) do { \
+    lrm_state_t *lrm = (lrm_state); \
+    remote_ra_data_t *ra = lrm->remote_ra_data; \
+    ra->status = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, "Remote", \
+                                      lrm->node_name, ra->status, \
+                                      (flags_to_clear), #flags_to_clear); \
+        } while (0)
+
+enum remote_status {
+    expect_takeover     = (1 << 0),
+    takeover_complete   = (1 << 1),
+    remote_active       = (1 << 2),
+    /* Maintenance mode is difficult to determine from the controller's context,
+     * so we have it signalled back with the transition from the scheduler.
+     */
+    remote_in_maint     = (1 << 3),
+    /* Similar for whether we are controlling a guest node or remote node.
+     * Fortunately there is a meta-attribute in the transition already and
+     * as the situation doesn't change over time we can use the
+     * resource start for noting down the information for later use when
+     * the attributes aren't at hand.
+     */
+    controlling_guest   = (1 << 4),
+};
+
+typedef struct remote_ra_data_s {
+    crm_trigger_t *work;
+    remote_ra_cmd_t *cur_cmd;
+    GList *cmds;
+    GList *recurring_cmds;
+    uint32_t status;
+} remote_ra_data_t;
+
+static int handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms);
+static void handle_remote_ra_stop(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd);
+static GList *fail_all_monitor_cmds(GList * list);
+
+static void
+free_cmd(gpointer user_data)
+{
+    remote_ra_cmd_t *cmd = user_data;
+
+    if (!cmd) {
+        return;
+    }
+    if (cmd->delay_id) {
+        g_source_remove(cmd->delay_id);
+    }
+    if (cmd->interval_id) {
+        g_source_remove(cmd->interval_id);
+    }
+    if (cmd->monitor_timeout_id) {
+        g_source_remove(cmd->monitor_timeout_id);
+    }
+    if (cmd->takeover_timeout_id) {
+        g_source_remove(cmd->takeover_timeout_id);
+    }
+    free(cmd->owner);
+    free(cmd->rsc_id);
+    free(cmd->action);
+    free(cmd->userdata);
+    pcmk__reset_result(&(cmd->result));
+    lrmd_key_value_freeall(cmd->params);
+    free(cmd);
+}
+
+static int
+generate_callid(void)
+{
+    static int remote_ra_callid = 0;
+
+    remote_ra_callid++;
+    if (remote_ra_callid <= 0) {
+        remote_ra_callid = 1;
+    }
+
+    return remote_ra_callid;
+}
+
+static gboolean
+recurring_helper(gpointer data)
+{
+    remote_ra_cmd_t *cmd = data;
+    lrm_state_t *connection_rsc = NULL;
+
+    cmd->interval_id = 0;
+    connection_rsc = lrm_state_find(cmd->rsc_id);
+    if (connection_rsc && connection_rsc->remote_ra_data) {
+        remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;
+
+        ra_data->recurring_cmds = g_list_remove(ra_data->recurring_cmds, cmd);
+
+        ra_data->cmds = g_list_append(ra_data->cmds, cmd);
+        mainloop_set_trigger(ra_data->work);
+    }
+    return FALSE;
+}
+
+static gboolean
+start_delay_helper(gpointer data)
+{
+    remote_ra_cmd_t *cmd = data;
+    lrm_state_t *connection_rsc = NULL;
+
+    cmd->delay_id = 0;
+    connection_rsc = lrm_state_find(cmd->rsc_id);
+    if (connection_rsc && connection_rsc->remote_ra_data) {
+        remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;
+
+        mainloop_set_trigger(ra_data->work);
+    }
+    return FALSE;
+}
+
+static bool
+should_purge_attributes(crm_node_t *node)
+{
+    bool purge = true;
+    crm_node_t *conn_node = NULL;
+    lrm_state_t *connection_rsc = NULL;
+
+    if (!node->conn_host) {
+        return purge;
+    }
+
+    /* Get the node that was hosting the remote connection resource from the
+     * peer cache.  That's the one we really care about here.
+     */
+    conn_node = crm_get_peer(0, node->conn_host);
+    if (conn_node == NULL) {
+        return purge;
+    }
+
+    /* Check the uptime of connection_rsc.  If it hasn't been running long
+     * enough, set purge=true.  "Long enough" means it started running earlier
+     * than the timestamp when we noticed it went away in the first place.
+     */
+    connection_rsc = lrm_state_find(node->uname);
+
+    if (connection_rsc != NULL) {
+        lrmd_t *lrm = connection_rsc->conn;
+        time_t uptime = lrmd__uptime(lrm);
+        time_t now = time(NULL);
+
+        /* Add 20s of fuzziness to give corosync a while to notice the remote
+         * host is gone.  On various error conditions (failure to get uptime,
+         * peer_lost isn't set) we default to purging.
+         */
+        if (uptime > 0 &&
+            conn_node->peer_lost > 0 &&
+            uptime + 20 >= now - conn_node->peer_lost) {
+            purge = false;
+        }
+    }
+
+    return purge;
+}
+
+static enum controld_section_e
+section_to_delete(bool purge)
+{
+    if (pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
+        if (purge) {
+            return controld_section_all_unlocked;
+        } else {
+            return controld_section_lrm_unlocked;
+        }
+    } else {
+        if (purge) {
+            return controld_section_all;
+        } else {
+            return controld_section_lrm;
+        }
+    }
+}
+
+static void
+purge_remote_node_attrs(int call_opt, crm_node_t *node)
+{
+    bool purge = should_purge_attributes(node);
+    enum controld_section_e section = section_to_delete(purge);
+
+    /* Purge node from attrd's memory */
+    if (purge) {
+        update_attrd_remote_node_removed(node->uname, NULL);
+    }
+
+    controld_delete_node_state(node->uname, section, call_opt);
+}
+
+/*!
+ * \internal
+ * \brief Handle cluster communication related to pacemaker_remote node joining
+ *
+ * \param[in] node_name  Name of newly integrated pacemaker_remote node
+ */
+static void
+remote_node_up(const char *node_name)
+{
+    int call_opt;
+    xmlNode *update, *state;
+    crm_node_t *node;
+
+    CRM_CHECK(node_name != NULL, return);
+    crm_info("Announcing Pacemaker Remote node %s", node_name);
+
+    call_opt = crmd_cib_smart_opt();
+
+    /* Delete node's probe_complete attribute. This serves two purposes:
+     *
+     * - @COMPAT DCs < 1.1.14 in a rolling upgrade might use it
+     * - deleting it (or any attribute for that matter) here ensures the
+     *   attribute manager learns the node is remote
+     */
+    update_attrd(node_name, CRM_OP_PROBED, NULL, NULL, TRUE);
+
+    /* Ensure node is in the remote peer cache with member status */
+    node = crm_remote_peer_get(node_name);
+    CRM_CHECK(node != NULL, return);
+
+    purge_remote_node_attrs(call_opt, node);
+    pcmk__update_peer_state(__func__, node, CRM_NODE_MEMBER, 0);
+
+    /* pacemaker_remote nodes don't participate in the membership layer,
+     * so cluster nodes don't automatically get notified when they come and go.
+     * We send a cluster message to the DC, and update the CIB node state entry,
+     * so the DC will get it sooner (via message) or later (via CIB refresh),
+     * and any other interested parties can query the CIB.
+     */
+    broadcast_remote_state_message(node_name, true);
+
+    update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
+    state = create_node_state_update(node, node_update_cluster, update,
+                                     __func__);
+
+    /* Clear the XML_NODE_IS_FENCED flag in the node state. If the node ever
+     * needs to be fenced, this flag will allow various actions to determine
+     * whether the fencing has happened yet.
+     */
+    crm_xml_add(state, XML_NODE_IS_FENCED, "0");
+
+    /* TODO: If the remote connection drops, and this (async) CIB update either
+     * failed or has not yet completed, later actions could mistakenly think the
+     * node has already been fenced (if the XML_NODE_IS_FENCED attribute was
+     * previously set, because it won't have been cleared). This could prevent
+     * actual fencing or allow recurring monitor failures to be cleared too
+     * soon. Ideally, we wouldn't rely on the CIB for the fenced status.
+     */
+    controld_update_cib(XML_CIB_TAG_STATUS, update, call_opt, NULL);
+    free_xml(update);
+}
+
+enum down_opts {
+    DOWN_KEEP_LRM,
+    DOWN_ERASE_LRM
+};
+
+/*!
+ * \internal
+ * \brief Handle cluster communication related to pacemaker_remote node leaving
+ *
+ * \param[in] node_name  Name of lost node
+ * \param[in] opts       Whether to keep or erase LRM history
+ */
+static void
+remote_node_down(const char *node_name, const enum down_opts opts)
+{
+    xmlNode *update;
+    int call_opt = crmd_cib_smart_opt();
+    crm_node_t *node;
+
+    /* Purge node from attrd's memory */
+    update_attrd_remote_node_removed(node_name, NULL);
+
+    /* Normally, only node attributes should be erased, and the resource history
+     * should be kept until the node comes back up. However, after a successful
+     * fence, we want to clear the history as well, so we don't think resources
+     * are still running on the node.
+     */
+    if (opts == DOWN_ERASE_LRM) {
+        controld_delete_node_state(node_name, controld_section_all, call_opt);
+    } else {
+        controld_delete_node_state(node_name, controld_section_attrs, call_opt);
+    }
+
+    /* Ensure node is in the remote peer cache with lost state */
+    node = crm_remote_peer_get(node_name);
+    CRM_CHECK(node != NULL, return);
+    pcmk__update_peer_state(__func__, node, CRM_NODE_LOST, 0);
+
+    /* Notify DC */
+    broadcast_remote_state_message(node_name, false);
+
+    /* Update CIB node state */
+    update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
+    create_node_state_update(node, node_update_cluster, update, __func__);
+    controld_update_cib(XML_CIB_TAG_STATUS, update, call_opt, NULL);
+    free_xml(update);
+}
+
+/*!
+ * \internal
+ * \brief Handle effects of a remote RA command on node state
+ *
+ * \param[in] cmd  Completed remote RA command
+ */
+static void
+check_remote_node_state(const remote_ra_cmd_t *cmd)
+{
+    /* Only successful actions can change node state */
+    if (!pcmk__result_ok(&(cmd->result))) {
+        return;
+    }
+
+    if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {
+        remote_node_up(cmd->rsc_id);
+
+    } else if (pcmk__str_eq(cmd->action, "migrate_from", pcmk__str_casei)) {
+        /* After a successful migration, we don't need to do remote_node_up()
+         * because the DC already knows the node is up, and we don't want to
+         * clear LRM history etc. We do need to add the remote node to this
+         * host's remote peer cache, because (unless it happens to be DC)
+         * it hasn't been tracking the remote node, and other code relies on
+         * the cache to distinguish remote nodes from unseen cluster nodes.
+         */
+        crm_node_t *node = crm_remote_peer_get(cmd->rsc_id);
+
+        CRM_CHECK(node != NULL, return);
+        pcmk__update_peer_state(__func__, node, CRM_NODE_MEMBER, 0);
+
+    } else if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
+        lrm_state_t *lrm_state = lrm_state_find(cmd->rsc_id);
+        remote_ra_data_t *ra_data = lrm_state? lrm_state->remote_ra_data : NULL;
+
+        if (ra_data) {
+            if (!pcmk_is_set(ra_data->status, takeover_complete)) {
+                /* Stop means down if we didn't successfully migrate elsewhere */
+                remote_node_down(cmd->rsc_id, DOWN_KEEP_LRM);
+            } else if (AM_I_DC == FALSE) {
+                /* Only the connection host and DC track node state,
+                 * so if the connection migrated elsewhere and we aren't DC,
+                 * un-cache the node, so we don't have stale info
+                 */
+                crm_remote_peer_cache_remove(cmd->rsc_id);
+            }
+        }
+    }
+
+    /* We don't do anything for successful monitors, which is correct for
+     * routine recurring monitors, and for monitors on nodes where the
+     * connection isn't supposed to be (the cluster will stop the connection in
+     * that case). However, if the initial probe finds the connection already
+     * active on the node where we want it, we probably should do
+     * remote_node_up(). Unfortunately, we can't distinguish that case here.
+     * Given that connections have to be initiated by the cluster, the chance of
+     * that should be close to zero.
+     */
+}
+
+static void
+report_remote_ra_result(remote_ra_cmd_t * cmd)
+{
+    lrmd_event_data_t op = { 0, };
+
+    check_remote_node_state(cmd);
+
+    op.type = lrmd_event_exec_complete;
+    op.rsc_id = cmd->rsc_id;
+    op.op_type = cmd->action;
+    op.user_data = cmd->userdata;
+    op.timeout = cmd->timeout;
+    op.interval_ms = cmd->interval_ms;
+    op.t_run = (unsigned int) cmd->start_time;
+    op.t_rcchange = (unsigned int) cmd->start_time;
+
+    lrmd__set_result(&op, cmd->result.exit_status, cmd->result.execution_status,
+                     cmd->result.exit_reason);
+
+    if (pcmk_is_set(cmd->status, cmd_reported_success) && !pcmk__result_ok(&(cmd->result))) {
+        op.t_rcchange = (unsigned int) time(NULL);
+        /* This edge case will likely never ever occur, but if it does the
+         * result is that a failure will not be processed correctly. This is only
+         * remotely possible because we are able to detect a connection resource's tcp
+         * connection has failed at any moment after start has completed. The actual
+         * recurring operation is just a connectivity ping.
+         *
+         * basically, we are not guaranteed that the first successful monitor op and
+         * a subsequent failed monitor op will not occur in the same timestamp. We have to
+         * make it look like the operations occurred at separate times though. */
+        if (op.t_rcchange == op.t_run) {
+            op.t_rcchange++;
+        }
+    }
+
+    if (cmd->params) {
+        lrmd_key_value_t *tmp;
+
+        op.params = pcmk__strkey_table(free, free);
+        for (tmp = cmd->params; tmp; tmp = tmp->next) {
+            g_hash_table_insert(op.params, strdup(tmp->key), strdup(tmp->value));
+        }
+
+    }
+    op.call_id = cmd->call_id;
+    op.remote_nodename = cmd->owner;
+
+    lrm_op_callback(&op);
+
+    if (op.params) {
+        g_hash_table_destroy(op.params);
+    }
+    lrmd__reset_result(&op);
+}
+
+static void
+update_remaining_timeout(remote_ra_cmd_t * cmd)
+{
+    cmd->remaining_timeout = ((cmd->timeout / 1000) - (time(NULL) - cmd->start_time)) * 1000;
+}
+
+static gboolean
+retry_start_cmd_cb(gpointer data)
+{
+    lrm_state_t *lrm_state = data;
+    remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
+    remote_ra_cmd_t *cmd = NULL;
+    int rc = ETIME;
+
+    if (!ra_data || !ra_data->cur_cmd) {
+        return FALSE;
+    }
+    cmd = ra_data->cur_cmd;
+    if (!pcmk__strcase_any_of(cmd->action, "start", "migrate_from", NULL)) {
+        return FALSE;
+    }
+    update_remaining_timeout(cmd);
+
+    if (cmd->remaining_timeout > 0) {
+        rc = handle_remote_ra_start(lrm_state, cmd, cmd->remaining_timeout);
+    } else {
+        pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
+                         PCMK_EXEC_TIMEOUT,
+                         "Not enough time remains to retry remote connection");
+    }
+
+    if (rc != pcmk_rc_ok) {
+        report_remote_ra_result(cmd);
+
+        if (ra_data->cmds) {
+            mainloop_set_trigger(ra_data->work);
+        }
+        ra_data->cur_cmd = NULL;
+        free_cmd(cmd);
+    } else {
+        /* wait for connection event */
+    }
+
+    return FALSE;
+}
+
+
+static gboolean
+connection_takeover_timeout_cb(gpointer data)
+{
+    lrm_state_t *lrm_state = NULL;
+    remote_ra_cmd_t *cmd = data;
+
+    crm_info("takeover event timed out for node %s", cmd->rsc_id);
+    cmd->takeover_timeout_id = 0;
+
+    lrm_state = lrm_state_find(cmd->rsc_id);
+
+    handle_remote_ra_stop(lrm_state, cmd);
+    free_cmd(cmd);
+
+    return FALSE;
+}
+
+static gboolean
+monitor_timeout_cb(gpointer data)
+{
+    lrm_state_t *lrm_state = NULL;
+    remote_ra_cmd_t *cmd = data;
+
+    lrm_state = lrm_state_find(cmd->rsc_id);
+
+    crm_info("Timed out waiting for remote poke response from %s%s",
+             cmd->rsc_id, (lrm_state? "" : " (no LRM state)"));
+    cmd->monitor_timeout_id = 0;
+    pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_TIMEOUT,
+                     "Remote executor did not respond");
+
+    if (lrm_state && lrm_state->remote_ra_data) {
+        remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
+
+        if (ra_data->cur_cmd == cmd) {
+            ra_data->cur_cmd = NULL;
+        }
+        if (ra_data->cmds) {
+            mainloop_set_trigger(ra_data->work);
+        }
+    }
+
+    report_remote_ra_result(cmd);
+    free_cmd(cmd);
+
+    if(lrm_state) {
+        lrm_state_disconnect(lrm_state);
+    }
+    return FALSE;
+}
+
+static void
+synthesize_lrmd_success(lrm_state_t *lrm_state, const char *rsc_id, const char *op_type)
+{
+    lrmd_event_data_t op = { 0, };
+
+    if (lrm_state == NULL) {
+        /* if lrm_state not given assume local */
+        lrm_state = lrm_state_find(controld_globals.our_nodename);
+    }
+    CRM_ASSERT(lrm_state != NULL);
+
+    op.type = lrmd_event_exec_complete;
+    op.rsc_id = rsc_id;
+    op.op_type = op_type;
+    op.t_run = (unsigned int) time(NULL);
+    op.t_rcchange = op.t_run;
+    op.call_id = generate_callid();
+    lrmd__set_result(&op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
+    process_lrm_event(lrm_state, &op, NULL, NULL);
+}
+
+void
+remote_lrm_op_callback(lrmd_event_data_t * op)
+{
+    gboolean cmd_handled = FALSE;
+    lrm_state_t *lrm_state = NULL;
+    remote_ra_data_t *ra_data = NULL;
+    remote_ra_cmd_t *cmd = NULL;
+
+    crm_debug("Processing '%s%s%s' event on remote connection to %s: %s "
+              "(%d) status=%s (%d)",
+              (op->op_type? op->op_type : ""), (op->op_type? " " : ""),
+              lrmd_event_type2str(op->type), op->remote_nodename,
+              services_ocf_exitcode_str(op->rc), op->rc,
+              pcmk_exec_status_str(op->op_status), op->op_status);
+
+    lrm_state = lrm_state_find(op->remote_nodename);
+    if (!lrm_state || !lrm_state->remote_ra_data) {
+        crm_debug("No state information found for remote connection event");
+        return;
+    }
+    ra_data = lrm_state->remote_ra_data;
+
+    if (op->type == lrmd_event_new_client) {
+        // Another client has connected to the remote daemon
+
+        if (pcmk_is_set(ra_data->status, expect_takeover)) {
+            // Great, we knew this was coming
+            lrm_remote_clear_flags(lrm_state, expect_takeover);
+            lrm_remote_set_flags(lrm_state, takeover_complete);
+
+        } else {
+            crm_err("Disconnecting from Pacemaker Remote node %s due to "
+                    "unexpected client takeover", op->remote_nodename);
+            /* In this case, lrmd_tls_connection_destroy() will be called under the control of mainloop. */
+            /* Do not free lrm_state->conn yet. */
+            /* It'll be freed in the following stop action. */
+            lrm_state_disconnect_only(lrm_state);
+        }
+        return;
+    }
+
+    /* filter all EXEC events up */
+    if (op->type == lrmd_event_exec_complete) {
+        if (pcmk_is_set(ra_data->status, takeover_complete)) {
+            crm_debug("ignoring event, this connection is taken over by another node");
+        } else {
+            lrm_op_callback(op);
+        }
+        return;
+    }
+
+    if ((op->type == lrmd_event_disconnect) && (ra_data->cur_cmd == NULL)) {
+
+        if (!pcmk_is_set(ra_data->status, remote_active)) {
+            crm_debug("Disconnection from Pacemaker Remote node %s complete",
+                      lrm_state->node_name);
+
+        } else if (!remote_ra_is_in_maintenance(lrm_state)) {
+            crm_err("Lost connection to Pacemaker Remote node %s",
+                    lrm_state->node_name);
+            ra_data->recurring_cmds = fail_all_monitor_cmds(ra_data->recurring_cmds);
+            ra_data->cmds = fail_all_monitor_cmds(ra_data->cmds);
+
+        } else {
+            crm_notice("Unmanaged Pacemaker Remote node %s disconnected",
+                       lrm_state->node_name);
+            /* Do roughly what a 'stop' on the remote-resource would do */
+            handle_remote_ra_stop(lrm_state, NULL);
+            remote_node_down(lrm_state->node_name, DOWN_KEEP_LRM);
+            /* now fake the reply of a successful 'stop' */
+            synthesize_lrmd_success(NULL, lrm_state->node_name, "stop");
+        }
+        return;
+    }
+
+    if (!ra_data->cur_cmd) {
+        crm_debug("no event to match");
+        return;
+    }
+
+    cmd = ra_data->cur_cmd;
+
+    /* Start actions and migrate from actions complete after connection
+     * comes back to us. */
+    if (op->type == lrmd_event_connect && pcmk__strcase_any_of(cmd->action, "start",
+                                                               "migrate_from", NULL)) {
+        if (op->connection_rc < 0) {
+            update_remaining_timeout(cmd);
+
+            if ((op->connection_rc == -ENOKEY)
+                || (op->connection_rc == -EKEYREJECTED)) {
+                // Hard error, don't retry
+                pcmk__set_result(&(cmd->result), PCMK_OCF_INVALID_PARAM,
+                                 PCMK_EXEC_ERROR,
+                                 pcmk_strerror(op->connection_rc));
+
+            } else if (cmd->remaining_timeout > 3000) {
+                crm_trace("rescheduling start, remaining timeout %d", cmd->remaining_timeout);
+                g_timeout_add(1000, retry_start_cmd_cb, lrm_state);
+                return;
+
+            } else {
+                crm_trace("can't reschedule start, remaining timeout too small %d",
+                          cmd->remaining_timeout);
+                pcmk__format_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
+                                    PCMK_EXEC_TIMEOUT,
+                                    "%s without enough time to retry",
+                                    pcmk_strerror(op->connection_rc));
+            }
+
+        } else {
+            lrm_state_reset_tables(lrm_state, TRUE);
+            pcmk__set_result(&(cmd->result), PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
+            lrm_remote_set_flags(lrm_state, remote_active);
+        }
+
+        crm_debug("Remote connection event matched %s action", cmd->action);
+        report_remote_ra_result(cmd);
+        cmd_handled = TRUE;
+
+    } else if (op->type == lrmd_event_poke && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
+
+        if (cmd->monitor_timeout_id) {
+            g_source_remove(cmd->monitor_timeout_id);
+            cmd->monitor_timeout_id = 0;
+        }
+
+        /* Only report success the first time, after that only worry about failures.
+         * For this function, if we get the poke pack, it is always a success. Pokes
+         * only fail if the send fails, or the response times out. */
+        if (!pcmk_is_set(cmd->status, cmd_reported_success)) {
+            pcmk__set_result(&(cmd->result), PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
+            report_remote_ra_result(cmd);
+            cmd_set_flags(cmd, cmd_reported_success);
+        }
+
+        crm_debug("Remote poke event matched %s action", cmd->action);
+
+        /* success, keep rescheduling if interval is present. */
+        if (cmd->interval_ms && !pcmk_is_set(cmd->status, cmd_cancel)) {
+            ra_data->recurring_cmds = g_list_append(ra_data->recurring_cmds, cmd);
+            cmd->interval_id = g_timeout_add(cmd->interval_ms,
+                                             recurring_helper, cmd);
+            cmd = NULL;         /* prevent free */
+        }
+        cmd_handled = TRUE;
+
+    } else if (op->type == lrmd_event_disconnect && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
+        if (pcmk_is_set(ra_data->status, remote_active) &&
+            !pcmk_is_set(cmd->status, cmd_cancel)) {
+            pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
+                             PCMK_EXEC_ERROR,
+                             "Remote connection unexpectedly dropped "
+                             "during monitor");
+            report_remote_ra_result(cmd);
+            crm_err("Remote connection to %s unexpectedly dropped during monitor",
+                    lrm_state->node_name);
+        }
+        cmd_handled = TRUE;
+
+    } else if (op->type == lrmd_event_new_client && pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
+
+        handle_remote_ra_stop(lrm_state, cmd);
+        cmd_handled = TRUE;
+
+    } else {
+        crm_debug("Event did not match %s action", ra_data->cur_cmd->action);
+    }
+
+    if (cmd_handled) {
+        ra_data->cur_cmd = NULL;
+        if (ra_data->cmds) {
+            mainloop_set_trigger(ra_data->work);
+        }
+        free_cmd(cmd);
+    }
+}
+
+static void
+handle_remote_ra_stop(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd)
+{
+    remote_ra_data_t *ra_data = NULL;
+
+    CRM_ASSERT(lrm_state);
+    ra_data = lrm_state->remote_ra_data;
+
+    if (!pcmk_is_set(ra_data->status, takeover_complete)) {
+        /* delete pending ops when ever the remote connection is intentionally stopped */
+        g_hash_table_remove_all(lrm_state->active_ops);
+    } else {
+        /* we no longer hold the history if this connection has been migrated,
+         * however, we keep metadata cache for future use */
+        lrm_state_reset_tables(lrm_state, FALSE);
+    }
+
+    lrm_remote_clear_flags(lrm_state, remote_active);
+    lrm_state_disconnect(lrm_state);
+
+    if (ra_data->cmds) {
+        g_list_free_full(ra_data->cmds, free_cmd);
+    }
+    if (ra_data->recurring_cmds) {
+        g_list_free_full(ra_data->recurring_cmds, free_cmd);
+    }
+    ra_data->cmds = NULL;
+    ra_data->recurring_cmds = NULL;
+    ra_data->cur_cmd = NULL;
+
+    if (cmd) {
+        pcmk__set_result(&(cmd->result), PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
+        report_remote_ra_result(cmd);
+    }
+}
+
+// \return Standard Pacemaker return code
+static int
+handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms)
+{
+    const char *server = NULL;
+    lrmd_key_value_t *tmp = NULL;
+    int port = 0;
+    int timeout_used = timeout_ms > MAX_START_TIMEOUT_MS ? MAX_START_TIMEOUT_MS : timeout_ms;
+    int rc = pcmk_rc_ok;
+
+    for (tmp = cmd->params; tmp; tmp = tmp->next) {
+        if (pcmk__strcase_any_of(tmp->key, XML_RSC_ATTR_REMOTE_RA_ADDR,
+                                 XML_RSC_ATTR_REMOTE_RA_SERVER, NULL)) {
+            server = tmp->value;
+        } else if (pcmk__str_eq(tmp->key, XML_RSC_ATTR_REMOTE_RA_PORT, pcmk__str_casei)) {
+            port = atoi(tmp->value);
+        } else if (pcmk__str_eq(tmp->key, CRM_META "_" XML_RSC_ATTR_CONTAINER, pcmk__str_casei)) {
+            lrm_remote_set_flags(lrm_state, controlling_guest);
+        }
+    }
+
+    rc = controld_connect_remote_executor(lrm_state, server, port,
+                                          timeout_used);
+    if (rc != pcmk_rc_ok) {
+        pcmk__format_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
+                            PCMK_EXEC_ERROR,
+                            "Could not connect to Pacemaker Remote node %s: %s",
+                            lrm_state->node_name, pcmk_rc_str(rc));
+    }
+    return rc;
+}
+
+static gboolean
+handle_remote_ra_exec(gpointer user_data)
+{
+    int rc = 0;
+    lrm_state_t *lrm_state = user_data;
+    remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
+    remote_ra_cmd_t *cmd;
+    GList *first = NULL;
+
+    if (ra_data->cur_cmd) {
+        /* still waiting on previous cmd */
+        return TRUE;
+    }
+
+    while (ra_data->cmds) {
+        first = ra_data->cmds;
+        cmd = first->data;
+        if (cmd->delay_id) {
+            /* still waiting for start delay timer to trip */
+            return TRUE;
+        }
+
+        ra_data->cmds = g_list_remove_link(ra_data->cmds, first);
+        g_list_free_1(first);
+
+        if (!strcmp(cmd->action, "start") || !strcmp(cmd->action, "migrate_from")) {
+            lrm_remote_clear_flags(lrm_state, expect_takeover | takeover_complete);
+            if (handle_remote_ra_start(lrm_state, cmd,
+                                       cmd->timeout) == pcmk_rc_ok) {
+                /* take care of this later when we get async connection result */
+                crm_debug("Initiated async remote connection, %s action will complete after connect event",
+                          cmd->action);
+                ra_data->cur_cmd = cmd;
+                return TRUE;
+            }
+            report_remote_ra_result(cmd);
+
+        } else if (!strcmp(cmd->action, "monitor")) {
+
+            if (lrm_state_is_connected(lrm_state) == TRUE) {
+                rc = lrm_state_poke_connection(lrm_state);
+                if (rc < 0) {
+                    pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
+                                     PCMK_EXEC_ERROR, pcmk_strerror(rc));
+                }
+            } else {
+                rc = -1;
+                pcmk__set_result(&(cmd->result), PCMK_OCF_NOT_RUNNING,
+                                 PCMK_EXEC_DONE, "Remote connection inactive");
+            }
+
+            if (rc == 0) {
+                crm_debug("Poked Pacemaker Remote at node %s, waiting for async response",
+                          cmd->rsc_id);
+                ra_data->cur_cmd = cmd;
+                cmd->monitor_timeout_id = g_timeout_add(cmd->timeout, monitor_timeout_cb, cmd);
+                return TRUE;
+            }
+            report_remote_ra_result(cmd);
+
+        } else if (!strcmp(cmd->action, "stop")) {
+
+            if (pcmk_is_set(ra_data->status, expect_takeover)) {
+                /* briefly wait on stop for the takeover event to occur. If the
+                 * takeover event does not occur during the wait period, that's fine.
+                 * It just means that the remote-node's lrm_status section is going to get
+                 * cleared which will require all the resources running in the remote-node
+                 * to be explicitly re-detected via probe actions.  If the takeover does occur
+                 * successfully, then we can leave the status section intact. */
+                cmd->takeover_timeout_id = g_timeout_add((cmd->timeout/2), connection_takeover_timeout_cb, cmd);
+                ra_data->cur_cmd = cmd;
+                return TRUE;
+            }
+
+            handle_remote_ra_stop(lrm_state, cmd);
+
+        } else if (!strcmp(cmd->action, "migrate_to")) {
+            lrm_remote_clear_flags(lrm_state, takeover_complete);
+            lrm_remote_set_flags(lrm_state, expect_takeover);
+            pcmk__set_result(&(cmd->result), PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
+            report_remote_ra_result(cmd);
+        } else if (pcmk__str_any_of(cmd->action, CRMD_ACTION_RELOAD,
+                                    CRMD_ACTION_RELOAD_AGENT, NULL))  {
+            /* Currently the only reloadable parameter is reconnect_interval,
+             * which is only used by the scheduler via the CIB, so reloads are a
+             * no-op.
+             *
+             * @COMPAT DC <2.1.0: We only need to check for "reload" in case
+             * we're in a rolling upgrade with a DC scheduling "reload" instead
+             * of "reload-agent". An OCF 1.1 "reload" would be a no-op anyway,
+             * so this would work for that purpose as well.
+             */
+            pcmk__set_result(&(cmd->result), PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
+            report_remote_ra_result(cmd);
+        }
+
+        free_cmd(cmd);
+    }
+
+    return TRUE;
+}
+
+static void
+remote_ra_data_init(lrm_state_t * lrm_state)
+{
+    remote_ra_data_t *ra_data = NULL;
+
+    if (lrm_state->remote_ra_data) {
+        return;
+    }
+
+    ra_data = calloc(1, sizeof(remote_ra_data_t));
+    ra_data->work = mainloop_add_trigger(G_PRIORITY_HIGH, handle_remote_ra_exec, lrm_state);
+    lrm_state->remote_ra_data = ra_data;
+}
+
+void
+remote_ra_cleanup(lrm_state_t * lrm_state)
+{
+    remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
+
+    if (!ra_data) {
+        return;
+    }
+
+    if (ra_data->cmds) {
+        g_list_free_full(ra_data->cmds, free_cmd);
+    }
+
+    if (ra_data->recurring_cmds) {
+        g_list_free_full(ra_data->recurring_cmds, free_cmd);
+    }
+    mainloop_destroy_trigger(ra_data->work);
+    free(ra_data);
+    lrm_state->remote_ra_data = NULL;
+}
+
+gboolean
+is_remote_lrmd_ra(const char *agent, const char *provider, const char *id)
+{
+    if (agent && provider && !strcmp(agent, REMOTE_LRMD_RA) && !strcmp(provider, "pacemaker")) {
+        return TRUE;
+    }
+    if ((id != NULL) && (lrm_state_find(id) != NULL)
+        && !pcmk__str_eq(id, controld_globals.our_nodename, pcmk__str_casei)) {
+        return TRUE;
+    }
+
+    return FALSE;
+}
+
+lrmd_rsc_info_t *
+remote_ra_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id)
+{
+    lrmd_rsc_info_t *info = NULL;
+
+    if ((lrm_state_find(rsc_id))) {
+        info = calloc(1, sizeof(lrmd_rsc_info_t));
+
+        info->id = strdup(rsc_id);
+        info->type = strdup(REMOTE_LRMD_RA);
+        info->standard = strdup(PCMK_RESOURCE_CLASS_OCF);
+        info->provider = strdup("pacemaker");
+    }
+
+    return info;
+}
+
+static gboolean
+is_remote_ra_supported_action(const char *action)
+{
+    return pcmk__str_any_of(action,
+                            CRMD_ACTION_START,
+                            CRMD_ACTION_STOP,
+                            CRMD_ACTION_STATUS,
+                            CRMD_ACTION_MIGRATE,
+                            CRMD_ACTION_MIGRATED,
+                            CRMD_ACTION_RELOAD_AGENT,
+                            CRMD_ACTION_RELOAD,
+                            NULL);
+}
+
+static GList *
+fail_all_monitor_cmds(GList * list)
+{
+    GList *rm_list = NULL;
+    remote_ra_cmd_t *cmd = NULL;
+    GList *gIter = NULL;
+
+    for (gIter = list; gIter != NULL; gIter = gIter->next) {
+        cmd = gIter->data;
+        if ((cmd->interval_ms > 0) && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
+            rm_list = g_list_append(rm_list, cmd);
+        }
+    }
+
+    for (gIter = rm_list; gIter != NULL; gIter = gIter->next) {
+        cmd = gIter->data;
+
+        pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
+                         PCMK_EXEC_ERROR, "Lost connection to remote executor");
+        crm_trace("Pre-emptively failing %s %s (interval=%u, %s)",
+                  cmd->action, cmd->rsc_id, cmd->interval_ms, cmd->userdata);
+        report_remote_ra_result(cmd);
+
+        list = g_list_remove(list, cmd);
+        free_cmd(cmd);
+    }
+
+    /* frees only the list data, not the cmds */
+    g_list_free(rm_list);
+    return list;
+}
+
+static GList *
+remove_cmd(GList * list, const char *action, guint interval_ms)
+{
+    remote_ra_cmd_t *cmd = NULL;
+    GList *gIter = NULL;
+
+    for (gIter = list; gIter != NULL; gIter = gIter->next) {
+        cmd = gIter->data;
+        if ((cmd->interval_ms == interval_ms)
+            && pcmk__str_eq(cmd->action, action, pcmk__str_casei)) {
+            break;
+        }
+        cmd = NULL;
+    }
+    if (cmd) {
+        list = g_list_remove(list, cmd);
+        free_cmd(cmd);
+    }
+    return list;
+}
+
+int
+remote_ra_cancel(lrm_state_t *lrm_state, const char *rsc_id,
+                 const char *action, guint interval_ms)
+{
+    lrm_state_t *connection_rsc = NULL;
+    remote_ra_data_t *ra_data = NULL;
+
+    connection_rsc = lrm_state_find(rsc_id);
+    if (!connection_rsc || !connection_rsc->remote_ra_data) {
+        return -EINVAL;
+    }
+
+    ra_data = connection_rsc->remote_ra_data;
+    ra_data->cmds = remove_cmd(ra_data->cmds, action, interval_ms);
+    ra_data->recurring_cmds = remove_cmd(ra_data->recurring_cmds, action,
+                                         interval_ms);
+    if (ra_data->cur_cmd &&
+        (ra_data->cur_cmd->interval_ms == interval_ms) &&
+        (pcmk__str_eq(ra_data->cur_cmd->action, action, pcmk__str_casei))) {
+
+        cmd_set_flags(ra_data->cur_cmd, cmd_cancel);
+    }
+
+    return 0;
+}
+
+static remote_ra_cmd_t *
+handle_dup_monitor(remote_ra_data_t *ra_data, guint interval_ms,
+                   const char *userdata)
+{
+    GList *gIter = NULL;
+    remote_ra_cmd_t *cmd = NULL;
+
+    /* there are 3 places a potential duplicate monitor operation
+     * could exist.
+     * 1. recurring_cmds list. where the op is waiting for its next interval
+     * 2. cmds list, where the op is queued to get executed immediately
+     * 3. cur_cmd, which means the monitor op is in flight right now.
+     */
+    if (interval_ms == 0) {
+        return NULL;
+    }
+
+    if (ra_data->cur_cmd &&
+        !pcmk_is_set(ra_data->cur_cmd->status, cmd_cancel) &&
+        (ra_data->cur_cmd->interval_ms == interval_ms) &&
+        pcmk__str_eq(ra_data->cur_cmd->action, "monitor", pcmk__str_casei)) {
+
+        cmd = ra_data->cur_cmd;
+        goto handle_dup;
+    }
+
+    for (gIter = ra_data->recurring_cmds; gIter != NULL; gIter = gIter->next) {
+        cmd = gIter->data;
+        if ((cmd->interval_ms == interval_ms)
+            && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
+            goto handle_dup;
+        }
+    }
+
+    for (gIter = ra_data->cmds; gIter != NULL; gIter = gIter->next) {
+        cmd = gIter->data;
+        if ((cmd->interval_ms == interval_ms)
+            && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
+            goto handle_dup;
+        }
+    }
+
+    return NULL;
+
+handle_dup:
+
+    crm_trace("merging duplicate monitor cmd " PCMK__OP_FMT,
+              cmd->rsc_id, "monitor", interval_ms);
+
+    /* update the userdata */
+    if (userdata) {
+       free(cmd->userdata);
+       cmd->userdata = strdup(userdata);
+    }
+
+    /* if we've already reported success, generate a new call id */
+    if (pcmk_is_set(cmd->status, cmd_reported_success)) {
+        cmd->start_time = time(NULL);
+        cmd->call_id = generate_callid();
+        cmd_clear_flags(cmd, cmd_reported_success);
+    }
+
+    /* if we have an interval_id set, that means we are in the process of
+     * waiting for this cmd's next interval. instead of waiting, cancel
+     * the timer and execute the action immediately */
+    if (cmd->interval_id) {
+        g_source_remove(cmd->interval_id);
+        cmd->interval_id = 0;
+        recurring_helper(cmd);
+    }
+
+    return cmd;
+}
+
+/*!
+ * \internal
+ * \brief Execute an action using the (internal) ocf:pacemaker:remote agent
+ *
+ * \param[in]     lrm_state      Executor state object for remote connection
+ * \param[in]     rsc_id         Connection resource ID
+ * \param[in]     action         Action to execute
+ * \param[in]     userdata       String to copy and pass to execution callback
+ * \param[in]     interval_ms    Action interval (in milliseconds)
+ * \param[in]     timeout_ms     Action timeout (in milliseconds)
+ * \param[in]     start_delay_ms Delay (in milliseconds) before executing action
+ * \param[in,out] params         Connection resource parameters
+ * \param[out]    call_id        Where to store call ID on success
+ *
+ * \return Standard Pacemaker return code
+ * \note This takes ownership of \p params, which should not be used or freed
+ *       after calling this function.
+ */
+int
+controld_execute_remote_agent(const lrm_state_t *lrm_state, const char *rsc_id,
+                              const char *action, const char *userdata,
+                              guint interval_ms, int timeout_ms,
+                              int start_delay_ms, lrmd_key_value_t *params,
+                              int *call_id)
+{
+    lrm_state_t *connection_rsc = NULL;
+    remote_ra_cmd_t *cmd = NULL;
+    remote_ra_data_t *ra_data = NULL;
+
+    *call_id = 0;
+
+    CRM_CHECK((lrm_state != NULL) && (rsc_id != NULL) && (action != NULL)
+              && (userdata != NULL) && (call_id != NULL),
+              lrmd_key_value_freeall(params); return EINVAL);
+
+    if (!is_remote_ra_supported_action(action)) {
+        lrmd_key_value_freeall(params);
+        return EOPNOTSUPP;
+    }
+
+    connection_rsc = lrm_state_find(rsc_id);
+    if (connection_rsc == NULL) {
+        lrmd_key_value_freeall(params);
+        return ENOTCONN;
+    }
+
+    remote_ra_data_init(connection_rsc);
+    ra_data = connection_rsc->remote_ra_data;
+
+    cmd = handle_dup_monitor(ra_data, interval_ms, userdata);
+    if (cmd) {
+        *call_id = cmd->call_id;
+        lrmd_key_value_freeall(params);
+        return pcmk_rc_ok;
+    }
+
+    cmd = calloc(1, sizeof(remote_ra_cmd_t));
+    if (cmd == NULL) {
+        lrmd_key_value_freeall(params);
+        return ENOMEM;
+    }
+
+    cmd->owner = strdup(lrm_state->node_name);
+    cmd->rsc_id = strdup(rsc_id);
+    cmd->action = strdup(action);
+    cmd->userdata = strdup(userdata);
+    if ((cmd->owner == NULL) || (cmd->rsc_id == NULL) || (cmd->action == NULL)
+        || (cmd->userdata == NULL)) {
+        free_cmd(cmd);
+        lrmd_key_value_freeall(params);
+        return ENOMEM;
+    }
+
+    cmd->interval_ms = interval_ms;
+    cmd->timeout = timeout_ms;
+    cmd->start_delay = start_delay_ms;
+    cmd->params = params;
+    cmd->start_time = time(NULL);
+
+    cmd->call_id = generate_callid();
+
+    if (cmd->start_delay) {
+        cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
+    }
+
+    ra_data->cmds = g_list_append(ra_data->cmds, cmd);
+    mainloop_set_trigger(ra_data->work);
+
+    *call_id = cmd->call_id;
+    return pcmk_rc_ok;
+}
+
+/*!
+ * \internal
+ * \brief Immediately fail all monitors of a remote node, if proxied here
+ *
+ * \param[in] node_name  Name of pacemaker_remote node
+ */
+void
+remote_ra_fail(const char *node_name)
+{
+    lrm_state_t *lrm_state = lrm_state_find(node_name);
+
+    if (lrm_state && lrm_state_is_connected(lrm_state)) {
+        remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
+
+        crm_info("Failing monitors on Pacemaker Remote node %s", node_name);
+        ra_data->recurring_cmds = fail_all_monitor_cmds(ra_data->recurring_cmds);
+        ra_data->cmds = fail_all_monitor_cmds(ra_data->cmds);
+    }
+}
+
+/* A guest node fencing implied by host fencing looks like:
+ *
+ *  <pseudo_event id="103" operation="stonith" operation_key="stonith-lxc1-off"
+ *                on_node="lxc1" on_node_uuid="lxc1">
+ *     <attributes CRM_meta_on_node="lxc1" CRM_meta_on_node_uuid="lxc1"
+ *                 CRM_meta_stonith_action="off" crm_feature_set="3.0.12"/>
+ *     <downed>
+ *       <node id="lxc1"/>
+ *     </downed>
+ *  </pseudo_event>
+ */
+#define XPATH_PSEUDO_FENCE "/" XML_GRAPH_TAG_PSEUDO_EVENT \
+    "[@" XML_LRM_ATTR_TASK "='stonith']/" XML_GRAPH_TAG_DOWNED \
+    "/" XML_CIB_TAG_NODE
+
+/*!
+ * \internal
+ * \brief Check a pseudo-action for Pacemaker Remote node side effects
+ *
+ * \param[in,out] xml  XML of pseudo-action to check
+ */
+void
+remote_ra_process_pseudo(xmlNode *xml)
+{
+    xmlXPathObjectPtr search = xpath_search(xml, XPATH_PSEUDO_FENCE);
+
+    if (numXpathResults(search) == 1) {
+        xmlNode *result = getXpathResult(search, 0);
+
+        /* Normally, we handle the necessary side effects of a guest node stop
+         * action when reporting the remote agent's result. However, if the stop
+         * is implied due to fencing, it will be a fencing pseudo-event, and
+         * there won't be a result to report. Handle that case here.
+         *
+         * This will result in a duplicate call to remote_node_down() if the
+         * guest stop was real instead of implied, but that shouldn't hurt.
+         *
+         * There is still one corner case that isn't handled: if a guest node
+         * isn't running any resources when its host is fenced, it will appear
+         * to be cleanly stopped, so there will be no pseudo-fence, and our
+         * peer cache state will be incorrect unless and until the guest is
+         * recovered.
+         */
+        if (result) {
+            const char *remote = ID(result);
+
+            if (remote) {
+                remote_node_down(remote, DOWN_ERASE_LRM);
+            }
+        }
+    }
+    freeXpathObject(search);
+}
+
+static void
+remote_ra_maintenance(lrm_state_t * lrm_state, gboolean maintenance)
+{
+    xmlNode *update, *state;
+    int call_opt;
+    crm_node_t *node;
+
+    call_opt = crmd_cib_smart_opt();
+    node = crm_remote_peer_get(lrm_state->node_name);
+    CRM_CHECK(node != NULL, return);
+    update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
+    state = create_node_state_update(node, node_update_none, update,
+                                     __func__);
+    crm_xml_add(state, XML_NODE_IS_MAINTENANCE, maintenance?"1":"0");
+    if (controld_update_cib(XML_CIB_TAG_STATUS, update, call_opt,
+                            NULL) == pcmk_rc_ok) {
+        /* TODO: still not 100% sure that async update will succeed ... */
+        if (maintenance) {
+            lrm_remote_set_flags(lrm_state, remote_in_maint);
+        } else {
+            lrm_remote_clear_flags(lrm_state, remote_in_maint);
+        }
+    }
+    free_xml(update);
+}
+
+#define XPATH_PSEUDO_MAINTENANCE "//" XML_GRAPH_TAG_PSEUDO_EVENT \
+    "[@" XML_LRM_ATTR_TASK "='" CRM_OP_MAINTENANCE_NODES "']/" \
+    XML_GRAPH_TAG_MAINTENANCE
+
+/*!
+ * \internal
+ * \brief Check a pseudo-action holding updates for maintenance state
+ *
+ * \param[in,out] xml  XML of pseudo-action to check
+ */
+void
+remote_ra_process_maintenance_nodes(xmlNode *xml)
+{
+    xmlXPathObjectPtr search = xpath_search(xml, XPATH_PSEUDO_MAINTENANCE);
+
+    if (numXpathResults(search) == 1) {
+        xmlNode *node;
+        int cnt = 0, cnt_remote = 0;
+
+        for (node =
+                first_named_child(getXpathResult(search, 0), XML_CIB_TAG_NODE);
+            node != NULL; node = pcmk__xml_next(node)) {
+            lrm_state_t *lrm_state = lrm_state_find(ID(node));
+
+            cnt++;
+            if (lrm_state && lrm_state->remote_ra_data &&
+                pcmk_is_set(((remote_ra_data_t *) lrm_state->remote_ra_data)->status, remote_active)) {
+                int is_maint;
+
+                cnt_remote++;
+                pcmk__scan_min_int(crm_element_value(node, XML_NODE_IS_MAINTENANCE),
+                                   &is_maint, 0);
+                remote_ra_maintenance(lrm_state, is_maint);
+            }
+        }
+        crm_trace("Action holds %d nodes (%d remotes found) "
+                    "adjusting maintenance-mode", cnt, cnt_remote);
+    }
+    freeXpathObject(search);
+}
+
+gboolean
+remote_ra_is_in_maintenance(lrm_state_t * lrm_state)
+{
+    remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
+    return pcmk_is_set(ra_data->status, remote_in_maint);
+}
+
+gboolean
+remote_ra_controlling_guest(lrm_state_t * lrm_state)
+{
+    remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
+    return pcmk_is_set(ra_data->status, controlling_guest);
+}
diff --git a/daemons/controld/controld_schedulerd.c b/daemons/controld/controld_schedulerd.c
new file mode 100644
index 0000000..912f9a5
--- /dev/null
+++ b/daemons/controld/controld_schedulerd.c
@@ -0,0 +1,506 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <unistd.h>  /* pid_t, sleep, ssize_t */
+
+#include <crm/cib.h>
+#include <crm/cluster.h>
+#include <crm/common/xml.h>
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml_internal.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_schedulerd.h>
+
+#include <pacemaker-controld.h>
+
+static void handle_disconnect(void);
+
+static pcmk_ipc_api_t *schedulerd_api = NULL;
+
+/*!
+ * \internal
+ * \brief Close any scheduler connection and free associated memory
+ */
+void
+controld_shutdown_schedulerd_ipc(void)
+{
+    controld_clear_fsa_input_flags(R_PE_REQUIRED);
+    pcmk_disconnect_ipc(schedulerd_api);
+    handle_disconnect();
+
+    pcmk_free_ipc_api(schedulerd_api);
+    schedulerd_api = NULL;
+}
+
+/*!
+ * \internal
+ * \brief Save CIB query result to file, raising FSA error
+ *
+ * \param[in]     msg        Ignored
+ * \param[in]     call_id    Call ID of CIB query
+ * \param[in]     rc         Return code of CIB query
+ * \param[in,out] output     Result of CIB query
+ * \param[in]     user_data  Unique identifier for filename
+ *
+ * \note This is intended to be called after a scheduler connection fails.
+ */
+static void
+save_cib_contents(xmlNode *msg, int call_id, int rc, xmlNode *output,
+                  void *user_data)
+{
+    const char *id = user_data;
+
+    register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);
+    CRM_CHECK(id != NULL, return);
+
+    if (rc == pcmk_ok) {
+        char *filename = crm_strdup_printf(PE_STATE_DIR "/pe-core-%s.bz2", id);
+
+        if (write_xml_file(output, filename, TRUE) < 0) {
+            crm_err("Could not save Cluster Information Base to %s after scheduler crash",
+                    filename);
+        } else {
+            crm_notice("Saved Cluster Information Base to %s after scheduler crash",
+                       filename);
+        }
+        free(filename);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Respond to scheduler connection failure
+ */
+static void
+handle_disconnect(void)
+{
+    // If we aren't connected to the scheduler, we can't expect a reply
+    controld_expect_sched_reply(NULL);
+
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_PE_REQUIRED)) {
+        int rc = pcmk_ok;
+        char *uuid_str = crm_generate_uuid();
+
+        crm_crit("Connection to the scheduler failed "
+                 CRM_XS " uuid=%s", uuid_str);
+
+        /*
+         * The scheduler died...
+         *
+         * Save the current CIB so that we have a chance of
+         * figuring out what killed it.
+         *
+         * Delay raising the I_ERROR until the query below completes or
+         * 5s is up, whichever comes first.
+         *
+         */
+        rc = controld_globals.cib_conn->cmds->query(controld_globals.cib_conn,
+                                                    NULL, NULL,
+                                                    cib_scope_local);
+        fsa_register_cib_callback(rc, uuid_str, save_cib_contents);
+
+    } else {
+        crm_info("Connection to the scheduler released");
+    }
+
+    controld_clear_fsa_input_flags(R_PE_CONNECTED);
+    controld_trigger_fsa();
+    return;
+}
+
+static void
+handle_reply(pcmk_schedulerd_api_reply_t *reply)
+{
+    const char *msg_ref = NULL;
+
+    if (!AM_I_DC) {
+        return;
+    }
+
+    msg_ref = reply->data.graph.reference;
+
+    if (msg_ref == NULL) {
+        crm_err("%s - Ignoring calculation with no reference", CRM_OP_PECALC);
+
+    } else if (pcmk__str_eq(msg_ref, controld_globals.fsa_pe_ref,
+                            pcmk__str_none)) {
+        ha_msg_input_t fsa_input;
+        xmlNode *crm_data_node;
+
+        controld_stop_sched_timer();
+
+        /* do_te_invoke (which will eventually process the fsa_input we are constructing
+         * here) requires that fsa_input.xml be non-NULL.  That will only happen if
+         * copy_ha_msg_input (which is called by register_fsa_input_adv) sees the
+         * fsa_input.msg that it is expecting. The scheduler's IPC dispatch function
+         * gave us the values we need, we just need to put them into XML.
+         *
+         * The name of the top level element here is irrelevant.  Nothing checks it.
+         */
+        fsa_input.msg = create_xml_node(NULL, "dummy-reply");
+        crm_xml_add(fsa_input.msg, XML_ATTR_REFERENCE, msg_ref);
+        crm_xml_add(fsa_input.msg, F_CRM_TGRAPH_INPUT, reply->data.graph.input);
+
+        crm_data_node = create_xml_node(fsa_input.msg, F_CRM_DATA);
+        add_node_copy(crm_data_node, reply->data.graph.tgraph);
+        register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input);
+
+        free_xml(fsa_input.msg);
+
+    } else {
+        crm_info("%s calculation %s is obsolete", CRM_OP_PECALC, msg_ref);
+    }
+}
+
+static void
+scheduler_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
+                         crm_exit_t status, void *event_data, void *user_data)
+{
+    pcmk_schedulerd_api_reply_t *reply = event_data;
+
+    switch (event_type) {
+        case pcmk_ipc_event_disconnect:
+            handle_disconnect();
+            break;
+
+        case pcmk_ipc_event_reply:
+            handle_reply(reply);
+            break;
+
+        default:
+            break;
+    }
+}
+
+static bool
+new_schedulerd_ipc_connection(void)
+{
+    int rc;
+
+    controld_set_fsa_input_flags(R_PE_REQUIRED);
+
+    if (schedulerd_api == NULL) {
+        rc = pcmk_new_ipc_api(&schedulerd_api, pcmk_ipc_schedulerd);
+
+        if (rc != pcmk_rc_ok) {
+            crm_err("Error connecting to the scheduler: %s", pcmk_rc_str(rc));
+            return false;
+        }
+    }
+
+    pcmk_register_ipc_callback(schedulerd_api, scheduler_event_callback, NULL);
+
+    rc = pcmk_connect_ipc(schedulerd_api, pcmk_ipc_dispatch_main);
+    if (rc != pcmk_rc_ok) {
+        crm_err("Error connecting to the scheduler: %s", pcmk_rc_str(rc));
+        return false;
+    }
+
+    controld_set_fsa_input_flags(R_PE_CONNECTED);
+    return true;
+}
+
+static void do_pe_invoke_callback(xmlNode *msg, int call_id, int rc,
+                                  xmlNode *output, void *user_data);
+
+/*	 A_PE_START, A_PE_STOP, O_PE_RESTART	*/
+void
+do_pe_control(long long action,
+              enum crmd_fsa_cause cause,
+              enum crmd_fsa_state cur_state,
+              enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    if (pcmk_is_set(action, A_PE_STOP)) {
+        controld_clear_fsa_input_flags(R_PE_REQUIRED);
+        pcmk_disconnect_ipc(schedulerd_api);
+        handle_disconnect();
+    }
+    if (pcmk_is_set(action, A_PE_START)
+        && !pcmk_is_set(controld_globals.fsa_input_register, R_PE_CONNECTED)) {
+
+        if (cur_state == S_STOPPING) {
+            crm_info("Ignoring request to connect to scheduler while shutting down");
+
+        } else if (!new_schedulerd_ipc_connection()) {
+            crm_warn("Could not connect to scheduler");
+            register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
+        }
+    }
+}
+
+static int fsa_pe_query = 0;
+static mainloop_timer_t *controld_sched_timer = NULL;
+
+// @TODO Make this a configurable cluster option if there's demand for it
+#define SCHED_TIMEOUT_MS (120000)
+
+/*!
+ * \internal
+ * \brief Handle a timeout waiting for scheduler reply
+ *
+ * \param[in] user_data  Ignored
+ *
+ * \return FALSE (indicating that timer should not be restarted)
+ */
+static gboolean
+controld_sched_timeout(gpointer user_data)
+{
+    if (AM_I_DC) {
+        /* If this node is the DC but can't communicate with the scheduler, just
+         * exit (and likely get fenced) so this node doesn't interfere with any
+         * further DC elections.
+         *
+         * @TODO We could try something less drastic first, like disconnecting
+         * and reconnecting to the scheduler, but something is likely going
+         * seriously wrong, so perhaps it's better to just fail as quickly as
+         * possible.
+         */
+        crmd_exit(CRM_EX_FATAL);
+    }
+    return FALSE;
+}
+
+void
+controld_stop_sched_timer(void)
+{
+    if ((controld_sched_timer != NULL)
+        && (controld_globals.fsa_pe_ref != NULL)) {
+        crm_trace("Stopping timer for scheduler reply %s",
+                  controld_globals.fsa_pe_ref);
+    }
+    mainloop_timer_stop(controld_sched_timer);
+}
+
+/*!
+ * \internal
+ * \brief Set the scheduler request currently being waited on
+ *
+ * \param[in] ref  Request to expect reply to (or NULL for none)
+ *
+ * \note This function takes ownership of \p ref.
+ */
+void
+controld_expect_sched_reply(char *ref)
+{
+    if (ref) {
+        if (controld_sched_timer == NULL) {
+            controld_sched_timer = mainloop_timer_add("scheduler_reply_timer",
+                                                      SCHED_TIMEOUT_MS, FALSE,
+                                                      controld_sched_timeout,
+                                                      NULL);
+        }
+        mainloop_timer_start(controld_sched_timer);
+    } else {
+        controld_stop_sched_timer();
+    }
+    free(controld_globals.fsa_pe_ref);
+    controld_globals.fsa_pe_ref = ref;
+}
+
+/*!
+ * \internal
+ * \brief Free the scheduler reply timer
+ */
+void
+controld_free_sched_timer(void)
+{
+    if (controld_sched_timer != NULL) {
+        mainloop_timer_del(controld_sched_timer);
+        controld_sched_timer = NULL;
+    }
+}
+
+/*	 A_PE_INVOKE	*/
+void
+do_pe_invoke(long long action,
+             enum crmd_fsa_cause cause,
+             enum crmd_fsa_state cur_state,
+             enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    cib_t *cib_conn = controld_globals.cib_conn;
+
+    if (AM_I_DC == FALSE) {
+        crm_err("Not invoking scheduler because not DC: %s",
+                fsa_action2string(action));
+        return;
+    }
+
+    if (!pcmk_is_set(controld_globals.fsa_input_register, R_PE_CONNECTED)) {
+        if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+            crm_err("Cannot shut down gracefully without the scheduler");
+            register_fsa_input_before(C_FSA_INTERNAL, I_TERMINATE, NULL);
+
+        } else {
+            crm_info("Waiting for the scheduler to connect");
+            crmd_fsa_stall(FALSE);
+            controld_set_fsa_action_flags(A_PE_START);
+            controld_trigger_fsa();
+        }
+        return;
+    }
+
+    if (cur_state != S_POLICY_ENGINE) {
+        crm_notice("Not invoking scheduler because in state %s",
+                   fsa_state2string(cur_state));
+        return;
+    }
+    if (!pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
+        crm_err("Attempted to invoke scheduler without consistent Cluster Information Base!");
+
+        /* start the join from scratch */
+        register_fsa_input_before(C_FSA_INTERNAL, I_ELECTION, NULL);
+        return;
+    }
+
+    fsa_pe_query = cib_conn->cmds->query(cib_conn, NULL, NULL, cib_scope_local);
+
+    crm_debug("Query %d: Requesting the current CIB: %s", fsa_pe_query,
+              fsa_state2string(controld_globals.fsa_state));
+
+    controld_expect_sched_reply(NULL);
+    fsa_register_cib_callback(fsa_pe_query, NULL, do_pe_invoke_callback);
+}
+
+static void
+force_local_option(xmlNode *xml, const char *attr_name, const char *attr_value)
+{
+    int max = 0;
+    int lpc = 0;
+    const char *xpath_base = NULL;
+    char *xpath_string = NULL;
+    xmlXPathObjectPtr xpathObj = NULL;
+
+    xpath_base = pcmk_cib_xpath_for(XML_CIB_TAG_CRMCONFIG);
+    if (xpath_base == NULL) {
+        crm_err(XML_CIB_TAG_CRMCONFIG " CIB element not known (bug?)");
+        return;
+    }
+
+    xpath_string = crm_strdup_printf("%s//%s//nvpair[@name='%s']",
+                                     xpath_base, XML_CIB_TAG_PROPSET,
+                                     attr_name);
+    xpathObj = xpath_search(xml, xpath_string);
+    max = numXpathResults(xpathObj);
+    free(xpath_string);
+
+    for (lpc = 0; lpc < max; lpc++) {
+        xmlNode *match = getXpathResult(xpathObj, lpc);
+        crm_trace("Forcing %s/%s = %s", ID(match), attr_name, attr_value);
+        crm_xml_add(match, XML_NVPAIR_ATTR_VALUE, attr_value);
+    }
+
+    if(max == 0) {
+        xmlNode *configuration = NULL;
+        xmlNode *crm_config = NULL;
+        xmlNode *cluster_property_set = NULL;
+
+        crm_trace("Creating %s-%s for %s=%s",
+                  CIB_OPTIONS_FIRST, attr_name, attr_name, attr_value);
+
+        configuration = pcmk__xe_match(xml, XML_CIB_TAG_CONFIGURATION, NULL,
+                                       NULL);
+        if (configuration == NULL) {
+            configuration = create_xml_node(xml, XML_CIB_TAG_CONFIGURATION);
+        }
+
+        crm_config = pcmk__xe_match(configuration, XML_CIB_TAG_CRMCONFIG, NULL,
+                                    NULL);
+        if (crm_config == NULL) {
+            crm_config = create_xml_node(configuration, XML_CIB_TAG_CRMCONFIG);
+        }
+
+        cluster_property_set = pcmk__xe_match(crm_config, XML_CIB_TAG_PROPSET,
+                                              NULL, NULL);
+        if (cluster_property_set == NULL) {
+            cluster_property_set = create_xml_node(crm_config, XML_CIB_TAG_PROPSET);
+            crm_xml_add(cluster_property_set, XML_ATTR_ID, CIB_OPTIONS_FIRST);
+        }
+
+        xml = create_xml_node(cluster_property_set, XML_CIB_TAG_NVPAIR);
+
+        crm_xml_set_id(xml, "%s-%s", CIB_OPTIONS_FIRST, attr_name);
+        crm_xml_add(xml, XML_NVPAIR_ATTR_NAME, attr_name);
+        crm_xml_add(xml, XML_NVPAIR_ATTR_VALUE, attr_value);
+    }
+    freeXpathObject(xpathObj);
+}
+
+static void
+do_pe_invoke_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    char *ref = NULL;
+    pid_t watchdog = pcmk__locate_sbd();
+
+    if (rc != pcmk_ok) {
+        crm_err("Could not retrieve the Cluster Information Base: %s "
+                CRM_XS " rc=%d call=%d", pcmk_strerror(rc), rc, call_id);
+        register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);
+        return;
+
+    } else if (call_id != fsa_pe_query) {
+        crm_trace("Skipping superseded CIB query: %d (current=%d)", call_id, fsa_pe_query);
+        return;
+
+    } else if (!AM_I_DC
+               || !pcmk_is_set(controld_globals.fsa_input_register,
+                               R_PE_CONNECTED)) {
+        crm_debug("No need to invoke the scheduler anymore");
+        return;
+
+    } else if (controld_globals.fsa_state != S_POLICY_ENGINE) {
+        crm_debug("Discarding scheduler request in state: %s",
+                  fsa_state2string(controld_globals.fsa_state));
+        return;
+
+    /* this callback counts as 1 */
+    } else if (num_cib_op_callbacks() > 1) {
+        crm_debug("Re-asking for the CIB: %d other peer updates still pending",
+                  (num_cib_op_callbacks() - 1));
+        sleep(1);
+        controld_set_fsa_action_flags(A_PE_INVOKE);
+        controld_trigger_fsa();
+        return;
+    }
+
+    CRM_LOG_ASSERT(output != NULL);
+
+    /* Refresh the remote node cache and the known node cache when the
+     * scheduler is invoked */
+    pcmk__refresh_node_caches_from_cib(output);
+
+    crm_xml_add(output, XML_ATTR_DC_UUID, controld_globals.our_uuid);
+    pcmk__xe_set_bool_attr(output, XML_ATTR_HAVE_QUORUM,
+                           pcmk_is_set(controld_globals.flags,
+                                       controld_has_quorum));
+
+    force_local_option(output, XML_ATTR_HAVE_WATCHDOG, pcmk__btoa(watchdog));
+
+    if (pcmk_is_set(controld_globals.flags, controld_ever_had_quorum)
+        && !crm_have_quorum) {
+        crm_xml_add_int(output, XML_ATTR_QUORUM_PANIC, 1);
+    }
+
+    rc = pcmk_rc2legacy(pcmk_schedulerd_api_graph(schedulerd_api, output, &ref));
+
+    if (rc < 0) {
+        crm_err("Could not contact the scheduler: %s " CRM_XS " rc=%d",
+                pcmk_strerror(rc), rc);
+        register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);
+    } else {
+        CRM_ASSERT(ref != NULL);
+        controld_expect_sched_reply(ref);
+        crm_debug("Invoking the scheduler: query=%d, ref=%s, seq=%llu, "
+                  "quorate=%s", fsa_pe_query, controld_globals.fsa_pe_ref,
+                  crm_peer_seq, pcmk__btoa(pcmk_is_set(controld_globals.flags,
+                                                       controld_has_quorum)));
+    }
+}
diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c
new file mode 100644
index 0000000..d8cfcad
--- /dev/null
+++ b/daemons/controld/controld_te_actions.c
@@ -0,0 +1,746 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <crm/crm.h>
+#include <crm/cib.h>
+#include <crm/lrmd.h>               // lrmd_event_data_t, lrmd_free_event()
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/cluster.h>
+
+#include <pacemaker-internal.h>
+#include <pacemaker-controld.h>
+
+static GHashTable *te_targets = NULL;
+void send_rsc_command(pcmk__graph_action_t *action);
+static void te_update_job_count(pcmk__graph_action_t *action, int offset);
+
+static void
+te_start_action_timer(const pcmk__graph_t *graph, pcmk__graph_action_t *action)
+{
+    action->timer = g_timeout_add(action->timeout + graph->network_delay,
+                                  action_timer_callback, (void *) action);
+    CRM_ASSERT(action->timer != 0);
+}
+
+/*!
+ * \internal
+ * \brief Execute a graph pseudo-action
+ *
+ * \param[in,out] graph   Transition graph being executed
+ * \param[in,out] pseudo  Pseudo-action to execute
+ *
+ * \return Standard Pacemaker return code
+ */
+static int
+execute_pseudo_action(pcmk__graph_t *graph, pcmk__graph_action_t *pseudo)
+{
+    const char *task = crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK);
+
+    /* send to peers as well? */
+    if (pcmk__str_eq(task, CRM_OP_MAINTENANCE_NODES, pcmk__str_casei)) {
+        GHashTableIter iter;
+        crm_node_t *node = NULL;
+
+        g_hash_table_iter_init(&iter, crm_peer_cache);
+        while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
+            xmlNode *cmd = NULL;
+
+            if (pcmk__str_eq(controld_globals.our_nodename, node->uname,
+                             pcmk__str_casei)) {
+                continue;
+            }
+
+            cmd = create_request(task, pseudo->xml, node->uname,
+                                 CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
+            send_cluster_message(node, crm_msg_crmd, cmd, FALSE);
+            free_xml(cmd);
+        }
+
+        remote_ra_process_maintenance_nodes(pseudo->xml);
+    } else {
+        /* Check action for Pacemaker Remote node side effects */
+        remote_ra_process_pseudo(pseudo->xml);
+    }
+
+    crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id,
+              crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK_KEY));
+    te_action_confirmed(pseudo, graph);
+    return pcmk_rc_ok;
+}
+
+static int
+get_target_rc(pcmk__graph_action_t *action)
+{
+    int exit_status;
+
+    pcmk__scan_min_int(crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC),
+                       &exit_status, 0);
+    return exit_status;
+}
+
+/*!
+ * \internal
+ * \brief Execute a cluster action from a transition graph
+ *
+ * \param[in,out] graph   Transition graph being executed
+ * \param[in,out] action  Cluster action to execute
+ *
+ * \return Standard Pacemaker return code
+ */
+static int
+execute_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
+{
+    char *counter = NULL;
+    xmlNode *cmd = NULL;
+    gboolean is_local = FALSE;
+
+    const char *id = NULL;
+    const char *task = NULL;
+    const char *value = NULL;
+    const char *on_node = NULL;
+    const char *router_node = NULL;
+
+    gboolean rc = TRUE;
+    gboolean no_wait = FALSE;
+
+    id = ID(action->xml);
+    CRM_CHECK(!pcmk__str_empty(id), return EPROTO);
+
+    task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
+    CRM_CHECK(!pcmk__str_empty(task), return EPROTO);
+
+    on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
+    CRM_CHECK(!pcmk__str_empty(on_node), return pcmk_rc_node_unknown);
+
+    router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
+    if (router_node == NULL) {
+        router_node = on_node;
+        if (pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_none)) {
+            const char *mode = crm_element_value(action->xml, PCMK__XA_MODE);
+
+            if (pcmk__str_eq(mode, XML_TAG_CIB, pcmk__str_none)) {
+                router_node = controld_globals.our_nodename;
+            }
+        }
+    }
+
+    if (pcmk__str_eq(router_node, controld_globals.our_nodename,
+                     pcmk__str_casei)) {
+        is_local = TRUE;
+    }
+
+    value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
+    if (crm_is_true(value)) {
+        no_wait = TRUE;
+    }
+
+    crm_info("Handling controller request '%s' (%s on %s)%s%s",
+             id, task, on_node, (is_local? " locally" : ""),
+             (no_wait? " without waiting" : ""));
+
+    if (is_local && pcmk__str_eq(task, CRM_OP_SHUTDOWN, pcmk__str_none)) {
+        /* defer until everything else completes */
+        crm_info("Controller request '%s' is a local shutdown", id);
+        graph->completion_action = pcmk__graph_shutdown;
+        graph->abort_reason = "local shutdown";
+        te_action_confirmed(action, graph);
+        return pcmk_rc_ok;
+
+    } else if (pcmk__str_eq(task, CRM_OP_SHUTDOWN, pcmk__str_none)) {
+        crm_node_t *peer = crm_get_peer(0, router_node);
+
+        pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN);
+    }
+
+    cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
+
+    counter = pcmk__transition_key(controld_globals.transition_graph->id,
+                                   action->id, get_target_rc(action),
+                                   controld_globals.te_uuid);
+    crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);
+
+    rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_crmd, cmd, TRUE);
+    free(counter);
+    free_xml(cmd);
+
+    if (rc == FALSE) {
+        crm_err("Action %d failed: send", action->id);
+        return ECOMM;
+
+    } else if (no_wait) {
+        te_action_confirmed(action, graph);
+
+    } else {
+        if (action->timeout <= 0) {
+            crm_err("Action %d: %s on %s had an invalid timeout (%dms).  Using %ums instead",
+                    action->id, task, on_node, action->timeout, graph->network_delay);
+            action->timeout = (int) graph->network_delay;
+        }
+        te_start_action_timer(graph, action);
+    }
+
+    return pcmk_rc_ok;
+}
+
+/*!
+ * \internal
+ * \brief Synthesize an executor event for a resource action timeout
+ *
+ * \param[in] action     Resource action that timed out
+ * \param[in] target_rc  Expected result of action that timed out
+ *
+ * Synthesize an executor event for a resource action timeout. (If the executor
+ * gets a timeout while waiting for a resource action to complete, that will be
+ * reported via the usual callback. This timeout means we didn't hear from the
+ * executor itself or the controller that relayed the action to the executor.)
+ *
+ * \return Newly created executor event for result of \p action
+ * \note The caller is responsible for freeing the return value using
+ *       lrmd_free_event().
+ */
+static lrmd_event_data_t *
+synthesize_timeout_event(const pcmk__graph_action_t *action, int target_rc)
+{
+    lrmd_event_data_t *op = NULL;
+    const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
+    const char *reason = NULL;
+    char *dynamic_reason = NULL;
+
+    if (pcmk__str_eq(target, get_local_node_name(), pcmk__str_casei)) {
+        reason = "Local executor did not return result in time";
+    } else {
+        const char *router_node = NULL;
+
+        router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
+        if (router_node == NULL) {
+            router_node = target;
+        }
+        dynamic_reason = crm_strdup_printf("Controller on %s did not return "
+                                           "result in time", router_node);
+        reason = dynamic_reason;
+    }
+
+    op = pcmk__event_from_graph_action(NULL, action, PCMK_EXEC_TIMEOUT,
+                                       PCMK_OCF_UNKNOWN_ERROR, reason);
+    op->call_id = -1;
+    op->user_data = pcmk__transition_key(controld_globals.transition_graph->id,
+                                         action->id, target_rc,
+                                         controld_globals.te_uuid);
+    free(dynamic_reason);
+    return op;
+}
+
+static void
+controld_record_action_event(pcmk__graph_action_t *action,
+                             lrmd_event_data_t *op)
+{
+    cib_t *cib_conn = controld_globals.cib_conn;
+
+    xmlNode *state = NULL;
+    xmlNode *rsc = NULL;
+    xmlNode *action_rsc = NULL;
+
+    int rc = pcmk_ok;
+
+    const char *rsc_id = NULL;
+    const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
+    const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
+    const char *target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
+
+    int target_rc = get_target_rc(action);
+
+    action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE);
+    if (action_rsc == NULL) {
+        return;
+    }
+
+    rsc_id = ID(action_rsc);
+    CRM_CHECK(rsc_id != NULL,
+              crm_log_xml_err(action->xml, "Bad:action"); return);
+
+/*
+  update the CIB
+
+<node_state id="hadev">
+      <lrm>
+        <lrm_resources>
+          <lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
+*/
+
+    state = create_xml_node(NULL, XML_CIB_TAG_STATE);
+
+    crm_xml_add(state, XML_ATTR_ID, target_uuid);
+    crm_xml_add(state, XML_ATTR_UNAME, target);
+
+    rsc = create_xml_node(state, XML_CIB_TAG_LRM);
+    crm_xml_add(rsc, XML_ATTR_ID, target_uuid);
+
+    rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES);
+    rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE);
+    crm_xml_add(rsc, XML_ATTR_ID, rsc_id);
+
+
+    crm_copy_xml_element(action_rsc, rsc, XML_ATTR_TYPE);
+    crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_CLASS);
+    crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_PROVIDER);
+
+    pcmk__create_history_xml(rsc, op, CRM_FEATURE_SET, target_rc, target,
+                             __func__);
+
+    rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_STATUS, state,
+                                cib_scope_local);
+    fsa_register_cib_callback(rc, NULL, cib_action_updated);
+    free_xml(state);
+
+    crm_trace("Sent CIB update (call ID %d) for synthesized event of action %d (%s on %s)",
+              rc, action->id, task_uuid, target);
+    pcmk__set_graph_action_flags(action, pcmk__graph_action_sent_update);
+}
+
+void
+controld_record_action_timeout(pcmk__graph_action_t *action)
+{
+    lrmd_event_data_t *op = NULL;
+
+    const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
+    const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
+
+    int target_rc = get_target_rc(action);
+
+    crm_warn("%s %d: %s on %s timed out",
+             crm_element_name(action->xml), action->id, task_uuid, target);
+
+    op = synthesize_timeout_event(action, target_rc);
+    controld_record_action_event(action, op);
+    lrmd_free_event(op);
+}
+
+/*!
+ * \internal
+ * \brief Execute a resource action from a transition graph
+ *
+ * \param[in,out] graph   Transition graph being executed
+ * \param[in,out] action  Resource action to execute
+ *
+ * \return Standard Pacemaker return code
+ */
+static int
+execute_rsc_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
+{
+    /* never overwrite stop actions in the CIB with
+     *   anything other than completed results
+     *
+     * Writing pending stops makes it look like the
+     *   resource is running again
+     */
+    xmlNode *cmd = NULL;
+    xmlNode *rsc_op = NULL;
+
+    gboolean rc = TRUE;
+    gboolean no_wait = FALSE;
+    gboolean is_local = FALSE;
+
+    char *counter = NULL;
+    const char *task = NULL;
+    const char *value = NULL;
+    const char *on_node = NULL;
+    const char *router_node = NULL;
+    const char *task_uuid = NULL;
+
+    CRM_ASSERT(action != NULL);
+    CRM_ASSERT(action->xml != NULL);
+
+    pcmk__clear_graph_action_flags(action, pcmk__graph_action_executed);
+    on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
+
+    CRM_CHECK(!pcmk__str_empty(on_node),
+              crm_err("Corrupted command(id=%s) %s: no node",
+                      ID(action->xml), pcmk__s(task, "without task"));
+              return pcmk_rc_node_unknown);
+
+    rsc_op = action->xml;
+    task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
+    task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
+    router_node = crm_element_value(rsc_op, XML_LRM_ATTR_ROUTER_NODE);
+
+    if (!router_node) {
+        router_node = on_node;
+    }
+
+    counter = pcmk__transition_key(controld_globals.transition_graph->id,
+                                   action->id, get_target_rc(action),
+                                   controld_globals.te_uuid);
+    crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);
+
+    if (pcmk__str_eq(router_node, controld_globals.our_nodename,
+                     pcmk__str_casei)) {
+        is_local = TRUE;
+    }
+
+    value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
+    if (crm_is_true(value)) {
+        no_wait = TRUE;
+    }
+
+    crm_notice("Initiating %s operation %s%s on %s%s "CRM_XS" action %d",
+               task, task_uuid, (is_local? " locally" : ""), on_node,
+               (no_wait? " without waiting" : ""), action->id);
+
+    cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
+                         CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
+
+    if (is_local) {
+        /* shortcut local resource commands */
+        ha_msg_input_t data = {
+            .msg = cmd,
+            .xml = rsc_op,
+        };
+
+        fsa_data_t msg = {
+            .id = 0,
+            .data = &data,
+            .data_type = fsa_dt_ha_msg,
+            .fsa_input = I_NULL,
+            .fsa_cause = C_FSA_INTERNAL,
+            .actions = A_LRM_INVOKE,
+            .origin = __func__,
+        };
+
+        do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, controld_globals.fsa_state,
+                      I_NULL, &msg);
+
+    } else {
+        rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_lrmd, cmd, TRUE);
+    }
+
+    free(counter);
+    free_xml(cmd);
+
+    pcmk__set_graph_action_flags(action, pcmk__graph_action_executed);
+
+    if (rc == FALSE) {
+        crm_err("Action %d failed: send", action->id);
+        return ECOMM;
+
+    } else if (no_wait) {
+        /* Just mark confirmed. Don't bump the job count only to immediately
+         * decrement it.
+         */
+        crm_info("Action %d confirmed - no wait", action->id);
+        pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
+        pcmk__update_graph(controld_globals.transition_graph, action);
+        trigger_graph();
+
+    } else if (pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
+        crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
+                  action->id, task, task_uuid, on_node, action->timeout);
+    } else {
+        if (action->timeout <= 0) {
+            crm_err("Action %d: %s %s on %s had an invalid timeout (%dms).  Using %ums instead",
+                    action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
+            action->timeout = (int) graph->network_delay;
+        }
+        te_update_job_count(action, 1);
+        te_start_action_timer(graph, action);
+    }
+
+    return pcmk_rc_ok;
+}
+
+struct te_peer_s
+{
+        char *name;
+        int jobs;
+        int migrate_jobs;
+};
+
+static void te_peer_free(gpointer p)
+{
+    struct te_peer_s *peer = p;
+
+    free(peer->name);
+    free(peer);
+}
+
+void te_reset_job_counts(void)
+{
+    GHashTableIter iter;
+    struct te_peer_s *peer = NULL;
+
+    if(te_targets == NULL) {
+        te_targets = pcmk__strkey_table(NULL, te_peer_free);
+    }
+
+    g_hash_table_iter_init(&iter, te_targets);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) {
+        peer->jobs = 0;
+        peer->migrate_jobs = 0;
+    }
+}
+
+static void
+te_update_job_count_on(const char *target, int offset, bool migrate)
+{
+    struct te_peer_s *r = NULL;
+
+    if(target == NULL || te_targets == NULL) {
+        return;
+    }
+
+    r = g_hash_table_lookup(te_targets, target);
+    if(r == NULL) {
+        r = calloc(1, sizeof(struct te_peer_s));
+        r->name = strdup(target);
+        g_hash_table_insert(te_targets, r->name, r);
+    }
+
+    r->jobs += offset;
+    if(migrate) {
+        r->migrate_jobs += offset;
+    }
+    crm_trace("jobs[%s] = %d", target, r->jobs);
+}
+
+static void
+te_update_job_count(pcmk__graph_action_t *action, int offset)
+{
+    const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
+    const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
+
+    if ((action->type != pcmk__rsc_graph_action) || (target == NULL)) {
+        /* No limit on these */
+        return;
+    }
+
+    /* if we have a router node, this means the action is performing
+     * on a remote node. For now, we count all actions occurring on a
+     * remote node against the job list on the cluster node hosting
+     * the connection resources */
+    target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
+
+    if ((target == NULL) && pcmk__strcase_any_of(task, CRMD_ACTION_MIGRATE,
+                                                 CRMD_ACTION_MIGRATED, NULL)) {
+
+        const char *t1 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
+        const char *t2 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);
+
+        te_update_job_count_on(t1, offset, TRUE);
+        te_update_job_count_on(t2, offset, TRUE);
+        return;
+    } else if (target == NULL) {
+        target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
+    }
+
+    te_update_job_count_on(target, offset, FALSE);
+}
+
+/*!
+ * \internal
+ * \brief Check whether a graph action is allowed to be executed on a node
+ *
+ * \param[in] graph   Transition graph being executed
+ * \param[in] action  Graph action being executed
+ * \param[in] target  Name of node where action should be executed
+ *
+ * \return true if action is allowed, otherwise false
+ */
+static bool
+allowed_on_node(const pcmk__graph_t *graph, const pcmk__graph_action_t *action,
+                const char *target)
+{
+    int limit = 0;
+    struct te_peer_s *r = NULL;
+    const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
+    const char *id = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
+
+    if(target == NULL) {
+        /* No limit on these */
+        return true;
+
+    } else if(te_targets == NULL) {
+        return false;
+    }
+
+    r = g_hash_table_lookup(te_targets, target);
+    limit = throttle_get_job_limit(target);
+
+    if(r == NULL) {
+        r = calloc(1, sizeof(struct te_peer_s));
+        r->name = strdup(target);
+        g_hash_table_insert(te_targets, r->name, r);
+    }
+
+    if(limit <= r->jobs) {
+        crm_trace("Peer %s is over their job limit of %d (%d): deferring %s",
+                  target, limit, r->jobs, id);
+        return false;
+
+    } else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) {
+        if (pcmk__strcase_any_of(task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
+            crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s",
+                      target, graph->migration_limit, r->migrate_jobs, id);
+            return false;
+        }
+    }
+
+    crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit);
+
+    return true;
+}
+
+/*!
+ * \internal
+ * \brief Check whether a graph action is allowed to be executed
+ *
+ * \param[in] graph   Transition graph being executed
+ * \param[in] action  Graph action being executed
+ *
+ * \return true if action is allowed, otherwise false
+ */
+static bool
+graph_action_allowed(pcmk__graph_t *graph, pcmk__graph_action_t *action)
+{
+    const char *target = NULL;
+    const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
+
+    if (action->type != pcmk__rsc_graph_action) {
+        /* No limit on these */
+        return true;
+    }
+
+    /* if we have a router node, this means the action is performing
+     * on a remote node. For now, we count all actions occurring on a
+     * remote node against the job list on the cluster node hosting
+     * the connection resources */
+    target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
+
+    if ((target == NULL) && pcmk__strcase_any_of(task, CRMD_ACTION_MIGRATE,
+                                                 CRMD_ACTION_MIGRATED, NULL)) {
+        target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
+        if (!allowed_on_node(graph, action, target)) {
+            return false;
+        }
+
+        target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);
+
+    } else if (target == NULL) {
+        target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
+    }
+
+    return allowed_on_node(graph, action, target);
+}
+
+/*!
+ * \brief Confirm a graph action (and optionally update graph)
+ *
+ * \param[in,out] action  Action to confirm
+ * \param[in,out] graph   Update and trigger this graph (if non-NULL)
+ */
+void
+te_action_confirmed(pcmk__graph_action_t *action, pcmk__graph_t *graph)
+{
+    if (!pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
+        if ((action->type == pcmk__rsc_graph_action)
+            && (crm_element_value(action->xml, XML_LRM_ATTR_TARGET) != NULL)) {
+            te_update_job_count(action, -1);
+        }
+        pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
+    }
+    if (graph) {
+        pcmk__update_graph(graph, action);
+        trigger_graph();
+    }
+}
+
+static pcmk__graph_functions_t te_graph_fns = {
+    execute_pseudo_action,
+    execute_rsc_action,
+    execute_cluster_action,
+    controld_execute_fence_action,
+    graph_action_allowed,
+};
+
+/*
+ * \internal
+ * \brief Register the transitioner's graph functions with \p libpacemaker
+ */
+void
+controld_register_graph_functions(void)
+{
+    pcmk__set_graph_functions(&te_graph_fns);
+}
+
+void
+notify_crmd(pcmk__graph_t *graph)
+{
+    const char *type = "unknown";
+    enum crmd_fsa_input event = I_NULL;
+
+    crm_debug("Processing transition completion in state %s",
+              fsa_state2string(controld_globals.fsa_state));
+
+    CRM_CHECK(graph->complete, graph->complete = true);
+
+    switch (graph->completion_action) {
+        case pcmk__graph_wait:
+            type = "stop";
+            if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
+                event = I_TE_SUCCESS;
+            }
+            break;
+        case pcmk__graph_done:
+            type = "done";
+            if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
+                event = I_TE_SUCCESS;
+            }
+            break;
+
+        case pcmk__graph_restart:
+            type = "restart";
+            if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
+                if (controld_get_period_transition_timer() > 0) {
+                    controld_stop_transition_timer();
+                    controld_start_transition_timer();
+                } else {
+                    event = I_PE_CALC;
+                }
+
+            } else if (controld_globals.fsa_state == S_POLICY_ENGINE) {
+                controld_set_fsa_action_flags(A_PE_INVOKE);
+                controld_trigger_fsa();
+            }
+            break;
+
+        case pcmk__graph_shutdown:
+            type = "shutdown";
+            if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+                event = I_STOP;
+
+            } else {
+                crm_err("We didn't ask to be shut down, yet the scheduler is telling us to");
+                event = I_TERMINATE;
+            }
+    }
+
+    crm_debug("Transition %d status: %s - %s", graph->id, type,
+              pcmk__s(graph->abort_reason, "unspecified reason"));
+
+    graph->abort_reason = NULL;
+    graph->completion_action = pcmk__graph_done;
+
+    if (event != I_NULL) {
+        register_fsa_input(C_FSA_INTERNAL, event, NULL);
+    } else {
+        controld_trigger_fsa();
+    }
+}
diff --git a/daemons/controld/controld_te_callbacks.c b/daemons/controld/controld_te_callbacks.c
new file mode 100644
index 0000000..cf9de83
--- /dev/null
+++ b/daemons/controld/controld_te_callbacks.c
@@ -0,0 +1,689 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/stat.h>
+
+#include <crm/crm.h>
+#include <crm/common/xml.h>
+#include <crm/common/xml_internal.h>
+#include <crm/msg_xml.h>
+#include <crm/cluster.h>        /* For ONLINESTATUS etc */
+
+#include <pacemaker-controld.h>
+
+void te_update_confirm(const char *event, xmlNode * msg);
+
+#define RSC_OP_PREFIX "//" XML_TAG_DIFF_ADDED "//" XML_TAG_CIB \
+                      "//" XML_LRM_TAG_RSC_OP "[@" XML_ATTR_ID "='"
+
+// An explicit shutdown-lock of 0 means the lock has been cleared
+static bool
+shutdown_lock_cleared(xmlNode *lrm_resource)
+{
+    time_t shutdown_lock = 0;
+
+    return (crm_element_value_epoch(lrm_resource, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
+                                    &shutdown_lock) == pcmk_ok)
+           && (shutdown_lock == 0);
+}
+
+static void
+te_update_diff_v1(const char *event, xmlNode *diff)
+{
+    int lpc, max;
+    xmlXPathObject *xpathObj = NULL;
+    GString *rsc_op_xpath = NULL;
+
+    CRM_CHECK(diff != NULL, return);
+
+    pcmk__output_set_log_level(controld_globals.logger_out, LOG_TRACE);
+    controld_globals.logger_out->message(controld_globals.logger_out,
+                                         "xml-patchset", diff);
+
+    if (cib__config_changed_v1(NULL, NULL, &diff)) {
+        abort_transition(INFINITY, pcmk__graph_restart, "Non-status change",
+                         diff);
+        goto bail;              /* configuration changed */
+    }
+
+    /* Tickets Attributes - Added/Updated */
+    xpathObj =
+        xpath_search(diff,
+                     "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_TICKETS);
+    if (numXpathResults(xpathObj) > 0) {
+        xmlNode *aborted = getXpathResult(xpathObj, 0);
+
+        abort_transition(INFINITY, pcmk__graph_restart,
+                         "Ticket attribute: update", aborted);
+        goto bail;
+
+    }
+    freeXpathObject(xpathObj);
+
+    /* Tickets Attributes - Removed */
+    xpathObj =
+        xpath_search(diff,
+                     "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_TICKETS);
+    if (numXpathResults(xpathObj) > 0) {
+        xmlNode *aborted = getXpathResult(xpathObj, 0);
+
+        abort_transition(INFINITY, pcmk__graph_restart,
+                         "Ticket attribute: removal", aborted);
+        goto bail;
+    }
+    freeXpathObject(xpathObj);
+
+    /* Transient Attributes - Removed */
+    xpathObj =
+        xpath_search(diff,
+                     "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//"
+                     XML_TAG_TRANSIENT_NODEATTRS);
+    if (numXpathResults(xpathObj) > 0) {
+        xmlNode *aborted = getXpathResult(xpathObj, 0);
+
+        abort_transition(INFINITY, pcmk__graph_restart,
+                         "Transient attribute: removal", aborted);
+        goto bail;
+
+    }
+    freeXpathObject(xpathObj);
+
+    // Check for lrm_resource entries
+    xpathObj = xpath_search(diff,
+                            "//" F_CIB_UPDATE_RESULT
+                            "//" XML_TAG_DIFF_ADDED
+                            "//" XML_LRM_TAG_RESOURCE);
+    max = numXpathResults(xpathObj);
+
+    /*
+     * Updates by, or in response to, graph actions will never affect more than
+     * one resource at a time, so such updates indicate an LRM refresh. In that
+     * case, start a new transition rather than check each result individually,
+     * which can result in _huge_ speedups in large clusters.
+     *
+     * Unfortunately, we can only do so when there are no pending actions.
+     * Otherwise, we could mistakenly throw away those results here, and
+     * the cluster will stall waiting for them and time out the operation.
+     */
+    if ((controld_globals.transition_graph->pending == 0) && (max > 1)) {
+        crm_debug("Ignoring resource operation updates due to history refresh of %d resources",
+                  max);
+        crm_log_xml_trace(diff, "lrm-refresh");
+        abort_transition(INFINITY, pcmk__graph_restart, "History refresh",
+                         NULL);
+        goto bail;
+    }
+
+    if (max == 1) {
+        xmlNode *lrm_resource = getXpathResult(xpathObj, 0);
+
+        if (shutdown_lock_cleared(lrm_resource)) {
+            // @TODO would be more efficient to abort once after transition done
+            abort_transition(INFINITY, pcmk__graph_restart,
+                             "Shutdown lock cleared", lrm_resource);
+            // Still process results, so we stop timers and update failcounts
+        }
+    }
+    freeXpathObject(xpathObj);
+
+    /* Process operation updates */
+    xpathObj =
+        xpath_search(diff,
+                     "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_LRM_TAG_RSC_OP);
+    max = numXpathResults(xpathObj);
+    if (max > 0) {
+        int lpc = 0;
+
+        for (lpc = 0; lpc < max; lpc++) {
+            xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
+            const char *node = get_node_id(rsc_op);
+
+            process_graph_event(rsc_op, node);
+        }
+    }
+    freeXpathObject(xpathObj);
+
+    /* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
+    xpathObj = xpath_search(diff, "//" XML_TAG_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
+    max = numXpathResults(xpathObj);
+    for (lpc = 0; lpc < max; lpc++) {
+        const char *op_id = NULL;
+        xmlXPathObject *op_match = NULL;
+        xmlNode *match = getXpathResult(xpathObj, lpc);
+
+        CRM_LOG_ASSERT(match != NULL);
+        if(match == NULL) { continue; };
+
+        op_id = ID(match);
+
+        if (rsc_op_xpath == NULL) {
+            rsc_op_xpath = g_string_new(RSC_OP_PREFIX);
+        } else {
+            g_string_truncate(rsc_op_xpath, sizeof(RSC_OP_PREFIX) - 1);
+        }
+        pcmk__g_strcat(rsc_op_xpath, op_id, "']", NULL);
+
+        op_match = xpath_search(diff, (const char *) rsc_op_xpath->str);
+        if (numXpathResults(op_match) == 0) {
+            /* Prevent false positives by matching cancelations too */
+            const char *node = get_node_id(match);
+            pcmk__graph_action_t *cancelled = get_cancel_action(op_id, node);
+
+            if (cancelled == NULL) {
+                crm_debug("No match for deleted action %s (%s on %s)",
+                          (const char *) rsc_op_xpath->str, op_id, node);
+                abort_transition(INFINITY, pcmk__graph_restart,
+                                 "Resource op removal", match);
+                freeXpathObject(op_match);
+                goto bail;
+
+            } else {
+                crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
+                          op_id, node, cancelled->id);
+            }
+        }
+
+        freeXpathObject(op_match);
+    }
+
+  bail:
+    freeXpathObject(xpathObj);
+    if (rsc_op_xpath != NULL) {
+        g_string_free(rsc_op_xpath, TRUE);
+    }
+}
+
+static void
+process_lrm_resource_diff(xmlNode *lrm_resource, const char *node)
+{
+    for (xmlNode *rsc_op = pcmk__xml_first_child(lrm_resource); rsc_op != NULL;
+         rsc_op = pcmk__xml_next(rsc_op)) {
+        process_graph_event(rsc_op, node);
+    }
+    if (shutdown_lock_cleared(lrm_resource)) {
+        // @TODO would be more efficient to abort once after transition done
+        abort_transition(INFINITY, pcmk__graph_restart, "Shutdown lock cleared",
+                         lrm_resource);
+    }
+}
+
+static void
+process_resource_updates(const char *node, xmlNode *xml, xmlNode *change,
+                         const char *op, const char *xpath)
+{
+    xmlNode *rsc = NULL;
+
+    if (xml == NULL) {
+        return;
+    }
+
+    if (strcmp(TYPE(xml), XML_CIB_TAG_LRM) == 0) {
+        xml = first_named_child(xml, XML_LRM_TAG_RESOURCES);
+        CRM_CHECK(xml != NULL, return);
+    }
+
+    CRM_CHECK(strcmp(TYPE(xml), XML_LRM_TAG_RESOURCES) == 0, return);
+
+    /*
+     * Updates by, or in response to, TE actions will never contain updates
+     * for more than one resource at a time, so such updates indicate an
+     * LRM refresh.
+     *
+     * In that case, start a new transition rather than check each result
+     * individually, which can result in _huge_ speedups in large clusters.
+     *
+     * Unfortunately, we can only do so when there are no pending actions.
+     * Otherwise, we could mistakenly throw away those results here, and
+     * the cluster will stall waiting for them and time out the operation.
+     */
+    if ((controld_globals.transition_graph->pending == 0)
+        && (xml->children != NULL) && (xml->children->next != NULL)) {
+
+        crm_log_xml_trace(change, "lrm-refresh");
+        abort_transition(INFINITY, pcmk__graph_restart, "History refresh",
+                         NULL);
+        return;
+    }
+
+    for (rsc = pcmk__xml_first_child(xml); rsc != NULL;
+         rsc = pcmk__xml_next(rsc)) {
+        crm_trace("Processing %s", ID(rsc));
+        process_lrm_resource_diff(rsc, node);
+    }
+}
+
+static char *extract_node_uuid(const char *xpath) 
+{
+    char *mutable_path = strdup(xpath);
+    char *node_uuid = NULL;
+    char *search = NULL;
+    char *match = NULL;
+
+    match = strstr(mutable_path, "node_state[@" XML_ATTR_ID "=\'");
+    if (match == NULL) {
+        free(mutable_path);
+        return NULL;
+    }
+    match += strlen("node_state[@" XML_ATTR_ID "=\'");
+
+    search = strchr(match, '\'');
+    if (search == NULL) {
+        free(mutable_path);
+        return NULL;
+    }
+    search[0] = 0;
+
+    node_uuid = strdup(match);
+    free(mutable_path);
+    return node_uuid;
+}
+
+static void
+abort_unless_down(const char *xpath, const char *op, xmlNode *change,
+                  const char *reason)
+{
+    char *node_uuid = NULL;
+    pcmk__graph_action_t *down = NULL;
+
+    if(!pcmk__str_eq(op, "delete", pcmk__str_casei)) {
+        abort_transition(INFINITY, pcmk__graph_restart, reason, change);
+        return;
+    }
+
+    node_uuid = extract_node_uuid(xpath);
+    if(node_uuid == NULL) {
+        crm_err("Could not extract node ID from %s", xpath);
+        abort_transition(INFINITY, pcmk__graph_restart, reason, change);
+        return;
+    }
+
+    down = match_down_event(node_uuid);
+    if (down == NULL) {
+        crm_trace("Not expecting %s to be down (%s)", node_uuid, xpath);
+        abort_transition(INFINITY, pcmk__graph_restart, reason, change);
+    } else {
+        crm_trace("Expecting changes to %s (%s)", node_uuid, xpath);
+    }
+    free(node_uuid);
+}
+
+static void
+process_op_deletion(const char *xpath, xmlNode *change)
+{
+    char *mutable_key = strdup(xpath);
+    char *key;
+    char *node_uuid;
+
+    // Extract the part of xpath between last pair of single quotes
+    key = strrchr(mutable_key, '\'');
+    if (key != NULL) {
+        *key = '\0';
+        key = strrchr(mutable_key, '\'');
+    }
+    if (key == NULL) {
+        crm_warn("Ignoring malformed CIB update (resource deletion of %s)",
+                 xpath);
+        free(mutable_key);
+        return;
+    }
+    ++key;
+
+    node_uuid = extract_node_uuid(xpath);
+    if (confirm_cancel_action(key, node_uuid) == FALSE) {
+        abort_transition(INFINITY, pcmk__graph_restart,
+                         "Resource operation removal", change);
+    }
+    free(mutable_key);
+    free(node_uuid);
+}
+
+static void
+process_delete_diff(const char *xpath, const char *op, xmlNode *change)
+{
+    if (strstr(xpath, "/" XML_LRM_TAG_RSC_OP "[")) {
+        process_op_deletion(xpath, change);
+
+    } else if (strstr(xpath, "/" XML_CIB_TAG_LRM "[")) {
+        abort_unless_down(xpath, op, change, "Resource state removal");
+
+    } else if (strstr(xpath, "/" XML_CIB_TAG_STATE "[")) {
+        abort_unless_down(xpath, op, change, "Node state removal");
+
+    } else {
+        crm_trace("Ignoring delete of %s", xpath);
+    }
+}
+
+static void
+process_node_state_diff(xmlNode *state, xmlNode *change, const char *op,
+                        const char *xpath)
+{
+    xmlNode *lrm = first_named_child(state, XML_CIB_TAG_LRM);
+
+    process_resource_updates(ID(state), lrm, change, op, xpath);
+}
+
+static void
+process_status_diff(xmlNode *status, xmlNode *change, const char *op,
+                    const char *xpath)
+{
+    for (xmlNode *state = pcmk__xml_first_child(status); state != NULL;
+         state = pcmk__xml_next(state)) {
+        process_node_state_diff(state, change, op, xpath);
+    }
+}
+
+static void
+process_cib_diff(xmlNode *cib, xmlNode *change, const char *op,
+                 const char *xpath)
+{
+    xmlNode *status = first_named_child(cib, XML_CIB_TAG_STATUS);
+    xmlNode *config = first_named_child(cib, XML_CIB_TAG_CONFIGURATION);
+
+    if (status) {
+        process_status_diff(status, change, op, xpath);
+    }
+    if (config) {
+        abort_transition(INFINITY, pcmk__graph_restart,
+                         "Non-status-only change", change);
+    }
+}
+
+static void
+te_update_diff_v2(xmlNode *diff)
+{
+    crm_log_xml_trace(diff, "Patch:Raw");
+
+    for (xmlNode *change = pcmk__xml_first_child(diff); change != NULL;
+         change = pcmk__xml_next(change)) {
+
+        xmlNode *match = NULL;
+        const char *name = NULL;
+        const char *xpath = crm_element_value(change, XML_DIFF_PATH);
+
+        // Possible ops: create, modify, delete, move
+        const char *op = crm_element_value(change, XML_DIFF_OP);
+
+        // Ignore uninteresting updates
+        if (op == NULL) {
+            continue;
+
+        } else if (xpath == NULL) {
+            crm_trace("Ignoring %s change for version field", op);
+            continue;
+
+        } else if ((strcmp(op, "move") == 0)
+                   && (strstr(xpath,
+                              "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION
+                              "/" XML_CIB_TAG_RESOURCES) == NULL)) {
+            /* We still need to consider moves within the resources section,
+             * since they affect placement order.
+             */
+            crm_trace("Ignoring move change at %s", xpath);
+            continue;
+        }
+
+        // Find the result of create/modify ops
+        if (strcmp(op, "create") == 0) {
+            match = change->children;
+
+        } else if (strcmp(op, "modify") == 0) {
+            match = first_named_child(change, XML_DIFF_RESULT);
+            if(match) {
+                match = match->children;
+            }
+
+        } else if (!pcmk__str_any_of(op, "delete", "move", NULL)) {
+            crm_warn("Ignoring malformed CIB update (%s operation on %s is unrecognized)",
+                     op, xpath);
+            continue;
+        }
+
+        if (match) {
+            if (match->type == XML_COMMENT_NODE) {
+                crm_trace("Ignoring %s operation for comment at %s", op, xpath);
+                continue;
+            }
+            name = (const char *)match->name;
+        }
+
+        crm_trace("Handling %s operation for %s%s%s",
+                  op, (xpath? xpath : "CIB"),
+                  (name? " matched by " : ""), (name? name : ""));
+
+        if (strstr(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION)) {
+            abort_transition(INFINITY, pcmk__graph_restart,
+                             "Configuration change", change);
+            break; // Won't be packaged with operation results we may be waiting for
+
+        } else if (strstr(xpath, "/" XML_CIB_TAG_TICKETS)
+                   || pcmk__str_eq(name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
+            abort_transition(INFINITY, pcmk__graph_restart,
+                             "Ticket attribute change", change);
+            break; // Won't be packaged with operation results we may be waiting for
+
+        } else if (strstr(xpath, "/" XML_TAG_TRANSIENT_NODEATTRS "[")
+                   || pcmk__str_eq(name, XML_TAG_TRANSIENT_NODEATTRS,
+                                   pcmk__str_none)) {
+            abort_unless_down(xpath, op, change, "Transient attribute change");
+            break; // Won't be packaged with operation results we may be waiting for
+
+        } else if (strcmp(op, "delete") == 0) {
+            process_delete_diff(xpath, op, change);
+
+        } else if (name == NULL) {
+            crm_warn("Ignoring malformed CIB update (%s at %s has no result)",
+                     op, xpath);
+
+        } else if (strcmp(name, XML_TAG_CIB) == 0) {
+            process_cib_diff(match, change, op, xpath);
+
+        } else if (strcmp(name, XML_CIB_TAG_STATUS) == 0) {
+            process_status_diff(match, change, op, xpath);
+
+        } else if (strcmp(name, XML_CIB_TAG_STATE) == 0) {
+            process_node_state_diff(match, change, op, xpath);
+
+        } else if (strcmp(name, XML_CIB_TAG_LRM) == 0) {
+            process_resource_updates(ID(match), match, change, op, xpath);
+
+        } else if (strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
+            char *local_node = pcmk__xpath_node_id(xpath, "lrm");
+
+            process_resource_updates(local_node, match, change, op, xpath);
+            free(local_node);
+
+        } else if (strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
+            char *local_node = pcmk__xpath_node_id(xpath, "lrm");
+
+            process_lrm_resource_diff(match, local_node);
+            free(local_node);
+
+        } else if (strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
+            char *local_node = pcmk__xpath_node_id(xpath, "lrm");
+
+            process_graph_event(match, local_node);
+            free(local_node);
+
+        } else {
+            crm_warn("Ignoring malformed CIB update (%s at %s has unrecognized result %s)",
+                     op, xpath, name);
+        }
+    }
+}
+
+void
+te_update_diff(const char *event, xmlNode * msg)
+{
+    xmlNode *diff = NULL;
+    const char *op = NULL;
+    int rc = -EINVAL;
+    int format = 1;
+    int p_add[] = { 0, 0, 0 };
+    int p_del[] = { 0, 0, 0 };
+
+    CRM_CHECK(msg != NULL, return);
+    crm_element_value_int(msg, F_CIB_RC, &rc);
+
+    if (controld_globals.transition_graph == NULL) {
+        crm_trace("No graph");
+        return;
+
+    } else if (rc < pcmk_ok) {
+        crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
+        return;
+
+    } else if (controld_globals.transition_graph->complete
+               && (controld_globals.fsa_state != S_IDLE)
+               && (controld_globals.fsa_state != S_TRANSITION_ENGINE)
+               && (controld_globals.fsa_state != S_POLICY_ENGINE)) {
+        crm_trace("Filter state=%s (complete)",
+                  fsa_state2string(controld_globals.fsa_state));
+        return;
+    }
+
+    op = crm_element_value(msg, F_CIB_OPERATION);
+    diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
+
+    xml_patch_versions(diff, p_add, p_del);
+    crm_debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op,
+              p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2],
+              fsa_state2string(controld_globals.fsa_state));
+
+    crm_element_value_int(diff, "format", &format);
+    switch (format) {
+        case 1:
+            te_update_diff_v1(event, diff);
+            break;
+        case 2:
+            te_update_diff_v2(diff);
+            break;
+        default:
+            crm_warn("Ignoring malformed CIB update (unknown patch format %d)",
+                     format);
+    }
+    controld_remove_all_outside_events();
+}
+
+void
+process_te_message(xmlNode * msg, xmlNode * xml_data)
+{
+    const char *value = NULL;
+    xmlXPathObject *xpathObj = NULL;
+    int nmatches = 0;
+
+    CRM_CHECK(msg != NULL, return);
+
+    // Transition requests must specify transition engine as subsystem
+    value = crm_element_value(msg, F_CRM_SYS_TO);
+    if (pcmk__str_empty(value)
+        || !pcmk__str_eq(value, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
+        crm_info("Received invalid transition request: subsystem '%s' not '"
+                 CRM_SYSTEM_TENGINE "'", pcmk__s(value, ""));
+        return;
+    }
+
+    // Only the lrm_invoke command is supported as a transition request
+    value = crm_element_value(msg, F_CRM_TASK);
+    if (!pcmk__str_eq(value, CRM_OP_INVOKE_LRM, pcmk__str_none)) {
+        crm_info("Received invalid transition request: command '%s' not '"
+                 CRM_OP_INVOKE_LRM "'", pcmk__s(value, ""));
+        return;
+    }
+
+    // Transition requests must be marked as coming from the executor
+    value = crm_element_value(msg, F_CRM_SYS_FROM);
+    if (!pcmk__str_eq(value, CRM_SYSTEM_LRMD, pcmk__str_none)) {
+        crm_info("Received invalid transition request: from '%s' not '"
+                 CRM_SYSTEM_LRMD "'", pcmk__s(value, ""));
+        return;
+    }
+
+    crm_debug("Processing transition request with ref='%s' origin='%s'",
+              pcmk__s(crm_element_value(msg, F_CRM_REFERENCE), ""),
+              pcmk__s(crm_element_value(msg, F_ORIG), ""));
+
+    xpathObj = xpath_search(xml_data, "//" XML_LRM_TAG_RSC_OP);
+    nmatches = numXpathResults(xpathObj);
+    if (nmatches == 0) {
+        crm_err("Received transition request with no results (bug?)");
+    } else {
+        for (int lpc = 0; lpc < nmatches; lpc++) {
+            xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
+            const char *node = get_node_id(rsc_op);
+
+            process_graph_event(rsc_op, node);
+        }
+    }
+    freeXpathObject(xpathObj);
+}
+
+void
+cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    if (rc < pcmk_ok) {
+        crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
+    }
+}
+
+/*!
+ * \brief Handle a timeout in node-to-node communication
+ *
+ * \param[in,out] data  Pointer to graph action
+ *
+ * \return FALSE (indicating that source should be not be re-added)
+ */
+gboolean
+action_timer_callback(gpointer data)
+{
+    pcmk__graph_action_t *action = (pcmk__graph_action_t *) data;
+    const char *task = NULL;
+    const char *on_node = NULL;
+    const char *via_node = NULL;
+
+    CRM_CHECK(data != NULL, return FALSE);
+
+    stop_te_timer(action);
+
+    task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
+    on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
+    via_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
+
+    if (controld_globals.transition_graph->complete) {
+        crm_notice("Node %s did not send %s result (via %s) within %dms "
+                   "(ignoring because transition not in progress)",
+                   (on_node? on_node : ""), (task? task : "unknown action"),
+                   (via_node? via_node : "controller"), action->timeout);
+    } else {
+        /* fail the action */
+
+        crm_err("Node %s did not send %s result (via %s) within %dms "
+                "(action timeout plus cluster-delay)",
+                (on_node? on_node : ""), (task? task : "unknown action"),
+                (via_node? via_node : "controller"),
+                (action->timeout
+                 + controld_globals.transition_graph->network_delay));
+        pcmk__log_graph_action(LOG_ERR, action);
+
+        pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
+
+        te_action_confirmed(action, controld_globals.transition_graph);
+        abort_transition(INFINITY, pcmk__graph_restart, "Action lost", NULL);
+
+        // Record timeout in the CIB if appropriate
+        if ((action->type == pcmk__rsc_graph_action)
+            && controld_action_is_recordable(task)) {
+            controld_record_action_timeout(action);
+        }
+    }
+
+    return FALSE;
+}
diff --git a/daemons/controld/controld_te_events.c b/daemons/controld/controld_te_events.c
new file mode 100644
index 0000000..d4e2b0f
--- /dev/null
+++ b/daemons/controld/controld_te_events.c
@@ -0,0 +1,601 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <crm/crm.h>
+#include <crm/cib.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+
+#include <pacemaker-controld.h>
+
+#include <crm/common/attrd_internal.h>
+#include <crm/common/ipc_attrd_internal.h>
+
+/*!
+ * \internal
+ * \brief Action numbers of outside events processed in current update diff
+ *
+ * This table is to be used as a set. It should be empty when the transitioner
+ * begins processing a CIB update diff. It ensures that if there are multiple
+ * events (for example, "_last_0" and "_last_failure_0") for the same action,
+ * only one of them updates the failcount. Events that originate outside the
+ * cluster can't be confirmed, since they're not in the transition graph.
+ */
+static GHashTable *outside_events = NULL;
+
+/*!
+ * \internal
+ * \brief Empty the hash table containing action numbers of outside events
+ */
+void
+controld_remove_all_outside_events(void)
+{
+    if (outside_events != NULL) {
+        g_hash_table_remove_all(outside_events);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Destroy the hash table containing action numbers of outside events
+ */
+void
+controld_destroy_outside_events_table(void)
+{
+    if (outside_events != NULL) {
+        g_hash_table_destroy(outside_events);
+        outside_events = NULL;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Add an outside event's action number to a set
+ *
+ * \return Standard Pacemaker return code. Specifically, \p pcmk_rc_ok if the
+ *         event was not already in the set, or \p pcmk_rc_already otherwise.
+ */
+static int
+record_outside_event(gint action_num)
+{
+    if (outside_events == NULL) {
+        outside_events = g_hash_table_new(NULL, NULL);
+    }
+
+    if (g_hash_table_add(outside_events, GINT_TO_POINTER(action_num))) {
+        return pcmk_rc_ok;
+    }
+    return pcmk_rc_already;
+}
+
+gboolean
+fail_incompletable_actions(pcmk__graph_t *graph, const char *down_node)
+{
+    const char *target_uuid = NULL;
+    const char *router = NULL;
+    const char *router_uuid = NULL;
+    xmlNode *last_action = NULL;
+
+    GList *gIter = NULL;
+    GList *gIter2 = NULL;
+
+    if (graph == NULL || graph->complete) {
+        return FALSE;
+    }
+
+    gIter = graph->synapses;
+    for (; gIter != NULL; gIter = gIter->next) {
+        pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) gIter->data;
+
+        if (pcmk_any_flags_set(synapse->flags, pcmk__synapse_confirmed|pcmk__synapse_failed)) {
+            /* We've already been here */
+            continue;
+        }
+
+        gIter2 = synapse->actions;
+        for (; gIter2 != NULL; gIter2 = gIter2->next) {
+            pcmk__graph_action_t *action = (pcmk__graph_action_t *) gIter2->data;
+
+            if ((action->type == pcmk__pseudo_graph_action)
+                || pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
+                continue;
+            } else if (action->type == pcmk__cluster_graph_action) {
+                const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
+
+                if (pcmk__str_eq(task, CRM_OP_FENCE, pcmk__str_casei)) {
+                    continue;
+                }
+            }
+
+            target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
+            router = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
+            if (router) {
+                crm_node_t *node = crm_get_peer(0, router);
+                if (node) {
+                    router_uuid = node->uuid;
+                }
+            }
+
+            if (pcmk__str_eq(target_uuid, down_node, pcmk__str_casei) || pcmk__str_eq(router_uuid, down_node, pcmk__str_casei)) {
+                pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
+                pcmk__set_synapse_flags(synapse, pcmk__synapse_failed);
+                last_action = action->xml;
+                stop_te_timer(action);
+                pcmk__update_graph(graph, action);
+
+                if (pcmk_is_set(synapse->flags, pcmk__synapse_executed)) {
+                    crm_notice("Action %d (%s) was pending on %s (offline)",
+                               action->id, crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY), down_node);
+                } else {
+                    crm_info("Action %d (%s) is scheduled for %s (offline)",
+                             action->id, crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY), down_node);
+                }
+            }
+        }
+    }
+
+    if (last_action != NULL) {
+        crm_info("Node %s shutdown resulted in un-runnable actions", down_node);
+        abort_transition(INFINITY, pcmk__graph_restart, "Node failure",
+                         last_action);
+        return TRUE;
+    }
+
+    return FALSE;
+}
+
+/*!
+ * \internal
+ * \brief Update failure-related node attributes if warranted
+ *
+ * \param[in] event            XML describing operation that (maybe) failed
+ * \param[in] event_node_uuid  Node that event occurred on
+ * \param[in] rc               Actual operation return code
+ * \param[in] target_rc        Expected operation return code
+ * \param[in] do_update        If TRUE, do update regardless of operation type
+ * \param[in] ignore_failures  If TRUE, update last failure but not fail count
+ *
+ * \return TRUE if this was not a direct nack, success or lrm status refresh
+ */
+static gboolean
+update_failcount(const xmlNode *event, const char *event_node_uuid, int rc,
+                 int target_rc, gboolean do_update, gboolean ignore_failures)
+{
+    guint interval_ms = 0;
+
+    char *task = NULL;
+    char *rsc_id = NULL;
+
+    const char *value = NULL;
+    const char *id = crm_element_value(event, XML_LRM_ATTR_TASK_KEY);
+    const char *on_uname = crm_peer_uname(event_node_uuid);
+    const char *origin = crm_element_value(event, XML_ATTR_ORIGIN);
+
+    // Nothing needs to be done for success or status refresh
+    if (rc == target_rc) {
+        return FALSE;
+    } else if (pcmk__str_eq(origin, "build_active_RAs", pcmk__str_casei)) {
+        crm_debug("No update for %s (rc=%d) on %s: Old failure from lrm status refresh",
+                  id, rc, on_uname);
+        return FALSE;
+    }
+
+    /* Sanity check */
+    CRM_CHECK(on_uname != NULL, return TRUE);
+    CRM_CHECK(parse_op_key(id, &rsc_id, &task, &interval_ms),
+              crm_err("Couldn't parse: %s", ID(event)); goto bail);
+
+    /* Decide whether update is necessary and what value to use */
+    if ((interval_ms > 0)
+        || pcmk__str_eq(task, CRMD_ACTION_PROMOTE, pcmk__str_none)
+        || pcmk__str_eq(task, CRMD_ACTION_DEMOTE, pcmk__str_none)) {
+        do_update = TRUE;
+
+    } else if (pcmk__str_eq(task, CRMD_ACTION_START, pcmk__str_none)) {
+        do_update = TRUE;
+        value = pcmk__s(controld_globals.transition_graph->failed_start_offset,
+                        CRM_INFINITY_S);
+
+    } else if (pcmk__str_eq(task, CRMD_ACTION_STOP, pcmk__str_none)) {
+        do_update = TRUE;
+        value = pcmk__s(controld_globals.transition_graph->failed_stop_offset,
+                        CRM_INFINITY_S);
+    }
+
+    if (do_update) {
+        pcmk__attrd_query_pair_t *fail_pair = NULL;
+        pcmk__attrd_query_pair_t *last_pair = NULL;
+        char *fail_name = NULL;
+        char *last_name = NULL;
+        GList *attrs = NULL;
+
+        uint32_t opts = pcmk__node_attr_none;
+
+        char *now = pcmk__ttoa(time(NULL));
+
+        // Fail count will be either incremented or set to infinity
+        if (!pcmk_str_is_infinity(value)) {
+            value = XML_NVPAIR_ATTR_VALUE "++";
+        }
+
+        if (g_hash_table_lookup(crm_remote_peer_cache, event_node_uuid)) {
+            opts |= pcmk__node_attr_remote;
+        }
+
+        crm_info("Updating %s for %s on %s after failed %s: rc=%d (update=%s, time=%s)",
+                 (ignore_failures? "last failure" : "failcount"),
+                 rsc_id, on_uname, task, rc, value, now);
+
+        /* Update the fail count, if we're not ignoring failures */
+        if (!ignore_failures) {
+            fail_pair = calloc(1, sizeof(pcmk__attrd_query_pair_t));
+            CRM_ASSERT(fail_pair != NULL);
+
+            fail_name = pcmk__failcount_name(rsc_id, task, interval_ms);
+            fail_pair->name = fail_name;
+            fail_pair->value = value;
+            fail_pair->node = on_uname;
+
+            attrs = g_list_prepend(attrs, fail_pair);
+        }
+
+        /* Update the last failure time (even if we're ignoring failures,
+         * so that failure can still be detected and shown, e.g. by crm_mon)
+         */
+        last_pair = calloc(1, sizeof(pcmk__attrd_query_pair_t));
+        CRM_ASSERT(last_pair != NULL);
+
+        last_name = pcmk__lastfailure_name(rsc_id, task, interval_ms);
+        last_pair->name = last_name;
+        last_pair->value = now;
+        last_pair->node = on_uname;
+
+        attrs = g_list_prepend(attrs, last_pair);
+
+        update_attrd_list(attrs, opts);
+
+        free(fail_name);
+        free(fail_pair);
+
+        free(last_name);
+        free(last_pair);
+        g_list_free(attrs);
+
+        free(now);
+    }
+
+  bail:
+    free(rsc_id);
+    free(task);
+    return TRUE;
+}
+
+pcmk__graph_action_t *
+controld_get_action(int id)
+{
+    for (GList *item = controld_globals.transition_graph->synapses;
+         item != NULL; item = item->next) {
+        pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) item->data;
+
+        for (GList *item2 = synapse->actions; item2; item2 = item2->next) {
+            pcmk__graph_action_t *action = (pcmk__graph_action_t *) item2->data;
+
+            if (action->id == id) {
+                return action;
+            }
+        }
+    }
+    return NULL;
+}
+
+pcmk__graph_action_t *
+get_cancel_action(const char *id, const char *node)
+{
+    GList *gIter = NULL;
+    GList *gIter2 = NULL;
+
+    gIter = controld_globals.transition_graph->synapses;
+    for (; gIter != NULL; gIter = gIter->next) {
+        pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) gIter->data;
+
+        gIter2 = synapse->actions;
+        for (; gIter2 != NULL; gIter2 = gIter2->next) {
+            const char *task = NULL;
+            const char *target = NULL;
+            pcmk__graph_action_t *action = (pcmk__graph_action_t *) gIter2->data;
+
+            task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
+            if (!pcmk__str_eq(CRMD_ACTION_CANCEL, task, pcmk__str_casei)) {
+                continue;
+            }
+
+            task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
+            if (!pcmk__str_eq(task, id, pcmk__str_casei)) {
+                crm_trace("Wrong key %s for %s on %s", task, id, node);
+                continue;
+            }
+
+            target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
+            if (node && !pcmk__str_eq(target, node, pcmk__str_casei)) {
+                crm_trace("Wrong node %s for %s on %s", target, id, node);
+                continue;
+            }
+
+            crm_trace("Found %s on %s", id, node);
+            return action;
+        }
+    }
+
+    return NULL;
+}
+
+bool
+confirm_cancel_action(const char *id, const char *node_id)
+{
+    const char *op_key = NULL;
+    const char *node_name = NULL;
+    pcmk__graph_action_t *cancel = get_cancel_action(id, node_id);
+
+    if (cancel == NULL) {
+        return FALSE;
+    }
+    op_key = crm_element_value(cancel->xml, XML_LRM_ATTR_TASK_KEY);
+    node_name = crm_element_value(cancel->xml, XML_LRM_ATTR_TARGET);
+
+    stop_te_timer(cancel);
+    te_action_confirmed(cancel, controld_globals.transition_graph);
+
+    crm_info("Cancellation of %s on %s confirmed (action %d)",
+             op_key, node_name, cancel->id);
+    return TRUE;
+}
+
+/* downed nodes are listed like: <downed> <node id="UUID1" /> ... </downed> */
+#define XPATH_DOWNED "//" XML_GRAPH_TAG_DOWNED \
+                     "/" XML_CIB_TAG_NODE "[@" XML_ATTR_ID "='%s']"
+
+/*!
+ * \brief Find a transition event that would have made a specified node down
+ *
+ * \param[in] target  UUID of node to match
+ *
+ * \return Matching event if found, NULL otherwise
+ */
+pcmk__graph_action_t *
+match_down_event(const char *target)
+{
+    pcmk__graph_action_t *match = NULL;
+    xmlXPathObjectPtr xpath_ret = NULL;
+    GList *gIter, *gIter2;
+
+    char *xpath = crm_strdup_printf(XPATH_DOWNED, target);
+
+    for (gIter = controld_globals.transition_graph->synapses;
+         gIter != NULL && match == NULL;
+         gIter = gIter->next) {
+
+        for (gIter2 = ((pcmk__graph_synapse_t * ) gIter->data)->actions;
+             gIter2 != NULL && match == NULL;
+             gIter2 = gIter2->next) {
+
+            match = (pcmk__graph_action_t *) gIter2->data;
+            if (pcmk_is_set(match->flags, pcmk__graph_action_executed)) {
+                xpath_ret = xpath_search(match->xml, xpath);
+                if (numXpathResults(xpath_ret) < 1) {
+                    match = NULL;
+                }
+                freeXpathObject(xpath_ret);
+            } else {
+                // Only actions that were actually started can match
+                match = NULL;
+            }
+        }
+    }
+
+    free(xpath);
+
+    if (match != NULL) {
+        crm_debug("Shutdown action %d (%s) found for node %s", match->id,
+                  crm_element_value(match->xml, XML_LRM_ATTR_TASK_KEY), target);
+    } else {
+        crm_debug("No reason to expect node %s to be down", target);
+    }
+    return match;
+}
+
+void
+process_graph_event(xmlNode *event, const char *event_node)
+{
+    int rc = -1;                // Actual result
+    int target_rc = -1;         // Expected result
+    int status = -1;            // Executor status
+    int callid = -1;            // Executor call ID
+    int transition_num = -1;    // Transition number
+    int action_num = -1;        // Action number within transition
+    char *update_te_uuid = NULL;
+    bool ignore_failures = FALSE;
+    const char *id = NULL;
+    const char *desc = NULL;
+    const char *magic = NULL;
+    const char *uname = NULL;
+
+    CRM_ASSERT(event != NULL);
+
+/*
+<lrm_rsc_op id="rsc_east-05_last_0" operation_key="rsc_east-05_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="9:2:7:be2e97d9-05e2-439d-863e-48f7aecab2aa" transition-magic="0:7;9:2:7:be2e97d9-05e2-439d-863e-48f7aecab2aa" call-id="17" rc-code="7" op-status="0" interval="0" last-rc-change="1355361636" exec-time="128" queue-time="0" op-digest="c81f5f40b1c9e859c992e800b1aa6972"/>
+*/
+
+    magic = crm_element_value(event, XML_ATTR_TRANSITION_KEY);
+    if (magic == NULL) {
+        /* non-change */
+        return;
+    }
+
+    crm_element_value_int(event, XML_LRM_ATTR_OPSTATUS, &status);
+    if (status == PCMK_EXEC_PENDING) {
+        return;
+    }
+
+    id = crm_element_value(event, XML_LRM_ATTR_TASK_KEY);
+    crm_element_value_int(event, XML_LRM_ATTR_RC, &rc);
+    crm_element_value_int(event, XML_LRM_ATTR_CALLID, &callid);
+
+    rc = pcmk__effective_rc(rc);
+
+    if (decode_transition_key(magic, &update_te_uuid, &transition_num,
+                              &action_num, &target_rc) == FALSE) {
+        // decode_transition_key() already logged the bad key
+        crm_err("Can't process action %s result: Incompatible versions? "
+                CRM_XS " call-id=%d", id, callid);
+        abort_transition(INFINITY, pcmk__graph_restart, "Bad event", event);
+        return;
+    }
+
+    if (transition_num == -1) {
+        // E.g. crm_resource --fail
+        if (record_outside_event(action_num) != pcmk_rc_ok) {
+            crm_debug("Outside event with transition key '%s' has already been "
+                      "processed", magic);
+            goto bail;
+        }
+        desc = "initiated outside of the cluster";
+        abort_transition(INFINITY, pcmk__graph_restart, "Unexpected event",
+                         event);
+
+    } else if ((action_num < 0)
+               || !pcmk__str_eq(update_te_uuid, controld_globals.te_uuid,
+                                pcmk__str_none)) {
+        desc = "initiated by a different DC";
+        abort_transition(INFINITY, pcmk__graph_restart, "Foreign event", event);
+
+    } else if ((controld_globals.transition_graph->id != transition_num)
+               || controld_globals.transition_graph->complete) {
+
+        // Action is not from currently active transition
+
+        guint interval_ms = 0;
+
+        if (parse_op_key(id, NULL, NULL, &interval_ms)
+            && (interval_ms != 0)) {
+            /* Recurring actions have the transition number they were first
+             * scheduled in.
+             */
+
+            if (status == PCMK_EXEC_CANCELLED) {
+                confirm_cancel_action(id, get_node_id(event));
+                goto bail;
+            }
+
+            desc = "arrived after initial scheduling";
+            abort_transition(INFINITY, pcmk__graph_restart,
+                             "Change in recurring result", event);
+
+        } else if (controld_globals.transition_graph->id != transition_num) {
+            desc = "arrived really late";
+            abort_transition(INFINITY, pcmk__graph_restart, "Old event", event);
+        } else {
+            desc = "arrived late";
+            abort_transition(INFINITY, pcmk__graph_restart, "Inactive graph",
+                             event);
+        }
+
+    } else {
+        // Event is result of an action from currently active transition
+        pcmk__graph_action_t *action = controld_get_action(action_num);
+
+        if (action == NULL) {
+            // Should never happen
+            desc = "unknown";
+            abort_transition(INFINITY, pcmk__graph_restart, "Unknown event",
+                             event);
+
+        } else if (pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
+            /* Nothing further needs to be done if the action has already been
+             * confirmed. This can happen e.g. when processing both an
+             * "xxx_last_0" or "xxx_last_failure_0" record as well as the main
+             * history record, which would otherwise result in incorrectly
+             * bumping the fail count twice.
+             */
+            crm_log_xml_debug(event, "Event already confirmed:");
+            goto bail;
+
+        } else {
+            /* An action result needs to be confirmed.
+             * (This is the only case where desc == NULL.)
+             */
+
+            if (pcmk__str_eq(crm_meta_value(action->params, XML_OP_ATTR_ON_FAIL), "ignore", pcmk__str_casei)) {
+                ignore_failures = TRUE;
+
+            } else if (rc != target_rc) {
+                pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
+            }
+
+            stop_te_timer(action);
+            te_action_confirmed(action, controld_globals.transition_graph);
+
+            if (pcmk_is_set(action->flags, pcmk__graph_action_failed)) {
+                abort_transition(action->synapse->priority + 1,
+                                 pcmk__graph_restart, "Event failed", event);
+            }
+        }
+    }
+
+    if (id == NULL) {
+        id = "unknown action";
+    }
+    uname = crm_element_value(event, XML_LRM_ATTR_TARGET);
+    if (uname == NULL) {
+        uname = "unknown node";
+    }
+
+    if (status == PCMK_EXEC_INVALID) {
+        // We couldn't attempt the action
+        crm_info("Transition %d action %d (%s on %s): %s",
+                 transition_num, action_num, id, uname,
+                 pcmk_exec_status_str(status));
+
+    } else if (desc && update_failcount(event, event_node, rc, target_rc,
+                                        (transition_num == -1), FALSE)) {
+        crm_notice("Transition %d action %d (%s on %s): expected '%s' but got '%s' "
+                   CRM_XS " target-rc=%d rc=%d call-id=%d event='%s'",
+                   transition_num, action_num, id, uname,
+                   services_ocf_exitcode_str(target_rc),
+                   services_ocf_exitcode_str(rc),
+                   target_rc, rc, callid, desc);
+
+    } else if (desc) {
+        crm_info("Transition %d action %d (%s on %s): %s "
+                 CRM_XS " rc=%d target-rc=%d call-id=%d",
+                 transition_num, action_num, id, uname,
+                 desc, rc, target_rc, callid);
+
+    } else if (rc == target_rc) {
+        crm_info("Transition %d action %d (%s on %s) confirmed: %s "
+                 CRM_XS " rc=%d call-id=%d",
+                 transition_num, action_num, id, uname,
+                 services_ocf_exitcode_str(rc), rc, callid);
+
+    } else {
+        update_failcount(event, event_node, rc, target_rc,
+                         (transition_num == -1), ignore_failures);
+        crm_notice("Transition %d action %d (%s on %s): expected '%s' but got '%s' "
+                   CRM_XS " target-rc=%d rc=%d call-id=%d",
+                   transition_num, action_num, id, uname,
+                   services_ocf_exitcode_str(target_rc),
+                   services_ocf_exitcode_str(rc),
+                   target_rc, rc, callid);
+    }
+
+  bail:
+    free(update_te_uuid);
+}
diff --git a/daemons/controld/controld_te_utils.c b/daemons/controld/controld_te_utils.c
new file mode 100644
index 0000000..ecbc0b2
--- /dev/null
+++ b/daemons/controld/controld_te_utils.c
@@ -0,0 +1,367 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+
+#include <pacemaker-controld.h>
+
+//! Triggers transition graph processing
+static crm_trigger_t *transition_trigger = NULL;
+
+gboolean
+stop_te_timer(pcmk__graph_action_t *action)
+{
+    if (action == NULL) {
+        return FALSE;
+    }
+    if (action->timer != 0) {
+        crm_trace("Stopping action timer");
+        g_source_remove(action->timer);
+        action->timer = 0;
+    } else {
+        crm_trace("Action timer was already stopped");
+        return FALSE;
+    }
+    return TRUE;
+}
+
+static gboolean
+te_graph_trigger(gpointer user_data)
+{
+    if (controld_globals.transition_graph == NULL) {
+        crm_debug("Nothing to do");
+        return TRUE;
+    }
+
+    crm_trace("Invoking graph %d in state %s",
+              controld_globals.transition_graph->id,
+              fsa_state2string(controld_globals.fsa_state));
+
+    switch (controld_globals.fsa_state) {
+        case S_STARTING:
+        case S_PENDING:
+        case S_NOT_DC:
+        case S_HALT:
+        case S_ILLEGAL:
+        case S_STOPPING:
+        case S_TERMINATE:
+            return TRUE;
+        default:
+            break;
+    }
+
+    if (!controld_globals.transition_graph->complete) {
+        enum pcmk__graph_status graph_rc;
+        int orig_limit = controld_globals.transition_graph->batch_limit;
+        int throttled_limit = throttle_get_total_job_limit(orig_limit);
+
+        controld_globals.transition_graph->batch_limit = throttled_limit;
+        graph_rc = pcmk__execute_graph(controld_globals.transition_graph);
+        controld_globals.transition_graph->batch_limit = orig_limit;
+
+        if (graph_rc == pcmk__graph_active) {
+            crm_trace("Transition not yet complete");
+            return TRUE;
+
+        } else if (graph_rc == pcmk__graph_pending) {
+            crm_trace("Transition not yet complete - no actions fired");
+            return TRUE;
+        }
+
+        if (graph_rc != pcmk__graph_complete) {
+            crm_warn("Transition failed: %s",
+                     pcmk__graph_status2text(graph_rc));
+            pcmk__log_graph(LOG_NOTICE, controld_globals.transition_graph);
+        }
+    }
+
+    crm_debug("Transition %d is now complete",
+              controld_globals.transition_graph->id);
+    controld_globals.transition_graph->complete = true;
+    notify_crmd(controld_globals.transition_graph);
+
+    return TRUE;
+}
+
+/*!
+ * \internal
+ * \brief Initialize transition trigger
+ */
+void
+controld_init_transition_trigger(void)
+{
+    transition_trigger = mainloop_add_trigger(G_PRIORITY_LOW, te_graph_trigger,
+                                              NULL);
+}
+
+/*!
+ * \internal
+ * \brief Destroy transition trigger
+ */
+void
+controld_destroy_transition_trigger(void)
+{
+    mainloop_destroy_trigger(transition_trigger);
+    transition_trigger = NULL;
+}
+
+void
+controld_trigger_graph_as(const char *fn, int line)
+{
+    crm_trace("%s:%d - Triggered graph processing", fn, line);
+    mainloop_set_trigger(transition_trigger);
+}
+
+static struct abort_timer_s {
+    bool aborted;
+    guint id;
+    int priority;
+    enum pcmk__graph_next action;
+    const char *text;
+} abort_timer = { 0, };
+
+static gboolean
+abort_timer_popped(gpointer data)
+{
+    if (AM_I_DC && (abort_timer.aborted == FALSE)) {
+        abort_transition(abort_timer.priority, abort_timer.action,
+                         abort_timer.text, NULL);
+    }
+    abort_timer.id = 0;
+    return FALSE; // do not immediately reschedule timer
+}
+
+/*!
+ * \internal
+ * \brief Abort transition after delay, if not already aborted in that time
+ *
+ * \param[in] abort_text  Must be literal string
+ */
+void
+abort_after_delay(int abort_priority, enum pcmk__graph_next abort_action,
+                  const char *abort_text, guint delay_ms)
+{
+    if (abort_timer.id) {
+        // Timer already in progress, stop and reschedule
+        g_source_remove(abort_timer.id);
+    }
+    abort_timer.aborted = FALSE;
+    abort_timer.priority = abort_priority;
+    abort_timer.action = abort_action;
+    abort_timer.text = abort_text;
+    abort_timer.id = g_timeout_add(delay_ms, abort_timer_popped, NULL);
+}
+
+static const char *
+abort2text(enum pcmk__graph_next abort_action)
+{
+    switch (abort_action) {
+        case pcmk__graph_done:      return "done";
+        case pcmk__graph_wait:      return "stop";
+        case pcmk__graph_restart:   return "restart";
+        case pcmk__graph_shutdown:  return "shutdown";
+    }
+    return "unknown";
+}
+
+static bool
+update_abort_priority(pcmk__graph_t *graph, int priority,
+                      enum pcmk__graph_next action, const char *abort_reason)
+{
+    bool change = FALSE;
+
+    if (graph == NULL) {
+        return change;
+    }
+
+    if (graph->abort_priority < priority) {
+        crm_debug("Abort priority upgraded from %d to %d", graph->abort_priority, priority);
+        graph->abort_priority = priority;
+        if (graph->abort_reason != NULL) {
+            crm_debug("'%s' abort superseded by %s", graph->abort_reason, abort_reason);
+        }
+        graph->abort_reason = abort_reason;
+        change = TRUE;
+    }
+
+    if (graph->completion_action < action) {
+        crm_debug("Abort action %s superseded by %s: %s",
+                  abort2text(graph->completion_action), abort2text(action), abort_reason);
+        graph->completion_action = action;
+        change = TRUE;
+    }
+
+    return change;
+}
+
+void
+abort_transition_graph(int abort_priority, enum pcmk__graph_next abort_action,
+                       const char *abort_text, const xmlNode *reason,
+                       const char *fn, int line)
+{
+    int add[] = { 0, 0, 0 };
+    int del[] = { 0, 0, 0 };
+    int level = LOG_INFO;
+    const xmlNode *diff = NULL;
+    const xmlNode *change = NULL;
+
+    CRM_CHECK(controld_globals.transition_graph != NULL, return);
+
+    switch (controld_globals.fsa_state) {
+        case S_STARTING:
+        case S_PENDING:
+        case S_NOT_DC:
+        case S_HALT:
+        case S_ILLEGAL:
+        case S_STOPPING:
+        case S_TERMINATE:
+            crm_info("Abort %s suppressed: state=%s (%scomplete)",
+                     abort_text, fsa_state2string(controld_globals.fsa_state),
+                     (controld_globals.transition_graph->complete? "" : "in"));
+            return;
+        default:
+            break;
+    }
+
+    abort_timer.aborted = TRUE;
+    controld_expect_sched_reply(NULL);
+
+    if (!controld_globals.transition_graph->complete
+        && update_abort_priority(controld_globals.transition_graph,
+                                 abort_priority, abort_action,
+                                 abort_text)) {
+        level = LOG_NOTICE;
+    }
+
+    if (reason != NULL) {
+        const xmlNode *search = NULL;
+
+        for(search = reason; search; search = search->parent) {
+            if (pcmk__str_eq(XML_TAG_DIFF, TYPE(search), pcmk__str_casei)) {
+                diff = search;
+                break;
+            }
+        }
+
+        if(diff) {
+            xml_patch_versions(diff, add, del);
+            for(search = reason; search; search = search->parent) {
+                if (pcmk__str_eq(XML_DIFF_CHANGE, TYPE(search), pcmk__str_casei)) {
+                    change = search;
+                    break;
+                }
+            }
+        }
+    }
+
+    if (reason == NULL) {
+        do_crm_log(level,
+                   "Transition %d aborted: %s " CRM_XS " source=%s:%d "
+                   "complete=%s", controld_globals.transition_graph->id,
+                   abort_text, fn, line,
+                   pcmk__btoa(controld_globals.transition_graph->complete));
+
+    } else if(change == NULL) {
+        GString *local_path = pcmk__element_xpath(reason);
+        CRM_ASSERT(local_path != NULL);
+
+        do_crm_log(level, "Transition %d aborted by %s.%s: %s "
+                   CRM_XS " cib=%d.%d.%d source=%s:%d path=%s complete=%s",
+                   controld_globals.transition_graph->id, TYPE(reason),
+                   ID(reason), abort_text, add[0], add[1], add[2], fn, line,
+                   (const char *) local_path->str,
+                   pcmk__btoa(controld_globals.transition_graph->complete));
+        g_string_free(local_path, TRUE);
+
+    } else {
+        const char *kind = NULL;
+        const char *op = crm_element_value(change, XML_DIFF_OP);
+        const char *path = crm_element_value(change, XML_DIFF_PATH);
+
+        if(change == reason) {
+            if(strcmp(op, "create") == 0) {
+                reason = reason->children;
+
+            } else if(strcmp(op, "modify") == 0) {
+                reason = first_named_child(reason, XML_DIFF_RESULT);
+                if(reason) {
+                    reason = reason->children;
+                }
+            }
+        }
+
+        kind = TYPE(reason);
+        if(strcmp(op, "delete") == 0) {
+            const char *shortpath = strrchr(path, '/');
+
+            do_crm_log(level, "Transition %d aborted by deletion of %s: %s "
+                       CRM_XS " cib=%d.%d.%d source=%s:%d path=%s complete=%s",
+                       controld_globals.transition_graph->id,
+                       (shortpath? (shortpath + 1) : path), abort_text,
+                       add[0], add[1], add[2], fn, line, path,
+                       pcmk__btoa(controld_globals.transition_graph->complete));
+
+        } else if (pcmk__str_eq(XML_CIB_TAG_NVPAIR, kind, pcmk__str_none)) {
+            do_crm_log(level, "Transition %d aborted by %s doing %s %s=%s: %s "
+                       CRM_XS " cib=%d.%d.%d source=%s:%d path=%s complete=%s",
+                       controld_globals.transition_graph->id,
+                       crm_element_value(reason, XML_ATTR_ID), op,
+                       crm_element_value(reason, XML_NVPAIR_ATTR_NAME),
+                       crm_element_value(reason, XML_NVPAIR_ATTR_VALUE),
+                       abort_text, add[0], add[1], add[2], fn, line, path,
+                       pcmk__btoa(controld_globals.transition_graph->complete));
+
+        } else if (pcmk__str_eq(XML_LRM_TAG_RSC_OP, kind, pcmk__str_none)) {
+            const char *magic = crm_element_value(reason, XML_ATTR_TRANSITION_MAGIC);
+
+            do_crm_log(level, "Transition %d aborted by operation %s '%s' on %s: %s "
+                       CRM_XS " magic=%s cib=%d.%d.%d source=%s:%d complete=%s",
+                       controld_globals.transition_graph->id,
+                       crm_element_value(reason, XML_LRM_ATTR_TASK_KEY), op,
+                       crm_element_value(reason, XML_LRM_ATTR_TARGET), abort_text,
+                       magic, add[0], add[1], add[2], fn, line,
+                       pcmk__btoa(controld_globals.transition_graph->complete));
+
+        } else if (pcmk__str_any_of(kind, XML_CIB_TAG_STATE, XML_CIB_TAG_NODE, NULL)) {
+            const char *uname = crm_peer_uname(ID(reason));
+
+            do_crm_log(level, "Transition %d aborted by %s '%s' on %s: %s "
+                       CRM_XS " cib=%d.%d.%d source=%s:%d complete=%s",
+                       controld_globals.transition_graph->id,
+                       kind, op, (uname? uname : ID(reason)), abort_text,
+                       add[0], add[1], add[2], fn, line,
+                       pcmk__btoa(controld_globals.transition_graph->complete));
+
+        } else {
+            const char *id = ID(reason);
+
+            do_crm_log(level, "Transition %d aborted by %s.%s '%s': %s "
+                       CRM_XS " cib=%d.%d.%d source=%s:%d path=%s complete=%s",
+                       controld_globals.transition_graph->id,
+                       TYPE(reason), (id? id : ""), (op? op : "change"),
+                       abort_text, add[0], add[1], add[2], fn, line, path,
+                       pcmk__btoa(controld_globals.transition_graph->complete));
+        }
+    }
+
+    if (controld_globals.transition_graph->complete) {
+        if (controld_get_period_transition_timer() > 0) {
+            controld_stop_transition_timer();
+            controld_start_transition_timer();
+        } else {
+            register_fsa_input(C_FSA_INTERNAL, I_PE_CALC, NULL);
+        }
+        return;
+    }
+
+    trigger_graph();
+}
diff --git a/daemons/controld/controld_throttle.c b/daemons/controld/controld_throttle.c
new file mode 100644
index 0000000..5b7f9c0
--- /dev/null
+++ b/daemons/controld/controld_throttle.c
@@ -0,0 +1,574 @@
+/*
+ * Copyright 2013-2021 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include <unistd.h>
+#include <ctype.h>
+#include <dirent.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/cluster.h>
+
+#include <pacemaker-controld.h>
+
+/* These values don't need to be bits, but these particular values must be kept
+ * for backward compatibility during rolling upgrades.
+ */
+enum throttle_state_e {
+    throttle_none       = 0x0000,
+    throttle_low        = 0x0001,
+    throttle_med        = 0x0010,
+    throttle_high       = 0x0100,
+    throttle_extreme    = 0x1000,
+};
+
+struct throttle_record_s {
+    int max;
+    enum throttle_state_e mode;
+    char *node;
+};
+
+static int throttle_job_max = 0;
+static float throttle_load_target = 0.0;
+
+#define THROTTLE_FACTOR_LOW    1.2
+#define THROTTLE_FACTOR_MEDIUM 1.6
+#define THROTTLE_FACTOR_HIGH   2.0
+
+static GHashTable *throttle_records = NULL;
+static mainloop_timer_t *throttle_timer = NULL;
+
+static const char *
+load2str(enum throttle_state_e mode)
+{
+    switch (mode) {
+        case throttle_extreme:  return "extreme";
+        case throttle_high:     return "high";
+        case throttle_med:      return "medium";
+        case throttle_low:      return "low";
+        case throttle_none:     return "negligible";
+        default:                return "undetermined";
+    }
+}
+
+#if HAVE_LINUX_PROCFS
+/*!
+ * \internal
+ * \brief Return name of /proc file containing the CIB daemon's load statistics
+ *
+ * \return Newly allocated memory with file name on success, NULL otherwise
+ *
+ * \note It is the caller's responsibility to free the return value.
+ *       This will return NULL if the daemon is being run via valgrind.
+ *       This should be called only on Linux systems.
+ */
+static char *
+find_cib_loadfile(void)
+{
+    pid_t pid = pcmk__procfs_pid_of("pacemaker-based");
+
+    return pid? crm_strdup_printf("/proc/%lld/stat", (long long) pid) : NULL;
+}
+
+static bool
+throttle_cib_load(float *load)
+{
+/*
+       /proc/[pid]/stat
+              Status information about the process.  This is used by ps(1).  It is defined in /usr/src/linux/fs/proc/array.c.
+
+              The fields, in order, with their proper scanf(3) format specifiers, are:
+
+              pid %d      (1) The process ID.
+
+              comm %s     (2) The filename of the executable, in parentheses.  This is visible whether or not the executable is swapped out.
+
+              state %c    (3) One character from the string "RSDZTW" where R is running, S is sleeping in an interruptible wait, D is waiting in uninterruptible disk sleep, Z is zombie, T is traced or stopped (on a signal), and W is paging.
+
+              ppid %d     (4) The PID of the parent.
+
+              pgrp %d     (5) The process group ID of the process.
+
+              session %d  (6) The session ID of the process.
+
+              tty_nr %d   (7) The controlling terminal of the process.  (The minor device number is contained in the combination of bits 31 to 20 and 7 to 0; the major device number is in bits 15 to 8.)
+
+              tpgid %d    (8) The ID of the foreground process group of the controlling terminal of the process.
+
+              flags %u (%lu before Linux 2.6.22)
+                          (9) The kernel flags word of the process.  For bit meanings, see the PF_* defines in the Linux kernel source file include/linux/sched.h.  Details depend on the kernel version.
+
+              minflt %lu  (10) The number of minor faults the process has made which have not required loading a memory page from disk.
+
+              cminflt %lu (11) The number of minor faults that the process's waited-for children have made.
+
+              majflt %lu  (12) The number of major faults the process has made which have required loading a memory page from disk.
+
+              cmajflt %lu (13) The number of major faults that the process's waited-for children have made.
+
+              utime %lu   (14) Amount of time that this process has been scheduled in user mode, measured in clock ticks (divide by sysconf(_SC_CLK_TCK)).  This includes guest time, guest_time (time spent running a virtual CPU, see below), so that applications that are not aware of the guest time field do not lose that time from their calculations.
+
+              stime %lu   (15) Amount of time that this process has been scheduled in kernel mode, measured in clock ticks (divide by sysconf(_SC_CLK_TCK)).
+ */
+
+    static char *loadfile = NULL;
+    static time_t last_call = 0;
+    static long ticks_per_s = 0;
+    static unsigned long last_utime, last_stime;
+
+    char buffer[64*1024];
+    FILE *stream = NULL;
+    time_t now = time(NULL);
+
+    if(load == NULL) {
+        return FALSE;
+    } else {
+        *load = 0.0;
+    }
+
+    if(loadfile == NULL) {
+        last_call = 0;
+        last_utime = 0;
+        last_stime = 0;
+        loadfile = find_cib_loadfile();
+        if (loadfile == NULL) {
+            crm_warn("Couldn't find CIB load file");
+            return FALSE;
+        }
+        ticks_per_s = sysconf(_SC_CLK_TCK);
+        crm_trace("Found %s", loadfile);
+    }
+
+    stream = fopen(loadfile, "r");
+    if(stream == NULL) {
+        int rc = errno;
+
+        crm_warn("Couldn't read %s: %s (%d)", loadfile, pcmk_strerror(rc), rc);
+        free(loadfile); loadfile = NULL;
+        return FALSE;
+    }
+
+    if(fgets(buffer, sizeof(buffer), stream)) {
+        char *comm = calloc(1, 256);
+        char state = 0;
+        int rc = 0, pid = 0, ppid = 0, pgrp = 0, session = 0, tty_nr = 0, tpgid = 0;
+        unsigned long flags = 0, minflt = 0, cminflt = 0, majflt = 0, cmajflt = 0, utime = 0, stime = 0;
+
+        rc = sscanf(buffer,  "%d %[^ ] %c %d %d %d %d %d %lu %lu %lu %lu %lu %lu %lu",
+                    &pid, comm, &state,
+                    &ppid, &pgrp, &session, &tty_nr, &tpgid,
+                    &flags, &minflt, &cminflt, &majflt, &cmajflt, &utime, &stime);
+        free(comm);
+
+        if(rc != 15) {
+            crm_err("Only %d of 15 fields found in %s", rc, loadfile);
+            fclose(stream);
+            return FALSE;
+
+        } else if(last_call > 0
+           && last_call < now
+           && last_utime <= utime
+           && last_stime <= stime) {
+
+            time_t elapsed = now - last_call;
+            unsigned long delta_utime = utime - last_utime;
+            unsigned long delta_stime = stime - last_stime;
+
+            *load = (delta_utime + delta_stime); /* Cast to a float before division */
+            *load /= ticks_per_s;
+            *load /= elapsed;
+            crm_debug("cib load: %f (%lu ticks in %lds)", *load, delta_utime + delta_stime, (long)elapsed);
+
+        } else {
+            crm_debug("Init %lu + %lu ticks at %ld (%lu tps)", utime, stime, (long)now, ticks_per_s);
+        }
+
+        last_call = now;
+        last_utime = utime;
+        last_stime = stime;
+
+        fclose(stream);
+        return TRUE;
+    }
+
+    fclose(stream);
+    return FALSE;
+}
+
+static bool
+throttle_load_avg(float *load)
+{
+    char buffer[256];
+    FILE *stream = NULL;
+    const char *loadfile = "/proc/loadavg";
+
+    if(load == NULL) {
+        return FALSE;
+    }
+
+    stream = fopen(loadfile, "r");
+    if(stream == NULL) {
+        int rc = errno;
+        crm_warn("Couldn't read %s: %s (%d)", loadfile, pcmk_strerror(rc), rc);
+        return FALSE;
+    }
+
+    if(fgets(buffer, sizeof(buffer), stream)) {
+        char *nl = strstr(buffer, "\n");
+
+        /* Grab the 1-minute average, ignore the rest */
+        *load = strtof(buffer, NULL);
+        if(nl) { nl[0] = 0; }
+
+        fclose(stream);
+        return TRUE;
+    }
+
+    fclose(stream);
+    return FALSE;
+}
+
+/*!
+ * \internal
+ * \brief Check a load value against throttling thresholds
+ *
+ * \param[in] load        Load value to check
+ * \param[in] desc        Description of metric (for logging)
+ * \param[in] thresholds  Low/medium/high/extreme thresholds
+ *
+ * \return Throttle mode corresponding to load value
+ */
+static enum throttle_state_e
+throttle_check_thresholds(float load, const char *desc,
+                          const float thresholds[4])
+{
+    if (load > thresholds[3]) {
+        crm_notice("Extreme %s detected: %f", desc, load);
+        return throttle_extreme;
+
+    } else if (load > thresholds[2]) {
+        crm_notice("High %s detected: %f", desc, load);
+        return throttle_high;
+
+    } else if (load > thresholds[1]) {
+        crm_info("Moderate %s detected: %f", desc, load);
+        return throttle_med;
+
+    } else if (load > thresholds[0]) {
+        crm_debug("Noticeable %s detected: %f", desc, load);
+        return throttle_low;
+    }
+
+    crm_trace("Negligible %s detected: %f", desc, load);
+    return throttle_none;
+}
+
+static enum throttle_state_e
+throttle_handle_load(float load, const char *desc, int cores)
+{
+    float normalize;
+    float thresholds[4];
+
+    if (cores == 1) {
+        /* On a single core machine, a load of 1.0 is already too high */
+        normalize = 0.6;
+
+    } else {
+        /* Normalize the load to be per-core */
+        normalize = cores;
+    }
+    thresholds[0] = throttle_load_target * normalize * THROTTLE_FACTOR_LOW;
+    thresholds[1] = throttle_load_target * normalize * THROTTLE_FACTOR_MEDIUM;
+    thresholds[2] = throttle_load_target * normalize * THROTTLE_FACTOR_HIGH;
+    thresholds[3] = load + 1.0; /* never extreme */
+
+    return throttle_check_thresholds(load, desc, thresholds);
+}
+#endif // HAVE_LINUX_PROCFS
+
+static enum throttle_state_e
+throttle_mode(void)
+{
+    enum throttle_state_e mode = throttle_none;
+
+#if HAVE_LINUX_PROCFS
+    unsigned int cores;
+    float load;
+    float thresholds[4];
+
+    cores = pcmk__procfs_num_cores();
+    if(throttle_cib_load(&load)) {
+        float cib_max_cpu = 0.95;
+
+        /* The CIB is a single-threaded task and thus cannot consume
+         * more than 100% of a CPU (and 1/cores of the overall system
+         * load).
+         *
+         * On a many-cored system, the CIB might therefore be maxed out
+         * (causing operations to fail or appear to fail) even though
+         * the overall system load is still reasonable.
+         *
+         * Therefore, the 'normal' thresholds can not apply here, and we
+         * need a special case.
+         */
+        if(cores == 1) {
+            cib_max_cpu = 0.4;
+        }
+        if(throttle_load_target > 0.0 && throttle_load_target < cib_max_cpu) {
+            cib_max_cpu = throttle_load_target;
+        }
+
+        thresholds[0] = cib_max_cpu * 0.8;
+        thresholds[1] = cib_max_cpu * 0.9;
+        thresholds[2] = cib_max_cpu;
+        /* Can only happen on machines with a low number of cores */
+        thresholds[3] = cib_max_cpu * 1.5;
+
+        mode = throttle_check_thresholds(load, "CIB load", thresholds);
+    }
+
+    if(throttle_load_target <= 0) {
+        /* If we ever make this a valid value, the cluster will at least behave as expected */
+        return mode;
+    }
+
+    if(throttle_load_avg(&load)) {
+        enum throttle_state_e cpu_load;
+
+        cpu_load = throttle_handle_load(load, "CPU load", cores);
+        if (cpu_load > mode) {
+            mode = cpu_load;
+        }
+        crm_debug("Current load is %f across %u core(s)", load, cores);
+    }
+#endif // HAVE_LINUX_PROCFS
+    return mode;
+}
+
+static void
+throttle_send_command(enum throttle_state_e mode)
+{
+    xmlNode *xml = NULL;
+    static enum throttle_state_e last = -1;
+
+    if(mode != last) {
+        crm_info("New throttle mode: %s load (was %s)",
+                 load2str(mode), load2str(last));
+        last = mode;
+
+        xml = create_request(CRM_OP_THROTTLE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
+        crm_xml_add_int(xml, F_CRM_THROTTLE_MODE, mode);
+        crm_xml_add_int(xml, F_CRM_THROTTLE_MAX, throttle_job_max);
+
+        send_cluster_message(NULL, crm_msg_crmd, xml, TRUE);
+        free_xml(xml);
+    }
+}
+
+static gboolean
+throttle_timer_cb(gpointer data)
+{
+    throttle_send_command(throttle_mode());
+    return TRUE;
+}
+
+static void
+throttle_record_free(gpointer p)
+{
+    struct throttle_record_s *r = p;
+    free(r->node);
+    free(r);
+}
+
+static void
+throttle_set_load_target(float target)
+{
+    throttle_load_target = target;
+}
+
+/*!
+ * \internal
+ * \brief Update the maximum number of simultaneous jobs
+ *
+ * \param[in] preference  Cluster-wide node-action-limit from the CIB
+ */
+static void
+throttle_update_job_max(const char *preference)
+{
+    long long max = 0LL;
+    const char *env_limit = getenv("PCMK_node_action_limit");
+
+    if (env_limit != NULL) {
+        preference = env_limit; // Per-node override
+    }
+    if (preference != NULL) {
+        pcmk__scan_ll(preference, &max, 0LL);
+    }
+    if (max > 0) {
+        throttle_job_max = (int) max;
+    } else {
+        // Default is based on the number of cores detected
+        throttle_job_max = 2 * pcmk__procfs_num_cores();
+    }
+}
+
+void
+throttle_init(void)
+{
+    if(throttle_records == NULL) {
+        throttle_records = pcmk__strkey_table(NULL, throttle_record_free);
+        throttle_timer = mainloop_timer_add("throttle", 30 * 1000, TRUE, throttle_timer_cb, NULL);
+    }
+
+    throttle_update_job_max(NULL);
+    mainloop_timer_start(throttle_timer);
+}
+
+/*!
+ * \internal
+ * \brief Configure throttle options based on the CIB
+ *
+ * \param[in,out] options  Name/value pairs for configured options
+ */
+void
+controld_configure_throttle(GHashTable *options)
+{
+    const char *value = g_hash_table_lookup(options, "load-threshold");
+
+    if (value != NULL) {
+        throttle_set_load_target(strtof(value, NULL) / 100.0);
+    }
+
+    value = g_hash_table_lookup(options, "node-action-limit");
+    throttle_update_job_max(value);
+}
+
+void
+throttle_fini(void)
+{
+    if (throttle_timer != NULL) {
+        mainloop_timer_del(throttle_timer);
+        throttle_timer = NULL;
+    }
+    if (throttle_records != NULL) {
+        g_hash_table_destroy(throttle_records);
+        throttle_records = NULL;
+    }
+}
+
+int
+throttle_get_total_job_limit(int l)
+{
+    /* Cluster-wide limit */
+    GHashTableIter iter;
+    int limit = l;
+    int peers = crm_active_peers();
+    struct throttle_record_s *r = NULL;
+
+    g_hash_table_iter_init(&iter, throttle_records);
+
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &r)) {
+        switch(r->mode) {
+
+            case throttle_extreme:
+                if(limit == 0 || limit > peers/4) {
+                    limit = QB_MAX(1, peers/4);
+                }
+                break;
+
+            case throttle_high:
+                if(limit == 0 || limit > peers/2) {
+                    limit = QB_MAX(1, peers/2);
+                }
+                break;
+            default:
+                break;
+        }
+    }
+    if(limit == l) {
+        /* crm_trace("No change to batch-limit=%d", limit); */
+
+    } else if(l == 0) {
+        crm_trace("Using batch-limit=%d", limit);
+
+    } else {
+        crm_trace("Using batch-limit=%d instead of %d", limit, l);
+    }
+    return limit;
+}
+
+int
+throttle_get_job_limit(const char *node)
+{
+    int jobs = 1;
+    struct throttle_record_s *r = NULL;
+
+    r = g_hash_table_lookup(throttle_records, node);
+    if(r == NULL) {
+        r = calloc(1, sizeof(struct throttle_record_s));
+        r->node = strdup(node);
+        r->mode = throttle_low;
+        r->max = throttle_job_max;
+        crm_trace("Defaulting to local values for unknown node %s", node);
+
+        g_hash_table_insert(throttle_records, r->node, r);
+    }
+
+    switch(r->mode) {
+        case throttle_extreme:
+        case throttle_high:
+            jobs = 1; /* At least one job must always be allowed */
+            break;
+        case throttle_med:
+            jobs = QB_MAX(1, r->max / 4);
+            break;
+        case throttle_low:
+            jobs = QB_MAX(1, r->max / 2);
+            break;
+        case throttle_none:
+            jobs = QB_MAX(1, r->max);
+            break;
+        default:
+            crm_err("Unknown throttle mode %.4x on %s", r->mode, node);
+            break;
+    }
+    return jobs;
+}
+
+void
+throttle_update(xmlNode *xml)
+{
+    int max = 0;
+    int mode = 0;
+    struct throttle_record_s *r = NULL;
+    const char *from = crm_element_value(xml, F_CRM_HOST_FROM);
+
+    crm_element_value_int(xml, F_CRM_THROTTLE_MODE, &mode);
+    crm_element_value_int(xml, F_CRM_THROTTLE_MAX, &max);
+
+    r = g_hash_table_lookup(throttle_records, from);
+
+    if(r == NULL) {
+        r = calloc(1, sizeof(struct throttle_record_s));
+        r->node = strdup(from);
+        g_hash_table_insert(throttle_records, r->node, r);
+    }
+
+    r->max = max;
+    r->mode = (enum throttle_state_e) mode;
+
+    crm_debug("Node %s has %s load and supports at most %d jobs; new job limit %d",
+              from, load2str((enum throttle_state_e) mode), max,
+              throttle_get_job_limit(from));
+}
diff --git a/daemons/controld/controld_throttle.h b/daemons/controld/controld_throttle.h
new file mode 100644
index 0000000..a798c6c
--- /dev/null
+++ b/daemons/controld/controld_throttle.h
@@ -0,0 +1,16 @@
+/*
+ * Copyright 2013-2021 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+void throttle_init(void);
+void throttle_fini(void);
+void controld_configure_throttle(GHashTable *options);
+
+void throttle_update(xmlNode *xml);
+int throttle_get_job_limit(const char *node);
+int throttle_get_total_job_limit(int l);
diff --git a/daemons/controld/controld_timers.c b/daemons/controld/controld_timers.c
new file mode 100644
index 0000000..a65bef5
--- /dev/null
+++ b/daemons/controld/controld_timers.c
@@ -0,0 +1,509 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <time.h>
+#include <stdlib.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <pacemaker-controld.h>
+
+//! FSA mainloop timer type
+typedef struct fsa_timer_s {
+    guint source_id;                        //!< Timer source ID
+    guint period_ms;                        //!< Timer period
+    enum crmd_fsa_input fsa_input;          //!< Input to register if timer pops
+    gboolean (*callback) (gpointer data);   //!< What do if timer pops
+    bool log_error;                         //!< Timer popping indicates error
+    int counter;                            //!< For detecting loops
+} fsa_timer_t;
+
+//! Wait before retrying a failed cib or executor connection
+static fsa_timer_t *wait_timer = NULL;
+
+//! Periodically re-run scheduler (for date_spec evaluation and as a failsafe)
+static fsa_timer_t *recheck_timer = NULL;
+
+//! Wait at start-up, or after an election, for DC to make contact
+static fsa_timer_t *election_timer = NULL;
+
+//! Delay start of new transition with expectation something else might happen
+static fsa_timer_t *transition_timer = NULL;
+
+//! join-integration-timeout
+static fsa_timer_t *integration_timer = NULL;
+
+//! join-finalization-timeout
+static fsa_timer_t *finalization_timer = NULL;
+
+// Wait for DC to stop all resources and give us the all-clear to shut down
+fsa_timer_t *shutdown_escalation_timer = NULL;
+
+//! Cluster recheck interval (from configuration)
+static guint recheck_interval_ms = 0;
+
+static const char *
+get_timer_desc(fsa_timer_t * timer)
+{
+    if (timer == election_timer) {
+        return "Election Trigger";
+
+    } else if (timer == shutdown_escalation_timer) {
+        return "Shutdown Escalation";
+
+    } else if (timer == integration_timer) {
+        return "Integration Timer";
+
+    } else if (timer == finalization_timer) {
+        return "Finalization Timer";
+
+    } else if (timer == transition_timer) {
+        return "New Transition Timer";
+
+    } else if (timer == wait_timer) {
+        return "Wait Timer";
+
+    } else if (timer == recheck_timer) {
+        return "Cluster Recheck Timer";
+
+    }
+    return "Unknown Timer";
+}
+
+/*!
+ * \internal
+ * \brief Stop an FSA timer
+ *
+ * \param[in,out] timer  Timer to stop
+ *
+ * \return true if the timer was running, or false otherwise
+ */
+static bool
+controld_stop_timer(fsa_timer_t *timer)
+{
+    CRM_CHECK(timer != NULL, return false);
+
+    if (timer->source_id != 0) {
+        crm_trace("Stopping %s (would inject %s if popped after %ums, src=%d)",
+                  get_timer_desc(timer), fsa_input2string(timer->fsa_input),
+                  timer->period_ms, timer->source_id);
+        g_source_remove(timer->source_id);
+        timer->source_id = 0;
+
+    } else {
+        crm_trace("%s already stopped (would inject %s if popped after %ums)",
+                  get_timer_desc(timer), fsa_input2string(timer->fsa_input),
+                  timer->period_ms);
+        return false;
+    }
+    return true;
+}
+
+/*!
+ * \internal
+ * \brief Start an FSA timer
+ *
+ * \param[in,out] timer  Timer to start
+ */
+static void
+controld_start_timer(fsa_timer_t *timer)
+{
+    if (timer->source_id == 0 && timer->period_ms > 0) {
+        timer->source_id = g_timeout_add(timer->period_ms, timer->callback, (void *)timer);
+        CRM_ASSERT(timer->source_id != 0);
+        crm_debug("Started %s (inject %s if pops after %ums, source=%d)",
+                  get_timer_desc(timer), fsa_input2string(timer->fsa_input),
+                  timer->period_ms, timer->source_id);
+    } else {
+        crm_debug("%s already running (inject %s if pops after %ums, source=%d)",
+                  get_timer_desc(timer), fsa_input2string(timer->fsa_input),
+                  timer->period_ms, timer->source_id);
+    }
+}
+
+/*	A_DC_TIMER_STOP, A_DC_TIMER_START,
+ *	A_FINALIZE_TIMER_STOP, A_FINALIZE_TIMER_START
+ *	A_INTEGRATE_TIMER_STOP, A_INTEGRATE_TIMER_START
+ */
+void
+do_timer_control(long long action,
+                 enum crmd_fsa_cause cause,
+                 enum crmd_fsa_state cur_state,
+                 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    gboolean timer_op_ok = TRUE;
+
+    if (action & A_DC_TIMER_STOP) {
+        timer_op_ok = controld_stop_timer(election_timer);
+
+    } else if (action & A_FINALIZE_TIMER_STOP) {
+        timer_op_ok = controld_stop_timer(finalization_timer);
+
+    } else if (action & A_INTEGRATE_TIMER_STOP) {
+        timer_op_ok = controld_stop_timer(integration_timer);
+    }
+
+    /* don't start a timer that wasn't already running */
+    if (action & A_DC_TIMER_START && timer_op_ok) {
+        controld_start_timer(election_timer);
+        if (AM_I_DC) {
+            /* there can be only one */
+            register_fsa_input(cause, I_ELECTION, NULL);
+        }
+
+    } else if (action & A_FINALIZE_TIMER_START) {
+        controld_start_timer(finalization_timer);
+
+    } else if (action & A_INTEGRATE_TIMER_START) {
+        controld_start_timer(integration_timer);
+    }
+}
+
+static gboolean
+crm_timer_popped(gpointer data)
+{
+    fsa_timer_t *timer = (fsa_timer_t *) data;
+
+    if (timer->log_error) {
+        crm_err("%s just popped in state %s! " CRM_XS " input=%s time=%ums",
+                get_timer_desc(timer),
+                fsa_state2string(controld_globals.fsa_state),
+                fsa_input2string(timer->fsa_input), timer->period_ms);
+    } else {
+        crm_info("%s just popped " CRM_XS " input=%s time=%ums",
+                 get_timer_desc(timer), fsa_input2string(timer->fsa_input),
+                 timer->period_ms);
+        timer->counter++;
+    }
+
+    if ((timer == election_timer) && (election_timer->counter > 5)) {
+        crm_notice("We appear to be in an election loop, something may be wrong");
+        crm_write_blackbox(0, NULL);
+        election_timer->counter = 0;
+    }
+
+    controld_stop_timer(timer);  // Make timer _not_ go off again
+
+    if (timer->fsa_input == I_INTEGRATED) {
+        crm_info("Welcomed: %d, Integrated: %d",
+                 crmd_join_phase_count(crm_join_welcomed),
+                 crmd_join_phase_count(crm_join_integrated));
+        if (crmd_join_phase_count(crm_join_welcomed) == 0) {
+            // If we don't even have ourselves, start again
+            register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, NULL,
+                                   __func__);
+
+        } else {
+            register_fsa_input_before(C_TIMER_POPPED, timer->fsa_input, NULL);
+        }
+
+    } else if ((timer == recheck_timer)
+               && (controld_globals.fsa_state != S_IDLE)) {
+        crm_debug("Discarding %s event in state: %s",
+                  fsa_input2string(timer->fsa_input),
+                  fsa_state2string(controld_globals.fsa_state));
+
+    } else if ((timer == finalization_timer)
+               && (controld_globals.fsa_state != S_FINALIZE_JOIN)) {
+        crm_debug("Discarding %s event in state: %s",
+                  fsa_input2string(timer->fsa_input),
+                  fsa_state2string(controld_globals.fsa_state));
+
+    } else if (timer->fsa_input != I_NULL) {
+        register_fsa_input(C_TIMER_POPPED, timer->fsa_input, NULL);
+    }
+
+    controld_trigger_fsa();
+
+    return TRUE;
+}
+
+bool
+controld_init_fsa_timers(void)
+{
+    transition_timer = calloc(1, sizeof(fsa_timer_t));
+    if (transition_timer == NULL) {
+        return FALSE;
+    }
+
+    integration_timer = calloc(1, sizeof(fsa_timer_t));
+    if (integration_timer == NULL) {
+        return FALSE;
+    }
+
+    finalization_timer = calloc(1, sizeof(fsa_timer_t));
+    if (finalization_timer == NULL) {
+        return FALSE;
+    }
+
+    election_timer = calloc(1, sizeof(fsa_timer_t));
+    if (election_timer == NULL) {
+        return FALSE;
+    }
+
+    shutdown_escalation_timer = calloc(1, sizeof(fsa_timer_t));
+    if (shutdown_escalation_timer == NULL) {
+        return FALSE;
+    }
+
+    wait_timer = calloc(1, sizeof(fsa_timer_t));
+    if (wait_timer == NULL) {
+        return FALSE;
+    }
+
+    recheck_timer = calloc(1, sizeof(fsa_timer_t));
+    if (recheck_timer == NULL) {
+        return FALSE;
+    }
+
+    election_timer->source_id = 0;
+    election_timer->period_ms = 0;
+    election_timer->fsa_input = I_DC_TIMEOUT;
+    election_timer->callback = crm_timer_popped;
+    election_timer->log_error = FALSE;
+
+    transition_timer->source_id = 0;
+    transition_timer->period_ms = 0;
+    transition_timer->fsa_input = I_PE_CALC;
+    transition_timer->callback = crm_timer_popped;
+    transition_timer->log_error = FALSE;
+
+    integration_timer->source_id = 0;
+    integration_timer->period_ms = 0;
+    integration_timer->fsa_input = I_INTEGRATED;
+    integration_timer->callback = crm_timer_popped;
+    integration_timer->log_error = TRUE;
+
+    finalization_timer->source_id = 0;
+    finalization_timer->period_ms = 0;
+    finalization_timer->fsa_input = I_FINALIZED;
+    finalization_timer->callback = crm_timer_popped;
+    finalization_timer->log_error = FALSE;
+
+    /* We can't use I_FINALIZED here, because that creates a bug in the join
+     * process where a joining node can be stuck in S_PENDING while we think it
+     * is in S_NOT_DC. This created an infinite transition loop in which we
+     * continually send probes which the node NACKs because it's pending.
+     *
+     * If we have nodes where the cluster layer is active but the controller is
+     * not, we can avoid this causing an election/join loop, in the integration
+     * phase.
+     */
+    finalization_timer->fsa_input = I_ELECTION;
+
+    shutdown_escalation_timer->source_id = 0;
+    shutdown_escalation_timer->period_ms = 0;
+    shutdown_escalation_timer->fsa_input = I_STOP;
+    shutdown_escalation_timer->callback = crm_timer_popped;
+    shutdown_escalation_timer->log_error = TRUE;
+
+    wait_timer->source_id = 0;
+    wait_timer->period_ms = 2000;
+    wait_timer->fsa_input = I_NULL;
+    wait_timer->callback = crm_timer_popped;
+    wait_timer->log_error = FALSE;
+
+    recheck_timer->source_id = 0;
+    recheck_timer->period_ms = 0;
+    recheck_timer->fsa_input = I_PE_CALC;
+    recheck_timer->callback = crm_timer_popped;
+    recheck_timer->log_error = FALSE;
+
+    return TRUE;
+}
+
+/*!
+ * \internal
+ * \brief Configure timers based on the CIB
+ *
+ * \param[in,out] options  Name/value pairs for configured options
+ */
+void
+controld_configure_fsa_timers(GHashTable *options)
+{
+    const char *value = NULL;
+
+    // Election timer
+    value = g_hash_table_lookup(options, XML_CONFIG_ATTR_DC_DEADTIME);
+    election_timer->period_ms = crm_parse_interval_spec(value);
+
+    // Integration timer
+    value = g_hash_table_lookup(options, "join-integration-timeout");
+    integration_timer->period_ms = crm_parse_interval_spec(value);
+
+    // Finalization timer
+    value = g_hash_table_lookup(options, "join-finalization-timeout");
+    finalization_timer->period_ms = crm_parse_interval_spec(value);
+
+    // Shutdown escalation timer
+    value = g_hash_table_lookup(options, XML_CONFIG_ATTR_FORCE_QUIT);
+    shutdown_escalation_timer->period_ms = crm_parse_interval_spec(value);
+    crm_debug("Shutdown escalation occurs if DC has not responded to request "
+              "in %ums", shutdown_escalation_timer->period_ms);
+
+    // Transition timer
+    value = g_hash_table_lookup(options, "transition-delay");
+    transition_timer->period_ms = crm_parse_interval_spec(value);
+
+    // Recheck interval
+    value = g_hash_table_lookup(options, XML_CONFIG_ATTR_RECHECK);
+    recheck_interval_ms = crm_parse_interval_spec(value);
+    crm_debug("Re-run scheduler after %dms of inactivity", recheck_interval_ms);
+}
+
+void
+controld_free_fsa_timers(void)
+{
+    controld_stop_timer(transition_timer);
+    controld_stop_timer(integration_timer);
+    controld_stop_timer(finalization_timer);
+    controld_stop_timer(election_timer);
+    controld_stop_timer(shutdown_escalation_timer);
+    controld_stop_timer(wait_timer);
+    controld_stop_timer(recheck_timer);
+
+    free(transition_timer); transition_timer = NULL;
+    free(integration_timer); integration_timer = NULL;
+    free(finalization_timer); finalization_timer = NULL;
+    free(election_timer); election_timer = NULL;
+    free(shutdown_escalation_timer); shutdown_escalation_timer = NULL;
+    free(wait_timer); wait_timer = NULL;
+    free(recheck_timer); recheck_timer = NULL;
+}
+
+/*!
+ * \internal
+ * \brief Check whether the transition timer is started
+ * \return true if the transition timer is started, or false otherwise
+ */
+bool
+controld_is_started_transition_timer(void)
+{
+    return (transition_timer->period_ms > 0)
+           && (transition_timer->source_id != 0);
+}
+
+/*!
+ * \internal
+ * \brief Start the recheck timer
+ */
+void
+controld_start_recheck_timer(void)
+{
+    // Default to recheck interval configured in CIB (if any)
+    guint period_ms = recheck_interval_ms;
+
+    // If scheduler supplied a "recheck by" time, check whether that's sooner
+    if (controld_globals.transition_graph->recheck_by > 0) {
+        time_t diff_seconds = controld_globals.transition_graph->recheck_by
+                              - time(NULL);
+
+        if (diff_seconds < 1) {
+            // We're already past the desired time
+            period_ms = 500;
+        } else {
+            period_ms = (guint) diff_seconds * 1000;
+        }
+
+        // Use "recheck by" only if it's sooner than interval from CIB
+        if (period_ms > recheck_interval_ms) {
+            period_ms = recheck_interval_ms;
+        }
+    }
+
+    if (period_ms > 0) {
+        recheck_timer->period_ms = period_ms;
+        controld_start_timer(recheck_timer);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Start the wait timer
+ */
+void
+controld_start_wait_timer(void)
+{
+    controld_start_timer(wait_timer);
+}
+
+/*!
+ * \internal
+ * \brief Stop the recheck timer
+ *
+ * \return true if the recheck timer was running, or false otherwise
+ */
+bool
+controld_stop_recheck_timer(void)
+{
+    return controld_stop_timer(recheck_timer);
+}
+
+/*!
+ * \brief Get the transition timer's configured period
+ * \return The transition_timer's period
+ */
+guint
+controld_get_period_transition_timer(void)
+{
+    return transition_timer->period_ms;
+}
+
+/*!
+ * \internal
+ * \brief Reset the election timer's counter to 0
+ */
+void
+controld_reset_counter_election_timer(void)
+{
+    election_timer->counter = 0;
+}
+
+/*!
+ * \internal
+ * \brief Stop the transition timer
+ *
+ * \return true if the transition timer was running, or false otherwise
+ */
+bool
+controld_stop_transition_timer(void)
+{
+    return controld_stop_timer(transition_timer);
+}
+
+/*!
+ * \internal
+ * \brief Start the transition timer
+ */
+void
+controld_start_transition_timer(void)
+{
+    controld_start_timer(transition_timer);
+}
+
+/*!
+ * \internal
+ * \brief Start the countdown sequence for a shutdown
+ *
+ * \param[in] default_period_ms  Period to use if the shutdown escalation
+ *                               timer's period is 0
+ */
+void
+controld_shutdown_start_countdown(guint default_period_ms)
+{
+    if (shutdown_escalation_timer->period_ms == 0) {
+        shutdown_escalation_timer->period_ms = default_period_ms;
+    }
+
+    crm_notice("Initiating controller shutdown sequence " CRM_XS " limit=%ums",
+               shutdown_escalation_timer->period_ms);
+    controld_start_timer(shutdown_escalation_timer);
+}
diff --git a/daemons/controld/controld_timers.h b/daemons/controld/controld_timers.h
new file mode 100644
index 0000000..587f4d1
--- /dev/null
+++ b/daemons/controld/controld_timers.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef CONTROLD_TIMERS__H
+#  define CONTROLD_TIMERS__H
+
+#  include <stdbool.h>              // bool
+#  include <glib.h>                 // gboolean, gpointer, guint
+#  include <controld_fsa.h>         // crmd_fsa_input
+
+bool controld_init_fsa_timers(void);
+void controld_free_fsa_timers(void);
+void controld_configure_fsa_timers(GHashTable *options);
+
+bool controld_stop_recheck_timer(void);
+bool controld_stop_transition_timer(void);
+
+void controld_start_recheck_timer(void);
+void controld_start_transition_timer(void);
+void controld_start_wait_timer(void);
+
+bool controld_is_started_transition_timer(void);
+
+guint controld_get_period_transition_timer(void);
+
+void controld_reset_counter_election_timer(void);
+
+void controld_shutdown_start_countdown(guint default_period_ms);
+
+#endif
diff --git a/daemons/controld/controld_transition.c b/daemons/controld/controld_transition.c
new file mode 100644
index 0000000..c8a342c
--- /dev/null
+++ b/daemons/controld/controld_transition.c
@@ -0,0 +1,197 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+
+#include <pacemaker-controld.h>
+
+static void
+global_cib_callback(const xmlNode * msg, int callid, int rc, xmlNode * output)
+{
+}
+
+static pcmk__graph_t *
+create_blank_graph(void)
+{
+    pcmk__graph_t *a_graph = pcmk__unpack_graph(NULL, NULL);
+
+    a_graph->complete = true;
+    a_graph->abort_reason = "DC Takeover";
+    a_graph->completion_action = pcmk__graph_restart;
+    return a_graph;
+}
+
+/*	 A_TE_START, A_TE_STOP, O_TE_RESTART	*/
+void
+do_te_control(long long action,
+              enum crmd_fsa_cause cause,
+              enum crmd_fsa_state cur_state,
+              enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    cib_t *cib_conn = controld_globals.cib_conn;
+    gboolean init_ok = TRUE;
+
+    if (pcmk_is_set(action, A_TE_STOP)) {
+        pcmk__free_graph(controld_globals.transition_graph);
+        controld_globals.transition_graph = NULL;
+
+        if (cib_conn != NULL) {
+            cib_conn->cmds->del_notify_callback(cib_conn, T_CIB_DIFF_NOTIFY,
+                                                te_update_diff);
+        }
+
+        controld_clear_fsa_input_flags(R_TE_CONNECTED);
+        crm_info("Transitioner is now inactive");
+    }
+
+    if ((action & A_TE_START) == 0) {
+        return;
+
+    } else if (pcmk_is_set(controld_globals.fsa_input_register,
+                           R_TE_CONNECTED)) {
+        crm_debug("The transitioner is already active");
+        return;
+
+    } else if ((action & A_TE_START) && cur_state == S_STOPPING) {
+        crm_info("Ignoring request to start the transitioner while shutting down");
+        return;
+    }
+
+    if (controld_globals.te_uuid == NULL) {
+        controld_globals.te_uuid = crm_generate_uuid();
+        crm_info("Registering TE UUID: %s", controld_globals.te_uuid);
+    }
+
+    if (cib_conn == NULL) {
+        crm_err("Could not set CIB callbacks");
+        init_ok = FALSE;
+
+    } else {
+        if (cib_conn->cmds->add_notify_callback(cib_conn, T_CIB_DIFF_NOTIFY,
+                                                te_update_diff) != pcmk_ok) {
+            crm_err("Could not set CIB notification callback");
+            init_ok = FALSE;
+        }
+
+        if (cib_conn->cmds->set_op_callback(cib_conn,
+                                            global_cib_callback) != pcmk_ok) {
+            crm_err("Could not set CIB global callback");
+            init_ok = FALSE;
+        }
+    }
+
+    if (init_ok) {
+        controld_register_graph_functions();
+        pcmk__free_graph(controld_globals.transition_graph);
+
+        /* create a blank one */
+        crm_debug("Transitioner is now active");
+        controld_globals.transition_graph = create_blank_graph();
+        controld_set_fsa_input_flags(R_TE_CONNECTED);
+    }
+}
+
+/*	 A_TE_INVOKE, A_TE_CANCEL	*/
+void
+do_te_invoke(long long action,
+             enum crmd_fsa_cause cause,
+             enum crmd_fsa_state cur_state,
+             enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+
+    if (!AM_I_DC
+        || ((controld_globals.fsa_state != S_TRANSITION_ENGINE)
+            && pcmk_is_set(action, A_TE_INVOKE))) {
+        crm_notice("No need to invoke the TE (%s) in state %s",
+                   fsa_action2string(action),
+                   fsa_state2string(controld_globals.fsa_state));
+        return;
+    }
+
+    if (action & A_TE_CANCEL) {
+        crm_debug("Cancelling the transition: %sactive",
+                  controld_globals.transition_graph->complete? "in" : "");
+        abort_transition(INFINITY, pcmk__graph_restart, "Peer Cancelled", NULL);
+        if (!controld_globals.transition_graph->complete) {
+            crmd_fsa_stall(FALSE);
+        }
+
+    } else if (action & A_TE_HALT) {
+        abort_transition(INFINITY, pcmk__graph_wait, "Peer Halt", NULL);
+        if (!controld_globals.transition_graph->complete) {
+            crmd_fsa_stall(FALSE);
+        }
+
+    } else if (action & A_TE_INVOKE) {
+        ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
+        xmlNode *graph_data = input->xml;
+        const char *ref = crm_element_value(input->msg, XML_ATTR_REFERENCE);
+        const char *graph_file = crm_element_value(input->msg, F_CRM_TGRAPH);
+        const char *graph_input = crm_element_value(input->msg, F_CRM_TGRAPH_INPUT);
+
+        if (graph_file == NULL && graph_data == NULL) {
+            crm_log_xml_err(input->msg, "Bad command");
+            register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
+            return;
+        }
+
+        if (!controld_globals.transition_graph->complete) {
+            crm_info("Another transition is already active");
+            abort_transition(INFINITY, pcmk__graph_restart, "Transition Active",
+                             NULL);
+            return;
+        }
+
+        if ((controld_globals.fsa_pe_ref == NULL)
+            || !pcmk__str_eq(controld_globals.fsa_pe_ref, ref,
+                             pcmk__str_none)) {
+            crm_info("Transition is redundant: %s expected but %s received",
+                     pcmk__s(controld_globals.fsa_pe_ref, "no reference"),
+                     pcmk__s(ref, "no reference"));
+            abort_transition(INFINITY, pcmk__graph_restart,
+                             "Transition Redundant", NULL);
+        }
+
+        if (graph_data == NULL && graph_file != NULL) {
+            graph_data = filename2xml(graph_file);
+        }
+
+        if (controld_is_started_transition_timer()) {
+            crm_debug("The transitioner wait for a transition timer");
+            return;
+        }
+
+        CRM_CHECK(graph_data != NULL,
+                  crm_err("Input raised by %s is invalid", msg_data->origin);
+                  crm_log_xml_err(input->msg, "Bad command");
+                  return);
+
+        pcmk__free_graph(controld_globals.transition_graph);
+        controld_globals.transition_graph = pcmk__unpack_graph(graph_data,
+                                                               graph_input);
+        CRM_CHECK(controld_globals.transition_graph != NULL,
+                  controld_globals.transition_graph = create_blank_graph();
+                  return);
+        crm_info("Processing graph %d (ref=%s) derived from %s",
+                 controld_globals.transition_graph->id, ref, graph_input);
+
+        te_reset_job_counts();
+
+        trigger_graph();
+        pcmk__log_graph(LOG_TRACE, controld_globals.transition_graph);
+
+        if (graph_data != input->xml) {
+            free_xml(graph_data);
+        }
+    }
+}
diff --git a/daemons/controld/controld_transition.h b/daemons/controld/controld_transition.h
new file mode 100644
index 0000000..2da4221
--- /dev/null
+++ b/daemons/controld/controld_transition.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef TENGINE__H
+#  define TENGINE__H
+
+#  include <crm/common/mainloop.h>
+#  include <crm/stonith-ng.h>
+#  include <crm/services.h>
+#  include <pacemaker-internal.h>
+
+/* tengine */
+pcmk__graph_action_t *match_down_event(const char *target);
+pcmk__graph_action_t *get_cancel_action(const char *id, const char *node);
+bool confirm_cancel_action(const char *id, const char *node_id);
+
+void controld_record_action_timeout(pcmk__graph_action_t *action);
+
+void controld_destroy_outside_events_table(void);
+void controld_remove_all_outside_events(void);
+
+gboolean fail_incompletable_actions(pcmk__graph_t *graph, const char *down_node);
+void process_graph_event(xmlNode *event, const char *event_node);
+
+/* utils */
+pcmk__graph_action_t *controld_get_action(int id);
+gboolean stop_te_timer(pcmk__graph_action_t *action);
+const char *get_rsc_state(const char *task, enum pcmk_exec_status status);
+
+void process_te_message(xmlNode *msg, xmlNode *xml_data);
+
+void controld_register_graph_functions(void);
+
+void notify_crmd(pcmk__graph_t * graph);
+
+void cib_action_updated(xmlNode *msg, int call_id, int rc, xmlNode *output,
+                        void *user_data);
+gboolean action_timer_callback(gpointer data);
+void te_update_diff(const char *event, xmlNode *msg);
+
+void controld_init_transition_trigger(void);
+void controld_destroy_transition_trigger(void);
+
+void controld_trigger_graph_as(const char *fn, int line);
+void abort_after_delay(int abort_priority, enum pcmk__graph_next abort_action,
+                       const char *abort_text, guint delay_ms);
+void abort_transition_graph(int abort_priority,
+                            enum pcmk__graph_next abort_action,
+                            const char *abort_text, const xmlNode *reason,
+                            const char *fn, int line);
+
+#  define trigger_graph()   controld_trigger_graph_as(__func__, __LINE__)
+#  define abort_transition(pri, action, text, reason)			\
+	abort_transition_graph(pri, action, text, reason,__func__,__LINE__);
+
+void te_action_confirmed(pcmk__graph_action_t *action, pcmk__graph_t *graph);
+void te_reset_job_counts(void);
+
+#endif
diff --git a/daemons/controld/controld_utils.c b/daemons/controld/controld_utils.c
new file mode 100644
index 0000000..4ce09d9
--- /dev/null
+++ b/daemons/controld/controld_utils.c
@@ -0,0 +1,837 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdlib.h>
+#include <stdint.h>                 // uint64_t
+
+#include <crm/crm.h>
+#include <crm/cib.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+
+#include <pacemaker-controld.h>
+
+const char *
+fsa_input2string(enum crmd_fsa_input input)
+{
+    const char *inputAsText = NULL;
+
+    switch (input) {
+        case I_NULL:
+            inputAsText = "I_NULL";
+            break;
+        case I_CIB_OP:
+            inputAsText = "I_CIB_OP (unused)";
+            break;
+        case I_CIB_UPDATE:
+            inputAsText = "I_CIB_UPDATE";
+            break;
+        case I_DC_TIMEOUT:
+            inputAsText = "I_DC_TIMEOUT";
+            break;
+        case I_ELECTION:
+            inputAsText = "I_ELECTION";
+            break;
+        case I_PE_CALC:
+            inputAsText = "I_PE_CALC";
+            break;
+        case I_RELEASE_DC:
+            inputAsText = "I_RELEASE_DC";
+            break;
+        case I_ELECTION_DC:
+            inputAsText = "I_ELECTION_DC";
+            break;
+        case I_ERROR:
+            inputAsText = "I_ERROR";
+            break;
+        case I_FAIL:
+            inputAsText = "I_FAIL";
+            break;
+        case I_INTEGRATED:
+            inputAsText = "I_INTEGRATED";
+            break;
+        case I_FINALIZED:
+            inputAsText = "I_FINALIZED";
+            break;
+        case I_NODE_JOIN:
+            inputAsText = "I_NODE_JOIN";
+            break;
+        case I_JOIN_OFFER:
+            inputAsText = "I_JOIN_OFFER";
+            break;
+        case I_JOIN_REQUEST:
+            inputAsText = "I_JOIN_REQUEST";
+            break;
+        case I_JOIN_RESULT:
+            inputAsText = "I_JOIN_RESULT";
+            break;
+        case I_NOT_DC:
+            inputAsText = "I_NOT_DC";
+            break;
+        case I_RECOVERED:
+            inputAsText = "I_RECOVERED";
+            break;
+        case I_RELEASE_FAIL:
+            inputAsText = "I_RELEASE_FAIL";
+            break;
+        case I_RELEASE_SUCCESS:
+            inputAsText = "I_RELEASE_SUCCESS";
+            break;
+        case I_RESTART:
+            inputAsText = "I_RESTART";
+            break;
+        case I_PE_SUCCESS:
+            inputAsText = "I_PE_SUCCESS";
+            break;
+        case I_ROUTER:
+            inputAsText = "I_ROUTER";
+            break;
+        case I_SHUTDOWN:
+            inputAsText = "I_SHUTDOWN";
+            break;
+        case I_STARTUP:
+            inputAsText = "I_STARTUP";
+            break;
+        case I_TE_SUCCESS:
+            inputAsText = "I_TE_SUCCESS";
+            break;
+        case I_STOP:
+            inputAsText = "I_STOP";
+            break;
+        case I_DC_HEARTBEAT:
+            inputAsText = "I_DC_HEARTBEAT";
+            break;
+        case I_WAIT_FOR_EVENT:
+            inputAsText = "I_WAIT_FOR_EVENT";
+            break;
+        case I_LRM_EVENT:
+            inputAsText = "I_LRM_EVENT";
+            break;
+        case I_PENDING:
+            inputAsText = "I_PENDING";
+            break;
+        case I_HALT:
+            inputAsText = "I_HALT";
+            break;
+        case I_TERMINATE:
+            inputAsText = "I_TERMINATE";
+            break;
+        case I_ILLEGAL:
+            inputAsText = "I_ILLEGAL";
+            break;
+    }
+
+    if (inputAsText == NULL) {
+        crm_err("Input %d is unknown", input);
+        inputAsText = "<UNKNOWN_INPUT>";
+    }
+
+    return inputAsText;
+}
+
+const char *
+fsa_state2string(enum crmd_fsa_state state)
+{
+    const char *stateAsText = NULL;
+
+    switch (state) {
+        case S_IDLE:
+            stateAsText = "S_IDLE";
+            break;
+        case S_ELECTION:
+            stateAsText = "S_ELECTION";
+            break;
+        case S_INTEGRATION:
+            stateAsText = "S_INTEGRATION";
+            break;
+        case S_FINALIZE_JOIN:
+            stateAsText = "S_FINALIZE_JOIN";
+            break;
+        case S_NOT_DC:
+            stateAsText = "S_NOT_DC";
+            break;
+        case S_POLICY_ENGINE:
+            stateAsText = "S_POLICY_ENGINE";
+            break;
+        case S_RECOVERY:
+            stateAsText = "S_RECOVERY";
+            break;
+        case S_RELEASE_DC:
+            stateAsText = "S_RELEASE_DC";
+            break;
+        case S_PENDING:
+            stateAsText = "S_PENDING";
+            break;
+        case S_STOPPING:
+            stateAsText = "S_STOPPING";
+            break;
+        case S_TERMINATE:
+            stateAsText = "S_TERMINATE";
+            break;
+        case S_TRANSITION_ENGINE:
+            stateAsText = "S_TRANSITION_ENGINE";
+            break;
+        case S_STARTING:
+            stateAsText = "S_STARTING";
+            break;
+        case S_HALT:
+            stateAsText = "S_HALT";
+            break;
+        case S_ILLEGAL:
+            stateAsText = "S_ILLEGAL";
+            break;
+    }
+
+    if (stateAsText == NULL) {
+        crm_err("State %d is unknown", state);
+        stateAsText = "<UNKNOWN_STATE>";
+    }
+
+    return stateAsText;
+}
+
+const char *
+fsa_cause2string(enum crmd_fsa_cause cause)
+{
+    const char *causeAsText = NULL;
+
+    switch (cause) {
+        case C_UNKNOWN:
+            causeAsText = "C_UNKNOWN";
+            break;
+        case C_STARTUP:
+            causeAsText = "C_STARTUP";
+            break;
+        case C_IPC_MESSAGE:
+            causeAsText = "C_IPC_MESSAGE";
+            break;
+        case C_HA_MESSAGE:
+            causeAsText = "C_HA_MESSAGE";
+            break;
+        case C_TIMER_POPPED:
+            causeAsText = "C_TIMER_POPPED";
+            break;
+        case C_SHUTDOWN:
+            causeAsText = "C_SHUTDOWN";
+            break;
+        case C_LRM_OP_CALLBACK:
+            causeAsText = "C_LRM_OP_CALLBACK";
+            break;
+        case C_CRMD_STATUS_CALLBACK:
+            causeAsText = "C_CRMD_STATUS_CALLBACK";
+            break;
+        case C_FSA_INTERNAL:
+            causeAsText = "C_FSA_INTERNAL";
+            break;
+    }
+
+    if (causeAsText == NULL) {
+        crm_err("Cause %d is unknown", cause);
+        causeAsText = "<UNKNOWN_CAUSE>";
+    }
+
+    return causeAsText;
+}
+
+const char *
+fsa_action2string(long long action)
+{
+    const char *actionAsText = NULL;
+
+    switch (action) {
+
+        case A_NOTHING:
+            actionAsText = "A_NOTHING";
+            break;
+        case A_ELECTION_START:
+            actionAsText = "A_ELECTION_START";
+            break;
+        case A_DC_JOIN_FINAL:
+            actionAsText = "A_DC_JOIN_FINAL";
+            break;
+        case A_READCONFIG:
+            actionAsText = "A_READCONFIG";
+            break;
+        case O_RELEASE:
+            actionAsText = "O_RELEASE";
+            break;
+        case A_STARTUP:
+            actionAsText = "A_STARTUP";
+            break;
+        case A_STARTED:
+            actionAsText = "A_STARTED";
+            break;
+        case A_HA_CONNECT:
+            actionAsText = "A_HA_CONNECT";
+            break;
+        case A_HA_DISCONNECT:
+            actionAsText = "A_HA_DISCONNECT";
+            break;
+        case A_LRM_CONNECT:
+            actionAsText = "A_LRM_CONNECT";
+            break;
+        case A_LRM_EVENT:
+            actionAsText = "A_LRM_EVENT";
+            break;
+        case A_LRM_INVOKE:
+            actionAsText = "A_LRM_INVOKE";
+            break;
+        case A_LRM_DISCONNECT:
+            actionAsText = "A_LRM_DISCONNECT";
+            break;
+        case O_LRM_RECONNECT:
+            actionAsText = "O_LRM_RECONNECT";
+            break;
+        case A_CL_JOIN_QUERY:
+            actionAsText = "A_CL_JOIN_QUERY";
+            break;
+        case A_DC_TIMER_STOP:
+            actionAsText = "A_DC_TIMER_STOP";
+            break;
+        case A_DC_TIMER_START:
+            actionAsText = "A_DC_TIMER_START";
+            break;
+        case A_INTEGRATE_TIMER_START:
+            actionAsText = "A_INTEGRATE_TIMER_START";
+            break;
+        case A_INTEGRATE_TIMER_STOP:
+            actionAsText = "A_INTEGRATE_TIMER_STOP";
+            break;
+        case A_FINALIZE_TIMER_START:
+            actionAsText = "A_FINALIZE_TIMER_START";
+            break;
+        case A_FINALIZE_TIMER_STOP:
+            actionAsText = "A_FINALIZE_TIMER_STOP";
+            break;
+        case A_ELECTION_COUNT:
+            actionAsText = "A_ELECTION_COUNT";
+            break;
+        case A_ELECTION_VOTE:
+            actionAsText = "A_ELECTION_VOTE";
+            break;
+        case A_ELECTION_CHECK:
+            actionAsText = "A_ELECTION_CHECK";
+            break;
+        case A_CL_JOIN_ANNOUNCE:
+            actionAsText = "A_CL_JOIN_ANNOUNCE";
+            break;
+        case A_CL_JOIN_REQUEST:
+            actionAsText = "A_CL_JOIN_REQUEST";
+            break;
+        case A_CL_JOIN_RESULT:
+            actionAsText = "A_CL_JOIN_RESULT";
+            break;
+        case A_DC_JOIN_OFFER_ALL:
+            actionAsText = "A_DC_JOIN_OFFER_ALL";
+            break;
+        case A_DC_JOIN_OFFER_ONE:
+            actionAsText = "A_DC_JOIN_OFFER_ONE";
+            break;
+        case A_DC_JOIN_PROCESS_REQ:
+            actionAsText = "A_DC_JOIN_PROCESS_REQ";
+            break;
+        case A_DC_JOIN_PROCESS_ACK:
+            actionAsText = "A_DC_JOIN_PROCESS_ACK";
+            break;
+        case A_DC_JOIN_FINALIZE:
+            actionAsText = "A_DC_JOIN_FINALIZE";
+            break;
+        case A_MSG_PROCESS:
+            actionAsText = "A_MSG_PROCESS";
+            break;
+        case A_MSG_ROUTE:
+            actionAsText = "A_MSG_ROUTE";
+            break;
+        case A_RECOVER:
+            actionAsText = "A_RECOVER";
+            break;
+        case A_DC_RELEASE:
+            actionAsText = "A_DC_RELEASE";
+            break;
+        case A_DC_RELEASED:
+            actionAsText = "A_DC_RELEASED";
+            break;
+        case A_DC_TAKEOVER:
+            actionAsText = "A_DC_TAKEOVER";
+            break;
+        case A_SHUTDOWN:
+            actionAsText = "A_SHUTDOWN";
+            break;
+        case A_SHUTDOWN_REQ:
+            actionAsText = "A_SHUTDOWN_REQ";
+            break;
+        case A_STOP:
+            actionAsText = "A_STOP  ";
+            break;
+        case A_EXIT_0:
+            actionAsText = "A_EXIT_0";
+            break;
+        case A_EXIT_1:
+            actionAsText = "A_EXIT_1";
+            break;
+        case O_CIB_RESTART:
+            actionAsText = "O_CIB_RESTART";
+            break;
+        case A_CIB_START:
+            actionAsText = "A_CIB_START";
+            break;
+        case A_CIB_STOP:
+            actionAsText = "A_CIB_STOP";
+            break;
+        case A_TE_INVOKE:
+            actionAsText = "A_TE_INVOKE";
+            break;
+        case O_TE_RESTART:
+            actionAsText = "O_TE_RESTART";
+            break;
+        case A_TE_START:
+            actionAsText = "A_TE_START";
+            break;
+        case A_TE_STOP:
+            actionAsText = "A_TE_STOP";
+            break;
+        case A_TE_HALT:
+            actionAsText = "A_TE_HALT";
+            break;
+        case A_TE_CANCEL:
+            actionAsText = "A_TE_CANCEL";
+            break;
+        case A_PE_INVOKE:
+            actionAsText = "A_PE_INVOKE";
+            break;
+        case O_PE_RESTART:
+            actionAsText = "O_PE_RESTART";
+            break;
+        case A_PE_START:
+            actionAsText = "A_PE_START";
+            break;
+        case A_PE_STOP:
+            actionAsText = "A_PE_STOP";
+            break;
+        case A_NODE_BLOCK:
+            actionAsText = "A_NODE_BLOCK";
+            break;
+        case A_UPDATE_NODESTATUS:
+            actionAsText = "A_UPDATE_NODESTATUS";
+            break;
+        case A_LOG:
+            actionAsText = "A_LOG   ";
+            break;
+        case A_ERROR:
+            actionAsText = "A_ERROR ";
+            break;
+        case A_WARN:
+            actionAsText = "A_WARN  ";
+            break;
+            /* Composite actions */
+        case A_DC_TIMER_START | A_CL_JOIN_QUERY:
+            actionAsText = "A_DC_TIMER_START|A_CL_JOIN_QUERY";
+            break;
+    }
+
+    if (actionAsText == NULL) {
+        crm_err("Action %.16llx is unknown", action);
+        actionAsText = "<UNKNOWN_ACTION>";
+    }
+
+    return actionAsText;
+}
+
+void
+fsa_dump_inputs(int log_level, const char *text, long long input_register)
+{
+    if (input_register == A_NOTHING) {
+        return;
+    }
+    if (text == NULL) {
+        text = "Input register contents:";
+    }
+
+    if (pcmk_is_set(input_register, R_THE_DC)) {
+        crm_trace("%s %.16llx (R_THE_DC)", text, R_THE_DC);
+    }
+    if (pcmk_is_set(input_register, R_STARTING)) {
+        crm_trace("%s %.16llx (R_STARTING)", text, R_STARTING);
+    }
+    if (pcmk_is_set(input_register, R_SHUTDOWN)) {
+        crm_trace("%s %.16llx (R_SHUTDOWN)", text, R_SHUTDOWN);
+    }
+    if (pcmk_is_set(input_register, R_STAYDOWN)) {
+        crm_trace("%s %.16llx (R_STAYDOWN)", text, R_STAYDOWN);
+    }
+    if (pcmk_is_set(input_register, R_JOIN_OK)) {
+        crm_trace("%s %.16llx (R_JOIN_OK)", text, R_JOIN_OK);
+    }
+    if (pcmk_is_set(input_register, R_READ_CONFIG)) {
+        crm_trace("%s %.16llx (R_READ_CONFIG)", text, R_READ_CONFIG);
+    }
+    if (pcmk_is_set(input_register, R_INVOKE_PE)) {
+        crm_trace("%s %.16llx (R_INVOKE_PE)", text, R_INVOKE_PE);
+    }
+    if (pcmk_is_set(input_register, R_CIB_CONNECTED)) {
+        crm_trace("%s %.16llx (R_CIB_CONNECTED)", text, R_CIB_CONNECTED);
+    }
+    if (pcmk_is_set(input_register, R_PE_CONNECTED)) {
+        crm_trace("%s %.16llx (R_PE_CONNECTED)", text, R_PE_CONNECTED);
+    }
+    if (pcmk_is_set(input_register, R_TE_CONNECTED)) {
+        crm_trace("%s %.16llx (R_TE_CONNECTED)", text, R_TE_CONNECTED);
+    }
+    if (pcmk_is_set(input_register, R_LRM_CONNECTED)) {
+        crm_trace("%s %.16llx (R_LRM_CONNECTED)", text, R_LRM_CONNECTED);
+    }
+    if (pcmk_is_set(input_register, R_CIB_REQUIRED)) {
+        crm_trace("%s %.16llx (R_CIB_REQUIRED)", text, R_CIB_REQUIRED);
+    }
+    if (pcmk_is_set(input_register, R_PE_REQUIRED)) {
+        crm_trace("%s %.16llx (R_PE_REQUIRED)", text, R_PE_REQUIRED);
+    }
+    if (pcmk_is_set(input_register, R_TE_REQUIRED)) {
+        crm_trace("%s %.16llx (R_TE_REQUIRED)", text, R_TE_REQUIRED);
+    }
+    if (pcmk_is_set(input_register, R_REQ_PEND)) {
+        crm_trace("%s %.16llx (R_REQ_PEND)", text, R_REQ_PEND);
+    }
+    if (pcmk_is_set(input_register, R_PE_PEND)) {
+        crm_trace("%s %.16llx (R_PE_PEND)", text, R_PE_PEND);
+    }
+    if (pcmk_is_set(input_register, R_TE_PEND)) {
+        crm_trace("%s %.16llx (R_TE_PEND)", text, R_TE_PEND);
+    }
+    if (pcmk_is_set(input_register, R_RESP_PEND)) {
+        crm_trace("%s %.16llx (R_RESP_PEND)", text, R_RESP_PEND);
+    }
+    if (pcmk_is_set(input_register, R_CIB_DONE)) {
+        crm_trace("%s %.16llx (R_CIB_DONE)", text, R_CIB_DONE);
+    }
+    if (pcmk_is_set(input_register, R_HAVE_CIB)) {
+        crm_trace("%s %.16llx (R_HAVE_CIB)", text, R_HAVE_CIB);
+    }
+    if (pcmk_is_set(input_register, R_MEMBERSHIP)) {
+        crm_trace("%s %.16llx (R_MEMBERSHIP)", text, R_MEMBERSHIP);
+    }
+    if (pcmk_is_set(input_register, R_PEER_DATA)) {
+        crm_trace("%s %.16llx (R_PEER_DATA)", text, R_PEER_DATA);
+    }
+    if (pcmk_is_set(input_register, R_IN_RECOVERY)) {
+        crm_trace("%s %.16llx (R_IN_RECOVERY)", text, R_IN_RECOVERY);
+    }
+}
+
+void
+fsa_dump_actions(uint64_t action, const char *text)
+{
+    if (pcmk_is_set(action, A_READCONFIG)) {
+        crm_trace("Action %.16llx (A_READCONFIG) %s", A_READCONFIG, text);
+    }
+    if (pcmk_is_set(action, A_STARTUP)) {
+        crm_trace("Action %.16llx (A_STARTUP) %s", A_STARTUP, text);
+    }
+    if (pcmk_is_set(action, A_STARTED)) {
+        crm_trace("Action %.16llx (A_STARTED) %s", A_STARTED, text);
+    }
+    if (pcmk_is_set(action, A_HA_CONNECT)) {
+        crm_trace("Action %.16llx (A_CONNECT) %s", A_HA_CONNECT, text);
+    }
+    if (pcmk_is_set(action, A_HA_DISCONNECT)) {
+        crm_trace("Action %.16llx (A_DISCONNECT) %s", A_HA_DISCONNECT, text);
+    }
+    if (pcmk_is_set(action, A_LRM_CONNECT)) {
+        crm_trace("Action %.16llx (A_LRM_CONNECT) %s", A_LRM_CONNECT, text);
+    }
+    if (pcmk_is_set(action, A_LRM_EVENT)) {
+        crm_trace("Action %.16llx (A_LRM_EVENT) %s", A_LRM_EVENT, text);
+    }
+    if (pcmk_is_set(action, A_LRM_INVOKE)) {
+        crm_trace("Action %.16llx (A_LRM_INVOKE) %s", A_LRM_INVOKE, text);
+    }
+    if (pcmk_is_set(action, A_LRM_DISCONNECT)) {
+        crm_trace("Action %.16llx (A_LRM_DISCONNECT) %s", A_LRM_DISCONNECT, text);
+    }
+    if (pcmk_is_set(action, A_DC_TIMER_STOP)) {
+        crm_trace("Action %.16llx (A_DC_TIMER_STOP) %s", A_DC_TIMER_STOP, text);
+    }
+    if (pcmk_is_set(action, A_DC_TIMER_START)) {
+        crm_trace("Action %.16llx (A_DC_TIMER_START) %s", A_DC_TIMER_START, text);
+    }
+    if (pcmk_is_set(action, A_INTEGRATE_TIMER_START)) {
+        crm_trace("Action %.16llx (A_INTEGRATE_TIMER_START) %s", A_INTEGRATE_TIMER_START, text);
+    }
+    if (pcmk_is_set(action, A_INTEGRATE_TIMER_STOP)) {
+        crm_trace("Action %.16llx (A_INTEGRATE_TIMER_STOP) %s", A_INTEGRATE_TIMER_STOP, text);
+    }
+    if (pcmk_is_set(action, A_FINALIZE_TIMER_START)) {
+        crm_trace("Action %.16llx (A_FINALIZE_TIMER_START) %s", A_FINALIZE_TIMER_START, text);
+    }
+    if (pcmk_is_set(action, A_FINALIZE_TIMER_STOP)) {
+        crm_trace("Action %.16llx (A_FINALIZE_TIMER_STOP) %s", A_FINALIZE_TIMER_STOP, text);
+    }
+    if (pcmk_is_set(action, A_ELECTION_COUNT)) {
+        crm_trace("Action %.16llx (A_ELECTION_COUNT) %s", A_ELECTION_COUNT, text);
+    }
+    if (pcmk_is_set(action, A_ELECTION_VOTE)) {
+        crm_trace("Action %.16llx (A_ELECTION_VOTE) %s", A_ELECTION_VOTE, text);
+    }
+    if (pcmk_is_set(action, A_ELECTION_CHECK)) {
+        crm_trace("Action %.16llx (A_ELECTION_CHECK) %s", A_ELECTION_CHECK, text);
+    }
+    if (pcmk_is_set(action, A_CL_JOIN_ANNOUNCE)) {
+        crm_trace("Action %.16llx (A_CL_JOIN_ANNOUNCE) %s", A_CL_JOIN_ANNOUNCE, text);
+    }
+    if (pcmk_is_set(action, A_CL_JOIN_REQUEST)) {
+        crm_trace("Action %.16llx (A_CL_JOIN_REQUEST) %s", A_CL_JOIN_REQUEST, text);
+    }
+    if (pcmk_is_set(action, A_CL_JOIN_RESULT)) {
+        crm_trace("Action %.16llx (A_CL_JOIN_RESULT) %s", A_CL_JOIN_RESULT, text);
+    }
+    if (pcmk_is_set(action, A_DC_JOIN_OFFER_ALL)) {
+        crm_trace("Action %.16llx (A_DC_JOIN_OFFER_ALL) %s", A_DC_JOIN_OFFER_ALL, text);
+    }
+    if (pcmk_is_set(action, A_DC_JOIN_OFFER_ONE)) {
+        crm_trace("Action %.16llx (A_DC_JOIN_OFFER_ONE) %s", A_DC_JOIN_OFFER_ONE, text);
+    }
+    if (pcmk_is_set(action, A_DC_JOIN_PROCESS_REQ)) {
+        crm_trace("Action %.16llx (A_DC_JOIN_PROCESS_REQ) %s", A_DC_JOIN_PROCESS_REQ, text);
+    }
+    if (pcmk_is_set(action, A_DC_JOIN_PROCESS_ACK)) {
+        crm_trace("Action %.16llx (A_DC_JOIN_PROCESS_ACK) %s", A_DC_JOIN_PROCESS_ACK, text);
+    }
+    if (pcmk_is_set(action, A_DC_JOIN_FINALIZE)) {
+        crm_trace("Action %.16llx (A_DC_JOIN_FINALIZE) %s", A_DC_JOIN_FINALIZE, text);
+    }
+    if (pcmk_is_set(action, A_MSG_PROCESS)) {
+        crm_trace("Action %.16llx (A_MSG_PROCESS) %s", A_MSG_PROCESS, text);
+    }
+    if (pcmk_is_set(action, A_MSG_ROUTE)) {
+        crm_trace("Action %.16llx (A_MSG_ROUTE) %s", A_MSG_ROUTE, text);
+    }
+    if (pcmk_is_set(action, A_RECOVER)) {
+        crm_trace("Action %.16llx (A_RECOVER) %s", A_RECOVER, text);
+    }
+    if (pcmk_is_set(action, A_DC_RELEASE)) {
+        crm_trace("Action %.16llx (A_DC_RELEASE) %s", A_DC_RELEASE, text);
+    }
+    if (pcmk_is_set(action, A_DC_RELEASED)) {
+        crm_trace("Action %.16llx (A_DC_RELEASED) %s", A_DC_RELEASED, text);
+    }
+    if (pcmk_is_set(action, A_DC_TAKEOVER)) {
+        crm_trace("Action %.16llx (A_DC_TAKEOVER) %s", A_DC_TAKEOVER, text);
+    }
+    if (pcmk_is_set(action, A_SHUTDOWN)) {
+        crm_trace("Action %.16llx (A_SHUTDOWN) %s", A_SHUTDOWN, text);
+    }
+    if (pcmk_is_set(action, A_SHUTDOWN_REQ)) {
+        crm_trace("Action %.16llx (A_SHUTDOWN_REQ) %s", A_SHUTDOWN_REQ, text);
+    }
+    if (pcmk_is_set(action, A_STOP)) {
+        crm_trace("Action %.16llx (A_STOP  ) %s", A_STOP, text);
+    }
+    if (pcmk_is_set(action, A_EXIT_0)) {
+        crm_trace("Action %.16llx (A_EXIT_0) %s", A_EXIT_0, text);
+    }
+    if (pcmk_is_set(action, A_EXIT_1)) {
+        crm_trace("Action %.16llx (A_EXIT_1) %s", A_EXIT_1, text);
+    }
+    if (pcmk_is_set(action, A_CIB_START)) {
+        crm_trace("Action %.16llx (A_CIB_START) %s", A_CIB_START, text);
+    }
+    if (pcmk_is_set(action, A_CIB_STOP)) {
+        crm_trace("Action %.16llx (A_CIB_STOP) %s", A_CIB_STOP, text);
+    }
+    if (pcmk_is_set(action, A_TE_INVOKE)) {
+        crm_trace("Action %.16llx (A_TE_INVOKE) %s", A_TE_INVOKE, text);
+    }
+    if (pcmk_is_set(action, A_TE_START)) {
+        crm_trace("Action %.16llx (A_TE_START) %s", A_TE_START, text);
+    }
+    if (pcmk_is_set(action, A_TE_STOP)) {
+        crm_trace("Action %.16llx (A_TE_STOP) %s", A_TE_STOP, text);
+    }
+    if (pcmk_is_set(action, A_TE_CANCEL)) {
+        crm_trace("Action %.16llx (A_TE_CANCEL) %s", A_TE_CANCEL, text);
+    }
+    if (pcmk_is_set(action, A_PE_INVOKE)) {
+        crm_trace("Action %.16llx (A_PE_INVOKE) %s", A_PE_INVOKE, text);
+    }
+    if (pcmk_is_set(action, A_PE_START)) {
+        crm_trace("Action %.16llx (A_PE_START) %s", A_PE_START, text);
+    }
+    if (pcmk_is_set(action, A_PE_STOP)) {
+        crm_trace("Action %.16llx (A_PE_STOP) %s", A_PE_STOP, text);
+    }
+    if (pcmk_is_set(action, A_NODE_BLOCK)) {
+        crm_trace("Action %.16llx (A_NODE_BLOCK) %s", A_NODE_BLOCK, text);
+    }
+    if (pcmk_is_set(action, A_UPDATE_NODESTATUS)) {
+        crm_trace("Action %.16llx (A_UPDATE_NODESTATUS) %s", A_UPDATE_NODESTATUS, text);
+    }
+    if (pcmk_is_set(action, A_LOG)) {
+        crm_trace("Action %.16llx (A_LOG   ) %s", A_LOG, text);
+    }
+    if (pcmk_is_set(action, A_ERROR)) {
+        crm_trace("Action %.16llx (A_ERROR ) %s", A_ERROR, text);
+    }
+    if (pcmk_is_set(action, A_WARN)) {
+        crm_trace("Action %.16llx (A_WARN  ) %s", A_WARN, text);
+    }
+}
+
+gboolean
+update_dc(xmlNode * msg)
+{
+    char *last_dc = controld_globals.dc_name;
+    const char *dc_version = NULL;
+    const char *welcome_from = NULL;
+
+    if (msg != NULL) {
+        gboolean invalid = FALSE;
+
+        dc_version = crm_element_value(msg, F_CRM_VERSION);
+        welcome_from = crm_element_value(msg, F_CRM_HOST_FROM);
+
+        CRM_CHECK(dc_version != NULL, return FALSE);
+        CRM_CHECK(welcome_from != NULL, return FALSE);
+
+        if (AM_I_DC
+            && !pcmk__str_eq(welcome_from, controld_globals.our_nodename,
+                             pcmk__str_casei)) {
+            invalid = TRUE;
+
+        } else if ((controld_globals.dc_name != NULL)
+                   && !pcmk__str_eq(welcome_from, controld_globals.dc_name,
+                                    pcmk__str_casei)) {
+            invalid = TRUE;
+        }
+
+        if (invalid) {
+            if (AM_I_DC) {
+                crm_err("Not updating DC to %s (%s): we are also a DC",
+                        welcome_from, dc_version);
+            } else {
+                crm_warn("New DC %s is not %s",
+                         welcome_from, controld_globals.dc_name);
+            }
+
+            controld_set_fsa_action_flags(A_CL_JOIN_QUERY | A_DC_TIMER_START);
+            controld_trigger_fsa();
+            return FALSE;
+        }
+    }
+
+    controld_globals.dc_name = NULL;    // freed as last_dc
+    pcmk__str_update(&(controld_globals.dc_name), welcome_from);
+    pcmk__str_update(&(controld_globals.dc_version), dc_version);
+
+    if (pcmk__str_eq(controld_globals.dc_name, last_dc, pcmk__str_casei)) {
+        /* do nothing */
+
+    } else if (controld_globals.dc_name != NULL) {
+        crm_node_t *dc_node = crm_get_peer(0, controld_globals.dc_name);
+
+        crm_info("Set DC to %s (%s)",
+                 controld_globals.dc_name,
+                 pcmk__s(controld_globals.dc_version, "unknown version"));
+        pcmk__update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_MEMBER);
+
+    } else if (last_dc != NULL) {
+        crm_info("Unset DC (was %s)", last_dc);
+    }
+
+    free(last_dc);
+    return TRUE;
+}
+
+void crmd_peer_down(crm_node_t *peer, bool full) 
+{
+    if(full && peer->state == NULL) {
+        pcmk__update_peer_state(__func__, peer, CRM_NODE_LOST, 0);
+        crm_update_peer_proc(__func__, peer, crm_proc_none, NULL);
+    }
+    crm_update_peer_join(__func__, peer, crm_join_none);
+    pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN);
+}
+
+/*!
+ * \internal
+ * \brief Check feature set compatibility of DC and joining node
+ *
+ * Return true if a joining node's CRM feature set is compatible with the
+ * current DC's. The feature sets are compatible if they have the same major
+ * version number, and the DC's minor version number is the same or older than
+ * the joining node's. The minor-minor version is intended solely to allow
+ * resource agents to detect feature support, and so is ignored.
+ *
+ * \param[in] dc_version    DC's feature set
+ * \param[in] join_version  Joining node's version
+ */
+bool
+feature_set_compatible(const char *dc_version, const char *join_version)
+{
+    char *dc_minor = NULL;
+    char *join_minor = NULL;
+    long dc_v = 0;
+    long join_v = 0;
+
+    // Get DC's major version
+    errno = 0;
+    dc_v = strtol(dc_version, &dc_minor, 10);
+    if (errno) {
+        return FALSE;
+    }
+
+    // Get joining node's major version
+    errno = 0;
+    join_v = strtol(join_version, &join_minor, 10);
+    if (errno) {
+        return FALSE;
+    }
+
+    // Major version component must be identical
+    if (dc_v != join_v) {
+        return FALSE;
+    }
+
+    // Get DC's minor version
+    if (*dc_minor == '.') {
+        ++dc_minor;
+    }
+    errno = 0;
+    dc_v = strtol(dc_minor, NULL, 10);
+    if (errno) {
+        return FALSE;
+    }
+
+    // Get joining node's minor version
+    if (*join_minor == '.') {
+        ++join_minor;
+    }
+    errno = 0;
+    join_v = strtol(join_minor, NULL, 10);
+    if (errno) {
+        return FALSE;
+    }
+
+    // DC's minor version must be the same or older
+    return dc_v <= join_v;
+}
+
+const char *
+get_node_id(xmlNode *lrm_rsc_op)
+{
+    xmlNode *node = lrm_rsc_op;
+
+    while (node != NULL && !pcmk__str_eq(XML_CIB_TAG_STATE, TYPE(node), pcmk__str_casei)) {
+        node = node->parent;
+    }
+
+    CRM_CHECK(node != NULL, return NULL);
+    return ID(node);
+}
diff --git a/daemons/controld/controld_utils.h b/daemons/controld/controld_utils.h
new file mode 100644
index 0000000..6ce413d
--- /dev/null
+++ b/daemons/controld/controld_utils.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef CRMD_UTILS__H
+#  define CRMD_UTILS__H
+
+#  include <crm/crm.h>
+#  include <crm/common/xml.h>
+
+#  define FAKE_TE_ID	"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+
+enum node_update_flags {
+    node_update_none = 0x0000,
+    node_update_quick = 0x0001,
+    node_update_cluster = 0x0010,
+    node_update_peer = 0x0020,
+    node_update_join = 0x0040,
+    node_update_expected = 0x0100,
+    node_update_all = node_update_cluster|node_update_peer|node_update_join|node_update_expected,
+};
+
+crm_exit_t crmd_exit(crm_exit_t exit_code);
+_Noreturn void crmd_fast_exit(crm_exit_t exit_code);
+void controld_shutdown_schedulerd_ipc(void);
+void controld_stop_sched_timer(void);
+void controld_free_sched_timer(void);
+void controld_expect_sched_reply(char *ref);
+
+void fsa_dump_actions(uint64_t action, const char *text);
+void fsa_dump_inputs(int log_level, const char *text, long long input_register);
+
+gboolean update_dc(xmlNode * msg);
+void crm_update_peer_join(const char *source, crm_node_t * node, enum crm_join_phase phase);
+xmlNode *create_node_state_update(crm_node_t *node, int flags,
+                                  xmlNode *parent, const char *source);
+void populate_cib_nodes(enum node_update_flags flags, const char *source);
+void crm_update_quorum(gboolean quorum, gboolean force_update);
+void controld_close_attrd_ipc(void);
+void update_attrd(const char *host, const char *name, const char *value, const char *user_name, gboolean is_remote_node);
+void update_attrd_list(GList *attrs, uint32_t opts);
+void update_attrd_remote_node_removed(const char *host, const char *user_name);
+void update_attrd_clear_failures(const char *host, const char *rsc,
+                                 const char *op, const char *interval_spec,
+                                 gboolean is_remote_node);
+
+int crmd_join_phase_count(enum crm_join_phase phase);
+void crmd_join_phase_log(int level);
+
+void crmd_peer_down(crm_node_t *peer, bool full);
+
+bool feature_set_compatible(const char *dc_version, const char *join_version);
+
+const char *get_node_id(xmlNode *lrm_rsc_op);
+
+#endif
diff --git a/daemons/controld/pacemaker-controld.c b/daemons/controld/pacemaker-controld.c
new file mode 100644
index 0000000..5858898
--- /dev/null
+++ b/daemons/controld/pacemaker-controld.c
@@ -0,0 +1,205 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+
+#include <crm/crm.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/ipc.h>
+#include <crm/common/output_internal.h>
+#include <crm/common/xml.h>
+
+#include <pacemaker-controld.h>
+
+#define SUMMARY "daemon for coordinating a Pacemaker cluster's response "   \
+                "to events"
+
+_Noreturn void crmd_init(void);
+extern void init_dotfile(void);
+
+controld_globals_t controld_globals = {
+    // Automatic initialization to 0, false, or NULL is fine for most members
+    .fsa_state = S_STARTING,
+    .fsa_actions = A_NOTHING,
+};
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group)
+{
+    return pcmk__build_arg_context(args, "text (default), xml", group,
+                                   "[metadata]");
+}
+
+int
+main(int argc, char **argv)
+{
+    int rc = pcmk_rc_ok;
+    crm_exit_t exit_code = CRM_EX_OK;
+    bool initialize = true;
+
+    crm_ipc_t *old_instance = NULL;
+
+    pcmk__output_t *out = NULL;
+
+    GError *error = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, NULL);
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    crm_log_preinit(NULL, argc, argv);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        initialize = false;
+        goto done;
+    }
+
+    if ((g_strv_length(processed_args) >= 2)
+        && pcmk__str_eq(processed_args[1], "metadata", pcmk__str_none)) {
+        crmd_metadata();
+        initialize = false;
+        goto done;
+    }
+
+    pcmk__cli_init_logging("pacemaker-controld", args->verbosity);
+    crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
+    crm_notice("Starting Pacemaker controller");
+
+    old_instance = crm_ipc_new(CRM_SYSTEM_CRMD, 0);
+    if (old_instance == NULL) {
+        /* crm_ipc_new will have already printed an error message with crm_err. */
+        exit_code = CRM_EX_FATAL;
+        goto done;
+    }
+
+    if (crm_ipc_connect(old_instance)) {
+        /* IPC end-point already up */
+        crm_ipc_close(old_instance);
+        crm_ipc_destroy(old_instance);
+        crm_err("pacemaker-controld is already active, aborting startup");
+        initialize = false;
+        goto done;
+
+    } else {
+        /* not up or not authentic, we'll proceed either way */
+        crm_ipc_destroy(old_instance);
+        old_instance = NULL;
+    }
+
+    if (pcmk__daemon_can_write(PE_STATE_DIR, NULL) == FALSE) {
+        exit_code = CRM_EX_FATAL;
+        crm_err("Terminating due to bad permissions on " PE_STATE_DIR);
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Bad permissions on " PE_STATE_DIR
+                    " (see logs for details)");
+        goto done;
+
+    } else if (pcmk__daemon_can_write(CRM_CONFIG_DIR, NULL) == FALSE) {
+        exit_code = CRM_EX_FATAL;
+        crm_err("Terminating due to bad permissions on " CRM_CONFIG_DIR);
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Bad permissions on " CRM_CONFIG_DIR
+                    " (see logs for details)");
+        goto done;
+    }
+
+    if (pcmk__log_output_new(&(controld_globals.logger_out)) != pcmk_rc_ok) {
+        exit_code = CRM_EX_FATAL;
+        goto done;
+    }
+
+    pcmk__output_set_log_level(controld_globals.logger_out, LOG_TRACE);
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+    pcmk__unregister_formats();
+
+    if ((exit_code == CRM_EX_OK) && initialize) {
+        // Does not return
+        crmd_init();
+    }
+    crm_exit(exit_code);
+}
+
+void
+crmd_init(void)
+{
+    crm_exit_t exit_code = CRM_EX_OK;
+    enum crmd_fsa_state state;
+
+    init_dotfile();
+    register_fsa_input(C_STARTUP, I_STARTUP, NULL);
+
+    crm_peer_init();
+    state = s_crmd_fsa(C_STARTUP);
+
+    if (state == S_PENDING || state == S_STARTING) {
+        /* Create the mainloop and run it... */
+        crm_trace("Starting %s's mainloop", crm_system_name);
+        controld_globals.mainloop = g_main_loop_new(NULL, FALSE);
+        g_main_loop_run(controld_globals.mainloop);
+        if (pcmk_is_set(controld_globals.fsa_input_register, R_STAYDOWN)) {
+            crm_info("Inhibiting automated respawn");
+            exit_code = CRM_EX_FATAL;
+        }
+
+    } else {
+        crm_err("Startup of %s failed.  Current state: %s",
+                crm_system_name, fsa_state2string(state));
+        exit_code = CRM_EX_ERROR;
+    }
+
+    crm_info("%s[%lu] exiting with status %d (%s)",
+             crm_system_name, (unsigned long) getpid(), exit_code,
+             crm_exit_str(exit_code));
+
+    crmd_fast_exit(exit_code);
+}
diff --git a/daemons/controld/pacemaker-controld.h b/daemons/controld/pacemaker-controld.h
new file mode 100644
index 0000000..1484a00
--- /dev/null
+++ b/daemons/controld/pacemaker-controld.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef CRMD__H
+#  define CRMD__H
+
+#include <controld_alerts.h>
+#include <controld_callbacks.h>
+#include <controld_cib.h>
+#include <controld_fencing.h>
+#include <controld_fsa.h>
+#include <controld_globals.h>
+#include <controld_timers.h>
+#include <controld_lrm.h>
+#include <controld_membership.h>
+#include <controld_messages.h>
+#include <controld_metadata.h>
+#include <controld_throttle.h>
+#include <controld_transition.h>
+#include <controld_utils.h>
+
+#  define controld_trigger_config()  \
+    controld_trigger_config_as(__func__, __LINE__)
+
+void crmd_metadata(void);
+void controld_trigger_config_as(const char *fn, int line);
+void controld_election_init(const char *uname);
+void controld_configure_election(GHashTable *options);
+void controld_remove_voter(const char *uname);
+void controld_election_fini(void);
+void controld_stop_current_election_timeout(void);
+
+#endif
diff --git a/daemons/execd/Makefile.am b/daemons/execd/Makefile.am
new file mode 100644
index 0000000..466f0df
--- /dev/null
+++ b/daemons/execd/Makefile.am
@@ -0,0 +1,76 @@
+#
+# Copyright 2012-2021 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU Lesser General Public License
+# version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+#
+
+include $(top_srcdir)/mk/common.mk
+include $(top_srcdir)/mk/man.mk
+
+halibdir		= $(CRM_DAEMON_DIR)
+
+halib_PROGRAMS		= pacemaker-execd cts-exec-helper
+
+EXTRA_DIST	= pacemaker-remoted.8.inc
+
+pacemaker_execd_CFLAGS		= $(CFLAGS_HARDENED_EXE)
+pacemaker_execd_LDFLAGS		= $(LDFLAGS_HARDENED_EXE)
+
+pacemaker_execd_LDADD		= $(top_builddir)/lib/common/libcrmcommon.la \
+				  $(top_builddir)/lib/services/libcrmservice.la	\
+				  $(top_builddir)/lib/fencing/libstonithd.la
+pacemaker_execd_SOURCES		= pacemaker-execd.c execd_commands.c \
+				  execd_alerts.c
+
+if BUILD_REMOTE
+sbin_PROGRAMS		= pacemaker-remoted
+if BUILD_SYSTEMD
+systemdsystemunit_DATA	= pacemaker_remote.service
+else
+initdir			= $(INITDIR)
+init_SCRIPTS		= pacemaker_remote
+endif
+
+pacemaker_remoted_CPPFLAGS	= -DPCMK__COMPILE_REMOTE $(AM_CPPFLAGS)
+
+pacemaker_remoted_CFLAGS	= $(CFLAGS_HARDENED_EXE)
+pacemaker_remoted_LDFLAGS	= $(LDFLAGS_HARDENED_EXE)
+
+pacemaker_remoted_LDADD		= $(pacemaker_execd_LDADD) \
+				  $(top_builddir)/lib/lrmd/liblrmd.la
+pacemaker_remoted_SOURCES	= $(pacemaker_execd_SOURCES) \
+				  remoted_tls.c remoted_pidone.c remoted_proxy.c
+endif
+
+cts_exec_helper_LDADD	= $(top_builddir)/lib/common/libcrmcommon.la    \
+			  $(top_builddir)/lib/lrmd/liblrmd.la		\
+			  $(top_builddir)/lib/cib/libcib.la		\
+			  $(top_builddir)/lib/services/libcrmservice.la \
+			  $(top_builddir)/lib/pengine/libpe_status.la
+cts_exec_helper_SOURCES	= cts-exec-helper.c
+
+noinst_HEADERS  = pacemaker-execd.h
+
+CLEANFILES = $(man8_MANS)
+
+# Always create a symlink for the old pacemaker_remoted name, so that bundle
+# container images using a current Pacemaker will run on cluster nodes running
+# Pacemaker 1 (>=1.1.17).
+install-exec-hook:
+if BUILD_LEGACY_LINKS
+	cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f lrmd && $(LN_S) pacemaker-execd lrmd
+endif
+if BUILD_REMOTE
+	cd $(DESTDIR)$(sbindir) && rm -f pacemaker_remoted && $(LN_S) pacemaker-remoted pacemaker_remoted
+endif
+
+uninstall-hook:
+if BUILD_LEGACY_LINKS
+	cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f lrmd
+endif
+if BUILD_REMOTE
+	cd $(DESTDIR)$(sbindir) && rm -f pacemaker_remoted
+endif
diff --git a/daemons/execd/cts-exec-helper.c b/daemons/execd/cts-exec-helper.c
new file mode 100644
index 0000000..2af5e16
--- /dev/null
+++ b/daemons/execd/cts-exec-helper.c
@@ -0,0 +1,624 @@
+/*
+ * Copyright 2012-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <glib.h>
+#include <unistd.h>
+
+#include <crm/crm.h>
+#include <crm/services.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/mainloop.h>
+
+#include <crm/pengine/status.h>
+#include <crm/pengine/internal.h>
+#include <crm/cib.h>
+#include <crm/cib/internal.h>
+#include <crm/lrmd.h>
+
+#define SUMMARY "cts-exec-helper - inject commands into the Pacemaker executor and watch for events"
+
+static int exec_call_id = 0;
+static gboolean start_test(gpointer user_data);
+static void try_connect(void);
+
+static char *key = NULL;
+static char *val = NULL;
+
+static struct {
+    int verbose;
+    int quiet;
+    guint interval_ms;
+    int timeout;
+    int start_delay;
+    int cancel_call_id;
+    gboolean no_wait;
+    gboolean is_running;
+    gboolean no_connect;
+    int exec_call_opts;
+    const char *api_call;
+    const char *rsc_id;
+    const char *provider;
+    const char *class;
+    const char *type;
+    const char *action;
+    const char *listen;
+    gboolean use_tls;
+    lrmd_key_value_t *params;
+} options;
+
+static gboolean
+interval_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    options.interval_ms = crm_parse_interval_spec(optarg);
+    return errno == 0;
+}
+
+static gboolean
+notify_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_any_of(option_name, "--notify-orig", "-n", NULL)) {
+        options.exec_call_opts = lrmd_opt_notify_orig_only;
+    } else if (pcmk__str_any_of(option_name, "--notify-changes", "-o", NULL)) {
+        options.exec_call_opts = lrmd_opt_notify_changes_only;
+    }
+
+    return TRUE;
+}
+
+static gboolean
+param_key_val_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_any_of(option_name, "--param-key", "-k", NULL)) {
+        pcmk__str_update(&key, optarg);
+    } else if (pcmk__str_any_of(option_name, "--param-val", "-v", NULL)) {
+        pcmk__str_update(&val, optarg);
+    }
+
+    if (key != NULL && val != NULL) {
+        options.params = lrmd_key_value_add(options.params, key, val);
+        pcmk__str_update(&key, NULL);
+        pcmk__str_update(&val, NULL);
+    }
+
+    return TRUE;
+}
+
+static GOptionEntry basic_entries[] = {
+    { "api-call", 'c', 0, G_OPTION_ARG_STRING, &options.api_call,
+      "Directly relates to executor API functions",
+      NULL },
+
+    { "is-running", 'R', 0, G_OPTION_ARG_NONE, &options.is_running,
+      "Determine if a resource is registered and running",
+      NULL },
+
+    { "listen", 'l', 0, G_OPTION_ARG_STRING, &options.listen,
+      "Listen for a specific event string",
+      NULL },
+
+    { "no-wait", 'w', 0, G_OPTION_ARG_NONE, &options.no_wait,
+      "Make api call and do not wait for result",
+      NULL },
+
+    { "notify-changes", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, notify_cb,
+      "Only notify client changes to recurring operations",
+      NULL },
+
+    { "notify-orig", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, notify_cb,
+      "Only notify this client of the results of an API action",
+      NULL },
+
+    { "tls", 'S', 0, G_OPTION_ARG_NONE, &options.use_tls,
+      "Use TLS backend for local connection",
+      NULL },
+
+    { NULL }
+};
+
+static GOptionEntry api_call_entries[] = {
+    { "action", 'a', 0, G_OPTION_ARG_STRING, &options.action,
+      NULL, NULL },
+
+    { "cancel-call-id", 'x', 0, G_OPTION_ARG_INT, &options.cancel_call_id,
+      NULL, NULL },
+
+    { "class", 'C', 0, G_OPTION_ARG_STRING, &options.class,
+      NULL, NULL },
+
+    { "interval", 'i', 0, G_OPTION_ARG_CALLBACK, interval_cb,
+      NULL, NULL },
+
+    { "param-key", 'k', 0, G_OPTION_ARG_CALLBACK, param_key_val_cb,
+      NULL, NULL },
+
+    { "param-val", 'v', 0, G_OPTION_ARG_CALLBACK, param_key_val_cb,
+      NULL, NULL },
+
+    { "provider", 'P', 0, G_OPTION_ARG_STRING, &options.provider,
+      NULL, NULL },
+
+    { "rsc-id", 'r', 0, G_OPTION_ARG_STRING, &options.rsc_id,
+      NULL, NULL },
+
+    { "start-delay", 's', 0, G_OPTION_ARG_INT, &options.start_delay,
+      NULL, NULL },
+
+    { "timeout", 't', 0, G_OPTION_ARG_INT, &options.timeout,
+      NULL, NULL },
+
+    { "type", 'T', 0, G_OPTION_ARG_STRING, &options.type,
+      NULL, NULL },
+
+    { NULL }
+};
+
+static GMainLoop *mainloop = NULL;
+static lrmd_t *lrmd_conn = NULL;
+
+static char event_buf_v0[1024];
+
+static crm_exit_t
+test_exit(crm_exit_t exit_code)
+{
+    lrmd_api_delete(lrmd_conn);
+    return crm_exit(exit_code);
+}
+
+#define print_result(fmt, args...)  \
+    if (!options.quiet) {           \
+        printf(fmt "\n" , ##args);  \
+    }
+
+#define report_event(event)                                             \
+    snprintf(event_buf_v0, sizeof(event_buf_v0), "NEW_EVENT event_type:%s rsc_id:%s action:%s rc:%s op_status:%s", \
+             lrmd_event_type2str(event->type),                          \
+             event->rsc_id,                                             \
+             event->op_type ? event->op_type : "none",                  \
+             services_ocf_exitcode_str(event->rc),                      \
+             pcmk_exec_status_str(event->op_status));                   \
+    crm_info("%s", event_buf_v0);
+
+static void
+test_shutdown(int nsig)
+{
+    lrmd_api_delete(lrmd_conn);
+    lrmd_conn = NULL;
+}
+
+static void
+read_events(lrmd_event_data_t * event)
+{
+    report_event(event);
+    if (options.listen) {
+        if (pcmk__str_eq(options.listen, event_buf_v0, pcmk__str_casei)) {
+            print_result("LISTEN EVENT SUCCESSFUL");
+            test_exit(CRM_EX_OK);
+        }
+    }
+
+    if (exec_call_id && (event->call_id == exec_call_id)) {
+        if (event->op_status == 0 && event->rc == 0) {
+            print_result("API-CALL SUCCESSFUL for 'exec'");
+        } else {
+            print_result("API-CALL FAILURE for 'exec', rc:%d lrmd_op_status:%s",
+                         event->rc, pcmk_exec_status_str(event->op_status));
+            test_exit(CRM_EX_ERROR);
+        }
+
+        if (!options.listen) {
+            test_exit(CRM_EX_OK);
+        }
+    }
+}
+
+static gboolean
+timeout_err(gpointer data)
+{
+    print_result("LISTEN EVENT FAILURE - timeout occurred, never found");
+    test_exit(CRM_EX_TIMEOUT);
+    return FALSE;
+}
+
+static void
+connection_events(lrmd_event_data_t * event)
+{
+    int rc = event->connection_rc;
+
+    if (event->type != lrmd_event_connect) {
+        /* ignore */
+        return;
+    }
+
+    if (!rc) {
+        crm_info("Executor client connection established");
+        start_test(NULL);
+        return;
+    } else {
+        sleep(1);
+        try_connect();
+        crm_notice("Executor client connection failed");
+    }
+}
+
+static void
+try_connect(void)
+{
+    int tries = 10;
+    static int num_tries = 0;
+    int rc = 0;
+
+    lrmd_conn->cmds->set_callback(lrmd_conn, connection_events);
+    for (; num_tries < tries; num_tries++) {
+        rc = lrmd_conn->cmds->connect_async(lrmd_conn, crm_system_name, 3000);
+
+        if (!rc) {
+            return;             /* we'll hear back in async callback */
+        }
+        sleep(1);
+    }
+
+    print_result("API CONNECTION FAILURE");
+    test_exit(CRM_EX_ERROR);
+}
+
+static gboolean
+start_test(gpointer user_data)
+{
+    int rc = 0;
+
+    if (!options.no_connect) {
+        if (!lrmd_conn->cmds->is_connected(lrmd_conn)) {
+            try_connect();
+            /* async connect -- this function will get called back into */
+            return 0;
+        }
+    }
+    lrmd_conn->cmds->set_callback(lrmd_conn, read_events);
+
+    if (options.timeout) {
+        g_timeout_add(options.timeout, timeout_err, NULL);
+    }
+
+    if (!options.api_call) {
+        return 0;
+    }
+
+    if (pcmk__str_eq(options.api_call, "exec", pcmk__str_casei)) {
+        rc = lrmd_conn->cmds->exec(lrmd_conn,
+                                   options.rsc_id,
+                                   options.action,
+                                   NULL,
+                                   options.interval_ms,
+                                   options.timeout,
+                                   options.start_delay,
+                                   options.exec_call_opts,
+                                   options.params);
+
+        if (rc > 0) {
+            exec_call_id = rc;
+            print_result("API-CALL 'exec' action pending, waiting on response");
+        }
+
+    } else if (pcmk__str_eq(options.api_call, "register_rsc", pcmk__str_casei)) {
+        rc = lrmd_conn->cmds->register_rsc(lrmd_conn,
+                                           options.rsc_id,
+                                           options.class, options.provider, options.type, 0);
+    } else if (pcmk__str_eq(options.api_call, "get_rsc_info", pcmk__str_casei)) {
+        lrmd_rsc_info_t *rsc_info;
+
+        rsc_info = lrmd_conn->cmds->get_rsc_info(lrmd_conn, options.rsc_id, 0);
+
+        if (rsc_info) {
+            print_result("RSC_INFO: id:%s class:%s provider:%s type:%s",
+                         rsc_info->id, rsc_info->standard,
+                         (rsc_info->provider? rsc_info->provider : "<none>"),
+                         rsc_info->type);
+            lrmd_free_rsc_info(rsc_info);
+            rc = pcmk_ok;
+        } else {
+            rc = -1;
+        }
+    } else if (pcmk__str_eq(options.api_call, "unregister_rsc", pcmk__str_casei)) {
+        rc = lrmd_conn->cmds->unregister_rsc(lrmd_conn, options.rsc_id, 0);
+    } else if (pcmk__str_eq(options.api_call, "cancel", pcmk__str_casei)) {
+        rc = lrmd_conn->cmds->cancel(lrmd_conn, options.rsc_id, options.action,
+                                     options.interval_ms);
+    } else if (pcmk__str_eq(options.api_call, "metadata", pcmk__str_casei)) {
+        char *output = NULL;
+
+        rc = lrmd_conn->cmds->get_metadata(lrmd_conn,
+                                           options.class,
+                                           options.provider, options.type, &output, 0);
+        if (rc == pcmk_ok) {
+            print_result("%s", output);
+            free(output);
+        }
+    } else if (pcmk__str_eq(options.api_call, "list_agents", pcmk__str_casei)) {
+        lrmd_list_t *list = NULL;
+        lrmd_list_t *iter = NULL;
+
+        rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, options.class, options.provider);
+
+        if (rc > 0) {
+            print_result("%d agents found", rc);
+            for (iter = list; iter != NULL; iter = iter->next) {
+                print_result("%s", iter->val);
+            }
+            lrmd_list_freeall(list);
+            rc = 0;
+        } else {
+            print_result("API_CALL FAILURE - no agents found");
+            rc = -1;
+        }
+    } else if (pcmk__str_eq(options.api_call, "list_ocf_providers", pcmk__str_casei)) {
+        lrmd_list_t *list = NULL;
+        lrmd_list_t *iter = NULL;
+
+        rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, options.type, &list);
+
+        if (rc > 0) {
+            print_result("%d providers found", rc);
+            for (iter = list; iter != NULL; iter = iter->next) {
+                print_result("%s", iter->val);
+            }
+            lrmd_list_freeall(list);
+            rc = 0;
+        } else {
+            print_result("API_CALL FAILURE - no providers found");
+            rc = -1;
+        }
+
+    } else if (pcmk__str_eq(options.api_call, "list_standards", pcmk__str_casei)) {
+        lrmd_list_t *list = NULL;
+        lrmd_list_t *iter = NULL;
+
+        rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);
+
+        if (rc > 0) {
+            print_result("%d standards found", rc);
+            for (iter = list; iter != NULL; iter = iter->next) {
+                print_result("%s", iter->val);
+            }
+            lrmd_list_freeall(list);
+            rc = 0;
+        } else {
+            print_result("API_CALL FAILURE - no providers found");
+            rc = -1;
+        }
+
+    } else if (pcmk__str_eq(options.api_call, "get_recurring_ops", pcmk__str_casei)) {
+        GList *op_list = NULL;
+        GList *op_item = NULL;
+        rc = lrmd_conn->cmds->get_recurring_ops(lrmd_conn, options.rsc_id, 0, 0,
+                                                &op_list);
+
+        for (op_item = op_list; op_item != NULL; op_item = op_item->next) {
+            lrmd_op_info_t *op_info = op_item->data;
+
+            print_result("RECURRING_OP: %s_%s_%s timeout=%sms",
+                         op_info->rsc_id, op_info->action,
+                         op_info->interval_ms_s, op_info->timeout_ms_s);
+            lrmd_free_op_info(op_info);
+        }
+        g_list_free(op_list);
+
+    } else if (options.api_call) {
+        print_result("API-CALL FAILURE unknown action '%s'", options.action);
+        test_exit(CRM_EX_ERROR);
+    }
+
+    if (rc < 0) {
+        print_result("API-CALL FAILURE for '%s' api_rc:%d",
+                     options.api_call, rc);
+        test_exit(CRM_EX_ERROR);
+    }
+
+    if (options.api_call && rc == pcmk_ok) {
+        print_result("API-CALL SUCCESSFUL for '%s'", options.api_call);
+        if (!options.listen) {
+            test_exit(CRM_EX_OK);
+        }
+    }
+
+    if (options.no_wait) {
+        /* just make the call and exit regardless of anything else. */
+        test_exit(CRM_EX_OK);
+    }
+
+    return 0;
+}
+
+/*!
+ * \internal
+ * \brief Generate resource parameters from CIB if none explicitly given
+ *
+ * \return Standard Pacemaker return code
+ */
+static int
+generate_params(void)
+{
+    int rc = pcmk_rc_ok;
+    pe_working_set_t *data_set = NULL;
+    xmlNode *cib_xml_copy = NULL;
+    pe_resource_t *rsc = NULL;
+    GHashTable *params = NULL;
+    GHashTable *meta = NULL;
+    GHashTableIter iter;
+    char *key = NULL;
+    char *value = NULL;
+
+    if (options.params != NULL) {
+        return pcmk_rc_ok; // User specified parameters explicitly
+    }
+
+    // Retrieve and update CIB
+    rc = cib__signon_query(NULL, NULL, &cib_xml_copy);
+    if (rc != pcmk_rc_ok) {
+        return rc;
+    }
+    if (!cli_config_update(&cib_xml_copy, NULL, FALSE)) {
+        crm_err("Could not update CIB");
+        return pcmk_rc_cib_corrupt;
+    }
+
+    // Calculate cluster status
+    data_set = pe_new_working_set();
+    if (data_set == NULL) {
+        crm_crit("Could not allocate working set");
+        return ENOMEM;
+    }
+    pe__set_working_set_flags(data_set, pe_flag_no_counts|pe_flag_no_compat);
+    data_set->input = cib_xml_copy;
+    data_set->now = crm_time_new(NULL);
+    cluster_status(data_set);
+
+    // Find resource in CIB
+    rsc = pe_find_resource_with_flags(data_set->resources, options.rsc_id,
+                                      pe_find_renamed|pe_find_any);
+    if (rsc == NULL) {
+        crm_err("Resource does not exist in config");
+        pe_free_working_set(data_set);
+        return EINVAL;
+    }
+
+    // Add resource instance parameters to options.params
+    params = pe_rsc_params(rsc, NULL, data_set);
+    if (params != NULL) {
+        g_hash_table_iter_init(&iter, params);
+        while (g_hash_table_iter_next(&iter, (gpointer *) &key,
+                                      (gpointer *) &value)) {
+            options.params = lrmd_key_value_add(options.params, key, value);
+        }
+    }
+
+    // Add resource meta-attributes to options.params
+    meta = pcmk__strkey_table(free, free);
+    get_meta_attributes(meta, rsc, NULL, data_set);
+    g_hash_table_iter_init(&iter, meta);
+    while (g_hash_table_iter_next(&iter, (gpointer *) &key,
+                                  (gpointer *) &value)) {
+        char *crm_name = crm_meta_name(key);
+
+        options.params = lrmd_key_value_add(options.params, crm_name, value);
+        free(crm_name);
+    }
+    g_hash_table_destroy(meta);
+
+    pe_free_working_set(data_set);
+    return rc;
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    context = pcmk__build_arg_context(args, NULL, group, NULL);
+
+    pcmk__add_main_args(context, basic_entries);
+    pcmk__add_arg_group(context, "api-call", "API Call Options:",
+                        "Parameters for api-call option", api_call_entries);
+
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    GError *error = NULL;
+    crm_exit_t exit_code = CRM_EX_OK;
+    crm_trigger_t *trig = NULL;
+
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    /* Typically we'd pass all the single character options that take an argument
+     * as the second parameter here (and there's a bunch of those in this tool).
+     * However, we control how this program is called so we can just not call it
+     * in a way where the preprocessing ever matters.
+     */
+    gchar **processed_args = pcmk__cmdline_preproc(argv, NULL);
+    GOptionContext *context = build_arg_context(args, NULL);
+
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    /* We have to use crm_log_init here to set up the logging because there's
+     * different handling for daemons vs. command line programs, and
+     * pcmk__cli_init_logging is set up to only handle the latter.
+     */
+    crm_log_init(NULL, LOG_INFO, TRUE, (args->verbosity? TRUE : FALSE), argc,
+                 argv, FALSE);
+
+    for (int i = 0; i < args->verbosity; i++) {
+        crm_bump_log_level(argc, argv);
+    }
+
+    if (!options.listen && pcmk__strcase_any_of(options.api_call, "metadata", "list_agents",
+                                                "list_standards", "list_ocf_providers", NULL)) {
+        options.no_connect = TRUE;
+    }
+
+    if (options.is_running) {
+        int rc = pcmk_rc_ok;
+
+        if (options.rsc_id == NULL) {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "--is-running requires --rsc-id");
+            goto done;
+        }
+
+        options.interval_ms = 0;
+        if (options.timeout == 0) {
+            options.timeout = 30000;
+        }
+
+        rc = generate_params();
+        if (rc != pcmk_rc_ok) {
+            exit_code = pcmk_rc2exitc(rc);
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Can not determine resource status: "
+                        "unable to get parameters from CIB");
+            goto done;
+        }
+        options.api_call = "exec";
+        options.action = "monitor";
+        options.exec_call_opts = lrmd_opt_notify_orig_only;
+    }
+
+    if (!options.api_call && !options.listen) {
+        exit_code = CRM_EX_USAGE;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Must specify at least one of --api-call, --listen, "
+                    "or --is-running");
+        goto done;
+    }
+
+    if (options.use_tls) {
+        lrmd_conn = lrmd_remote_api_new(NULL, "localhost", 0);
+    } else {
+        lrmd_conn = lrmd_api_new();
+    }
+    trig = mainloop_add_trigger(G_PRIORITY_HIGH, start_test, NULL);
+    mainloop_set_trigger(trig);
+    mainloop_add_signal(SIGTERM, test_shutdown);
+
+    crm_info("Starting");
+    mainloop = g_main_loop_new(NULL, FALSE);
+    g_main_loop_run(mainloop);
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    free(key);
+    free(val);
+
+    pcmk__output_and_clear_error(&error, NULL);
+    return test_exit(exit_code);
+}
diff --git a/daemons/execd/execd_alerts.c b/daemons/execd/execd_alerts.c
new file mode 100644
index 0000000..5944d93
--- /dev/null
+++ b/daemons/execd/execd_alerts.c
@@ -0,0 +1,205 @@
+/*
+ * Copyright 2016-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <glib.h>
+
+#include <crm/crm.h>
+#include <crm/services.h>
+#include <crm/services_internal.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/common/alerts_internal.h>
+#include <crm/msg_xml.h>
+
+#include "pacemaker-execd.h"
+
+/* Track in-flight alerts so we can wait for them at shutdown */
+static GHashTable *inflight_alerts; /* key = call_id, value = timeout */
+static gboolean draining_alerts = FALSE;
+
+static inline void
+add_inflight_alert(int call_id, int timeout)
+{
+    if (inflight_alerts == NULL) {
+        inflight_alerts = pcmk__intkey_table(NULL);
+    }
+    pcmk__intkey_table_insert(inflight_alerts, call_id,
+                              GINT_TO_POINTER(timeout));
+}
+
+static inline void
+remove_inflight_alert(int call_id)
+{
+    if (inflight_alerts != NULL) {
+        pcmk__intkey_table_remove(inflight_alerts, call_id);
+    }
+}
+
+static int
+max_inflight_timeout(void)
+{
+    GHashTableIter iter;
+    gpointer timeout;
+    int max_timeout = 0;
+
+    if (inflight_alerts) {
+        g_hash_table_iter_init(&iter, inflight_alerts);
+        while (g_hash_table_iter_next(&iter, NULL, &timeout)) {
+            if (GPOINTER_TO_INT(timeout) > max_timeout) {
+                max_timeout = GPOINTER_TO_INT(timeout);
+            }
+        }
+    }
+    return max_timeout;
+}
+
+struct alert_cb_s {
+    char *client_id;
+    int call_id;
+};
+
+static void
+alert_complete(svc_action_t *action)
+{
+    struct alert_cb_s *cb_data = (struct alert_cb_s *) (action->cb_data);
+
+    CRM_CHECK(cb_data != NULL, return);
+
+    remove_inflight_alert(cb_data->call_id);
+
+    if (action->status != PCMK_EXEC_DONE) {
+        const char *reason = services__exit_reason(action);
+
+        crm_notice("Could not send alert: %s%s%s%s " CRM_XS " client=%s",
+                   pcmk_exec_status_str(action->status),
+                   (reason == NULL)? "" : " (",
+                   (reason == NULL)? "" : reason,
+                   (reason == NULL)? "" : ")",
+                   cb_data->client_id);
+
+    } else if (action->rc != 0) {
+        crm_notice("Alert [%d] completed but exited with status %d "
+                   CRM_XS " client=%s",
+                   action->pid, action->rc, cb_data->client_id);
+
+    } else {
+        crm_debug("Alert [%d] completed " CRM_XS " client=%s",
+                  action->pid, cb_data->client_id);
+    }
+
+    free(cb_data->client_id);
+    free(action->cb_data);
+    action->cb_data = NULL;
+}
+
+int
+process_lrmd_alert_exec(pcmk__client_t *client, uint32_t id, xmlNode *request)
+{
+    static int alert_sequence_no = 0;
+
+    xmlNode *alert_xml = get_xpath_object("//" F_LRMD_ALERT, request, LOG_ERR);
+    const char *alert_id = crm_element_value(alert_xml, F_LRMD_ALERT_ID);
+    const char *alert_path = crm_element_value(alert_xml, F_LRMD_ALERT_PATH);
+    svc_action_t *action = NULL;
+    int alert_timeout = 0;
+    int rc = pcmk_ok;
+    GHashTable *params = NULL;
+    struct alert_cb_s *cb_data = NULL;
+
+    if ((alert_id == NULL) || (alert_path == NULL) ||
+        (client == NULL) || (client->id == NULL)) { /* hint static analyzer */
+        return -EINVAL;
+    }
+    if (draining_alerts) {
+        return pcmk_ok;
+    }
+
+    crm_element_value_int(alert_xml, F_LRMD_TIMEOUT, &alert_timeout);
+
+    crm_info("Executing alert %s for %s", alert_id, client->id);
+
+    params = xml2list(alert_xml);
+    pcmk__add_alert_key_int(params, PCMK__alert_key_node_sequence,
+                            ++alert_sequence_no);
+
+    cb_data = calloc(1, sizeof(struct alert_cb_s));
+    if (cb_data == NULL) {
+        rc = -errno;
+        goto err;
+    }
+
+    /* coverity[deref_ptr] False Positive */
+    cb_data->client_id = strdup(client->id);
+    if (cb_data->client_id == NULL) {
+        rc = -errno;
+        goto err;
+    }
+
+    crm_element_value_int(request, F_LRMD_CALLID, &(cb_data->call_id));
+
+    action = services_alert_create(alert_id, alert_path, alert_timeout, params,
+                                   alert_sequence_no, cb_data);
+    if (action->rc != PCMK_OCF_UNKNOWN) {
+        rc = -E2BIG;
+        goto err;
+    }
+
+    rc = services_action_user(action, CRM_DAEMON_USER);
+    if (rc < 0) {
+        goto err;
+    }
+
+    add_inflight_alert(cb_data->call_id, alert_timeout);
+    if (services_alert_async(action, alert_complete) == FALSE) {
+        services_action_free(action);
+    }
+    return pcmk_ok;
+
+err:
+    if (cb_data) {
+        if (cb_data->client_id) {
+            free(cb_data->client_id);
+        }
+        free(cb_data);
+    }
+    services_action_free(action);
+    return rc;
+}
+
+static bool
+drain_check(guint remaining_timeout_ms)
+{
+    if (inflight_alerts != NULL) {
+        guint count = g_hash_table_size(inflight_alerts);
+
+        if (count > 0) {
+            crm_trace("%d alerts pending (%.3fs timeout remaining)",
+                      count, remaining_timeout_ms / 1000.0);
+            return TRUE;
+        }
+    }
+    return FALSE;
+}
+
+void
+lrmd_drain_alerts(GMainLoop *mloop)
+{
+    if (inflight_alerts != NULL) {
+        guint timer_ms = max_inflight_timeout() + 5000;
+
+        crm_trace("Draining in-flight alerts (timeout %.3fs)",
+                  timer_ms / 1000.0);
+        draining_alerts = TRUE;
+        pcmk_drain_main_loop(mloop, timer_ms, drain_check);
+        g_hash_table_destroy(inflight_alerts);
+        inflight_alerts = NULL;
+    }
+}
diff --git a/daemons/execd/execd_commands.c b/daemons/execd/execd_commands.c
new file mode 100644
index 0000000..fa2761e
--- /dev/null
+++ b/daemons/execd/execd_commands.c
@@ -0,0 +1,1927 @@
+/*
+ * Copyright 2012-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include <crm/fencing/internal.h>
+
+#include <glib.h>
+
+// Check whether we have a high-resolution monotonic clock
+#undef PCMK__TIME_USE_CGT
+#if HAVE_DECL_CLOCK_MONOTONIC && defined(CLOCK_MONOTONIC)
+#  define PCMK__TIME_USE_CGT
+#  include <time.h>  /* clock_gettime */
+#endif
+
+#include <unistd.h>
+
+#include <crm/crm.h>
+#include <crm/fencing/internal.h>
+#include <crm/services.h>
+#include <crm/services_internal.h>
+#include <crm/common/mainloop.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/msg_xml.h>
+
+#include "pacemaker-execd.h"
+
+GHashTable *rsc_list = NULL;
+
+typedef struct lrmd_cmd_s {
+    int timeout;
+    guint interval_ms;
+    int start_delay;
+    int timeout_orig;
+
+    int call_id;
+
+    int call_opts;
+    /* Timer ids, must be removed on cmd destruction. */
+    int delay_id;
+    int stonith_recurring_id;
+
+    int rsc_deleted;
+
+    int service_flags;
+
+    char *client_id;
+    char *origin;
+    char *rsc_id;
+    char *action;
+    char *real_action;
+    char *userdata_str;
+
+    pcmk__action_result_t result;
+
+    /* We can track operation queue time and run time, to be saved with the CIB
+     * resource history (and displayed in cluster status). We need
+     * high-resolution monotonic time for this purpose, so we use
+     * clock_gettime(CLOCK_MONOTONIC, ...) (if available, otherwise this feature
+     * is disabled).
+     *
+     * However, we also need epoch timestamps for recording the time the command
+     * last ran and the time its return value last changed, for use in time
+     * displays (as opposed to interval calculations). We keep time_t values for
+     * this purpose.
+     *
+     * The last run time is used for both purposes, so we keep redundant
+     * monotonic and epoch values for this. Technically the two could represent
+     * different times, but since time_t has only second resolution and the
+     * values are used for distinct purposes, that is not significant.
+     */
+#ifdef PCMK__TIME_USE_CGT
+    /* Recurring and systemd operations may involve more than one executor
+     * command per operation, so they need info about the original and the most
+     * recent.
+     */
+    struct timespec t_first_run;    // When op first ran
+    struct timespec t_run;          // When op most recently ran
+    struct timespec t_first_queue;  // When op was first queued
+    struct timespec t_queue;        // When op was most recently queued
+#endif
+    time_t epoch_last_run;          // Epoch timestamp of when op last ran
+    time_t epoch_rcchange;          // Epoch timestamp of when rc last changed
+
+    bool first_notify_sent;
+    int last_notify_rc;
+    int last_notify_op_status;
+    int last_pid;
+
+    GHashTable *params;
+} lrmd_cmd_t;
+
+static void cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc);
+static gboolean execute_resource_action(gpointer user_data);
+static void cancel_all_recurring(lrmd_rsc_t * rsc, const char *client_id);
+
+#ifdef PCMK__TIME_USE_CGT
+
+/*!
+ * \internal
+ * \brief Check whether a struct timespec has been set
+ *
+ * \param[in] timespec  Time to check
+ *
+ * \return true if timespec has been set (i.e. is nonzero), false otherwise
+ */
+static inline bool
+time_is_set(const struct timespec *timespec)
+{
+    return (timespec != NULL) &&
+           ((timespec->tv_sec != 0) || (timespec->tv_nsec != 0));
+}
+
+/*
+ * \internal
+ * \brief Set a timespec (and its original if unset) to the current time
+ *
+ * \param[out] t_current  Where to store current time
+ * \param[out] t_orig     Where to copy t_current if unset
+ */
+static void
+get_current_time(struct timespec *t_current, struct timespec *t_orig)
+{
+    clock_gettime(CLOCK_MONOTONIC, t_current);
+    if ((t_orig != NULL) && !time_is_set(t_orig)) {
+        *t_orig = *t_current;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Return difference between two times in milliseconds
+ *
+ * \param[in] now  More recent time (or NULL to use current time)
+ * \param[in] old  Earlier time
+ *
+ * \return milliseconds difference (or 0 if old is NULL or unset)
+ *
+ * \note Can overflow on 32bit machines when the differences is around
+ *       24 days or more.
+ */
+static int
+time_diff_ms(const struct timespec *now, const struct timespec *old)
+{
+    int diff_ms = 0;
+
+    if (time_is_set(old)) {
+        struct timespec local_now = { 0, };
+
+        if (now == NULL) {
+            clock_gettime(CLOCK_MONOTONIC, &local_now);
+            now = &local_now;
+        }
+        diff_ms = (now->tv_sec - old->tv_sec) * 1000
+                  + (now->tv_nsec - old->tv_nsec) / 1000000;
+    }
+    return diff_ms;
+}
+
+/*!
+ * \internal
+ * \brief Reset a command's operation times to their original values.
+ *
+ * Reset a command's run and queued timestamps to the timestamps of the original
+ * command, so we report the entire time since then and not just the time since
+ * the most recent command (for recurring and systemd operations).
+ *
+ * \param[in,out] cmd  Executor command object to reset
+ *
+ * \note It's not obvious what the queued time should be for a systemd
+ *       start/stop operation, which might go like this:
+ *         initial command queued 5ms, runs 3s
+ *         monitor command queued 10ms, runs 10s
+ *         monitor command queued 10ms, runs 10s
+ *       Is the queued time for that operation 5ms, 10ms or 25ms? The current
+ *       implementation will report 5ms. If it's 25ms, then we need to
+ *       subtract 20ms from the total exec time so as not to count it twice.
+ *       We can implement that later if it matters to anyone ...
+ */
+static void
+cmd_original_times(lrmd_cmd_t * cmd)
+{
+    cmd->t_run = cmd->t_first_run;
+    cmd->t_queue = cmd->t_first_queue;
+}
+#endif
+
+static inline bool
+action_matches(const lrmd_cmd_t *cmd, const char *action, guint interval_ms)
+{
+    return (cmd->interval_ms == interval_ms)
+           && pcmk__str_eq(cmd->action, action, pcmk__str_casei);
+}
+
+/*!
+ * \internal
+ * \brief Log the result of an asynchronous command
+ *
+ * \param[in] cmd            Command to log result for
+ * \param[in] exec_time_ms   Execution time in milliseconds, if known
+ * \param[in] queue_time_ms  Queue time in milliseconds, if known
+ */
+static void
+log_finished(const lrmd_cmd_t *cmd, int exec_time_ms, int queue_time_ms)
+{
+    int log_level = LOG_INFO;
+    GString *str = g_string_sized_new(100); // reasonable starting size
+
+    if (pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
+        log_level = LOG_DEBUG;
+    }
+
+    g_string_append_printf(str, "%s %s (call %d",
+                           cmd->rsc_id, cmd->action, cmd->call_id);
+    if (cmd->last_pid != 0) {
+        g_string_append_printf(str, ", PID %d", cmd->last_pid);
+    }
+    if (cmd->result.execution_status == PCMK_EXEC_DONE) {
+        g_string_append_printf(str, ") exited with status %d",
+                               cmd->result.exit_status);
+    } else {
+        pcmk__g_strcat(str, ") could not be executed: ",
+                       pcmk_exec_status_str(cmd->result.execution_status),
+                       NULL);
+    }
+    if (cmd->result.exit_reason != NULL) {
+        pcmk__g_strcat(str, " (", cmd->result.exit_reason, ")", NULL);
+    }
+
+#ifdef PCMK__TIME_USE_CGT
+    pcmk__g_strcat(str, " (execution time ",
+                   pcmk__readable_interval(exec_time_ms), NULL);
+    if (queue_time_ms > 0) {
+        pcmk__g_strcat(str, " after being queued ",
+                       pcmk__readable_interval(queue_time_ms), NULL);
+    }
+    g_string_append_c(str, ')');
+#endif
+
+    do_crm_log(log_level, "%s", str->str);
+    g_string_free(str, TRUE);
+}
+
+static void
+log_execute(lrmd_cmd_t * cmd)
+{
+    int log_level = LOG_INFO;
+
+    if (pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
+        log_level = LOG_DEBUG;
+    }
+
+    do_crm_log(log_level, "executing - rsc:%s action:%s call_id:%d",
+               cmd->rsc_id, cmd->action, cmd->call_id);
+}
+
+static const char *
+normalize_action_name(lrmd_rsc_t * rsc, const char *action)
+{
+    if (pcmk__str_eq(action, "monitor", pcmk__str_casei) &&
+        pcmk_is_set(pcmk_get_ra_caps(rsc->class), pcmk_ra_cap_status)) {
+        return "status";
+    }
+    return action;
+}
+
+static lrmd_rsc_t *
+build_rsc_from_xml(xmlNode * msg)
+{
+    xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR);
+    lrmd_rsc_t *rsc = NULL;
+
+    rsc = calloc(1, sizeof(lrmd_rsc_t));
+
+    crm_element_value_int(msg, F_LRMD_CALLOPTS, &rsc->call_opts);
+
+    rsc->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID);
+    rsc->class = crm_element_value_copy(rsc_xml, F_LRMD_CLASS);
+    rsc->provider = crm_element_value_copy(rsc_xml, F_LRMD_PROVIDER);
+    rsc->type = crm_element_value_copy(rsc_xml, F_LRMD_TYPE);
+    rsc->work = mainloop_add_trigger(G_PRIORITY_HIGH, execute_resource_action,
+                                     rsc);
+
+    // Initialize fence device probes (to return "not running")
+    pcmk__set_result(&rsc->fence_probe_result, CRM_EX_ERROR,
+                     PCMK_EXEC_NO_FENCE_DEVICE, NULL);
+    return rsc;
+}
+
+static lrmd_cmd_t *
+create_lrmd_cmd(xmlNode *msg, pcmk__client_t *client)
+{
+    int call_options = 0;
+    xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR);
+    lrmd_cmd_t *cmd = NULL;
+
+    cmd = calloc(1, sizeof(lrmd_cmd_t));
+
+    crm_element_value_int(msg, F_LRMD_CALLOPTS, &call_options);
+    cmd->call_opts = call_options;
+    cmd->client_id = strdup(client->id);
+
+    crm_element_value_int(msg, F_LRMD_CALLID, &cmd->call_id);
+    crm_element_value_ms(rsc_xml, F_LRMD_RSC_INTERVAL, &cmd->interval_ms);
+    crm_element_value_int(rsc_xml, F_LRMD_TIMEOUT, &cmd->timeout);
+    crm_element_value_int(rsc_xml, F_LRMD_RSC_START_DELAY, &cmd->start_delay);
+    cmd->timeout_orig = cmd->timeout;
+
+    cmd->origin = crm_element_value_copy(rsc_xml, F_LRMD_ORIGIN);
+    cmd->action = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ACTION);
+    cmd->userdata_str = crm_element_value_copy(rsc_xml, F_LRMD_RSC_USERDATA_STR);
+    cmd->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID);
+
+    cmd->params = xml2list(rsc_xml);
+
+    if (pcmk__str_eq(g_hash_table_lookup(cmd->params, "CRM_meta_on_fail"), "block", pcmk__str_casei)) {
+        crm_debug("Setting flag to leave pid group on timeout and "
+                  "only kill action pid for " PCMK__OP_FMT,
+                  cmd->rsc_id, cmd->action, cmd->interval_ms);
+        cmd->service_flags = pcmk__set_flags_as(__func__, __LINE__,
+                                                LOG_TRACE, "Action",
+                                                cmd->action, 0,
+                                                SVC_ACTION_LEAVE_GROUP,
+                                                "SVC_ACTION_LEAVE_GROUP");
+    }
+    return cmd;
+}
+
+static void
+stop_recurring_timer(lrmd_cmd_t *cmd)
+{
+    if (cmd) {
+        if (cmd->stonith_recurring_id) {
+            g_source_remove(cmd->stonith_recurring_id);
+        }
+        cmd->stonith_recurring_id = 0;
+    }
+}
+
+static void
+free_lrmd_cmd(lrmd_cmd_t * cmd)
+{
+    stop_recurring_timer(cmd);
+    if (cmd->delay_id) {
+        g_source_remove(cmd->delay_id);
+    }
+    if (cmd->params) {
+        g_hash_table_destroy(cmd->params);
+    }
+    pcmk__reset_result(&(cmd->result));
+    free(cmd->origin);
+    free(cmd->action);
+    free(cmd->real_action);
+    free(cmd->userdata_str);
+    free(cmd->rsc_id);
+    free(cmd->client_id);
+    free(cmd);
+}
+
+static gboolean
+stonith_recurring_op_helper(gpointer data)
+{
+    lrmd_cmd_t *cmd = data;
+    lrmd_rsc_t *rsc;
+
+    cmd->stonith_recurring_id = 0;
+
+    if (!cmd->rsc_id) {
+        return FALSE;
+    }
+
+    rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);
+
+    CRM_ASSERT(rsc != NULL);
+    /* take it out of recurring_ops list, and put it in the pending ops
+     * to be executed */
+    rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
+    rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
+#ifdef PCMK__TIME_USE_CGT
+    get_current_time(&(cmd->t_queue), &(cmd->t_first_queue));
+#endif
+    mainloop_set_trigger(rsc->work);
+
+    return FALSE;
+}
+
+static inline void
+start_recurring_timer(lrmd_cmd_t *cmd)
+{
+    if (cmd && (cmd->interval_ms > 0)) {
+        cmd->stonith_recurring_id = g_timeout_add(cmd->interval_ms,
+                                                  stonith_recurring_op_helper,
+                                                  cmd);
+    }
+}
+
+static gboolean
+start_delay_helper(gpointer data)
+{
+    lrmd_cmd_t *cmd = data;
+    lrmd_rsc_t *rsc = NULL;
+
+    cmd->delay_id = 0;
+    rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;
+
+    if (rsc) {
+        mainloop_set_trigger(rsc->work);
+    }
+
+    return FALSE;
+}
+
+/*!
+ * \internal
+ * \brief Check whether a list already contains the equivalent of a given action
+ *
+ * \param[in] action_list  List to search
+ * \param[in] cmd          Action to search for
+ */
+static lrmd_cmd_t *
+find_duplicate_action(const GList *action_list, const lrmd_cmd_t *cmd)
+{
+    for (const GList *item = action_list; item != NULL; item = item->next) {
+        lrmd_cmd_t *dup = item->data;
+
+        if (action_matches(cmd, dup->action, dup->interval_ms)) {
+            return dup;
+        }
+    }
+    return NULL;
+}
+
+static bool
+merge_recurring_duplicate(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
+{
+    lrmd_cmd_t * dup = NULL;
+    bool dup_pending = true;
+
+    if (cmd->interval_ms == 0) {
+        return false;
+    }
+
+    // Search for a duplicate of this action (in-flight or not)
+    dup = find_duplicate_action(rsc->pending_ops, cmd);
+    if (dup == NULL) {
+        dup_pending = false;
+        dup = find_duplicate_action(rsc->recurring_ops, cmd);
+        if (dup == NULL) {
+            return false;
+        }
+    }
+
+    /* Do not merge fencing monitors marked for cancellation, so we can reply to
+     * the cancellation separately.
+     */
+    if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
+                     pcmk__str_casei)
+        && (dup->result.execution_status == PCMK_EXEC_CANCELLED)) {
+        return false;
+    }
+
+    /* This should not occur. If it does, we need to investigate how something
+     * like this is possible in the controller.
+     */
+    crm_warn("Duplicate recurring op entry detected (" PCMK__OP_FMT
+             "), merging with previous op entry",
+             rsc->rsc_id, normalize_action_name(rsc, dup->action),
+             dup->interval_ms);
+
+    // Merge new action's call ID and user data into existing action
+    dup->first_notify_sent = false;
+    free(dup->userdata_str);
+    dup->userdata_str = cmd->userdata_str;
+    cmd->userdata_str = NULL;
+    dup->call_id = cmd->call_id;
+    free_lrmd_cmd(cmd);
+    cmd = NULL;
+
+    /* If dup is not pending, that means it has already executed at least once
+     * and is waiting in the interval. In that case, stop waiting and initiate
+     * a new instance now.
+     */
+    if (!dup_pending) {
+        if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
+                         pcmk__str_casei)) {
+            stop_recurring_timer(dup);
+            stonith_recurring_op_helper(dup);
+        } else {
+            services_action_kick(rsc->rsc_id,
+                                 normalize_action_name(rsc, dup->action),
+                                 dup->interval_ms);
+        }
+    }
+    return true;
+}
+
+static void
+schedule_lrmd_cmd(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
+{
+    CRM_CHECK(cmd != NULL, return);
+    CRM_CHECK(rsc != NULL, return);
+
+    crm_trace("Scheduling %s on %s", cmd->action, rsc->rsc_id);
+
+    if (merge_recurring_duplicate(rsc, cmd)) {
+        // Equivalent of cmd has already been scheduled
+        return;
+    }
+
+    /* The controller expects the executor to automatically cancel
+     * recurring operations before a resource stops.
+     */
+    if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
+        cancel_all_recurring(rsc, NULL);
+    }
+
+    rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
+#ifdef PCMK__TIME_USE_CGT
+    get_current_time(&(cmd->t_queue), &(cmd->t_first_queue));
+#endif
+    mainloop_set_trigger(rsc->work);
+
+    if (cmd->start_delay) {
+        cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
+    }
+}
+
+static xmlNode *
+create_lrmd_reply(const char *origin, int rc, int call_id)
+{
+    xmlNode *reply = create_xml_node(NULL, T_LRMD_REPLY);
+
+    crm_xml_add(reply, F_LRMD_ORIGIN, origin);
+    crm_xml_add_int(reply, F_LRMD_RC, rc);
+    crm_xml_add_int(reply, F_LRMD_CALLID, call_id);
+    return reply;
+}
+
+static void
+send_client_notify(gpointer key, gpointer value, gpointer user_data)
+{
+    xmlNode *update_msg = user_data;
+    pcmk__client_t *client = value;
+    int rc;
+    int log_level = LOG_WARNING;
+    const char *msg = NULL;
+
+    CRM_CHECK(client != NULL, return);
+    if (client->name == NULL) {
+        crm_trace("Skipping notification to client without name");
+        return;
+    }
+    if (pcmk_is_set(client->flags, pcmk__client_to_proxy)) {
+        /* We only want to notify clients of the executor IPC API. If we are
+         * running as Pacemaker Remote, we may have clients proxied to other
+         * IPC services in the cluster, so skip those.
+         */
+        crm_trace("Skipping executor API notification to client %s",
+                  pcmk__client_name(client));
+        return;
+    }
+
+    rc = lrmd_server_send_notify(client, update_msg);
+    if (rc == pcmk_rc_ok) {
+        return;
+    }
+
+    switch (rc) {
+        case ENOTCONN:
+        case EPIPE: // Client exited without waiting for notification
+            log_level = LOG_INFO;
+            msg = "Disconnected";
+            break;
+
+        default:
+            msg = pcmk_rc_str(rc);
+            break;
+    }
+    do_crm_log(log_level, "Could not notify client %s: %s " CRM_XS " rc=%d",
+               pcmk__client_name(client), msg, rc);
+}
+
+static void
+send_cmd_complete_notify(lrmd_cmd_t * cmd)
+{
+    xmlNode *notify = NULL;
+    int exec_time = 0;
+    int queue_time = 0;
+
+#ifdef PCMK__TIME_USE_CGT
+    exec_time = time_diff_ms(NULL, &(cmd->t_run));
+    queue_time = time_diff_ms(&cmd->t_run, &(cmd->t_queue));
+#endif
+    log_finished(cmd, exec_time, queue_time);
+
+    /* If the originator requested to be notified only for changes in recurring
+     * operation results, skip the notification if the result hasn't changed.
+     */
+    if (cmd->first_notify_sent
+        && pcmk_is_set(cmd->call_opts, lrmd_opt_notify_changes_only)
+        && (cmd->last_notify_rc == cmd->result.exit_status)
+        && (cmd->last_notify_op_status == cmd->result.execution_status)) {
+        return;
+    }
+
+    cmd->first_notify_sent = true;
+    cmd->last_notify_rc = cmd->result.exit_status;
+    cmd->last_notify_op_status = cmd->result.execution_status;
+
+    notify = create_xml_node(NULL, T_LRMD_NOTIFY);
+
+    crm_xml_add(notify, F_LRMD_ORIGIN, __func__);
+    crm_xml_add_int(notify, F_LRMD_TIMEOUT, cmd->timeout);
+    crm_xml_add_ms(notify, F_LRMD_RSC_INTERVAL, cmd->interval_ms);
+    crm_xml_add_int(notify, F_LRMD_RSC_START_DELAY, cmd->start_delay);
+    crm_xml_add_int(notify, F_LRMD_EXEC_RC, cmd->result.exit_status);
+    crm_xml_add_int(notify, F_LRMD_OP_STATUS, cmd->result.execution_status);
+    crm_xml_add_int(notify, F_LRMD_CALLID, cmd->call_id);
+    crm_xml_add_int(notify, F_LRMD_RSC_DELETED, cmd->rsc_deleted);
+
+    crm_xml_add_ll(notify, F_LRMD_RSC_RUN_TIME,
+                   (long long) cmd->epoch_last_run);
+    crm_xml_add_ll(notify, F_LRMD_RSC_RCCHANGE_TIME,
+                   (long long) cmd->epoch_rcchange);
+#ifdef PCMK__TIME_USE_CGT
+    crm_xml_add_int(notify, F_LRMD_RSC_EXEC_TIME, exec_time);
+    crm_xml_add_int(notify, F_LRMD_RSC_QUEUE_TIME, queue_time);
+#endif
+
+    crm_xml_add(notify, F_LRMD_OPERATION, LRMD_OP_RSC_EXEC);
+    crm_xml_add(notify, F_LRMD_RSC_ID, cmd->rsc_id);
+    if(cmd->real_action) {
+        crm_xml_add(notify, F_LRMD_RSC_ACTION, cmd->real_action);
+    } else {
+        crm_xml_add(notify, F_LRMD_RSC_ACTION, cmd->action);
+    }
+    crm_xml_add(notify, F_LRMD_RSC_USERDATA_STR, cmd->userdata_str);
+    crm_xml_add(notify, F_LRMD_RSC_EXIT_REASON, cmd->result.exit_reason);
+
+    if (cmd->result.action_stderr != NULL) {
+        crm_xml_add(notify, F_LRMD_RSC_OUTPUT, cmd->result.action_stderr);
+
+    } else if (cmd->result.action_stdout != NULL) {
+        crm_xml_add(notify, F_LRMD_RSC_OUTPUT, cmd->result.action_stdout);
+    }
+
+    if (cmd->params) {
+        char *key = NULL;
+        char *value = NULL;
+        GHashTableIter iter;
+
+        xmlNode *args = create_xml_node(notify, XML_TAG_ATTRS);
+
+        g_hash_table_iter_init(&iter, cmd->params);
+        while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
+            hash2smartfield((gpointer) key, (gpointer) value, args);
+        }
+    }
+    if ((cmd->client_id != NULL)
+        && pcmk_is_set(cmd->call_opts, lrmd_opt_notify_orig_only)) {
+
+        pcmk__client_t *client = pcmk__find_client_by_id(cmd->client_id);
+
+        if (client != NULL) {
+            send_client_notify(client->id, client, notify);
+        }
+    } else {
+        pcmk__foreach_ipc_client(send_client_notify, notify);
+    }
+
+    free_xml(notify);
+}
+
+static void
+send_generic_notify(int rc, xmlNode * request)
+{
+    if (pcmk__ipc_client_count() != 0) {
+        int call_id = 0;
+        xmlNode *notify = NULL;
+        xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
+        const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
+        const char *op = crm_element_value(request, F_LRMD_OPERATION);
+
+        crm_element_value_int(request, F_LRMD_CALLID, &call_id);
+
+        notify = create_xml_node(NULL, T_LRMD_NOTIFY);
+        crm_xml_add(notify, F_LRMD_ORIGIN, __func__);
+        crm_xml_add_int(notify, F_LRMD_RC, rc);
+        crm_xml_add_int(notify, F_LRMD_CALLID, call_id);
+        crm_xml_add(notify, F_LRMD_OPERATION, op);
+        crm_xml_add(notify, F_LRMD_RSC_ID, rsc_id);
+
+        pcmk__foreach_ipc_client(send_client_notify, notify);
+
+        free_xml(notify);
+    }
+}
+
+static void
+cmd_reset(lrmd_cmd_t * cmd)
+{
+    cmd->last_pid = 0;
+#ifdef PCMK__TIME_USE_CGT
+    memset(&cmd->t_run, 0, sizeof(cmd->t_run));
+    memset(&cmd->t_queue, 0, sizeof(cmd->t_queue));
+#endif
+    cmd->epoch_last_run = 0;
+
+    pcmk__reset_result(&(cmd->result));
+    cmd->result.execution_status = PCMK_EXEC_DONE;
+}
+
+static void
+cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc)
+{
+    crm_trace("Resource operation rsc:%s action:%s completed (%p %p)", cmd->rsc_id, cmd->action,
+              rsc ? rsc->active : NULL, cmd);
+
+    if (rsc && (rsc->active == cmd)) {
+        rsc->active = NULL;
+        mainloop_set_trigger(rsc->work);
+    }
+
+    if (!rsc) {
+        cmd->rsc_deleted = 1;
+    }
+
+    /* reset original timeout so client notification has correct information */
+    cmd->timeout = cmd->timeout_orig;
+
+    send_cmd_complete_notify(cmd);
+
+    if ((cmd->interval_ms != 0)
+        && (cmd->result.execution_status == PCMK_EXEC_CANCELLED)) {
+
+        if (rsc) {
+            rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
+            rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
+        }
+        free_lrmd_cmd(cmd);
+    } else if (cmd->interval_ms == 0) {
+        if (rsc) {
+            rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
+        }
+        free_lrmd_cmd(cmd);
+    } else {
+        /* Clear all the values pertaining just to the last iteration of a recurring op. */
+        cmd_reset(cmd);
+    }
+}
+
+struct notify_new_client_data {
+    xmlNode *notify;
+    pcmk__client_t *new_client;
+};
+
+static void
+notify_one_client(gpointer key, gpointer value, gpointer user_data)
+{
+    pcmk__client_t *client = value;
+    struct notify_new_client_data *data = user_data;
+
+    if (!pcmk__str_eq(client->id, data->new_client->id, pcmk__str_casei)) {
+        send_client_notify(key, (gpointer) client, (gpointer) data->notify);
+    }
+}
+
+void
+notify_of_new_client(pcmk__client_t *new_client)
+{
+    struct notify_new_client_data data;
+
+    data.new_client = new_client;
+    data.notify = create_xml_node(NULL, T_LRMD_NOTIFY);
+    crm_xml_add(data.notify, F_LRMD_ORIGIN, __func__);
+    crm_xml_add(data.notify, F_LRMD_OPERATION, LRMD_OP_NEW_CLIENT);
+    pcmk__foreach_ipc_client(notify_one_client, &data);
+    free_xml(data.notify);
+}
+
+void
+client_disconnect_cleanup(const char *client_id)
+{
+    GHashTableIter iter;
+    lrmd_rsc_t *rsc = NULL;
+    char *key = NULL;
+
+    g_hash_table_iter_init(&iter, rsc_list);
+    while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & rsc)) {
+        if (pcmk_all_flags_set(rsc->call_opts, lrmd_opt_drop_recurring)) {
+            /* This client is disconnecting, drop any recurring operations
+             * it may have initiated on the resource */
+            cancel_all_recurring(rsc, client_id);
+        }
+    }
+}
+
+static void
+action_complete(svc_action_t * action)
+{
+    lrmd_rsc_t *rsc;
+    lrmd_cmd_t *cmd = action->cb_data;
+    enum ocf_exitcode code;
+
+#ifdef PCMK__TIME_USE_CGT
+    const char *rclass = NULL;
+    bool goagain = false;
+#endif
+
+    if (!cmd) {
+        crm_err("Completed executor action (%s) does not match any known operations",
+                action->id);
+        return;
+    }
+
+#ifdef PCMK__TIME_USE_CGT
+    if (cmd->result.exit_status != action->rc) {
+        cmd->epoch_rcchange = time(NULL);
+    }
+#endif
+
+    cmd->last_pid = action->pid;
+
+    // Cast variable instead of function return to keep compilers happy
+    code = services_result2ocf(action->standard, cmd->action, action->rc);
+    pcmk__set_result(&(cmd->result), (int) code,
+                     action->status, services__exit_reason(action));
+
+    rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;
+
+#ifdef PCMK__TIME_USE_CGT
+    if (rsc && pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
+        rclass = resources_find_service_class(rsc->type);
+    } else if(rsc) {
+        rclass = rsc->class;
+    }
+
+    if (pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_SYSTEMD, pcmk__str_casei)) {
+        if (pcmk__result_ok(&(cmd->result))
+            && pcmk__strcase_any_of(cmd->action, "start", "stop", NULL)) {
+            /* systemd returns from start and stop actions after the action
+             * begins, not after it completes. We have to jump through a few
+             * hoops so that we don't report 'complete' to the rest of pacemaker
+             * until it's actually done.
+             */
+            goagain = true;
+            cmd->real_action = cmd->action;
+            cmd->action = strdup("monitor");
+
+        } else if (cmd->real_action != NULL) {
+            // This is follow-up monitor to check whether start/stop completed
+            if (cmd->result.execution_status == PCMK_EXEC_PENDING) {
+                goagain = true;
+
+            } else if (pcmk__result_ok(&(cmd->result))
+                       && pcmk__str_eq(cmd->real_action, "stop", pcmk__str_casei)) {
+                goagain = true;
+
+            } else {
+                int time_sum = time_diff_ms(NULL, &(cmd->t_first_run));
+                int timeout_left = cmd->timeout_orig - time_sum;
+
+                crm_debug("%s systemd %s is now complete (elapsed=%dms, "
+                          "remaining=%dms): %s (%d)",
+                          cmd->rsc_id, cmd->real_action, time_sum, timeout_left,
+                          services_ocf_exitcode_str(cmd->result.exit_status),
+                          cmd->result.exit_status);
+                cmd_original_times(cmd);
+
+                // Monitors may return "not running", but start/stop shouldn't
+                if ((cmd->result.execution_status == PCMK_EXEC_DONE)
+                    && (cmd->result.exit_status == PCMK_OCF_NOT_RUNNING)) {
+
+                    if (pcmk__str_eq(cmd->real_action, "start", pcmk__str_casei)) {
+                        cmd->result.exit_status = PCMK_OCF_UNKNOWN_ERROR;
+                    } else if (pcmk__str_eq(cmd->real_action, "stop", pcmk__str_casei)) {
+                        cmd->result.exit_status = PCMK_OCF_OK;
+                    }
+                }
+            }
+        }
+    }
+#endif
+
+#if SUPPORT_NAGIOS
+    if (rsc && pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
+        if (action_matches(cmd, "monitor", 0)
+            && pcmk__result_ok(&(cmd->result))) {
+            /* Successfully executed --version for the nagios plugin */
+            cmd->result.exit_status = PCMK_OCF_NOT_RUNNING;
+
+        } else if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)
+                   && !pcmk__result_ok(&(cmd->result))) {
+#ifdef PCMK__TIME_USE_CGT
+            goagain = true;
+#endif
+        }
+    }
+#endif
+
+#ifdef PCMK__TIME_USE_CGT
+    if (goagain) {
+        int time_sum = time_diff_ms(NULL, &(cmd->t_first_run));
+        int timeout_left = cmd->timeout_orig - time_sum;
+        int delay = cmd->timeout_orig / 10;
+
+        if(delay >= timeout_left && timeout_left > 20) {
+            delay = timeout_left/2;
+        }
+
+        delay = QB_MIN(2000, delay);
+        if (delay < timeout_left) {
+            cmd->start_delay = delay;
+            cmd->timeout = timeout_left;
+
+            if (pcmk__result_ok(&(cmd->result))) {
+                crm_debug("%s %s may still be in progress: re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
+                          cmd->rsc_id, cmd->real_action, time_sum, timeout_left, delay);
+
+            } else if (cmd->result.execution_status == PCMK_EXEC_PENDING) {
+                crm_info("%s %s is still in progress: re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
+                         cmd->rsc_id, cmd->action, time_sum, timeout_left, delay);
+
+            } else {
+                crm_notice("%s %s failed '%s' (%d): re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
+                           cmd->rsc_id, cmd->action,
+                           services_ocf_exitcode_str(cmd->result.exit_status),
+                           cmd->result.exit_status, time_sum, timeout_left,
+                           delay);
+            }
+
+            cmd_reset(cmd);
+            if(rsc) {
+                rsc->active = NULL;
+            }
+            schedule_lrmd_cmd(rsc, cmd);
+
+            /* Don't finalize cmd, we're not done with it yet */
+            return;
+
+        } else {
+            crm_notice("Giving up on %s %s (rc=%d): timeout (elapsed=%dms, remaining=%dms)",
+                       cmd->rsc_id,
+                       (cmd->real_action? cmd->real_action : cmd->action),
+                       cmd->result.exit_status, time_sum, timeout_left);
+            pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
+                             PCMK_EXEC_TIMEOUT,
+                             "Investigate reason for timeout, and adjust "
+                             "configured operation timeout if necessary");
+            cmd_original_times(cmd);
+        }
+    }
+#endif
+
+    pcmk__set_result_output(&(cmd->result), services__grab_stdout(action),
+                            services__grab_stderr(action));
+    cmd_finalize(cmd, rsc);
+}
+
+/*!
+ * \internal
+ * \brief Process the result of a fence device action (start, stop, or monitor)
+ *
+ * \param[in,out] cmd               Fence device action that completed
+ * \param[in]     exit_status       Fencer API exit status for action
+ * \param[in]     execution_status  Fencer API execution status for action
+ * \param[in]     exit_reason       Human-friendly detail, if action failed
+ */
+static void
+stonith_action_complete(lrmd_cmd_t *cmd, int exit_status,
+                        enum pcmk_exec_status execution_status,
+                        const char *exit_reason)
+{
+    // This can be NULL if resource was removed before command completed
+    lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);
+
+    // Simplify fencer exit status to uniform exit status
+    if (exit_status != CRM_EX_OK) {
+        exit_status = PCMK_OCF_UNKNOWN_ERROR;
+    }
+
+    if (cmd->result.execution_status == PCMK_EXEC_CANCELLED) {
+        /* An in-flight fence action was cancelled. The execution status is
+         * already correct, so don't overwrite it.
+         */
+        execution_status = PCMK_EXEC_CANCELLED;
+
+    } else {
+        /* Some execution status codes have specific meanings for the fencer
+         * that executor clients may not expect, so map them to a simple error
+         * status.
+         */
+        switch (execution_status) {
+            case PCMK_EXEC_NOT_CONNECTED:
+            case PCMK_EXEC_INVALID:
+                execution_status = PCMK_EXEC_ERROR;
+                break;
+
+            case PCMK_EXEC_NO_FENCE_DEVICE:
+                /* This should be possible only for probes in practice, but
+                 * interpret for all actions to be safe.
+                 */
+                if (pcmk__str_eq(cmd->action, CRMD_ACTION_STATUS,
+                                 pcmk__str_none)) {
+                    exit_status = PCMK_OCF_NOT_RUNNING;
+
+                } else if (pcmk__str_eq(cmd->action, CRMD_ACTION_STOP,
+                                        pcmk__str_none)) {
+                    exit_status = PCMK_OCF_OK;
+
+                } else {
+                    exit_status = PCMK_OCF_NOT_INSTALLED;
+                }
+                execution_status = PCMK_EXEC_ERROR;
+                break;
+
+            case PCMK_EXEC_NOT_SUPPORTED:
+                exit_status = PCMK_OCF_UNIMPLEMENT_FEATURE;
+                break;
+
+            default:
+                break;
+        }
+    }
+
+    pcmk__set_result(&cmd->result, exit_status, execution_status, exit_reason);
+
+    // Certain successful actions change the known state of the resource
+    if ((rsc != NULL) && pcmk__result_ok(&(cmd->result))) {
+
+        if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {
+            pcmk__set_result(&rsc->fence_probe_result, CRM_EX_OK,
+                             PCMK_EXEC_DONE, NULL); // "running"
+
+        } else if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
+            pcmk__set_result(&rsc->fence_probe_result, CRM_EX_ERROR,
+                             PCMK_EXEC_NO_FENCE_DEVICE, NULL); // "not running"
+        }
+    }
+
+    /* The recurring timer should not be running at this point in any case, but
+     * as a failsafe, stop it if it is.
+     */
+    stop_recurring_timer(cmd);
+
+    /* Reschedule this command if appropriate. If a recurring command is *not*
+     * rescheduled, its status must be PCMK_EXEC_CANCELLED, otherwise it will
+     * not be removed from recurring_ops by cmd_finalize().
+     */
+    if (rsc && (cmd->interval_ms > 0)
+        && (cmd->result.execution_status != PCMK_EXEC_CANCELLED)) {
+        start_recurring_timer(cmd);
+    }
+
+    cmd_finalize(cmd, rsc);
+}
+
+static void
+lrmd_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data)
+{
+    if ((data == NULL) || (data->userdata == NULL)) {
+        crm_err("Ignoring fence action result: "
+                "Invalid callback arguments (bug?)");
+    } else {
+        stonith_action_complete((lrmd_cmd_t *) data->userdata,
+                                stonith__exit_status(data),
+                                stonith__execution_status(data),
+                                stonith__exit_reason(data));
+    }
+}
+
+void
+stonith_connection_failed(void)
+{
+    GHashTableIter iter;
+    lrmd_rsc_t *rsc = NULL;
+
+    crm_warn("Connection to fencer lost (any pending operations for "
+             "fence devices will be considered failed)");
+
+    g_hash_table_iter_init(&iter, rsc_list);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &rsc)) {
+        if (!pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
+                          pcmk__str_none)) {
+            continue;
+        }
+
+        /* If we registered this fence device, we don't know whether the
+         * fencer still has the registration or not. Cause future probes to
+         * return an error until the resource is stopped or started
+         * successfully. This is especially important if the controller also
+         * went away (possibly due to a cluster layer restart) and won't
+         * receive our client notification of any monitors finalized below.
+         */
+        if (rsc->fence_probe_result.execution_status == PCMK_EXEC_DONE) {
+            pcmk__set_result(&rsc->fence_probe_result, CRM_EX_ERROR,
+                             PCMK_EXEC_NOT_CONNECTED,
+                             "Lost connection to fencer");
+        }
+
+        // Consider any active, pending, or recurring operations as failed
+
+        for (GList *op = rsc->recurring_ops; op != NULL; op = op->next) {
+            lrmd_cmd_t *cmd = op->data;
+
+            /* This won't free a recurring op but instead restart its timer.
+             * If cmd is rsc->active, this will set rsc->active to NULL, so we
+             * don't have to worry about finalizing it a second time below.
+             */
+            stonith_action_complete(cmd,
+                                    CRM_EX_ERROR, PCMK_EXEC_NOT_CONNECTED,
+                                    "Lost connection to fencer");
+        }
+
+        if (rsc->active != NULL) {
+            rsc->pending_ops = g_list_prepend(rsc->pending_ops, rsc->active);
+        }
+        while (rsc->pending_ops != NULL) {
+            // This will free the op and remove it from rsc->pending_ops
+            stonith_action_complete((lrmd_cmd_t *) rsc->pending_ops->data,
+                                    CRM_EX_ERROR, PCMK_EXEC_NOT_CONNECTED,
+                                    "Lost connection to fencer");
+        }
+    }
+}
+
+/*!
+ * \internal
+ * \brief Execute a stonith resource "start" action
+ *
+ * Start a stonith resource by registering it with the fencer.
+ * (Stonith agents don't have a start command.)
+ *
+ * \param[in,out] stonith_api  Connection to fencer
+ * \param[in]     rsc          Stonith resource to start
+ * \param[in]     cmd          Start command to execute
+ *
+ * \return pcmk_ok on success, -errno otherwise
+ */
+static int
+execd_stonith_start(stonith_t *stonith_api, const lrmd_rsc_t *rsc,
+                    const lrmd_cmd_t *cmd)
+{
+    char *key = NULL;
+    char *value = NULL;
+    stonith_key_value_t *device_params = NULL;
+    int rc = pcmk_ok;
+
+    // Convert command parameters to stonith API key/values
+    if (cmd->params) {
+        GHashTableIter iter;
+
+        g_hash_table_iter_init(&iter, cmd->params);
+        while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
+            device_params = stonith_key_value_add(device_params, key, value);
+        }
+    }
+
+    /* The fencer will automatically register devices via CIB notifications
+     * when the CIB changes, but to avoid a possible race condition between
+     * the fencer receiving the notification and the executor requesting that
+     * resource, the executor registers the device as well. The fencer knows how
+     * to handle duplicate registrations.
+     */
+    rc = stonith_api->cmds->register_device(stonith_api, st_opt_sync_call,
+                                            cmd->rsc_id, rsc->provider,
+                                            rsc->type, device_params);
+
+    stonith_key_value_freeall(device_params, 1, 1);
+    return rc;
+}
+
+/*!
+ * \internal
+ * \brief Execute a stonith resource "stop" action
+ *
+ * Stop a stonith resource by unregistering it with the fencer.
+ * (Stonith agents don't have a stop command.)
+ *
+ * \param[in,out] stonith_api  Connection to fencer
+ * \param[in]     rsc          Stonith resource to stop
+ *
+ * \return pcmk_ok on success, -errno otherwise
+ */
+static inline int
+execd_stonith_stop(stonith_t *stonith_api, const lrmd_rsc_t *rsc)
+{
+    /* @TODO Failure would indicate a problem communicating with fencer;
+     * perhaps we should try reconnecting and retrying a few times?
+     */
+    return stonith_api->cmds->remove_device(stonith_api, st_opt_sync_call,
+                                            rsc->rsc_id);
+}
+
+/*!
+ * \internal
+ * \brief Initiate a stonith resource agent recurring "monitor" action
+ *
+ * \param[in,out] stonith_api  Connection to fencer
+ * \param[in,out] rsc          Stonith resource to monitor
+ * \param[in]     cmd          Monitor command being executed
+ *
+ * \return pcmk_ok if monitor was successfully initiated, -errno otherwise
+ */
+static inline int
+execd_stonith_monitor(stonith_t *stonith_api, lrmd_rsc_t *rsc, lrmd_cmd_t *cmd)
+{
+    int rc = stonith_api->cmds->monitor(stonith_api, 0, cmd->rsc_id,
+                                        cmd->timeout / 1000);
+
+    rc = stonith_api->cmds->register_callback(stonith_api, rc, 0, 0, cmd,
+                                              "lrmd_stonith_callback",
+                                              lrmd_stonith_callback);
+    if (rc == TRUE) {
+        rsc->active = cmd;
+        rc = pcmk_ok;
+    } else {
+        rc = -pcmk_err_generic;
+    }
+    return rc;
+}
+
+static void
+execute_stonith_action(lrmd_rsc_t *rsc, lrmd_cmd_t *cmd)
+{
+    int rc = 0;
+    bool do_monitor = FALSE;
+
+    stonith_t *stonith_api = get_stonith_connection();
+
+    if (pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)
+        && (cmd->interval_ms == 0)) {
+        // Probes don't require a fencer connection
+        stonith_action_complete(cmd, rsc->fence_probe_result.exit_status,
+                                rsc->fence_probe_result.execution_status,
+                                rsc->fence_probe_result.exit_reason);
+        return;
+
+    } else if (stonith_api == NULL) {
+        stonith_action_complete(cmd, PCMK_OCF_UNKNOWN_ERROR,
+                                PCMK_EXEC_NOT_CONNECTED,
+                                "No connection to fencer");
+        return;
+
+    } else if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {
+        rc = execd_stonith_start(stonith_api, rsc, cmd);
+        if (rc == pcmk_ok) {
+            do_monitor = TRUE;
+        }
+
+    } else if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
+        rc = execd_stonith_stop(stonith_api, rsc);
+
+    } else if (pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
+        do_monitor = TRUE;
+
+    } else {
+        stonith_action_complete(cmd, PCMK_OCF_UNIMPLEMENT_FEATURE,
+                                PCMK_EXEC_ERROR,
+                                "Invalid fence device action (bug?)");
+        return;
+    }
+
+    if (do_monitor) {
+        rc = execd_stonith_monitor(stonith_api, rsc, cmd);
+        if (rc == pcmk_ok) {
+            // Don't clean up yet, we will find out result of the monitor later
+            return;
+        }
+    }
+
+    stonith_action_complete(cmd,
+                            ((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR),
+                            stonith__legacy2status(rc),
+                            ((rc == -pcmk_err_generic)? NULL : pcmk_strerror(rc)));
+}
+
+static void
+execute_nonstonith_action(lrmd_rsc_t *rsc, lrmd_cmd_t *cmd)
+{
+    svc_action_t *action = NULL;
+    GHashTable *params_copy = NULL;
+
+    CRM_ASSERT(rsc);
+    CRM_ASSERT(cmd);
+
+    crm_trace("Creating action, resource:%s action:%s class:%s provider:%s agent:%s",
+              rsc->rsc_id, cmd->action, rsc->class, rsc->provider, rsc->type);
+
+#if SUPPORT_NAGIOS
+    /* Recurring operations are cancelled anyway for a stop operation */
+    if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)
+        && pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
+
+        cmd->result.exit_status = PCMK_OCF_OK;
+        cmd_finalize(cmd, rsc);
+        return;
+    }
+#endif
+
+    params_copy = pcmk__str_table_dup(cmd->params);
+
+    action = services__create_resource_action(rsc->rsc_id, rsc->class, rsc->provider,
+                                     rsc->type,
+                                     normalize_action_name(rsc, cmd->action),
+                                     cmd->interval_ms, cmd->timeout,
+                                     params_copy, cmd->service_flags);
+
+    if (action == NULL) {
+        pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
+                         PCMK_EXEC_ERROR, strerror(ENOMEM));
+        cmd_finalize(cmd, rsc);
+        return;
+    }
+
+    if (action->rc != PCMK_OCF_UNKNOWN) {
+        pcmk__set_result(&(cmd->result), action->rc, action->status,
+                         services__exit_reason(action));
+        services_action_free(action);
+        cmd_finalize(cmd, rsc);
+        return;
+    }
+
+    action->cb_data = cmd;
+
+    if (services_action_async(action, action_complete)) {
+        /* The services library has taken responsibility for the action. It
+         * could be pending, blocked, or merged into a duplicate recurring
+         * action, in which case the action callback (action_complete())
+         * will be called when the action completes, otherwise the callback has
+         * already been called.
+         *
+         * action_complete() calls cmd_finalize() which can free cmd, so cmd
+         * cannot be used here.
+         */
+    } else {
+        /* This is a recurring action that is not being cancelled and could not
+         * be initiated. It has been rescheduled, and the action callback
+         * (action_complete()) has been called, which in this case has already
+         * called cmd_finalize(), which in this case should only reset (not
+         * free) cmd.
+         */
+
+        pcmk__set_result(&(cmd->result), action->rc, action->status,
+                         services__exit_reason(action));
+        services_action_free(action);
+    }
+}
+
+static gboolean
+execute_resource_action(gpointer user_data)
+{
+    lrmd_rsc_t *rsc = (lrmd_rsc_t *) user_data;
+    lrmd_cmd_t *cmd = NULL;
+
+    CRM_CHECK(rsc != NULL, return FALSE);
+
+    if (rsc->active) {
+        crm_trace("%s is still active", rsc->rsc_id);
+        return TRUE;
+    }
+
+    if (rsc->pending_ops) {
+        GList *first = rsc->pending_ops;
+
+        cmd = first->data;
+        if (cmd->delay_id) {
+            crm_trace
+                ("Command %s %s was asked to run too early, waiting for start_delay timeout of %dms",
+                 cmd->rsc_id, cmd->action, cmd->start_delay);
+            return TRUE;
+        }
+        rsc->pending_ops = g_list_remove_link(rsc->pending_ops, first);
+        g_list_free_1(first);
+
+#ifdef PCMK__TIME_USE_CGT
+        get_current_time(&(cmd->t_run), &(cmd->t_first_run));
+#endif
+        cmd->epoch_last_run = time(NULL);
+    }
+
+    if (!cmd) {
+        crm_trace("Nothing further to do for %s", rsc->rsc_id);
+        return TRUE;
+    }
+
+    rsc->active = cmd;          /* only one op at a time for a rsc */
+    if (cmd->interval_ms) {
+        rsc->recurring_ops = g_list_append(rsc->recurring_ops, cmd);
+    }
+
+    log_execute(cmd);
+
+    if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
+        execute_stonith_action(rsc, cmd);
+    } else {
+        execute_nonstonith_action(rsc, cmd);
+    }
+
+    return TRUE;
+}
+
+void
+free_rsc(gpointer data)
+{
+    GList *gIter = NULL;
+    lrmd_rsc_t *rsc = data;
+    int is_stonith = pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
+                                  pcmk__str_casei);
+
+    gIter = rsc->pending_ops;
+    while (gIter != NULL) {
+        GList *next = gIter->next;
+        lrmd_cmd_t *cmd = gIter->data;
+
+        /* command was never executed */
+        cmd->result.execution_status = PCMK_EXEC_CANCELLED;
+        cmd_finalize(cmd, NULL);
+
+        gIter = next;
+    }
+    /* frees list, but not list elements. */
+    g_list_free(rsc->pending_ops);
+
+    gIter = rsc->recurring_ops;
+    while (gIter != NULL) {
+        GList *next = gIter->next;
+        lrmd_cmd_t *cmd = gIter->data;
+
+        if (is_stonith) {
+            cmd->result.execution_status = PCMK_EXEC_CANCELLED;
+            /* If a stonith command is in-flight, just mark it as cancelled;
+             * it is not safe to finalize/free the cmd until the stonith api
+             * says it has either completed or timed out.
+             */
+            if (rsc->active != cmd) {
+                cmd_finalize(cmd, NULL);
+            }
+        } else {
+            /* This command is already handed off to service library,
+             * let service library cancel it and tell us via the callback
+             * when it is cancelled. The rsc can be safely destroyed
+             * even if we are waiting for the cancel result */
+            services_action_cancel(rsc->rsc_id,
+                                   normalize_action_name(rsc, cmd->action),
+                                   cmd->interval_ms);
+        }
+
+        gIter = next;
+    }
+    /* frees list, but not list elements. */
+    g_list_free(rsc->recurring_ops);
+
+    free(rsc->rsc_id);
+    free(rsc->class);
+    free(rsc->provider);
+    free(rsc->type);
+    mainloop_destroy_trigger(rsc->work);
+
+    free(rsc);
+}
+
+static int
+process_lrmd_signon(pcmk__client_t *client, xmlNode *request, int call_id,
+                    xmlNode **reply)
+{
+    int rc = pcmk_ok;
+    time_t now = time(NULL);
+    const char *protocol_version = crm_element_value(request, F_LRMD_PROTOCOL_VERSION);
+
+    if (compare_version(protocol_version, LRMD_MIN_PROTOCOL_VERSION) < 0) {
+        crm_err("Cluster API version must be greater than or equal to %s, not %s",
+                LRMD_MIN_PROTOCOL_VERSION, protocol_version);
+        rc = -EPROTO;
+    }
+
+    if (pcmk__xe_attr_is_true(request, F_LRMD_IS_IPC_PROVIDER)) {
+#ifdef PCMK__COMPILE_REMOTE
+        if ((client->remote != NULL)
+            && pcmk_is_set(client->flags,
+                           pcmk__client_tls_handshake_complete)) {
+
+            // This is a remote connection from a cluster node's controller
+            ipc_proxy_add_provider(client);
+        } else {
+            rc = -EACCES;
+        }
+#else
+        rc = -EPROTONOSUPPORT;
+#endif
+    }
+
+    *reply = create_lrmd_reply(__func__, rc, call_id);
+    crm_xml_add(*reply, F_LRMD_OPERATION, CRM_OP_REGISTER);
+    crm_xml_add(*reply, F_LRMD_CLIENTID, client->id);
+    crm_xml_add(*reply, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
+    crm_xml_add_ll(*reply, PCMK__XA_UPTIME, now - start_time);
+
+    return rc;
+}
+
+static int
+process_lrmd_rsc_register(pcmk__client_t *client, uint32_t id, xmlNode *request)
+{
+    int rc = pcmk_ok;
+    lrmd_rsc_t *rsc = build_rsc_from_xml(request);
+    lrmd_rsc_t *dup = g_hash_table_lookup(rsc_list, rsc->rsc_id);
+
+    if (dup &&
+        pcmk__str_eq(rsc->class, dup->class, pcmk__str_casei) &&
+        pcmk__str_eq(rsc->provider, dup->provider, pcmk__str_casei) && pcmk__str_eq(rsc->type, dup->type, pcmk__str_casei)) {
+
+        crm_notice("Ignoring duplicate registration of '%s'", rsc->rsc_id);
+        free_rsc(rsc);
+        return rc;
+    }
+
+    g_hash_table_replace(rsc_list, rsc->rsc_id, rsc);
+    crm_info("Cached agent information for '%s'", rsc->rsc_id);
+    return rc;
+}
+
+static xmlNode *
+process_lrmd_get_rsc_info(xmlNode *request, int call_id)
+{
+    int rc = pcmk_ok;
+    xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
+    const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
+    xmlNode *reply = NULL;
+    lrmd_rsc_t *rsc = NULL;
+
+    if (rsc_id == NULL) {
+        rc = -ENODEV;
+    } else {
+        rsc = g_hash_table_lookup(rsc_list, rsc_id);
+        if (rsc == NULL) {
+            crm_info("Agent information for '%s' not in cache", rsc_id);
+            rc = -ENODEV;
+        }
+    }
+
+    reply = create_lrmd_reply(__func__, rc, call_id);
+    if (rsc) {
+        crm_xml_add(reply, F_LRMD_RSC_ID, rsc->rsc_id);
+        crm_xml_add(reply, F_LRMD_CLASS, rsc->class);
+        crm_xml_add(reply, F_LRMD_PROVIDER, rsc->provider);
+        crm_xml_add(reply, F_LRMD_TYPE, rsc->type);
+    }
+    return reply;
+}
+
+static int
+process_lrmd_rsc_unregister(pcmk__client_t *client, uint32_t id,
+                            xmlNode *request)
+{
+    int rc = pcmk_ok;
+    lrmd_rsc_t *rsc = NULL;
+    xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
+    const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
+
+    if (!rsc_id) {
+        return -ENODEV;
+    }
+
+    rsc = g_hash_table_lookup(rsc_list, rsc_id);
+    if (rsc == NULL) {
+        crm_info("Ignoring unregistration of resource '%s', which is not registered",
+                 rsc_id);
+        return pcmk_ok;
+    }
+
+    if (rsc->active) {
+        /* let the caller know there are still active ops on this rsc to watch for */
+        crm_trace("Operation (%p) still in progress for unregistered resource %s",
+                  rsc->active, rsc_id);
+        rc = -EINPROGRESS;
+    }
+
+    g_hash_table_remove(rsc_list, rsc_id);
+
+    return rc;
+}
+
+static int
+process_lrmd_rsc_exec(pcmk__client_t *client, uint32_t id, xmlNode *request)
+{
+    lrmd_rsc_t *rsc = NULL;
+    lrmd_cmd_t *cmd = NULL;
+    xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
+    const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
+    int call_id;
+
+    if (!rsc_id) {
+        return -EINVAL;
+    }
+    if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
+        crm_info("Resource '%s' not found (%d active resources)",
+                 rsc_id, g_hash_table_size(rsc_list));
+        return -ENODEV;
+    }
+
+    cmd = create_lrmd_cmd(request, client);
+    call_id = cmd->call_id;
+
+    /* Don't reference cmd after handing it off to be scheduled.
+     * The cmd could get merged and freed. */
+    schedule_lrmd_cmd(rsc, cmd);
+
+    return call_id;
+}
+
+static int
+cancel_op(const char *rsc_id, const char *action, guint interval_ms)
+{
+    GList *gIter = NULL;
+    lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, rsc_id);
+
+    /* How to cancel an action.
+     * 1. Check pending ops list, if it hasn't been handed off
+     *    to the service library or stonith recurring list remove
+     *    it there and that will stop it.
+     * 2. If it isn't in the pending ops list, then it's either a
+     *    recurring op in the stonith recurring list, or the service
+     *    library's recurring list.  Stop it there
+     * 3. If not found in any lists, then this operation has either
+     *    been executed already and is not a recurring operation, or
+     *    never existed.
+     */
+    if (!rsc) {
+        return -ENODEV;
+    }
+
+    for (gIter = rsc->pending_ops; gIter != NULL; gIter = gIter->next) {
+        lrmd_cmd_t *cmd = gIter->data;
+
+        if (action_matches(cmd, action, interval_ms)) {
+            cmd->result.execution_status = PCMK_EXEC_CANCELLED;
+            cmd_finalize(cmd, rsc);
+            return pcmk_ok;
+        }
+    }
+
+    if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
+        /* The service library does not handle stonith operations.
+         * We have to handle recurring stonith operations ourselves. */
+        for (gIter = rsc->recurring_ops; gIter != NULL; gIter = gIter->next) {
+            lrmd_cmd_t *cmd = gIter->data;
+
+            if (action_matches(cmd, action, interval_ms)) {
+                cmd->result.execution_status = PCMK_EXEC_CANCELLED;
+                if (rsc->active != cmd) {
+                    cmd_finalize(cmd, rsc);
+                }
+                return pcmk_ok;
+            }
+        }
+    } else if (services_action_cancel(rsc_id,
+                                      normalize_action_name(rsc, action),
+                                      interval_ms) == TRUE) {
+        /* The service library will tell the action_complete callback function
+         * this action was cancelled, which will destroy the cmd and remove
+         * it from the recurring_op list. Do not do that in this function
+         * if the service library says it cancelled it. */
+        return pcmk_ok;
+    }
+
+    return -EOPNOTSUPP;
+}
+
+static void
+cancel_all_recurring(lrmd_rsc_t * rsc, const char *client_id)
+{
+    GList *cmd_list = NULL;
+    GList *cmd_iter = NULL;
+
+    /* Notice a copy of each list is created when concat is called.
+     * This prevents odd behavior from occurring when the cmd_list
+     * is iterated through later on.  It is possible the cancel_op
+     * function may end up modifying the recurring_ops and pending_ops
+     * lists.  If we did not copy those lists, our cmd_list iteration
+     * could get messed up.*/
+    if (rsc->recurring_ops) {
+        cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->recurring_ops));
+    }
+    if (rsc->pending_ops) {
+        cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->pending_ops));
+    }
+    if (!cmd_list) {
+        return;
+    }
+
+    for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) {
+        lrmd_cmd_t *cmd = cmd_iter->data;
+
+        if (cmd->interval_ms == 0) {
+            continue;
+        }
+
+        if (client_id && !pcmk__str_eq(cmd->client_id, client_id, pcmk__str_casei)) {
+            continue;
+        }
+
+        cancel_op(rsc->rsc_id, cmd->action, cmd->interval_ms);
+    }
+    /* frees only the copied list data, not the cmds */
+    g_list_free(cmd_list);
+}
+
+static int
+process_lrmd_rsc_cancel(pcmk__client_t *client, uint32_t id, xmlNode *request)
+{
+    xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
+    const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
+    const char *action = crm_element_value(rsc_xml, F_LRMD_RSC_ACTION);
+    guint interval_ms = 0;
+
+    crm_element_value_ms(rsc_xml, F_LRMD_RSC_INTERVAL, &interval_ms);
+
+    if (!rsc_id || !action) {
+        return -EINVAL;
+    }
+
+    return cancel_op(rsc_id, action, interval_ms);
+}
+
+static void
+add_recurring_op_xml(xmlNode *reply, lrmd_rsc_t *rsc)
+{
+    xmlNode *rsc_xml = create_xml_node(reply, F_LRMD_RSC);
+
+    crm_xml_add(rsc_xml, F_LRMD_RSC_ID, rsc->rsc_id);
+    for (GList *item = rsc->recurring_ops; item != NULL; item = item->next) {
+        lrmd_cmd_t *cmd = item->data;
+        xmlNode *op_xml = create_xml_node(rsc_xml, T_LRMD_RSC_OP);
+
+        crm_xml_add(op_xml, F_LRMD_RSC_ACTION,
+                    (cmd->real_action? cmd->real_action : cmd->action));
+        crm_xml_add_ms(op_xml, F_LRMD_RSC_INTERVAL, cmd->interval_ms);
+        crm_xml_add_int(op_xml, F_LRMD_TIMEOUT, cmd->timeout_orig);
+    }
+}
+
+static xmlNode *
+process_lrmd_get_recurring(xmlNode *request, int call_id)
+{
+    int rc = pcmk_ok;
+    const char *rsc_id = NULL;
+    lrmd_rsc_t *rsc = NULL;
+    xmlNode *reply = NULL;
+    xmlNode *rsc_xml = NULL;
+
+    // Resource ID is optional
+    rsc_xml = first_named_child(request, F_LRMD_CALLDATA);
+    if (rsc_xml) {
+        rsc_xml = first_named_child(rsc_xml, F_LRMD_RSC);
+    }
+    if (rsc_xml) {
+        rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
+    }
+
+    // If resource ID is specified, resource must exist
+    if (rsc_id != NULL) {
+        rsc = g_hash_table_lookup(rsc_list, rsc_id);
+        if (rsc == NULL) {
+            crm_info("Resource '%s' not found (%d active resources)",
+                     rsc_id, g_hash_table_size(rsc_list));
+            rc = -ENODEV;
+        }
+    }
+
+    reply = create_lrmd_reply(__func__, rc, call_id);
+
+    // If resource ID is not specified, check all resources
+    if (rsc_id == NULL) {
+        GHashTableIter iter;
+        char *key = NULL;
+
+        g_hash_table_iter_init(&iter, rsc_list);
+        while (g_hash_table_iter_next(&iter, (gpointer *) &key,
+                                      (gpointer *) &rsc)) {
+            add_recurring_op_xml(reply, rsc);
+        }
+    } else if (rsc) {
+        add_recurring_op_xml(reply, rsc);
+    }
+    return reply;
+}
+
+void
+process_lrmd_message(pcmk__client_t *client, uint32_t id, xmlNode *request)
+{
+    int rc = pcmk_ok;
+    int call_id = 0;
+    const char *op = crm_element_value(request, F_LRMD_OPERATION);
+    int do_reply = 0;
+    int do_notify = 0;
+    xmlNode *reply = NULL;
+
+    /* Certain IPC commands may be done only by privileged users (i.e. root or
+     * hacluster), because they would otherwise provide a means of bypassing
+     * ACLs.
+     */
+    bool allowed = pcmk_is_set(client->flags, pcmk__client_privileged);
+
+    crm_trace("Processing %s operation from %s", op, client->id);
+    crm_element_value_int(request, F_LRMD_CALLID, &call_id);
+
+    if (pcmk__str_eq(op, CRM_OP_IPC_FWD, pcmk__str_none)) {
+#ifdef PCMK__COMPILE_REMOTE
+        if (allowed) {
+            ipc_proxy_forward_client(client, request);
+        } else {
+            rc = -EACCES;
+        }
+#else
+        rc = -EPROTONOSUPPORT;
+#endif
+        do_reply = 1;
+    } else if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
+        rc = process_lrmd_signon(client, request, call_id, &reply);
+        do_reply = 1;
+    } else if (pcmk__str_eq(op, LRMD_OP_RSC_REG, pcmk__str_none)) {
+        if (allowed) {
+            rc = process_lrmd_rsc_register(client, id, request);
+            do_notify = 1;
+        } else {
+            rc = -EACCES;
+        }
+        do_reply = 1;
+    } else if (pcmk__str_eq(op, LRMD_OP_RSC_INFO, pcmk__str_none)) {
+        if (allowed) {
+            reply = process_lrmd_get_rsc_info(request, call_id);
+        } else {
+            rc = -EACCES;
+        }
+        do_reply = 1;
+    } else if (pcmk__str_eq(op, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
+        if (allowed) {
+            rc = process_lrmd_rsc_unregister(client, id, request);
+            /* don't notify anyone about failed un-registers */
+            if (rc == pcmk_ok || rc == -EINPROGRESS) {
+                do_notify = 1;
+            }
+        } else {
+            rc = -EACCES;
+        }
+        do_reply = 1;
+    } else if (pcmk__str_eq(op, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
+        if (allowed) {
+            rc = process_lrmd_rsc_exec(client, id, request);
+        } else {
+            rc = -EACCES;
+        }
+        do_reply = 1;
+    } else if (pcmk__str_eq(op, LRMD_OP_RSC_CANCEL, pcmk__str_none)) {
+        if (allowed) {
+            rc = process_lrmd_rsc_cancel(client, id, request);
+        } else {
+            rc = -EACCES;
+        }
+        do_reply = 1;
+    } else if (pcmk__str_eq(op, LRMD_OP_POKE, pcmk__str_none)) {
+        do_notify = 1;
+        do_reply = 1;
+    } else if (pcmk__str_eq(op, LRMD_OP_CHECK, pcmk__str_none)) {
+        if (allowed) {
+            xmlNode *data = get_message_xml(request, F_LRMD_CALLDATA);
+
+            CRM_LOG_ASSERT(data != NULL);
+            pcmk__valid_sbd_timeout(crm_element_value(data, F_LRMD_WATCHDOG));
+        } else {
+            rc = -EACCES;
+        }
+    } else if (pcmk__str_eq(op, LRMD_OP_ALERT_EXEC, pcmk__str_none)) {
+        if (allowed) {
+            rc = process_lrmd_alert_exec(client, id, request);
+        } else {
+            rc = -EACCES;
+        }
+        do_reply = 1;
+    } else if (pcmk__str_eq(op, LRMD_OP_GET_RECURRING, pcmk__str_none)) {
+        if (allowed) {
+            reply = process_lrmd_get_recurring(request, call_id);
+        } else {
+            rc = -EACCES;
+        }
+        do_reply = 1;
+    } else {
+        rc = -EOPNOTSUPP;
+        do_reply = 1;
+        crm_err("Unknown IPC request '%s' from client %s",
+                op, pcmk__client_name(client));
+    }
+
+    if (rc == -EACCES) {
+        crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
+                 op, pcmk__client_name(client));
+    }
+
+    crm_debug("Processed %s operation from %s: rc=%d, reply=%d, notify=%d",
+              op, client->id, rc, do_reply, do_notify);
+
+    if (do_reply) {
+        int send_rc = pcmk_rc_ok;
+
+        if (reply == NULL) {
+            reply = create_lrmd_reply(__func__, rc, call_id);
+        }
+        send_rc = lrmd_server_send_reply(client, id, reply);
+        free_xml(reply);
+        if (send_rc != pcmk_rc_ok) {
+            crm_warn("Reply to client %s failed: %s " CRM_XS " rc=%d",
+                     pcmk__client_name(client), pcmk_rc_str(send_rc), send_rc);
+        }
+    }
+
+    if (do_notify) {
+        send_generic_notify(rc, request);
+    }
+}
diff --git a/daemons/execd/pacemaker-execd.c b/daemons/execd/pacemaker-execd.c
new file mode 100644
index 0000000..83a8cd7
--- /dev/null
+++ b/daemons/execd/pacemaker-execd.c
@@ -0,0 +1,582 @@
+/*
+ * Copyright 2012-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <glib.h>
+#include <signal.h>
+#include <sys/types.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/services.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/common/mainloop.h>
+#include <crm/common/output_internal.h>
+#include <crm/common/remote_internal.h>
+#include <crm/lrmd_internal.h>
+
+#include "pacemaker-execd.h"
+
+#ifdef PCMK__COMPILE_REMOTE
+#  define EXECD_TYPE "remote"
+#  define EXECD_NAME "pacemaker-remoted"
+#  define SUMMARY "resource agent executor daemon for Pacemaker Remote nodes"
+#else
+#  define EXECD_TYPE "local"
+#  define EXECD_NAME "pacemaker-execd"
+#  define SUMMARY "resource agent executor daemon for Pacemaker cluster nodes"
+#endif
+
+static GMainLoop *mainloop = NULL;
+static qb_ipcs_service_t *ipcs = NULL;
+static stonith_t *stonith_api = NULL;
+int lrmd_call_id = 0;
+time_t start_time;
+
+static struct {
+    gchar **log_files;
+#ifdef PCMK__COMPILE_REMOTE
+    gchar *port;
+#endif  // PCMK__COMPILE_REMOTE
+} options;
+
+#ifdef PCMK__COMPILE_REMOTE
+/* whether shutdown request has been sent */
+static gboolean shutting_down = FALSE;
+
+/* timer for waiting for acknowledgment of shutdown request */
+static guint shutdown_ack_timer = 0;
+
+static gboolean lrmd_exit(gpointer data);
+#endif
+
+static void
+stonith_connection_destroy_cb(stonith_t * st, stonith_event_t * e)
+{
+    stonith_api->state = stonith_disconnected;
+    stonith_connection_failed();
+}
+
+stonith_t *
+get_stonith_connection(void)
+{
+    if (stonith_api && stonith_api->state == stonith_disconnected) {
+        stonith_api_delete(stonith_api);
+        stonith_api = NULL;
+    }
+
+    if (stonith_api == NULL) {
+        int rc = pcmk_ok;
+
+        stonith_api = stonith_api_new();
+        if (stonith_api == NULL) {
+            crm_err("Could not connect to fencer: API memory allocation failed");
+            return NULL;
+        }
+        rc = stonith_api_connect_retry(stonith_api, crm_system_name, 10);
+        if (rc != pcmk_ok) {
+            crm_err("Could not connect to fencer in 10 attempts: %s "
+                    CRM_XS " rc=%d", pcmk_strerror(rc), rc);
+            stonith_api_delete(stonith_api);
+            stonith_api = NULL;
+        } else {
+            stonith_api->cmds->register_notification(stonith_api,
+                                                     T_STONITH_NOTIFY_DISCONNECT,
+                                                     stonith_connection_destroy_cb);
+        }
+    }
+    return stonith_api;
+}
+
+static int32_t
+lrmd_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
+{
+    crm_trace("Connection %p", c);
+    if (pcmk__new_client(c, uid, gid) == NULL) {
+        return -EIO;
+    }
+    return 0;
+}
+
+static void
+lrmd_ipc_created(qb_ipcs_connection_t * c)
+{
+    pcmk__client_t *new_client = pcmk__find_client(c);
+
+    crm_trace("Connection %p", c);
+    CRM_ASSERT(new_client != NULL);
+    /* Now that the connection is offically established, alert
+     * the other clients a new connection exists. */
+
+    notify_of_new_client(new_client);
+}
+
+static int32_t
+lrmd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
+{
+    uint32_t id = 0;
+    uint32_t flags = 0;
+    pcmk__client_t *client = pcmk__find_client(c);
+    xmlNode *request = pcmk__client_data2xml(client, data, &id, &flags);
+
+    CRM_CHECK(client != NULL, crm_err("Invalid client");
+              return FALSE);
+    CRM_CHECK(client->id != NULL, crm_err("Invalid client: %p", client);
+              return FALSE);
+
+    CRM_CHECK(flags & crm_ipc_client_response, crm_err("Invalid client request: %p", client);
+              return FALSE);
+
+    if (!request) {
+        return 0;
+    }
+
+    if (!client->name) {
+        const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);
+
+        if (value == NULL) {
+            client->name = pcmk__itoa(pcmk__client_pid(c));
+        } else {
+            client->name = strdup(value);
+        }
+    }
+
+    lrmd_call_id++;
+    if (lrmd_call_id < 1) {
+        lrmd_call_id = 1;
+    }
+
+    crm_xml_add(request, F_LRMD_CLIENTID, client->id);
+    crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
+    crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);
+
+    process_lrmd_message(client, id, request);
+
+    free_xml(request);
+    return 0;
+}
+
+/*!
+ * \internal
+ * \brief Free a client connection, and exit if appropriate
+ *
+ * \param[in,out] client  Client connection to free
+ */
+void
+lrmd_client_destroy(pcmk__client_t *client)
+{
+    pcmk__free_client(client);
+
+#ifdef PCMK__COMPILE_REMOTE
+    /* If we were waiting to shut down, we can now safely do so
+     * if there are no more proxied IPC providers
+     */
+    if (shutting_down && (ipc_proxy_get_provider() == NULL)) {
+        lrmd_exit(NULL);
+    }
+#endif
+}
+
+static int32_t
+lrmd_ipc_closed(qb_ipcs_connection_t * c)
+{
+    pcmk__client_t *client = pcmk__find_client(c);
+
+    if (client == NULL) {
+        return 0;
+    }
+
+    crm_trace("Connection %p", c);
+    client_disconnect_cleanup(client->id);
+#ifdef PCMK__COMPILE_REMOTE
+    ipc_proxy_remove_provider(client);
+#endif
+    lrmd_client_destroy(client);
+    return 0;
+}
+
+static void
+lrmd_ipc_destroy(qb_ipcs_connection_t * c)
+{
+    lrmd_ipc_closed(c);
+    crm_trace("Connection %p", c);
+}
+
+static struct qb_ipcs_service_handlers lrmd_ipc_callbacks = {
+    .connection_accept = lrmd_ipc_accept,
+    .connection_created = lrmd_ipc_created,
+    .msg_process = lrmd_ipc_dispatch,
+    .connection_closed = lrmd_ipc_closed,
+    .connection_destroyed = lrmd_ipc_destroy
+};
+
+// \return Standard Pacemaker return code
+int
+lrmd_server_send_reply(pcmk__client_t *client, uint32_t id, xmlNode *reply)
+{
+    crm_trace("Sending reply (%d) to client (%s)", id, client->id);
+    switch (PCMK__CLIENT_TYPE(client)) {
+        case pcmk__client_ipc:
+            return pcmk__ipc_send_xml(client, id, reply, FALSE);
+#ifdef PCMK__COMPILE_REMOTE
+        case pcmk__client_tls:
+            return lrmd__remote_send_xml(client->remote, reply, id, "reply");
+#endif
+        default:
+            crm_err("Could not send reply: unknown type for client %s "
+                    CRM_XS " flags=%#llx",
+                    pcmk__client_name(client), client->flags);
+    }
+    return ENOTCONN;
+}
+
+// \return Standard Pacemaker return code
+int
+lrmd_server_send_notify(pcmk__client_t *client, xmlNode *msg)
+{
+    crm_trace("Sending notification to client (%s)", client->id);
+    switch (PCMK__CLIENT_TYPE(client)) {
+        case pcmk__client_ipc:
+            if (client->ipcs == NULL) {
+                crm_trace("Could not notify local client: disconnected");
+                return ENOTCONN;
+            }
+            return pcmk__ipc_send_xml(client, 0, msg, crm_ipc_server_event);
+#ifdef PCMK__COMPILE_REMOTE
+        case pcmk__client_tls:
+            if (client->remote == NULL) {
+                crm_trace("Could not notify remote client: disconnected");
+                return ENOTCONN;
+            } else {
+                return lrmd__remote_send_xml(client->remote, msg, 0, "notify");
+            }
+#endif
+        default:
+            crm_err("Could not notify client %s with unknown transport "
+                    CRM_XS " flags=%#llx",
+                    pcmk__client_name(client), client->flags);
+    }
+    return ENOTCONN;
+}
+
+/*!
+ * \internal
+ * \brief Clean up and exit immediately
+ *
+ * \param[in] data  Ignored
+ *
+ * \return Doesn't return
+ * \note   This can be used as a timer callback.
+ */
+static gboolean
+lrmd_exit(gpointer data)
+{
+    crm_info("Terminating with %d clients", pcmk__ipc_client_count());
+    if (stonith_api) {
+        stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT);
+        stonith_api->cmds->disconnect(stonith_api);
+        stonith_api_delete(stonith_api);
+    }
+    if (ipcs) {
+        mainloop_del_ipc_server(ipcs);
+    }
+
+#ifdef PCMK__COMPILE_REMOTE
+    execd_stop_tls_server();
+    ipc_proxy_cleanup();
+#endif
+
+    pcmk__client_cleanup();
+    g_hash_table_destroy(rsc_list);
+
+    if (mainloop) {
+        lrmd_drain_alerts(mainloop);
+    }
+
+    crm_exit(CRM_EX_OK);
+    return FALSE;
+}
+
+/*!
+ * \internal
+ * \brief Request cluster shutdown if appropriate, otherwise exit immediately
+ *
+ * \param[in] nsig  Signal that caused invocation (ignored)
+ */
+static void
+lrmd_shutdown(int nsig)
+{
+#ifdef PCMK__COMPILE_REMOTE
+    pcmk__client_t *ipc_proxy = ipc_proxy_get_provider();
+
+    /* If there are active proxied IPC providers, then we may be running
+     * resources, so notify the cluster that we wish to shut down.
+     */
+    if (ipc_proxy) {
+        if (shutting_down) {
+            crm_notice("Waiting for cluster to stop resources before exiting");
+            return;
+        }
+
+        crm_info("Sending shutdown request to cluster");
+        if (ipc_proxy_shutdown_req(ipc_proxy) < 0) {
+            crm_crit("Shutdown request failed, exiting immediately");
+
+        } else {
+            /* We requested a shutdown. Now, we need to wait for an
+             * acknowledgement from the proxy host (which ensures the proxy host
+             * supports shutdown requests), then wait for all proxy hosts to
+             * disconnect (which ensures that all resources have been stopped).
+             */
+            shutting_down = TRUE;
+
+            /* Stop accepting new proxy connections */
+            execd_stop_tls_server();
+
+            /* Older controller versions will never acknowledge our request, so
+             * set a fairly short timeout to exit quickly in that case. If we
+             * get the ack, we'll defuse this timer.
+             */
+            shutdown_ack_timer = g_timeout_add_seconds(20, lrmd_exit, NULL);
+
+            /* Currently, we let the OS kill us if the clients don't disconnect
+             * in a reasonable time. We could instead set a long timer here
+             * (shorter than what the OS is likely to use) and exit immediately
+             * if it pops.
+             */
+            return;
+        }
+    }
+#endif
+    lrmd_exit(NULL);
+}
+
+/*!
+ * \internal
+ * \brief Defuse short exit timer if shutting down
+ */
+void
+handle_shutdown_ack(void)
+{
+#ifdef PCMK__COMPILE_REMOTE
+    if (shutting_down) {
+        crm_info("Received shutdown ack");
+        if (shutdown_ack_timer > 0) {
+            g_source_remove(shutdown_ack_timer);
+            shutdown_ack_timer = 0;
+        }
+        return;
+    }
+#endif
+    crm_debug("Ignoring unexpected shutdown ack");
+}
+
+/*!
+ * \internal
+ * \brief Make short exit timer fire immediately
+ */
+void
+handle_shutdown_nack(void)
+{
+#ifdef PCMK__COMPILE_REMOTE
+    if (shutting_down) {
+        crm_info("Received shutdown nack");
+        if (shutdown_ack_timer > 0) {
+            g_source_remove(shutdown_ack_timer);
+            shutdown_ack_timer = g_timeout_add(0, lrmd_exit, NULL);
+        }
+        return;
+    }
+#endif
+    crm_debug("Ignoring unexpected shutdown nack");
+}
+
+static GOptionEntry entries[] = {
+    { "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY,
+      &options.log_files, "Send logs to the additional named logfile", NULL },
+
+#ifdef PCMK__COMPILE_REMOTE
+
+    { "port", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.port,
+      "Port to listen on (defaults to " G_STRINGIFY(DEFAULT_REMOTE_PORT) ")", NULL },
+#endif  // PCMK__COMPILE_REMOTE
+
+    { NULL }
+};
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group)
+{
+    GOptionContext *context = NULL;
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
+    pcmk__add_main_args(context, entries);
+    return context;
+}
+
+int
+main(int argc, char **argv, char **envp)
+{
+    int rc = pcmk_rc_ok;
+    crm_exit_t exit_code = CRM_EX_OK;
+
+    const char *option = NULL;
+
+    pcmk__output_t *out = NULL;
+
+    GError *error = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+#ifdef PCMK__COMPILE_REMOTE
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "lp");
+#else
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "l");
+#endif  // PCMK__COMPILE_REMOTE
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+#ifdef PCMK__COMPILE_REMOTE
+    // If necessary, create PID 1 now before any file descriptors are opened
+    remoted_spawn_pidone(argc, argv, envp);
+#endif
+
+    crm_log_preinit(EXECD_NAME, argc, argv);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    // Open additional log files
+    if (options.log_files != NULL) {
+        for (gchar **fname = options.log_files; *fname != NULL; fname++) {
+            rc = pcmk__add_logfile(*fname);
+
+            if (rc != pcmk_rc_ok) {
+                out->err(out, "Logging to %s is disabled: %s",
+                         *fname, pcmk_rc_str(rc));
+            }
+        }
+    }
+
+    pcmk__cli_init_logging(EXECD_NAME, args->verbosity);
+    crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
+
+    option = pcmk__env_option(PCMK__ENV_LOGFACILITY);
+    if (!pcmk__str_eq(option, PCMK__VALUE_NONE,
+                      pcmk__str_casei|pcmk__str_null_matches)
+        && !pcmk__str_eq(option, "/dev/null", pcmk__str_none)) {
+        setenv("HA_LOGFACILITY", option, 1);  /* Used by the ocf_log/ha_log OCF macro */
+    }
+
+    option = pcmk__env_option(PCMK__ENV_LOGFILE);
+    if (!pcmk__str_eq(option, PCMK__VALUE_NONE,
+                      pcmk__str_casei|pcmk__str_null_matches)) {
+        setenv("HA_LOGFILE", option, 1);      /* Used by the ocf_log/ha_log OCF macro */
+
+        if (pcmk__env_option_enabled(crm_system_name, PCMK__ENV_DEBUG)) {
+            setenv("HA_DEBUGLOG", option, 1); /* Used by the ocf_log/ha_debug OCF macro */
+        }
+    }
+
+#ifdef PCMK__COMPILE_REMOTE
+    if (options.port != NULL) {
+        setenv("PCMK_remote_port", options.port, 1);
+    }
+#endif  // PCMK__COMPILE_REMOTE
+
+    start_time = time(NULL);
+
+    crm_notice("Starting Pacemaker " EXECD_TYPE " executor");
+
+    /* The presence of this variable allegedly controls whether child
+     * processes like httpd will try and use Systemd's sd_notify
+     * API
+     */
+    unsetenv("NOTIFY_SOCKET");
+
+    {
+        // Temporary directory for resource agent use (leave owned by root)
+        int rc = pcmk__build_path(CRM_RSCTMP_DIR, 0755);
+
+        if (rc != pcmk_rc_ok) {
+            crm_warn("Could not create resource agent temporary directory "
+                     CRM_RSCTMP_DIR ": %s", pcmk_rc_str(rc));
+        }
+    }
+
+    rsc_list = pcmk__strkey_table(NULL, free_rsc);
+    ipcs = mainloop_add_ipc_server(CRM_SYSTEM_LRMD, QB_IPC_SHM, &lrmd_ipc_callbacks);
+    if (ipcs == NULL) {
+        crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
+        exit_code = CRM_EX_FATAL;
+        goto done;
+    }
+
+#ifdef PCMK__COMPILE_REMOTE
+    if (lrmd_init_remote_tls_server() < 0) {
+        crm_err("Failed to create TLS listener: shutting down and staying down");
+        exit_code = CRM_EX_FATAL;
+        goto done;
+    }
+    ipc_proxy_init();
+#endif
+
+    mainloop_add_signal(SIGTERM, lrmd_shutdown);
+    mainloop = g_main_loop_new(NULL, FALSE);
+    crm_notice("Pacemaker " EXECD_TYPE " executor successfully started and accepting connections");
+    crm_notice("OCF resource agent search path is %s", OCF_RA_PATH);
+    g_main_loop_run(mainloop);
+
+    /* should never get here */
+    lrmd_exit(NULL);
+
+done:
+    g_strfreev(options.log_files);
+#ifdef PCMK__COMPILE_REMOTE
+    g_free(options.port);
+#endif  // PCMK__COMPILE_REMOTE
+
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+    pcmk__unregister_formats();
+    crm_exit(exit_code);
+}
diff --git a/daemons/execd/pacemaker-execd.h b/daemons/execd/pacemaker-execd.h
new file mode 100644
index 0000000..9c1d173
--- /dev/null
+++ b/daemons/execd/pacemaker-execd.h
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2012-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef PACEMAKER_EXECD__H
+#  define PACEMAKER_EXECD__H
+
+#  include <glib.h>
+#  include <crm/common/ipc_internal.h>
+#  include <crm/lrmd.h>
+#  include <crm/stonith-ng.h>
+
+#  ifdef HAVE_GNUTLS_GNUTLS_H
+#    include <gnutls/gnutls.h>
+#  endif
+
+extern GHashTable *rsc_list;
+extern time_t start_time;
+
+typedef struct lrmd_rsc_s {
+    char *rsc_id;
+    char *class;
+    char *provider;
+    char *type;
+
+    int call_opts;
+
+    /* NEVER dereference this pointer,
+     * It simply exists as a switch to let us know
+     * when the currently active operation has completed */
+    void *active;
+
+    /* Operations in this list
+     * have not been executed yet. */
+    GList *pending_ops;
+    /* Operations in this list are recurring operations
+     * that have been handed off from the pending ops list. */
+    GList *recurring_ops;
+
+    /* If this resource is a fence device, probes are handled internally by the
+     * executor, and this value indicates the result that should currently be
+     * returned for probes. It should be one of:
+     * PCMK_EXEC_DONE (to indicate "running"),
+     * PCMK_EXEC_NO_FENCE_DEVICE ("not running"), or
+     * PCMK_EXEC_NOT_CONNECTED ("unknown because fencer connection was lost").
+     */
+    pcmk__action_result_t fence_probe_result;
+
+    crm_trigger_t *work;
+} lrmd_rsc_t;
+
+#  ifdef HAVE_GNUTLS_GNUTLS_H
+// in remoted_tls.c
+int lrmd_init_remote_tls_server(void);
+void execd_stop_tls_server(void);
+#  endif
+
+int lrmd_server_send_reply(pcmk__client_t *client, uint32_t id, xmlNode *reply);
+
+int lrmd_server_send_notify(pcmk__client_t *client, xmlNode *msg);
+
+void notify_of_new_client(pcmk__client_t *new_client);
+
+void process_lrmd_message(pcmk__client_t *client, uint32_t id,
+                          xmlNode *request);
+
+void free_rsc(gpointer data);
+
+void handle_shutdown_ack(void);
+
+void handle_shutdown_nack(void);
+
+void lrmd_client_destroy(pcmk__client_t *client);
+
+void client_disconnect_cleanup(const char *client_id);
+
+/*!
+ * \brief Don't worry about freeing this connection. It is
+ *        taken care of after mainloop exits by the main() function.
+ */
+stonith_t *get_stonith_connection(void);
+
+/*!
+ * \brief This is a callback that tells the lrmd
+ * the current stonith connection has gone away. This allows
+ * us to timeout any pending stonith commands
+ */
+void stonith_connection_failed(void);
+
+#ifdef PCMK__COMPILE_REMOTE
+void ipc_proxy_init(void);
+void ipc_proxy_cleanup(void);
+void ipc_proxy_add_provider(pcmk__client_t *client);
+void ipc_proxy_remove_provider(pcmk__client_t *client);
+void ipc_proxy_forward_client(pcmk__client_t *client, xmlNode *xml);
+pcmk__client_t *ipc_proxy_get_provider(void);
+int ipc_proxy_shutdown_req(pcmk__client_t *ipc_proxy);
+void remoted_spawn_pidone(int argc, char **argv, char **envp);
+#endif
+
+int process_lrmd_alert_exec(pcmk__client_t *client, uint32_t id,
+                            xmlNode *request);
+void lrmd_drain_alerts(GMainLoop *mloop);
+
+#endif // PACEMAKER_EXECD__H
diff --git a/daemons/execd/pacemaker-remoted.8.inc b/daemons/execd/pacemaker-remoted.8.inc
new file mode 100644
index 0000000..bc86acc
--- /dev/null
+++ b/daemons/execd/pacemaker-remoted.8.inc
@@ -0,0 +1,5 @@
+[synopsis]
+pacemaker-remoted [options]
+
+/for Pacemaker Remote nodes/
+.SH OPTIONS
diff --git a/daemons/execd/pacemaker_remote.in b/daemons/execd/pacemaker_remote.in
new file mode 100644
index 0000000..2096c5f
--- /dev/null
+++ b/daemons/execd/pacemaker_remote.in
@@ -0,0 +1,176 @@
+#!@BASH_PATH@
+
+# Authors:
+#  Andrew Beekhof <abeekhof@redhat.com>
+#
+# License: Revised BSD
+
+# chkconfig: - 99 01
+# description: Pacemaker Cluster Manager
+# processname: pacemaker-remoted
+#
+### BEGIN INIT INFO
+# Provides:		pacemaker_remote
+# Required-Start:	$network $remote_fs
+# Should-Start:		$syslog
+# Required-Stop:	$network $remote_fs
+# Default-Start:
+# Default-Stop:
+# Short-Description:	Manage the executor for Pacemaker Remote nodes
+# Description:		Manage the executor for Pacemaker Remote nodes
+### END INIT INFO
+
+desc="Pacemaker Remote Executor"
+prog="pacemaker-remoted"
+
+# set secure PATH
+PATH="/sbin:/bin:/usr/sbin:/usr/bin:@sbindir@"
+
+checkrc() {
+    if [ $? = 0 ]; then
+	success
+    else
+	failure
+    fi
+}
+
+success()
+{
+	echo -ne "[  OK  ]\r"
+}
+
+failure()
+{
+	echo -ne "[FAILED]\r"
+}
+
+status()
+{
+	pid=$(pidof $1 2>/dev/null)
+	local rtrn=$?
+	if [ $rtrn -ne 0 ]; then
+		echo "$1 is stopped"
+		if [ -f "@localstatedir@/run/$prog.pid" ]; then
+			rtrn=1
+		else
+			rtrn=3
+		fi
+	else
+		echo "$1 (pid $pid) is running..."
+	fi
+	return $rtrn
+}
+
+if [ -d @CONFIGDIR@ ]; then
+	[ -f @INITDIR@/functions ] && . @INITDIR@/functions
+set -a
+	[ -f @CONFIGDIR@/pacemaker ] && . @CONFIGDIR@/pacemaker
+	[ -f @CONFIGDIR@/sbd ] && . @CONFIGDIR@/sbd
+set +a
+fi
+
+LOCK_DIR="."
+if [ -d "@localstatedir@/lock/subsys" ]; then
+	LOCK_DIR="@localstatedir@/lock/subsys" 
+elif [ -d "@localstatedir@/lock" ]; then
+	LOCK_DIR="@localstatedir@/lock" 
+fi
+[ -z "$LOCK_FILE" ] && LOCK_FILE="$LOCK_DIR/pacemaker_remote"
+
+# Check if there is a valid watchdog-device configured in sbd config
+if [ x != "x$SBD_WATCHDOG_DEV" -a "/dev/null" != "$SBD_WATCHDOG_DEV" -a -c "$SBD_WATCHDOG_DEV" ]; then
+	# enhance for unavailable chkconfig - don't touch sbd for now
+	if chkconfig --list sbd_remote_helper 2>/dev/null | grep -q ":on"; then
+		SBD_SERVICE=sbd_remote_helper
+	fi
+fi
+
+start()
+{
+	echo -n "Starting $desc: "
+
+	# most recent distributions use tmpfs for $@localstatedir@/run
+	# to avoid to clean it up on every boot.
+	# they also assume that init scripts will create
+	# required subdirectories for proper operations
+	mkdir -p "@localstatedir@/run"
+
+	if status $prog > /dev/null 2>&1; then
+		success
+	else
+		$prog > /dev/null 2>&1 &
+
+		# Time to connect to corosync and fail
+		sleep 5
+
+		if status $prog > /dev/null 2>&1; then
+			touch "$LOCK_FILE"
+			pidof $prog > "@localstatedir@/run/$prog.pid"
+			success
+		else
+			failure
+			rtrn=1
+		fi
+	fi
+	echo
+
+	[ "x$SBD_SERVICE" = "x" ] || service $SBD_SERVICE start
+}
+
+stop()
+{
+	if status $prog > /dev/null 2>&1; then
+	    echo -n "Signaling $desc to terminate: "
+	    kill -TERM $(pidof $prog) > /dev/null 2>&1
+	    success
+	    echo
+
+	    echo -n "Waiting for $desc to unload:"
+	    while status $prog > /dev/null 2>&1; do
+		sleep 1
+		echo -n "."
+	    done
+	else
+		echo -n "$desc is already stopped"
+	fi
+
+	rm -f "$LOCK_FILE"
+	rm -f "@localstatedir@/run/$prog.pid"
+	success
+	echo
+
+	[ "x$SBD_SERVICE" = "x" ] || service $SBD_SERVICE stop
+}
+
+rtrn=0
+
+case "$1" in
+start)
+	start
+;;
+restart|reload|force-reload)
+	stop
+	start
+;;
+condrestart|try-restart)
+	if status $prog > /dev/null 2>&1; then
+	    stop
+	    start
+	    rtrn=$?
+	fi
+;;
+status)
+	status $prog
+	rtrn=$?
+;;
+stop)
+	stop
+	rtrn=$?
+;;
+*)
+	echo "usage: $0 {start|stop|restart|reload|force-reload|condrestart|try-restart|status}"
+	rtrn=2
+;;
+esac
+
+exit $rtrn
diff --git a/daemons/execd/pacemaker_remote.service.in b/daemons/execd/pacemaker_remote.service.in
new file mode 100644
index 0000000..1e48d14
--- /dev/null
+++ b/daemons/execd/pacemaker_remote.service.in
@@ -0,0 +1,52 @@
+[Unit]
+Description=Pacemaker Remote executor daemon
+Documentation=man:pacemaker-remoted
+Documentation=https://clusterlabs.org/pacemaker/doc/
+
+# See main pacemaker unit file for descriptions of why these are needed
+After=network.target
+After=time-sync.target
+After=dbus.service
+Wants=dbus.service
+After=resource-agents-deps.target
+Wants=resource-agents-deps.target
+After=syslog.service
+After=rsyslog.service
+
+[Install]
+Alias=pacemaker-remote.service
+WantedBy=multi-user.target
+
+[Service]
+Type=simple
+KillMode=process
+NotifyAccess=none
+EnvironmentFile=-@CONFIGDIR@/pacemaker
+EnvironmentFile=-@CONFIGDIR@/sbd
+
+# Not actually success, but fatal failure -- this ensures no respawn
+SuccessExitStatus=100
+
+ExecStart=@sbindir@/pacemaker-remoted
+
+# Systemd v227 and above can limit the number of processes spawned by a
+# service. That is a bad idea for an HA cluster resource manager, so disable it
+# by default. The administrator can create a local override if they really want
+# a limit. If your systemd version does not support TasksMax, and you want to
+# get rid of the resulting log warnings, comment out this option.
+TasksMax=infinity
+
+# If connected to the cluster and when the service functions properly, it will
+# wait to exit until the cluster notifies it all resources on the remote node
+# have been stopped.  The default of 30min should cover most typical cluster
+# configurations, but it may need an increase to adapt to local conditions
+# (e.g. a large, clustered database could conceivably take longer to stop).
+TimeoutStopSec=30min
+TimeoutStartSec=30s
+
+# Restart options include: no, on-success, on-failure, on-abort or always
+Restart=on-failure
+
+# crm_perror() writes directly to stderr, so ignore it here
+# to avoid double-logging with the wrong format
+StandardError=null
diff --git a/daemons/execd/remoted_pidone.c b/daemons/execd/remoted_pidone.c
new file mode 100644
index 0000000..4f914eb
--- /dev/null
+++ b/daemons/execd/remoted_pidone.c
@@ -0,0 +1,298 @@
+/*
+ * Copyright 2017-2020 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include <crm/crm.h>
+#include "pacemaker-execd.h"
+
+static pid_t main_pid = 0;
+
+static void
+sigdone(void)
+{
+    exit(CRM_EX_OK);
+}
+
+static void
+sigreap(void)
+{
+    pid_t pid = 0;
+    int status;
+
+    do {
+        /*
+         * Opinions seem to differ as to what to put here:
+         *  -1, any child process
+         *  0,  any child process whose process group ID is equal to that of the calling process
+         */
+        pid = waitpid(-1, &status, WNOHANG);
+        if (pid == main_pid) {
+            /* Exit when pacemaker-remote exits and use the same return code */
+            if (WIFEXITED(status)) {
+                exit(WEXITSTATUS(status));
+            }
+            exit(CRM_EX_ERROR);
+        }
+    } while (pid > 0);
+}
+
+static struct {
+    int sig;
+    void (*handler)(void);
+} sigmap[] = {
+    { SIGCHLD, sigreap },
+    { SIGINT,  sigdone },
+};
+
+/*!
+ * \internal
+ * \brief Check a line of text for a valid environment variable name
+ *
+ * \param[in]  line  Text to check
+ * \param[out] first  First character of valid name if found, NULL otherwise
+ * \param[out] last   Last character of valid name if found, NULL otherwise
+ *
+ * \return TRUE if valid name found, FALSE otherwise
+ * \note It's reasonable to impose limitations on environment variable names
+ *       beyond what C or setenv() does: We only allow names that contain only
+ *       [a-zA-Z0-9_] characters and do not start with a digit.
+ */
+static bool
+find_env_var_name(char *line, char **first, char **last)
+{
+    // Skip leading whitespace
+    *first = line;
+    while (isspace(**first)) {
+        ++*first;
+    }
+
+    if (isalpha(**first) || (**first == '_')) { // Valid first character
+        *last = *first;
+        while (isalnum(*(*last + 1)) || (*(*last + 1) == '_')) {
+            ++*last;
+        }
+        return TRUE;
+    }
+
+    *first = *last = NULL;
+    return FALSE;
+}
+
+static void
+load_env_vars(const char *filename)
+{
+    /* We haven't forked or initialized logging yet, so don't leave any file
+     * descriptors open, and don't log -- silently ignore errors.
+     */
+    FILE *fp = fopen(filename, "r");
+
+    if (fp != NULL) {
+        char line[LINE_MAX] = { '\0', };
+
+        while (fgets(line, LINE_MAX, fp) != NULL) {
+            char *name = NULL;
+            char *end = NULL;
+            char *value = NULL;
+            char *quote = NULL;
+
+            // Look for valid name immediately followed by equals sign
+            if (find_env_var_name(line, &name, &end) && (*++end == '=')) {
+
+                // Null-terminate name, and advance beyond equals sign
+                *end++ = '\0';
+
+                // Check whether value is quoted
+                if ((*end == '\'') || (*end == '"')) {
+                    quote = end++;
+                }
+                value = end;
+
+                if (quote) {
+                    /* Value is remaining characters up to next non-backslashed
+                     * matching quote character.
+                     */
+                    while (((*end != *quote) || (*(end - 1) == '\\'))
+                           && (*end != '\0')) {
+                        end++;
+                    }
+                    if (*end == *quote) {
+                        // Null-terminate value, and advance beyond close quote
+                        *end++ = '\0';
+                    } else {
+                        // Matching closing quote wasn't found
+                        value = NULL;
+                    }
+
+                } else {
+                    /* Value is remaining characters up to next non-backslashed
+                     * whitespace.
+                     */
+                    while ((!isspace(*end) || (*(end - 1) == '\\'))
+                           && (*end != '\0')) {
+                        ++end;
+                    }
+
+                    if (end == (line + LINE_MAX - 1)) {
+                        // Line was too long
+                        value = NULL;
+                    }
+                    // Do NOT null-terminate value (yet)
+                }
+
+                /* We have a valid name and value, and end is now the character
+                 * after the closing quote or the first whitespace after the
+                 * unquoted value. Make sure the rest of the line is just
+                 * whitespace or a comment.
+                 */
+                if (value) {
+                    char *value_end = end;
+
+                    while (isspace(*end) && (*end != '\n')) {
+                        ++end;
+                    }
+                    if ((*end == '\n') || (*end == '#')) {
+                        if (quote == NULL) {
+                            // Now we can null-terminate an unquoted value
+                            *value_end = '\0';
+                        }
+
+                        // Don't overwrite (bundle options take precedence)
+                        setenv(name, value, 0);
+
+                    } else {
+                        value = NULL;
+                    }
+                }
+            }
+
+            if ((value == NULL) && (strchr(line, '\n') == NULL)) {
+                // Eat remainder of line beyond LINE_MAX
+                if (fscanf(fp, "%*[^\n]\n") == EOF) {
+                    value = NULL; // Don't care, make compiler happy
+                }
+            }
+        }
+        fclose(fp);
+    }
+}
+
+void
+remoted_spawn_pidone(int argc, char **argv, char **envp)
+{
+    sigset_t set;
+
+    /* This environment variable exists for two purposes:
+     * - For testing, setting it to "full" enables full PID 1 behavior even
+     *   when PID is not 1
+     * - Setting to "vars" enables just the loading of environment variables
+     *   from /etc/pacemaker/pcmk-init.env, which could be useful for testing or
+     *   containers with a custom PID 1 script that launches pacemaker-remoted.
+     */
+    const char *pid1 = (getpid() == 1)? "full" : getenv("PCMK_remote_pid1");
+
+    if (pid1 == NULL) {
+        return;
+    }
+
+    /* When a container is launched, it may be given specific environment
+     * variables, which for Pacemaker bundles are given in the bundle
+     * configuration. However, that does not allow for host-specific values.
+     * To allow for that, look for a special file containing a shell-like syntax
+     * of name/value pairs, and export those into the environment.
+     */
+    load_env_vars("/etc/pacemaker/pcmk-init.env");
+
+    if (strcmp(pid1, "full")) {
+        return;
+    }
+
+    /* Containers can be expected to have /var/log, but they may not have
+     * /var/log/pacemaker, so use a different default if no value has been
+     * explicitly configured in the container's environment.
+     */
+    if (pcmk__env_option(PCMK__ENV_LOGFILE) == NULL) {
+        pcmk__set_env_option(PCMK__ENV_LOGFILE, "/var/log/pcmk-init.log");
+    }
+
+    sigfillset(&set);
+    sigprocmask(SIG_BLOCK, &set, 0);
+
+    main_pid = fork();
+    switch (main_pid) {
+        case 0:
+            sigprocmask(SIG_UNBLOCK, &set, NULL);
+            setsid();
+            setpgid(0, 0);
+
+            // Child remains as pacemaker-remoted
+            return;
+        case -1:
+            perror("fork");
+    }
+
+    /* Parent becomes the reaper of zombie processes */
+    /* Safe to initialize logging now if needed */
+
+#  ifdef HAVE_PROGNAME
+    /* Differentiate ourselves in the 'ps' output */
+    {
+        char *p;
+        int i, maxlen;
+        char *LastArgv = NULL;
+        const char *name = "pcmk-init";
+
+        for (i = 0; i < argc; i++) {
+            if (!i || (LastArgv + 1 == argv[i]))
+                LastArgv = argv[i] + strlen(argv[i]);
+        }
+
+        for (i = 0; envp[i] != NULL; i++) {
+            if ((LastArgv + 1) == envp[i]) {
+                LastArgv = envp[i] + strlen(envp[i]);
+            }
+        }
+
+        maxlen = (LastArgv - argv[0]) - 2;
+
+        i = strlen(name);
+
+        /* We can overwrite individual argv[] arguments */
+        snprintf(argv[0], maxlen, "%s", name);
+
+        /* Now zero out everything else */
+        p = &argv[0][i];
+        while (p < LastArgv) {
+            *p++ = '\0';
+        }
+        argv[1] = NULL;
+    }
+#  endif // HAVE_PROGNAME
+
+    while (1) {
+        int sig;
+        size_t i;
+
+        sigwait(&set, &sig);
+        for (i = 0; i < PCMK__NELEM(sigmap); i++) {
+            if (sigmap[i].sig == sig) {
+                sigmap[i].handler();
+                break;
+            }
+        }
+    }
+}
diff --git a/daemons/execd/remoted_proxy.c b/daemons/execd/remoted_proxy.c
new file mode 100644
index 0000000..62c8c3a
--- /dev/null
+++ b/daemons/execd/remoted_proxy.c
@@ -0,0 +1,470 @@
+/*
+ * Copyright 2012-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <glib.h>
+#include <unistd.h>
+
+#include "pacemaker-execd.h"
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/services.h>
+#include <crm/common/mainloop.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/cib/internal.h>
+#include <crm/fencing/internal.h>
+
+static qb_ipcs_service_t *cib_ro = NULL;
+static qb_ipcs_service_t *cib_rw = NULL;
+static qb_ipcs_service_t *cib_shm = NULL;
+
+static qb_ipcs_service_t *attrd_ipcs = NULL;
+static qb_ipcs_service_t *crmd_ipcs = NULL;
+static qb_ipcs_service_t *stonith_ipcs = NULL;
+static qb_ipcs_service_t *pacemakerd_ipcs = NULL;
+
+// An IPC provider is a cluster node controller connecting as a client
+static GList *ipc_providers = NULL;
+/* ipc clients == things like cibadmin, crm_resource, connecting locally */
+static GHashTable *ipc_clients = NULL;
+
+/*!
+ * \internal
+ * \brief Get an IPC proxy provider
+ *
+ * \return Pointer to a provider if one exists, NULL otherwise
+ *
+ * \note Grab the first provider, which is the most recent connection. That way,
+ *       if we haven't yet timed out an old, failed connection, we don't try to
+ *       use it.
+ */
+pcmk__client_t *
+ipc_proxy_get_provider(void)
+{
+    return ipc_providers? (pcmk__client_t *) (ipc_providers->data) : NULL;
+}
+
+/*!
+ * \internal
+ * \brief Accept a client connection on a proxy IPC server
+ *
+ * \param[in] c            Client's IPC connection
+ * \param[in] uid          Client's user ID
+ * \param[in] gid          Client's group ID
+ * \param[in] ipc_channel  Name of IPC server to proxy
+ *
+ * \return pcmk_ok on success, -errno on error
+ */
+static int32_t
+ipc_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid, const char *ipc_channel)
+{
+    pcmk__client_t *client;
+    pcmk__client_t *ipc_proxy = ipc_proxy_get_provider();
+    xmlNode *msg;
+
+    if (ipc_proxy == NULL) {
+        crm_warn("Cannot proxy IPC connection from uid %d gid %d to %s "
+                 "because not connected to cluster", uid, gid, ipc_channel);
+        return -EREMOTEIO;
+    }
+
+    /* This new client is a local IPC client on a Pacemaker Remote controlled
+     * node, needing to access cluster node IPC services.
+     */
+    client = pcmk__new_client(c, uid, gid);
+    if (client == NULL) {
+        return -EREMOTEIO;
+    }
+
+    /* This ipc client is bound to a single ipc provider. If the
+     * provider goes away, this client is disconnected */
+    client->userdata = strdup(ipc_proxy->id);
+    client->name = crm_strdup_printf("proxy-%s-%d-%.8s", ipc_channel, client->pid, client->id);
+
+    /* Allow remote executor to distinguish between proxied local clients and
+     * actual executor API clients
+     */
+    pcmk__set_client_flags(client, pcmk__client_to_proxy);
+
+    g_hash_table_insert(ipc_clients, client->id, client);
+
+    msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
+    crm_xml_add(msg, F_LRMD_IPC_OP, LRMD_IPC_OP_NEW);
+    crm_xml_add(msg, F_LRMD_IPC_IPC_SERVER, ipc_channel);
+    crm_xml_add(msg, F_LRMD_IPC_SESSION, client->id);
+    lrmd_server_send_notify(ipc_proxy, msg);
+    free_xml(msg);
+    crm_debug("Accepted IPC proxy connection (session ID %s) "
+              "from uid %d gid %d on channel %s",
+              client->id, uid, gid, ipc_channel);
+    return 0;
+}
+
+static int32_t
+crmd_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
+{
+    return ipc_proxy_accept(c, uid, gid, CRM_SYSTEM_CRMD);
+}
+
+static int32_t
+attrd_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
+{
+    return ipc_proxy_accept(c, uid, gid, T_ATTRD);
+}
+
+static int32_t
+stonith_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
+{
+    return ipc_proxy_accept(c, uid, gid, "stonith-ng");
+}
+
+static int32_t
+pacemakerd_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
+{
+    return -EREMOTEIO;
+}
+
+static int32_t
+cib_proxy_accept_rw(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
+{
+    return ipc_proxy_accept(c, uid, gid, PCMK__SERVER_BASED_RW);
+}
+
+static int32_t
+cib_proxy_accept_ro(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
+{
+    return ipc_proxy_accept(c, uid, gid, PCMK__SERVER_BASED_RO);
+}
+
+void
+ipc_proxy_forward_client(pcmk__client_t *ipc_proxy, xmlNode *xml)
+{
+    const char *session = crm_element_value(xml, F_LRMD_IPC_SESSION);
+    const char *msg_type = crm_element_value(xml, F_LRMD_IPC_OP);
+    xmlNode *msg = get_message_xml(xml, F_LRMD_IPC_MSG);
+    pcmk__client_t *ipc_client;
+    int rc = pcmk_rc_ok;
+
+    /* If the IPC provider is acknowledging our shutdown request,
+     * defuse the short exit timer to give the cluster time to
+     * stop any resources we're running.
+     */
+    if (pcmk__str_eq(msg_type, LRMD_IPC_OP_SHUTDOWN_ACK, pcmk__str_casei)) {
+        handle_shutdown_ack();
+        return;
+    }
+
+    if (pcmk__str_eq(msg_type, LRMD_IPC_OP_SHUTDOWN_NACK, pcmk__str_casei)) {
+        handle_shutdown_nack();
+        return;
+    }
+
+    ipc_client = pcmk__find_client_by_id(session);
+    if (ipc_client == NULL) {
+        xmlNode *msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
+        crm_xml_add(msg, F_LRMD_IPC_OP, LRMD_IPC_OP_DESTROY);
+        crm_xml_add(msg, F_LRMD_IPC_SESSION, session);
+        lrmd_server_send_notify(ipc_proxy, msg);
+        free_xml(msg);
+        return;
+    }
+
+    /* This is an event or response from the ipc provider
+     * going to the local ipc client.
+     *
+     * Looking at the chain of events.
+     *
+     * -----remote node----------------|---- cluster node ------
+     * ipc_client <--1--> this code
+     *    <--2--> pacemaker-controld:remote_proxy_cb/remote_proxy_relay_event()
+     *    <--3--> ipc server
+     *
+     * This function is receiving a msg from connection 2
+     * and forwarding it to connection 1.
+     */
+
+    if (pcmk__str_eq(msg_type, LRMD_IPC_OP_EVENT, pcmk__str_casei)) {
+        crm_trace("Sending event to %s", ipc_client->id);
+        rc = pcmk__ipc_send_xml(ipc_client, 0, msg, crm_ipc_server_event);
+
+    } else if (pcmk__str_eq(msg_type, LRMD_IPC_OP_RESPONSE, pcmk__str_casei)) {
+        int msg_id = 0;
+
+        crm_element_value_int(xml, F_LRMD_IPC_MSG_ID, &msg_id);
+        crm_trace("Sending response to %d - %s", ipc_client->request_id, ipc_client->id);
+        rc = pcmk__ipc_send_xml(ipc_client, msg_id, msg, FALSE);
+
+        CRM_LOG_ASSERT(msg_id == ipc_client->request_id);
+        ipc_client->request_id = 0;
+
+    } else if (pcmk__str_eq(msg_type, LRMD_IPC_OP_DESTROY, pcmk__str_casei)) {
+        qb_ipcs_disconnect(ipc_client->ipcs);
+
+    } else {
+        crm_err("Unknown ipc proxy msg type %s" , msg_type);
+    }
+
+    if (rc != pcmk_rc_ok) {
+        crm_warn("Could not proxy IPC to client %s: %s " CRM_XS " rc=%d",
+                 ipc_client->id, pcmk_rc_str(rc), rc);
+    }
+}
+
+static int32_t
+ipc_proxy_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
+{
+    uint32_t id = 0;
+    uint32_t flags = 0;
+    pcmk__client_t *client = pcmk__find_client(c);
+    pcmk__client_t *ipc_proxy = pcmk__find_client_by_id(client->userdata);
+    xmlNode *request = NULL;
+    xmlNode *msg = NULL;
+
+    if (!ipc_proxy) {
+        qb_ipcs_disconnect(client->ipcs);
+        return 0;
+    }
+
+    /* This is a request from the local ipc client going
+     * to the ipc provider.
+     *
+     * Looking at the chain of events.
+     *
+     * -----remote node----------------|---- cluster node ------
+     * ipc_client <--1--> this code
+     *     <--2--> pacemaker-controld:remote_proxy_dispatch_internal()
+     *     <--3--> ipc server
+     *
+     * This function is receiving a request from connection
+     * 1 and forwarding it to connection 2.
+     */
+    request = pcmk__client_data2xml(client, data, &id, &flags);
+
+    if (!request) {
+        return 0;
+    }
+
+    CRM_CHECK(client != NULL, crm_err("Invalid client");
+              free_xml(request); return FALSE);
+    CRM_CHECK(client->id != NULL, crm_err("Invalid client: %p", client);
+              free_xml(request); return FALSE);
+
+    /* This ensures that synced request/responses happen over the event channel
+     * in the controller, allowing the controller to process the messages async.
+     */
+    pcmk__set_ipc_flags(flags, pcmk__client_name(client), crm_ipc_proxied);
+    client->request_id = id;
+
+    msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
+    crm_xml_add(msg, F_LRMD_IPC_OP, LRMD_IPC_OP_REQUEST);
+    crm_xml_add(msg, F_LRMD_IPC_SESSION, client->id);
+    crm_xml_add(msg, F_LRMD_IPC_CLIENT, pcmk__client_name(client));
+    crm_xml_add(msg, F_LRMD_IPC_USER, client->user);
+    crm_xml_add_int(msg, F_LRMD_IPC_MSG_ID, id);
+    crm_xml_add_int(msg, F_LRMD_IPC_MSG_FLAGS, flags);
+    add_message_xml(msg, F_LRMD_IPC_MSG, request);
+    lrmd_server_send_notify(ipc_proxy, msg);
+    free_xml(request);
+    free_xml(msg);
+
+    return 0;
+}
+
+/*!
+ * \internal
+ * \brief Notify a proxy provider that we wish to shut down
+ *
+ * \param[in,out] ipc_proxy  IPC client connection to proxy provider
+ *
+ * \return 0 on success, -1 on error
+ */
+int
+ipc_proxy_shutdown_req(pcmk__client_t *ipc_proxy)
+{
+    xmlNode *msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
+    int rc;
+
+    crm_xml_add(msg, F_LRMD_IPC_OP, LRMD_IPC_OP_SHUTDOWN_REQ);
+
+    /* We don't really have a session, but the controller needs this attribute
+     * to recognize this as proxy communication.
+     */
+    crm_xml_add(msg, F_LRMD_IPC_SESSION, "0");
+
+    rc = (lrmd_server_send_notify(ipc_proxy, msg) != pcmk_rc_ok)? -1 : 0;
+    free_xml(msg);
+    return rc;
+}
+
+static int32_t
+ipc_proxy_closed(qb_ipcs_connection_t * c)
+{
+    pcmk__client_t *client = pcmk__find_client(c);
+    pcmk__client_t *ipc_proxy;
+
+    if (client == NULL) {
+        return 0;
+    }
+
+    ipc_proxy = pcmk__find_client_by_id(client->userdata);
+
+    crm_trace("Connection %p", c);
+
+    if (ipc_proxy) {
+        xmlNode *msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
+        crm_xml_add(msg, F_LRMD_IPC_OP, LRMD_IPC_OP_DESTROY);
+        crm_xml_add(msg, F_LRMD_IPC_SESSION, client->id);
+        lrmd_server_send_notify(ipc_proxy, msg);
+        free_xml(msg);
+    }
+
+    g_hash_table_remove(ipc_clients, client->id);
+
+    free(client->userdata);
+    client->userdata = NULL;
+    pcmk__free_client(client);
+    return 0;
+}
+
+static void
+ipc_proxy_destroy(qb_ipcs_connection_t * c)
+{
+    crm_trace("Connection %p", c);
+    ipc_proxy_closed(c);
+}
+
+static struct qb_ipcs_service_handlers crmd_proxy_callbacks = {
+    .connection_accept = crmd_proxy_accept,
+    .connection_created = NULL,
+    .msg_process = ipc_proxy_dispatch,
+    .connection_closed = ipc_proxy_closed,
+    .connection_destroyed = ipc_proxy_destroy
+};
+
+static struct qb_ipcs_service_handlers attrd_proxy_callbacks = {
+    .connection_accept = attrd_proxy_accept,
+    .connection_created = NULL,
+    .msg_process = ipc_proxy_dispatch,
+    .connection_closed = ipc_proxy_closed,
+    .connection_destroyed = ipc_proxy_destroy
+};
+
+static struct qb_ipcs_service_handlers stonith_proxy_callbacks = {
+    .connection_accept = stonith_proxy_accept,
+    .connection_created = NULL,
+    .msg_process = ipc_proxy_dispatch,
+    .connection_closed = ipc_proxy_closed,
+    .connection_destroyed = ipc_proxy_destroy
+};
+
+static struct qb_ipcs_service_handlers pacemakerd_proxy_callbacks = {
+    .connection_accept = pacemakerd_proxy_accept,
+    .connection_created = NULL,
+    .msg_process = NULL,
+    .connection_closed = NULL,
+    .connection_destroyed = NULL
+};
+
+static struct qb_ipcs_service_handlers cib_proxy_callbacks_ro = {
+    .connection_accept = cib_proxy_accept_ro,
+    .connection_created = NULL,
+    .msg_process = ipc_proxy_dispatch,
+    .connection_closed = ipc_proxy_closed,
+    .connection_destroyed = ipc_proxy_destroy
+};
+
+static struct qb_ipcs_service_handlers cib_proxy_callbacks_rw = {
+    .connection_accept = cib_proxy_accept_rw,
+    .connection_created = NULL,
+    .msg_process = ipc_proxy_dispatch,
+    .connection_closed = ipc_proxy_closed,
+    .connection_destroyed = ipc_proxy_destroy
+};
+
+void
+ipc_proxy_add_provider(pcmk__client_t *ipc_proxy)
+{
+    // Prepending ensures the most recent connection is always first
+    ipc_providers = g_list_prepend(ipc_providers, ipc_proxy);
+}
+
+void
+ipc_proxy_remove_provider(pcmk__client_t *ipc_proxy)
+{
+    GHashTableIter iter;
+    pcmk__client_t *ipc_client = NULL;
+    char *key = NULL;
+    GList *remove_these = NULL;
+    GList *gIter = NULL;
+
+    ipc_providers = g_list_remove(ipc_providers, ipc_proxy);
+
+    g_hash_table_iter_init(&iter, ipc_clients);
+    while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & ipc_client)) {
+        const char *proxy_id = ipc_client->userdata;
+        if (pcmk__str_eq(proxy_id, ipc_proxy->id, pcmk__str_casei)) {
+            crm_info("ipc proxy connection for client %s pid %d destroyed because cluster node disconnected.",
+                ipc_client->id, ipc_client->pid);
+            /* we can't remove during the iteration, so copy items
+             * to a list we can destroy later */
+            remove_these = g_list_append(remove_these, ipc_client);
+        }
+    }
+
+    for (gIter = remove_these; gIter != NULL; gIter = gIter->next) {
+        ipc_client = gIter->data;
+
+        // Disconnection callback will free the client here
+        qb_ipcs_disconnect(ipc_client->ipcs);
+    }
+
+    /* just frees the list, not the elements in the list */
+    g_list_free(remove_these);
+}
+
+void
+ipc_proxy_init(void)
+{
+    ipc_clients = pcmk__strkey_table(NULL, NULL);
+
+    pcmk__serve_based_ipc(&cib_ro, &cib_rw, &cib_shm, &cib_proxy_callbacks_ro,
+                          &cib_proxy_callbacks_rw);
+    pcmk__serve_attrd_ipc(&attrd_ipcs, &attrd_proxy_callbacks);
+    pcmk__serve_fenced_ipc(&stonith_ipcs, &stonith_proxy_callbacks);
+    pcmk__serve_pacemakerd_ipc(&pacemakerd_ipcs, &pacemakerd_proxy_callbacks);
+    crmd_ipcs = pcmk__serve_controld_ipc(&crmd_proxy_callbacks);
+    if (crmd_ipcs == NULL) {
+        crm_err("Failed to create controller: exiting and inhibiting respawn");
+        crm_warn("Verify pacemaker and pacemaker_remote are not both enabled");
+        crm_exit(CRM_EX_FATAL);
+    }
+}
+
+void
+ipc_proxy_cleanup(void)
+{
+    if (ipc_providers) {
+        g_list_free(ipc_providers);
+        ipc_providers = NULL;
+    }
+    if (ipc_clients) {
+        g_hash_table_destroy(ipc_clients);
+        ipc_clients = NULL;
+    }
+    pcmk__stop_based_ipc(cib_ro, cib_rw, cib_shm);
+    qb_ipcs_destroy(attrd_ipcs);
+    qb_ipcs_destroy(stonith_ipcs);
+    qb_ipcs_destroy(pacemakerd_ipcs);
+    qb_ipcs_destroy(crmd_ipcs);
+    cib_ro = NULL;
+    cib_rw = NULL;
+    cib_shm = NULL;
+}
diff --git a/daemons/execd/remoted_tls.c b/daemons/execd/remoted_tls.c
new file mode 100644
index 0000000..c65e3f3
--- /dev/null
+++ b/daemons/execd/remoted_tls.c
@@ -0,0 +1,428 @@
+/*
+ * Copyright 2012-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <glib.h>
+#include <unistd.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/mainloop.h>
+#include <crm/common/remote_internal.h>
+#include <crm/lrmd_internal.h>
+
+#include <netdb.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <arpa/inet.h>
+
+#include "pacemaker-execd.h"
+
+#ifdef HAVE_GNUTLS_GNUTLS_H
+
+#  include <gnutls/gnutls.h>
+
+#  define LRMD_REMOTE_AUTH_TIMEOUT 10000
+gnutls_psk_server_credentials_t psk_cred_s;
+gnutls_dh_params_t dh_params;
+static int ssock = -1;
+extern int lrmd_call_id;
+
+static void
+debug_log(int level, const char *str)
+{
+    fputs(str, stderr);
+}
+
+/*!
+ * \internal
+ * \brief Read (more) TLS handshake data from client
+ *
+ * \param[in,out] client  IPC client doing handshake
+ *
+ * \return 0 on success or more data needed, -1 on error
+ */
+static int
+remoted__read_handshake_data(pcmk__client_t *client)
+{
+    int rc = pcmk__read_handshake_data(client);
+
+    if (rc == EAGAIN) {
+        /* No more data is available at the moment. Just return for now;
+         * we'll get invoked again once the client sends more.
+         */
+        return 0;
+    } else if (rc != pcmk_rc_ok) {
+        return -1;
+    }
+
+    if (client->remote->auth_timeout) {
+        g_source_remove(client->remote->auth_timeout);
+    }
+    client->remote->auth_timeout = 0;
+
+    pcmk__set_client_flags(client, pcmk__client_tls_handshake_complete);
+    crm_notice("Remote client connection accepted");
+
+    /* Only a client with access to the TLS key can connect, so we can treat
+     * it as privileged.
+     */
+    pcmk__set_client_flags(client, pcmk__client_privileged);
+
+    // Alert other clients of the new connection
+    notify_of_new_client(client);
+    return 0;
+}
+
+static int
+lrmd_remote_client_msg(gpointer data)
+{
+    int id = 0;
+    int rc;
+    xmlNode *request = NULL;
+    pcmk__client_t *client = data;
+
+    if (!pcmk_is_set(client->flags,
+                     pcmk__client_tls_handshake_complete)) {
+        return remoted__read_handshake_data(client);
+    }
+
+    switch (pcmk__remote_ready(client->remote, 0)) {
+        case pcmk_rc_ok:
+            break;
+        case ETIME: // No message available to read
+            return 0;
+        default:    // Error
+            crm_info("Remote client disconnected while polling it");
+            return -1;
+    }
+
+    rc = pcmk__read_remote_message(client->remote, -1);
+
+    request = pcmk__remote_message_xml(client->remote);
+    while (request) {
+        crm_element_value_int(request, F_LRMD_REMOTE_MSG_ID, &id);
+        crm_trace("Processing remote client request %d", id);
+        if (!client->name) {
+            const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);
+
+            if (value) {
+                client->name = strdup(value);
+            }
+        }
+
+        lrmd_call_id++;
+        if (lrmd_call_id < 1) {
+            lrmd_call_id = 1;
+        }
+
+        crm_xml_add(request, F_LRMD_CLIENTID, client->id);
+        crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
+        crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);
+
+        process_lrmd_message(client, id, request);
+        free_xml(request);
+
+        /* process all the messages in the current buffer */
+        request = pcmk__remote_message_xml(client->remote);
+    }
+
+    if (rc == ENOTCONN) {
+        crm_info("Remote client disconnected while reading from it");
+        return -1;
+    }
+
+    return 0;
+}
+
+static void
+lrmd_remote_client_destroy(gpointer user_data)
+{
+    pcmk__client_t *client = user_data;
+
+    if (client == NULL) {
+        return;
+    }
+
+    crm_notice("Cleaning up after remote client %s disconnected",
+               pcmk__client_name(client));
+
+    ipc_proxy_remove_provider(client);
+
+    /* if this is the last remote connection, stop recurring
+     * operations */
+    if (pcmk__ipc_client_count() == 1) {
+        client_disconnect_cleanup(NULL);
+    }
+
+    if (client->remote->tls_session) {
+        void *sock_ptr;
+        int csock;
+
+        sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
+        csock = GPOINTER_TO_INT(sock_ptr);
+
+        gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_RDWR);
+        gnutls_deinit(*client->remote->tls_session);
+        gnutls_free(client->remote->tls_session);
+        close(csock);
+    }
+
+    lrmd_client_destroy(client);
+    return;
+}
+
+static gboolean
+lrmd_auth_timeout_cb(gpointer data)
+{
+    pcmk__client_t *client = data;
+
+    client->remote->auth_timeout = 0;
+
+    if (pcmk_is_set(client->flags,
+                    pcmk__client_tls_handshake_complete)) {
+        return FALSE;
+    }
+
+    mainloop_del_fd(client->remote->source);
+    client->remote->source = NULL;
+    crm_err("Remote client authentication timed out");
+
+    return FALSE;
+}
+
+// Dispatch callback for remote server socket
+static int
+lrmd_remote_listen(gpointer data)
+{
+    int csock = -1;
+    gnutls_session_t *session = NULL;
+    pcmk__client_t *new_client = NULL;
+
+    // For client socket
+    static struct mainloop_fd_callbacks lrmd_remote_fd_cb = {
+        .dispatch = lrmd_remote_client_msg,
+        .destroy = lrmd_remote_client_destroy,
+    };
+
+    CRM_CHECK(ssock >= 0, return TRUE);
+
+    if (pcmk__accept_remote_connection(ssock, &csock) != pcmk_rc_ok) {
+        return TRUE;
+    }
+
+    session = pcmk__new_tls_session(csock, GNUTLS_SERVER, GNUTLS_CRD_PSK,
+                                    psk_cred_s);
+    if (session == NULL) {
+        close(csock);
+        return TRUE;
+    }
+
+    new_client = pcmk__new_unauth_client(NULL);
+    new_client->remote = calloc(1, sizeof(pcmk__remote_t));
+    pcmk__set_client_flags(new_client, pcmk__client_tls);
+    new_client->remote->tls_session = session;
+
+    // Require the client to authenticate within this time
+    new_client->remote->auth_timeout = g_timeout_add(LRMD_REMOTE_AUTH_TIMEOUT,
+                                                     lrmd_auth_timeout_cb,
+                                                     new_client);
+    crm_info("Remote client pending authentication "
+             CRM_XS " %p id: %s", new_client, new_client->id);
+
+    new_client->remote->source =
+        mainloop_add_fd("pacemaker-remote-client", G_PRIORITY_DEFAULT, csock,
+                        new_client, &lrmd_remote_fd_cb);
+    return TRUE;
+}
+
+static void
+tls_server_dropped(gpointer user_data)
+{
+    crm_notice("TLS server session ended");
+    return;
+}
+
+// \return 0 on success, -1 on error (gnutls_psk_server_credentials_function)
+static int
+lrmd_tls_server_key_cb(gnutls_session_t session, const char *username, gnutls_datum_t * key)
+{
+    return (lrmd__init_remote_key(key) == pcmk_rc_ok)? 0 : -1;
+}
+
+static int
+bind_and_listen(struct addrinfo *addr)
+{
+    int optval;
+    int fd;
+    int rc;
+    char buffer[INET6_ADDRSTRLEN] = { 0, };
+
+    pcmk__sockaddr2str(addr->ai_addr, buffer);
+    crm_trace("Attempting to bind to address %s", buffer);
+
+    fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
+    if (fd < 0) {
+        crm_perror(LOG_ERR, "Listener socket creation failed");
+        return -1;
+    }
+
+    /* reuse address */
+    optval = 1;
+    rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
+    if (rc < 0) {
+        crm_perror(LOG_ERR, "Local address reuse not allowed on %s", buffer);
+        close(fd);
+        return -1;
+    }
+
+    if (addr->ai_family == AF_INET6) {
+        optval = 0;
+        rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &optval, sizeof(optval));
+        if (rc < 0) {
+            crm_perror(LOG_INFO, "Couldn't disable IPV6-only on %s", buffer);
+            close(fd);
+            return -1;
+        }
+    }
+
+    if (bind(fd, addr->ai_addr, addr->ai_addrlen) != 0) {
+        crm_perror(LOG_ERR, "Cannot bind to %s", buffer);
+        close(fd);
+        return -1;
+    }
+
+    if (listen(fd, 10) == -1) {
+        crm_perror(LOG_ERR, "Cannot listen on %s", buffer);
+        close(fd);
+        return -1;
+    }
+    return fd;
+}
+
+static int
+get_address_info(const char *bind_name, int port, struct addrinfo **res)
+{
+    int rc;
+    char port_str[6]; // at most "65535"
+    struct addrinfo hints;
+
+    memset(&hints, 0, sizeof(struct addrinfo));
+    hints.ai_flags = AI_PASSIVE;
+    hints.ai_family = AF_UNSPEC; // IPv6 or IPv4
+    hints.ai_socktype = SOCK_STREAM;
+    hints.ai_protocol = IPPROTO_TCP;
+
+    snprintf(port_str, sizeof(port_str), "%d", port);
+    rc = getaddrinfo(bind_name, port_str, &hints, res);
+    if (rc) {
+        crm_err("Unable to get IP address(es) for %s: %s",
+                (bind_name? bind_name : "local node"), gai_strerror(rc));
+        return -EADDRNOTAVAIL;
+    }
+    return pcmk_ok;
+}
+
+int
+lrmd_init_remote_tls_server(void)
+{
+    int filter;
+    int port = crm_default_remote_port();
+    struct addrinfo *res = NULL, *iter;
+    gnutls_datum_t psk_key = { NULL, 0 };
+    const char *bind_name = getenv("PCMK_remote_address");
+
+    static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
+        .dispatch = lrmd_remote_listen,
+        .destroy = tls_server_dropped,
+    };
+
+    CRM_CHECK(ssock == -1, return ssock);
+
+    crm_debug("Starting TLS listener on %s port %d",
+              (bind_name? bind_name : "all addresses on"), port);
+    crm_gnutls_global_init();
+    gnutls_global_set_log_function(debug_log);
+
+    if (pcmk__init_tls_dh(&dh_params) != pcmk_rc_ok) {
+        return -1;
+    }
+    gnutls_psk_allocate_server_credentials(&psk_cred_s);
+    gnutls_psk_set_server_credentials_function(psk_cred_s, lrmd_tls_server_key_cb);
+    gnutls_psk_set_server_dh_params(psk_cred_s, dh_params);
+
+    /* The key callback won't get called until the first client connection
+     * attempt. Do it once here, so we can warn the user at start-up if we can't
+     * read the key. We don't error out, though, because it's fine if the key is
+     * going to be added later.
+     */
+    if (lrmd__init_remote_key(&psk_key) != pcmk_rc_ok) {
+        crm_warn("A cluster connection will not be possible until the key is available");
+    }
+    gnutls_free(psk_key.data);
+
+    if (get_address_info(bind_name, port, &res) != pcmk_ok) {
+        return -1;
+    }
+
+    /* Currently we listen on only one address from the resulting list (the
+     * first IPv6 address we can bind to if possible, otherwise the first IPv4
+     * address we can bind to). When bind_name is NULL, this should be the
+     * respective wildcard address.
+     *
+     * @TODO If there is demand for specifying more than one address, allow
+     * bind_name to be a space-separated list, call getaddrinfo() for each,
+     * and create a socket for each result (set IPV6_V6ONLY on IPv6 sockets
+     * since IPv4 listeners will have their own sockets).
+     */
+    iter = res;
+    filter = AF_INET6;
+    while (iter) {
+        if (iter->ai_family == filter) {
+            ssock = bind_and_listen(iter);
+        }
+        if (ssock != -1) {
+            break;
+        }
+
+        iter = iter->ai_next;
+        if (iter == NULL && filter == AF_INET6) {
+            iter = res;
+            filter = AF_INET;
+        }
+    }
+
+    if (ssock >= 0) {
+        mainloop_add_fd("pacemaker-remote-server", G_PRIORITY_DEFAULT, ssock,
+                        NULL, &remote_listen_fd_callbacks);
+        crm_debug("Started TLS listener on %s port %d",
+                  (bind_name? bind_name : "all addresses on"), port);
+    }
+    freeaddrinfo(res);
+    return ssock;
+}
+
+void
+execd_stop_tls_server(void)
+{
+    if (psk_cred_s) {
+        gnutls_psk_free_server_credentials(psk_cred_s);
+        psk_cred_s = 0;
+    }
+
+    if (ssock >= 0) {
+        close(ssock);
+        ssock = -1;
+    }
+}
+#endif
diff --git a/daemons/fenced/Makefile.am b/daemons/fenced/Makefile.am
new file mode 100644
index 0000000..2ca0088
--- /dev/null
+++ b/daemons/fenced/Makefile.am
@@ -0,0 +1,52 @@
+#
+# Original Author: Sun Jiang Dong <sunjd@cn.ibm.com>
+# Copyright 2004 International Business Machines
+#
+# with later changes copyright 2004-2023 the Pacemaker project contributors.
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+include $(top_srcdir)/mk/common.mk
+include $(top_srcdir)/mk/man.mk
+
+halibdir	= $(CRM_DAEMON_DIR)
+
+halib_PROGRAMS	= pacemaker-fenced cts-fence-helper
+
+noinst_HEADERS	= pacemaker-fenced.h
+
+if BUILD_XML_HELP
+man7_MANS	= pacemaker-fenced.7
+endif
+
+cts_fence_helper_SOURCES	= cts-fence-helper.c
+cts_fence_helper_LDADD		= $(top_builddir)/lib/common/libcrmcommon.la	\
+				  $(top_builddir)/lib/fencing/libstonithd.la
+
+pacemaker_fenced_YFLAGS		= -d
+pacemaker_fenced_CFLAGS		= $(CFLAGS_HARDENED_EXE)
+pacemaker_fenced_LDFLAGS	= $(LDFLAGS_HARDENED_EXE)
+pacemaker_fenced_LDADD		= $(top_builddir)/lib/common/libcrmcommon.la		\
+				  $(top_builddir)/lib/cib/libcib.la		\
+				  $(top_builddir)/lib/cluster/libcrmcluster.la		\
+				  $(top_builddir)/lib/fencing/libstonithd.la		\
+				  $(top_builddir)/lib/pengine/libpe_status.la		\
+				  $(top_builddir)/lib/pacemaker/libpacemaker.la	\
+				  $(CLUSTERLIBS)
+pacemaker_fenced_SOURCES	= pacemaker-fenced.c \
+				  fenced_commands.c \
+				  fenced_remote.c \
+				  fenced_history.c
+
+CLEANFILES = $(man7_MANS) $(man8_MANS)
+
+if BUILD_LEGACY_LINKS
+install-exec-hook:
+	cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f stonithd && $(LN_S) pacemaker-fenced stonithd
+
+uninstall-hook:
+	cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f stonithd
+endif
diff --git a/daemons/fenced/cts-fence-helper.c b/daemons/fenced/cts-fence-helper.c
new file mode 100644
index 0000000..e18a1f4
--- /dev/null
+++ b/daemons/fenced/cts-fence-helper.c
@@ -0,0 +1,681 @@
+/*
+ * Copyright 2009-2023 the Pacemaker project contributors
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/utsname.h>
+
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/ipc.h>
+#include <crm/cluster/internal.h>
+
+#include <crm/stonith-ng.h>
+#include <crm/fencing/internal.h>
+#include <crm/common/agents.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/xml.h>
+
+#include <crm/common/mainloop.h>
+
+#define SUMMARY "cts-fence-helper - inject commands into the Pacemaker fencer and watch for events"
+
+static GMainLoop *mainloop = NULL;
+static crm_trigger_t *trig = NULL;
+static int mainloop_iter = 0;
+static pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
+
+typedef void (*mainloop_test_iteration_cb) (int check_event);
+
+#define MAINLOOP_DEFAULT_TIMEOUT 2
+
+enum test_modes {
+    test_standard = 0,  // test using a specific developer environment
+    test_passive,       // watch notifications only
+    test_api_sanity,    // sanity-test stonith client API using fence_dummy
+    test_api_mainloop,  // sanity-test mainloop code with async responses
+};
+
+struct {
+    enum test_modes mode;
+} options = {
+    .mode = test_standard
+};
+
+static gboolean
+mode_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
+    if (pcmk__str_any_of(option_name, "--mainloop_api_test", "-m", NULL)) {
+        options.mode = test_api_mainloop;
+    } else if (pcmk__str_any_of(option_name, "--api_test", "-t", NULL)) {
+        options.mode = test_api_sanity;
+    } else if (pcmk__str_any_of(option_name, "--passive", "-p", NULL)) {
+        options.mode = test_passive;
+    }
+
+    return TRUE;
+}
+
+static GOptionEntry entries[] = {
+    { "mainloop_api_test", 'm', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, mode_cb,
+      NULL, NULL,
+    },
+
+    { "api_test", 't', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, mode_cb,
+      NULL, NULL,
+    },
+
+    { "passive", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, mode_cb,
+      NULL, NULL,
+    },
+
+    { NULL }
+};
+
+static stonith_t *st = NULL;
+static struct pollfd pollfd;
+static const int st_opts = st_opt_sync_call;
+static int expected_notifications = 0;
+static int verbose = 0;
+
+static void
+mainloop_test_done(const char *origin, bool pass)
+{
+    if (pass) {
+        crm_info("SUCCESS - %s", origin);
+        mainloop_iter++;
+        mainloop_set_trigger(trig);
+        result.execution_status = PCMK_EXEC_DONE;
+        result.exit_status = CRM_EX_OK;
+    } else {
+        crm_err("FAILURE - %s (%d: %s)", origin, result.exit_status,
+                pcmk_exec_status_str(result.execution_status));
+        crm_exit(CRM_EX_ERROR);
+    }
+}
+
+
+static void
+dispatch_helper(int timeout)
+{
+    int rc;
+
+    crm_debug("Looking for notification");
+    pollfd.events = POLLIN;
+    while (true) {
+        rc = poll(&pollfd, 1, timeout); /* wait 10 minutes, -1 forever */
+        if (rc > 0) {
+            if (!stonith_dispatch(st)) {
+                break;
+            }
+        } else {
+            break;
+        }
+    }
+}
+
+static void
+st_callback(stonith_t * st, stonith_event_t * e)
+{
+    char *desc = NULL;
+
+    if (st->state == stonith_disconnected) {
+        crm_exit(CRM_EX_DISCONNECT);
+    }
+
+    desc = stonith__event_description(e);
+    crm_notice("%s", desc);
+    free(desc);
+
+    if (expected_notifications) {
+        expected_notifications--;
+    }
+}
+
+static void
+st_global_callback(stonith_t * stonith, stonith_callback_data_t * data)
+{
+    crm_notice("Call %d exited %d: %s (%s)",
+               data->call_id, stonith__exit_status(data),
+               stonith__execution_status(data),
+               pcmk__s(stonith__exit_reason(data), "unspecified reason"));
+}
+
+static void
+passive_test(void)
+{
+    int rc = 0;
+
+    rc = st->cmds->connect(st, crm_system_name, &pollfd.fd);
+    if (rc != pcmk_ok) {
+        stonith_api_delete(st);
+        crm_exit(CRM_EX_DISCONNECT);
+    }
+    st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT, st_callback);
+    st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, st_callback);
+    st->cmds->register_notification(st, STONITH_OP_DEVICE_ADD, st_callback);
+    st->cmds->register_notification(st, STONITH_OP_DEVICE_DEL, st_callback);
+    st->cmds->register_callback(st, 0, 120, st_opt_timeout_updates, NULL, "st_global_callback",
+                                st_global_callback);
+
+    dispatch_helper(600 * 1000);
+}
+
+#define single_test(cmd, str, num_notifications, expected_rc) \
+{ \
+    int rc = 0; \
+    rc = cmd; \
+    expected_notifications = 0;  \
+    if (num_notifications) { \
+        expected_notifications = num_notifications; \
+        dispatch_helper(500);  \
+    } \
+    if (rc != expected_rc) { \
+        crm_err("FAILURE - expected rc %d != %d(%s) for cmd - %s", expected_rc, rc, pcmk_strerror(rc), str); \
+        crm_exit(CRM_EX_ERROR); \
+    } else if (expected_notifications) { \
+        crm_err("FAILURE - expected %d notifications, got only %d for cmd - %s", \
+            num_notifications, num_notifications - expected_notifications, str); \
+        crm_exit(CRM_EX_ERROR); \
+    } else { \
+        if (verbose) {                   \
+            crm_info("SUCCESS - %s: %d", str, rc);    \
+        } else {   \
+            crm_debug("SUCCESS - %s: %d", str, rc);    \
+        }                          \
+    } \
+}\
+
+static void
+run_fence_failure_test(void)
+{
+    stonith_key_value_t *params = NULL;
+
+    params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
+                                   "false_1_node1=1,2 false_1_node2=3,4");
+    params = stonith_key_value_add(params, "mode", "fail");
+
+    single_test(st->
+                cmds->register_device(st, st_opts, "test-id1", "stonith-ng", "fence_dummy", params),
+                "Register device1 for failure test", 1, 0);
+
+    single_test(st->cmds->fence(st, st_opts, "false_1_node2", "off", 3, 0),
+                "Fence failure results off", 1, -ENODATA);
+
+    single_test(st->cmds->fence(st, st_opts, "false_1_node2", "reboot", 3, 0),
+                "Fence failure results reboot", 1, -ENODATA);
+
+    single_test(st->cmds->remove_device(st, st_opts, "test-id1"),
+                "Remove device1 for failure test", 1, 0);
+
+    stonith_key_value_freeall(params, 1, 1);
+}
+
+static void
+run_fence_failure_rollover_test(void)
+{
+    stonith_key_value_t *params = NULL;
+
+    params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
+                                   "false_1_node1=1,2 false_1_node2=3,4");
+    params = stonith_key_value_add(params, "mode", "fail");
+
+    single_test(st->
+                cmds->register_device(st, st_opts, "test-id1", "stonith-ng", "fence_dummy", params),
+                "Register device1 for rollover test", 1, 0);
+    stonith_key_value_freeall(params, 1, 1);
+    params = NULL;
+    params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
+                                   "false_1_node1=1,2 false_1_node2=3,4");
+    params = stonith_key_value_add(params, "mode", "pass");
+
+    single_test(st->
+                cmds->register_device(st, st_opts, "test-id2", "stonith-ng", "fence_dummy", params),
+                "Register device2 for rollover test", 1, 0);
+
+    single_test(st->cmds->fence(st, st_opts, "false_1_node2", "off", 3, 0),
+                "Fence rollover results off", 1, 0);
+
+    /* Expect -ENODEV because fence_dummy requires 'on' to be executed on target */
+    single_test(st->cmds->fence(st, st_opts, "false_1_node2", "on", 3, 0),
+                "Fence rollover results on", 1, -ENODEV);
+
+    single_test(st->cmds->remove_device(st, st_opts, "test-id1"),
+                "Remove device1 for rollover tests", 1, 0);
+
+    single_test(st->cmds->remove_device(st, st_opts, "test-id2"),
+                "Remove device2 for rollover tests", 1, 0);
+
+    stonith_key_value_freeall(params, 1, 1);
+}
+
+static void
+run_standard_test(void)
+{
+    stonith_key_value_t *params = NULL;
+
+    params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
+                                   "false_1_node1=1,2 false_1_node2=3,4");
+    params = stonith_key_value_add(params, "mode", "pass");
+    params = stonith_key_value_add(params, "mock_dynamic_hosts", "false_1_node1 false_1_node2");
+
+    single_test(st->
+                cmds->register_device(st, st_opts, "test-id", "stonith-ng", "fence_dummy", params),
+                "Register", 1, 0);
+    stonith_key_value_freeall(params, 1, 1);
+    params = NULL;
+
+    single_test(st->cmds->list(st, st_opts, "test-id", NULL, 1), "list", 1, 0);
+
+    single_test(st->cmds->monitor(st, st_opts, "test-id", 1), "Monitor", 1, 0);
+
+    single_test(st->cmds->status(st, st_opts, "test-id", "false_1_node2", 1),
+                "Status false_1_node2", 1, 0);
+
+    single_test(st->cmds->status(st, st_opts, "test-id", "false_1_node1", 1),
+                "Status false_1_node1", 1, 0);
+
+    single_test(st->cmds->fence(st, st_opts, "unknown-host", "off", 1, 0),
+                "Fence unknown-host (expected failure)", 0, -ENODEV);
+
+    single_test(st->cmds->fence(st, st_opts, "false_1_node1", "off", 1, 0),
+                "Fence false_1_node1", 1, 0);
+
+    /* Expect -ENODEV because fence_dummy requires 'on' to be executed on target */
+    single_test(st->cmds->fence(st, st_opts, "false_1_node1", "on", 1, 0),
+                "Unfence false_1_node1", 1, -ENODEV);
+
+    /* Confirm that an invalid level index is rejected */
+    single_test(st->cmds->register_level(st, st_opts, "node1", 999, params),
+                "Attempt to register an invalid level index", 0, -EINVAL);
+
+    single_test(st->cmds->remove_device(st, st_opts, "test-id"), "Remove test-id", 1, 0);
+
+    stonith_key_value_freeall(params, 1, 1);
+}
+
+static void
+sanity_tests(void)
+{
+    int rc = 0;
+
+    rc = st->cmds->connect(st, crm_system_name, &pollfd.fd);
+    if (rc != pcmk_ok) {
+        stonith_api_delete(st);
+        crm_exit(CRM_EX_DISCONNECT);
+    }
+    st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT, st_callback);
+    st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, st_callback);
+    st->cmds->register_notification(st, STONITH_OP_DEVICE_ADD, st_callback);
+    st->cmds->register_notification(st, STONITH_OP_DEVICE_DEL, st_callback);
+    st->cmds->register_callback(st, 0, 120, st_opt_timeout_updates, NULL, "st_global_callback",
+                                st_global_callback);
+
+    crm_info("Starting API Sanity Tests");
+    run_standard_test();
+    run_fence_failure_test();
+    run_fence_failure_rollover_test();
+    crm_info("Sanity Tests Passed");
+}
+
+static void
+standard_dev_test(void)
+{
+    int rc = 0;
+    char *tmp = NULL;
+    stonith_key_value_t *params = NULL;
+
+    rc = st->cmds->connect(st, crm_system_name, &pollfd.fd);
+    if (rc != pcmk_ok) {
+        stonith_api_delete(st);
+        crm_exit(CRM_EX_DISCONNECT);
+    }
+
+    params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
+                                   "some-host=pcmk-7 true_1_node1=3,4");
+
+    rc = st->cmds->register_device(st, st_opts, "test-id", "stonith-ng", "fence_xvm", params);
+    crm_debug("Register: %d", rc);
+
+    rc = st->cmds->list(st, st_opts, "test-id", &tmp, 10);
+    crm_debug("List: %d output: %s", rc, tmp ? tmp : "<none>");
+
+    rc = st->cmds->monitor(st, st_opts, "test-id", 10);
+    crm_debug("Monitor: %d", rc);
+
+    rc = st->cmds->status(st, st_opts, "test-id", "false_1_node2", 10);
+    crm_debug("Status false_1_node2: %d", rc);
+
+    rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10);
+    crm_debug("Status false_1_node1: %d", rc);
+
+    rc = st->cmds->fence(st, st_opts, "unknown-host", "off", 60, 0);
+    crm_debug("Fence unknown-host: %d", rc);
+
+    rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10);
+    crm_debug("Status false_1_node1: %d", rc);
+
+    rc = st->cmds->fence(st, st_opts, "false_1_node1", "off", 60, 0);
+    crm_debug("Fence false_1_node1: %d", rc);
+
+    rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10);
+    crm_debug("Status false_1_node1: %d", rc);
+
+    rc = st->cmds->fence(st, st_opts, "false_1_node1", "on", 10, 0);
+    crm_debug("Unfence false_1_node1: %d", rc);
+
+    rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10);
+    crm_debug("Status false_1_node1: %d", rc);
+
+    rc = st->cmds->fence(st, st_opts, "some-host", "off", 10, 0);
+    crm_debug("Fence alias: %d", rc);
+
+    rc = st->cmds->status(st, st_opts, "test-id", "some-host", 10);
+    crm_debug("Status alias: %d", rc);
+
+    rc = st->cmds->fence(st, st_opts, "false_1_node1", "on", 10, 0);
+    crm_debug("Unfence false_1_node1: %d", rc);
+
+    rc = st->cmds->remove_device(st, st_opts, "test-id");
+    crm_debug("Remove test-id: %d", rc);
+
+    stonith_key_value_freeall(params, 1, 1);
+}
+
+static void
+ iterate_mainloop_tests(gboolean event_ready);
+
+static void
+mainloop_callback(stonith_t * stonith, stonith_callback_data_t * data)
+{
+    pcmk__set_result(&result, stonith__exit_status(data),
+                     stonith__execution_status(data),
+                     stonith__exit_reason(data));
+    iterate_mainloop_tests(TRUE);
+}
+
+static int
+register_callback_helper(int callid)
+{
+    return st->cmds->register_callback(st,
+                                       callid,
+                                       MAINLOOP_DEFAULT_TIMEOUT,
+                                       st_opt_timeout_updates, NULL, "callback", mainloop_callback);
+}
+
+static void
+test_async_fence_pass(int check_event)
+{
+    int rc = 0;
+
+    if (check_event) {
+        mainloop_test_done(__func__, (result.exit_status == CRM_EX_OK));
+        return;
+    }
+
+    rc = st->cmds->fence(st, 0, "true_1_node1", "off", MAINLOOP_DEFAULT_TIMEOUT, 0);
+    if (rc < 0) {
+        crm_err("fence failed with rc %d", rc);
+        mainloop_test_done(__func__, false);
+    }
+    register_callback_helper(rc);
+    /* wait for event */
+}
+
+#define CUSTOM_TIMEOUT_ADDITION 10
+static void
+test_async_fence_custom_timeout(int check_event)
+{
+    int rc = 0;
+    static time_t begin = 0;
+
+    if (check_event) {
+        uint32_t diff = (time(NULL) - begin);
+
+        if (result.execution_status != PCMK_EXEC_TIMEOUT) {
+            mainloop_test_done(__func__, false);
+        } else if (diff < CUSTOM_TIMEOUT_ADDITION + MAINLOOP_DEFAULT_TIMEOUT) {
+            crm_err
+                ("Custom timeout test failed, callback expiration should be updated to %d, actual timeout was %d",
+                 CUSTOM_TIMEOUT_ADDITION + MAINLOOP_DEFAULT_TIMEOUT, diff);
+            mainloop_test_done(__func__, false);
+        } else {
+            mainloop_test_done(__func__, true);
+        }
+        return;
+    }
+    begin = time(NULL);
+
+    rc = st->cmds->fence(st, 0, "custom_timeout_node1", "off", MAINLOOP_DEFAULT_TIMEOUT, 0);
+    if (rc < 0) {
+        crm_err("fence failed with rc %d", rc);
+        mainloop_test_done(__func__, false);
+    }
+    register_callback_helper(rc);
+    /* wait for event */
+}
+
+static void
+test_async_fence_timeout(int check_event)
+{
+    int rc = 0;
+
+    if (check_event) {
+        mainloop_test_done(__func__,
+                           (result.execution_status == PCMK_EXEC_NO_FENCE_DEVICE));
+        return;
+    }
+
+    rc = st->cmds->fence(st, 0, "false_1_node2", "off", MAINLOOP_DEFAULT_TIMEOUT, 0);
+    if (rc < 0) {
+        crm_err("fence failed with rc %d", rc);
+        mainloop_test_done(__func__, false);
+    }
+    register_callback_helper(rc);
+    /* wait for event */
+}
+
+static void
+test_async_monitor(int check_event)
+{
+    int rc = 0;
+
+    if (check_event) {
+        mainloop_test_done(__func__, (result.exit_status == CRM_EX_OK));
+        return;
+    }
+
+    rc = st->cmds->monitor(st, 0, "false_1", MAINLOOP_DEFAULT_TIMEOUT);
+    if (rc < 0) {
+        crm_err("monitor failed with rc %d", rc);
+        mainloop_test_done(__func__, false);
+    }
+
+    register_callback_helper(rc);
+    /* wait for event */
+}
+
+static void
+test_register_async_devices(int check_event)
+{
+    char buf[16] = { 0, };
+    stonith_key_value_t *params = NULL;
+
+    params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
+                                   "false_1_node1=1,2");
+    params = stonith_key_value_add(params, "mode", "fail");
+    st->cmds->register_device(st, st_opts, "false_1", "stonith-ng", "fence_dummy", params);
+    stonith_key_value_freeall(params, 1, 1);
+
+    params = NULL;
+    params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
+                                   "true_1_node1=1,2");
+    params = stonith_key_value_add(params, "mode", "pass");
+    st->cmds->register_device(st, st_opts, "true_1", "stonith-ng", "fence_dummy", params);
+    stonith_key_value_freeall(params, 1, 1);
+
+    params = NULL;
+    params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
+                                   "custom_timeout_node1=1,2");
+    params = stonith_key_value_add(params, "mode", "fail");
+    params = stonith_key_value_add(params, "delay", "1000");
+    snprintf(buf, sizeof(buf) - 1, "%d", MAINLOOP_DEFAULT_TIMEOUT + CUSTOM_TIMEOUT_ADDITION);
+    params = stonith_key_value_add(params, "pcmk_off_timeout", buf);
+    st->cmds->register_device(st, st_opts, "false_custom_timeout", "stonith-ng", "fence_dummy",
+                              params);
+    stonith_key_value_freeall(params, 1, 1);
+
+    mainloop_test_done(__func__, true);
+}
+
+static void
+try_mainloop_connect(int check_event)
+{
+    int rc = stonith_api_connect_retry(st, crm_system_name, 10);
+
+    if (rc == pcmk_ok) {
+        mainloop_test_done(__func__, true);
+        return;
+    }
+    crm_err("API CONNECTION FAILURE");
+    mainloop_test_done(__func__, false);
+}
+
+static void
+iterate_mainloop_tests(gboolean event_ready)
+{
+    static mainloop_test_iteration_cb callbacks[] = {
+        try_mainloop_connect,
+        test_register_async_devices,
+        test_async_monitor,
+        test_async_fence_pass,
+        test_async_fence_timeout,
+        test_async_fence_custom_timeout,
+    };
+
+    if (mainloop_iter == (sizeof(callbacks) / sizeof(mainloop_test_iteration_cb))) {
+        /* all tests ran, everything passed */
+        crm_info("ALL MAINLOOP TESTS PASSED!");
+        crm_exit(CRM_EX_OK);
+    }
+
+    callbacks[mainloop_iter] (event_ready);
+}
+
+static gboolean
+trigger_iterate_mainloop_tests(gpointer user_data)
+{
+    iterate_mainloop_tests(FALSE);
+    return TRUE;
+}
+
+static void
+test_shutdown(int nsig)
+{
+    int rc = 0;
+
+    if (st) {
+        rc = st->cmds->disconnect(st);
+        crm_info("Disconnect: %d", rc);
+
+        crm_debug("Destroy");
+        stonith_api_delete(st);
+    }
+
+    if (rc) {
+        crm_exit(CRM_EX_ERROR);
+    }
+}
+
+static void
+mainloop_tests(void)
+{
+    trig = mainloop_add_trigger(G_PRIORITY_HIGH, trigger_iterate_mainloop_tests, NULL);
+    mainloop_set_trigger(trig);
+    mainloop_add_signal(SIGTERM, test_shutdown);
+
+    crm_info("Starting");
+    mainloop = g_main_loop_new(NULL, FALSE);
+    g_main_loop_run(mainloop);
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    context = pcmk__build_arg_context(args, NULL, group, NULL);
+    pcmk__add_main_args(context, entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    GError *error = NULL;
+    crm_exit_t exit_code = CRM_EX_OK;
+
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, NULL);
+    GOptionContext *context = build_arg_context(args, NULL);
+
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    /* We have to use crm_log_init here to set up the logging because there's
+     * different handling for daemons vs. command line programs, and
+     * pcmk__cli_init_logging is set up to only handle the latter.
+     */
+    crm_log_init(NULL, LOG_INFO, TRUE, (verbose? TRUE : FALSE), argc, argv,
+                 FALSE);
+
+    for (int i = 0; i < args->verbosity; i++) {
+        crm_bump_log_level(argc, argv);
+    }
+
+    st = stonith_api_new();
+    if (st == NULL) {
+        exit_code = CRM_EX_DISCONNECT;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Could not connect to fencer: API memory allocation failed");
+        goto done;
+    }
+
+    switch (options.mode) {
+        case test_standard:
+            standard_dev_test();
+            break;
+        case test_passive:
+            passive_test();
+            break;
+        case test_api_sanity:
+            sanity_tests();
+            break;
+        case test_api_mainloop:
+            mainloop_tests();
+            break;
+    }
+
+    test_shutdown(0);
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    pcmk__output_and_clear_error(&error, NULL);
+    crm_exit(exit_code);
+}
diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c
new file mode 100644
index 0000000..ba63cf8
--- /dev/null
+++ b/daemons/fenced/fenced_commands.c
@@ -0,0 +1,3674 @@
+/*
+ * Copyright 2009-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/utsname.h>
+
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <ctype.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/cluster/internal.h>
+#include <crm/common/mainloop.h>
+
+#include <crm/stonith-ng.h>
+#include <crm/fencing/internal.h>
+#include <crm/common/xml.h>
+
+#include <pacemaker-fenced.h>
+
+GHashTable *device_list = NULL;
+GHashTable *topology = NULL;
+static GList *cmd_list = NULL;
+
+static GHashTable *fenced_handlers = NULL;
+
+struct device_search_s {
+    /* target of fence action */
+    char *host;
+    /* requested fence action */
+    char *action;
+    /* timeout to use if a device is queried dynamically for possible targets */
+    int per_device_timeout;
+    /* number of registered fencing devices at time of request */
+    int replies_needed;
+    /* number of device replies received so far */
+    int replies_received;
+    /* whether the target is eligible to perform requested action (or off) */
+    bool allow_suicide;
+
+    /* private data to pass to search callback function */
+    void *user_data;
+    /* function to call when all replies have been received */
+    void (*callback) (GList * devices, void *user_data);
+    /* devices capable of performing requested action (or off if remapping) */
+    GList *capable;
+    /* Whether to perform searches that support the action */
+    uint32_t support_action_only;
+};
+
+static gboolean stonith_device_dispatch(gpointer user_data);
+static void st_child_done(int pid, const pcmk__action_result_t *result,
+                          void *user_data);
+static void stonith_send_reply(xmlNode * reply, int call_options, const char *remote_peer,
+                               pcmk__client_t *client);
+
+static void search_devices_record_result(struct device_search_s *search, const char *device,
+                                         gboolean can_fence);
+
+static int get_agent_metadata(const char *agent, xmlNode **metadata);
+static void read_action_metadata(stonith_device_t *device);
+static enum fenced_target_by unpack_level_kind(const xmlNode *level);
+
+typedef struct async_command_s {
+
+    int id;
+    int pid;
+    int fd_stdout;
+    int options;
+    int default_timeout; /* seconds */
+    int timeout; /* seconds */
+
+    int start_delay; // seconds (-1 means disable static/random fencing delays)
+    int delay_id;
+
+    char *op;
+    char *origin;
+    char *client;
+    char *client_name;
+    char *remote_op_id;
+
+    char *target;
+    uint32_t target_nodeid;
+    char *action;
+    char *device;
+
+    GList *device_list;
+    GList *next_device_iter; // device_list entry for next device to execute
+
+    void *internal_user_data;
+    void (*done_cb) (int pid, const pcmk__action_result_t *result,
+                     void *user_data);
+    guint timer_sigterm;
+    guint timer_sigkill;
+    /*! If the operation timed out, this is the last signal
+     *  we sent to the process to get it to terminate */
+    int last_timeout_signo;
+
+    stonith_device_t *active_on;
+    stonith_device_t *activating_on;
+} async_command_t;
+
+static xmlNode *construct_async_reply(const async_command_t *cmd,
+                                      const pcmk__action_result_t *result);
+
+static gboolean
+is_action_required(const char *action, const stonith_device_t *device)
+{
+    return (device != NULL) && device->automatic_unfencing
+           && pcmk__str_eq(action, "on", pcmk__str_none);
+}
+
+static int
+get_action_delay_max(const stonith_device_t *device, const char *action)
+{
+    const char *value = NULL;
+    int delay_max = 0;
+
+    if (!pcmk__is_fencing_action(action)) {
+        return 0;
+    }
+
+    value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_MAX);
+    if (value) {
+       delay_max = crm_parse_interval_spec(value) / 1000;
+    }
+
+    return delay_max;
+}
+
+static int
+get_action_delay_base(const stonith_device_t *device, const char *action,
+                      const char *target)
+{
+    char *hash_value = NULL;
+    int delay_base = 0;
+
+    if (!pcmk__is_fencing_action(action)) {
+        return 0;
+    }
+
+    hash_value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_BASE);
+
+    if (hash_value) {
+        char *value = strdup(hash_value);
+        char *valptr = value;
+
+        CRM_ASSERT(value != NULL);
+
+        if (target != NULL) {
+            for (char *val = strtok(value, "; \t"); val != NULL; val = strtok(NULL, "; \t")) {
+                char *mapval = strchr(val, ':');
+
+                if (mapval == NULL || mapval[1] == 0) {
+                    crm_err("pcmk_delay_base: empty value in mapping", val);
+                    continue;
+                }
+
+                if (mapval != val && strncasecmp(target, val, (size_t)(mapval - val)) == 0) {
+                    value = mapval + 1;
+                    crm_debug("pcmk_delay_base mapped to %s for %s",
+                              value, target);
+                    break;
+                }
+            }
+        }
+
+        if (strchr(value, ':') == 0) {
+           delay_base = crm_parse_interval_spec(value) / 1000;
+        }
+
+        free(valptr);
+    }
+
+    return delay_base;
+}
+
+/*!
+ * \internal
+ * \brief Override STONITH timeout with pcmk_*_timeout if available
+ *
+ * \param[in] device           STONITH device to use
+ * \param[in] action           STONITH action name
+ * \param[in] default_timeout  Timeout to use if device does not have
+ *                             a pcmk_*_timeout parameter for action
+ *
+ * \return Value of pcmk_(action)_timeout if available, otherwise default_timeout
+ * \note For consistency, it would be nice if reboot/off/on timeouts could be
+ *       set the same way as start/stop/monitor timeouts, i.e. with an
+ *       <operation> entry in the fencing resource configuration. However that
+ *       is insufficient because fencing devices may be registered directly via
+ *       the fencer's register_device() API instead of going through the CIB
+ *       (e.g. stonith_admin uses it for its -R option, and the executor uses it
+ *       to ensure a device is registered when a command is issued). As device
+ *       properties, pcmk_*_timeout parameters can be grabbed by the fencer when
+ *       the device is registered, whether by CIB change or API call.
+ */
+static int
+get_action_timeout(const stonith_device_t *device, const char *action,
+                   int default_timeout)
+{
+    if (action && device && device->params) {
+        char buffer[64] = { 0, };
+        const char *value = NULL;
+
+        /* If "reboot" was requested but the device does not support it,
+         * we will remap to "off", so check timeout for "off" instead
+         */
+        if (pcmk__str_eq(action, "reboot", pcmk__str_none)
+            && !pcmk_is_set(device->flags, st_device_supports_reboot)) {
+            crm_trace("%s doesn't support reboot, using timeout for off instead",
+                      device->id);
+            action = "off";
+        }
+
+        /* If the device config specified an action-specific timeout, use it */
+        snprintf(buffer, sizeof(buffer), "pcmk_%s_timeout", action);
+        value = g_hash_table_lookup(device->params, buffer);
+        if (value) {
+            return atoi(value);
+        }
+    }
+    return default_timeout;
+}
+
+/*!
+ * \internal
+ * \brief Get the currently executing device for a fencing operation
+ *
+ * \param[in] cmd  Fencing operation to check
+ *
+ * \return Currently executing device for \p cmd if any, otherwise NULL
+ */
+static stonith_device_t *
+cmd_device(const async_command_t *cmd)
+{
+    if ((cmd == NULL) || (cmd->device == NULL) || (device_list == NULL)) {
+        return NULL;
+    }
+    return g_hash_table_lookup(device_list, cmd->device);
+}
+
+/*!
+ * \internal
+ * \brief Return the configured reboot action for a given device
+ *
+ * \param[in] device_id  Device ID
+ *
+ * \return Configured reboot action for \p device_id
+ */
+const char *
+fenced_device_reboot_action(const char *device_id)
+{
+    const char *action = NULL;
+
+    if ((device_list != NULL) && (device_id != NULL)) {
+        stonith_device_t *device = g_hash_table_lookup(device_list, device_id);
+
+        if ((device != NULL) && (device->params != NULL)) {
+            action = g_hash_table_lookup(device->params, "pcmk_reboot_action");
+        }
+    }
+    return pcmk__s(action, "reboot");
+}
+
+/*!
+ * \internal
+ * \brief Check whether a given device supports the "on" action
+ *
+ * \param[in] device_id  Device ID
+ *
+ * \return true if \p device_id supports "on", otherwise false
+ */
+bool
+fenced_device_supports_on(const char *device_id)
+{
+    if ((device_list != NULL) && (device_id != NULL)) {
+        stonith_device_t *device = g_hash_table_lookup(device_list, device_id);
+
+        if (device != NULL) {
+            return pcmk_is_set(device->flags, st_device_supports_on);
+        }
+    }
+    return false;
+}
+
+static void
+free_async_command(async_command_t * cmd)
+{
+    if (!cmd) {
+        return;
+    }
+
+    if (cmd->delay_id) {
+        g_source_remove(cmd->delay_id);
+    }
+
+    cmd_list = g_list_remove(cmd_list, cmd);
+
+    g_list_free_full(cmd->device_list, free);
+    free(cmd->device);
+    free(cmd->action);
+    free(cmd->target);
+    free(cmd->remote_op_id);
+    free(cmd->client);
+    free(cmd->client_name);
+    free(cmd->origin);
+    free(cmd->op);
+    free(cmd);
+}
+
+/*!
+ * \internal
+ * \brief Create a new asynchronous fencing operation from request XML
+ *
+ * \param[in] msg  Fencing request XML (from IPC or CPG)
+ *
+ * \return Newly allocated fencing operation on success, otherwise NULL
+ *
+ * \note This asserts on memory errors, so a NULL return indicates an
+ *       unparseable message.
+ */
+static async_command_t *
+create_async_command(xmlNode *msg)
+{
+    xmlNode *op = NULL;
+    async_command_t *cmd = NULL;
+
+    if (msg == NULL) {
+        return NULL;
+    }
+
+    op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
+    if (op == NULL) {
+        return NULL;
+    }
+
+    cmd = calloc(1, sizeof(async_command_t));
+    CRM_ASSERT(cmd != NULL);
+
+    // All messages must include these
+    cmd->action = crm_element_value_copy(op, F_STONITH_ACTION);
+    cmd->op = crm_element_value_copy(msg, F_STONITH_OPERATION);
+    cmd->client = crm_element_value_copy(msg, F_STONITH_CLIENTID);
+    if ((cmd->action == NULL) || (cmd->op == NULL) || (cmd->client == NULL)) {
+        free_async_command(cmd);
+        return NULL;
+    }
+
+    crm_element_value_int(msg, F_STONITH_CALLID, &(cmd->id));
+    crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options));
+    crm_element_value_int(msg, F_STONITH_DELAY, &(cmd->start_delay));
+    crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->default_timeout));
+    cmd->timeout = cmd->default_timeout;
+
+    cmd->origin = crm_element_value_copy(msg, F_ORIG);
+    cmd->remote_op_id = crm_element_value_copy(msg, F_STONITH_REMOTE_OP_ID);
+    cmd->client_name = crm_element_value_copy(msg, F_STONITH_CLIENTNAME);
+    cmd->target = crm_element_value_copy(op, F_STONITH_TARGET);
+    cmd->device = crm_element_value_copy(op, F_STONITH_DEVICE);
+
+    cmd->done_cb = st_child_done;
+
+    // Track in global command list
+    cmd_list = g_list_append(cmd_list, cmd);
+
+    return cmd;
+}
+
+static int
+get_action_limit(stonith_device_t * device)
+{
+    const char *value = NULL;
+    int action_limit = 1;
+
+    value = g_hash_table_lookup(device->params, PCMK_STONITH_ACTION_LIMIT);
+    if ((value == NULL)
+        || (pcmk__scan_min_int(value, &action_limit, INT_MIN) != pcmk_rc_ok)
+        || (action_limit == 0)) {
+        action_limit = 1;
+    }
+    return action_limit;
+}
+
+static int
+get_active_cmds(stonith_device_t * device)
+{
+    int counter = 0;
+    GList *gIter = NULL;
+    GList *gIterNext = NULL;
+
+    CRM_CHECK(device != NULL, return 0);
+
+    for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
+        async_command_t *cmd = gIter->data;
+
+        gIterNext = gIter->next;
+
+        if (cmd->active_on == device) {
+            counter++;
+        }
+    }
+
+    return counter;
+}
+
+static void
+fork_cb(int pid, void *user_data)
+{
+    async_command_t *cmd = (async_command_t *) user_data;
+    stonith_device_t * device =
+        /* in case of a retry we've done the move from
+           activating_on to active_on already
+         */
+        cmd->activating_on?cmd->activating_on:cmd->active_on;
+
+    CRM_ASSERT(device);
+    crm_debug("Operation '%s' [%d]%s%s using %s now running with %ds timeout",
+              cmd->action, pid,
+              ((cmd->target == NULL)? "" : " targeting "),
+              pcmk__s(cmd->target, ""), device->id, cmd->timeout);
+    cmd->active_on = device;
+    cmd->activating_on = NULL;
+}
+
+static int
+get_agent_metadata_cb(gpointer data) {
+    stonith_device_t *device = data;
+    guint period_ms;
+
+    switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
+        case pcmk_rc_ok:
+            if (device->agent_metadata) {
+                read_action_metadata(device);
+                stonith__device_parameter_flags(&(device->flags), device->id,
+                                        device->agent_metadata);
+            }
+            return G_SOURCE_REMOVE;
+
+        case EAGAIN:
+            period_ms = pcmk__mainloop_timer_get_period(device->timer);
+            if (period_ms < 160 * 1000) {
+                mainloop_timer_set_period(device->timer, 2 * period_ms);
+            }
+            return G_SOURCE_CONTINUE;
+
+        default:
+            return G_SOURCE_REMOVE;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Call a command's action callback for an internal (not library) result
+ *
+ * \param[in,out] cmd               Command to report result for
+ * \param[in]     execution_status  Execution status to use for result
+ * \param[in]     exit_status       Exit status to use for result
+ * \param[in]     exit_reason       Exit reason to use for result
+ */
+static void
+report_internal_result(async_command_t *cmd, int exit_status,
+                       int execution_status, const char *exit_reason)
+{
+    pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
+
+    pcmk__set_result(&result, exit_status, execution_status, exit_reason);
+    cmd->done_cb(0, &result, cmd);
+    pcmk__reset_result(&result);
+}
+
+static gboolean
+stonith_device_execute(stonith_device_t * device)
+{
+    int exec_rc = 0;
+    const char *action_str = NULL;
+    const char *host_arg = NULL;
+    async_command_t *cmd = NULL;
+    stonith_action_t *action = NULL;
+    int active_cmds = 0;
+    int action_limit = 0;
+    GList *gIter = NULL;
+    GList *gIterNext = NULL;
+
+    CRM_CHECK(device != NULL, return FALSE);
+
+    active_cmds = get_active_cmds(device);
+    action_limit = get_action_limit(device);
+    if (action_limit > -1 && active_cmds >= action_limit) {
+        crm_trace("%s is over its action limit of %d (%u active action%s)",
+                  device->id, action_limit, active_cmds,
+                  pcmk__plural_s(active_cmds));
+        return TRUE;
+    }
+
+    for (gIter = device->pending_ops; gIter != NULL; gIter = gIterNext) {
+        async_command_t *pending_op = gIter->data;
+
+        gIterNext = gIter->next;
+
+        if (pending_op && pending_op->delay_id) {
+            crm_trace("Operation '%s'%s%s using %s was asked to run too early, "
+                      "waiting for start delay of %ds",
+                      pending_op->action,
+                      ((pending_op->target == NULL)? "" : " targeting "),
+                      pcmk__s(pending_op->target, ""),
+                      device->id, pending_op->start_delay);
+            continue;
+        }
+
+        device->pending_ops = g_list_remove_link(device->pending_ops, gIter);
+        g_list_free_1(gIter);
+
+        cmd = pending_op;
+        break;
+    }
+
+    if (cmd == NULL) {
+        crm_trace("No actions using %s are needed", device->id);
+        return TRUE;
+    }
+
+    if (pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
+                         STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
+        if (pcmk__is_fencing_action(cmd->action)) {
+            if (node_does_watchdog_fencing(stonith_our_uname)) {
+                pcmk__panic(__func__);
+                goto done;
+            }
+        } else {
+            crm_info("Faking success for %s watchdog operation", cmd->action);
+            report_internal_result(cmd, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+            goto done;
+        }
+    }
+
+#if SUPPORT_CIBSECRETS
+    exec_rc = pcmk__substitute_secrets(device->id, device->params);
+    if (exec_rc != pcmk_rc_ok) {
+        if (pcmk__str_eq(cmd->action, "stop", pcmk__str_none)) {
+            crm_info("Proceeding with stop operation for %s "
+                     "despite being unable to load CIB secrets (%s)",
+                     device->id, pcmk_rc_str(exec_rc));
+        } else {
+            crm_err("Considering %s unconfigured "
+                    "because unable to load CIB secrets: %s",
+                     device->id, pcmk_rc_str(exec_rc));
+            report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_SECRETS,
+                                   "Failed to get CIB secrets");
+            goto done;
+        }
+    }
+#endif
+
+    action_str = cmd->action;
+    if (pcmk__str_eq(cmd->action, "reboot", pcmk__str_none)
+        && !pcmk_is_set(device->flags, st_device_supports_reboot)) {
+
+        crm_notice("Remapping 'reboot' action%s%s using %s to 'off' "
+                   "because agent '%s' does not support reboot",
+                   ((cmd->target == NULL)? "" : " targeting "),
+                   pcmk__s(cmd->target, ""), device->id, device->agent);
+        action_str = "off";
+    }
+
+    if (pcmk_is_set(device->flags, st_device_supports_parameter_port)) {
+        host_arg = "port";
+
+    } else if (pcmk_is_set(device->flags, st_device_supports_parameter_plug)) {
+        host_arg = "plug";
+    }
+
+    action = stonith__action_create(device->agent, action_str, cmd->target,
+                                    cmd->target_nodeid, cmd->timeout,
+                                    device->params, device->aliases, host_arg);
+
+    /* for async exec, exec_rc is negative for early error exit
+       otherwise handling of success/errors is done via callbacks */
+    cmd->activating_on = device;
+    exec_rc = stonith__execute_async(action, (void *)cmd, cmd->done_cb,
+                                     fork_cb);
+    if (exec_rc < 0) {
+        cmd->activating_on = NULL;
+        cmd->done_cb(0, stonith__action_result(action), cmd);
+        stonith__destroy_action(action);
+    }
+
+done:
+    /* Device might get triggered to work by multiple fencing commands
+     * simultaneously. Trigger the device again to make sure any
+     * remaining concurrent commands get executed. */
+    if (device->pending_ops) {
+        mainloop_set_trigger(device->work);
+    }
+    return TRUE;
+}
+
+static gboolean
+stonith_device_dispatch(gpointer user_data)
+{
+    return stonith_device_execute(user_data);
+}
+
+static gboolean
+start_delay_helper(gpointer data)
+{
+    async_command_t *cmd = data;
+    stonith_device_t *device = cmd_device(cmd);
+
+    cmd->delay_id = 0;
+    if (device) {
+        mainloop_set_trigger(device->work);
+    }
+
+    return FALSE;
+}
+
+static void
+schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
+{
+    int delay_max = 0;
+    int delay_base = 0;
+    int requested_delay = cmd->start_delay;
+
+    CRM_CHECK(cmd != NULL, return);
+    CRM_CHECK(device != NULL, return);
+
+    if (cmd->device) {
+        free(cmd->device);
+    }
+
+    if (device->include_nodeid && (cmd->target != NULL)) {
+        crm_node_t *node = crm_get_peer(0, cmd->target);
+
+        cmd->target_nodeid = node->id;
+    }
+
+    cmd->device = strdup(device->id);
+    cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout);
+
+    if (cmd->remote_op_id) {
+        crm_debug("Scheduling '%s' action%s%s using %s for remote peer %s "
+                  "with op id %.8s and timeout %ds",
+                  cmd->action,
+                  (cmd->target == NULL)? "" : " targeting ",
+                  pcmk__s(cmd->target, ""),
+                  device->id, cmd->origin, cmd->remote_op_id, cmd->timeout);
+    } else {
+        crm_debug("Scheduling '%s' action%s%s using %s for %s with timeout %ds",
+                  cmd->action,
+                  (cmd->target == NULL)? "" : " targeting ",
+                  pcmk__s(cmd->target, ""),
+                  device->id, cmd->client, cmd->timeout);
+    }
+
+    device->pending_ops = g_list_append(device->pending_ops, cmd);
+    mainloop_set_trigger(device->work);
+
+    // Value -1 means disable any static/random fencing delays
+    if (requested_delay < 0) {
+        return;
+    }
+
+    delay_max = get_action_delay_max(device, cmd->action);
+    delay_base = get_action_delay_base(device, cmd->action, cmd->target);
+    if (delay_max == 0) {
+        delay_max = delay_base;
+    }
+    if (delay_max < delay_base) {
+        crm_warn(PCMK_STONITH_DELAY_BASE " (%ds) is larger than "
+                 PCMK_STONITH_DELAY_MAX " (%ds) for %s using %s "
+                 "(limiting to maximum delay)",
+                 delay_base, delay_max, cmd->action, device->id);
+        delay_base = delay_max;
+    }
+    if (delay_max > 0) {
+        // coverity[dont_call] We're not using rand() for security
+        cmd->start_delay +=
+            ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0)
+            + delay_base;
+    }
+
+    if (cmd->start_delay > 0) {
+        crm_notice("Delaying '%s' action%s%s using %s for %ds " CRM_XS
+                   " timeout=%ds requested_delay=%ds base=%ds max=%ds",
+                   cmd->action,
+                   (cmd->target == NULL)? "" : " targeting ",
+                   pcmk__s(cmd->target, ""),
+                   device->id, cmd->start_delay, cmd->timeout,
+                   requested_delay, delay_base, delay_max);
+        cmd->delay_id =
+            g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd);
+    }
+}
+
+static void
+free_device(gpointer data)
+{
+    GList *gIter = NULL;
+    stonith_device_t *device = data;
+
+    g_hash_table_destroy(device->params);
+    g_hash_table_destroy(device->aliases);
+
+    for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) {
+        async_command_t *cmd = gIter->data;
+
+        crm_warn("Removal of device '%s' purged operation '%s'", device->id, cmd->action);
+        report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
+                               "Device was removed before action could be executed");
+    }
+    g_list_free(device->pending_ops);
+
+    g_list_free_full(device->targets, free);
+
+    if (device->timer) {
+        mainloop_timer_stop(device->timer);
+        mainloop_timer_del(device->timer);
+    }
+
+    mainloop_destroy_trigger(device->work);
+
+    free_xml(device->agent_metadata);
+    free(device->namespace);
+    if (device->on_target_actions != NULL) {
+        g_string_free(device->on_target_actions, TRUE);
+    }
+    free(device->agent);
+    free(device->id);
+    free(device);
+}
+
+void free_device_list(void)
+{
+    if (device_list != NULL) {
+        g_hash_table_destroy(device_list);
+        device_list = NULL;
+    }
+}
+
+void
+init_device_list(void)
+{
+    if (device_list == NULL) {
+        device_list = pcmk__strkey_table(NULL, free_device);
+    }
+}
+
+static GHashTable *
+build_port_aliases(const char *hostmap, GList ** targets)
+{
+    char *name = NULL;
+    int last = 0, lpc = 0, max = 0, added = 0;
+    GHashTable *aliases = pcmk__strikey_table(free, free);
+
+    if (hostmap == NULL) {
+        return aliases;
+    }
+
+    max = strlen(hostmap);
+    for (; lpc <= max; lpc++) {
+        switch (hostmap[lpc]) {
+                /* Skip escaped chars */
+            case '\\':
+                lpc++;
+                break;
+
+                /* Assignment chars */
+            case '=':
+            case ':':
+                if (lpc > last) {
+                    free(name);
+                    name = calloc(1, 1 + lpc - last);
+                    memcpy(name, hostmap + last, lpc - last);
+                }
+                last = lpc + 1;
+                break;
+
+                /* Delimeter chars */
+                /* case ',': Potentially used to specify multiple ports */
+            case 0:
+            case ';':
+            case ' ':
+            case '\t':
+                if (name) {
+                    char *value = NULL;
+                    int k = 0;
+
+                    value = calloc(1, 1 + lpc - last);
+                    memcpy(value, hostmap + last, lpc - last);
+
+                    for (int i = 0; value[i] != '\0'; i++) {
+                        if (value[i] != '\\') {
+                            value[k++] = value[i];
+                        }
+                    }
+                    value[k] = '\0';
+
+                    crm_debug("Adding alias '%s'='%s'", name, value);
+                    g_hash_table_replace(aliases, name, value);
+                    if (targets) {
+                        *targets = g_list_append(*targets, strdup(value));
+                    }
+                    value = NULL;
+                    name = NULL;
+                    added++;
+
+                } else if (lpc > last) {
+                    crm_debug("Parse error at offset %d near '%s'", lpc - last, hostmap + last);
+                }
+
+                last = lpc + 1;
+                break;
+        }
+
+        if (hostmap[lpc] == 0) {
+            break;
+        }
+    }
+
+    if (added == 0) {
+        crm_info("No host mappings detected in '%s'", hostmap);
+    }
+
+    free(name);
+    return aliases;
+}
+
+GHashTable *metadata_cache = NULL;
+
+void
+free_metadata_cache(void) {
+    if (metadata_cache != NULL) {
+        g_hash_table_destroy(metadata_cache);
+        metadata_cache = NULL;
+    }
+}
+
+static void
+init_metadata_cache(void) {
+    if (metadata_cache == NULL) {
+        metadata_cache = pcmk__strkey_table(free, free);
+    }
+}
+
+int
+get_agent_metadata(const char *agent, xmlNode ** metadata)
+{
+    char *buffer = NULL;
+
+    if (metadata == NULL) {
+        return EINVAL;
+    }
+    *metadata = NULL;
+    if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT_INTERNAL, pcmk__str_none)) {
+        return pcmk_rc_ok;
+    }
+    init_metadata_cache();
+    buffer = g_hash_table_lookup(metadata_cache, agent);
+    if (buffer == NULL) {
+        stonith_t *st = stonith_api_new();
+        int rc;
+
+        if (st == NULL) {
+            crm_warn("Could not get agent meta-data: "
+                     "API memory allocation failed");
+            return EAGAIN;
+        }
+        rc = st->cmds->metadata(st, st_opt_sync_call, agent,
+                                NULL, &buffer, 10);
+        stonith_api_delete(st);
+        if (rc || !buffer) {
+            crm_err("Could not retrieve metadata for fencing agent %s", agent);
+            return EAGAIN;
+        }
+        g_hash_table_replace(metadata_cache, strdup(agent), buffer);
+    }
+
+    *metadata = string2xml(buffer);
+    return pcmk_rc_ok;
+}
+
+static gboolean
+is_nodeid_required(xmlNode * xml)
+{
+    xmlXPathObjectPtr xpath = NULL;
+
+    if (stand_alone) {
+        return FALSE;
+    }
+
+    if (!xml) {
+        return FALSE;
+    }
+
+    xpath = xpath_search(xml, "//parameter[@name='nodeid']");
+    if (numXpathResults(xpath)  <= 0) {
+        freeXpathObject(xpath);
+        return FALSE;
+    }
+
+    freeXpathObject(xpath);
+    return TRUE;
+}
+
+static void
+read_action_metadata(stonith_device_t *device)
+{
+    xmlXPathObjectPtr xpath = NULL;
+    int max = 0;
+    int lpc = 0;
+
+    if (device->agent_metadata == NULL) {
+        return;
+    }
+
+    xpath = xpath_search(device->agent_metadata, "//action");
+    max = numXpathResults(xpath);
+
+    if (max <= 0) {
+        freeXpathObject(xpath);
+        return;
+    }
+
+    for (lpc = 0; lpc < max; lpc++) {
+        const char *action = NULL;
+        xmlNode *match = getXpathResult(xpath, lpc);
+
+        CRM_LOG_ASSERT(match != NULL);
+        if(match == NULL) { continue; };
+
+        action = crm_element_value(match, "name");
+
+        if (pcmk__str_eq(action, "list", pcmk__str_none)) {
+            stonith__set_device_flags(device->flags, device->id,
+                                      st_device_supports_list);
+        } else if (pcmk__str_eq(action, "status", pcmk__str_none)) {
+            stonith__set_device_flags(device->flags, device->id,
+                                      st_device_supports_status);
+        } else if (pcmk__str_eq(action, "reboot", pcmk__str_none)) {
+            stonith__set_device_flags(device->flags, device->id,
+                                      st_device_supports_reboot);
+        } else if (pcmk__str_eq(action, "on", pcmk__str_none)) {
+            /* "automatic" means the cluster will unfence node when it joins */
+            /* "required" is a deprecated synonym for "automatic" */
+            if (pcmk__xe_attr_is_true(match, "automatic") || pcmk__xe_attr_is_true(match, "required")) {
+                device->automatic_unfencing = TRUE;
+            }
+            stonith__set_device_flags(device->flags, device->id,
+                                      st_device_supports_on);
+        }
+
+        if ((action != NULL) && pcmk__xe_attr_is_true(match, "on_target")) {
+            pcmk__add_word(&(device->on_target_actions), 64, action);
+        }
+    }
+
+    freeXpathObject(xpath);
+}
+
+/*!
+ * \internal
+ * \brief Set a pcmk_*_action parameter if not already set
+ *
+ * \param[in,out] params  Device parameters
+ * \param[in]     action  Name of action
+ * \param[in]     value   Value to use if action is not already set
+ */
+static void
+map_action(GHashTable *params, const char *action, const char *value)
+{
+    char *key = crm_strdup_printf("pcmk_%s_action", action);
+
+    if (g_hash_table_lookup(params, key)) {
+        crm_warn("Ignoring %s='%s', see %s instead",
+                 STONITH_ATTR_ACTION_OP, value, key);
+        free(key);
+    } else {
+        crm_warn("Mapping %s='%s' to %s='%s'",
+                 STONITH_ATTR_ACTION_OP, value, key, value);
+        g_hash_table_insert(params, key, strdup(value));
+    }
+}
+
+/*!
+ * \internal
+ * \brief Create device parameter table from XML
+ *
+ * \param[in]  name    Device name (used for logging only)
+ * \param[in]  dev     XML containing device parameters
+ */
+static GHashTable *
+xml2device_params(const char *name, const xmlNode *dev)
+{
+    GHashTable *params = xml2list(dev);
+    const char *value;
+
+    /* Action should never be specified in the device configuration,
+     * but we support it for users who are familiar with other software
+     * that worked that way.
+     */
+    value = g_hash_table_lookup(params, STONITH_ATTR_ACTION_OP);
+    if (value != NULL) {
+        crm_warn("%s has '%s' parameter, which should never be specified in configuration",
+                 name, STONITH_ATTR_ACTION_OP);
+
+        if (*value == '\0') {
+            crm_warn("Ignoring empty '%s' parameter", STONITH_ATTR_ACTION_OP);
+
+        } else if (strcmp(value, "reboot") == 0) {
+            crm_warn("Ignoring %s='reboot' (see stonith-action cluster property instead)",
+                     STONITH_ATTR_ACTION_OP);
+
+        } else if (strcmp(value, "off") == 0) {
+            map_action(params, "reboot", value);
+
+        } else {
+            map_action(params, "off", value);
+            map_action(params, "reboot", value);
+        }
+
+        g_hash_table_remove(params, STONITH_ATTR_ACTION_OP);
+    }
+
+    return params;
+}
+
+static const char *
+target_list_type(stonith_device_t * dev)
+{
+    const char *check_type = NULL;
+
+    check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK);
+
+    if (check_type == NULL) {
+
+        if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) {
+            check_type = "static-list";
+        } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) {
+            check_type = "static-list";
+        } else if (pcmk_is_set(dev->flags, st_device_supports_list)) {
+            check_type = "dynamic-list";
+        } else if (pcmk_is_set(dev->flags, st_device_supports_status)) {
+            check_type = "status";
+        } else {
+            check_type = PCMK__VALUE_NONE;
+        }
+    }
+
+    return check_type;
+}
+
+static stonith_device_t *
+build_device_from_xml(xmlNode *dev)
+{
+    const char *value;
+    stonith_device_t *device = NULL;
+    char *agent = crm_element_value_copy(dev, "agent");
+
+    CRM_CHECK(agent != NULL, return device);
+
+    device = calloc(1, sizeof(stonith_device_t));
+
+    CRM_CHECK(device != NULL, {free(agent); return device;});
+
+    device->id = crm_element_value_copy(dev, XML_ATTR_ID);
+    device->agent = agent;
+    device->namespace = crm_element_value_copy(dev, "namespace");
+    device->params = xml2device_params(device->id, dev);
+
+    value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_LIST);
+    if (value) {
+        device->targets = stonith__parse_targets(value);
+    }
+
+    value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP);
+    device->aliases = build_port_aliases(value, &(device->targets));
+
+    value = target_list_type(device);
+    if (!pcmk__str_eq(value, "static-list", pcmk__str_casei) && device->targets) {
+        /* Other than "static-list", dev-> targets is unnecessary. */
+        g_list_free_full(device->targets, free);
+        device->targets = NULL;
+    }
+    switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
+        case pcmk_rc_ok:
+            if (device->agent_metadata) {
+                read_action_metadata(device);
+                stonith__device_parameter_flags(&(device->flags), device->id,
+                                                device->agent_metadata);
+            }
+            break;
+
+        case EAGAIN:
+            if (device->timer == NULL) {
+                device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000,
+                                           TRUE, get_agent_metadata_cb, device);
+            }
+            if (!mainloop_timer_running(device->timer)) {
+                mainloop_timer_start(device->timer);
+            }
+            break;
+
+        default:
+            break;
+    }
+
+    value = g_hash_table_lookup(device->params, "nodeid");
+    if (!value) {
+        device->include_nodeid = is_nodeid_required(device->agent_metadata);
+    }
+
+    value = crm_element_value(dev, "rsc_provides");
+    if (pcmk__str_eq(value, PCMK__VALUE_UNFENCING, pcmk__str_casei)) {
+        device->automatic_unfencing = TRUE;
+    }
+
+    if (is_action_required("on", device)) {
+        crm_info("Fencing device '%s' requires unfencing", device->id);
+    }
+
+    if (device->on_target_actions != NULL) {
+        crm_info("Fencing device '%s' requires actions (%s) to be executed "
+                 "on target", device->id,
+                 (const char *) device->on_target_actions->str);
+    }
+
+    device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device);
+    /* TODO: Hook up priority */
+
+    return device;
+}
+
+static void
+schedule_internal_command(const char *origin,
+                          stonith_device_t * device,
+                          const char *action,
+                          const char *target,
+                          int timeout,
+                          void *internal_user_data,
+                          void (*done_cb) (int pid,
+                                           const pcmk__action_result_t *result,
+                                           void *user_data))
+{
+    async_command_t *cmd = NULL;
+
+    cmd = calloc(1, sizeof(async_command_t));
+
+    cmd->id = -1;
+    cmd->default_timeout = timeout ? timeout : 60;
+    cmd->timeout = cmd->default_timeout;
+    cmd->action = strdup(action);
+    pcmk__str_update(&cmd->target, target);
+    cmd->device = strdup(device->id);
+    cmd->origin = strdup(origin);
+    cmd->client = strdup(crm_system_name);
+    cmd->client_name = strdup(crm_system_name);
+
+    cmd->internal_user_data = internal_user_data;
+    cmd->done_cb = done_cb; /* cmd, not internal_user_data, is passed to 'done_cb' as the userdata */
+
+    schedule_stonith_command(cmd, device);
+}
+
+// Fence agent status commands use custom exit status codes
+enum fence_status_code {
+    fence_status_invalid    = -1,
+    fence_status_active     = 0,
+    fence_status_unknown    = 1,
+    fence_status_inactive   = 2,
+};
+
+static void
+status_search_cb(int pid, const pcmk__action_result_t *result, void *user_data)
+{
+    async_command_t *cmd = user_data;
+    struct device_search_s *search = cmd->internal_user_data;
+    stonith_device_t *dev = cmd_device(cmd);
+    gboolean can = FALSE;
+
+    free_async_command(cmd);
+
+    if (!dev) {
+        search_devices_record_result(search, NULL, FALSE);
+        return;
+    }
+
+    mainloop_set_trigger(dev->work);
+
+    if (result->execution_status != PCMK_EXEC_DONE) {
+        crm_warn("Assuming %s cannot fence %s "
+                 "because status could not be executed: %s%s%s%s",
+                 dev->id, search->host,
+                 pcmk_exec_status_str(result->execution_status),
+                 ((result->exit_reason == NULL)? "" : " ("),
+                 ((result->exit_reason == NULL)? "" : result->exit_reason),
+                 ((result->exit_reason == NULL)? "" : ")"));
+        search_devices_record_result(search, dev->id, FALSE);
+        return;
+    }
+
+    switch (result->exit_status) {
+        case fence_status_unknown:
+            crm_trace("%s reported it cannot fence %s", dev->id, search->host);
+            break;
+
+        case fence_status_active:
+        case fence_status_inactive:
+            crm_trace("%s reported it can fence %s", dev->id, search->host);
+            can = TRUE;
+            break;
+
+        default:
+            crm_warn("Assuming %s cannot fence %s "
+                     "(status returned unknown code %d)",
+                     dev->id, search->host, result->exit_status);
+            break;
+    }
+    search_devices_record_result(search, dev->id, can);
+}
+
+static void
+dynamic_list_search_cb(int pid, const pcmk__action_result_t *result,
+                       void *user_data)
+{
+    async_command_t *cmd = user_data;
+    struct device_search_s *search = cmd->internal_user_data;
+    stonith_device_t *dev = cmd_device(cmd);
+    gboolean can_fence = FALSE;
+
+    free_async_command(cmd);
+
+    /* Host/alias must be in the list output to be eligible to be fenced
+     *
+     * Will cause problems if down'd nodes aren't listed or (for virtual nodes)
+     *  if the guest is still listed despite being moved to another machine
+     */
+    if (!dev) {
+        search_devices_record_result(search, NULL, FALSE);
+        return;
+    }
+
+    mainloop_set_trigger(dev->work);
+
+    if (pcmk__result_ok(result)) {
+        crm_info("Refreshing target list for %s", dev->id);
+        g_list_free_full(dev->targets, free);
+        dev->targets = stonith__parse_targets(result->action_stdout);
+        dev->targets_age = time(NULL);
+
+    } else if (dev->targets != NULL) {
+        if (result->execution_status == PCMK_EXEC_DONE) {
+            crm_info("Reusing most recent target list for %s "
+                     "because list returned error code %d",
+                     dev->id, result->exit_status);
+        } else {
+            crm_info("Reusing most recent target list for %s "
+                     "because list could not be executed: %s%s%s%s",
+                     dev->id, pcmk_exec_status_str(result->execution_status),
+                     ((result->exit_reason == NULL)? "" : " ("),
+                     ((result->exit_reason == NULL)? "" : result->exit_reason),
+                     ((result->exit_reason == NULL)? "" : ")"));
+        }
+
+    } else { // We have never successfully executed list
+        if (result->execution_status == PCMK_EXEC_DONE) {
+            crm_warn("Assuming %s cannot fence %s "
+                     "because list returned error code %d",
+                     dev->id, search->host, result->exit_status);
+        } else {
+            crm_warn("Assuming %s cannot fence %s "
+                     "because list could not be executed: %s%s%s%s",
+                     dev->id, search->host,
+                     pcmk_exec_status_str(result->execution_status),
+                     ((result->exit_reason == NULL)? "" : " ("),
+                     ((result->exit_reason == NULL)? "" : result->exit_reason),
+                     ((result->exit_reason == NULL)? "" : ")"));
+        }
+
+        /* Fall back to pcmk_host_check="status" if the user didn't explicitly
+         * specify "dynamic-list".
+         */
+        if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK) == NULL) {
+            crm_notice("Switching to pcmk_host_check='status' for %s", dev->id);
+            g_hash_table_replace(dev->params, strdup(PCMK_STONITH_HOST_CHECK),
+                                 strdup("status"));
+        }
+    }
+
+    if (dev->targets) {
+        const char *alias = g_hash_table_lookup(dev->aliases, search->host);
+
+        if (!alias) {
+            alias = search->host;
+        }
+        if (pcmk__str_in_list(alias, dev->targets, pcmk__str_casei)) {
+            can_fence = TRUE;
+        }
+    }
+    search_devices_record_result(search, dev->id, can_fence);
+}
+
+/*!
+ * \internal
+ * \brief Returns true if any key in first is not in second or second has a different value for key
+ */
+static int
+device_params_diff(GHashTable *first, GHashTable *second) {
+    char *key = NULL;
+    char *value = NULL;
+    GHashTableIter gIter;
+
+    g_hash_table_iter_init(&gIter, first);
+    while (g_hash_table_iter_next(&gIter, (void **)&key, (void **)&value)) {
+
+        if(strstr(key, "CRM_meta") == key) {
+            continue;
+        } else if(strcmp(key, "crm_feature_set") == 0) {
+            continue;
+        } else {
+            char *other_value = g_hash_table_lookup(second, key);
+
+            if (!other_value || !pcmk__str_eq(other_value, value, pcmk__str_casei)) {
+                crm_trace("Different value for %s: %s != %s", key, other_value, value);
+                return 1;
+            }
+        }
+    }
+
+    return 0;
+}
+
+/*!
+ * \internal
+ * \brief Checks to see if an identical device already exists in the device_list
+ */
+static stonith_device_t *
+device_has_duplicate(const stonith_device_t *device)
+{
+    stonith_device_t *dup = g_hash_table_lookup(device_list, device->id);
+
+    if (!dup) {
+        crm_trace("No match for %s", device->id);
+        return NULL;
+
+    } else if (!pcmk__str_eq(dup->agent, device->agent, pcmk__str_casei)) {
+        crm_trace("Different agent: %s != %s", dup->agent, device->agent);
+        return NULL;
+    }
+
+    /* Use calculate_operation_digest() here? */
+    if (device_params_diff(device->params, dup->params) ||
+        device_params_diff(dup->params, device->params)) {
+        return NULL;
+    }
+
+    crm_trace("Match");
+    return dup;
+}
+
+int
+stonith_device_register(xmlNode *dev, gboolean from_cib)
+{
+    stonith_device_t *dup = NULL;
+    stonith_device_t *device = build_device_from_xml(dev);
+    guint ndevices = 0;
+    int rv = pcmk_ok;
+
+    CRM_CHECK(device != NULL, return -ENOMEM);
+
+    /* do we have a watchdog-device? */
+    if (pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none) ||
+        pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
+                     STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) do {
+        if (stonith_watchdog_timeout_ms <= 0) {
+            crm_err("Ignoring watchdog fence device without "
+                    "stonith-watchdog-timeout set.");
+            rv = -ENODEV;
+            /* fall through to cleanup & return */
+        } else if (!pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
+                                 STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
+            crm_err("Ignoring watchdog fence device with unknown "
+                    "agent '%s' unequal '" STONITH_WATCHDOG_AGENT "'.",
+                    device->agent?device->agent:"");
+            rv = -ENODEV;
+            /* fall through to cleanup & return */
+        } else if (!pcmk__str_eq(device->id, STONITH_WATCHDOG_ID,
+                                 pcmk__str_none)) {
+            crm_err("Ignoring watchdog fence device "
+                    "named %s !='"STONITH_WATCHDOG_ID"'.",
+                    device->id?device->id:"");
+            rv = -ENODEV;
+            /* fall through to cleanup & return */
+        } else {
+            if (pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT,
+                             pcmk__str_none)) {
+                /* this either has an empty list or the targets
+                   configured for watchdog-fencing
+                 */
+                g_list_free_full(stonith_watchdog_targets, free);
+                stonith_watchdog_targets = device->targets;
+                device->targets = NULL;
+            }
+            if (node_does_watchdog_fencing(stonith_our_uname)) {
+                g_list_free_full(device->targets, free);
+                device->targets = stonith__parse_targets(stonith_our_uname);
+                g_hash_table_replace(device->params,
+                                     strdup(PCMK_STONITH_HOST_LIST),
+                                     strdup(stonith_our_uname));
+                /* proceed as with any other stonith-device */
+                break;
+            }
+
+            crm_debug("Skip registration of watchdog fence device on node not in host-list.");
+            /* cleanup and fall through to more cleanup and return */
+            device->targets = NULL;
+            stonith_device_remove(device->id, from_cib);
+        }
+        free_device(device);
+        return rv;
+    } while (0);
+
+    dup = device_has_duplicate(device);
+    if (dup) {
+        ndevices = g_hash_table_size(device_list);
+        crm_debug("Device '%s' already in device list (%d active device%s)",
+                  device->id, ndevices, pcmk__plural_s(ndevices));
+        free_device(device);
+        device = dup;
+        dup = g_hash_table_lookup(device_list, device->id);
+        dup->dirty = FALSE;
+
+    } else {
+        stonith_device_t *old = g_hash_table_lookup(device_list, device->id);
+
+        if (from_cib && old && old->api_registered) {
+            /* If the cib is writing over an entry that is shared with a stonith client,
+             * copy any pending ops that currently exist on the old entry to the new one.
+             * Otherwise the pending ops will be reported as failures
+             */
+            crm_info("Overwriting existing entry for %s from CIB", device->id);
+            device->pending_ops = old->pending_ops;
+            device->api_registered = TRUE;
+            old->pending_ops = NULL;
+            if (device->pending_ops) {
+                mainloop_set_trigger(device->work);
+            }
+        }
+        g_hash_table_replace(device_list, device->id, device);
+
+        ndevices = g_hash_table_size(device_list);
+        crm_notice("Added '%s' to device list (%d active device%s)",
+                   device->id, ndevices, pcmk__plural_s(ndevices));
+    }
+
+    if (from_cib) {
+        device->cib_registered = TRUE;
+    } else {
+        device->api_registered = TRUE;
+    }
+
+    return pcmk_ok;
+}
+
+void
+stonith_device_remove(const char *id, bool from_cib)
+{
+    stonith_device_t *device = g_hash_table_lookup(device_list, id);
+    guint ndevices = 0;
+
+    if (!device) {
+        ndevices = g_hash_table_size(device_list);
+        crm_info("Device '%s' not found (%d active device%s)",
+                 id, ndevices, pcmk__plural_s(ndevices));
+        return;
+    }
+
+    if (from_cib) {
+        device->cib_registered = FALSE;
+    } else {
+        device->verified = FALSE;
+        device->api_registered = FALSE;
+    }
+
+    if (!device->cib_registered && !device->api_registered) {
+        g_hash_table_remove(device_list, id);
+        ndevices = g_hash_table_size(device_list);
+        crm_info("Removed '%s' from device list (%d active device%s)",
+                 id, ndevices, pcmk__plural_s(ndevices));
+    } else {
+        crm_trace("Not removing '%s' from device list (%d active) because "
+                  "still registered via:%s%s",
+                  id, g_hash_table_size(device_list),
+                  (device->cib_registered? " cib" : ""),
+                  (device->api_registered? " api" : ""));
+    }
+}
+
+/*!
+ * \internal
+ * \brief Return the number of stonith levels registered for a node
+ *
+ * \param[in] tp  Node's topology table entry
+ *
+ * \return Number of non-NULL levels in topology entry
+ * \note This function is used only for log messages.
+ */
+static int
+count_active_levels(const stonith_topology_t *tp)
+{
+    int lpc = 0;
+    int count = 0;
+
+    for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
+        if (tp->levels[lpc] != NULL) {
+            count++;
+        }
+    }
+    return count;
+}
+
+static void
+free_topology_entry(gpointer data)
+{
+    stonith_topology_t *tp = data;
+
+    int lpc = 0;
+
+    for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
+        if (tp->levels[lpc] != NULL) {
+            g_list_free_full(tp->levels[lpc], free);
+        }
+    }
+    free(tp->target);
+    free(tp->target_value);
+    free(tp->target_pattern);
+    free(tp->target_attribute);
+    free(tp);
+}
+
+void
+free_topology_list(void)
+{
+    if (topology != NULL) {
+        g_hash_table_destroy(topology);
+        topology = NULL;
+    }
+}
+
+void
+init_topology_list(void)
+{
+    if (topology == NULL) {
+        topology = pcmk__strkey_table(NULL, free_topology_entry);
+    }
+}
+
+char *
+stonith_level_key(const xmlNode *level, enum fenced_target_by mode)
+{
+    if (mode == fenced_target_by_unknown) {
+        mode = unpack_level_kind(level);
+    }
+    switch (mode) {
+        case fenced_target_by_name:
+            return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET);
+
+        case fenced_target_by_pattern:
+            return crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN);
+
+        case fenced_target_by_attribute:
+            return crm_strdup_printf("%s=%s",
+                crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE),
+                crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE));
+
+        default:
+            return crm_strdup_printf("unknown-%s", ID(level));
+    }
+}
+
+/*!
+ * \internal
+ * \brief Parse target identification from topology level XML
+ *
+ * \param[in] level  Topology level XML to parse
+ *
+ * \return How to identify target of \p level
+ */
+static enum fenced_target_by
+unpack_level_kind(const xmlNode *level)
+{
+    if (crm_element_value(level, XML_ATTR_STONITH_TARGET) != NULL) {
+        return fenced_target_by_name;
+    }
+    if (crm_element_value(level, XML_ATTR_STONITH_TARGET_PATTERN) != NULL) {
+        return fenced_target_by_pattern;
+    }
+    if (!stand_alone /* if standalone, there's no attribute manager */
+        && (crm_element_value(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE) != NULL)
+        && (crm_element_value(level, XML_ATTR_STONITH_TARGET_VALUE) != NULL)) {
+        return fenced_target_by_attribute;
+    }
+    return fenced_target_by_unknown;
+}
+
+static stonith_key_value_t *
+parse_device_list(const char *devices)
+{
+    int lpc = 0;
+    int max = 0;
+    int last = 0;
+    stonith_key_value_t *output = NULL;
+
+    if (devices == NULL) {
+        return output;
+    }
+
+    max = strlen(devices);
+    for (lpc = 0; lpc <= max; lpc++) {
+        if (devices[lpc] == ',' || devices[lpc] == 0) {
+            char *line = strndup(devices + last, lpc - last);
+
+            output = stonith_key_value_add(output, NULL, line);
+            free(line);
+
+            last = lpc + 1;
+        }
+    }
+
+    return output;
+}
+
+/*!
+ * \internal
+ * \brief Unpack essential information from topology request XML
+ *
+ * \param[in]  xml     Request XML to search
+ * \param[out] mode    If not NULL, where to store level kind
+ * \param[out] target  If not NULL, where to store representation of target
+ * \param[out] id      If not NULL, where to store level number
+ * \param[out] desc    If not NULL, where to store log-friendly level description
+ *
+ * \return Topology level XML from within \p xml, or NULL if not found
+ * \note The caller is responsible for freeing \p *target and \p *desc if set.
+ */
+static xmlNode *
+unpack_level_request(xmlNode *xml, enum fenced_target_by *mode, char **target,
+                     int *id, char **desc)
+{
+    enum fenced_target_by local_mode = fenced_target_by_unknown;
+    char *local_target = NULL;
+    int local_id = 0;
+
+    /* The level element can be the top element or lower. If top level, don't
+     * search by xpath, because it might give multiple hits if the XML is the
+     * CIB.
+     */
+    if ((xml != NULL)
+        && !pcmk__str_eq(TYPE(xml), XML_TAG_FENCING_LEVEL, pcmk__str_none)) {
+        xml = get_xpath_object("//" XML_TAG_FENCING_LEVEL, xml, LOG_WARNING);
+    }
+
+    if (xml == NULL) {
+        if (desc != NULL) {
+            *desc = crm_strdup_printf("missing");
+        }
+    } else {
+        local_mode = unpack_level_kind(xml);
+        local_target = stonith_level_key(xml, local_mode);
+        crm_element_value_int(xml, XML_ATTR_STONITH_INDEX, &local_id);
+        if (desc != NULL) {
+            *desc = crm_strdup_printf("%s[%d]", local_target, local_id);
+        }
+    }
+
+    if (mode != NULL) {
+        *mode = local_mode;
+    }
+    if (id != NULL) {
+        *id = local_id;
+    }
+
+    if (target != NULL) {
+        *target = local_target;
+    } else {
+        free(local_target);
+    }
+
+    return xml;
+}
+
+/*!
+ * \internal
+ * \brief Register a fencing topology level for a target
+ *
+ * Given an XML request specifying the target name, level index, and device IDs
+ * for the level, this will create an entry for the target in the global topology
+ * table if one does not already exist, then append the specified device IDs to
+ * the entry's device list for the specified level.
+ *
+ * \param[in]  msg     XML request for STONITH level registration
+ * \param[out] desc    If not NULL, set to string representation "TARGET[LEVEL]"
+ * \param[out] result  Where to set result of registration
+ */
+void
+fenced_register_level(xmlNode *msg, char **desc, pcmk__action_result_t *result)
+{
+    int id = 0;
+    xmlNode *level;
+    enum fenced_target_by mode;
+    char *target;
+
+    stonith_topology_t *tp;
+    stonith_key_value_t *dIter = NULL;
+    stonith_key_value_t *devices = NULL;
+
+    CRM_CHECK((msg != NULL) && (result != NULL), return);
+
+    level = unpack_level_request(msg, &mode, &target, &id, desc);
+    if (level == NULL) {
+        fenced_set_protocol_error(result);
+        return;
+    }
+
+    // Ensure an ID was given (even the client API adds an ID)
+    if (pcmk__str_empty(ID(level))) {
+        crm_warn("Ignoring registration for topology level without ID");
+        free(target);
+        crm_log_xml_trace(level, "Bad level");
+        pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
+                            "Topology level is invalid without ID");
+        return;
+    }
+
+    // Ensure a valid target was specified
+    if (mode == fenced_target_by_unknown) {
+        crm_warn("Ignoring registration for topology level '%s' "
+                 "without valid target", ID(level));
+        free(target);
+        crm_log_xml_trace(level, "Bad level");
+        pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
+                            "Invalid target for topology level '%s'",
+                            ID(level));
+        return;
+    }
+
+    // Ensure level ID is in allowed range
+    if ((id <= 0) || (id >= ST_LEVEL_MAX)) {
+        crm_warn("Ignoring topology registration for %s with invalid level %d",
+                  target, id);
+        free(target);
+        crm_log_xml_trace(level, "Bad level");
+        pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
+                            "Invalid level number '%s' for topology level '%s'",
+                            pcmk__s(crm_element_value(level,
+                                                      XML_ATTR_STONITH_INDEX),
+                                    ""),
+                            ID(level));
+        return;
+    }
+
+    /* Find or create topology table entry */
+    tp = g_hash_table_lookup(topology, target);
+    if (tp == NULL) {
+        tp = calloc(1, sizeof(stonith_topology_t));
+        if (tp == NULL) {
+            pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
+                             strerror(ENOMEM));
+            free(target);
+            return;
+        }
+        tp->kind = mode;
+        tp->target = target;
+        tp->target_value = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_VALUE);
+        tp->target_pattern = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_PATTERN);
+        tp->target_attribute = crm_element_value_copy(level, XML_ATTR_STONITH_TARGET_ATTRIBUTE);
+
+        g_hash_table_replace(topology, tp->target, tp);
+        crm_trace("Added %s (%d) to the topology (%d active entries)",
+                  target, (int) mode, g_hash_table_size(topology));
+    } else {
+        free(target);
+    }
+
+    if (tp->levels[id] != NULL) {
+        crm_info("Adding to the existing %s[%d] topology entry",
+                 tp->target, id);
+    }
+
+    devices = parse_device_list(crm_element_value(level, XML_ATTR_STONITH_DEVICES));
+    for (dIter = devices; dIter; dIter = dIter->next) {
+        const char *device = dIter->value;
+
+        crm_trace("Adding device '%s' for %s[%d]", device, tp->target, id);
+        tp->levels[id] = g_list_append(tp->levels[id], strdup(device));
+    }
+    stonith_key_value_freeall(devices, 1, 1);
+
+    {
+        int nlevels = count_active_levels(tp);
+
+        crm_info("Target %s has %d active fencing level%s",
+                 tp->target, nlevels, pcmk__plural_s(nlevels));
+    }
+
+    pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+}
+
+/*!
+ * \internal
+ * \brief Unregister a fencing topology level for a target
+ *
+ * Given an XML request specifying the target name and level index (or 0 for all
+ * levels), this will remove any corresponding entry for the target from the
+ * global topology table.
+ *
+ * \param[in]  msg     XML request for STONITH level registration
+ * \param[out] desc    If not NULL, set to string representation "TARGET[LEVEL]"
+ * \param[out] result  Where to set result of unregistration
+ */
+void
+fenced_unregister_level(xmlNode *msg, char **desc,
+                        pcmk__action_result_t *result)
+{
+    int id = -1;
+    stonith_topology_t *tp;
+    char *target;
+    xmlNode *level = NULL;
+
+    CRM_CHECK(result != NULL, return);
+
+    level = unpack_level_request(msg, NULL, &target, &id, desc);
+    if (level == NULL) {
+        fenced_set_protocol_error(result);
+        return;
+    }
+
+    // Ensure level ID is in allowed range
+    if ((id < 0) || (id >= ST_LEVEL_MAX)) {
+        crm_warn("Ignoring topology unregistration for %s with invalid level %d",
+                  target, id);
+        free(target);
+        crm_log_xml_trace(level, "Bad level");
+        pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
+                            "Invalid level number '%s' for topology level %s",
+                            pcmk__s(crm_element_value(level,
+                                                      XML_ATTR_STONITH_INDEX),
+                                    "<null>"),
+
+                            // Client API doesn't add ID to unregistration XML
+                            pcmk__s(ID(level), ""));
+        return;
+    }
+
+    tp = g_hash_table_lookup(topology, target);
+    if (tp == NULL) {
+        guint nentries = g_hash_table_size(topology);
+
+        crm_info("No fencing topology found for %s (%d active %s)",
+                 target, nentries,
+                 pcmk__plural_alt(nentries, "entry", "entries"));
+
+    } else if (id == 0 && g_hash_table_remove(topology, target)) {
+        guint nentries = g_hash_table_size(topology);
+
+        crm_info("Removed all fencing topology entries related to %s "
+                 "(%d active %s remaining)", target, nentries,
+                 pcmk__plural_alt(nentries, "entry", "entries"));
+
+    } else if (tp->levels[id] != NULL) {
+        guint nlevels;
+
+        g_list_free_full(tp->levels[id], free);
+        tp->levels[id] = NULL;
+
+        nlevels = count_active_levels(tp);
+        crm_info("Removed level %d from fencing topology for %s "
+                 "(%d active level%s remaining)",
+                 id, target, nlevels, pcmk__plural_s(nlevels));
+    }
+
+    free(target);
+    pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+}
+
+static char *
+list_to_string(GList *list, const char *delim, gboolean terminate_with_delim)
+{
+    int max = g_list_length(list);
+    size_t delim_len = delim?strlen(delim):0;
+    size_t alloc_size = 1 + (max?((max-1+(terminate_with_delim?1:0))*delim_len):0);
+    char *rv;
+    GList *gIter;
+
+    for (gIter = list; gIter != NULL; gIter = gIter->next) {
+        const char *value = (const char *) gIter->data;
+
+        alloc_size += strlen(value);
+    }
+    rv = calloc(alloc_size, sizeof(char));
+    if (rv) {
+        char *pos = rv;
+        const char *lead_delim = "";
+
+        for (gIter = list; gIter != NULL; gIter = gIter->next) {
+            const char *value = (const char *) gIter->data;
+
+            pos = &pos[sprintf(pos, "%s%s", lead_delim, value)];
+            lead_delim = delim;
+        }
+        if (max && terminate_with_delim) {
+            sprintf(pos, "%s", delim);
+        }
+    }
+    return rv;
+}
+
+/*!
+ * \internal
+ * \brief Execute a fence agent action directly (and asynchronously)
+ *
+ * Handle a STONITH_OP_EXEC API message by scheduling a requested agent action
+ * directly on a specified device. Only list, monitor, and status actions are
+ * expected to use this call, though it should work with any agent command.
+ *
+ * \param[in]  msg     Request XML specifying action
+ * \param[out] result  Where to store result of action
+ *
+ * \note If the action is monitor, the device must be registered via the API
+ *       (CIB registration is not sufficient), because monitor should not be
+ *       possible unless the device is "started" (API registered).
+ */
+static void
+execute_agent_action(xmlNode *msg, pcmk__action_result_t *result)
+{
+    xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, msg, LOG_ERR);
+    xmlNode *op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
+    const char *id = crm_element_value(dev, F_STONITH_DEVICE);
+    const char *action = crm_element_value(op, F_STONITH_ACTION);
+    async_command_t *cmd = NULL;
+    stonith_device_t *device = NULL;
+
+    if ((id == NULL) || (action == NULL)) {
+        crm_info("Malformed API action request: device %s, action %s",
+                 (id? id : "not specified"),
+                 (action? action : "not specified"));
+        fenced_set_protocol_error(result);
+        return;
+    }
+
+    if (pcmk__str_eq(id, STONITH_WATCHDOG_ID, pcmk__str_none)) {
+        // Watchdog agent actions are implemented internally
+        if (stonith_watchdog_timeout_ms <= 0) {
+            pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
+                             "Watchdog fence device not configured");
+            return;
+
+        } else if (pcmk__str_eq(action, "list", pcmk__str_none)) {
+            pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+            pcmk__set_result_output(result,
+                                    list_to_string(stonith_watchdog_targets,
+                                                   "\n", TRUE),
+                                    NULL);
+            return;
+
+        } else if (pcmk__str_eq(action, "monitor", pcmk__str_none)) {
+            pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+            return;
+        }
+    }
+
+    device = g_hash_table_lookup(device_list, id);
+    if (device == NULL) {
+        crm_info("Ignoring API '%s' action request because device %s not found",
+                 action, id);
+        pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
+                            "'%s' not found", id);
+        return;
+
+    } else if (!device->api_registered && !strcmp(action, "monitor")) {
+        // Monitors may run only on "started" (API-registered) devices
+        crm_info("Ignoring API '%s' action request because device %s not active",
+                 action, id);
+        pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
+                            "'%s' not active", id);
+        return;
+    }
+
+    cmd = create_async_command(msg);
+    if (cmd == NULL) {
+        crm_log_xml_warn(msg, "invalid");
+        fenced_set_protocol_error(result);
+        return;
+    }
+
+    schedule_stonith_command(cmd, device);
+    pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
+}
+
+static void
+search_devices_record_result(struct device_search_s *search, const char *device, gboolean can_fence)
+{
+    search->replies_received++;
+    if (can_fence && device) {
+        if (search->support_action_only != st_device_supports_none) {
+            stonith_device_t *dev = g_hash_table_lookup(device_list, device);
+            if (dev && !pcmk_is_set(dev->flags, search->support_action_only)) {
+                return;
+            }
+        }
+        search->capable = g_list_append(search->capable, strdup(device));
+    }
+
+    if (search->replies_needed == search->replies_received) {
+
+        guint ndevices = g_list_length(search->capable);
+
+        crm_debug("Search found %d device%s that can perform '%s' targeting %s",
+                  ndevices, pcmk__plural_s(ndevices),
+                  (search->action? search->action : "unknown action"),
+                  (search->host? search->host : "any node"));
+
+        search->callback(search->capable, search->user_data);
+        free(search->host);
+        free(search->action);
+        free(search);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Check whether the local host is allowed to execute a fencing action
+ *
+ * \param[in] device         Fence device to check
+ * \param[in] action         Fence action to check
+ * \param[in] target         Hostname of fence target
+ * \param[in] allow_suicide  Whether self-fencing is allowed for this operation
+ *
+ * \return TRUE if local host is allowed to execute action, FALSE otherwise
+ */
+static gboolean
+localhost_is_eligible(const stonith_device_t *device, const char *action,
+                      const char *target, gboolean allow_suicide)
+{
+    gboolean localhost_is_target = pcmk__str_eq(target, stonith_our_uname,
+                                                pcmk__str_casei);
+
+    if ((device != NULL) && (action != NULL)
+        && (device->on_target_actions != NULL)
+        && (strstr((const char*) device->on_target_actions->str,
+                   action) != NULL)) {
+
+        if (!localhost_is_target) {
+            crm_trace("Operation '%s' using %s can only be executed for local "
+                      "host, not %s", action, device->id, target);
+            return FALSE;
+        }
+
+    } else if (localhost_is_target && !allow_suicide) {
+        crm_trace("'%s' operation does not support self-fencing", action);
+        return FALSE;
+    }
+    return TRUE;
+}
+
+/*!
+ * \internal
+ * \brief Check if local node is allowed to execute (possibly remapped) action
+ *
+ * \param[in] device      Fence device to check
+ * \param[in] action      Fence action to check
+ * \param[in] target      Node name of fence target
+ * \param[in] allow_self  Whether self-fencing is allowed for this operation
+ *
+ * \return true if local node is allowed to execute \p action or any actions it
+ *         might be remapped to, otherwise false
+ */
+static bool
+localhost_is_eligible_with_remap(const stonith_device_t *device,
+                                 const char *action, const char *target,
+                                 gboolean allow_self)
+{
+    // Check exact action
+    if (localhost_is_eligible(device, action, target, allow_self)) {
+        return true;
+    }
+
+    // Check potential remaps
+
+    if (pcmk__str_eq(action, "reboot", pcmk__str_none)) {
+        /* "reboot" might get remapped to "off" then "on", so even if reboot is
+         * disallowed, return true if either of those is allowed. We'll report
+         * the disallowed actions with the results. We never allow self-fencing
+         * for remapped "on" actions because the target is off at that point.
+         */
+        if (localhost_is_eligible(device, "off", target, allow_self)
+            || localhost_is_eligible(device, "on", target, FALSE)) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static void
+can_fence_host_with_device(stonith_device_t *dev,
+                           struct device_search_s *search)
+{
+    gboolean can = FALSE;
+    const char *check_type = "Internal bug";
+    const char *target = NULL;
+    const char *alias = NULL;
+    const char *dev_id = "Unspecified device";
+    const char *action = (search == NULL)? NULL : search->action;
+
+    CRM_CHECK((dev != NULL) && (action != NULL), goto search_report_results);
+
+    if (dev->id != NULL) {
+        dev_id = dev->id;
+    }
+
+    target = search->host;
+    if (target == NULL) {
+        can = TRUE;
+        check_type = "No target";
+        goto search_report_results;
+    }
+
+    /* Answer immediately if the device does not support the action
+     * or the local node is not allowed to perform it
+     */
+    if (pcmk__str_eq(action, "on", pcmk__str_none)
+        && !pcmk_is_set(dev->flags, st_device_supports_on)) {
+        check_type = "Agent does not support 'on'";
+        goto search_report_results;
+
+    } else if (!localhost_is_eligible_with_remap(dev, action, target,
+                                                 search->allow_suicide)) {
+        check_type = "This node is not allowed to execute action";
+        goto search_report_results;
+    }
+
+    // Check eligibility as specified by pcmk_host_check
+    check_type = target_list_type(dev);
+    alias = g_hash_table_lookup(dev->aliases, target);
+    if (pcmk__str_eq(check_type, PCMK__VALUE_NONE, pcmk__str_casei)) {
+        can = TRUE;
+
+    } else if (pcmk__str_eq(check_type, "static-list", pcmk__str_casei)) {
+        if (pcmk__str_in_list(target, dev->targets, pcmk__str_casei)) {
+            can = TRUE;
+        } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)
+                   && g_hash_table_lookup(dev->aliases, target)) {
+            can = TRUE;
+        }
+
+    } else if (pcmk__str_eq(check_type, "dynamic-list", pcmk__str_casei)) {
+        time_t now = time(NULL);
+
+        if (dev->targets == NULL || dev->targets_age + 60 < now) {
+            int device_timeout = get_action_timeout(dev, "list", search->per_device_timeout);
+
+            if (device_timeout > search->per_device_timeout) {
+                crm_notice("Since the pcmk_list_timeout(%ds) parameter of %s is larger than stonith-timeout(%ds), timeout may occur",
+                    device_timeout, dev_id, search->per_device_timeout);
+            }
+
+            crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
+                      check_type, dev_id, target, action);
+
+            schedule_internal_command(__func__, dev, "list", NULL,
+                                      search->per_device_timeout, search, dynamic_list_search_cb);
+
+            /* we'll respond to this search request async in the cb */
+            return;
+        }
+
+        if (pcmk__str_in_list(((alias == NULL)? target : alias), dev->targets,
+                              pcmk__str_casei)) {
+            can = TRUE;
+        }
+
+    } else if (pcmk__str_eq(check_type, "status", pcmk__str_casei)) {
+        int device_timeout = get_action_timeout(dev, check_type, search->per_device_timeout);
+
+        if (device_timeout > search->per_device_timeout) {
+            crm_notice("Since the pcmk_status_timeout(%ds) parameter of %s is larger than stonith-timeout(%ds), timeout may occur",
+                device_timeout, dev_id, search->per_device_timeout);
+        }
+
+        crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
+                  check_type, dev_id, target, action);
+        schedule_internal_command(__func__, dev, "status", target,
+                                  search->per_device_timeout, search, status_search_cb);
+        /* we'll respond to this search request async in the cb */
+        return;
+    } else {
+        crm_err("Invalid value for " PCMK_STONITH_HOST_CHECK ": %s", check_type);
+        check_type = "Invalid " PCMK_STONITH_HOST_CHECK;
+    }
+
+  search_report_results:
+    crm_info("%s is%s eligible to fence (%s) %s%s%s%s: %s",
+             dev_id, (can? "" : " not"), pcmk__s(action, "unspecified action"),
+             pcmk__s(target, "unspecified target"),
+             (alias == NULL)? "" : " (as '", pcmk__s(alias, ""),
+             (alias == NULL)? "" : "')", check_type);
+    search_devices_record_result(search, ((dev == NULL)? NULL : dev_id), can);
+}
+
+static void
+search_devices(gpointer key, gpointer value, gpointer user_data)
+{
+    stonith_device_t *dev = value;
+    struct device_search_s *search = user_data;
+
+    can_fence_host_with_device(dev, search);
+}
+
+#define DEFAULT_QUERY_TIMEOUT 20
+static void
+get_capable_devices(const char *host, const char *action, int timeout, bool suicide, void *user_data,
+                    void (*callback) (GList * devices, void *user_data), uint32_t support_action_only)
+{
+    struct device_search_s *search;
+    guint ndevices = g_hash_table_size(device_list);
+
+    if (ndevices == 0) {
+        callback(NULL, user_data);
+        return;
+    }
+
+    search = calloc(1, sizeof(struct device_search_s));
+    if (!search) {
+        crm_crit("Cannot search for capable fence devices: %s",
+                 strerror(ENOMEM));
+        callback(NULL, user_data);
+        return;
+    }
+
+    pcmk__str_update(&search->host, host);
+    pcmk__str_update(&search->action, action);
+    search->per_device_timeout = timeout;
+    search->allow_suicide = suicide;
+    search->callback = callback;
+    search->user_data = user_data;
+    search->support_action_only = support_action_only;
+
+    /* We are guaranteed this many replies, even if a device is
+     * unregistered while the search is in progress.
+     */
+    search->replies_needed = ndevices;
+
+    crm_debug("Searching %d device%s to see which can execute '%s' targeting %s",
+              ndevices, pcmk__plural_s(ndevices),
+              (search->action? search->action : "unknown action"),
+              (search->host? search->host : "any node"));
+    g_hash_table_foreach(device_list, search_devices, search);
+}
+
+struct st_query_data {
+    xmlNode *reply;
+    char *remote_peer;
+    char *client_id;
+    char *target;
+    char *action;
+    int call_options;
+};
+
+/*!
+ * \internal
+ * \brief Add action-specific attributes to query reply XML
+ *
+ * \param[in,out] xml     XML to add attributes to
+ * \param[in]     action  Fence action
+ * \param[in]     device  Fence device
+ * \param[in]     target  Fence target
+ */
+static void
+add_action_specific_attributes(xmlNode *xml, const char *action,
+                               const stonith_device_t *device,
+                               const char *target)
+{
+    int action_specific_timeout;
+    int delay_max;
+    int delay_base;
+
+    CRM_CHECK(xml && action && device, return);
+
+    if (is_action_required(action, device)) {
+        crm_trace("Action '%s' is required using %s", action, device->id);
+        crm_xml_add_int(xml, F_STONITH_DEVICE_REQUIRED, 1);
+    }
+
+    action_specific_timeout = get_action_timeout(device, action, 0);
+    if (action_specific_timeout) {
+        crm_trace("Action '%s' has timeout %dms using %s",
+                  action, action_specific_timeout, device->id);
+        crm_xml_add_int(xml, F_STONITH_ACTION_TIMEOUT, action_specific_timeout);
+    }
+
+    delay_max = get_action_delay_max(device, action);
+    if (delay_max > 0) {
+        crm_trace("Action '%s' has maximum random delay %ds using %s",
+                  action, delay_max, device->id);
+        crm_xml_add_int(xml, F_STONITH_DELAY_MAX, delay_max);
+    }
+
+    delay_base = get_action_delay_base(device, action, target);
+    if (delay_base > 0) {
+        crm_xml_add_int(xml, F_STONITH_DELAY_BASE, delay_base);
+    }
+
+    if ((delay_max > 0) && (delay_base == 0)) {
+        crm_trace("Action '%s' has maximum random delay %ds using %s",
+                  action, delay_max, device->id);
+    } else if ((delay_max == 0) && (delay_base > 0)) {
+        crm_trace("Action '%s' has a static delay of %ds using %s",
+                  action, delay_base, device->id);
+    } else if ((delay_max > 0) && (delay_base > 0)) {
+        crm_trace("Action '%s' has a minimum delay of %ds and a randomly chosen "
+                  "maximum delay of %ds using %s",
+                  action, delay_base, delay_max, device->id);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Add "disallowed" attribute to query reply XML if appropriate
+ *
+ * \param[in,out] xml            XML to add attribute to
+ * \param[in]     action         Fence action
+ * \param[in]     device         Fence device
+ * \param[in]     target         Fence target
+ * \param[in]     allow_suicide  Whether self-fencing is allowed
+ */
+static void
+add_disallowed(xmlNode *xml, const char *action, const stonith_device_t *device,
+               const char *target, gboolean allow_suicide)
+{
+    if (!localhost_is_eligible(device, action, target, allow_suicide)) {
+        crm_trace("Action '%s' using %s is disallowed for local host",
+                  action, device->id);
+        pcmk__xe_set_bool_attr(xml, F_STONITH_ACTION_DISALLOWED, true);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Add child element with action-specific values to query reply XML
+ *
+ * \param[in,out] xml            XML to add attribute to
+ * \param[in]     action         Fence action
+ * \param[in]     device         Fence device
+ * \param[in]     target         Fence target
+ * \param[in]     allow_suicide  Whether self-fencing is allowed
+ */
+static void
+add_action_reply(xmlNode *xml, const char *action,
+                 const stonith_device_t *device, const char *target,
+                 gboolean allow_suicide)
+{
+    xmlNode *child = create_xml_node(xml, F_STONITH_ACTION);
+
+    crm_xml_add(child, XML_ATTR_ID, action);
+    add_action_specific_attributes(child, action, device, target);
+    add_disallowed(child, action, device, target, allow_suicide);
+}
+
+static void
+stonith_query_capable_device_cb(GList * devices, void *user_data)
+{
+    struct st_query_data *query = user_data;
+    int available_devices = 0;
+    xmlNode *dev = NULL;
+    xmlNode *list = NULL;
+    GList *lpc = NULL;
+    pcmk__client_t *client = NULL;
+
+    if (query->client_id != NULL) {
+        client = pcmk__find_client_by_id(query->client_id);
+        if ((client == NULL) && (query->remote_peer == NULL)) {
+            crm_trace("Skipping reply to %s: no longer a client",
+                      query->client_id);
+            goto done;
+        }
+    }
+
+    /* Pack the results into XML */
+    list = create_xml_node(NULL, __func__);
+    crm_xml_add(list, F_STONITH_TARGET, query->target);
+    for (lpc = devices; lpc != NULL; lpc = lpc->next) {
+        stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data);
+        const char *action = query->action;
+
+        if (!device) {
+            /* It is possible the device got unregistered while
+             * determining who can fence the target */
+            continue;
+        }
+
+        available_devices++;
+
+        dev = create_xml_node(list, F_STONITH_DEVICE);
+        crm_xml_add(dev, XML_ATTR_ID, device->id);
+        crm_xml_add(dev, "namespace", device->namespace);
+        crm_xml_add(dev, "agent", device->agent);
+        crm_xml_add_int(dev, F_STONITH_DEVICE_VERIFIED, device->verified);
+        crm_xml_add_int(dev, F_STONITH_DEVICE_SUPPORT_FLAGS, device->flags);
+
+        /* If the originating fencer wants to reboot the node, and we have a
+         * capable device that doesn't support "reboot", remap to "off" instead.
+         */
+        if (!pcmk_is_set(device->flags, st_device_supports_reboot)
+            && pcmk__str_eq(query->action, "reboot", pcmk__str_none)) {
+            crm_trace("%s doesn't support reboot, using values for off instead",
+                      device->id);
+            action = "off";
+        }
+
+        /* Add action-specific values if available */
+        add_action_specific_attributes(dev, action, device, query->target);
+        if (pcmk__str_eq(query->action, "reboot", pcmk__str_none)) {
+            /* A "reboot" *might* get remapped to "off" then "on", so after
+             * sending the "reboot"-specific values in the main element, we add
+             * sub-elements for "off" and "on" values.
+             *
+             * We short-circuited earlier if "reboot", "off" and "on" are all
+             * disallowed for the local host. However if only one or two are
+             * disallowed, we send back the results and mark which ones are
+             * disallowed. If "reboot" is disallowed, this might cause problems
+             * with older fencer versions, which won't check for it. Older
+             * versions will ignore "off" and "on", so they are not a problem.
+             */
+            add_disallowed(dev, action, device, query->target,
+                           pcmk_is_set(query->call_options, st_opt_allow_suicide));
+            add_action_reply(dev, "off", device, query->target,
+                             pcmk_is_set(query->call_options, st_opt_allow_suicide));
+            add_action_reply(dev, "on", device, query->target, FALSE);
+        }
+
+        /* A query without a target wants device parameters */
+        if (query->target == NULL) {
+            xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS);
+
+            g_hash_table_foreach(device->params, hash2field, attrs);
+        }
+    }
+
+    crm_xml_add_int(list, F_STONITH_AVAILABLE_DEVICES, available_devices);
+    if (query->target) {
+        crm_debug("Found %d matching device%s for target '%s'",
+                  available_devices, pcmk__plural_s(available_devices),
+                  query->target);
+    } else {
+        crm_debug("%d device%s installed",
+                  available_devices, pcmk__plural_s(available_devices));
+    }
+
+    if (list != NULL) {
+        crm_log_xml_trace(list, "Add query results");
+        add_message_xml(query->reply, F_STONITH_CALLDATA, list);
+    }
+
+    stonith_send_reply(query->reply, query->call_options, query->remote_peer,
+                       client);
+
+done:
+    free_xml(query->reply);
+    free(query->remote_peer);
+    free(query->client_id);
+    free(query->target);
+    free(query->action);
+    free(query);
+    free_xml(list);
+    g_list_free_full(devices, free);
+}
+
+/*!
+ * \internal
+ * \brief Log the result of an asynchronous command
+ *
+ * \param[in] cmd        Command the result is for
+ * \param[in] result     Result of command
+ * \param[in] pid        Process ID of command, if available
+ * \param[in] next       Alternate device that will be tried if command failed
+ * \param[in] op_merged  Whether this command was merged with an earlier one
+ */
+static void
+log_async_result(const async_command_t *cmd,
+                 const pcmk__action_result_t *result,
+                 int pid, const char *next, bool op_merged)
+{
+    int log_level = LOG_ERR;
+    int output_log_level = LOG_NEVER;
+    guint devices_remaining = g_list_length(cmd->next_device_iter);
+
+    GString *msg = g_string_sized_new(80); // Reasonable starting size
+
+    // Choose log levels appropriately if we have a result
+    if (pcmk__result_ok(result)) {
+        log_level = (cmd->target == NULL)? LOG_DEBUG : LOG_NOTICE;
+        if ((result->action_stdout != NULL)
+            && !pcmk__str_eq(cmd->action, "metadata", pcmk__str_none)) {
+            output_log_level = LOG_DEBUG;
+        }
+        next = NULL;
+    } else {
+        log_level = (cmd->target == NULL)? LOG_NOTICE : LOG_ERR;
+        if ((result->action_stdout != NULL)
+            && !pcmk__str_eq(cmd->action, "metadata", pcmk__str_none)) {
+            output_log_level = LOG_WARNING;
+        }
+    }
+
+    // Build the log message piece by piece
+    pcmk__g_strcat(msg, "Operation '", cmd->action, "' ", NULL);
+    if (pid != 0) {
+        g_string_append_printf(msg, "[%d] ", pid);
+    }
+    if (cmd->target != NULL) {
+        pcmk__g_strcat(msg, "targeting ", cmd->target, " ", NULL);
+    }
+    if (cmd->device != NULL) {
+        pcmk__g_strcat(msg, "using ", cmd->device, " ", NULL);
+    }
+
+    // Add exit status or execution status as appropriate
+    if (result->execution_status == PCMK_EXEC_DONE) {
+        g_string_append_printf(msg, "returned %d", result->exit_status);
+    } else {
+        pcmk__g_strcat(msg, "could not be executed: ",
+                       pcmk_exec_status_str(result->execution_status), NULL);
+    }
+
+    // Add exit reason and next device if appropriate
+    if (result->exit_reason != NULL) {
+        pcmk__g_strcat(msg, " (", result->exit_reason, ")", NULL);
+    }
+    if (next != NULL) {
+        pcmk__g_strcat(msg, ", retrying with ", next, NULL);
+    }
+    if (devices_remaining > 0) {
+        g_string_append_printf(msg, " (%u device%s remaining)",
+                               (unsigned int) devices_remaining,
+                               pcmk__plural_s(devices_remaining));
+    }
+    g_string_append_printf(msg, " " CRM_XS " %scall %d from %s",
+                           (op_merged? "merged " : ""), cmd->id,
+                           cmd->client_name);
+
+    // Log the result
+    do_crm_log(log_level, "%s", msg->str);
+    g_string_free(msg, TRUE);
+
+    // Log the output (which may have multiple lines), if appropriate
+    if (output_log_level != LOG_NEVER) {
+        char *prefix = crm_strdup_printf("%s[%d]", cmd->device, pid);
+
+        crm_log_output(output_log_level, prefix, result->action_stdout);
+        free(prefix);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Reply to requester after asynchronous command completion
+ *
+ * \param[in] cmd      Command that completed
+ * \param[in] result   Result of command
+ * \param[in] pid      Process ID of command, if available
+ * \param[in] merged   If true, command was merged with another, not executed
+ */
+static void
+send_async_reply(const async_command_t *cmd, const pcmk__action_result_t *result,
+                 int pid, bool merged)
+{
+    xmlNode *reply = NULL;
+    pcmk__client_t *client = NULL;
+
+    CRM_CHECK((cmd != NULL) && (result != NULL), return);
+
+    log_async_result(cmd, result, pid, NULL, merged);
+
+    if (cmd->client != NULL) {
+        client = pcmk__find_client_by_id(cmd->client);
+        if ((client == NULL) && (cmd->origin == NULL)) {
+            crm_trace("Skipping reply to %s: no longer a client", cmd->client);
+            return;
+        }
+    }
+
+    reply = construct_async_reply(cmd, result);
+    if (merged) {
+        pcmk__xe_set_bool_attr(reply, F_STONITH_MERGED, true);
+    }
+
+    if (!stand_alone && pcmk__is_fencing_action(cmd->action)
+        && pcmk__str_eq(cmd->origin, cmd->target, pcmk__str_casei)) {
+        /* The target was also the originator, so broadcast the result on its
+         * behalf (since it will be unable to).
+         */
+        crm_trace("Broadcast '%s' result for %s (target was also originator)",
+                  cmd->action, cmd->target);
+        crm_xml_add(reply, F_SUBTYPE, "broadcast");
+        crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY);
+        send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE);
+    } else {
+        // Reply only to the originator
+        stonith_send_reply(reply, cmd->options, cmd->origin, client);
+    }
+
+    crm_log_xml_trace(reply, "Reply");
+    free_xml(reply);
+
+    if (stand_alone) {
+        /* Do notification with a clean data object */
+        xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);
+
+        stonith__xe_set_result(notify_data, result);
+        crm_xml_add(notify_data, F_STONITH_TARGET, cmd->target);
+        crm_xml_add(notify_data, F_STONITH_OPERATION, cmd->op);
+        crm_xml_add(notify_data, F_STONITH_DELEGATE, "localhost");
+        crm_xml_add(notify_data, F_STONITH_DEVICE, cmd->device);
+        crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
+        crm_xml_add(notify_data, F_STONITH_ORIGIN, cmd->client);
+
+        fenced_send_notification(T_STONITH_NOTIFY_FENCE, result, notify_data);
+        fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
+    }
+}
+
+static void
+cancel_stonith_command(async_command_t * cmd)
+{
+    stonith_device_t *device = cmd_device(cmd);
+
+    if (device) {
+        crm_trace("Cancel scheduled '%s' action using %s",
+                  cmd->action, device->id);
+        device->pending_ops = g_list_remove(device->pending_ops, cmd);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Cancel and reply to any duplicates of a just-completed operation
+ *
+ * Check whether any fencing operations are scheduled to do the same thing as
+ * one that just succeeded. If so, rather than performing the same operation
+ * twice, return the result of this operation for all matching pending commands.
+ *
+ * \param[in,out] cmd     Fencing operation that just succeeded
+ * \param[in]     result  Result of \p cmd
+ * \param[in]     pid     If nonzero, process ID of agent invocation (for logs)
+ *
+ * \note Duplicate merging will do the right thing for either type of remapped
+ *       reboot. If the executing fencer remapped an unsupported reboot to off,
+ *       then cmd->action will be "reboot" and will be merged with any other
+ *       reboot requests. If the originating fencer remapped a topology reboot
+ *       to off then on, we will get here once with cmd->action "off" and once
+ *       with "on", and they will be merged separately with similar requests.
+ */
+static void
+reply_to_duplicates(async_command_t *cmd, const pcmk__action_result_t *result,
+                    int pid)
+{
+    GList *next = NULL;
+
+    for (GList *iter = cmd_list; iter != NULL; iter = next) {
+        async_command_t *cmd_other = iter->data;
+
+        next = iter->next; // We might delete this entry, so grab next now
+
+        if (cmd == cmd_other) {
+            continue;
+        }
+
+        /* A pending operation matches if:
+         * 1. The client connections are different.
+         * 2. The target is the same.
+         * 3. The fencing action is the same.
+         * 4. The device scheduled to execute the action is the same.
+         */
+        if (pcmk__str_eq(cmd->client, cmd_other->client, pcmk__str_casei) ||
+            !pcmk__str_eq(cmd->target, cmd_other->target, pcmk__str_casei) ||
+            !pcmk__str_eq(cmd->action, cmd_other->action, pcmk__str_none) ||
+            !pcmk__str_eq(cmd->device, cmd_other->device, pcmk__str_casei)) {
+
+            continue;
+        }
+
+        crm_notice("Merging fencing action '%s'%s%s originating from "
+                   "client %s with identical fencing request from client %s",
+                   cmd_other->action,
+                   (cmd_other->target == NULL)? "" : " targeting ",
+                   pcmk__s(cmd_other->target, ""), cmd_other->client_name,
+                   cmd->client_name);
+
+        // Stop tracking the duplicate, send its result, and cancel it
+        cmd_list = g_list_remove_link(cmd_list, iter);
+        send_async_reply(cmd_other, result, pid, true);
+        cancel_stonith_command(cmd_other);
+
+        free_async_command(cmd_other);
+        g_list_free_1(iter);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Return the next required device (if any) for an operation
+ *
+ * \param[in,out] cmd  Fencing operation that just succeeded
+ *
+ * \return Next device required for action if any, otherwise NULL
+ */
+static stonith_device_t *
+next_required_device(async_command_t *cmd)
+{
+    for (GList *iter = cmd->next_device_iter; iter != NULL; iter = iter->next) {
+        stonith_device_t *next_device = g_hash_table_lookup(device_list,
+                                                            iter->data);
+
+        if (is_action_required(cmd->action, next_device)) {
+            /* This is only called for successful actions, so it's OK to skip
+             * non-required devices.
+             */
+            cmd->next_device_iter = iter->next;
+            return next_device;
+        }
+    }
+    return NULL;
+}
+
+static void
+st_child_done(int pid, const pcmk__action_result_t *result, void *user_data)
+{
+    async_command_t *cmd = user_data;
+
+    stonith_device_t *device = NULL;
+    stonith_device_t *next_device = NULL;
+
+    CRM_CHECK(cmd != NULL, return);
+
+    device = cmd_device(cmd);
+    cmd->active_on = NULL;
+
+    /* The device is ready to do something else now */
+    if (device) {
+        if (!device->verified && pcmk__result_ok(result) &&
+            (pcmk__strcase_any_of(cmd->action, "list", "monitor", "status", NULL))) {
+
+            device->verified = TRUE;
+        }
+
+        mainloop_set_trigger(device->work);
+    }
+
+    if (pcmk__result_ok(result)) {
+        next_device = next_required_device(cmd);
+
+    } else if ((cmd->next_device_iter != NULL)
+               && !is_action_required(cmd->action, device)) {
+        /* if this device didn't work out, see if there are any others we can try.
+         * if the failed device was 'required', we can't pick another device. */
+        next_device = g_hash_table_lookup(device_list,
+                                          cmd->next_device_iter->data);
+        cmd->next_device_iter = cmd->next_device_iter->next;
+    }
+
+    if (next_device == NULL) {
+        send_async_reply(cmd, result, pid, false);
+        if (pcmk__result_ok(result)) {
+            reply_to_duplicates(cmd, result, pid);
+        }
+        free_async_command(cmd);
+
+    } else { // This operation requires more fencing
+        log_async_result(cmd, result, pid, next_device->id, false);
+        schedule_stonith_command(cmd, next_device);
+    }
+}
+
+static gint
+sort_device_priority(gconstpointer a, gconstpointer b)
+{
+    const stonith_device_t *dev_a = a;
+    const stonith_device_t *dev_b = b;
+
+    if (dev_a->priority > dev_b->priority) {
+        return -1;
+    } else if (dev_a->priority < dev_b->priority) {
+        return 1;
+    }
+    return 0;
+}
+
+static void
+stonith_fence_get_devices_cb(GList * devices, void *user_data)
+{
+    async_command_t *cmd = user_data;
+    stonith_device_t *device = NULL;
+    guint ndevices = g_list_length(devices);
+
+    crm_info("Found %d matching device%s for target '%s'",
+             ndevices, pcmk__plural_s(ndevices), cmd->target);
+
+    if (devices != NULL) {
+        /* Order based on priority */
+        devices = g_list_sort(devices, sort_device_priority);
+        device = g_hash_table_lookup(device_list, devices->data);
+    }
+
+    if (device == NULL) { // No device found
+        pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
+
+        pcmk__format_result(&result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
+                            "No device configured for target '%s'",
+                            cmd->target);
+        send_async_reply(cmd, &result, 0, false);
+        pcmk__reset_result(&result);
+        free_async_command(cmd);
+        g_list_free_full(devices, free);
+
+    } else { // Device found, schedule it for fencing
+        cmd->device_list = devices;
+        cmd->next_device_iter = devices->next;
+        schedule_stonith_command(cmd, device);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Execute a fence action via the local node
+ *
+ * \param[in]  msg     Fencing request
+ * \param[out] result  Where to store result of fence action
+ */
+static void
+fence_locally(xmlNode *msg, pcmk__action_result_t *result)
+{
+    const char *device_id = NULL;
+    stonith_device_t *device = NULL;
+    async_command_t *cmd = NULL;
+    xmlNode *dev = NULL;
+
+    CRM_CHECK((msg != NULL) && (result != NULL), return);
+
+    dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR);
+
+    cmd = create_async_command(msg);
+    if (cmd == NULL) {
+        crm_log_xml_warn(msg, "invalid");
+        fenced_set_protocol_error(result);
+        return;
+    }
+
+    device_id = crm_element_value(dev, F_STONITH_DEVICE);
+    if (device_id != NULL) {
+        device = g_hash_table_lookup(device_list, device_id);
+        if (device == NULL) {
+            crm_err("Requested device '%s' is not available", device_id);
+            pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
+                                "Requested device '%s' not found", device_id);
+            return;
+        }
+        schedule_stonith_command(cmd, device);
+
+    } else {
+        const char *host = crm_element_value(dev, F_STONITH_TARGET);
+
+        if (pcmk_is_set(cmd->options, st_opt_cs_nodeid)) {
+            int nodeid = 0;
+            crm_node_t *node = NULL;
+
+            pcmk__scan_min_int(host, &nodeid, 0);
+            node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY);
+            if (node != NULL) {
+                host = node->uname;
+            }
+        }
+
+        /* If we get to here, then self-fencing is implicitly allowed */
+        get_capable_devices(host, cmd->action, cmd->default_timeout,
+                            TRUE, cmd, stonith_fence_get_devices_cb,
+                            fenced_support_flag(cmd->action));
+    }
+
+    pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
+}
+
+/*!
+ * \internal
+ * \brief Build an XML reply for a fencing operation
+ *
+ * \param[in] request  Request that reply is for
+ * \param[in] data     If not NULL, add to reply as call data
+ * \param[in] result   Full result of fencing operation
+ *
+ * \return Newly created XML reply
+ * \note The caller is responsible for freeing the result.
+ * \note This has some overlap with construct_async_reply(), but that copies
+ *       values from an async_command_t, whereas this one copies them from the
+ *       request.
+ */
+xmlNode *
+fenced_construct_reply(const xmlNode *request, xmlNode *data,
+                       const pcmk__action_result_t *result)
+{
+    xmlNode *reply = NULL;
+
+    reply = create_xml_node(NULL, T_STONITH_REPLY);
+
+    crm_xml_add(reply, "st_origin", __func__);
+    crm_xml_add(reply, F_TYPE, T_STONITH_NG);
+    stonith__xe_set_result(reply, result);
+
+    if (request == NULL) {
+        /* Most likely, this is the result of a stonith operation that was
+         * initiated before we came up. Unfortunately that means we lack enough
+         * information to provide clients with a full result.
+         *
+         * @TODO Maybe synchronize this information at start-up?
+         */
+        crm_warn("Missing request information for client notifications for "
+                 "operation with result '%s' (initiated before we came up?)",
+                 pcmk_exec_status_str(result->execution_status));
+
+    } else {
+        const char *name = NULL;
+        const char *value = NULL;
+
+        // Attributes to copy from request to reply
+        const char *names[] = {
+            F_STONITH_OPERATION,
+            F_STONITH_CALLID,
+            F_STONITH_CLIENTID,
+            F_STONITH_CLIENTNAME,
+            F_STONITH_REMOTE_OP_ID,
+            F_STONITH_CALLOPTS
+        };
+
+        for (int lpc = 0; lpc < PCMK__NELEM(names); lpc++) {
+            name = names[lpc];
+            value = crm_element_value(request, name);
+            crm_xml_add(reply, name, value);
+        }
+        if (data != NULL) {
+            add_message_xml(reply, F_STONITH_CALLDATA, data);
+        }
+    }
+    return reply;
+}
+
+/*!
+ * \internal
+ * \brief Build an XML reply to an asynchronous fencing command
+ *
+ * \param[in] cmd     Fencing command that reply is for
+ * \param[in] result  Command result
+ */
+static xmlNode *
+construct_async_reply(const async_command_t *cmd,
+                      const pcmk__action_result_t *result)
+{
+    xmlNode *reply = create_xml_node(NULL, T_STONITH_REPLY);
+
+    crm_xml_add(reply, "st_origin", __func__);
+    crm_xml_add(reply, F_TYPE, T_STONITH_NG);
+    crm_xml_add(reply, F_STONITH_OPERATION, cmd->op);
+    crm_xml_add(reply, F_STONITH_DEVICE, cmd->device);
+    crm_xml_add(reply, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
+    crm_xml_add(reply, F_STONITH_CLIENTID, cmd->client);
+    crm_xml_add(reply, F_STONITH_CLIENTNAME, cmd->client_name);
+    crm_xml_add(reply, F_STONITH_TARGET, cmd->target);
+    crm_xml_add(reply, F_STONITH_ACTION, cmd->op);
+    crm_xml_add(reply, F_STONITH_ORIGIN, cmd->origin);
+    crm_xml_add_int(reply, F_STONITH_CALLID, cmd->id);
+    crm_xml_add_int(reply, F_STONITH_CALLOPTS, cmd->options);
+
+    stonith__xe_set_result(reply, result);
+    return reply;
+}
+
+bool fencing_peer_active(crm_node_t *peer)
+{
+    if (peer == NULL) {
+        return FALSE;
+    } else if (peer->uname == NULL) {
+        return FALSE;
+    } else if (pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
+        return TRUE;
+    }
+    return FALSE;
+}
+
+void
+set_fencing_completed(remote_fencing_op_t *op)
+{
+    struct timespec tv;
+
+    qb_util_timespec_from_epoch_get(&tv);
+    op->completed = tv.tv_sec;
+    op->completed_nsec = tv.tv_nsec;
+}
+
+/*!
+ * \internal
+ * \brief Look for alternate node needed if local node shouldn't fence target
+ *
+ * \param[in] target  Node that must be fenced
+ *
+ * \return Name of an alternate node that should fence \p target if any,
+ *         or NULL otherwise
+ */
+static const char *
+check_alternate_host(const char *target)
+{
+    if (pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
+        GHashTableIter gIter;
+        crm_node_t *entry = NULL;
+
+        g_hash_table_iter_init(&gIter, crm_peer_cache);
+        while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
+            if (fencing_peer_active(entry)
+                && !pcmk__str_eq(entry->uname, target, pcmk__str_casei)) {
+                crm_notice("Forwarding self-fencing request to %s",
+                           entry->uname);
+                return entry->uname;
+            }
+        }
+        crm_warn("Will handle own fencing because no peer can");
+    }
+    return NULL;
+}
+
+/*!
+ * \internal
+ * \brief Send a reply to a CPG peer or IPC client
+ *
+ * \param[in]     reply         XML reply to send
+ * \param[in]     call_options  Send synchronously if st_opt_sync_call is set
+ * \param[in]     remote_peer   If not NULL, name of peer node to send CPG reply
+ * \param[in,out] client        If not NULL, client to send IPC reply
+ */
+static void
+stonith_send_reply(xmlNode *reply, int call_options, const char *remote_peer,
+                   pcmk__client_t *client)
+{
+    CRM_CHECK((reply != NULL) && ((remote_peer != NULL) || (client != NULL)),
+              return);
+
+    if (remote_peer == NULL) {
+        do_local_reply(reply, client, call_options);
+    } else {
+        send_cluster_message(crm_get_peer(0, remote_peer), crm_msg_stonith_ng,
+                             reply, FALSE);
+    }
+}
+
+static void 
+remove_relay_op(xmlNode * request)
+{
+    xmlNode *dev = get_xpath_object("//@" F_STONITH_ACTION, request, LOG_TRACE);
+    const char *relay_op_id = NULL; 
+    const char *op_id = NULL;
+    const char *client_name = NULL;
+    const char *target = NULL; 
+    remote_fencing_op_t *relay_op = NULL; 
+
+    if (dev) { 
+        target = crm_element_value(dev, F_STONITH_TARGET); 
+    }
+
+    relay_op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID_RELAY);
+    op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID);
+    client_name = crm_element_value(request, F_STONITH_CLIENTNAME);
+
+    /* Delete RELAY operation. */
+    if (relay_op_id && target && pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
+        relay_op = g_hash_table_lookup(stonith_remote_op_list, relay_op_id);
+
+        if (relay_op) {
+            GHashTableIter iter;
+            remote_fencing_op_t *list_op = NULL; 
+            g_hash_table_iter_init(&iter, stonith_remote_op_list);
+
+            /* If the operation to be deleted is registered as a duplicate, delete the registration. */
+            while (g_hash_table_iter_next(&iter, NULL, (void **)&list_op)) {
+                GList *dup_iter = NULL;
+                if (list_op != relay_op) {
+                    for (dup_iter = list_op->duplicates; dup_iter != NULL; dup_iter = dup_iter->next) {
+                        remote_fencing_op_t *other = dup_iter->data;
+                        if (other == relay_op) {
+                            other->duplicates = g_list_remove(other->duplicates, relay_op);
+                            break;
+                        }
+                    }
+                }
+            }
+            crm_debug("Deleting relay op %s ('%s'%s%s for %s), "
+                      "replaced by op %s ('%s'%s%s for %s)",
+                      relay_op->id, relay_op->action,
+                      (relay_op->target == NULL)? "" : " targeting ",
+                      pcmk__s(relay_op->target, ""),
+                      relay_op->client_name, op_id, relay_op->action,
+                      (target == NULL)? "" : " targeting ", pcmk__s(target, ""),
+                      client_name);
+
+            g_hash_table_remove(stonith_remote_op_list, relay_op_id);
+        }
+    }
+}
+
+/*!
+ * \internal
+ * \brief Check whether an API request was sent by a privileged user
+ *
+ * API commands related to fencing configuration may be done only by privileged
+ * IPC users (i.e. root or hacluster), because all other users should go through
+ * the CIB to have ACLs applied. If no client was given, this is a peer request,
+ * which is always allowed.
+ *
+ * \param[in] c   IPC client that sent request (or NULL if sent by CPG peer)
+ * \param[in] op  Requested API operation (for logging only)
+ *
+ * \return true if sender is peer or privileged client, otherwise false
+ */
+static inline bool
+is_privileged(const pcmk__client_t *c, const char *op)
+{
+    if ((c == NULL) || pcmk_is_set(c->flags, pcmk__client_privileged)) {
+        return true;
+    } else {
+        crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
+                 pcmk__s(op, ""), pcmk__client_name(c));
+        return false;
+    }
+}
+
+// CRM_OP_REGISTER
+static xmlNode *
+handle_register_request(pcmk__request_t *request)
+{
+    xmlNode *reply = create_xml_node(NULL, "reply");
+
+    CRM_ASSERT(request->ipc_client != NULL);
+    crm_xml_add(reply, F_STONITH_OPERATION, CRM_OP_REGISTER);
+    crm_xml_add(reply, F_STONITH_CLIENTID, request->ipc_client->id);
+    pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    pcmk__set_request_flags(request, pcmk__request_reuse_options);
+    return reply;
+}
+
+// STONITH_OP_EXEC
+static xmlNode *
+handle_agent_request(pcmk__request_t *request)
+{
+    execute_agent_action(request->xml, &request->result);
+    if (request->result.execution_status == PCMK_EXEC_PENDING) {
+        return NULL;
+    }
+    return fenced_construct_reply(request->xml, NULL, &request->result);
+}
+
+// STONITH_OP_TIMEOUT_UPDATE
+static xmlNode *
+handle_update_timeout_request(pcmk__request_t *request)
+{
+    const char *call_id = crm_element_value(request->xml, F_STONITH_CALLID);
+    const char *client_id = crm_element_value(request->xml, F_STONITH_CLIENTID);
+    int op_timeout = 0;
+
+    crm_element_value_int(request->xml, F_STONITH_TIMEOUT, &op_timeout);
+    do_stonith_async_timeout_update(client_id, call_id, op_timeout);
+    pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    return NULL;
+}
+
+// STONITH_OP_QUERY
+static xmlNode *
+handle_query_request(pcmk__request_t *request)
+{
+    int timeout = 0;
+    xmlNode *dev = NULL;
+    const char *action = NULL;
+    const char *target = NULL;
+    const char *client_id = crm_element_value(request->xml, F_STONITH_CLIENTID);
+    struct st_query_data *query = NULL;
+
+    if (request->peer != NULL) {
+        // Record it for the future notification
+        create_remote_stonith_op(client_id, request->xml, TRUE);
+    }
+
+    /* Delete the DC node RELAY operation. */
+    remove_relay_op(request->xml);
+
+    pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+
+    dev = get_xpath_object("//@" F_STONITH_ACTION, request->xml, LOG_NEVER);
+    if (dev != NULL) {
+        const char *device = crm_element_value(dev, F_STONITH_DEVICE);
+
+        if (pcmk__str_eq(device, "manual_ack", pcmk__str_casei)) {
+            return NULL; // No query or reply necessary
+        }
+        target = crm_element_value(dev, F_STONITH_TARGET);
+        action = crm_element_value(dev, F_STONITH_ACTION);
+    }
+
+    crm_log_xml_trace(request->xml, "Query");
+
+    query = calloc(1, sizeof(struct st_query_data));
+    CRM_ASSERT(query != NULL);
+
+    query->reply = fenced_construct_reply(request->xml, NULL, &request->result);
+    pcmk__str_update(&query->remote_peer, request->peer);
+    pcmk__str_update(&query->client_id, client_id);
+    pcmk__str_update(&query->target, target);
+    pcmk__str_update(&query->action, action);
+    query->call_options = request->call_options;
+
+    crm_element_value_int(request->xml, F_STONITH_TIMEOUT, &timeout);
+    get_capable_devices(target, action, timeout,
+                        pcmk_is_set(query->call_options, st_opt_allow_suicide),
+                        query, stonith_query_capable_device_cb, st_device_supports_none);
+    return NULL;
+}
+
+// T_STONITH_NOTIFY
+static xmlNode *
+handle_notify_request(pcmk__request_t *request)
+{
+    const char *flag_name = NULL;
+
+    CRM_ASSERT(request->ipc_client != NULL);
+    flag_name = crm_element_value(request->xml, F_STONITH_NOTIFY_ACTIVATE);
+    if (flag_name != NULL) {
+        crm_debug("Enabling %s callbacks for client %s",
+                  flag_name, pcmk__request_origin(request));
+        pcmk__set_client_flags(request->ipc_client, get_stonith_flag(flag_name));
+    }
+
+    flag_name = crm_element_value(request->xml, F_STONITH_NOTIFY_DEACTIVATE);
+    if (flag_name != NULL) {
+        crm_debug("Disabling %s callbacks for client %s",
+                  flag_name, pcmk__request_origin(request));
+        pcmk__clear_client_flags(request->ipc_client,
+                                 get_stonith_flag(flag_name));
+    }
+
+    pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    pcmk__set_request_flags(request, pcmk__request_reuse_options);
+
+    return pcmk__ipc_create_ack(request->ipc_flags, "ack", NULL, CRM_EX_OK);
+}
+
+// STONITH_OP_RELAY
+static xmlNode *
+handle_relay_request(pcmk__request_t *request)
+{
+    xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request->xml,
+                                    LOG_TRACE);
+
+    crm_notice("Received forwarded fencing request from "
+               "%s %s to fence (%s) peer %s",
+               pcmk__request_origin_type(request),
+               pcmk__request_origin(request),
+               crm_element_value(dev, F_STONITH_ACTION),
+               crm_element_value(dev, F_STONITH_TARGET));
+
+    if (initiate_remote_stonith_op(NULL, request->xml, FALSE) == NULL) {
+        fenced_set_protocol_error(&request->result);
+        return fenced_construct_reply(request->xml, NULL, &request->result);
+    }
+
+    pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
+    return NULL;
+}
+
+// STONITH_OP_FENCE
+static xmlNode *
+handle_fence_request(pcmk__request_t *request)
+{
+    if ((request->peer != NULL) || stand_alone) {
+        fence_locally(request->xml, &request->result);
+
+    } else if (pcmk_is_set(request->call_options, st_opt_manual_ack)) {
+        switch (fenced_handle_manual_confirmation(request->ipc_client,
+                                                  request->xml)) {
+            case pcmk_rc_ok:
+                pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
+                                 NULL);
+                break;
+            case EINPROGRESS:
+                pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
+                                 NULL);
+                break;
+            default:
+                fenced_set_protocol_error(&request->result);
+                break;
+        }
+
+    } else {
+        const char *alternate_host = NULL;
+        xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request->xml,
+                                        LOG_TRACE);
+        const char *target = crm_element_value(dev, F_STONITH_TARGET);
+        const char *action = crm_element_value(dev, F_STONITH_ACTION);
+        const char *device = crm_element_value(dev, F_STONITH_DEVICE);
+
+        if (request->ipc_client != NULL) {
+            int tolerance = 0;
+
+            crm_notice("Client %s wants to fence (%s) %s using %s",
+                       pcmk__request_origin(request), action,
+                       target, (device? device : "any device"));
+            crm_element_value_int(dev, F_STONITH_TOLERANCE, &tolerance);
+            if (stonith_check_fence_tolerance(tolerance, target, action)) {
+                pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
+                                 NULL);
+                return fenced_construct_reply(request->xml, NULL,
+                                              &request->result);
+            }
+            alternate_host = check_alternate_host(target);
+
+        } else {
+            crm_notice("Peer %s wants to fence (%s) '%s' with device '%s'",
+                       request->peer, action, target,
+                       (device == NULL)? "(any)" : device);
+        }
+
+        if (alternate_host != NULL) {
+            const char *client_id = NULL;
+            remote_fencing_op_t *op = NULL;
+
+            if (request->ipc_client->id == 0) {
+                client_id = crm_element_value(request->xml, F_STONITH_CLIENTID);
+            } else {
+                client_id = request->ipc_client->id;
+            }
+
+            /* Create a duplicate fencing operation to relay with the client ID.
+             * When a query response is received, this operation should be
+             * deleted to avoid keeping the duplicate around.
+             */
+            op = create_remote_stonith_op(client_id, request->xml, FALSE);
+
+            crm_xml_add(request->xml, F_STONITH_OPERATION, STONITH_OP_RELAY);
+            crm_xml_add(request->xml, F_STONITH_CLIENTID,
+                        request->ipc_client->id);
+            crm_xml_add(request->xml, F_STONITH_REMOTE_OP_ID, op->id);
+            send_cluster_message(crm_get_peer(0, alternate_host),
+                                 crm_msg_stonith_ng, request->xml, FALSE);
+            pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
+                             NULL);
+
+        } else if (initiate_remote_stonith_op(request->ipc_client, request->xml,
+                                              FALSE) == NULL) {
+            fenced_set_protocol_error(&request->result);
+
+        } else {
+            pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
+                             NULL);
+        }
+    }
+
+    if (request->result.execution_status == PCMK_EXEC_PENDING) {
+        return NULL;
+    }
+    return fenced_construct_reply(request->xml, NULL, &request->result);
+}
+
+// STONITH_OP_FENCE_HISTORY
+static xmlNode *
+handle_history_request(pcmk__request_t *request)
+{
+    xmlNode *reply = NULL;
+    xmlNode *data = NULL;
+
+    stonith_fence_history(request->xml, &data, request->peer,
+                          request->call_options);
+    pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    if (!pcmk_is_set(request->call_options, st_opt_discard_reply)) {
+        /* When the local node broadcasts its history, it sets
+         * st_opt_discard_reply and doesn't need a reply.
+         */
+        reply = fenced_construct_reply(request->xml, data, &request->result);
+    }
+    free_xml(data);
+    return reply;
+}
+
+// STONITH_OP_DEVICE_ADD
+static xmlNode *
+handle_device_add_request(pcmk__request_t *request)
+{
+    const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
+    xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request->xml,
+                                    LOG_ERR);
+
+    if (is_privileged(request->ipc_client, op)) {
+        int rc = stonith_device_register(dev, FALSE);
+
+        pcmk__set_result(&request->result,
+                         ((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR),
+                         stonith__legacy2status(rc),
+                         ((rc == pcmk_ok)? NULL : pcmk_strerror(rc)));
+    } else {
+        pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
+                         PCMK_EXEC_INVALID,
+                         "Unprivileged users must register device via CIB");
+    }
+    fenced_send_device_notification(op, &request->result,
+                                    (dev == NULL)? NULL : ID(dev));
+    return fenced_construct_reply(request->xml, NULL, &request->result);
+}
+
+// STONITH_OP_DEVICE_DEL
+static xmlNode *
+handle_device_delete_request(pcmk__request_t *request)
+{
+    xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request->xml,
+                                    LOG_ERR);
+    const char *device_id = crm_element_value(dev, XML_ATTR_ID);
+    const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
+
+    if (is_privileged(request->ipc_client, op)) {
+        stonith_device_remove(device_id, false);
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    } else {
+        pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
+                         PCMK_EXEC_INVALID,
+                         "Unprivileged users must delete device via CIB");
+    }
+    fenced_send_device_notification(op, &request->result, device_id);
+    return fenced_construct_reply(request->xml, NULL, &request->result);
+}
+
+// STONITH_OP_LEVEL_ADD
+static xmlNode *
+handle_level_add_request(pcmk__request_t *request)
+{
+    char *desc = NULL;
+    const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
+
+    if (is_privileged(request->ipc_client, op)) {
+        fenced_register_level(request->xml, &desc, &request->result);
+    } else {
+        unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
+        pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
+                         PCMK_EXEC_INVALID,
+                         "Unprivileged users must add level via CIB");
+    }
+    fenced_send_level_notification(op, &request->result, desc);
+    free(desc);
+    return fenced_construct_reply(request->xml, NULL, &request->result);
+}
+
+// STONITH_OP_LEVEL_DEL
+static xmlNode *
+handle_level_delete_request(pcmk__request_t *request)
+{
+    char *desc = NULL;
+    const char *op = crm_element_value(request->xml, F_STONITH_OPERATION);
+
+    if (is_privileged(request->ipc_client, op)) {
+        fenced_unregister_level(request->xml, &desc, &request->result);
+    } else {
+        unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
+        pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
+                         PCMK_EXEC_INVALID,
+                         "Unprivileged users must delete level via CIB");
+    }
+    fenced_send_level_notification(op, &request->result, desc);
+    free(desc);
+    return fenced_construct_reply(request->xml, NULL, &request->result);
+}
+
+// CRM_OP_RM_NODE_CACHE
+static xmlNode *
+handle_cache_request(pcmk__request_t *request)
+{
+    int node_id = 0;
+    const char *name = NULL;
+
+    crm_element_value_int(request->xml, XML_ATTR_ID, &node_id);
+    name = crm_element_value(request->xml, XML_ATTR_UNAME);
+    reap_crm_member(node_id, name);
+    pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    return NULL;
+}
+
+static xmlNode *
+handle_unknown_request(pcmk__request_t *request)
+{
+    crm_err("Unknown IPC request %s from %s %s",
+            request->op, pcmk__request_origin_type(request),
+            pcmk__request_origin(request));
+    pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
+                        "Unknown IPC request type '%s' (bug?)", request->op);
+    return fenced_construct_reply(request->xml, NULL, &request->result);
+}
+
+static void
+fenced_register_handlers(void)
+{
+    pcmk__server_command_t handlers[] = {
+        { CRM_OP_REGISTER, handle_register_request },
+        { STONITH_OP_EXEC, handle_agent_request },
+        { STONITH_OP_TIMEOUT_UPDATE, handle_update_timeout_request },
+        { STONITH_OP_QUERY, handle_query_request },
+        { T_STONITH_NOTIFY, handle_notify_request },
+        { STONITH_OP_RELAY, handle_relay_request },
+        { STONITH_OP_FENCE, handle_fence_request },
+        { STONITH_OP_FENCE_HISTORY, handle_history_request },
+        { STONITH_OP_DEVICE_ADD, handle_device_add_request },
+        { STONITH_OP_DEVICE_DEL, handle_device_delete_request },
+        { STONITH_OP_LEVEL_ADD, handle_level_add_request },
+        { STONITH_OP_LEVEL_DEL, handle_level_delete_request },
+        { CRM_OP_RM_NODE_CACHE, handle_cache_request },
+        { NULL, handle_unknown_request },
+    };
+
+    fenced_handlers = pcmk__register_handlers(handlers);
+}
+
+void
+fenced_unregister_handlers(void)
+{
+    if (fenced_handlers != NULL) {
+        g_hash_table_destroy(fenced_handlers);
+        fenced_handlers = NULL;
+    }
+}
+
+static void
+handle_request(pcmk__request_t *request)
+{
+    xmlNode *reply = NULL;
+    const char *reason = NULL;
+
+    if (fenced_handlers == NULL) {
+        fenced_register_handlers();
+    }
+    reply = pcmk__process_request(request, fenced_handlers);
+    if (reply != NULL) {
+        if (pcmk_is_set(request->flags, pcmk__request_reuse_options)
+            && (request->ipc_client != NULL)) {
+            /* Certain IPC-only commands must reuse the call options from the
+             * original request rather than the ones set by stonith_send_reply()
+             * -> do_local_reply().
+             */
+            pcmk__ipc_send_xml(request->ipc_client, request->ipc_id, reply,
+                               request->ipc_flags);
+            request->ipc_client->request_id = 0;
+        } else {
+            stonith_send_reply(reply, request->call_options,
+                               request->peer, request->ipc_client);
+        }
+        free_xml(reply);
+    }
+
+    reason = request->result.exit_reason;
+    crm_debug("Processed %s request from %s %s: %s%s%s%s",
+              request->op, pcmk__request_origin_type(request),
+              pcmk__request_origin(request),
+              pcmk_exec_status_str(request->result.execution_status),
+              (reason == NULL)? "" : " (",
+              (reason == NULL)? "" : reason,
+              (reason == NULL)? "" : ")");
+}
+
+static void
+handle_reply(pcmk__client_t *client, xmlNode *request, const char *remote_peer)
+{
+    // Copy, because request might be freed before we want to log this
+    char *op = crm_element_value_copy(request, F_STONITH_OPERATION);
+
+    if (pcmk__str_eq(op, STONITH_OP_QUERY, pcmk__str_none)) {
+        process_remote_stonith_query(request);
+    } else if (pcmk__str_any_of(op, T_STONITH_NOTIFY, STONITH_OP_FENCE, NULL)) {
+        fenced_process_fencing_reply(request);
+    } else {
+        crm_err("Ignoring unknown %s reply from %s %s",
+                pcmk__s(op, "untyped"), ((client == NULL)? "peer" : "client"),
+                ((client == NULL)? remote_peer : pcmk__client_name(client)));
+        crm_log_xml_warn(request, "UnknownOp");
+        free(op);
+        return;
+    }
+    crm_debug("Processed %s reply from %s %s",
+              op, ((client == NULL)? "peer" : "client"),
+              ((client == NULL)? remote_peer : pcmk__client_name(client)));
+    free(op);
+}
+
+/*!
+ * \internal
+ * \brief Handle a message from an IPC client or CPG peer
+ *
+ * \param[in,out] client      If not NULL, IPC client that sent message
+ * \param[in]     id          If from IPC client, IPC message ID
+ * \param[in]     flags       Message flags
+ * \param[in,out] message     Message XML
+ * \param[in]     remote_peer If not NULL, CPG peer that sent message
+ */
+void
+stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags,
+                xmlNode *message, const char *remote_peer)
+{
+    int call_options = st_opt_none;
+    bool is_reply = false;
+
+    CRM_CHECK(message != NULL, return);
+
+    if (get_xpath_object("//" T_STONITH_REPLY, message, LOG_NEVER) != NULL) {
+        is_reply = true;
+    }
+    crm_element_value_int(message, F_STONITH_CALLOPTS, &call_options);
+    crm_debug("Processing %ssynchronous %s %s %u from %s %s",
+              pcmk_is_set(call_options, st_opt_sync_call)? "" : "a",
+              crm_element_value(message, F_STONITH_OPERATION),
+              (is_reply? "reply" : "request"), id,
+              ((client == NULL)? "peer" : "client"),
+              ((client == NULL)? remote_peer : pcmk__client_name(client)));
+
+    if (pcmk_is_set(call_options, st_opt_sync_call)) {
+        CRM_ASSERT(client == NULL || client->request_id == id);
+    }
+
+    if (is_reply) {
+        handle_reply(client, message, remote_peer);
+    } else {
+        pcmk__request_t request = {
+            .ipc_client     = client,
+            .ipc_id         = id,
+            .ipc_flags      = flags,
+            .peer           = remote_peer,
+            .xml            = message,
+            .call_options   = call_options,
+            .result         = PCMK__UNKNOWN_RESULT,
+        };
+
+        request.op = crm_element_value_copy(request.xml, F_STONITH_OPERATION);
+        CRM_CHECK(request.op != NULL, return);
+
+        if (pcmk_is_set(request.call_options, st_opt_sync_call)) {
+            pcmk__set_request_flags(&request, pcmk__request_sync);
+        }
+
+        handle_request(&request);
+        pcmk__reset_request(&request);
+    }
+}
diff --git a/daemons/fenced/fenced_history.c b/daemons/fenced/fenced_history.c
new file mode 100644
index 0000000..a766477
--- /dev/null
+++ b/daemons/fenced/fenced_history.c
@@ -0,0 +1,548 @@
+/*
+ * Copyright 2009-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/cluster/internal.h>
+
+#include <crm/stonith-ng.h>
+#include <crm/fencing/internal.h>
+#include <crm/common/xml.h>
+#include <crm/common/xml_internal.h>
+
+#include <pacemaker-fenced.h>
+
+#define MAX_STONITH_HISTORY 500
+
+/*!
+ * \internal
+ * \brief Send a broadcast to all nodes to trigger cleanup or
+ *        history synchronisation
+ *
+ * \param[in] history   Optional history to be attached
+ * \param[in] callopts  We control cleanup via a flag in the callopts
+ * \param[in] target    Cleanup can be limited to certain fence-targets
+ */
+static void
+stonith_send_broadcast_history(xmlNode *history,
+                               int callopts,
+                               const char *target)
+{
+    xmlNode *bcast = create_xml_node(NULL, "stonith_command");
+    xmlNode *data = create_xml_node(NULL, __func__);
+
+    if (target) {
+        crm_xml_add(data, F_STONITH_TARGET, target);
+    }
+    crm_xml_add(bcast, F_TYPE, T_STONITH_NG);
+    crm_xml_add(bcast, F_SUBTYPE, "broadcast");
+    crm_xml_add(bcast, F_STONITH_OPERATION, STONITH_OP_FENCE_HISTORY);
+    crm_xml_add_int(bcast, F_STONITH_CALLOPTS, callopts);
+    if (history) {
+        add_node_copy(data, history);
+    }
+    add_message_xml(bcast, F_STONITH_CALLDATA, data);
+    send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE);
+
+    free_xml(data);
+    free_xml(bcast);
+}
+
+static gboolean
+stonith_remove_history_entry (gpointer key,
+                              gpointer value,
+                              gpointer user_data)
+{
+    remote_fencing_op_t *op = value;
+    const char *target = (const char *) user_data;
+
+    if ((op->state == st_failed) || (op->state == st_done)) {
+        if ((target) && (strcmp(op->target, target) != 0)) {
+            return FALSE;
+        }
+        return TRUE;
+    }
+
+    return FALSE; /* don't clean pending operations */
+}
+
+/*!
+ * \internal
+ * \brief Send out a cleanup broadcast or do a local history-cleanup
+ *
+ * \param[in] target    Cleanup can be limited to certain fence-targets
+ * \param[in] broadcast Send out a cleanup broadcast
+ */
+static void
+stonith_fence_history_cleanup(const char *target,
+                              gboolean broadcast)
+{
+    if (broadcast) {
+        stonith_send_broadcast_history(NULL,
+                                       st_opt_cleanup | st_opt_discard_reply,
+                                       target);
+        /* we'll do the local clean when we receive back our own broadcast */
+    } else if (stonith_remote_op_list) {
+        g_hash_table_foreach_remove(stonith_remote_op_list,
+                             stonith_remove_history_entry,
+                             (gpointer) target);
+        fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
+    }
+}
+
+/* keeping the length of fence-history within bounds
+ * =================================================
+ *
+ * If things are really running wild a lot of fencing-attempts
+ * might fill up the hash-map, eventually using up a lot
+ * of memory and creating huge history-sync messages.
+ * Before the history being synced across nodes at least
+ * the reboot of a cluster-node helped keeping the
+ * history within bounds even though not in a reliable
+ * manner.
+ *
+ * stonith_remote_op_list isn't sorted for time-stamps
+ * thus it would be kind of expensive to delete e.g.
+ * the oldest entry if it would grow past MAX_STONITH_HISTORY
+ * entries.
+ * It is more efficient to purge MAX_STONITH_HISTORY/2
+ * entries whenever the list grows beyond MAX_STONITH_HISTORY.
+ * (sort for age + purge the MAX_STONITH_HISTORY/2 oldest)
+ * That done on a per-node-base might raise the
+ * probability of large syncs to occur.
+ * Things like introducing a broadcast to purge
+ * MAX_STONITH_HISTORY/2 entries or not sync above a certain
+ * threshold coming to mind ...
+ * Simplest thing though is to purge the full history
+ * throughout the cluster once MAX_STONITH_HISTORY is reached.
+ * On the other hand this leads to purging the history in
+ * situations where it would be handy to have it probably.
+ */
+
+
+static int
+op_time_sort(const void *a_voidp, const void *b_voidp)
+{
+    const remote_fencing_op_t **a = (const remote_fencing_op_t **) a_voidp;
+    const remote_fencing_op_t **b = (const remote_fencing_op_t **) b_voidp;
+    gboolean a_pending = ((*a)->state != st_failed) && ((*a)->state != st_done);
+    gboolean b_pending = ((*b)->state != st_failed) && ((*b)->state != st_done);
+
+    if (a_pending && b_pending) {
+        return 0;
+    } else if (a_pending) {
+        return -1;
+    } else if (b_pending) {
+        return 1;
+    } else if ((*b)->completed == (*a)->completed) {
+        if ((*b)->completed_nsec > (*a)->completed_nsec) {
+            return 1;
+        } else if ((*b)->completed_nsec == (*a)->completed_nsec) {
+            return 0;
+        }
+    } else if ((*b)->completed > (*a)->completed) {
+        return 1;
+    }
+
+    return -1;
+}
+
+
+/*!
+ * \internal
+ * \brief Do a local history-trim to MAX_STONITH_HISTORY / 2 entries
+ *        once over MAX_STONITH_HISTORY
+ */
+void
+stonith_fence_history_trim(void)
+{
+    guint num_ops;
+
+    if (!stonith_remote_op_list) {
+        return;
+    }
+    num_ops = g_hash_table_size(stonith_remote_op_list);
+    if (num_ops > MAX_STONITH_HISTORY) {
+        remote_fencing_op_t *ops[num_ops];
+        remote_fencing_op_t *op = NULL;
+        GHashTableIter iter;
+        int i;
+
+        crm_trace("Fencing History growing beyond limit of %d so purge "
+                  "half of failed/successful attempts", MAX_STONITH_HISTORY);
+
+        /* write all ops into an array */
+        i = 0;
+        g_hash_table_iter_init(&iter, stonith_remote_op_list);
+        while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
+            ops[i++] = op;
+        }
+        /* run quicksort over the array so that we get pending ops
+         * first and then sorted most recent to oldest
+         */
+        qsort(ops, num_ops, sizeof(remote_fencing_op_t *), op_time_sort);
+        /* purgest oldest half of the history entries */
+        for (i = MAX_STONITH_HISTORY / 2; i < num_ops; i++) {
+            /* keep pending ops even if they shouldn't fill more than
+             * half of our buffer
+             */
+            if ((ops[i]->state == st_failed) || (ops[i]->state == st_done)) {
+                g_hash_table_remove(stonith_remote_op_list, ops[i]->id);
+            }
+        }
+        /* we've just purged valid data from the list so there is no need
+         * to create a notification - if displayed it can stay
+         */
+    }
+}
+
+/*!
+ * \internal
+ * \brief Convert xml fence-history to a hash-table like stonith_remote_op_list
+ *
+ * \param[in] history   Fence-history in xml
+ *
+ * \return Fence-history as hash-table
+ */
+static GHashTable *
+stonith_xml_history_to_list(const xmlNode *history)
+{
+    xmlNode *xml_op = NULL;
+    GHashTable *rv = NULL;
+
+    init_stonith_remote_op_hash_table(&rv);
+
+    CRM_LOG_ASSERT(rv != NULL);
+
+    for (xml_op = pcmk__xml_first_child(history); xml_op != NULL;
+         xml_op = pcmk__xml_next(xml_op)) {
+        remote_fencing_op_t *op = NULL;
+        char *id = crm_element_value_copy(xml_op, F_STONITH_REMOTE_OP_ID);
+        int state;
+        int exit_status = CRM_EX_OK;
+        int execution_status = PCMK_EXEC_DONE;
+        long long completed;
+        long long completed_nsec = 0L;
+
+        if (!id) {
+            crm_warn("Malformed fencing history received from peer");
+            continue;
+        }
+
+        crm_trace("Attaching op %s to hashtable", id);
+
+        op = calloc(1, sizeof(remote_fencing_op_t));
+
+        op->id = id;
+        op->target = crm_element_value_copy(xml_op, F_STONITH_TARGET);
+        op->action = crm_element_value_copy(xml_op, F_STONITH_ACTION);
+        op->originator = crm_element_value_copy(xml_op, F_STONITH_ORIGIN);
+        op->delegate = crm_element_value_copy(xml_op, F_STONITH_DELEGATE);
+        op->client_name = crm_element_value_copy(xml_op, F_STONITH_CLIENTNAME);
+        crm_element_value_ll(xml_op, F_STONITH_DATE, &completed);
+        op->completed = (time_t) completed;
+        crm_element_value_ll(xml_op, F_STONITH_DATE_NSEC, &completed_nsec);
+        op->completed_nsec = completed_nsec;
+        crm_element_value_int(xml_op, F_STONITH_STATE, &state);
+        op->state = (enum op_state) state;
+
+        /* @COMPAT We can't use stonith__xe_get_result() here because
+         * fencers <2.1.3 didn't include results, leading it to assume an error
+         * status. Instead, set an unknown status in that case.
+         */
+        if ((crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &exit_status) < 0)
+            || (crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS,
+                                      &execution_status) < 0)) {
+            exit_status = CRM_EX_INDETERMINATE;
+            execution_status = PCMK_EXEC_UNKNOWN;
+        }
+        pcmk__set_result(&op->result, exit_status, execution_status,
+                         crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON));
+        pcmk__set_result_output(&op->result,
+                                crm_element_value_copy(xml_op, F_STONITH_OUTPUT),
+                                NULL);
+
+
+        g_hash_table_replace(rv, id, op);
+        CRM_LOG_ASSERT(g_hash_table_lookup(rv, id) != NULL);
+    }
+
+    return rv;
+}
+
+/*!
+ * \internal
+ * \brief Craft xml difference between local fence-history and a history
+ *        coming from remote, and merge the remote history into the local
+ *
+ * \param[in,out] remote_history  Fence-history as hash-table (may be NULL)
+ * \param[in]     add_id          If crafting the answer for an API
+ *                                history-request there is no need for the id
+ * \param[in]     target          Optionally limit to certain fence-target
+ *
+ * \return The fence-history as xml
+ */
+static xmlNode *
+stonith_local_history_diff_and_merge(GHashTable *remote_history,
+                                     gboolean add_id, const char *target)
+{
+    xmlNode *history = NULL;
+    GHashTableIter iter;
+    remote_fencing_op_t *op = NULL;
+    gboolean updated = FALSE;
+    int cnt = 0;
+
+    if (stonith_remote_op_list) {
+            char *id = NULL;
+
+            history = create_xml_node(NULL, F_STONITH_HISTORY_LIST);
+
+            g_hash_table_iter_init(&iter, stonith_remote_op_list);
+            while (g_hash_table_iter_next(&iter, (void **)&id, (void **)&op)) {
+                xmlNode *entry = NULL;
+
+                if (remote_history) {
+                    remote_fencing_op_t *remote_op =
+                        g_hash_table_lookup(remote_history, op->id);
+
+                    if (remote_op) {
+                        if (stonith__op_state_pending(op->state)
+                            && !stonith__op_state_pending(remote_op->state)) {
+
+                            crm_debug("Updating outdated pending operation %.8s "
+                                      "(state=%s) according to the one (state=%s) from "
+                                      "remote peer history",
+                                      op->id, stonith_op_state_str(op->state),
+                                      stonith_op_state_str(remote_op->state));
+
+                            g_hash_table_steal(remote_history, op->id);
+                            op->id = remote_op->id;
+                            remote_op->id = id;
+                            g_hash_table_iter_replace(&iter, remote_op);
+
+                            updated = TRUE;
+                            continue; /* skip outdated entries */
+
+                        } else if (!stonith__op_state_pending(op->state)
+                                   && stonith__op_state_pending(remote_op->state)) {
+
+                            crm_debug("Broadcasting operation %.8s (state=%s) to "
+                                      "update the outdated pending one "
+                                      "(state=%s) in remote peer history",
+                                      op->id, stonith_op_state_str(op->state),
+                                      stonith_op_state_str(remote_op->state));
+
+                            g_hash_table_remove(remote_history, op->id);
+
+                        } else {
+                            g_hash_table_remove(remote_history, op->id);
+                            continue; /* skip entries broadcasted already */
+                        }
+                    }
+                }
+
+                if (!pcmk__str_eq(target, op->target, pcmk__str_null_matches)) {
+                    continue;
+                }
+
+                cnt++;
+                crm_trace("Attaching op %s", op->id);
+                entry = create_xml_node(history, STONITH_OP_EXEC);
+                if (add_id) {
+                    crm_xml_add(entry, F_STONITH_REMOTE_OP_ID, op->id);
+                }
+                crm_xml_add(entry, F_STONITH_TARGET, op->target);
+                crm_xml_add(entry, F_STONITH_ACTION, op->action);
+                crm_xml_add(entry, F_STONITH_ORIGIN, op->originator);
+                crm_xml_add(entry, F_STONITH_DELEGATE, op->delegate);
+                crm_xml_add(entry, F_STONITH_CLIENTNAME, op->client_name);
+                crm_xml_add_ll(entry, F_STONITH_DATE, op->completed);
+                crm_xml_add_ll(entry, F_STONITH_DATE_NSEC, op->completed_nsec);
+                crm_xml_add_int(entry, F_STONITH_STATE, op->state);
+                stonith__xe_set_result(entry, &op->result);
+            }
+    }
+
+    if (remote_history) {
+        init_stonith_remote_op_hash_table(&stonith_remote_op_list);
+
+        updated |= g_hash_table_size(remote_history);
+
+        g_hash_table_iter_init(&iter, remote_history);
+        while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
+            if (stonith__op_state_pending(op->state) &&
+                pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
+
+                crm_warn("Failing pending operation %.8s originated by us but "
+                         "known only from peer history", op->id);
+                op->state = st_failed;
+                set_fencing_completed(op);
+
+                /* CRM_EX_EXPIRED + PCMK_EXEC_INVALID prevents finalize_op()
+                 * from setting a delegate
+                 */
+                pcmk__set_result(&op->result, CRM_EX_EXPIRED, PCMK_EXEC_INVALID,
+                                 "Initiated by earlier fencer "
+                                 "process and presumed failed");
+                fenced_broadcast_op_result(op, false);
+            }
+
+            g_hash_table_iter_steal(&iter);
+            g_hash_table_replace(stonith_remote_op_list, op->id, op);
+            /* we could trim the history here but if we bail
+             * out after trim we might miss more recent entries
+             * of those that might still be in the list
+             * if we don't bail out trimming once is more
+             * efficient and memory overhead is minimal as
+             * we are just moving pointers from one hash to
+             * another
+             */
+        }
+
+        g_hash_table_destroy(remote_history); /* remove what is left */
+    }
+
+    if (updated) {
+        stonith_fence_history_trim();
+        fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
+    }
+
+    if (cnt == 0) {
+        free_xml(history);
+        return NULL;
+    } else {
+        return history;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Craft xml from the local fence-history
+ *
+ * \param[in] add_id            If crafting the answer for an API
+ *                              history-request there is no need for the id
+ * \param[in] target            Optionally limit to certain fence-target
+ *
+ * \return The fence-history as xml
+ */
+static xmlNode *
+stonith_local_history(gboolean add_id, const char *target)
+{
+    return stonith_local_history_diff_and_merge(NULL, add_id, target);
+}
+
+/*!
+ * \internal
+ * \brief Handle fence-history messages (from API or coming in as broadcasts)
+ *
+ * \param[in,out] msg          Request XML
+ * \param[out]    output       Where to set local history, if requested
+ * \param[in]     remote_peer  If broadcast, peer that sent it
+ * \param[in]     options      Call options from the request
+ */
+void
+stonith_fence_history(xmlNode *msg, xmlNode **output,
+                      const char *remote_peer, int options)
+{
+    const char *target = NULL;
+    xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_NEVER);
+    xmlNode *out_history = NULL;
+
+    if (dev) {
+        target = crm_element_value(dev, F_STONITH_TARGET);
+        if (target && (options & st_opt_cs_nodeid)) {
+            int nodeid;
+            crm_node_t *node;
+
+            pcmk__scan_min_int(target, &nodeid, 0);
+            node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY);
+            if (node) {
+                target = node->uname;
+            }
+        }
+    }
+
+    if (options & st_opt_cleanup) {
+        crm_trace("Cleaning up operations on %s in %p", target,
+                  stonith_remote_op_list);
+
+        stonith_fence_history_cleanup(target,
+            crm_element_value(msg, F_STONITH_CALLID) != NULL);
+    } else if (options & st_opt_broadcast) {
+        /* there is no clear sign atm for when a history sync
+           is done so send a notification for anything
+           that smells like history-sync
+         */
+        fenced_send_notification(T_STONITH_NOTIFY_HISTORY_SYNCED, NULL, NULL);
+        if (crm_element_value(msg, F_STONITH_CALLID)) {
+            /* this is coming from the stonith-API
+            *
+            * craft a broadcast with node's history
+            * so that every node can merge and broadcast
+            * what it has on top
+            */
+            out_history = stonith_local_history(TRUE, NULL);
+            crm_trace("Broadcasting history to peers");
+            stonith_send_broadcast_history(out_history,
+                                        st_opt_broadcast | st_opt_discard_reply,
+                                        NULL);
+        } else if (remote_peer &&
+                   !pcmk__str_eq(remote_peer, stonith_our_uname, pcmk__str_casei)) {
+            xmlNode *history = get_xpath_object("//" F_STONITH_HISTORY_LIST,
+                                                msg, LOG_NEVER);
+
+            /* either a broadcast created directly upon stonith-API request
+            * or a diff as response to such a thing
+            *
+            * in both cases it may have a history or not
+            * if we have differential data
+            * merge in what we've received and stop
+            * otherwise broadcast what we have on top
+            * marking as differential and merge in afterwards
+            */
+            if (!history || !pcmk__xe_attr_is_true(history, F_STONITH_DIFFERENTIAL)) {
+                GHashTable *received_history = NULL;
+
+                if (history != NULL) {
+                    received_history = stonith_xml_history_to_list(history);
+                }
+                out_history =
+                    stonith_local_history_diff_and_merge(received_history, TRUE, NULL);
+                if (out_history) {
+                    crm_trace("Broadcasting history-diff to peers");
+                    pcmk__xe_set_bool_attr(out_history, F_STONITH_DIFFERENTIAL, true);
+                    stonith_send_broadcast_history(out_history,
+                        st_opt_broadcast | st_opt_discard_reply,
+                        NULL);
+                } else {
+                    crm_trace("History-diff is empty - skip broadcast");
+                }
+            }
+        } else {
+            crm_trace("Skipping history-query-broadcast (%s%s)"
+                      " we sent ourselves",
+                      remote_peer?"remote-peer=":"local-ipc",
+                      remote_peer?remote_peer:"");
+        }
+    } else {
+        /* plain history request */
+        crm_trace("Looking for operations on %s in %p", target,
+                  stonith_remote_op_list);
+        *output = stonith_local_history(FALSE, target);
+    }
+    free_xml(out_history);
+}
diff --git a/daemons/fenced/fenced_remote.c b/daemons/fenced/fenced_remote.c
new file mode 100644
index 0000000..dc67947
--- /dev/null
+++ b/daemons/fenced/fenced_remote.c
@@ -0,0 +1,2509 @@
+/*
+ * Copyright 2009-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/utsname.h>
+
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <ctype.h>
+#include <regex.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/cluster/internal.h>
+
+#include <crm/stonith-ng.h>
+#include <crm/fencing/internal.h>
+#include <crm/common/xml.h>
+#include <crm/common/xml_internal.h>
+
+#include <crm/common/util.h>
+#include <pacemaker-fenced.h>
+
+#define TIMEOUT_MULTIPLY_FACTOR 1.2
+
+/* When one fencer queries its peers for devices able to handle a fencing
+ * request, each peer will reply with a list of such devices available to it.
+ * Each reply will be parsed into a peer_device_info_t, with each device's
+ * information kept in a device_properties_t.
+ */
+
+typedef struct device_properties_s {
+    /* Whether access to this device has been verified */
+    gboolean verified;
+
+    /* The remaining members are indexed by the operation's "phase" */
+
+    /* Whether this device has been executed in each phase */
+    gboolean executed[st_phase_max];
+    /* Whether this device is disallowed from executing in each phase */
+    gboolean disallowed[st_phase_max];
+    /* Action-specific timeout for each phase */
+    int custom_action_timeout[st_phase_max];
+    /* Action-specific maximum random delay for each phase */
+    int delay_max[st_phase_max];
+    /* Action-specific base delay for each phase */
+    int delay_base[st_phase_max];
+    /* Group of enum st_device_flags */
+    uint32_t device_support_flags;
+} device_properties_t;
+
+typedef struct {
+    /* Name of peer that sent this result */
+    char *host;
+    /* Only try peers for non-topology based operations once */
+    gboolean tried;
+    /* Number of entries in the devices table */
+    int ndevices;
+    /* Devices available to this host that are capable of fencing the target */
+    GHashTable *devices;
+} peer_device_info_t;
+
+GHashTable *stonith_remote_op_list = NULL;
+
+extern xmlNode *stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data,
+                                  int call_options);
+
+static void request_peer_fencing(remote_fencing_op_t *op,
+                                 peer_device_info_t *peer);
+static void finalize_op(remote_fencing_op_t *op, xmlNode *data, bool dup);
+static void report_timeout_period(remote_fencing_op_t * op, int op_timeout);
+static int get_op_total_timeout(const remote_fencing_op_t *op,
+                                const peer_device_info_t *chosen_peer);
+
+static gint
+sort_strings(gconstpointer a, gconstpointer b)
+{
+    return strcmp(a, b);
+}
+
+static void
+free_remote_query(gpointer data)
+{
+    if (data != NULL) {
+        peer_device_info_t *peer = data;
+
+        g_hash_table_destroy(peer->devices);
+        free(peer->host);
+        free(peer);
+    }
+}
+
+void
+free_stonith_remote_op_list(void)
+{
+    if (stonith_remote_op_list != NULL) {
+        g_hash_table_destroy(stonith_remote_op_list);
+        stonith_remote_op_list = NULL;
+    }
+}
+
+struct peer_count_data {
+    const remote_fencing_op_t *op;
+    gboolean verified_only;
+    uint32_t support_action_only;
+    int count;
+};
+
+/*!
+ * \internal
+ * \brief Increment a counter if a device has not been executed yet
+ *
+ * \param[in]     key        Device ID (ignored)
+ * \param[in]     value      Device properties
+ * \param[in,out] user_data  Peer count data
+ */
+static void
+count_peer_device(gpointer key, gpointer value, gpointer user_data)
+{
+    device_properties_t *props = (device_properties_t*)value;
+    struct peer_count_data *data = user_data;
+
+    if (!props->executed[data->op->phase]
+        && (!data->verified_only || props->verified)
+        && ((data->support_action_only == st_device_supports_none) || pcmk_is_set(props->device_support_flags, data->support_action_only))) {
+        ++(data->count);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Check the number of available devices in a peer's query results
+ *
+ * \param[in] op             Operation that results are for
+ * \param[in] peer           Peer to count
+ * \param[in] verified_only  Whether to count only verified devices
+ * \param[in] support_action_only Whether to count only devices that support action
+ *
+ * \return Number of devices available to peer that were not already executed
+ */
+static int
+count_peer_devices(const remote_fencing_op_t *op,
+                   const peer_device_info_t *peer, gboolean verified_only, uint32_t support_on_action_only)
+{
+    struct peer_count_data data;
+
+    data.op = op;
+    data.verified_only = verified_only;
+    data.support_action_only = support_on_action_only;
+    data.count = 0;
+    if (peer) {
+        g_hash_table_foreach(peer->devices, count_peer_device, &data);
+    }
+    return data.count;
+}
+
+/*!
+ * \internal
+ * \brief Search for a device in a query result
+ *
+ * \param[in] op      Operation that result is for
+ * \param[in] peer    Query result for a peer
+ * \param[in] device  Device ID to search for
+ *
+ * \return Device properties if found, NULL otherwise
+ */
+static device_properties_t *
+find_peer_device(const remote_fencing_op_t *op, const peer_device_info_t *peer,
+                 const char *device, uint32_t support_action_only)
+{
+    device_properties_t *props = g_hash_table_lookup(peer->devices, device);
+
+    if (props && support_action_only != st_device_supports_none && !pcmk_is_set(props->device_support_flags, support_action_only)) {
+        return NULL;
+    }
+    return (props && !props->executed[op->phase]
+           && !props->disallowed[op->phase])? props : NULL;
+}
+
+/*!
+ * \internal
+ * \brief Find a device in a peer's device list and mark it as executed
+ *
+ * \param[in]     op                     Operation that peer result is for
+ * \param[in,out] peer                   Peer with results to search
+ * \param[in]     device                 ID of device to mark as done
+ * \param[in]     verified_devices_only  Only consider verified devices
+ *
+ * \return TRUE if device was found and marked, FALSE otherwise
+ */
+static gboolean
+grab_peer_device(const remote_fencing_op_t *op, peer_device_info_t *peer,
+                 const char *device, gboolean verified_devices_only)
+{
+    device_properties_t *props = find_peer_device(op, peer, device,
+                                                  fenced_support_flag(op->action));
+
+    if ((props == NULL) || (verified_devices_only && !props->verified)) {
+        return FALSE;
+    }
+
+    crm_trace("Removing %s from %s (%d remaining)",
+              device, peer->host, count_peer_devices(op, peer, FALSE, st_device_supports_none));
+    props->executed[op->phase] = TRUE;
+    return TRUE;
+}
+
+static void
+clear_remote_op_timers(remote_fencing_op_t * op)
+{
+    if (op->query_timer) {
+        g_source_remove(op->query_timer);
+        op->query_timer = 0;
+    }
+    if (op->op_timer_total) {
+        g_source_remove(op->op_timer_total);
+        op->op_timer_total = 0;
+    }
+    if (op->op_timer_one) {
+        g_source_remove(op->op_timer_one);
+        op->op_timer_one = 0;
+    }
+}
+
+static void
+free_remote_op(gpointer data)
+{
+    remote_fencing_op_t *op = data;
+
+    crm_log_xml_debug(op->request, "Destroying");
+
+    clear_remote_op_timers(op);
+
+    free(op->id);
+    free(op->action);
+    free(op->delegate);
+    free(op->target);
+    free(op->client_id);
+    free(op->client_name);
+    free(op->originator);
+
+    if (op->query_results) {
+        g_list_free_full(op->query_results, free_remote_query);
+    }
+    if (op->request) {
+        free_xml(op->request);
+        op->request = NULL;
+    }
+    if (op->devices_list) {
+        g_list_free_full(op->devices_list, free);
+        op->devices_list = NULL;
+    }
+    g_list_free_full(op->automatic_list, free);
+    g_list_free(op->duplicates);
+
+    pcmk__reset_result(&op->result);
+    free(op);
+}
+
+void
+init_stonith_remote_op_hash_table(GHashTable **table)
+{
+    if (*table == NULL) {
+        *table = pcmk__strkey_table(NULL, free_remote_op);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Return an operation's originally requested action (before any remap)
+ *
+ * \param[in] op  Operation to check
+ *
+ * \return Operation's original action
+ */
+static const char *
+op_requested_action(const remote_fencing_op_t *op)
+{
+    return ((op->phase > st_phase_requested)? "reboot" : op->action);
+}
+
+/*!
+ * \internal
+ * \brief Remap a "reboot" operation to the "off" phase
+ *
+ * \param[in,out] op      Operation to remap
+ */
+static void
+op_phase_off(remote_fencing_op_t *op)
+{
+    crm_info("Remapping multiple-device reboot targeting %s to 'off' "
+             CRM_XS " id=%.8s", op->target, op->id);
+    op->phase = st_phase_off;
+
+    /* Happily, "off" and "on" are shorter than "reboot", so we can reuse the
+     * memory allocation at each phase.
+     */
+    strcpy(op->action, "off");
+}
+
+/*!
+ * \internal
+ * \brief Advance a remapped reboot operation to the "on" phase
+ *
+ * \param[in,out] op  Operation to remap
+ */
+static void
+op_phase_on(remote_fencing_op_t *op)
+{
+    GList *iter = NULL;
+
+    crm_info("Remapped 'off' targeting %s complete, "
+             "remapping to 'on' for %s " CRM_XS " id=%.8s",
+             op->target, op->client_name, op->id);
+    op->phase = st_phase_on;
+    strcpy(op->action, "on");
+
+    /* Skip devices with automatic unfencing, because the cluster will handle it
+     * when the node rejoins.
+     */
+    for (iter = op->automatic_list; iter != NULL; iter = iter->next) {
+        GList *match = g_list_find_custom(op->devices_list, iter->data,
+                                            sort_strings);
+
+        if (match) {
+            op->devices_list = g_list_remove(op->devices_list, match->data);
+        }
+    }
+    g_list_free_full(op->automatic_list, free);
+    op->automatic_list = NULL;
+
+    /* Rewind device list pointer */
+    op->devices = op->devices_list;
+}
+
+/*!
+ * \internal
+ * \brief Reset a remapped reboot operation
+ *
+ * \param[in,out] op  Operation to reset
+ */
+static void
+undo_op_remap(remote_fencing_op_t *op)
+{
+    if (op->phase > 0) {
+        crm_info("Undoing remap of reboot targeting %s for %s "
+                 CRM_XS " id=%.8s", op->target, op->client_name, op->id);
+        op->phase = st_phase_requested;
+        strcpy(op->action, "reboot");
+    }
+}
+
+/*!
+ * \internal
+ * \brief Create notification data XML for a fencing operation result
+ *
+ * \param[in] op      Fencer operation that completed
+ *
+ * \return Newly created XML to add as notification data
+ * \note The caller is responsible for freeing the result.
+ */
+static xmlNode *
+fencing_result2xml(const remote_fencing_op_t *op)
+{
+    xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);
+
+    crm_xml_add_int(notify_data, "state", op->state);
+    crm_xml_add(notify_data, F_STONITH_TARGET, op->target);
+    crm_xml_add(notify_data, F_STONITH_ACTION, op->action);
+    crm_xml_add(notify_data, F_STONITH_DELEGATE, op->delegate);
+    crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, op->id);
+    crm_xml_add(notify_data, F_STONITH_ORIGIN, op->originator);
+    crm_xml_add(notify_data, F_STONITH_CLIENTID, op->client_id);
+    crm_xml_add(notify_data, F_STONITH_CLIENTNAME, op->client_name);
+
+    return notify_data;
+}
+
+/*!
+ * \internal
+ * \brief Broadcast a fence result notification to all CPG peers
+ *
+ * \param[in] op         Fencer operation that completed
+ * \param[in] op_merged  Whether this operation is a duplicate of another
+ */
+void
+fenced_broadcast_op_result(const remote_fencing_op_t *op, bool op_merged)
+{
+    static int count = 0;
+    xmlNode *bcast = create_xml_node(NULL, T_STONITH_REPLY);
+    xmlNode *notify_data = fencing_result2xml(op);
+
+    count++;
+    crm_trace("Broadcasting result to peers");
+    crm_xml_add(bcast, F_TYPE, T_STONITH_NOTIFY);
+    crm_xml_add(bcast, F_SUBTYPE, "broadcast");
+    crm_xml_add(bcast, F_STONITH_OPERATION, T_STONITH_NOTIFY);
+    crm_xml_add_int(bcast, "count", count);
+
+    if (op_merged) {
+        pcmk__xe_set_bool_attr(bcast, F_STONITH_MERGED, true);
+    }
+
+    stonith__xe_set_result(notify_data, &op->result);
+
+    add_message_xml(bcast, F_STONITH_CALLDATA, notify_data);
+    send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE);
+    free_xml(notify_data);
+    free_xml(bcast);
+
+    return;
+}
+
+/*!
+ * \internal
+ * \brief Reply to a local request originator and notify all subscribed clients
+ *
+ * \param[in,out] op    Fencer operation that completed
+ * \param[in,out] data  Top-level XML to add notification to
+ */
+static void
+handle_local_reply_and_notify(remote_fencing_op_t *op, xmlNode *data)
+{
+    xmlNode *notify_data = NULL;
+    xmlNode *reply = NULL;
+    pcmk__client_t *client = NULL;
+
+    if (op->notify_sent == TRUE) {
+        /* nothing to do */
+        return;
+    }
+
+    /* Do notification with a clean data object */
+    crm_xml_add_int(data, "state", op->state);
+    crm_xml_add(data, F_STONITH_TARGET, op->target);
+    crm_xml_add(data, F_STONITH_OPERATION, op->action);
+
+    reply = fenced_construct_reply(op->request, data, &op->result);
+    crm_xml_add(reply, F_STONITH_DELEGATE, op->delegate);
+
+    /* Send fencing OP reply to local client that initiated fencing */
+    client = pcmk__find_client_by_id(op->client_id);
+    if (client == NULL) {
+        crm_trace("Skipping reply to %s: no longer a client", op->client_id);
+    } else {
+        do_local_reply(reply, client, op->call_options);
+    }
+
+    /* bcast to all local clients that the fencing operation happend */
+    notify_data = fencing_result2xml(op);
+    fenced_send_notification(T_STONITH_NOTIFY_FENCE, &op->result, notify_data);
+    free_xml(notify_data);
+    fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
+
+    /* mark this op as having notify's already sent */
+    op->notify_sent = TRUE;
+    free_xml(reply);
+}
+
+/*!
+ * \internal
+ * \brief Finalize all duplicates of a given fencer operation
+ *
+ * \param[in,out] op    Fencer operation that completed
+ * \param[in,out] data  Top-level XML to add notification to
+ */
+static void
+finalize_op_duplicates(remote_fencing_op_t *op, xmlNode *data)
+{
+    for (GList *iter = op->duplicates; iter != NULL; iter = iter->next) {
+        remote_fencing_op_t *other = iter->data;
+
+        if (other->state == st_duplicate) {
+            other->state = op->state;
+            crm_debug("Performing duplicate notification for %s@%s: %s "
+                      CRM_XS " id=%.8s",
+                      other->client_name, other->originator,
+                      pcmk_exec_status_str(op->result.execution_status),
+                      other->id);
+            pcmk__copy_result(&op->result, &other->result);
+            finalize_op(other, data, true);
+
+        } else {
+            // Possible if (for example) it timed out already
+            crm_err("Skipping duplicate notification for %s@%s "
+                    CRM_XS " state=%s id=%.8s",
+                    other->client_name, other->originator,
+                    stonith_op_state_str(other->state), other->id);
+        }
+    }
+}
+
+static char *
+delegate_from_xml(xmlNode *xml)
+{
+    xmlNode *match = get_xpath_object("//@" F_STONITH_DELEGATE, xml, LOG_NEVER);
+
+    if (match == NULL) {
+        return crm_element_value_copy(xml, F_ORIG);
+    } else {
+        return crm_element_value_copy(match, F_STONITH_DELEGATE);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Finalize a peer fencing operation
+ *
+ * Clean up after a fencing operation completes. This function has two code
+ * paths: the executioner uses it to broadcast the result to CPG peers, and then
+ * each peer (including the executioner) uses it to process that broadcast and
+ * notify its IPC clients of the result.
+ *
+ * \param[in,out] op      Fencer operation that completed
+ * \param[in,out] data    If not NULL, XML reply of last delegated operation
+ * \param[in]     dup     Whether this operation is a duplicate of another
+ *                        (in which case, do not broadcast the result)
+ *
+ *  \note The operation result should be set before calling this function.
+ */
+static void
+finalize_op(remote_fencing_op_t *op, xmlNode *data, bool dup)
+{
+    int level = LOG_ERR;
+    const char *subt = NULL;
+    xmlNode *local_data = NULL;
+    gboolean op_merged = FALSE;
+
+    CRM_CHECK((op != NULL), return);
+
+    // This is a no-op if timers have already been cleared
+    clear_remote_op_timers(op);
+
+    if (op->notify_sent) {
+        // Most likely, this is a timed-out action that eventually completed
+        crm_notice("Operation '%s'%s%s by %s for %s@%s%s: "
+                   "Result arrived too late " CRM_XS " id=%.8s",
+                   op->action, (op->target? " targeting " : ""),
+                   (op->target? op->target : ""),
+                   (op->delegate? op->delegate : "unknown node"),
+                   op->client_name, op->originator,
+                   (op_merged? " (merged)" : ""),
+                   op->id);
+        return;
+    }
+
+    set_fencing_completed(op);
+    undo_op_remap(op);
+
+    if (data == NULL) {
+        data = create_xml_node(NULL, "remote-op");
+        local_data = data;
+
+    } else if (op->delegate == NULL) {
+        switch (op->result.execution_status) {
+            case PCMK_EXEC_NO_FENCE_DEVICE:
+                break;
+
+            case PCMK_EXEC_INVALID:
+                if (op->result.exit_status != CRM_EX_EXPIRED) {
+                    op->delegate = delegate_from_xml(data);
+                }
+                break;
+
+            default:
+                op->delegate = delegate_from_xml(data);
+                break;
+        }
+    }
+
+    if (dup || (crm_element_value(data, F_STONITH_MERGED) != NULL)) {
+        op_merged = true;
+    }
+
+    /* Tell everyone the operation is done, we will continue
+     * with doing the local notifications once we receive
+     * the broadcast back. */
+    subt = crm_element_value(data, F_SUBTYPE);
+    if (!dup && !pcmk__str_eq(subt, "broadcast", pcmk__str_casei)) {
+        /* Defer notification until the bcast message arrives */
+        fenced_broadcast_op_result(op, op_merged);
+        free_xml(local_data);
+        return;
+    }
+
+    if (pcmk__result_ok(&op->result) || dup
+        || !pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
+        level = LOG_NOTICE;
+    }
+    do_crm_log(level, "Operation '%s'%s%s by %s for %s@%s%s: %s (%s%s%s) "
+               CRM_XS " id=%.8s", op->action, (op->target? " targeting " : ""),
+               (op->target? op->target : ""),
+               (op->delegate? op->delegate : "unknown node"),
+               op->client_name, op->originator,
+               (op_merged? " (merged)" : ""),
+               crm_exit_str(op->result.exit_status),
+               pcmk_exec_status_str(op->result.execution_status),
+               ((op->result.exit_reason == NULL)? "" : ": "),
+               ((op->result.exit_reason == NULL)? "" : op->result.exit_reason),
+               op->id);
+
+    handle_local_reply_and_notify(op, data);
+
+    if (!dup) {
+        finalize_op_duplicates(op, data);
+    }
+
+    /* Free non-essential parts of the record
+     * Keep the record around so we can query the history
+     */
+    if (op->query_results) {
+        g_list_free_full(op->query_results, free_remote_query);
+        op->query_results = NULL;
+    }
+    if (op->request) {
+        free_xml(op->request);
+        op->request = NULL;
+    }
+
+    free_xml(local_data);
+}
+
+/*!
+ * \internal
+ * \brief Finalize a watchdog fencer op after the waiting time expires
+ *
+ * \param[in,out] userdata  Fencer operation that completed
+ *
+ * \return G_SOURCE_REMOVE (which tells glib not to restart timer)
+ */
+static gboolean
+remote_op_watchdog_done(gpointer userdata)
+{
+    remote_fencing_op_t *op = userdata;
+
+    op->op_timer_one = 0;
+
+    crm_notice("Self-fencing (%s) by %s for %s assumed complete "
+               CRM_XS " id=%.8s",
+               op->action, op->target, op->client_name, op->id);
+    op->state = st_done;
+    pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    finalize_op(op, NULL, false);
+    return G_SOURCE_REMOVE;
+}
+
+static gboolean
+remote_op_timeout_one(gpointer userdata)
+{
+    remote_fencing_op_t *op = userdata;
+
+    op->op_timer_one = 0;
+
+    crm_notice("Peer's '%s' action targeting %s for client %s timed out " CRM_XS
+               " id=%.8s", op->action, op->target, op->client_name, op->id);
+    pcmk__set_result(&op->result, CRM_EX_ERROR, PCMK_EXEC_TIMEOUT,
+                     "Peer did not return fence result within timeout");
+
+    // The requested delay has been applied for the first device
+    if (op->delay > 0) {
+        op->delay = 0;
+        crm_trace("Try another device for '%s' action targeting %s "
+                  "for client %s without delay " CRM_XS " id=%.8s",
+                  op->action, op->target, op->client_name, op->id);
+    }
+
+    // Try another device, if appropriate
+    request_peer_fencing(op, NULL);
+    return G_SOURCE_REMOVE;
+}
+
+/*!
+ * \internal
+ * \brief Finalize a remote fencer operation that timed out
+ *
+ * \param[in,out] op      Fencer operation that timed out
+ * \param[in]     reason  Readable description of what step timed out
+ */
+static void
+finalize_timed_out_op(remote_fencing_op_t *op, const char *reason)
+{
+    crm_debug("Action '%s' targeting %s for client %s timed out "
+              CRM_XS " id=%.8s",
+              op->action, op->target, op->client_name, op->id);
+
+    if (op->phase == st_phase_on) {
+        /* A remapped reboot operation timed out in the "on" phase, but the
+         * "off" phase completed successfully, so quit trying any further
+         * devices, and return success.
+         */
+        op->state = st_done;
+        pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    } else {
+        op->state = st_failed;
+        pcmk__set_result(&op->result, CRM_EX_ERROR, PCMK_EXEC_TIMEOUT, reason);
+    }
+    finalize_op(op, NULL, false);
+}
+
+/*!
+ * \internal
+ * \brief Finalize a remote fencer operation that timed out
+ *
+ * \param[in,out] userdata  Fencer operation that timed out
+ *
+ * \return G_SOURCE_REMOVE (which tells glib not to restart timer)
+ */
+static gboolean
+remote_op_timeout(gpointer userdata)
+{
+    remote_fencing_op_t *op = userdata;
+
+    op->op_timer_total = 0;
+
+    if (op->state == st_done) {
+        crm_debug("Action '%s' targeting %s for client %s already completed "
+                  CRM_XS " id=%.8s",
+                  op->action, op->target, op->client_name, op->id);
+    } else {
+        finalize_timed_out_op(userdata, "Fencing did not complete within a "
+                                        "total timeout based on the "
+                                        "configured timeout and retries for "
+                                        "any devices attempted");
+    }
+    return G_SOURCE_REMOVE;
+}
+
+static gboolean
+remote_op_query_timeout(gpointer data)
+{
+    remote_fencing_op_t *op = data;
+
+    op->query_timer = 0;
+
+    if (op->state == st_done) {
+        crm_debug("Operation %.8s targeting %s already completed",
+                  op->id, op->target);
+    } else if (op->state == st_exec) {
+        crm_debug("Operation %.8s targeting %s already in progress",
+                  op->id, op->target);
+    } else if (op->query_results) {
+        // Query succeeded, so attempt the actual fencing
+        crm_debug("Query %.8s targeting %s complete (state=%s)",
+                  op->id, op->target, stonith_op_state_str(op->state));
+        request_peer_fencing(op, NULL);
+    } else {
+        crm_debug("Query %.8s targeting %s timed out (state=%s)",
+                  op->id, op->target, stonith_op_state_str(op->state));
+        finalize_timed_out_op(op, "No capable peers replied to device query "
+                                  "within timeout");
+    }
+
+    return G_SOURCE_REMOVE;
+}
+
+static gboolean
+topology_is_empty(stonith_topology_t *tp)
+{
+    int i;
+
+    if (tp == NULL) {
+        return TRUE;
+    }
+
+    for (i = 0; i < ST_LEVEL_MAX; i++) {
+        if (tp->levels[i] != NULL) {
+            return FALSE;
+        }
+    }
+    return TRUE;
+}
+
+/*!
+ * \internal
+ * \brief Add a device to an operation's automatic unfencing list
+ *
+ * \param[in,out] op      Operation to modify
+ * \param[in]     device  Device ID to add
+ */
+static void
+add_required_device(remote_fencing_op_t *op, const char *device)
+{
+    GList *match  = g_list_find_custom(op->automatic_list, device,
+                                         sort_strings);
+
+    if (!match) {
+        op->automatic_list = g_list_prepend(op->automatic_list, strdup(device));
+    }
+}
+
+/*!
+ * \internal
+ * \brief Remove a device from the automatic unfencing list
+ *
+ * \param[in,out] op      Operation to modify
+ * \param[in]     device  Device ID to remove
+ */
+static void
+remove_required_device(remote_fencing_op_t *op, const char *device)
+{
+    GList *match = g_list_find_custom(op->automatic_list, device,
+                                        sort_strings);
+
+    if (match) {
+        op->automatic_list = g_list_remove(op->automatic_list, match->data);
+    }
+}
+
+/* deep copy the device list */
+static void
+set_op_device_list(remote_fencing_op_t * op, GList *devices)
+{
+    GList *lpc = NULL;
+
+    if (op->devices_list) {
+        g_list_free_full(op->devices_list, free);
+        op->devices_list = NULL;
+    }
+    for (lpc = devices; lpc != NULL; lpc = lpc->next) {
+        op->devices_list = g_list_append(op->devices_list, strdup(lpc->data));
+    }
+    op->devices = op->devices_list;
+}
+
+/*!
+ * \internal
+ * \brief Check whether a node matches a topology target
+ *
+ * \param[in] tp    Topology table entry to check
+ * \param[in] node  Name of node to check
+ *
+ * \return TRUE if node matches topology target
+ */
+static gboolean
+topology_matches(const stonith_topology_t *tp, const char *node)
+{
+    regex_t r_patt;
+
+    CRM_CHECK(node && tp && tp->target, return FALSE);
+    switch (tp->kind) {
+        case fenced_target_by_attribute:
+            /* This level targets by attribute, so tp->target is a NAME=VALUE pair
+             * of a permanent attribute applied to targeted nodes. The test below
+             * relies on the locally cached copy of the CIB, so if fencing needs to
+             * be done before the initial CIB is received or after a malformed CIB
+             * is received, then the topology will be unable to be used.
+             */
+            if (node_has_attr(node, tp->target_attribute, tp->target_value)) {
+                crm_notice("Matched %s with %s by attribute", node, tp->target);
+                return TRUE;
+            }
+            break;
+
+        case fenced_target_by_pattern:
+            /* This level targets node names matching a pattern, so tp->target
+             * (and tp->target_pattern) is a regular expression.
+             */
+            if (regcomp(&r_patt, tp->target_pattern, REG_EXTENDED|REG_NOSUB)) {
+                crm_info("Bad regex '%s' for fencing level", tp->target);
+            } else {
+                int status = regexec(&r_patt, node, 0, NULL, 0);
+
+                regfree(&r_patt);
+                if (status == 0) {
+                    crm_notice("Matched %s with %s by name", node, tp->target);
+                    return TRUE;
+                }
+            }
+            break;
+
+        case fenced_target_by_name:
+            crm_trace("Testing %s against %s", node, tp->target);
+            return pcmk__str_eq(tp->target, node, pcmk__str_casei);
+
+        default:
+            break;
+    }
+    crm_trace("No match for %s with %s", node, tp->target);
+    return FALSE;
+}
+
+stonith_topology_t *
+find_topology_for_host(const char *host) 
+{
+    GHashTableIter tIter;
+    stonith_topology_t *tp = g_hash_table_lookup(topology, host);
+
+    if(tp != NULL) {
+        crm_trace("Found %s for %s in %d entries", tp->target, host, g_hash_table_size(topology));
+        return tp;
+    }
+
+    g_hash_table_iter_init(&tIter, topology);
+    while (g_hash_table_iter_next(&tIter, NULL, (gpointer *) & tp)) {
+        if (topology_matches(tp, host)) {
+            crm_trace("Found %s for %s in %d entries", tp->target, host, g_hash_table_size(topology));
+            return tp;
+        }
+    }
+
+    crm_trace("No matches for %s in %d topology entries", host, g_hash_table_size(topology));
+    return NULL;
+}
+
+/*!
+ * \internal
+ * \brief Set fencing operation's device list to target's next topology level
+ *
+ * \param[in,out] op        Remote fencing operation to modify
+ * \param[in]     empty_ok  If true, an operation without a target (i.e.
+ *                          queries) or a target without a topology will get a
+ *                          pcmk_rc_ok return value instead of ENODEV
+ *
+ * \return Standard Pacemaker return value
+ */
+static int
+advance_topology_level(remote_fencing_op_t *op, bool empty_ok)
+{
+    stonith_topology_t *tp = NULL;
+
+    if (op->target) {
+        tp = find_topology_for_host(op->target);
+    }
+    if (topology_is_empty(tp)) {
+        return empty_ok? pcmk_rc_ok : ENODEV;
+    }
+
+    CRM_ASSERT(tp->levels != NULL);
+
+    stonith__set_call_options(op->call_options, op->id, st_opt_topology);
+
+    /* This is a new level, so undo any remapping left over from previous */
+    undo_op_remap(op);
+
+    do {
+        op->level++;
+
+    } while (op->level < ST_LEVEL_MAX && tp->levels[op->level] == NULL);
+
+    if (op->level < ST_LEVEL_MAX) {
+        crm_trace("Attempting fencing level %d targeting %s (%d devices) "
+                  "for client %s@%s (id=%.8s)",
+                  op->level, op->target, g_list_length(tp->levels[op->level]),
+                  op->client_name, op->originator, op->id);
+        set_op_device_list(op, tp->levels[op->level]);
+
+        // The requested delay has been applied for the first fencing level
+        if (op->level > 1 && op->delay > 0) {
+            op->delay = 0;
+        }
+
+        if ((g_list_next(op->devices_list) != NULL)
+            && pcmk__str_eq(op->action, "reboot", pcmk__str_none)) {
+            /* A reboot has been requested for a topology level with multiple
+             * devices. Instead of rebooting the devices sequentially, we will
+             * turn them all off, then turn them all on again. (Think about
+             * switched power outlets for redundant power supplies.)
+             */
+            op_phase_off(op);
+        }
+        return pcmk_rc_ok;
+    }
+
+    crm_info("All %sfencing options targeting %s for client %s@%s failed "
+             CRM_XS " id=%.8s",
+             (stonith_watchdog_timeout_ms > 0)?"non-watchdog ":"",
+             op->target, op->client_name, op->originator, op->id);
+    return ENODEV;
+}
+
+/*!
+ * \internal
+ * \brief If fencing operation is a duplicate, merge it into the other one
+ *
+ * \param[in,out] op  Fencing operation to check
+ */
+static void
+merge_duplicates(remote_fencing_op_t *op)
+{
+    GHashTableIter iter;
+    remote_fencing_op_t *other = NULL;
+
+    time_t now = time(NULL);
+
+    g_hash_table_iter_init(&iter, stonith_remote_op_list);
+    while (g_hash_table_iter_next(&iter, NULL, (void **)&other)) {
+        const char *other_action = op_requested_action(other);
+
+        if (!strcmp(op->id, other->id)) {
+            continue; // Don't compare against self
+        }
+        if (other->state > st_exec) {
+            crm_trace("%.8s not duplicate of %.8s: not in progress",
+                      op->id, other->id);
+            continue;
+        }
+        if (!pcmk__str_eq(op->target, other->target, pcmk__str_casei)) {
+            crm_trace("%.8s not duplicate of %.8s: node %s vs. %s",
+                      op->id, other->id, op->target, other->target);
+            continue;
+        }
+        if (!pcmk__str_eq(op->action, other_action, pcmk__str_none)) {
+            crm_trace("%.8s not duplicate of %.8s: action %s vs. %s",
+                      op->id, other->id, op->action, other_action);
+            continue;
+        }
+        if (pcmk__str_eq(op->client_name, other->client_name, pcmk__str_casei)) {
+            crm_trace("%.8s not duplicate of %.8s: same client %s",
+                      op->id, other->id, op->client_name);
+            continue;
+        }
+        if (pcmk__str_eq(other->target, other->originator, pcmk__str_casei)) {
+            crm_trace("%.8s not duplicate of %.8s: suicide for %s",
+                      op->id, other->id, other->target);
+            continue;
+        }
+        if (!fencing_peer_active(crm_get_peer(0, other->originator))) {
+            crm_notice("Failing action '%s' targeting %s originating from "
+                       "client %s@%s: Originator is dead " CRM_XS " id=%.8s",
+                       other->action, other->target, other->client_name,
+                       other->originator, other->id);
+            crm_trace("%.8s not duplicate of %.8s: originator dead",
+                      op->id, other->id);
+            other->state = st_failed;
+            continue;
+        }
+        if ((other->total_timeout > 0)
+            && (now > (other->total_timeout + other->created))) {
+            crm_trace("%.8s not duplicate of %.8s: old (%ld vs. %ld + %d)",
+                      op->id, other->id, now, other->created,
+                      other->total_timeout);
+            continue;
+        }
+
+        /* There is another in-flight request to fence the same host
+         * Piggyback on that instead.  If it fails, so do we.
+         */
+        other->duplicates = g_list_append(other->duplicates, op);
+        if (other->total_timeout == 0) {
+            other->total_timeout = op->total_timeout =
+                TIMEOUT_MULTIPLY_FACTOR * get_op_total_timeout(op, NULL);
+            crm_trace("Best guess as to timeout used for %.8s: %d",
+                      other->id, other->total_timeout);
+        }
+        crm_notice("Merging fencing action '%s' targeting %s originating from "
+                   "client %s with identical request from %s@%s "
+                   CRM_XS " original=%.8s duplicate=%.8s total_timeout=%ds",
+                   op->action, op->target, op->client_name,
+                   other->client_name, other->originator,
+                   op->id, other->id, other->total_timeout);
+        report_timeout_period(op, other->total_timeout);
+        op->state = st_duplicate;
+    }
+}
+
+static uint32_t fencing_active_peers(void)
+{
+    uint32_t count = 0;
+    crm_node_t *entry;
+    GHashTableIter gIter;
+
+    g_hash_table_iter_init(&gIter, crm_peer_cache);
+    while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
+        if(fencing_peer_active(entry)) {
+            count++;
+        }
+    }
+    return count;
+}
+
+/*!
+ * \internal
+ * \brief Process a manual confirmation of a pending fence action
+ *
+ * \param[in]     client  IPC client that sent confirmation
+ * \param[in,out] msg     Request XML with manual confirmation
+ *
+ * \return Standard Pacemaker return code
+ */
+int
+fenced_handle_manual_confirmation(const pcmk__client_t *client, xmlNode *msg)
+{
+    remote_fencing_op_t *op = NULL;
+    xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR);
+
+    CRM_CHECK(dev != NULL, return EPROTO);
+
+    crm_notice("Received manual confirmation that %s has been fenced",
+               pcmk__s(crm_element_value(dev, F_STONITH_TARGET),
+                       "unknown target"));
+    op = initiate_remote_stonith_op(client, msg, TRUE);
+    if (op == NULL) {
+        return EPROTO;
+    }
+    op->state = st_done;
+    set_fencing_completed(op);
+    op->delegate = strdup("a human");
+
+    // For the fencer's purposes, the fencing operation is done
+    pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    finalize_op(op, msg, false);
+
+    /* For the requester's purposes, the operation is still pending. The
+     * actual result will be sent asynchronously via the operation's done_cb().
+     */
+    return EINPROGRESS;
+}
+
+/*!
+ * \internal
+ * \brief Create a new remote stonith operation
+ *
+ * \param[in] client   ID of local stonith client that initiated the operation
+ * \param[in] request  The request from the client that started the operation
+ * \param[in] peer     TRUE if this operation is owned by another stonith peer
+ *                     (an operation owned by one peer is stored on all peers,
+ *                     but only the owner executes it; all nodes get the results
+ *                     once the owner finishes execution)
+ */
+void *
+create_remote_stonith_op(const char *client, xmlNode *request, gboolean peer)
+{
+    remote_fencing_op_t *op = NULL;
+    xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_NEVER);
+    int call_options = 0;
+    const char *operation = NULL;
+
+    init_stonith_remote_op_hash_table(&stonith_remote_op_list);
+
+    /* If this operation is owned by another node, check to make
+     * sure we haven't already created this operation. */
+    if (peer && dev) {
+        const char *op_id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
+
+        CRM_CHECK(op_id != NULL, return NULL);
+
+        op = g_hash_table_lookup(stonith_remote_op_list, op_id);
+        if (op) {
+            crm_debug("Reusing existing remote fencing op %.8s for %s",
+                      op_id, ((client == NULL)? "unknown client" : client));
+            return op;
+        }
+    }
+
+    op = calloc(1, sizeof(remote_fencing_op_t));
+    CRM_ASSERT(op != NULL);
+
+    crm_element_value_int(request, F_STONITH_TIMEOUT, &(op->base_timeout));
+    // Value -1 means disable any static/random fencing delays
+    crm_element_value_int(request, F_STONITH_DELAY, &(op->delay));
+
+    if (peer && dev) {
+        op->id = crm_element_value_copy(dev, F_STONITH_REMOTE_OP_ID);
+    } else {
+        op->id = crm_generate_uuid();
+    }
+
+    g_hash_table_replace(stonith_remote_op_list, op->id, op);
+
+    op->state = st_query;
+    op->replies_expected = fencing_active_peers();
+    op->action = crm_element_value_copy(dev, F_STONITH_ACTION);
+    op->originator = crm_element_value_copy(dev, F_STONITH_ORIGIN);
+    op->delegate = crm_element_value_copy(dev, F_STONITH_DELEGATE); /* May not be set */
+    op->created = time(NULL);
+
+    if (op->originator == NULL) {
+        /* Local or relayed request */
+        op->originator = strdup(stonith_our_uname);
+    }
+
+    CRM_LOG_ASSERT(client != NULL);
+    if (client) {
+        op->client_id = strdup(client);
+    }
+
+
+    /* For a RELAY operation, set fenced on the client. */
+    operation = crm_element_value(request, F_STONITH_OPERATION);
+
+    if (pcmk__str_eq(operation, STONITH_OP_RELAY, pcmk__str_none)) {
+        op->client_name = crm_strdup_printf("%s.%lu", crm_system_name,
+                                         (unsigned long) getpid());
+    } else {
+        op->client_name = crm_element_value_copy(request, F_STONITH_CLIENTNAME);
+    }
+
+    op->target = crm_element_value_copy(dev, F_STONITH_TARGET);
+    op->request = copy_xml(request);    /* TODO: Figure out how to avoid this */
+    crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
+    op->call_options = call_options;
+
+    crm_element_value_int(request, F_STONITH_CALLID, &(op->client_callid));
+
+    crm_trace("%s new fencing op %s ('%s' targeting %s for client %s, "
+              "base timeout %d, %u %s expected)",
+              (peer && dev)? "Recorded" : "Generated", op->id, op->action,
+              op->target, op->client_name, op->base_timeout,
+              op->replies_expected,
+              pcmk__plural_alt(op->replies_expected, "reply", "replies"));
+
+    if (op->call_options & st_opt_cs_nodeid) {
+        int nodeid;
+        crm_node_t *node;
+
+        pcmk__scan_min_int(op->target, &nodeid, 0);
+        node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY);
+
+        /* Ensure the conversion only happens once */
+        stonith__clear_call_options(op->call_options, op->id, st_opt_cs_nodeid);
+
+        if (node && node->uname) {
+            free(op->target);
+            op->target = strdup(node->uname);
+
+        } else {
+            crm_warn("Could not expand nodeid '%s' into a host name", op->target);
+        }
+    }
+
+    /* check to see if this is a duplicate operation of another in-flight operation */
+    merge_duplicates(op);
+
+    if (op->state != st_duplicate) {
+        /* kick history readers */
+        fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
+    }
+
+    /* safe to trim as long as that doesn't touch pending ops */
+    stonith_fence_history_trim();
+
+    return op;
+}
+
+/*!
+ * \internal
+ * \brief Create a peer fencing operation from a request, and initiate it
+ *
+ * \param[in] client     IPC client that made request (NULL to get from request)
+ * \param[in] request    Request XML
+ * \param[in] manual_ack Whether this is a manual action confirmation
+ *
+ * \return Newly created operation on success, otherwise NULL
+ */
+remote_fencing_op_t *
+initiate_remote_stonith_op(const pcmk__client_t *client, xmlNode *request,
+                           gboolean manual_ack)
+{
+    int query_timeout = 0;
+    xmlNode *query = NULL;
+    const char *client_id = NULL;
+    remote_fencing_op_t *op = NULL;
+    const char *relay_op_id = NULL;
+    const char *operation = NULL;
+
+    if (client) {
+        client_id = client->id;
+    } else {
+        client_id = crm_element_value(request, F_STONITH_CLIENTID);
+    }
+
+    CRM_LOG_ASSERT(client_id != NULL);
+    op = create_remote_stonith_op(client_id, request, FALSE);
+    op->owner = TRUE;
+    if (manual_ack) {
+        return op;
+    }
+
+    CRM_CHECK(op->action, return NULL);
+
+    if (advance_topology_level(op, true) != pcmk_rc_ok) {
+        op->state = st_failed;
+    }
+
+    switch (op->state) {
+        case st_failed:
+            // advance_topology_level() exhausted levels
+            pcmk__set_result(&op->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
+                             "All topology levels failed");
+            crm_warn("Could not request peer fencing (%s) targeting %s "
+                     CRM_XS " id=%.8s", op->action, op->target, op->id);
+            finalize_op(op, NULL, false);
+            return op;
+
+        case st_duplicate:
+            crm_info("Requesting peer fencing (%s) targeting %s (duplicate) "
+                     CRM_XS " id=%.8s", op->action, op->target, op->id);
+            return op;
+
+        default:
+            crm_notice("Requesting peer fencing (%s) targeting %s "
+                       CRM_XS " id=%.8s state=%s base_timeout=%d",
+                       op->action, op->target, op->id,
+                       stonith_op_state_str(op->state), op->base_timeout);
+    }
+
+    query = stonith_create_op(op->client_callid, op->id, STONITH_OP_QUERY,
+                              NULL, op->call_options);
+
+    crm_xml_add(query, F_STONITH_REMOTE_OP_ID, op->id);
+    crm_xml_add(query, F_STONITH_TARGET, op->target);
+    crm_xml_add(query, F_STONITH_ACTION, op_requested_action(op));
+    crm_xml_add(query, F_STONITH_ORIGIN, op->originator);
+    crm_xml_add(query, F_STONITH_CLIENTID, op->client_id);
+    crm_xml_add(query, F_STONITH_CLIENTNAME, op->client_name);
+    crm_xml_add_int(query, F_STONITH_TIMEOUT, op->base_timeout);
+
+    /* In case of RELAY operation, RELAY information is added to the query to delete the original operation of RELAY. */
+    operation = crm_element_value(request, F_STONITH_OPERATION);
+    if (pcmk__str_eq(operation, STONITH_OP_RELAY, pcmk__str_none)) {
+        relay_op_id = crm_element_value(request, F_STONITH_REMOTE_OP_ID);
+        if (relay_op_id) {
+            crm_xml_add(query, F_STONITH_REMOTE_OP_ID_RELAY, relay_op_id);
+        }
+    }
+
+    send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE);
+    free_xml(query);
+
+    query_timeout = op->base_timeout * TIMEOUT_MULTIPLY_FACTOR;
+    op->query_timer = g_timeout_add((1000 * query_timeout), remote_op_query_timeout, op);
+
+    return op;
+}
+
+enum find_best_peer_options {
+    /*! Skip checking the target peer for capable fencing devices */
+    FIND_PEER_SKIP_TARGET = 0x0001,
+    /*! Only check the target peer for capable fencing devices */
+    FIND_PEER_TARGET_ONLY = 0x0002,
+    /*! Skip peers and devices that are not verified */
+    FIND_PEER_VERIFIED_ONLY = 0x0004,
+};
+
+static peer_device_info_t *
+find_best_peer(const char *device, remote_fencing_op_t * op, enum find_best_peer_options options)
+{
+    GList *iter = NULL;
+    gboolean verified_devices_only = (options & FIND_PEER_VERIFIED_ONLY) ? TRUE : FALSE;
+
+    if (!device && pcmk_is_set(op->call_options, st_opt_topology)) {
+        return NULL;
+    }
+
+    for (iter = op->query_results; iter != NULL; iter = iter->next) {
+        peer_device_info_t *peer = iter->data;
+
+        crm_trace("Testing result from %s targeting %s with %d device%s: %d %x",
+                  peer->host, op->target, peer->ndevices,
+                  pcmk__plural_s(peer->ndevices), peer->tried, options);
+        if ((options & FIND_PEER_SKIP_TARGET) && pcmk__str_eq(peer->host, op->target, pcmk__str_casei)) {
+            continue;
+        }
+        if ((options & FIND_PEER_TARGET_ONLY) && !pcmk__str_eq(peer->host, op->target, pcmk__str_casei)) {
+            continue;
+        }
+
+        if (pcmk_is_set(op->call_options, st_opt_topology)) {
+
+            if (grab_peer_device(op, peer, device, verified_devices_only)) {
+                return peer;
+            }
+
+        } else if (!peer->tried
+                   && count_peer_devices(op, peer, verified_devices_only,
+                                         fenced_support_flag(op->action))) {
+            /* No topology: Use the current best peer */
+            crm_trace("Simple fencing");
+            return peer;
+        }
+    }
+
+    return NULL;
+}
+
+static peer_device_info_t *
+stonith_choose_peer(remote_fencing_op_t * op)
+{
+    const char *device = NULL;
+    peer_device_info_t *peer = NULL;
+    uint32_t active = fencing_active_peers();
+
+    do {
+        if (op->devices) {
+            device = op->devices->data;
+            crm_trace("Checking for someone to fence (%s) %s using %s",
+                      op->action, op->target, device);
+        } else {
+            crm_trace("Checking for someone to fence (%s) %s",
+                      op->action, op->target);
+        }
+
+        /* Best choice is a peer other than the target with verified access */
+        peer = find_best_peer(device, op, FIND_PEER_SKIP_TARGET|FIND_PEER_VERIFIED_ONLY);
+        if (peer) {
+            crm_trace("Found verified peer %s for %s", peer->host, device?device:"<any>");
+            return peer;
+        }
+
+        if(op->query_timer != 0 && op->replies < QB_MIN(op->replies_expected, active)) {
+            crm_trace("Waiting before looking for unverified devices to fence %s", op->target);
+            return NULL;
+        }
+
+        /* If no other peer has verified access, next best is unverified access */
+        peer = find_best_peer(device, op, FIND_PEER_SKIP_TARGET);
+        if (peer) {
+            crm_trace("Found best unverified peer %s", peer->host);
+            return peer;
+        }
+
+        /* If no other peer can do it, last option is self-fencing
+         * (which is never allowed for the "on" phase of a remapped reboot)
+         */
+        if (op->phase != st_phase_on) {
+            peer = find_best_peer(device, op, FIND_PEER_TARGET_ONLY);
+            if (peer) {
+                crm_trace("%s will fence itself", peer->host);
+                return peer;
+            }
+        }
+
+        /* Try the next fencing level if there is one (unless we're in the "on"
+         * phase of a remapped "reboot", because we ignore errors in that case)
+         */
+    } while ((op->phase != st_phase_on)
+             && pcmk_is_set(op->call_options, st_opt_topology)
+             && (advance_topology_level(op, false) == pcmk_rc_ok));
+
+    if ((stonith_watchdog_timeout_ms > 0)
+        && pcmk__is_fencing_action(op->action)
+        && pcmk__str_eq(device, STONITH_WATCHDOG_ID, pcmk__str_none)
+        && node_does_watchdog_fencing(op->target)) {
+        crm_info("Couldn't contact watchdog-fencing target-node (%s)",
+                 op->target);
+        /* check_watchdog_fencing_and_wait will log additional info */
+    } else {
+        crm_notice("Couldn't find anyone to fence (%s) %s using %s",
+                   op->action, op->target, (device? device : "any device"));
+    }
+    return NULL;
+}
+
+static int
+get_device_timeout(const remote_fencing_op_t *op,
+                   const peer_device_info_t *peer, const char *device,
+                   bool with_delay)
+{
+    device_properties_t *props;
+    int delay = 0;
+
+    if (!peer || !device) {
+        return op->base_timeout;
+    }
+
+    props = g_hash_table_lookup(peer->devices, device);
+    if (!props) {
+        return op->base_timeout;
+    }
+
+    // op->delay < 0 means disable any static/random fencing delays
+    if (with_delay && op->delay >= 0) {
+        // delay_base is eventually limited by delay_max
+        delay = (props->delay_max[op->phase] > 0 ?
+                 props->delay_max[op->phase] : props->delay_base[op->phase]);
+    }
+
+    return (props->custom_action_timeout[op->phase]?
+            props->custom_action_timeout[op->phase] : op->base_timeout)
+           + delay;
+}
+
+struct timeout_data {
+    const remote_fencing_op_t *op;
+    const peer_device_info_t *peer;
+    int total_timeout;
+};
+
+/*!
+ * \internal
+ * \brief Add timeout to a total if device has not been executed yet
+ *
+ * \param[in]     key        GHashTable key (device ID)
+ * \param[in]     value      GHashTable value (device properties)
+ * \param[in,out] user_data  Timeout data
+ */
+static void
+add_device_timeout(gpointer key, gpointer value, gpointer user_data)
+{
+    const char *device_id = key;
+    device_properties_t *props = value;
+    struct timeout_data *timeout = user_data;
+
+    if (!props->executed[timeout->op->phase]
+        && !props->disallowed[timeout->op->phase]) {
+        timeout->total_timeout += get_device_timeout(timeout->op, timeout->peer,
+                                                     device_id, true);
+    }
+}
+
+static int
+get_peer_timeout(const remote_fencing_op_t *op, const peer_device_info_t *peer)
+{
+    struct timeout_data timeout;
+
+    timeout.op = op;
+    timeout.peer = peer;
+    timeout.total_timeout = 0;
+
+    g_hash_table_foreach(peer->devices, add_device_timeout, &timeout);
+
+    return (timeout.total_timeout? timeout.total_timeout : op->base_timeout);
+}
+
+static int
+get_op_total_timeout(const remote_fencing_op_t *op,
+                     const peer_device_info_t *chosen_peer)
+{
+    int total_timeout = 0;
+    stonith_topology_t *tp = find_topology_for_host(op->target);
+
+    if (pcmk_is_set(op->call_options, st_opt_topology) && tp) {
+        int i;
+        GList *device_list = NULL;
+        GList *iter = NULL;
+        GList *auto_list = NULL;
+
+        if (pcmk__str_eq(op->action, "on", pcmk__str_none)
+            && (op->automatic_list != NULL)) {
+            auto_list = g_list_copy(op->automatic_list);
+        }
+
+        /* Yep, this looks scary, nested loops all over the place.
+         * Here is what is going on.
+         * Loop1: Iterate through fencing levels.
+         * Loop2: If a fencing level has devices, loop through each device
+         * Loop3: For each device in a fencing level, see what peer owns it
+         *        and what that peer has reported the timeout is for the device.
+         */
+        for (i = 0; i < ST_LEVEL_MAX; i++) {
+            if (!tp->levels[i]) {
+                continue;
+            }
+            for (device_list = tp->levels[i]; device_list; device_list = device_list->next) {
+                /* in case of watchdog-device we add the timeout to the budget
+                   regardless of if we got a reply or not
+                 */
+                if ((stonith_watchdog_timeout_ms > 0)
+                    && pcmk__is_fencing_action(op->action)
+                    && pcmk__str_eq(device_list->data, STONITH_WATCHDOG_ID,
+                                    pcmk__str_none)
+                    && node_does_watchdog_fencing(op->target)) {
+                    total_timeout += stonith_watchdog_timeout_ms / 1000;
+                    continue;
+                }
+
+                for (iter = op->query_results; iter != NULL; iter = iter->next) {
+                    const peer_device_info_t *peer = iter->data;
+
+                    if (auto_list) {
+                        GList *match = g_list_find_custom(auto_list, device_list->data,
+                                        sort_strings);
+                        if (match) {
+                            auto_list = g_list_remove(auto_list, match->data);
+                        }
+                    }
+
+                    if (find_peer_device(op, peer, device_list->data,
+                                         fenced_support_flag(op->action))) {
+                        total_timeout += get_device_timeout(op, peer,
+                                                            device_list->data,
+                                                            true);
+                        break;
+                    }
+                }               /* End Loop3: match device with peer that owns device, find device's timeout period */
+            }                   /* End Loop2: iterate through devices at a specific level */
+        }                       /*End Loop1: iterate through fencing levels */
+
+        //Add only exists automatic_list device timeout
+        if (auto_list) {
+            for (iter = auto_list; iter != NULL; iter = iter->next) {
+                GList *iter2 = NULL;
+
+                for (iter2 = op->query_results; iter2 != NULL; iter = iter2->next) {
+                    peer_device_info_t *peer = iter2->data;
+                    if (find_peer_device(op, peer, iter->data, st_device_supports_on)) {
+                        total_timeout += get_device_timeout(op, peer,
+                                                            iter->data, true);
+                        break;
+                    }
+                }
+            }
+        }
+
+        g_list_free(auto_list);
+
+    } else if (chosen_peer) {
+        total_timeout = get_peer_timeout(op, chosen_peer);
+    } else {
+        total_timeout = op->base_timeout;
+    }
+
+    /* Take any requested fencing delay into account to prevent it from eating
+     * up the total timeout.
+     */
+    return ((total_timeout ? total_timeout : op->base_timeout)
+            + (op->delay > 0 ? op->delay : 0));
+}
+
+static void
+report_timeout_period(remote_fencing_op_t * op, int op_timeout)
+{
+    GList *iter = NULL;
+    xmlNode *update = NULL;
+    const char *client_node = NULL;
+    const char *client_id = NULL;
+    const char *call_id = NULL;
+
+    if (op->call_options & st_opt_sync_call) {
+        /* There is no reason to report the timeout for a synchronous call. It
+         * is impossible to use the reported timeout to do anything when the client
+         * is blocking for the response.  This update is only important for
+         * async calls that require a callback to report the results in. */
+        return;
+    } else if (!op->request) {
+        return;
+    }
+
+    crm_trace("Reporting timeout for %s (id=%.8s)", op->client_name, op->id);
+    client_node = crm_element_value(op->request, F_STONITH_CLIENTNODE);
+    call_id = crm_element_value(op->request, F_STONITH_CALLID);
+    client_id = crm_element_value(op->request, F_STONITH_CLIENTID);
+    if (!client_node || !call_id || !client_id) {
+        return;
+    }
+
+    if (pcmk__str_eq(client_node, stonith_our_uname, pcmk__str_casei)) {
+        // Client is connected to this node, so send update directly to them
+        do_stonith_async_timeout_update(client_id, call_id, op_timeout);
+        return;
+    }
+
+    /* The client is connected to another node, relay this update to them */
+    update = stonith_create_op(op->client_callid, op->id, STONITH_OP_TIMEOUT_UPDATE, NULL, 0);
+    crm_xml_add(update, F_STONITH_REMOTE_OP_ID, op->id);
+    crm_xml_add(update, F_STONITH_CLIENTID, client_id);
+    crm_xml_add(update, F_STONITH_CALLID, call_id);
+    crm_xml_add_int(update, F_STONITH_TIMEOUT, op_timeout);
+
+    send_cluster_message(crm_get_peer(0, client_node), crm_msg_stonith_ng, update, FALSE);
+
+    free_xml(update);
+
+    for (iter = op->duplicates; iter != NULL; iter = iter->next) {
+        remote_fencing_op_t *dup = iter->data;
+
+        crm_trace("Reporting timeout for duplicate %.8s to client %s",
+                  dup->id, dup->client_name);
+        report_timeout_period(iter->data, op_timeout);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Advance an operation to the next device in its topology
+ *
+ * \param[in,out] op      Fencer operation to advance
+ * \param[in]     device  ID of device that just completed
+ * \param[in,out] msg     If not NULL, XML reply of last delegated operation
+ */
+static void
+advance_topology_device_in_level(remote_fencing_op_t *op, const char *device,
+                                 xmlNode *msg)
+{
+    /* Advance to the next device at this topology level, if any */
+    if (op->devices) {
+        op->devices = op->devices->next;
+    }
+
+    /* Handle automatic unfencing if an "on" action was requested */
+    if ((op->phase == st_phase_requested)
+        && pcmk__str_eq(op->action, "on", pcmk__str_none)) {
+        /* If the device we just executed was required, it's not anymore */
+        remove_required_device(op, device);
+
+        /* If there are no more devices at this topology level, run through any
+         * remaining devices with automatic unfencing
+         */
+        if (op->devices == NULL) {
+            op->devices = op->automatic_list;
+        }
+    }
+
+    if ((op->devices == NULL) && (op->phase == st_phase_off)) {
+        /* We're done with this level and with required devices, but we had
+         * remapped "reboot" to "off", so start over with "on". If any devices
+         * need to be turned back on, op->devices will be non-NULL after this.
+         */
+        op_phase_on(op);
+    }
+
+    // This function is only called if the previous device succeeded
+    pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+
+    if (op->devices) {
+        /* Necessary devices remain, so execute the next one */
+        crm_trace("Next targeting %s on behalf of %s@%s",
+                  op->target, op->client_name, op->originator);
+
+        // The requested delay has been applied for the first device
+        if (op->delay > 0) {
+            op->delay = 0;
+        }
+
+        request_peer_fencing(op, NULL);
+    } else {
+        /* We're done with all devices and phases, so finalize operation */
+        crm_trace("Marking complex fencing op targeting %s as complete",
+                  op->target);
+        op->state = st_done;
+        finalize_op(op, msg, false);
+    }
+}
+
+static gboolean
+check_watchdog_fencing_and_wait(remote_fencing_op_t * op)
+{
+    if (node_does_watchdog_fencing(op->target)) {
+
+        crm_notice("Waiting %lds for %s to self-fence (%s) for "
+                   "client %s " CRM_XS " id=%.8s",
+                   (stonith_watchdog_timeout_ms / 1000),
+                   op->target, op->action, op->client_name, op->id);
+
+        if (op->op_timer_one) {
+            g_source_remove(op->op_timer_one);
+        }
+        op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms,
+                                         remote_op_watchdog_done, op);
+        return TRUE;
+    } else {
+        crm_debug("Skipping fallback to watchdog-fencing as %s is "
+                 "not in host-list", op->target);
+    }
+    return FALSE;
+}
+
+/*!
+ * \internal
+ * \brief Ask a peer to execute a fencing operation
+ *
+ * \param[in,out] op      Fencing operation to be executed
+ * \param[in,out] peer    If NULL or topology is in use, choose best peer to
+ *                        execute the fencing, otherwise use this peer
+ */
+static void
+request_peer_fencing(remote_fencing_op_t *op, peer_device_info_t *peer)
+{
+    const char *device = NULL;
+    int timeout;
+
+    CRM_CHECK(op != NULL, return);
+
+    crm_trace("Action %.8s targeting %s for %s is %s",
+              op->id, op->target, op->client_name,
+              stonith_op_state_str(op->state));
+
+    if ((op->phase == st_phase_on) && (op->devices != NULL)) {
+        /* We are in the "on" phase of a remapped topology reboot. If this
+         * device has pcmk_reboot_action="off", or doesn't support the "on"
+         * action, skip it.
+         *
+         * We can't check device properties at this point because we haven't
+         * chosen a peer for this stage yet. Instead, we check the local node's
+         * knowledge about the device. If different versions of the fence agent
+         * are installed on different nodes, there's a chance this could be
+         * mistaken, but the worst that could happen is we don't try turning the
+         * node back on when we should.
+         */
+        device = op->devices->data;
+        if (pcmk__str_eq(fenced_device_reboot_action(device), "off",
+                         pcmk__str_none)) {
+            crm_info("Not turning %s back on using %s because the device is "
+                     "configured to stay off (pcmk_reboot_action='off')",
+                     op->target, device);
+            advance_topology_device_in_level(op, device, NULL);
+            return;
+        }
+        if (!fenced_device_supports_on(device)) {
+            crm_info("Not turning %s back on using %s because the agent "
+                     "doesn't support 'on'", op->target, device);
+            advance_topology_device_in_level(op, device, NULL);
+            return;
+        }
+    }
+
+    timeout = op->base_timeout;
+    if ((peer == NULL) && !pcmk_is_set(op->call_options, st_opt_topology)) {
+        peer = stonith_choose_peer(op);
+    }
+
+    if (!op->op_timer_total) {
+        op->total_timeout = TIMEOUT_MULTIPLY_FACTOR * get_op_total_timeout(op, peer);
+        op->op_timer_total = g_timeout_add(1000 * op->total_timeout, remote_op_timeout, op);
+        report_timeout_period(op, op->total_timeout);
+        crm_info("Total timeout set to %d for peer's fencing targeting %s for %s"
+                 CRM_XS "id=%.8s",
+                 op->total_timeout, op->target, op->client_name, op->id);
+    }
+
+    if (pcmk_is_set(op->call_options, st_opt_topology) && op->devices) {
+        /* Ignore the caller's peer preference if topology is in use, because
+         * that peer might not have access to the required device. With
+         * topology, stonith_choose_peer() removes the device from further
+         * consideration, so the timeout must be calculated beforehand.
+         *
+         * @TODO Basing the total timeout on the caller's preferred peer (above)
+         *       is less than ideal.
+         */
+        peer = stonith_choose_peer(op);
+
+        device = op->devices->data;
+        /* Fencing timeout sent to peer takes no delay into account.
+         * The peer will add a dedicated timer for any delay upon
+         * schedule_stonith_command().
+         */
+        timeout = get_device_timeout(op, peer, device, false);
+    }
+
+    if (peer) {
+       /* Take any requested fencing delay into account to prevent it from eating
+        * up the timeout.
+        */
+        int timeout_one = (op->delay > 0 ?
+                           TIMEOUT_MULTIPLY_FACTOR * op->delay : 0);
+        xmlNode *remote_op = stonith_create_op(op->client_callid, op->id, STONITH_OP_FENCE, NULL, 0);
+
+        crm_xml_add(remote_op, F_STONITH_REMOTE_OP_ID, op->id);
+        crm_xml_add(remote_op, F_STONITH_TARGET, op->target);
+        crm_xml_add(remote_op, F_STONITH_ACTION, op->action);
+        crm_xml_add(remote_op, F_STONITH_ORIGIN, op->originator);
+        crm_xml_add(remote_op, F_STONITH_CLIENTID, op->client_id);
+        crm_xml_add(remote_op, F_STONITH_CLIENTNAME, op->client_name);
+        crm_xml_add_int(remote_op, F_STONITH_TIMEOUT, timeout);
+        crm_xml_add_int(remote_op, F_STONITH_CALLOPTS, op->call_options);
+        crm_xml_add_int(remote_op, F_STONITH_DELAY, op->delay);
+
+        if (device) {
+            timeout_one += TIMEOUT_MULTIPLY_FACTOR *
+                           get_device_timeout(op, peer, device, true);
+            crm_notice("Requesting that %s perform '%s' action targeting %s "
+                       "using %s " CRM_XS " for client %s (%ds)",
+                       peer->host, op->action, op->target, device,
+                       op->client_name, timeout_one);
+            crm_xml_add(remote_op, F_STONITH_DEVICE, device);
+
+        } else {
+            timeout_one += TIMEOUT_MULTIPLY_FACTOR * get_peer_timeout(op, peer);
+            crm_notice("Requesting that %s perform '%s' action targeting %s "
+                       CRM_XS " for client %s (%ds, %lds)",
+                       peer->host, op->action, op->target, op->client_name,
+                       timeout_one, stonith_watchdog_timeout_ms);
+        }
+
+        op->state = st_exec;
+        if (op->op_timer_one) {
+            g_source_remove(op->op_timer_one);
+            op->op_timer_one = 0;
+        }
+
+        if (!((stonith_watchdog_timeout_ms > 0)
+              && (pcmk__str_eq(device, STONITH_WATCHDOG_ID, pcmk__str_none)
+                  || (pcmk__str_eq(peer->host, op->target, pcmk__str_casei)
+                      && pcmk__is_fencing_action(op->action)))
+              && check_watchdog_fencing_and_wait(op))) {
+
+            /* Some thoughts about self-fencing cases reaching this point:
+               - Actually check in check_watchdog_fencing_and_wait
+                 shouldn't fail if STONITH_WATCHDOG_ID is
+                 chosen as fencing-device and it being present implies
+                 watchdog-fencing is enabled anyway
+               - If watchdog-fencing is disabled either in general or for
+                 a specific target - detected in check_watchdog_fencing_and_wait -
+                 for some other kind of self-fencing we can't expect
+                 a success answer but timeout is fine if the node doesn't
+                 come back in between
+               - Delicate might be the case where we have watchdog-fencing
+                 enabled for a node but the watchdog-fencing-device isn't
+                 explicitly chosen for suicide. Local pe-execution in sbd
+                 may detect the node as unclean and lead to timely suicide.
+                 Otherwise the selection of stonith-watchdog-timeout at
+                 least is questionable.
+             */
+
+            /* coming here we're not waiting for watchdog timeout -
+               thus engage timer with timout evaluated before */
+            op->op_timer_one = g_timeout_add((1000 * timeout_one), remote_op_timeout_one, op);
+        }
+
+        send_cluster_message(crm_get_peer(0, peer->host), crm_msg_stonith_ng, remote_op, FALSE);
+        peer->tried = TRUE;
+        free_xml(remote_op);
+        return;
+
+    } else if (op->phase == st_phase_on) {
+        /* A remapped "on" cannot be executed, but the node was already
+         * turned off successfully, so ignore the error and continue.
+         */
+        crm_warn("Ignoring %s 'on' failure (no capable peers) targeting %s "
+                 "after successful 'off'", device, op->target);
+        advance_topology_device_in_level(op, device, NULL);
+        return;
+
+    } else if (op->owner == FALSE) {
+        crm_err("Fencing (%s) targeting %s for client %s is not ours to control",
+                op->action, op->target, op->client_name);
+
+    } else if (op->query_timer == 0) {
+        /* We've exhausted all available peers */
+        crm_info("No remaining peers capable of fencing (%s) %s for client %s "
+                 CRM_XS " state=%s", op->action, op->target, op->client_name,
+                 stonith_op_state_str(op->state));
+        CRM_CHECK(op->state < st_done, return);
+        finalize_timed_out_op(op, "All nodes failed, or are unable, to "
+                                  "fence target");
+
+    } else if(op->replies >= op->replies_expected || op->replies >= fencing_active_peers()) {
+        /* if the operation never left the query state,
+         * but we have all the expected replies, then no devices
+         * are available to execute the fencing operation. */
+
+        if(stonith_watchdog_timeout_ms > 0 && pcmk__str_eq(device,
+           STONITH_WATCHDOG_ID, pcmk__str_null_matches)) {
+            if (check_watchdog_fencing_and_wait(op)) {
+                return;
+            }
+        }
+
+        if (op->state == st_query) {
+            crm_info("No peers (out of %d) have devices capable of fencing "
+                     "(%s) %s for client %s " CRM_XS " state=%s",
+                     op->replies, op->action, op->target, op->client_name,
+                     stonith_op_state_str(op->state));
+
+            pcmk__reset_result(&op->result);
+            pcmk__set_result(&op->result, CRM_EX_ERROR,
+                             PCMK_EXEC_NO_FENCE_DEVICE, NULL);
+        } else {
+            if (pcmk_is_set(op->call_options, st_opt_topology)) {
+                pcmk__reset_result(&op->result);
+                pcmk__set_result(&op->result, CRM_EX_ERROR,
+                                 PCMK_EXEC_NO_FENCE_DEVICE, NULL);
+            }
+            /* ... else use existing result from previous failed attempt
+             * (topology is not in use, and no devices remain to be attempted).
+             * Overwriting the result with PCMK_EXEC_NO_FENCE_DEVICE would
+             * prevent finalize_op() from setting the correct delegate if
+             * needed.
+             */
+
+            crm_info("No peers (out of %d) are capable of fencing (%s) %s "
+                     "for client %s " CRM_XS " state=%s",
+                     op->replies, op->action, op->target, op->client_name,
+                     stonith_op_state_str(op->state));
+        }
+
+        op->state = st_failed;
+        finalize_op(op, NULL, false);
+
+    } else {
+        crm_info("Waiting for additional peers capable of fencing (%s) %s%s%s "
+                 "for client %s " CRM_XS " id=%.8s",
+                 op->action, op->target, (device? " using " : ""),
+                 (device? device : ""), op->client_name, op->id);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Comparison function for sorting query results
+ *
+ * \param[in] a  GList item to compare
+ * \param[in] b  GList item to compare
+ *
+ * \return Per the glib documentation, "a negative integer if the first value
+ *         comes before the second, 0 if they are equal, or a positive integer
+ *         if the first value comes after the second."
+ */
+static gint
+sort_peers(gconstpointer a, gconstpointer b)
+{
+    const peer_device_info_t *peer_a = a;
+    const peer_device_info_t *peer_b = b;
+
+    return (peer_b->ndevices - peer_a->ndevices);
+}
+
+/*!
+ * \internal
+ * \brief Determine if all the devices in the topology are found or not
+ *
+ * \param[in] op  Fencing operation with topology to check
+ */
+static gboolean
+all_topology_devices_found(const remote_fencing_op_t *op)
+{
+    GList *device = NULL;
+    GList *iter = NULL;
+    device_properties_t *match = NULL;
+    stonith_topology_t *tp = NULL;
+    gboolean skip_target = FALSE;
+    int i;
+
+    tp = find_topology_for_host(op->target);
+    if (!tp) {
+        return FALSE;
+    }
+    if (pcmk__is_fencing_action(op->action)) {
+        /* Don't count the devices on the target node if we are killing
+         * the target node. */
+        skip_target = TRUE;
+    }
+
+    for (i = 0; i < ST_LEVEL_MAX; i++) {
+        for (device = tp->levels[i]; device; device = device->next) {
+            match = NULL;
+            for (iter = op->query_results; iter && !match; iter = iter->next) {
+                peer_device_info_t *peer = iter->data;
+
+                if (skip_target && pcmk__str_eq(peer->host, op->target, pcmk__str_casei)) {
+                    continue;
+                }
+                match = find_peer_device(op, peer, device->data, st_device_supports_none);
+            }
+            if (!match) {
+                return FALSE;
+            }
+        }
+    }
+
+    return TRUE;
+}
+
+/*!
+ * \internal
+ * \brief Parse action-specific device properties from XML
+ *
+ * \param[in]     xml     XML element containing the properties
+ * \param[in]     peer    Name of peer that sent XML (for logs)
+ * \param[in]     device  Device ID (for logs)
+ * \param[in]     action  Action the properties relate to (for logs)
+ * \param[in,out] op      Fencing operation that properties are being parsed for
+ * \param[in]     phase   Phase the properties relate to
+ * \param[in,out] props   Device properties to update
+ */
+static void
+parse_action_specific(const xmlNode *xml, const char *peer, const char *device,
+                      const char *action, remote_fencing_op_t *op,
+                      enum st_remap_phase phase, device_properties_t *props)
+{
+    props->custom_action_timeout[phase] = 0;
+    crm_element_value_int(xml, F_STONITH_ACTION_TIMEOUT,
+                          &props->custom_action_timeout[phase]);
+    if (props->custom_action_timeout[phase]) {
+        crm_trace("Peer %s with device %s returned %s action timeout %d",
+                  peer, device, action, props->custom_action_timeout[phase]);
+    }
+
+    props->delay_max[phase] = 0;
+    crm_element_value_int(xml, F_STONITH_DELAY_MAX, &props->delay_max[phase]);
+    if (props->delay_max[phase]) {
+        crm_trace("Peer %s with device %s returned maximum of random delay %d for %s",
+                  peer, device, props->delay_max[phase], action);
+    }
+
+    props->delay_base[phase] = 0;
+    crm_element_value_int(xml, F_STONITH_DELAY_BASE, &props->delay_base[phase]);
+    if (props->delay_base[phase]) {
+        crm_trace("Peer %s with device %s returned base delay %d for %s",
+                  peer, device, props->delay_base[phase], action);
+    }
+
+    /* Handle devices with automatic unfencing */
+    if (pcmk__str_eq(action, "on", pcmk__str_none)) {
+        int required = 0;
+
+        crm_element_value_int(xml, F_STONITH_DEVICE_REQUIRED, &required);
+        if (required) {
+            crm_trace("Peer %s requires device %s to execute for action %s",
+                      peer, device, action);
+            add_required_device(op, device);
+        }
+    }
+
+    /* If a reboot is remapped to off+on, it's possible that a node is allowed
+     * to perform one action but not another.
+     */
+    if (pcmk__xe_attr_is_true(xml, F_STONITH_ACTION_DISALLOWED)) {
+        props->disallowed[phase] = TRUE;
+        crm_trace("Peer %s is disallowed from executing %s for device %s",
+                  peer, action, device);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Parse one device's properties from peer's XML query reply
+ *
+ * \param[in]     xml       XML node containing device properties
+ * \param[in,out] op        Operation that query and reply relate to
+ * \param[in,out] peer      Peer's device information
+ * \param[in]     device    ID of device being parsed
+ */
+static void
+add_device_properties(const xmlNode *xml, remote_fencing_op_t *op,
+                      peer_device_info_t *peer, const char *device)
+{
+    xmlNode *child;
+    int verified = 0;
+    device_properties_t *props = calloc(1, sizeof(device_properties_t));
+    int flags = st_device_supports_on; /* Old nodes that don't set the flag assume they support the on action */
+
+    /* Add a new entry to this peer's devices list */
+    CRM_ASSERT(props != NULL);
+    g_hash_table_insert(peer->devices, strdup(device), props);
+
+    /* Peers with verified (monitored) access will be preferred */
+    crm_element_value_int(xml, F_STONITH_DEVICE_VERIFIED, &verified);
+    if (verified) {
+        crm_trace("Peer %s has confirmed a verified device %s",
+                  peer->host, device);
+        props->verified = TRUE;
+    }
+
+    crm_element_value_int(xml, F_STONITH_DEVICE_SUPPORT_FLAGS, &flags);
+    props->device_support_flags = flags;
+
+    /* Parse action-specific device properties */
+    parse_action_specific(xml, peer->host, device, op_requested_action(op),
+                          op, st_phase_requested, props);
+    for (child = pcmk__xml_first_child(xml); child != NULL;
+         child = pcmk__xml_next(child)) {
+        /* Replies for "reboot" operations will include the action-specific
+         * values for "off" and "on" in child elements, just in case the reboot
+         * winds up getting remapped.
+         */
+        if (pcmk__str_eq(ID(child), "off", pcmk__str_none)) {
+            parse_action_specific(child, peer->host, device, "off",
+                                  op, st_phase_off, props);
+        } else if (pcmk__str_eq(ID(child), "on", pcmk__str_none)) {
+            parse_action_specific(child, peer->host, device, "on",
+                                  op, st_phase_on, props);
+        }
+    }
+}
+
+/*!
+ * \internal
+ * \brief Parse a peer's XML query reply and add it to operation's results
+ *
+ * \param[in,out] op        Operation that query and reply relate to
+ * \param[in]     host      Name of peer that sent this reply
+ * \param[in]     ndevices  Number of devices expected in reply
+ * \param[in]     xml       XML node containing device list
+ *
+ * \return Newly allocated result structure with parsed reply
+ */
+static peer_device_info_t *
+add_result(remote_fencing_op_t *op, const char *host, int ndevices,
+           const xmlNode *xml)
+{
+    peer_device_info_t *peer = calloc(1, sizeof(peer_device_info_t));
+    xmlNode *child;
+
+    // cppcheck seems not to understand the abort logic in CRM_CHECK
+    // cppcheck-suppress memleak
+    CRM_CHECK(peer != NULL, return NULL);
+    peer->host = strdup(host);
+    peer->devices = pcmk__strkey_table(free, free);
+
+    /* Each child element describes one capable device available to the peer */
+    for (child = pcmk__xml_first_child(xml); child != NULL;
+         child = pcmk__xml_next(child)) {
+        const char *device = ID(child);
+
+        if (device) {
+            add_device_properties(child, op, peer, device);
+        }
+    }
+
+    peer->ndevices = g_hash_table_size(peer->devices);
+    CRM_CHECK(ndevices == peer->ndevices,
+              crm_err("Query claimed to have %d device%s but %d found",
+                      ndevices, pcmk__plural_s(ndevices), peer->ndevices));
+
+    op->query_results = g_list_insert_sorted(op->query_results, peer, sort_peers);
+    return peer;
+}
+
+/*!
+ * \internal
+ * \brief Handle a peer's reply to our fencing query
+ *
+ * Parse a query result from XML and store it in the remote operation
+ * table, and when enough replies have been received, issue a fencing request.
+ *
+ * \param[in] msg  XML reply received
+ *
+ * \return pcmk_ok on success, -errno on error
+ *
+ * \note See initiate_remote_stonith_op() for how the XML query was initially
+ *       formed, and stonith_query() for how the peer formed its XML reply.
+ */
+int
+process_remote_stonith_query(xmlNode *msg)
+{
+    int ndevices = 0;
+    gboolean host_is_target = FALSE;
+    gboolean have_all_replies = FALSE;
+    const char *id = NULL;
+    const char *host = NULL;
+    remote_fencing_op_t *op = NULL;
+    peer_device_info_t *peer = NULL;
+    uint32_t replies_expected;
+    xmlNode *dev = get_xpath_object("//@" F_STONITH_REMOTE_OP_ID, msg, LOG_ERR);
+
+    CRM_CHECK(dev != NULL, return -EPROTO);
+
+    id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
+    CRM_CHECK(id != NULL, return -EPROTO);
+
+    dev = get_xpath_object("//@" F_STONITH_AVAILABLE_DEVICES, msg, LOG_ERR);
+    CRM_CHECK(dev != NULL, return -EPROTO);
+    crm_element_value_int(dev, F_STONITH_AVAILABLE_DEVICES, &ndevices);
+
+    op = g_hash_table_lookup(stonith_remote_op_list, id);
+    if (op == NULL) {
+        crm_debug("Received query reply for unknown or expired operation %s",
+                  id);
+        return -EOPNOTSUPP;
+    }
+
+    replies_expected = fencing_active_peers();
+    if (op->replies_expected < replies_expected) {
+        replies_expected = op->replies_expected;
+    }
+    if ((++op->replies >= replies_expected) && (op->state == st_query)) {
+        have_all_replies = TRUE;
+    }
+    host = crm_element_value(msg, F_ORIG);
+    host_is_target = pcmk__str_eq(host, op->target, pcmk__str_casei);
+
+    crm_info("Query result %d of %d from %s for %s/%s (%d device%s) %s",
+             op->replies, replies_expected, host,
+             op->target, op->action, ndevices, pcmk__plural_s(ndevices), id);
+    if (ndevices > 0) {
+        peer = add_result(op, host, ndevices, dev);
+    }
+
+    pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+
+    if (pcmk_is_set(op->call_options, st_opt_topology)) {
+        /* If we start the fencing before all the topology results are in,
+         * it is possible fencing levels will be skipped because of the missing
+         * query results. */
+        if (op->state == st_query && all_topology_devices_found(op)) {
+            /* All the query results are in for the topology, start the fencing ops. */
+            crm_trace("All topology devices found");
+            request_peer_fencing(op, peer);
+
+        } else if (have_all_replies) {
+            crm_info("All topology query replies have arrived, continuing (%d expected/%d received) ",
+                     replies_expected, op->replies);
+            request_peer_fencing(op, NULL);
+        }
+
+    } else if (op->state == st_query) {
+        int nverified = count_peer_devices(op, peer, TRUE,
+                                           fenced_support_flag(op->action));
+
+        /* We have a result for a non-topology fencing op that looks promising,
+         * go ahead and start fencing before query timeout */
+        if ((peer != NULL) && !host_is_target && nverified) {
+            /* we have a verified device living on a peer that is not the target */
+            crm_trace("Found %d verified device%s",
+                      nverified, pcmk__plural_s(nverified));
+            request_peer_fencing(op, peer);
+
+        } else if (have_all_replies) {
+            crm_info("All query replies have arrived, continuing (%d expected/%d received) ",
+                     replies_expected, op->replies);
+            request_peer_fencing(op, NULL);
+
+        } else {
+            crm_trace("Waiting for more peer results before launching fencing operation");
+        }
+
+    } else if ((peer != NULL) && (op->state == st_done)) {
+        crm_info("Discarding query result from %s (%d device%s): "
+                 "Operation is %s", peer->host,
+                 peer->ndevices, pcmk__plural_s(peer->ndevices),
+                 stonith_op_state_str(op->state));
+    }
+
+    return pcmk_ok;
+}
+
+/*!
+ * \internal
+ * \brief Handle a peer's reply to a fencing request
+ *
+ * Parse a fencing reply from XML, and either finalize the operation
+ * or attempt another device as appropriate.
+ *
+ * \param[in] msg  XML reply received
+ */
+void
+fenced_process_fencing_reply(xmlNode *msg)
+{
+    const char *id = NULL;
+    const char *device = NULL;
+    remote_fencing_op_t *op = NULL;
+    xmlNode *dev = get_xpath_object("//@" F_STONITH_REMOTE_OP_ID, msg, LOG_ERR);
+    pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
+
+    CRM_CHECK(dev != NULL, return);
+
+    id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
+    CRM_CHECK(id != NULL, return);
+
+    dev = stonith__find_xe_with_result(msg);
+    CRM_CHECK(dev != NULL, return);
+
+    stonith__xe_get_result(dev, &result);
+
+    device = crm_element_value(dev, F_STONITH_DEVICE);
+
+    if (stonith_remote_op_list) {
+        op = g_hash_table_lookup(stonith_remote_op_list, id);
+    }
+
+    if ((op == NULL) && pcmk__result_ok(&result)) {
+        /* Record successful fencing operations */
+        const char *client_id = crm_element_value(dev, F_STONITH_CLIENTID);
+
+        op = create_remote_stonith_op(client_id, dev, TRUE);
+    }
+
+    if (op == NULL) {
+        /* Could be for an event that began before we started */
+        /* TODO: Record the op for later querying */
+        crm_info("Received peer result of unknown or expired operation %s", id);
+        pcmk__reset_result(&result);
+        return;
+    }
+
+    pcmk__reset_result(&op->result);
+    op->result = result; // The operation takes ownership of the result
+
+    if (op->devices && device && !pcmk__str_eq(op->devices->data, device, pcmk__str_casei)) {
+        crm_err("Received outdated reply for device %s (instead of %s) to "
+                "fence (%s) %s. Operation already timed out at peer level.",
+                device, (const char *) op->devices->data, op->action, op->target);
+        return;
+    }
+
+    if (pcmk__str_eq(crm_element_value(msg, F_SUBTYPE), "broadcast", pcmk__str_casei)) {
+        if (pcmk__result_ok(&op->result)) {
+            op->state = st_done;
+        } else {
+            op->state = st_failed;
+        }
+        finalize_op(op, msg, false);
+        return;
+
+    } else if (!pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
+        /* If this isn't a remote level broadcast, and we are not the
+         * originator of the operation, we should not be receiving this msg. */
+        crm_err("Received non-broadcast fencing result for operation %.8s "
+                "we do not own (device %s targeting %s)",
+                op->id, device, op->target);
+        return;
+    }
+
+    if (pcmk_is_set(op->call_options, st_opt_topology)) {
+        const char *device = NULL;
+        const char *reason = op->result.exit_reason;
+
+        /* We own the op, and it is complete. broadcast the result to all nodes
+         * and notify our local clients. */
+        if (op->state == st_done) {
+            finalize_op(op, msg, false);
+            return;
+        }
+
+        device = crm_element_value(msg, F_STONITH_DEVICE);
+
+        if ((op->phase == 2) && !pcmk__result_ok(&op->result)) {
+            /* A remapped "on" failed, but the node was already turned off
+             * successfully, so ignore the error and continue.
+             */
+            crm_warn("Ignoring %s 'on' failure (%s%s%s) targeting %s "
+                     "after successful 'off'",
+                     device, pcmk_exec_status_str(op->result.execution_status),
+                     (reason == NULL)? "" : ": ",
+                     (reason == NULL)? "" : reason,
+                     op->target);
+            pcmk__set_result(&op->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+        } else {
+            crm_notice("Action '%s' targeting %s%s%s on behalf of %s@%s: "
+                       "%s%s%s%s",
+                       op->action, op->target,
+                       ((device == NULL)? "" : " using "),
+                       ((device == NULL)? "" : device),
+                       op->client_name,
+                       op->originator,
+                       pcmk_exec_status_str(op->result.execution_status),
+                       (reason == NULL)? "" : " (",
+                       (reason == NULL)? "" : reason,
+                       (reason == NULL)? "" : ")");
+        }
+
+        if (pcmk__result_ok(&op->result)) {
+            /* An operation completed successfully. Try another device if
+             * necessary, otherwise mark the operation as done. */
+            advance_topology_device_in_level(op, device, msg);
+            return;
+        } else {
+            /* This device failed, time to try another topology level. If no other
+             * levels are available, mark this operation as failed and report results. */
+            if (advance_topology_level(op, false) != pcmk_rc_ok) {
+                op->state = st_failed;
+                finalize_op(op, msg, false);
+                return;
+            }
+        }
+
+    } else if (pcmk__result_ok(&op->result) && (op->devices == NULL)) {
+        op->state = st_done;
+        finalize_op(op, msg, false);
+        return;
+
+    } else if ((op->result.execution_status == PCMK_EXEC_TIMEOUT)
+               && (op->devices == NULL)) {
+        /* If the operation timed out don't bother retrying other peers. */
+        op->state = st_failed;
+        finalize_op(op, msg, false);
+        return;
+
+    } else {
+        /* fall-through and attempt other fencing action using another peer */
+    }
+
+    /* Retry on failure */
+    crm_trace("Next for %s on behalf of %s@%s (result was: %s)",
+              op->target, op->originator, op->client_name,
+              pcmk_exec_status_str(op->result.execution_status));
+    request_peer_fencing(op, NULL);
+}
+
+gboolean
+stonith_check_fence_tolerance(int tolerance, const char *target, const char *action)
+{
+    GHashTableIter iter;
+    time_t now = time(NULL);
+    remote_fencing_op_t *rop = NULL;
+
+    if (tolerance <= 0 || !stonith_remote_op_list || target == NULL ||
+        action == NULL) {
+        return FALSE;
+    }
+
+    g_hash_table_iter_init(&iter, stonith_remote_op_list);
+    while (g_hash_table_iter_next(&iter, NULL, (void **)&rop)) {
+        if (strcmp(rop->target, target) != 0) {
+            continue;
+        } else if (rop->state != st_done) {
+            continue;
+        /* We don't have to worry about remapped reboots here
+         * because if state is done, any remapping has been undone
+         */
+        } else if (strcmp(rop->action, action) != 0) {
+            continue;
+        } else if ((rop->completed + tolerance) < now) {
+            continue;
+        }
+
+        crm_notice("Target %s was fenced (%s) less than %ds ago by %s on behalf of %s",
+                   target, action, tolerance, rop->delegate, rop->originator);
+        return TRUE;
+    }
+    return FALSE;
+}
diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c
new file mode 100644
index 0000000..4edda6c
--- /dev/null
+++ b/daemons/fenced/pacemaker-fenced.c
@@ -0,0 +1,1751 @@
+/*
+ * Copyright 2009-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/utsname.h>
+
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>  // PRIu32, PRIx32
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/ipc.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/common/output_internal.h>
+#include <crm/cluster/internal.h>
+
+#include <crm/stonith-ng.h>
+#include <crm/fencing/internal.h>
+#include <crm/common/xml.h>
+#include <crm/common/xml_internal.h>
+
+#include <crm/common/mainloop.h>
+
+#include <crm/cib/internal.h>
+#include <crm/pengine/status.h>
+#include <pacemaker-internal.h>
+
+#include <pacemaker-fenced.h>
+
+#define SUMMARY "daemon for executing fencing devices in a Pacemaker cluster"
+
+char *stonith_our_uname = NULL;
+long stonith_watchdog_timeout_ms = 0;
+GList *stonith_watchdog_targets = NULL;
+
+static GMainLoop *mainloop = NULL;
+
+gboolean stand_alone = FALSE;
+static gboolean stonith_shutdown_flag = FALSE;
+
+static qb_ipcs_service_t *ipcs = NULL;
+static xmlNode *local_cib = NULL;
+static pe_working_set_t *fenced_data_set = NULL;
+static const unsigned long long data_set_flags = pe_flag_quick_location
+                                                 | pe_flag_no_compat
+                                                 | pe_flag_no_counts;
+
+static cib_t *cib_api = NULL;
+
+static pcmk__output_t *logger_out = NULL;
+static pcmk__output_t *out = NULL;
+
+pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+static struct {
+    bool no_cib_connect;
+    gchar **log_files;
+} options;
+
+static crm_exit_t exit_code = CRM_EX_OK;
+
+static void stonith_shutdown(int nsig);
+static void stonith_cleanup(void);
+
+static int32_t
+st_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
+{
+    if (stonith_shutdown_flag) {
+        crm_info("Ignoring new client [%d] during shutdown",
+                 pcmk__client_pid(c));
+        return -EPERM;
+    }
+
+    if (pcmk__new_client(c, uid, gid) == NULL) {
+        return -EIO;
+    }
+    return 0;
+}
+
+/* Exit code means? */
+static int32_t
+st_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
+{
+    uint32_t id = 0;
+    uint32_t flags = 0;
+    int call_options = 0;
+    xmlNode *request = NULL;
+    pcmk__client_t *c = pcmk__find_client(qbc);
+    const char *op = NULL;
+
+    if (c == NULL) {
+        crm_info("Invalid client: %p", qbc);
+        return 0;
+    }
+
+    request = pcmk__client_data2xml(c, data, &id, &flags);
+    if (request == NULL) {
+        pcmk__ipc_send_ack(c, id, flags, "nack", NULL, CRM_EX_PROTOCOL);
+        return 0;
+    }
+
+
+    op = crm_element_value(request, F_CRM_TASK);
+    if(pcmk__str_eq(op, CRM_OP_RM_NODE_CACHE, pcmk__str_casei)) {
+        crm_xml_add(request, F_TYPE, T_STONITH_NG);
+        crm_xml_add(request, F_STONITH_OPERATION, op);
+        crm_xml_add(request, F_STONITH_CLIENTID, c->id);
+        crm_xml_add(request, F_STONITH_CLIENTNAME, pcmk__client_name(c));
+        crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname);
+
+        send_cluster_message(NULL, crm_msg_stonith_ng, request, FALSE);
+        free_xml(request);
+        return 0;
+    }
+
+    if (c->name == NULL) {
+        const char *value = crm_element_value(request, F_STONITH_CLIENTNAME);
+
+        if (value == NULL) {
+            value = "unknown";
+        }
+        c->name = crm_strdup_printf("%s.%u", value, c->pid);
+    }
+
+    crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
+    crm_trace("Flags %#08" PRIx32 "/%#08x for command %" PRIu32
+              " from client %s", flags, call_options, id, pcmk__client_name(c));
+
+    if (pcmk_is_set(call_options, st_opt_sync_call)) {
+        CRM_ASSERT(flags & crm_ipc_client_response);
+        CRM_LOG_ASSERT(c->request_id == 0);     /* This means the client has two synchronous events in-flight */
+        c->request_id = id;     /* Reply only to the last one */
+    }
+
+    crm_xml_add(request, F_STONITH_CLIENTID, c->id);
+    crm_xml_add(request, F_STONITH_CLIENTNAME, pcmk__client_name(c));
+    crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname);
+
+    crm_log_xml_trace(request, "ipc-received");
+    stonith_command(c, id, flags, request, NULL);
+
+    free_xml(request);
+    return 0;
+}
+
+/* Error code means? */
+static int32_t
+st_ipc_closed(qb_ipcs_connection_t * c)
+{
+    pcmk__client_t *client = pcmk__find_client(c);
+
+    if (client == NULL) {
+        return 0;
+    }
+
+    crm_trace("Connection %p closed", c);
+    pcmk__free_client(client);
+
+    /* 0 means: yes, go ahead and destroy the connection */
+    return 0;
+}
+
+static void
+st_ipc_destroy(qb_ipcs_connection_t * c)
+{
+    crm_trace("Connection %p destroyed", c);
+    st_ipc_closed(c);
+}
+
+static void
+stonith_peer_callback(xmlNode * msg, void *private_data)
+{
+    const char *remote_peer = crm_element_value(msg, F_ORIG);
+    const char *op = crm_element_value(msg, F_STONITH_OPERATION);
+
+    if (pcmk__str_eq(op, "poke", pcmk__str_none)) {
+        return;
+    }
+
+    crm_log_xml_trace(msg, "Peer[inbound]");
+    stonith_command(NULL, 0, 0, msg, remote_peer);
+}
+
+#if SUPPORT_COROSYNC
+static void
+stonith_peer_ais_callback(cpg_handle_t handle,
+                          const struct cpg_name *groupName,
+                          uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
+{
+    uint32_t kind = 0;
+    xmlNode *xml = NULL;
+    const char *from = NULL;
+    char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);
+
+    if(data == NULL) {
+        return;
+    }
+    if (kind == crm_class_cluster) {
+        xml = string2xml(data);
+        if (xml == NULL) {
+            crm_err("Invalid XML: '%.120s'", data);
+            free(data);
+            return;
+        }
+        crm_xml_add(xml, F_ORIG, from);
+        /* crm_xml_add_int(xml, F_SEQ, wrapper->id); */
+        stonith_peer_callback(xml, NULL);
+    }
+
+    free_xml(xml);
+    free(data);
+    return;
+}
+
+static void
+stonith_peer_cs_destroy(gpointer user_data)
+{
+    crm_crit("Lost connection to cluster layer, shutting down");
+    stonith_shutdown(0);
+}
+#endif
+
+void
+do_local_reply(xmlNode *notify_src, pcmk__client_t *client, int call_options)
+{
+    /* send callback to originating child */
+    int local_rc = pcmk_rc_ok;
+    int rid = 0;
+    uint32_t ipc_flags = crm_ipc_server_event;
+
+    if (pcmk_is_set(call_options, st_opt_sync_call)) {
+        CRM_LOG_ASSERT(client->request_id);
+        rid = client->request_id;
+        client->request_id = 0;
+        ipc_flags = crm_ipc_flags_none;
+    }
+
+    local_rc = pcmk__ipc_send_xml(client, rid, notify_src, ipc_flags);
+    if (local_rc == pcmk_rc_ok) {
+        crm_trace("Sent response %d to client %s",
+                  rid, pcmk__client_name(client));
+    } else {
+        crm_warn("%synchronous reply to client %s failed: %s",
+                 (pcmk_is_set(call_options, st_opt_sync_call)? "S" : "As"),
+                 pcmk__client_name(client), pcmk_rc_str(local_rc));
+    }
+}
+
+uint64_t
+get_stonith_flag(const char *name)
+{
+    if (pcmk__str_eq(name, T_STONITH_NOTIFY_FENCE, pcmk__str_casei)) {
+        return st_callback_notify_fence;
+
+    } else if (pcmk__str_eq(name, STONITH_OP_DEVICE_ADD, pcmk__str_casei)) {
+        return st_callback_device_add;
+
+    } else if (pcmk__str_eq(name, STONITH_OP_DEVICE_DEL, pcmk__str_casei)) {
+        return st_callback_device_del;
+
+    } else if (pcmk__str_eq(name, T_STONITH_NOTIFY_HISTORY, pcmk__str_casei)) {
+        return st_callback_notify_history;
+
+    } else if (pcmk__str_eq(name, T_STONITH_NOTIFY_HISTORY_SYNCED, pcmk__str_casei)) {
+        return st_callback_notify_history_synced;
+
+    }
+    return st_callback_unknown;
+}
+
+static void
+stonith_notify_client(gpointer key, gpointer value, gpointer user_data)
+{
+
+    xmlNode *update_msg = user_data;
+    pcmk__client_t *client = value;
+    const char *type = NULL;
+
+    CRM_CHECK(client != NULL, return);
+    CRM_CHECK(update_msg != NULL, return);
+
+    type = crm_element_value(update_msg, F_SUBTYPE);
+    CRM_CHECK(type != NULL, crm_log_xml_err(update_msg, "notify"); return);
+
+    if (client->ipcs == NULL) {
+        crm_trace("Skipping client with NULL channel");
+        return;
+    }
+
+    if (pcmk_is_set(client->flags, get_stonith_flag(type))) {
+        int rc = pcmk__ipc_send_xml(client, 0, update_msg,
+                                    crm_ipc_server_event);
+
+        if (rc != pcmk_rc_ok) {
+            crm_warn("%s notification of client %s failed: %s "
+                     CRM_XS " id=%.8s rc=%d", type, pcmk__client_name(client),
+                     pcmk_rc_str(rc), client->id, rc);
+        } else {
+            crm_trace("Sent %s notification to client %s",
+                      type, pcmk__client_name(client));
+        }
+    }
+}
+
+void
+do_stonith_async_timeout_update(const char *client_id, const char *call_id, int timeout)
+{
+    pcmk__client_t *client = NULL;
+    xmlNode *notify_data = NULL;
+
+    if (!timeout || !call_id || !client_id) {
+        return;
+    }
+
+    client = pcmk__find_client_by_id(client_id);
+    if (!client) {
+        return;
+    }
+
+    notify_data = create_xml_node(NULL, T_STONITH_TIMEOUT_VALUE);
+    crm_xml_add(notify_data, F_TYPE, T_STONITH_TIMEOUT_VALUE);
+    crm_xml_add(notify_data, F_STONITH_CALLID, call_id);
+    crm_xml_add_int(notify_data, F_STONITH_TIMEOUT, timeout);
+
+    crm_trace("timeout update is %d for client %s and call id %s", timeout, client_id, call_id);
+
+    if (client) {
+        pcmk__ipc_send_xml(client, 0, notify_data, crm_ipc_server_event);
+    }
+
+    free_xml(notify_data);
+}
+
+/*!
+ * \internal
+ * \brief Notify relevant IPC clients of a fencing operation result
+ *
+ * \param[in] type     Notification type
+ * \param[in] result   Result of fencing operation (assume success if NULL)
+ * \param[in] data     If not NULL, add to notification as call data
+ */
+void
+fenced_send_notification(const char *type, const pcmk__action_result_t *result,
+                         xmlNode *data)
+{
+    /* TODO: Standardize the contents of data */
+    xmlNode *update_msg = create_xml_node(NULL, "notify");
+
+    CRM_LOG_ASSERT(type != NULL);
+
+    crm_xml_add(update_msg, F_TYPE, T_STONITH_NOTIFY);
+    crm_xml_add(update_msg, F_SUBTYPE, type);
+    crm_xml_add(update_msg, F_STONITH_OPERATION, type);
+    stonith__xe_set_result(update_msg, result);
+
+    if (data != NULL) {
+        add_message_xml(update_msg, F_STONITH_CALLDATA, data);
+    }
+
+    crm_trace("Notifying clients");
+    pcmk__foreach_ipc_client(stonith_notify_client, update_msg);
+    free_xml(update_msg);
+    crm_trace("Notify complete");
+}
+
+/*!
+ * \internal
+ * \brief Send notifications for a configuration change to subscribed clients
+ *
+ * \param[in] op      Notification type (STONITH_OP_DEVICE_ADD,
+ *                    STONITH_OP_DEVICE_DEL, STONITH_OP_LEVEL_ADD, or
+ *                    STONITH_OP_LEVEL_DEL)
+ * \param[in] result  Operation result
+ * \param[in] desc    Description of what changed
+ * \param[in] active  Current number of devices or topologies in use
+ */
+static void
+send_config_notification(const char *op, const pcmk__action_result_t *result,
+                         const char *desc, int active)
+{
+    xmlNode *notify_data = create_xml_node(NULL, op);
+
+    CRM_CHECK(notify_data != NULL, return);
+
+    crm_xml_add(notify_data, F_STONITH_DEVICE, desc);
+    crm_xml_add_int(notify_data, F_STONITH_ACTIVE, active);
+
+    fenced_send_notification(op, result, notify_data);
+    free_xml(notify_data);
+}
+
+/*!
+ * \internal
+ * \brief Send notifications for a device change to subscribed clients
+ *
+ * \param[in] op      Notification type (STONITH_OP_DEVICE_ADD or
+ *                    STONITH_OP_DEVICE_DEL)
+ * \param[in] result  Operation result
+ * \param[in] desc    ID of device that changed
+ */
+void
+fenced_send_device_notification(const char *op,
+                                const pcmk__action_result_t *result,
+                                const char *desc)
+{
+    send_config_notification(op, result, desc, g_hash_table_size(device_list));
+}
+
+/*!
+ * \internal
+ * \brief Send notifications for a topology level change to subscribed clients
+ *
+ * \param[in] op      Notification type (STONITH_OP_LEVEL_ADD or
+ *                    STONITH_OP_LEVEL_DEL)
+ * \param[in] result  Operation result
+ * \param[in] desc    String representation of level (<target>[<level_index>])
+ */
+void
+fenced_send_level_notification(const char *op,
+                               const pcmk__action_result_t *result,
+                               const char *desc)
+{
+    send_config_notification(op, result, desc, g_hash_table_size(topology));
+}
+
+static void
+topology_remove_helper(const char *node, int level)
+{
+    char *desc = NULL;
+    pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
+    xmlNode *data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL);
+
+    crm_xml_add(data, F_STONITH_ORIGIN, __func__);
+    crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level);
+    crm_xml_add(data, XML_ATTR_STONITH_TARGET, node);
+
+    fenced_unregister_level(data, &desc, &result);
+    fenced_send_level_notification(STONITH_OP_LEVEL_DEL, &result, desc);
+    pcmk__reset_result(&result);
+    free_xml(data);
+    free(desc);
+}
+
+static void
+remove_cib_device(xmlXPathObjectPtr xpathObj)
+{
+    int max = numXpathResults(xpathObj), lpc = 0;
+
+    for (lpc = 0; lpc < max; lpc++) {
+        const char *rsc_id = NULL;
+        const char *standard = NULL;
+        xmlNode *match = getXpathResult(xpathObj, lpc);
+
+        CRM_LOG_ASSERT(match != NULL);
+        if(match != NULL) {
+            standard = crm_element_value(match, XML_AGENT_ATTR_CLASS);
+        }
+
+        if (!pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
+            continue;
+        }
+
+        rsc_id = crm_element_value(match, XML_ATTR_ID);
+
+        stonith_device_remove(rsc_id, true);
+    }
+}
+
+static void
+remove_topology_level(xmlNode *match)
+{
+    int index = 0;
+    char *key = NULL;
+
+    CRM_CHECK(match != NULL, return);
+
+    key = stonith_level_key(match, fenced_target_by_unknown);
+    crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
+    topology_remove_helper(key, index);
+    free(key);
+}
+
+static void
+add_topology_level(xmlNode *match)
+{
+    char *desc = NULL;
+    pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
+
+    CRM_CHECK(match != NULL, return);
+
+    fenced_register_level(match, &desc, &result);
+    fenced_send_level_notification(STONITH_OP_LEVEL_ADD, &result, desc);
+    pcmk__reset_result(&result);
+    free(desc);
+}
+
+static void
+remove_fencing_topology(xmlXPathObjectPtr xpathObj)
+{
+    int max = numXpathResults(xpathObj), lpc = 0;
+
+    for (lpc = 0; lpc < max; lpc++) {
+        xmlNode *match = getXpathResult(xpathObj, lpc);
+
+        CRM_LOG_ASSERT(match != NULL);
+        if (match && crm_element_value(match, XML_DIFF_MARKER)) {
+            /* Deletion */
+            int index = 0;
+            char *target = stonith_level_key(match, fenced_target_by_unknown);
+
+            crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
+            if (target == NULL) {
+                crm_err("Invalid fencing target in element %s", ID(match));
+
+            } else if (index <= 0) {
+                crm_err("Invalid level for %s in element %s", target, ID(match));
+
+            } else {
+                topology_remove_helper(target, index);
+            }
+            /* } else { Deal with modifications during the 'addition' stage */
+        }
+    }
+}
+
+static void
+register_fencing_topology(xmlXPathObjectPtr xpathObj)
+{
+    int max = numXpathResults(xpathObj), lpc = 0;
+
+    for (lpc = 0; lpc < max; lpc++) {
+        xmlNode *match = getXpathResult(xpathObj, lpc);
+
+        remove_topology_level(match);
+        add_topology_level(match);
+    }
+}
+
+/* Fencing
+<diff crm_feature_set="3.0.6">
+  <diff-removed>
+    <fencing-topology>
+      <fencing-level id="f-p1.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="removed:top"/>
+      <fencing-level id="f-p1.2" target="pcmk-1" index="2" devices="power" __crm_diff_marker__="removed:top"/>
+      <fencing-level devices="disk,network" id="f-p2.1"/>
+    </fencing-topology>
+  </diff-removed>
+  <diff-added>
+    <fencing-topology>
+      <fencing-level id="f-p.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="added:top"/>
+      <fencing-level id="f-p2.1" target="pcmk-2" index="1" devices="disk,something"/>
+      <fencing-level id="f-p3.1" target="pcmk-2" index="2" devices="power" __crm_diff_marker__="added:top"/>
+    </fencing-topology>
+  </diff-added>
+</diff>
+*/
+
+static void
+fencing_topology_init(void)
+{
+    xmlXPathObjectPtr xpathObj = NULL;
+    const char *xpath = "//" XML_TAG_FENCING_LEVEL;
+
+    crm_trace("Full topology refresh");
+    free_topology_list();
+    init_topology_list();
+
+    /* Grab everything */
+    xpathObj = xpath_search(local_cib, xpath);
+    register_fencing_topology(xpathObj);
+
+    freeXpathObject(xpathObj);
+}
+
+#define rsc_name(x) x->clone_name?x->clone_name:x->id
+
+/*!
+ * \internal
+ * \brief Check whether our uname is in a resource's allowed node list
+ *
+ * \param[in] rsc  Resource to check
+ *
+ * \return Pointer to node object if found, NULL otherwise
+ */
+static pe_node_t *
+our_node_allowed_for(const pe_resource_t *rsc)
+{
+    GHashTableIter iter;
+    pe_node_t *node = NULL;
+
+    if (rsc && stonith_our_uname) {
+        g_hash_table_iter_init(&iter, rsc->allowed_nodes);
+        while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
+            if (node && strcmp(node->details->uname, stonith_our_uname) == 0) {
+                break;
+            }
+            node = NULL;
+        }
+    }
+    return node;
+}
+
+static void
+watchdog_device_update(void)
+{
+    if (stonith_watchdog_timeout_ms > 0) {
+        if (!g_hash_table_lookup(device_list, STONITH_WATCHDOG_ID) &&
+            !stonith_watchdog_targets) {
+            /* getting here watchdog-fencing enabled, no device there yet
+               and reason isn't stonith_watchdog_targets preventing that
+             */
+            int rc;
+            xmlNode *xml;
+
+            xml = create_device_registration_xml(
+                    STONITH_WATCHDOG_ID,
+                    st_namespace_internal,
+                    STONITH_WATCHDOG_AGENT,
+                    NULL, /* stonith_device_register will add our
+                             own name as PCMK_STONITH_HOST_LIST param
+                             so we can skip that here
+                           */
+                    NULL);
+            rc = stonith_device_register(xml, TRUE);
+            free_xml(xml);
+            if (rc != pcmk_ok) {
+                rc = pcmk_legacy2rc(rc);
+                exit_code = CRM_EX_FATAL;
+                crm_crit("Cannot register watchdog pseudo fence agent: %s",
+                         pcmk_rc_str(rc));
+                stonith_shutdown(0);
+            }
+        }
+
+    } else if (g_hash_table_lookup(device_list, STONITH_WATCHDOG_ID) != NULL) {
+        /* be silent if no device - todo parameter to stonith_device_remove */
+        stonith_device_remove(STONITH_WATCHDOG_ID, true);
+    }
+}
+
+static void
+update_stonith_watchdog_timeout_ms(xmlNode *cib)
+{
+    long timeout_ms = 0;
+    xmlNode *stonith_watchdog_xml = NULL;
+    const char *value = NULL;
+
+    stonith_watchdog_xml = get_xpath_object("//nvpair[@name='stonith-watchdog-timeout']",
+					    cib, LOG_NEVER);
+    if (stonith_watchdog_xml) {
+        value = crm_element_value(stonith_watchdog_xml, XML_NVPAIR_ATTR_VALUE);
+    }
+    if (value) {
+        timeout_ms = crm_get_msec(value);
+    }
+
+    if (timeout_ms < 0) {
+        timeout_ms = pcmk__auto_watchdog_timeout();
+    }
+
+    stonith_watchdog_timeout_ms = timeout_ms;
+}
+
+/*!
+ * \internal
+ * \brief If a resource or any of its children are STONITH devices, update their
+ *        definitions given a cluster working set.
+ *
+ * \param[in,out] rsc       Resource to check
+ * \param[in,out] data_set  Cluster working set with device information
+ */
+static void
+cib_device_update(pe_resource_t *rsc, pe_working_set_t *data_set)
+{
+    pe_node_t *node = NULL;
+    const char *value = NULL;
+    const char *rclass = NULL;
+    pe_node_t *parent = NULL;
+
+    /* If this is a complex resource, check children rather than this resource itself. */
+    if(rsc->children) {
+        GList *gIter = NULL;
+        for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
+            cib_device_update(gIter->data, data_set);
+            if(pe_rsc_is_clone(rsc)) {
+                crm_trace("Only processing one copy of the clone %s", rsc->id);
+                break;
+            }
+        }
+        return;
+    }
+
+    /* We only care about STONITH resources. */
+    rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
+    if (!pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
+        return;
+    }
+
+    /* If this STONITH resource is disabled, remove it. */
+    if (pe__resource_is_disabled(rsc)) {
+        crm_info("Device %s has been disabled", rsc->id);
+        return;
+    }
+
+    /* if watchdog-fencing is disabled handle any watchdog-fence
+       resource as if it was disabled
+     */
+    if ((stonith_watchdog_timeout_ms <= 0) &&
+        pcmk__str_eq(rsc->id, STONITH_WATCHDOG_ID, pcmk__str_none)) {
+        crm_info("Watchdog-fencing disabled thus handling "
+                 "device %s as disabled", rsc->id);
+        return;
+    }
+
+    /* Check whether our node is allowed for this resource (and its parent if in a group) */
+    node = our_node_allowed_for(rsc);
+    if (rsc->parent && (rsc->parent->variant == pe_group)) {
+        parent = our_node_allowed_for(rsc->parent);
+    }
+
+    if(node == NULL) {
+        /* Our node is disallowed, so remove the device */
+        GHashTableIter iter;
+
+        crm_info("Device %s has been disabled on %s: unknown", rsc->id, stonith_our_uname);
+        g_hash_table_iter_init(&iter, rsc->allowed_nodes);
+        while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
+            crm_trace("Available: %s = %d", pe__node_name(node), node->weight);
+        }
+
+        return;
+
+    } else if(node->weight < 0 || (parent && parent->weight < 0)) {
+        /* Our node (or its group) is disallowed by score, so remove the device */
+        int score = (node->weight < 0)? node->weight : parent->weight;
+
+        crm_info("Device %s has been disabled on %s: score=%s",
+                 rsc->id, stonith_our_uname, pcmk_readable_score(score));
+        return;
+
+    } else {
+        /* Our node is allowed, so update the device information */
+        int rc;
+        xmlNode *data;
+        GHashTable *rsc_params = NULL;
+        GHashTableIter gIter;
+        stonith_key_value_t *params = NULL;
+
+        const char *name = NULL;
+        const char *agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE);
+        const char *rsc_provides = NULL;
+
+        crm_debug("Device %s is allowed on %s: score=%d", rsc->id, stonith_our_uname, node->weight);
+        rsc_params = pe_rsc_params(rsc, node, data_set);
+        get_meta_attributes(rsc->meta, rsc, node, data_set);
+
+        rsc_provides = g_hash_table_lookup(rsc->meta, PCMK_STONITH_PROVIDES);
+
+        g_hash_table_iter_init(&gIter, rsc_params);
+        while (g_hash_table_iter_next(&gIter, (gpointer *) & name, (gpointer *) & value)) {
+            if (!name || !value) {
+                continue;
+            }
+            params = stonith_key_value_add(params, name, value);
+            crm_trace(" %s=%s", name, value);
+        }
+
+        data = create_device_registration_xml(rsc_name(rsc), st_namespace_any,
+                                              agent, params, rsc_provides);
+        stonith_key_value_freeall(params, 1, 1);
+        rc = stonith_device_register(data, TRUE);
+        CRM_ASSERT(rc == pcmk_ok);
+        free_xml(data);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Update all STONITH device definitions based on current CIB
+ */
+static void
+cib_devices_update(void)
+{
+    GHashTableIter iter;
+    stonith_device_t *device = NULL;
+
+    crm_info("Updating devices to version %s.%s.%s",
+             crm_element_value(local_cib, XML_ATTR_GENERATION_ADMIN),
+             crm_element_value(local_cib, XML_ATTR_GENERATION),
+             crm_element_value(local_cib, XML_ATTR_NUMUPDATES));
+
+    if (fenced_data_set->now != NULL) {
+        crm_time_free(fenced_data_set->now);
+        fenced_data_set->now = NULL;
+    }
+    fenced_data_set->localhost = stonith_our_uname;
+    pcmk__schedule_actions(local_cib, data_set_flags, fenced_data_set);
+
+    g_hash_table_iter_init(&iter, device_list);
+    while (g_hash_table_iter_next(&iter, NULL, (void **)&device)) {
+        if (device->cib_registered) {
+            device->dirty = TRUE;
+        }
+    }
+
+    /* have list repopulated if cib has a watchdog-fencing-resource
+       TODO: keep a cached list for queries happening while we are refreshing
+     */
+    g_list_free_full(stonith_watchdog_targets, free);
+    stonith_watchdog_targets = NULL;
+    g_list_foreach(fenced_data_set->resources, (GFunc) cib_device_update, fenced_data_set);
+
+    g_hash_table_iter_init(&iter, device_list);
+    while (g_hash_table_iter_next(&iter, NULL, (void **)&device)) {
+        if (device->dirty) {
+            g_hash_table_iter_remove(&iter);
+        }
+    }
+
+    fenced_data_set->input = NULL; // Wasn't a copy, so don't let API free it
+    pe_reset_working_set(fenced_data_set);
+}
+
+static void
+update_cib_stonith_devices_v2(const char *event, xmlNode * msg)
+{
+    xmlNode *change = NULL;
+    char *reason = NULL;
+    bool needs_update = FALSE;
+    xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
+
+    for (change = pcmk__xml_first_child(patchset); change != NULL;
+         change = pcmk__xml_next(change)) {
+        const char *op = crm_element_value(change, XML_DIFF_OP);
+        const char *xpath = crm_element_value(change, XML_DIFF_PATH);
+        const char *shortpath = NULL;
+
+        if ((op == NULL) ||
+            (strcmp(op, "move") == 0) ||
+            strstr(xpath, "/"XML_CIB_TAG_STATUS)) {
+            continue;
+        } else if (pcmk__str_eq(op, "delete", pcmk__str_casei) && strstr(xpath, "/"XML_CIB_TAG_RESOURCE)) {
+            const char *rsc_id = NULL;
+            char *search = NULL;
+            char *mutable = NULL;
+
+            if (strstr(xpath, XML_TAG_ATTR_SETS) ||
+                strstr(xpath, XML_TAG_META_SETS)) {
+                needs_update = TRUE;
+                pcmk__str_update(&reason,
+                                 "(meta) attribute deleted from resource");
+                break;
+            }
+            pcmk__str_update(&mutable, xpath);
+            rsc_id = strstr(mutable, "primitive[@" XML_ATTR_ID "=\'");
+            if (rsc_id != NULL) {
+                rsc_id += strlen("primitive[@" XML_ATTR_ID "=\'");
+                search = strchr(rsc_id, '\'');
+            }
+            if (search != NULL) {
+                *search = 0;
+                stonith_device_remove(rsc_id, true);
+                /* watchdog_device_update called afterwards
+                   to fall back to implicit definition if needed */
+            } else {
+                crm_warn("Ignoring malformed CIB update (resource deletion)");
+            }
+            free(mutable);
+
+        } else if (strstr(xpath, "/"XML_CIB_TAG_RESOURCES) ||
+                   strstr(xpath, "/"XML_CIB_TAG_CONSTRAINTS) ||
+                   strstr(xpath, "/"XML_CIB_TAG_RSCCONFIG)) {
+            shortpath = strrchr(xpath, '/'); CRM_ASSERT(shortpath);
+            reason = crm_strdup_printf("%s %s", op, shortpath+1);
+            needs_update = TRUE;
+            break;
+        }
+    }
+
+    if(needs_update) {
+        crm_info("Updating device list from CIB: %s", reason);
+        cib_devices_update();
+    } else {
+        crm_trace("No updates for device list found in CIB");
+    }
+    free(reason);
+}
+
+
+static void
+update_cib_stonith_devices_v1(const char *event, xmlNode * msg)
+{
+    const char *reason = "none";
+    gboolean needs_update = FALSE;
+    xmlXPathObjectPtr xpath_obj = NULL;
+
+    /* process new constraints */
+    xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_CONS_TAG_RSC_LOCATION);
+    if (numXpathResults(xpath_obj) > 0) {
+        int max = numXpathResults(xpath_obj), lpc = 0;
+
+        /* Safest and simplest to always recompute */
+        needs_update = TRUE;
+        reason = "new location constraint";
+
+        for (lpc = 0; lpc < max; lpc++) {
+            xmlNode *match = getXpathResult(xpath_obj, lpc);
+
+            crm_log_xml_trace(match, "new constraint");
+        }
+    }
+    freeXpathObject(xpath_obj);
+
+    /* process deletions */
+    xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_RESOURCE);
+    if (numXpathResults(xpath_obj) > 0) {
+        remove_cib_device(xpath_obj);
+    }
+    freeXpathObject(xpath_obj);
+
+    /* process additions */
+    xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_RESOURCE);
+    if (numXpathResults(xpath_obj) > 0) {
+        int max = numXpathResults(xpath_obj), lpc = 0;
+
+        for (lpc = 0; lpc < max; lpc++) {
+            const char *rsc_id = NULL;
+            const char *standard = NULL;
+            xmlNode *match = getXpathResult(xpath_obj, lpc);
+
+            rsc_id = crm_element_value(match, XML_ATTR_ID);
+            standard = crm_element_value(match, XML_AGENT_ATTR_CLASS);
+
+            if (!pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
+                continue;
+            }
+
+            crm_trace("Fencing resource %s was added or modified", rsc_id);
+            reason = "new resource";
+            needs_update = TRUE;
+        }
+    }
+    freeXpathObject(xpath_obj);
+
+    if(needs_update) {
+        crm_info("Updating device list from CIB: %s", reason);
+        cib_devices_update();
+    }
+}
+
+static void
+update_cib_stonith_devices(const char *event, xmlNode * msg)
+{
+    int format = 1;
+    xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
+
+    CRM_ASSERT(patchset);
+    crm_element_value_int(patchset, "format", &format);
+    switch(format) {
+        case 1:
+            update_cib_stonith_devices_v1(event, msg);
+            break;
+        case 2:
+            update_cib_stonith_devices_v2(event, msg);
+            break;
+        default:
+            crm_warn("Unknown patch format: %d", format);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Check whether a node has a specific attribute name/value
+ *
+ * \param[in] node    Name of node to check
+ * \param[in] name    Name of an attribute to look for
+ * \param[in] value   The value the named attribute needs to be set to in order to be considered a match
+ *
+ * \return TRUE if the locally cached CIB has the specified node attribute
+ */
+gboolean
+node_has_attr(const char *node, const char *name, const char *value)
+{
+    GString *xpath = NULL;
+    xmlNode *match;
+
+    CRM_CHECK((local_cib != NULL) && (node != NULL) && (name != NULL)
+              && (value != NULL), return FALSE);
+
+    /* Search for the node's attributes in the CIB. While the schema allows
+     * multiple sets of instance attributes, and allows instance attributes to
+     * use id-ref to reference values elsewhere, that is intended for resources,
+     * so we ignore that here.
+     */
+    xpath = g_string_sized_new(256);
+    pcmk__g_strcat(xpath,
+                   "//" XML_CIB_TAG_NODES "/" XML_CIB_TAG_NODE
+                   "[@" XML_ATTR_UNAME "='", node, "']/" XML_TAG_ATTR_SETS
+                   "/" XML_CIB_TAG_NVPAIR
+                   "[@" XML_NVPAIR_ATTR_NAME "='", name, "' "
+                   "and @" XML_NVPAIR_ATTR_VALUE "='", value, "']", NULL);
+
+    match = get_xpath_object((const char *) xpath->str, local_cib, LOG_NEVER);
+
+    g_string_free(xpath, TRUE);
+    return (match != NULL);
+}
+
+/*!
+ * \internal
+ * \brief Check whether a node does watchdog-fencing
+ *
+ * \param[in] node    Name of node to check
+ *
+ * \return TRUE if node found in stonith_watchdog_targets
+ *         or stonith_watchdog_targets is empty indicating
+ *         all nodes are doing watchdog-fencing
+ */
+gboolean
+node_does_watchdog_fencing(const char *node)
+{
+    return ((stonith_watchdog_targets == NULL) ||
+            pcmk__str_in_list(node, stonith_watchdog_targets, pcmk__str_casei));
+}
+
+
+static void
+update_fencing_topology(const char *event, xmlNode * msg)
+{
+    int format = 1;
+    const char *xpath;
+    xmlXPathObjectPtr xpathObj = NULL;
+    xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
+
+    CRM_ASSERT(patchset);
+    crm_element_value_int(patchset, "format", &format);
+
+    if(format == 1) {
+        /* Process deletions (only) */
+        xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_TAG_FENCING_LEVEL;
+        xpathObj = xpath_search(msg, xpath);
+
+        remove_fencing_topology(xpathObj);
+        freeXpathObject(xpathObj);
+
+        /* Process additions and changes */
+        xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_TAG_FENCING_LEVEL;
+        xpathObj = xpath_search(msg, xpath);
+
+        register_fencing_topology(xpathObj);
+        freeXpathObject(xpathObj);
+
+    } else if(format == 2) {
+        xmlNode *change = NULL;
+        int add[] = { 0, 0, 0 };
+        int del[] = { 0, 0, 0 };
+
+        xml_patch_versions(patchset, add, del);
+
+        for (change = pcmk__xml_first_child(patchset); change != NULL;
+             change = pcmk__xml_next(change)) {
+            const char *op = crm_element_value(change, XML_DIFF_OP);
+            const char *xpath = crm_element_value(change, XML_DIFF_PATH);
+
+            if(op == NULL) {
+                continue;
+
+            } else if(strstr(xpath, "/" XML_TAG_FENCING_LEVEL) != NULL) {
+                /* Change to a specific entry */
+
+                crm_trace("Handling %s operation %d.%d.%d for %s", op, add[0], add[1], add[2], xpath);
+                if(strcmp(op, "move") == 0) {
+                    continue;
+
+                } else if(strcmp(op, "create") == 0) {
+                    add_topology_level(change->children);
+
+                } else if(strcmp(op, "modify") == 0) {
+                    xmlNode *match = first_named_child(change, XML_DIFF_RESULT);
+
+                    if(match) {
+                        remove_topology_level(match->children);
+                        add_topology_level(match->children);
+                    }
+
+                } else if(strcmp(op, "delete") == 0) {
+                    /* Nuclear option, all we have is the path and an id... not enough to remove a specific entry */
+                    crm_info("Re-initializing fencing topology after %s operation %d.%d.%d for %s",
+                             op, add[0], add[1], add[2], xpath);
+                    fencing_topology_init();
+                    return;
+                }
+
+            } else if (strstr(xpath, "/" XML_TAG_FENCING_TOPOLOGY) != NULL) {
+                /* Change to the topology in general */
+                crm_info("Re-initializing fencing topology after top-level %s operation  %d.%d.%d for %s",
+                         op, add[0], add[1], add[2], xpath);
+                fencing_topology_init();
+                return;
+
+            } else if (strstr(xpath, "/" XML_CIB_TAG_CONFIGURATION)) {
+                /* Changes to the whole config section, possibly including the topology as a whild */
+                if(first_named_child(change, XML_TAG_FENCING_TOPOLOGY) == NULL) {
+                    crm_trace("Nothing for us in %s operation %d.%d.%d for %s.",
+                              op, add[0], add[1], add[2], xpath);
+
+                } else if(strcmp(op, "delete") == 0 || strcmp(op, "create") == 0) {
+                    crm_info("Re-initializing fencing topology after top-level %s operation %d.%d.%d for %s.",
+                             op, add[0], add[1], add[2], xpath);
+                    fencing_topology_init();
+                    return;
+                }
+
+            } else {
+                crm_trace("Nothing for us in %s operation %d.%d.%d for %s",
+                          op, add[0], add[1], add[2], xpath);
+            }
+        }
+
+    } else {
+        crm_warn("Unknown patch format: %d", format);
+    }
+}
+static bool have_cib_devices = FALSE;
+
+static void
+update_cib_cache_cb(const char *event, xmlNode * msg)
+{
+    int rc = pcmk_ok;
+    long timeout_ms_saved = stonith_watchdog_timeout_ms;
+    bool need_full_refresh = false;
+
+    if(!have_cib_devices) {
+        crm_trace("Skipping updates until we get a full dump");
+        return;
+
+    } else if(msg == NULL) {
+        crm_trace("Missing %s update", event);
+        return;
+    }
+
+    /* Maintain a local copy of the CIB so that we have full access
+     * to device definitions, location constraints, and node attributes
+     */
+    if (local_cib != NULL) {
+        int rc = pcmk_ok;
+        xmlNode *patchset = NULL;
+
+        crm_element_value_int(msg, F_CIB_RC, &rc);
+        if (rc != pcmk_ok) {
+            return;
+        }
+
+        patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
+        pcmk__output_set_log_level(logger_out, LOG_TRACE);
+        out->message(out, "xml-patchset", patchset);
+        rc = xml_apply_patchset(local_cib, patchset, TRUE);
+        switch (rc) {
+            case pcmk_ok:
+            case -pcmk_err_old_data:
+                break;
+            case -pcmk_err_diff_resync:
+            case -pcmk_err_diff_failed:
+                crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
+                free_xml(local_cib);
+                local_cib = NULL;
+                break;
+            default:
+                crm_warn("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
+                free_xml(local_cib);
+                local_cib = NULL;
+        }
+    }
+
+    if (local_cib == NULL) {
+        crm_trace("Re-requesting full CIB");
+        rc = cib_api->cmds->query(cib_api, NULL, &local_cib, cib_scope_local | cib_sync_call);
+        if(rc != pcmk_ok) {
+            crm_err("Couldn't retrieve the CIB: %s (%d)", pcmk_strerror(rc), rc);
+            return;
+        }
+        CRM_ASSERT(local_cib != NULL);
+        need_full_refresh = true;
+    }
+
+    pcmk__refresh_node_caches_from_cib(local_cib);
+    update_stonith_watchdog_timeout_ms(local_cib);
+
+    if (timeout_ms_saved != stonith_watchdog_timeout_ms) {
+        need_full_refresh = true;
+    }
+
+    if (need_full_refresh) {
+        fencing_topology_init();
+        cib_devices_update();
+    } else {
+        // Partial refresh
+        update_fencing_topology(event, msg);
+        update_cib_stonith_devices(event, msg);
+    }
+
+    watchdog_device_update();
+}
+
+static void
+init_cib_cache_cb(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    crm_info("Updating device list from CIB");
+    have_cib_devices = TRUE;
+    local_cib = copy_xml(output);
+
+    pcmk__refresh_node_caches_from_cib(local_cib);
+    update_stonith_watchdog_timeout_ms(local_cib);
+
+    fencing_topology_init();
+    cib_devices_update();
+    watchdog_device_update();
+}
+
+static void
+stonith_shutdown(int nsig)
+{
+    crm_info("Terminating with %d clients", pcmk__ipc_client_count());
+    stonith_shutdown_flag = TRUE;
+    if (mainloop != NULL && g_main_loop_is_running(mainloop)) {
+        g_main_loop_quit(mainloop);
+    }
+}
+
+static void
+cib_connection_destroy(gpointer user_data)
+{
+    if (stonith_shutdown_flag) {
+        crm_info("Connection to the CIB manager closed");
+        return;
+    } else {
+        crm_crit("Lost connection to the CIB manager, shutting down");
+    }
+    if (cib_api) {
+        cib_api->cmds->signoff(cib_api);
+    }
+    stonith_shutdown(0);
+}
+
+static void
+stonith_cleanup(void)
+{
+    if (cib_api) {
+        cib_api->cmds->del_notify_callback(cib_api, T_CIB_DIFF_NOTIFY, update_cib_cache_cb);
+        cib_api->cmds->signoff(cib_api);
+    }
+
+    if (ipcs) {
+        qb_ipcs_destroy(ipcs);
+    }
+
+    crm_peer_destroy();
+    pcmk__client_cleanup();
+    free_stonith_remote_op_list();
+    free_topology_list();
+    free_device_list();
+    free_metadata_cache();
+    fenced_unregister_handlers();
+
+    free(stonith_our_uname);
+    stonith_our_uname = NULL;
+
+    free_xml(local_cib);
+    local_cib = NULL;
+}
+
+static gboolean
+stand_alone_cpg_cb(const gchar *option_name, const gchar *optarg, gpointer data,
+                   GError **error)
+{
+    stand_alone = FALSE;
+    options.no_cib_connect = true;
+    return TRUE;
+}
+
+static void
+setup_cib(void)
+{
+    int rc, retries = 0;
+
+    cib_api = cib_new();
+    if (cib_api == NULL) {
+        crm_err("No connection to the CIB manager");
+        return;
+    }
+
+    do {
+        sleep(retries);
+        rc = cib_api->cmds->signon(cib_api, CRM_SYSTEM_STONITHD, cib_command);
+    } while (rc == -ENOTCONN && ++retries < 5);
+
+    if (rc != pcmk_ok) {
+        crm_err("Could not connect to the CIB manager: %s (%d)", pcmk_strerror(rc), rc);
+
+    } else if (pcmk_ok !=
+               cib_api->cmds->add_notify_callback(cib_api, T_CIB_DIFF_NOTIFY, update_cib_cache_cb)) {
+        crm_err("Could not set CIB notification callback");
+
+    } else {
+        rc = cib_api->cmds->query(cib_api, NULL, NULL, cib_scope_local);
+        cib_api->cmds->register_callback(cib_api, rc, 120, FALSE, NULL, "init_cib_cache_cb",
+                                         init_cib_cache_cb);
+        cib_api->cmds->set_connection_dnotify(cib_api, cib_connection_destroy);
+        crm_info("Watching for fencing topology changes");
+    }
+}
+
+struct qb_ipcs_service_handlers ipc_callbacks = {
+    .connection_accept = st_ipc_accept,
+    .connection_created = NULL,
+    .msg_process = st_ipc_dispatch,
+    .connection_closed = st_ipc_closed,
+    .connection_destroyed = st_ipc_destroy
+};
+
+/*!
+ * \internal
+ * \brief Callback for peer status changes
+ *
+ * \param[in] type  What changed
+ * \param[in] node  What peer had the change
+ * \param[in] data  Previous value of what changed
+ */
+static void
+st_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
+{
+    if ((type != crm_status_processes)
+        && !pcmk_is_set(node->flags, crm_remote_node)) {
+        /*
+         * This is a hack until we can send to a nodeid and/or we fix node name lookups
+         * These messages are ignored in stonith_peer_callback()
+         */
+        xmlNode *query = create_xml_node(NULL, "stonith_command");
+
+        crm_xml_add(query, F_XML_TAGNAME, "stonith_command");
+        crm_xml_add(query, F_TYPE, T_STONITH_NG);
+        crm_xml_add(query, F_STONITH_OPERATION, "poke");
+
+        crm_debug("Broadcasting our uname because of node %u", node->id);
+        send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE);
+
+        free_xml(query);
+    }
+}
+
+static pcmk__cluster_option_t fencer_options[] = {
+    /* name, old name, type, allowed values,
+     * default value, validator,
+     * short description,
+     * long description
+     */
+    {
+        PCMK_STONITH_HOST_ARGUMENT, NULL, "string", NULL, "port", NULL,
+        N_("Advanced use only: An alternate parameter to supply instead of 'port'"),
+        N_("some devices do not support the "
+           "standard 'port' parameter or may provide additional ones. Use "
+           "this to specify an alternate, device-specific, parameter "
+           "that should indicate the machine to be fenced. A value of "
+           "none can be used to tell the cluster not to supply any "
+           "additional parameters.")
+    },
+    {
+        PCMK_STONITH_HOST_MAP,NULL, "string", NULL, "", NULL,
+        N_("A mapping of host names to ports numbers for devices that do not support host names."),
+        N_("Eg. node1:1;node2:2,3 would tell the cluster to use port 1 for node1 and ports 2 and 3 for node2")
+    },
+    {
+        PCMK_STONITH_HOST_LIST,NULL, "string", NULL, "", NULL,
+        N_("Eg. node1,node2,node3"),
+        N_("A list of machines controlled by "
+               "this device (Optional unless pcmk_host_list=static-list)")
+    },
+    {
+        PCMK_STONITH_HOST_CHECK,NULL, "string", NULL, "dynamic-list", NULL,
+        N_("How to determine which machines are controlled by the device."),
+        N_("Allowed values: dynamic-list "
+               "(query the device via the 'list' command), static-list "
+               "(check the pcmk_host_list attribute), status "
+               "(query the device via the 'status' command), "
+               "none (assume every device can fence every "
+               "machine)")
+    },
+    {
+        PCMK_STONITH_DELAY_MAX,NULL, "time", NULL, "0s", NULL,
+        N_("Enable a base delay for fencing actions and specify base delay value."),
+        N_("Enable a delay of no more than the "
+               "time specified before executing fencing actions. Pacemaker "
+               "derives the overall delay by taking the value of "
+               "pcmk_delay_base and adding a random delay value such "
+               "that the sum is kept below this maximum.")
+    },
+    {
+        PCMK_STONITH_DELAY_BASE,NULL, "string", NULL, "0s", NULL,
+        N_("Enable a base delay for "
+               "fencing actions and specify base delay value."),
+        N_("This enables a static delay for "
+               "fencing actions, which can help avoid \"death matches\" where "
+               "two nodes try to fence each other at the same time. If "
+               "pcmk_delay_max  is also used, a random delay will be "
+               "added such that the total delay is kept below that value."
+               "This can be set to a single time value to apply to any node "
+               "targeted by this device (useful if a separate device is "
+               "configured for each target), or to a node map (for example, "
+               "\"node1:1s;node2:5\") to set a different value per target.")
+    },
+    {
+        PCMK_STONITH_ACTION_LIMIT,NULL, "integer", NULL, "1", NULL,
+        N_("The maximum number of actions can be performed in parallel on this device"),
+        N_("Cluster property concurrent-fencing=true needs to be configured first."
+             "Then use this to specify the maximum number of actions can be performed in parallel on this device. -1 is unlimited.")
+    },
+    {
+	"pcmk_reboot_action",NULL, "string", NULL, "reboot", NULL,
+	N_("Advanced use only: An alternate command to run instead of 'reboot'"),
+        N_("Some devices do not support the standard commands or may provide additional ones.\n"
+                 "Use this to specify an alternate, device-specific, command that implements the \'reboot\' action.")
+    },
+    {
+	"pcmk_reboot_timeout",NULL, "time", NULL, "60s", NULL,
+	N_("Advanced use only: Specify an alternate timeout to use for reboot actions instead of stonith-timeout"),
+        N_("Some devices need much more/less time to complete than normal."
+	   "Use this to specify an alternate, device-specific, timeout for \'reboot\' actions.")
+    },
+    {
+	"pcmk_reboot_retries",NULL, "integer", NULL, "2", NULL,
+	N_("Advanced use only: The maximum number of times to retry the 'reboot' command within the timeout period"),
+        N_("Some devices do not support multiple connections."
+           " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation,      if there is time remaining."
+           " Use this option to alter the number of times Pacemaker retries \'reboot\' actions before giving up.")
+    },
+    {
+	"pcmk_off_action",NULL, "string", NULL, "off", NULL,
+	N_("Advanced use only: An alternate command to run instead of \'off\'"),
+        N_("Some devices do not support the standard commands or may provide additional ones."
+                 "Use this to specify an alternate, device-specific, command that implements the \'off\' action.")
+    },
+    {
+	"pcmk_off_timeout",NULL, "time", NULL, "60s", NULL,
+	N_("Advanced use only: Specify an alternate timeout to use for off actions instead of stonith-timeout"),
+        N_("Some devices need much more/less time to complete than normal."
+	   "Use this to specify an alternate, device-specific, timeout for \'off\' actions.")
+    },
+    {
+	"pcmk_off_retries",NULL, "integer", NULL, "2", NULL,
+	N_("Advanced use only: The maximum number of times to retry the 'off' command within the timeout period"),
+        N_("Some devices do not support multiple connections."
+           " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation,      if there is time remaining."
+           " Use this option to alter the number of times Pacemaker retries \'off\' actions before giving up.")
+    },
+    {
+	"pcmk_on_action",NULL, "string", NULL, "on", NULL,
+	N_("Advanced use only: An alternate command to run instead of 'on'"),
+        N_("Some devices do not support the standard commands or may provide additional ones."
+                 "Use this to specify an alternate, device-specific, command that implements the \'on\' action.")
+    },
+    {
+	"pcmk_on_timeout",NULL, "time", NULL, "60s", NULL,
+	N_("Advanced use only: Specify an alternate timeout to use for on actions instead of stonith-timeout"),
+        N_("Some devices need much more/less time to complete than normal."
+	   "Use this to specify an alternate, device-specific, timeout for \'on\' actions.")
+    },
+    {
+	"pcmk_on_retries",NULL, "integer", NULL, "2", NULL,
+	N_("Advanced use only: The maximum number of times to retry the 'on' command within the timeout period"),
+        N_("Some devices do not support multiple connections."
+           " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation,      if there is time remaining."
+           " Use this option to alter the number of times Pacemaker retries \'on\' actions before giving up.")
+    },
+    {
+	"pcmk_list_action",NULL, "string", NULL, "list", NULL,
+	N_("Advanced use only: An alternate command to run instead of \'list\'"),
+        N_("Some devices do not support the standard commands or may provide additional ones."
+                 "Use this to specify an alternate, device-specific, command that implements the \'list\' action.")
+    },
+    {
+	"pcmk_list_timeout",NULL, "time", NULL, "60s", NULL,
+	N_("Advanced use only: Specify an alternate timeout to use for list actions instead of stonith-timeout"),
+        N_("Some devices need much more/less time to complete than normal."
+	   "Use this to specify an alternate, device-specific, timeout for \'list\' actions.")
+    },
+    {
+	"pcmk_list_retries",NULL, "integer", NULL, "2", NULL,
+	N_("Advanced use only: The maximum number of times to retry the \'list\' command within the timeout period"),
+        N_("Some devices do not support multiple connections."
+           " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation,      if there is time remaining."
+           " Use this option to alter the number of times Pacemaker retries \'list\' actions before giving up.")
+    },
+    {
+	"pcmk_monitor_action",NULL, "string", NULL, "monitor", NULL,
+	N_("Advanced use only: An alternate command to run instead of \'monitor\'"),
+        N_("Some devices do not support the standard commands or may provide additional ones."
+                 "Use this to specify an alternate, device-specific, command that implements the \'monitor\' action.")
+    },
+    {
+	"pcmk_monitor_timeout",NULL, "time", NULL, "60s", NULL,
+	N_("Advanced use only: Specify an alternate timeout to use for monitor actions instead of stonith-timeout"),
+        N_("Some devices need much more/less time to complete than normal.\n"
+	   "Use this to specify an alternate, device-specific, timeout for \'monitor\' actions.")
+    },
+    {
+	"pcmk_monitor_retries",NULL, "integer", NULL, "2", NULL,
+	N_("Advanced use only: The maximum number of times to retry the \'monitor\' command within the timeout period"),
+        N_("Some devices do not support multiple connections."
+           " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation,      if there is time remaining."
+           " Use this option to alter the number of times Pacemaker retries \'monitor\' actions before giving up.")
+    },
+    {
+	"pcmk_status_action",NULL, "string", NULL, "status", NULL,
+	N_("Advanced use only: An alternate command to run instead of \'status\'"),
+        N_("Some devices do not support the standard commands or may provide additional ones."
+                 "Use this to specify an alternate, device-specific, command that implements the \'status\' action.")
+    },
+    {
+	"pcmk_status_timeout",NULL, "time", NULL, "60s", NULL,
+	N_("Advanced use only: Specify an alternate timeout to use for status actions instead of stonith-timeout"),
+        N_("Some devices need much more/less time to complete than normal."
+	   "Use this to specify an alternate, device-specific, timeout for \'status\' actions.")
+    },
+    {
+	"pcmk_status_retries",NULL, "integer", NULL, "2", NULL,
+	N_("Advanced use only: The maximum number of times to retry the \'status\' command within the timeout period"),
+        N_("Some devices do not support multiple connections."
+           " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation,      if there is time remaining."
+           " Use this option to alter the number of times Pacemaker retries \'status\' actions before giving up.")
+    },
+};
+
+void
+fencer_metadata(void)
+{
+    const char *desc_short = N_("Instance attributes available for all "
+                             "\"stonith\"-class resources");
+    const char *desc_long = N_("Instance attributes available for all \"stonith\"-"
+                            "class resources and used by Pacemaker's fence "
+                            "daemon, formerly known as stonithd");
+
+    gchar *s = pcmk__format_option_metadata("pacemaker-fenced", desc_short,
+                                            desc_long, fencer_options,
+                                            PCMK__NELEM(fencer_options));
+    printf("%s", s);
+    g_free(s);
+}
+
+static GOptionEntry entries[] = {
+    { "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &stand_alone,
+      "Deprecated (will be removed in a future release)", NULL },
+
+    { "stand-alone-w-cpg", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
+      stand_alone_cpg_cb, "Intended for use in regression testing only", NULL },
+
+    { "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY,
+      &options.log_files, "Send logs to the additional named logfile", NULL },
+
+    { NULL }
+};
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group)
+{
+    GOptionContext *context = NULL;
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group,
+                                      "[metadata]");
+    pcmk__add_main_args(context, entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    int rc = pcmk_rc_ok;
+    crm_cluster_t *cluster = NULL;
+    crm_ipc_t *old_instance = NULL;
+
+    GError *error = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "l");
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    crm_log_preinit(NULL, argc, argv);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    if ((g_strv_length(processed_args) >= 2)
+        && pcmk__str_eq(processed_args[1], "metadata", pcmk__str_none)) {
+        fencer_metadata();
+        goto done;
+    }
+
+    // Open additional log files
+    pcmk__add_logfiles(options.log_files, out);
+
+    crm_log_init(NULL, LOG_INFO + args->verbosity, TRUE,
+                 (args->verbosity > 0), argc, argv, FALSE);
+
+    crm_notice("Starting Pacemaker fencer");
+
+    old_instance = crm_ipc_new("stonith-ng", 0);
+    if (old_instance == NULL) {
+        /* crm_ipc_new() will have already logged an error message with
+         * crm_err()
+         */
+        exit_code = CRM_EX_FATAL;
+        goto done;
+    }
+
+    if (crm_ipc_connect(old_instance)) {
+        // IPC endpoint already up
+        crm_ipc_close(old_instance);
+        crm_ipc_destroy(old_instance);
+        crm_err("pacemaker-fenced is already active, aborting startup");
+        goto done;
+    } else {
+        // Not up or not authentic, we'll proceed either way
+        crm_ipc_destroy(old_instance);
+        old_instance = NULL;
+    }
+
+    mainloop_add_signal(SIGTERM, stonith_shutdown);
+
+    crm_peer_init();
+
+    fenced_data_set = pe_new_working_set();
+    CRM_ASSERT(fenced_data_set != NULL);
+
+    cluster = pcmk_cluster_new();
+
+    /* Initialize the logger prior to setup_cib(). update_cib_cache_cb() may
+     * call the "xml-patchset" message function, which needs the logger, after
+     * setup_cib() has run.
+     */
+    rc = pcmk__log_output_new(&logger_out) != pcmk_rc_ok;
+    if (rc != pcmk_rc_ok) {
+        exit_code = CRM_EX_FATAL;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Error creating output format log: %s", pcmk_rc_str(rc));
+        goto done;
+    }
+    pe__register_messages(logger_out);
+    pcmk__register_lib_messages(logger_out);
+    pcmk__output_set_log_level(logger_out, LOG_TRACE);
+    fenced_data_set->priv = logger_out;
+
+    if (!stand_alone) {
+#if SUPPORT_COROSYNC
+        if (is_corosync_cluster()) {
+            cluster->destroy = stonith_peer_cs_destroy;
+            cluster->cpg.cpg_deliver_fn = stonith_peer_ais_callback;
+            cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership;
+        }
+#endif // SUPPORT_COROSYNC
+
+        crm_set_status_callback(&st_peer_update_callback);
+
+        if (crm_cluster_connect(cluster) == FALSE) {
+            exit_code = CRM_EX_FATAL;
+            crm_crit("Cannot sign in to the cluster... terminating");
+            goto done;
+        }
+        pcmk__str_update(&stonith_our_uname, cluster->uname);
+
+        if (!options.no_cib_connect) {
+            setup_cib();
+        }
+
+    } else {
+        pcmk__str_update(&stonith_our_uname, "localhost");
+        crm_warn("Stand-alone mode is deprecated and will be removed "
+                 "in a future release");
+    }
+
+    init_device_list();
+    init_topology_list();
+
+    pcmk__serve_fenced_ipc(&ipcs, &ipc_callbacks);
+
+    // Create the mainloop and run it...
+    mainloop = g_main_loop_new(NULL, FALSE);
+    crm_notice("Pacemaker fencer successfully started and accepting connections");
+    g_main_loop_run(mainloop);
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    g_strfreev(options.log_files);
+
+    stonith_cleanup();
+    pcmk_cluster_free(cluster);
+    pe_free_working_set(fenced_data_set);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    if (logger_out != NULL) {
+        logger_out->finish(logger_out, exit_code, true, NULL);
+        pcmk__output_free(logger_out);
+    }
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+
+    pcmk__unregister_formats();
+    crm_exit(exit_code);
+}
diff --git a/daemons/fenced/pacemaker-fenced.h b/daemons/fenced/pacemaker-fenced.h
new file mode 100644
index 0000000..a3d2e17
--- /dev/null
+++ b/daemons/fenced/pacemaker-fenced.h
@@ -0,0 +1,315 @@
+/*
+ * Copyright 2009-2023 the Pacemaker project contributors
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <stdint.h>                 // uint32_t, uint64_t
+#include <crm/common/mainloop.h>
+
+/*!
+ * \internal
+ * \brief Check whether target has already been fenced recently
+ *
+ * \param[in] tolerance  Number of seconds to look back in time
+ * \param[in] target     Name of node to search for
+ * \param[in] action     Action we want to match
+ *
+ * \return TRUE if an equivalent fencing operation took place in the last
+ *         \p tolerance seconds, FALSE otherwise
+ */
+gboolean stonith_check_fence_tolerance(int tolerance, const char *target, const char *action);
+
+typedef struct stonith_device_s {
+    char *id;
+    char *agent;
+    char *namespace;
+
+    /*! list of actions that must execute on the target node. Used for unfencing */
+    GString *on_target_actions;
+    GList *targets;
+    time_t targets_age;
+    gboolean has_attr_map;
+
+    // Whether target's nodeid should be passed as a parameter to the agent
+    gboolean include_nodeid;
+
+    /* whether the cluster should automatically unfence nodes with the device */
+    gboolean automatic_unfencing;
+    guint priority;
+
+    uint32_t flags; // Group of enum st_device_flags
+
+    GHashTable *params;
+    GHashTable *aliases;
+    GList *pending_ops;
+    mainloop_timer_t *timer;
+    crm_trigger_t *work;
+    xmlNode *agent_metadata;
+
+    /*! A verified device is one that has contacted the
+     * agent successfully to perform a monitor operation */
+    gboolean verified;
+
+    gboolean cib_registered;
+    gboolean api_registered;
+    gboolean dirty;
+} stonith_device_t;
+
+/* These values are used to index certain arrays by "phase". Usually an
+ * operation has only one "phase", so phase is always zero. However, some
+ * reboots are remapped to "off" then "on", in which case "reboot" will be
+ * phase 0, "off" will be phase 1 and "on" will be phase 2.
+ */
+enum st_remap_phase {
+    st_phase_requested = 0,
+    st_phase_off = 1,
+    st_phase_on = 2,
+    st_phase_max = 3
+};
+
+typedef struct remote_fencing_op_s {
+    /* The unique id associated with this operation */
+    char *id;
+    /*! The node this operation will fence */
+    char *target;
+    /*! The fencing action to perform on the target. (reboot, on, off) */
+    char *action;
+
+    /*! When was the fencing action recorded (seconds since epoch) */
+    time_t created;
+
+    /*! Marks if the final notifications have been sent to local stonith clients. */
+    gboolean notify_sent;
+    /*! The number of query replies received */
+    guint replies;
+    /*! The number of query replies expected */
+    guint replies_expected;
+    /*! Does this node own control of this operation */
+    gboolean owner;
+    /*! After query is complete, This the high level timer that expires the entire operation */
+    guint op_timer_total;
+    /*! This timer expires the current fencing request. Many fencing
+     * requests may exist in a single operation */
+    guint op_timer_one;
+    /*! This timer expires the query request sent out to determine
+     * what nodes are contain what devices, and who those devices can fence */
+    guint query_timer;
+    /*! This is the default timeout to use for each fencing device if no
+     * custom timeout is received in the query. */
+    gint base_timeout;
+    /*! This is the calculated total timeout an operation can take before
+     * expiring. This is calculated by adding together all the timeout
+     * values associated with the devices this fencing operation may call */
+    gint total_timeout;
+
+    /*! Requested fencing delay.
+     * Value -1 means disable any static/random fencing delays. */
+    int delay;
+
+    /*! Delegate is the node being asked to perform a fencing action
+     * on behalf of the node that owns the remote operation. Some operations
+     * will involve multiple delegates. This value represents the final delegate
+     * that is used. */
+    char *delegate;
+    /*! The point at which the remote operation completed */
+    time_t completed;
+    //! Group of enum stonith_call_options associated with this operation
+    uint32_t call_options;
+
+    /*! The current state of the remote operation. This indicates
+     * what stage the op is in, query, exec, done, duplicate, failed. */
+    enum op_state state;
+    /*! The node that owns the remote operation */
+    char *originator;
+    /*! The local client id that initiated the fencing request */
+    char *client_id;
+    /*! The client's call_id that initiated the fencing request */
+    int client_callid;
+    /*! The name of client that initiated the fencing request */
+    char *client_name;
+    /*! List of the received query results for all the nodes in the cpg group */
+    GList *query_results;
+    /*! The original request that initiated the remote stonith operation */
+    xmlNode *request;
+
+    /*! The current topology level being executed */
+    guint level;
+    /*! The current operation phase being executed */
+    enum st_remap_phase phase;
+
+    /*! Devices with automatic unfencing (always run if "on" requested, never if remapped) */
+    GList *automatic_list;
+    /*! List of all devices at the currently executing topology level */
+    GList *devices_list;
+    /*! Current entry in the topology device list */
+    GList *devices;
+
+    /*! List of duplicate operations attached to this operation. Once this operation
+     * completes, the duplicate operations will be closed out as well. */
+    GList *duplicates;
+
+    /*! The point at which the remote operation completed(nsec) */
+    long long completed_nsec;
+
+    /*! The (potentially intermediate) result of the operation */
+    pcmk__action_result_t result;
+} remote_fencing_op_t;
+
+void fenced_broadcast_op_result(const remote_fencing_op_t *op, bool op_merged);
+
+// Fencer-specific client flags
+enum st_client_flags {
+    st_callback_unknown               =  UINT64_C(0),
+    st_callback_notify_fence          = (UINT64_C(1) << 0),
+    st_callback_device_add            = (UINT64_C(1) << 2),
+    st_callback_device_del            = (UINT64_C(1) << 4),
+    st_callback_notify_history        = (UINT64_C(1) << 5),
+    st_callback_notify_history_synced = (UINT64_C(1) << 6)
+};
+
+// How the user specified the target of a topology level
+enum fenced_target_by {
+    fenced_target_by_unknown = -1,  // Invalid or not yet parsed
+    fenced_target_by_name,          // By target name
+    fenced_target_by_pattern,       // By a pattern matching target names
+    fenced_target_by_attribute,     // By a node attribute/value on target
+};
+
+/*
+ * Complex fencing requirements are specified via fencing topologies.
+ * A topology consists of levels; each level is a list of fencing devices.
+ * Topologies are stored in a hash table by node name. When a node needs to be
+ * fenced, if it has an entry in the topology table, the levels are tried
+ * sequentially, and the devices in each level are tried sequentially.
+ * Fencing is considered successful as soon as any level succeeds;
+ * a level is considered successful if all its devices succeed.
+ * Essentially, all devices at a given level are "and-ed" and the
+ * levels are "or-ed".
+ *
+ * This structure is used for the topology table entries.
+ * Topology levels start from 1, so levels[0] is unused and always NULL.
+ */
+typedef struct stonith_topology_s {
+    enum fenced_target_by kind; // How target was specified
+
+    /*! Node name regex or attribute name=value for which topology applies */
+    char *target;
+    char *target_value;
+    char *target_pattern;
+    char *target_attribute;
+
+    /*! Names of fencing devices at each topology level */
+    GList *levels[ST_LEVEL_MAX];
+
+} stonith_topology_t;
+
+void init_device_list(void);
+void free_device_list(void);
+void init_topology_list(void);
+void free_topology_list(void);
+void free_stonith_remote_op_list(void);
+void init_stonith_remote_op_hash_table(GHashTable **table);
+void free_metadata_cache(void);
+void fenced_unregister_handlers(void);
+
+uint64_t get_stonith_flag(const char *name);
+
+void stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags,
+                            xmlNode *op_request, const char *remote_peer);
+
+int stonith_device_register(xmlNode *msg, gboolean from_cib);
+
+void stonith_device_remove(const char *id, bool from_cib);
+
+char *stonith_level_key(const xmlNode *msg, enum fenced_target_by);
+void fenced_register_level(xmlNode *msg, char **desc,
+                           pcmk__action_result_t *result);
+void fenced_unregister_level(xmlNode *msg, char **desc,
+                             pcmk__action_result_t *result);
+
+stonith_topology_t *find_topology_for_host(const char *host);
+
+void do_local_reply(xmlNode *notify_src, pcmk__client_t *client,
+                    int call_options);
+
+xmlNode *fenced_construct_reply(const xmlNode *request, xmlNode *data,
+                                const pcmk__action_result_t *result);
+
+void
+ do_stonith_async_timeout_update(const char *client, const char *call_id, int timeout);
+
+void fenced_send_notification(const char *type,
+                              const pcmk__action_result_t *result,
+                              xmlNode *data);
+void fenced_send_device_notification(const char *op,
+                                     const pcmk__action_result_t *result,
+                                     const char *desc);
+void fenced_send_level_notification(const char *op,
+                                    const pcmk__action_result_t *result,
+                                    const char *desc);
+
+remote_fencing_op_t *initiate_remote_stonith_op(const pcmk__client_t *client,
+                                                xmlNode *request,
+                                                gboolean manual_ack);
+
+void fenced_process_fencing_reply(xmlNode *msg);
+
+int process_remote_stonith_query(xmlNode * msg);
+
+void *create_remote_stonith_op(const char *client, xmlNode * request, gboolean peer);
+
+void stonith_fence_history(xmlNode *msg, xmlNode **output,
+                           const char *remote_peer, int options);
+
+void stonith_fence_history_trim(void);
+
+bool fencing_peer_active(crm_node_t *peer);
+
+void set_fencing_completed(remote_fencing_op_t * op);
+
+int fenced_handle_manual_confirmation(const pcmk__client_t *client,
+                                      xmlNode *msg);
+void fencer_metadata(void);
+
+const char *fenced_device_reboot_action(const char *device_id);
+bool fenced_device_supports_on(const char *device_id);
+
+gboolean node_has_attr(const char *node, const char *name, const char *value);
+
+gboolean node_does_watchdog_fencing(const char *node);
+
+static inline void
+fenced_set_protocol_error(pcmk__action_result_t *result)
+{
+    pcmk__set_result(result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
+                     "Fencer API request missing required information (bug?)");
+}
+
+/*!
+ * \internal
+ * \brief Get the device flag to use with a given action when searching devices
+ *
+ * \param[in] action  Action to check
+ *
+ * \return st_device_supports_on if \p action is "on", otherwise
+ *         st_device_supports_none
+ */
+static inline uint32_t
+fenced_support_flag(const char *action)
+{
+    if (pcmk__str_eq(action, "on", pcmk__str_none)) {
+        return st_device_supports_on;
+    }
+    return st_device_supports_none;
+}
+
+extern char *stonith_our_uname;
+extern gboolean stand_alone;
+extern GHashTable *device_list;
+extern GHashTable *topology;
+extern long stonith_watchdog_timeout_ms;
+extern GList *stonith_watchdog_targets;
+
+extern GHashTable *stonith_remote_op_list;
diff --git a/daemons/pacemakerd/Makefile.am b/daemons/pacemakerd/Makefile.am
new file mode 100644
index 0000000..fc0e014
--- /dev/null
+++ b/daemons/pacemakerd/Makefile.am
@@ -0,0 +1,37 @@
+#
+# Copyright 2004-2021 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+include $(top_srcdir)/mk/common.mk
+include $(top_srcdir)/mk/man.mk
+
+sbin_PROGRAMS		= pacemakerd
+
+if BUILD_SYSTEMD
+systemdsystemunit_DATA	= pacemaker.service
+endif
+
+EXTRA_DIST 	= pacemakerd.8.inc
+
+## SOURCES
+
+noinst_HEADERS		= pacemakerd.h
+
+pacemakerd_CFLAGS	= $(CFLAGS_HARDENED_EXE)
+pacemakerd_LDFLAGS	= $(LDFLAGS_HARDENED_EXE)
+
+pacemakerd_LDADD	= $(top_builddir)/lib/cluster/libcrmcluster.la $(top_builddir)/lib/common/libcrmcommon.la
+pacemakerd_LDADD	+= $(CLUSTERLIBS)
+pacemakerd_SOURCES	= pacemakerd.c
+if BUILD_CS_SUPPORT
+pacemakerd_SOURCES	+= pcmkd_corosync.c
+endif
+pacemakerd_SOURCES	+= pcmkd_messages.c
+pacemakerd_SOURCES	+= pcmkd_subdaemons.c
+
+CLEANFILES = $(man8_MANS)
diff --git a/daemons/pacemakerd/pacemaker.combined.upstart.in b/daemons/pacemakerd/pacemaker.combined.upstart.in
new file mode 100644
index 0000000..af59ff0
--- /dev/null
+++ b/daemons/pacemakerd/pacemaker.combined.upstart.in
@@ -0,0 +1,67 @@
+# pacemaker-corosync - High-Availability cluster
+#
+# Starts Corosync cluster engine and Pacemaker cluster manager.
+
+# if you use automatic start, uncomment the line below.
+#start on started local and runlevel [2345]
+
+stop on runlevel [0123456]
+kill timeout 3600
+respawn
+
+env prog=pacemakerd
+env sysconf=@CONFIGDIR@/pacemaker
+env rpm_lockdir=@localstatedir@/lock/subsys
+env deb_lockdir=@localstatedir@/lock
+
+script
+    [ -f "$sysconf" ] && . "$sysconf"
+    exec $prog
+end script
+
+pre-start script
+    pidof corosync || start corosync
+
+    # if you use corosync-notifyd, uncomment the line below.
+    #start corosync-notifyd
+
+    # give it time to fail.
+    sleep 2
+    pidof corosync || { exit 1; }
+
+    # if you use crm_mon, uncomment the line below.
+    #start crm_mon
+end script
+
+post-start script
+    [ -f "$sysconf" ] && . "$sysconf"
+    [ -z "$LOCK_FILE" -a -d "$rpm_lockdir" ] && LOCK_FILE="$rpm_lockdir/pacemaker"
+    [ -z "$LOCK_FILE" -a -d "$deb_lockdir" ] && LOCK_FILE="$deb_lockdir/pacemaker"
+    touch "$LOCK_FILE"
+    pidof $prog > "@localstatedir@/run/$prog.pid"
+end script
+
+post-stop script
+    [ -f "$sysconf" ] && . "$sysconf"
+    [ -z "$LOCK_FILE" -a -d "$rpm_lockdir" ] && LOCK_FILE="$rpm_lockdir/pacemaker"
+    [ -z "$LOCK_FILE" -a -d "$deb_lockdir" ] && LOCK_FILE="$deb_lockdir/pacemaker"
+    rm -f "$LOCK_FILE"
+    rm -f "@localstatedir@/run/$prog.pid"
+
+    # if you use corosync-notifyd, uncomment the line below.
+    #stop corosync-notifyd || true
+
+    # if you use watchdog of corosync, uncomment the line below.
+    #pidof corosync || false
+
+    pidof pacemaker-controld || stop corosync
+
+    # if you want to reboot a machine by watchdog of corosync when
+    # pacemakerd disappeared unexpectedly, uncomment the line below
+    # and invalidate above "respawn" stanza.
+    #pidof pacemaker-controld && killall -q -9 corosync
+
+    # if you use crm_mon, uncomment the line below.
+    #stop crm_mon
+
+end script
diff --git a/daemons/pacemakerd/pacemaker.service.in b/daemons/pacemakerd/pacemaker.service.in
new file mode 100644
index 0000000..3fd53d9
--- /dev/null
+++ b/daemons/pacemakerd/pacemaker.service.in
@@ -0,0 +1,103 @@
+[Unit]
+Description=Pacemaker High Availability Cluster Manager
+Documentation=man:pacemakerd
+Documentation=https://clusterlabs.org/pacemaker/doc/
+
+# DefaultDependencies takes care of sysinit.target,
+# basic.target, and shutdown.target
+
+# We need networking to bind to a network address. It is recommended not to
+# use Wants or Requires with network.target, and not to use
+# network-online.target for server daemons.
+After=network.target
+
+# Time syncs can make the clock jump backward, which messes with logging
+# and failure timestamps, so wait until it's done.
+After=time-sync.target
+
+# Managing systemd resources requires DBus.
+After=dbus.service
+Wants=dbus.service
+
+# Some OCF resources may have dependencies that aren't managed by the cluster;
+# these must be started before Pacemaker and stopped after it. The
+# resource-agents package provides this target, which lets system adminstrators
+# add drop-ins for those dependencies.
+After=resource-agents-deps.target
+Wants=resource-agents-deps.target
+
+After=syslog.service
+After=rsyslog.service
+After=corosync.service
+Requires=corosync.service
+
+# If Pacemaker respawns repeatedly, give up after this many tries in this time
+StartLimitBurst=5
+StartLimitIntervalSec=25s
+
+[Install]
+WantedBy=multi-user.target
+
+
+[Service]
+Type=simple
+KillMode=process
+NotifyAccess=main
+EnvironmentFile=-@CONFIGDIR@/pacemaker
+EnvironmentFile=-@CONFIGDIR@/sbd
+SuccessExitStatus=100
+
+ExecStart=@sbindir@/pacemakerd
+
+# Systemd v227 and above can limit the number of processes spawned by a
+# service. That is a bad idea for an HA cluster resource manager, so disable it
+# by default. The administrator can create a local override if they really want
+# a limit. If your systemd version does not support TasksMax, and you want to
+# get rid of the resulting log warnings, comment out this option.
+TasksMax=infinity
+
+# If pacemakerd doesn't stop, it's probably waiting on a cluster
+# resource.  Sending -KILL will just get the node fenced
+SendSIGKILL=no
+
+# Systemd's default of respawning a failed service after 100ms is too aggressive
+RestartSec=1s
+
+# If we ever hit the StartLimitInterval/StartLimitBurst limit, and the
+# admin wants to stop the cluster while pacemakerd is not running, it
+# might be a good idea to enable the ExecStopPost directive below.
+#
+# However, the node will likely end up being fenced as a result, so it's
+# not enabled by default.
+#
+# ExecStopPost=/usr/bin/killall -TERM pacemaker-attrd pacemaker-based \
+#              pacemaker-controld pacemaker-execd pacemaker-fenced \
+#              pacemaker-schedulerd
+
+# If you want Corosync to stop whenever Pacemaker is stopped,
+# uncomment the next line too:
+#
+# ExecStopPost=/bin/sh -c 'pidof pacemaker-controld || killall -TERM corosync'
+
+# Pacemaker will restart along with Corosync if Corosync is stopped while
+# Pacemaker is running.
+# In this case, if you want to be fenced always (if you do not want to restart)
+# uncomment ExecStopPost below.
+# 
+# ExecStopPost=/bin/sh -c 'pidof corosync || \
+#              /usr/bin/systemctl --no-block stop pacemaker'
+
+# When the service functions properly, it will wait to exit until all resources
+# have been stopped on the local node, and potentially across all nodes that
+# are shutting down.  The default of 30min should cover most typical cluster
+# configurations, but it may need an increase to adapt to local conditions
+# (e.g. a large, clustered database could conceivably take longer to stop).
+TimeoutStopSec=30min
+TimeoutStartSec=60s
+
+# Restart options include: no, on-success, on-failure, on-abort or always
+Restart=on-failure
+
+# crm_perror() writes directly to stderr, so ignore it here
+# to avoid double-logging with the wrong format
+StandardError=null
diff --git a/daemons/pacemakerd/pacemaker.upstart.in b/daemons/pacemakerd/pacemaker.upstart.in
new file mode 100644
index 0000000..7a54bc0
--- /dev/null
+++ b/daemons/pacemakerd/pacemaker.upstart.in
@@ -0,0 +1,33 @@
+# pacemaker - High-Availability cluster resource manager
+#
+# Starts pacemakerd
+
+stop on runlevel [0123456]
+kill timeout 3600
+respawn
+
+env prog=pacemakerd
+env sysconf=@CONFIGDIR@/pacemaker
+env rpm_lockdir=@localstatedir@/lock/subsys
+env deb_lockdir=@localstatedir@/lock
+
+script
+    [ -f "$sysconf" ] && . "$sysconf"
+    exec $prog
+end script
+
+post-start script
+    [ -f "$sysconf" ] && . "$sysconf"
+    [ -z "$LOCK_FILE" -a -d "$rpm_lockdir" ] && LOCK_FILE="$rpm_lockdir/pacemaker"
+    [ -z "$LOCK_FILE" -a -d "$deb_lockdir" ] && LOCK_FILE="$deb_lockdir/pacemaker"
+    touch "$LOCK_FILE"
+    pidof $prog > "@localstatedir@/run/$prog.pid"
+end script
+
+post-stop script
+    [ -f "$sysconf" ] && . "$sysconf"
+    [ -z "$LOCK_FILE" -a -d "$rpm_lockdir" ] && LOCK_FILE="$rpm_lockdir/pacemaker"
+    [ -z "$LOCK_FILE" -a -d "$deb_lockdir" ] && LOCK_FILE="$deb_lockdir/pacemaker"
+    rm -f "$LOCK_FILE"
+    rm -f "@localstatedir@/run/$prog.pid"
+end script
diff --git a/daemons/pacemakerd/pacemakerd.8.inc b/daemons/pacemakerd/pacemakerd.8.inc
new file mode 100644
index 0000000..902af4e
--- /dev/null
+++ b/daemons/pacemakerd/pacemakerd.8.inc
@@ -0,0 +1,5 @@
+[synopsis]
+pacemakerd [options]
+
+/subsidiary Pacemaker daemons/
+.SH OPTIONS
diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
new file mode 100644
index 0000000..9f77ccc
--- /dev/null
+++ b/daemons/pacemakerd/pacemakerd.c
@@ -0,0 +1,483 @@
+/*
+ * Copyright 2010-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include "pacemakerd.h"
+
+#include <pwd.h>
+#include <errno.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdbool.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+
+#include <crm/crm.h>  /* indirectly: CRM_EX_* */
+#include <crm/msg_xml.h>
+#include <crm/common/mainloop.h>
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/ipc_pacemakerd.h>
+#include <crm/common/output_internal.h>
+#include <crm/cluster/internal.h>
+#include <crm/cluster.h>
+
+#define SUMMARY "pacemakerd - primary Pacemaker daemon that launches and monitors all subsidiary Pacemaker daemons"
+
+struct {
+    gboolean features;
+    gboolean foreground;
+    gboolean shutdown;
+    gboolean standby;
+} options;
+
+static pcmk__output_t *out = NULL;
+
+static pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+PCMK__OUTPUT_ARGS("features")
+static int
+pacemakerd_features(pcmk__output_t *out, va_list args) {
+    out->info(out, "Pacemaker %s (Build: %s)\n Supporting v%s: %s", PACEMAKER_VERSION,
+              BUILD_VERSION, CRM_FEATURE_SET, CRM_FEATURES);
+    return pcmk_rc_ok;
+}
+
+PCMK__OUTPUT_ARGS("features")
+static int
+pacemakerd_features_xml(pcmk__output_t *out, va_list args) {
+    gchar **feature_list = g_strsplit(CRM_FEATURES, " ", 0);
+
+    pcmk__output_xml_create_parent(out, "pacemakerd",
+                                   "version", PACEMAKER_VERSION,
+                                   "build", BUILD_VERSION,
+                                   "feature_set", CRM_FEATURE_SET,
+                                   NULL);
+    out->begin_list(out, NULL, NULL, "features");
+
+    for (char **s = feature_list; *s != NULL; s++) {
+        pcmk__output_create_xml_text_node(out, "feature", *s);
+    }
+
+    out->end_list(out);
+
+    g_strfreev(feature_list);
+    return pcmk_rc_ok;
+}
+
+static pcmk__message_entry_t fmt_functions[] = {
+    { "features", "default", pacemakerd_features },
+    { "features", "xml", pacemakerd_features_xml },
+
+    { NULL, NULL, NULL }
+};
+
+static gboolean
+pid_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    return TRUE;
+}
+
+static gboolean
+standby_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+    options.standby = TRUE;
+    pcmk__set_env_option("node_start_state", "standby");
+    return TRUE;
+}
+
+static GOptionEntry entries[] = {
+    { "features", 'F', 0, G_OPTION_ARG_NONE, &options.features,
+      "Display full version and list of features Pacemaker was built with",
+      NULL },
+    { "foreground", 'f', 0, G_OPTION_ARG_NONE, &options.foreground,
+      "(Ignored) Pacemaker always runs in the foreground",
+      NULL },
+    { "pid-file", 'p', 0, G_OPTION_ARG_CALLBACK, pid_cb,
+      "(Ignored) Daemon pid file location",
+      "FILE" },
+    { "shutdown", 'S', 0, G_OPTION_ARG_NONE, &options.shutdown,
+      "Instruct Pacemaker to shutdown on this machine",
+      NULL },
+    { "standby", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, standby_cb,
+      "Start node in standby state",
+      NULL },
+
+    { NULL }
+};
+
+static void
+pcmk_ignore(int nsig)
+{
+    crm_info("Ignoring signal %s (%d)", strsignal(nsig), nsig);
+}
+
+static void
+pcmk_sigquit(int nsig)
+{
+    pcmk__panic(__func__);
+}
+
+static void
+mcp_chown(const char *path, uid_t uid, gid_t gid)
+{
+    int rc = chown(path, uid, gid);
+
+    if (rc < 0) {
+        crm_warn("Cannot change the ownership of %s to user %s and gid %d: %s",
+                 path, CRM_DAEMON_USER, gid, pcmk_rc_str(errno));
+    }
+}
+
+static void
+create_pcmk_dirs(void)
+{
+    uid_t pcmk_uid = 0;
+    gid_t pcmk_gid = 0;
+
+    const char *dirs[] = {
+        CRM_PACEMAKER_DIR, // core/blackbox/scheduler/CIB files
+        CRM_CORE_DIR,      // core files
+        CRM_BLACKBOX_DIR,  // blackbox dumps
+        PE_STATE_DIR,      // scheduler inputs
+        CRM_CONFIG_DIR,    // the Cluster Information Base (CIB)
+        // Don't build CRM_RSCTMP_DIR, pacemaker-execd will do it
+        NULL
+    };
+
+    if (pcmk_daemon_user(&pcmk_uid, &pcmk_gid) < 0) {
+        crm_err("Cluster user %s does not exist, aborting Pacemaker startup",
+                CRM_DAEMON_USER);
+        crm_exit(CRM_EX_NOUSER);
+    }
+
+    // Used by some resource agents
+    if ((mkdir(CRM_STATE_DIR, 0750) < 0) && (errno != EEXIST)) {
+        crm_warn("Could not create directory " CRM_STATE_DIR ": %s",
+                 pcmk_rc_str(errno));
+    } else {
+        mcp_chown(CRM_STATE_DIR, pcmk_uid, pcmk_gid);
+    }
+
+    for (int i = 0; dirs[i] != NULL; ++i) {
+        int rc = pcmk__build_path(dirs[i], 0750);
+
+        if (rc != pcmk_rc_ok) {
+            crm_warn("Could not create directory %s: %s",
+                     dirs[i], pcmk_rc_str(rc));
+        } else {
+            mcp_chown(dirs[i], pcmk_uid, pcmk_gid);
+        }
+    }
+}
+
+static void
+remove_core_file_limit(void)
+{
+    struct rlimit cores;
+
+    // Get current limits
+    if (getrlimit(RLIMIT_CORE, &cores) < 0) {
+        crm_notice("Unable to check system core file limits "
+                   "(consider ensuring the size is unlimited): %s",
+                   strerror(errno));
+        return;
+    }
+
+    // Check whether core dumps are disabled
+    if (cores.rlim_max == 0) {
+        if (geteuid() != 0) { // Yes, and there's nothing we can do about it
+            crm_notice("Core dumps are disabled (consider enabling them)");
+            return;
+        }
+        cores.rlim_max = RLIM_INFINITY; // Yes, but we're root, so enable them
+    }
+
+    // Raise soft limit to hard limit (if not already done)
+    if (cores.rlim_cur != cores.rlim_max) {
+        cores.rlim_cur = cores.rlim_max;
+        if (setrlimit(RLIMIT_CORE, &cores) < 0) {
+            crm_notice("Unable to raise system limit on core file size "
+                       "(consider doing so manually): %s",
+                       strerror(errno));
+            return;
+        }
+    }
+
+    if (cores.rlim_cur == RLIM_INFINITY) {
+        crm_trace("Core file size is unlimited");
+    } else {
+        crm_trace("Core file size is limited to %llu bytes",
+                  (unsigned long long) cores.rlim_cur);
+    }
+}
+
+static void
+pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
+                    enum pcmk_ipc_event event_type, crm_exit_t status,
+                    void *event_data, void *user_data)
+{
+    pcmk_pacemakerd_api_reply_t *reply = event_data;
+
+    switch (event_type) {
+        case pcmk_ipc_event_reply:
+            break;
+
+        default:
+            return;
+    }
+
+    if (status != CRM_EX_OK) {
+        out->err(out, "Bad reply from pacemakerd: %s", crm_exit_str(status));
+        return;
+    }
+
+    if (reply->reply_type != pcmk_pacemakerd_reply_shutdown) {
+        out->err(out, "Unknown reply type %d from pacemakerd",
+                 reply->reply_type);
+    }
+}
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
+    pcmk__add_main_args(context, entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    int rc = pcmk_rc_ok;
+    crm_exit_t exit_code = CRM_EX_OK;
+
+    GError *error = NULL;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, "p");
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    bool old_instance_connected = false;
+
+    pcmk_ipc_api_t *old_instance = NULL;
+    qb_ipcs_service_t *ipcs = NULL;
+
+    subdaemon_check_progress = time(NULL);
+
+    setenv("LC_ALL", "C", 1); // Ensure logs are in a common language
+
+    crm_log_preinit(NULL, argc, argv);
+    mainloop_add_signal(SIGHUP, pcmk_ignore);
+    mainloop_add_signal(SIGQUIT, pcmk_sigquit);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if ((rc != pcmk_rc_ok) || (out == NULL)) {
+        exit_code = CRM_EX_ERROR;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    pcmk__force_args(context, &error, "%s --xml-simple-list", g_get_prgname());
+
+    pcmk__register_messages(out, fmt_functions);
+
+    if (options.features) {
+        out->message(out, "features");
+        exit_code = CRM_EX_OK;
+        goto done;
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    pcmk__set_env_option("mcp", "true");
+
+    if (options.shutdown) {
+        pcmk__cli_init_logging("pacemakerd", args->verbosity);
+    } else {
+        crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
+    }
+
+    crm_debug("Checking for existing Pacemaker instance");
+
+    rc = pcmk_new_ipc_api(&old_instance, pcmk_ipc_pacemakerd);
+    if (old_instance == NULL) {
+        out->err(out, "Could not check for existing pacemakerd: %s", pcmk_rc_str(rc));
+        exit_code = pcmk_rc2exitc(rc);
+        goto done;
+    }
+
+    pcmk_register_ipc_callback(old_instance, pacemakerd_event_cb, NULL);
+    rc = pcmk_connect_ipc(old_instance, pcmk_ipc_dispatch_sync);
+    old_instance_connected = pcmk_ipc_is_connected(old_instance);
+
+    if (options.shutdown) {
+        if (old_instance_connected) {
+            rc = pcmk_pacemakerd_api_shutdown(old_instance, crm_system_name);
+            pcmk_dispatch_ipc(old_instance);
+
+            exit_code = pcmk_rc2exitc(rc);
+
+            if (exit_code != CRM_EX_OK) {
+                pcmk_free_ipc_api(old_instance);
+                goto done;
+            }
+
+            /* We get the ACK immediately, and the response right after that,
+             * but it might take a while for pacemakerd to get around to
+             * shutting down.  Wait for that to happen (with 30-minute timeout).
+             */
+            for (int i = 0; i < 900; i++) {
+                if (!pcmk_ipc_is_connected(old_instance)) {
+                    exit_code = CRM_EX_OK;
+                    pcmk_free_ipc_api(old_instance);
+                    goto done;
+                }
+
+                sleep(2);
+            }
+
+            exit_code = CRM_EX_TIMEOUT;
+            pcmk_free_ipc_api(old_instance);
+            goto done;
+
+        } else {
+            out->err(out, "Could not request shutdown "
+                     "of existing Pacemaker instance: %s", pcmk_rc_str(rc));
+            pcmk_free_ipc_api(old_instance);
+            exit_code = CRM_EX_DISCONNECT;
+            goto done;
+        }
+
+    } else if (old_instance_connected) {
+        pcmk_free_ipc_api(old_instance);
+        crm_err("Aborting start-up because active Pacemaker instance found");
+        exit_code = CRM_EX_FATAL;
+        goto done;
+    }
+
+    pcmk_free_ipc_api(old_instance);
+
+    /* Don't allow any accidental output after this point. */
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+        out = NULL;
+    }
+
+#ifdef SUPPORT_COROSYNC
+    if (mcp_read_config() == FALSE) {
+        crm_exit(CRM_EX_UNAVAILABLE);
+    }
+#endif
+
+    // OCF shell functions and cluster-glue need facility under different name
+    {
+        const char *facility = pcmk__env_option(PCMK__ENV_LOGFACILITY);
+
+        if (!pcmk__str_eq(facility, PCMK__VALUE_NONE,
+                          pcmk__str_casei|pcmk__str_null_matches)) {
+            setenv("HA_LOGFACILITY", facility, 1);
+        }
+    }
+
+    crm_notice("Starting Pacemaker %s "CRM_XS" build=%s features:%s",
+               PACEMAKER_VERSION, BUILD_VERSION, CRM_FEATURES);
+    mainloop = g_main_loop_new(NULL, FALSE);
+
+    remove_core_file_limit();
+    create_pcmk_dirs();
+    pcmk__serve_pacemakerd_ipc(&ipcs, &mcp_ipc_callbacks);
+
+#ifdef SUPPORT_COROSYNC
+    /* Allows us to block shutdown */
+    if (!cluster_connect_cfg()) {
+        exit_code = CRM_EX_PROTOCOL;
+        goto done;
+    }
+#endif
+
+    if (pcmk__locate_sbd() > 0) {
+        setenv("PCMK_watchdog", "true", 1);
+        running_with_sbd = TRUE;
+    } else {
+        setenv("PCMK_watchdog", "false", 1);
+    }
+
+    switch (find_and_track_existing_processes()) {
+        case pcmk_rc_ok:
+            break;
+        case pcmk_rc_ipc_unauthorized:
+            exit_code = CRM_EX_CANTCREAT;
+            goto done;
+        default:
+            exit_code = CRM_EX_FATAL;
+            goto done;
+    };
+
+    mainloop_add_signal(SIGTERM, pcmk_shutdown);
+    mainloop_add_signal(SIGINT, pcmk_shutdown);
+
+    if ((running_with_sbd) && pcmk__get_sbd_sync_resource_startup()) {
+        crm_notice("Waiting for startup-trigger from SBD.");
+        pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_WAITPING;
+        startup_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, init_children_processes, NULL);
+    } else {
+        if (running_with_sbd) {
+            crm_warn("Enabling SBD_SYNC_RESOURCE_STARTUP would (if supported "
+                     "by your SBD version) improve reliability of "
+                     "interworking between SBD & pacemaker.");
+        }
+        pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
+        init_children_processes(NULL);
+    }
+
+    crm_notice("Pacemaker daemon successfully started and accepting connections");
+    g_main_loop_run(mainloop);
+
+    if (ipcs) {
+        crm_trace("Closing IPC server");
+        mainloop_del_ipc_server(ipcs);
+        ipcs = NULL;
+    }
+
+    g_main_loop_unref(mainloop);
+#ifdef SUPPORT_COROSYNC
+    cluster_disconnect_cfg();
+#endif
+
+done:
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    pcmk__output_and_clear_error(&error, out);
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+    }
+    pcmk__unregister_formats();
+    crm_exit(exit_code);
+}
diff --git a/daemons/pacemakerd/pacemakerd.h b/daemons/pacemakerd/pacemakerd.h
new file mode 100644
index 0000000..b2a6864
--- /dev/null
+++ b/daemons/pacemakerd/pacemakerd.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2010-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <stdint.h>
+
+#define MAX_RESPAWN		100
+
+extern GMainLoop *mainloop;
+extern struct qb_ipcs_service_handlers mcp_ipc_callbacks;
+extern const char *pacemakerd_state;
+extern gboolean running_with_sbd;
+extern unsigned int shutdown_complete_state_reported_to;
+extern gboolean shutdown_complete_state_reported_client_closed;
+extern crm_trigger_t *shutdown_trigger;
+extern crm_trigger_t *startup_trigger;
+extern time_t subdaemon_check_progress;
+
+gboolean mcp_read_config(void);
+
+gboolean cluster_connect_cfg(void);
+void cluster_disconnect_cfg(void);
+int find_and_track_existing_processes(void);
+gboolean init_children_processes(void *user_data);
+void restart_cluster_subdaemons(void);
+void pcmk_shutdown(int nsig);
+void pcmkd_shutdown_corosync(void);
+bool pcmkd_corosync_connected(void);
diff --git a/daemons/pacemakerd/pcmkd_corosync.c b/daemons/pacemakerd/pcmkd_corosync.c
new file mode 100644
index 0000000..2648756
--- /dev/null
+++ b/daemons/pacemakerd/pcmkd_corosync.c
@@ -0,0 +1,371 @@
+/*
+ * Copyright 2010-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include "pacemakerd.h"
+
+#include <sys/utsname.h>
+#include <sys/stat.h>           /* for calls to stat() */
+#include <libgen.h>             /* For basename() and dirname() */
+
+#include <sys/types.h>
+#include <pwd.h>                /* For getpwname() */
+
+#include <corosync/hdb.h>
+#include <corosync/cfg.h>
+#include <corosync/cpg.h>
+#include <corosync/cmap.h>
+
+#include <crm/cluster/internal.h>
+#include <crm/common/ipc.h>     /* for crm_ipc_is_authentic_process */
+#include <crm/common/mainloop.h>
+
+#include <crm/common/ipc_internal.h>  /* PCMK__SPECIAL_PID* */
+
+static corosync_cfg_handle_t cfg_handle = 0;
+static mainloop_timer_t *reconnect_timer = NULL;
+
+/* =::=::=::= CFG - Shutdown stuff =::=::=::= */
+
+static void
+cfg_shutdown_callback(corosync_cfg_handle_t h, corosync_cfg_shutdown_flags_t flags)
+{
+    crm_info("Corosync wants to shut down: %s",
+             (flags == COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE) ? "immediate" :
+             (flags == COROSYNC_CFG_SHUTDOWN_FLAG_REGARDLESS) ? "forced" : "optional");
+
+    /* Never allow corosync to shut down while we're running */
+    corosync_cfg_replyto_shutdown(h, COROSYNC_CFG_SHUTDOWN_FLAG_NO);
+}
+
+static corosync_cfg_callbacks_t cfg_callbacks = {
+    .corosync_cfg_shutdown_callback = cfg_shutdown_callback,
+};
+
+static int
+pcmk_cfg_dispatch(gpointer user_data)
+{
+    corosync_cfg_handle_t *handle = (corosync_cfg_handle_t *) user_data;
+    cs_error_t rc = corosync_cfg_dispatch(*handle, CS_DISPATCH_ALL);
+
+    if (rc != CS_OK) {
+        return -1;
+    }
+    return 0;
+}
+
+static void
+close_cfg(void)
+{
+    if (cfg_handle != 0) {
+#ifdef HAVE_COROSYNC_CFG_TRACKSTART
+        /* Ideally, we would call corosync_cfg_trackstop(cfg_handle) here, but a
+         * bug in corosync 3.1.1 and 3.1.2 makes it hang forever. Thankfully,
+         * it's not necessary since we exit immediately after this.
+         */
+#endif
+        corosync_cfg_finalize(cfg_handle);
+        cfg_handle = 0;
+    }
+}
+
+static gboolean
+cluster_reconnect_cb(gpointer data)
+{
+    if (cluster_connect_cfg()) {
+        mainloop_timer_del(reconnect_timer);
+        reconnect_timer = NULL;
+        crm_notice("Cluster reconnect succeeded");
+        mcp_read_config();
+        restart_cluster_subdaemons();
+        return G_SOURCE_REMOVE;
+    } else {
+        crm_info("Cluster reconnect failed "
+                 "(connection will be reattempted once per second)");
+    }
+    /*
+     * In theory this will continue forever. In practice the CIB connection from
+     * attrd will timeout and shut down Pacemaker when it gets bored.
+     */
+    return G_SOURCE_CONTINUE;
+}
+
+
+static void
+cfg_connection_destroy(gpointer user_data)
+{
+    crm_warn("Lost connection to cluster layer "
+             "(connection will be reattempted once per second)");
+    corosync_cfg_finalize(cfg_handle);
+    cfg_handle = 0;
+    reconnect_timer = mainloop_timer_add("corosync reconnect", 1000, TRUE, cluster_reconnect_cb, NULL);
+    mainloop_timer_start(reconnect_timer);
+}
+
+void
+cluster_disconnect_cfg(void)
+{
+    close_cfg();
+    if (reconnect_timer != NULL) {
+        /* The mainloop should be gone by this point, so this isn't necessary,
+         * but cleaning up memory should make valgrind happier.
+         */
+        mainloop_timer_del(reconnect_timer);
+        reconnect_timer = NULL;
+    }
+}
+
+#define cs_repeat(counter, max, code) do {		\
+	code;						\
+	if(rc == CS_ERR_TRY_AGAIN || rc == CS_ERR_QUEUE_FULL) {  \
+	    counter++;					\
+	    crm_debug("Retrying Corosync operation after %ds", counter);    \
+	    sleep(counter);				\
+	} else {                                        \
+            break;                                      \
+	}						\
+    } while(counter < max)
+
+gboolean
+cluster_connect_cfg(void)
+{
+    cs_error_t rc;
+    int fd = -1, retries = 0, rv;
+    uid_t found_uid = 0;
+    gid_t found_gid = 0;
+    pid_t found_pid = 0;
+    uint32_t nodeid;
+
+    static struct mainloop_fd_callbacks cfg_fd_callbacks = {
+        .dispatch = pcmk_cfg_dispatch,
+        .destroy = cfg_connection_destroy,
+    };
+
+    cs_repeat(retries, 30, rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks));
+
+    if (rc != CS_OK) {
+        crm_crit("Could not connect to Corosync CFG: %s " CRM_XS " rc=%d",
+                 cs_strerror(rc), rc);
+        return FALSE;
+    }
+
+    rc = corosync_cfg_fd_get(cfg_handle, &fd);
+    if (rc != CS_OK) {
+        crm_crit("Could not get Corosync CFG descriptor: %s " CRM_XS " rc=%d",
+                 cs_strerror(rc), rc);
+        goto bail;
+    }
+
+    /* CFG provider run as root (in given user namespace, anyway)? */
+    if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
+                                            &found_uid, &found_gid))) {
+        crm_crit("Rejecting Corosync CFG provider because process %lld "
+                 "is running as uid %lld gid %lld, not root",
+                  (long long) PCMK__SPECIAL_PID_AS_0(found_pid),
+                 (long long) found_uid, (long long) found_gid);
+        goto bail;
+    } else if (rv < 0) {
+        crm_crit("Could not authenticate Corosync CFG provider: %s "
+                 CRM_XS " rc=%d", strerror(-rv), -rv);
+        goto bail;
+    }
+
+    retries = 0;
+    cs_repeat(retries, 30, rc = corosync_cfg_local_get(cfg_handle, &nodeid));
+    if (rc != CS_OK) {
+        crm_crit("Could not get local node ID from Corosync: %s "
+                 CRM_XS " rc=%d", cs_strerror(rc), rc);
+        goto bail;
+    }
+    crm_debug("Corosync reports local node ID is %lu", (unsigned long) nodeid);
+
+#ifdef HAVE_COROSYNC_CFG_TRACKSTART
+    retries = 0;
+    cs_repeat(retries, 30, rc = corosync_cfg_trackstart(cfg_handle, 0));
+    if (rc != CS_OK) {
+        crm_crit("Could not enable Corosync CFG shutdown tracker: %s " CRM_XS " rc=%d",
+                 cs_strerror(rc), rc);
+        goto bail;
+    }
+#endif
+
+    mainloop_add_fd("corosync-cfg", G_PRIORITY_DEFAULT, fd, &cfg_handle, &cfg_fd_callbacks);
+    return TRUE;
+
+  bail:
+    corosync_cfg_finalize(cfg_handle);
+    return FALSE;
+}
+
+void
+pcmkd_shutdown_corosync(void)
+{
+    cs_error_t rc;
+
+    if (cfg_handle == 0) {
+        crm_warn("Unable to shut down Corosync: No connection");
+        return;
+    }
+    crm_info("Asking Corosync to shut down");
+    rc = corosync_cfg_try_shutdown(cfg_handle,
+                                    COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE);
+    if (rc == CS_OK) {
+        close_cfg();
+    } else {
+        crm_warn("Corosync shutdown failed: %s " CRM_XS " rc=%d",
+                 cs_strerror(rc), rc);
+    }
+}
+
+bool
+pcmkd_corosync_connected(void)
+{
+    cpg_handle_t local_handle = 0;
+    cpg_model_v1_data_t cpg_model_info = {CPG_MODEL_V1, NULL, NULL, NULL, 0};
+    int fd = -1;
+
+    if (cpg_model_initialize(&local_handle, CPG_MODEL_V1, (cpg_model_data_t *) &cpg_model_info, NULL) != CS_OK) {
+        return false;
+    }
+
+    if (cpg_fd_get(local_handle, &fd) != CS_OK) {
+        return false;
+    }
+
+    cpg_finalize(local_handle);
+
+    return true;
+}
+
+/* =::=::=::= Configuration =::=::=::= */
+static int
+get_config_opt(uint64_t unused, cmap_handle_t object_handle, const char *key, char **value,
+               const char *fallback)
+{
+    int rc = 0, retries = 0;
+
+    cs_repeat(retries, 5, rc = cmap_get_string(object_handle, key, value));
+    if (rc != CS_OK) {
+        crm_trace("Search for %s failed %d, defaulting to %s", key, rc, fallback);
+        pcmk__str_update(value, fallback);
+    }
+    crm_trace("%s: %s", key, *value);
+    return rc;
+}
+
+gboolean
+mcp_read_config(void)
+{
+    cs_error_t rc = CS_OK;
+    int retries = 0;
+    cmap_handle_t local_handle;
+    uint64_t config = 0;
+    int fd = -1;
+    uid_t found_uid = 0;
+    gid_t found_gid = 0;
+    pid_t found_pid = 0;
+    int rv;
+    enum cluster_type_e stack;
+
+    // There can be only one possibility
+    do {
+        rc = pcmk__init_cmap(&local_handle);
+        if (rc != CS_OK) {
+            retries++;
+            crm_info("Could not connect to Corosync CMAP: %s (retrying in %ds) "
+                     CRM_XS " rc=%d", cs_strerror(rc), retries, rc);
+            sleep(retries);
+
+        } else {
+            break;
+        }
+
+    } while (retries < 5);
+
+    if (rc != CS_OK) {
+        crm_crit("Could not connect to Corosync CMAP: %s "
+                 CRM_XS " rc=%d", cs_strerror(rc), rc);
+        return FALSE;
+    }
+
+    rc = cmap_fd_get(local_handle, &fd);
+    if (rc != CS_OK) {
+        crm_crit("Could not get Corosync CMAP descriptor: %s " CRM_XS " rc=%d",
+                 cs_strerror(rc), rc);
+        cmap_finalize(local_handle);
+        return FALSE;
+    }
+
+    /* CMAP provider run as root (in given user namespace, anyway)? */
+    if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
+                                            &found_uid, &found_gid))) {
+        crm_crit("Rejecting Corosync CMAP provider because process %lld "
+                 "is running as uid %lld gid %lld, not root",
+                 (long long) PCMK__SPECIAL_PID_AS_0(found_pid),
+                 (long long) found_uid, (long long) found_gid);
+        cmap_finalize(local_handle);
+        return FALSE;
+    } else if (rv < 0) {
+        crm_crit("Could not authenticate Corosync CMAP provider: %s "
+                 CRM_XS " rc=%d", strerror(-rv), -rv);
+        cmap_finalize(local_handle);
+        return FALSE;
+    }
+
+    stack = get_cluster_type();
+    if (stack != pcmk_cluster_corosync) {
+        crm_crit("Expected Corosync cluster layer but detected %s "
+                 CRM_XS " stack=%d", name_for_cluster_type(stack), stack);
+        return FALSE;
+    }
+
+    crm_info("Reading configuration for %s stack",
+             name_for_cluster_type(stack));
+    pcmk__set_env_option(PCMK__ENV_CLUSTER_TYPE, "corosync");
+    pcmk__set_env_option(PCMK__ENV_QUORUM_TYPE, "corosync");
+
+    // If debug logging is not configured, check whether corosync has it
+    if (pcmk__env_option(PCMK__ENV_DEBUG) == NULL) {
+        char *debug_enabled = NULL;
+
+        get_config_opt(config, local_handle, "logging.debug", &debug_enabled, "off");
+
+        if (crm_is_true(debug_enabled)) {
+            pcmk__set_env_option(PCMK__ENV_DEBUG, "1");
+            if (get_crm_log_level() < LOG_DEBUG) {
+                set_crm_log_level(LOG_DEBUG);
+            }
+
+        } else {
+            pcmk__set_env_option(PCMK__ENV_DEBUG, "0");
+        }
+
+        free(debug_enabled);
+    }
+
+    if(local_handle){
+        gid_t gid = 0;
+        if (pcmk_daemon_user(NULL, &gid) < 0) {
+            crm_warn("Could not authorize group with Corosync " CRM_XS
+                     " No group found for user %s", CRM_DAEMON_USER);
+
+        } else {
+            char key[PATH_MAX];
+            snprintf(key, PATH_MAX, "uidgid.gid.%u", gid);
+            rc = cmap_set_uint8(local_handle, key, 1);
+            if (rc != CS_OK) {
+                crm_warn("Could not authorize group with Corosync: %s " CRM_XS
+                         " group=%u rc=%d", pcmk__cs_err_str(rc), gid, rc);
+            }
+        }
+    }
+    cmap_finalize(local_handle);
+
+    return TRUE;
+}
diff --git a/daemons/pacemakerd/pcmkd_messages.c b/daemons/pacemakerd/pcmkd_messages.c
new file mode 100644
index 0000000..7ed9899
--- /dev/null
+++ b/daemons/pacemakerd/pcmkd_messages.c
@@ -0,0 +1,278 @@
+/*
+ * Copyright 2010-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include "pacemakerd.h"
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+
+#include <errno.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <string.h>
+#include <time.h>
+#include <sys/types.h>
+
+static GHashTable *pcmkd_handlers = NULL;
+
+static xmlNode *
+handle_node_cache_request(pcmk__request_t *request)
+{
+    crm_trace("Ignoring request from client %s to purge node "
+              "because peer cache is not used",
+              pcmk__client_name(request->ipc_client));
+
+    pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
+                       "ack", NULL, CRM_EX_OK);
+    return NULL;
+}
+
+static xmlNode *
+handle_ping_request(pcmk__request_t *request)
+{
+    xmlNode *msg = request->xml;
+
+    const char *value = NULL;
+    xmlNode *ping = NULL;
+    xmlNode *reply = NULL;
+    const char *from = crm_element_value(msg, F_CRM_SYS_FROM);
+
+    /* Pinged for status */
+    crm_trace("Pinged from " F_CRM_SYS_FROM "='%s' " F_CRM_ORIGIN "='%s'",
+              pcmk__s(from, ""),
+              pcmk__s(crm_element_value(msg, F_CRM_ORIGIN), ""));
+
+    pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
+                       "ack", NULL, CRM_EX_INDETERMINATE);
+
+    ping = create_xml_node(NULL, XML_CRM_TAG_PING);
+    value = crm_element_value(msg, F_CRM_SYS_TO);
+    crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value);
+    crm_xml_add(ping, XML_PING_ATTR_PACEMAKERDSTATE, pacemakerd_state);
+    crm_xml_add_ll(ping, XML_ATTR_TSTAMP,
+                   (long long) subdaemon_check_progress);
+    crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");
+    reply = create_reply(msg, ping);
+
+    free_xml(ping);
+
+    if (reply == NULL) {
+        pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
+                            "Failed building ping reply for client %s",
+                            pcmk__client_name(request->ipc_client));
+    } else {
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    }
+
+    /* just proceed state on sbd pinging us */
+    if (from && strstr(from, "sbd")) {
+        if (pcmk__str_eq(pacemakerd_state, XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE, pcmk__str_none)) {
+            if (pcmk__get_sbd_sync_resource_startup()) {
+                crm_notice("Shutdown-complete-state passed to SBD.");
+            }
+
+            shutdown_complete_state_reported_to = request->ipc_client->pid;
+
+        } else if (pcmk__str_eq(pacemakerd_state, XML_PING_ATTR_PACEMAKERDSTATE_WAITPING, pcmk__str_none)) {
+            crm_notice("Received startup-trigger from SBD.");
+            pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
+            mainloop_set_trigger(startup_trigger);
+        }
+    }
+
+    return reply;
+}
+
+static xmlNode *
+handle_shutdown_request(pcmk__request_t *request)
+{
+    xmlNode *msg = request->xml;
+
+    xmlNode *shutdown = NULL;
+    xmlNode *reply = NULL;
+
+    /* Only allow privileged users (i.e. root or hacluster) to shut down
+     * Pacemaker from the command line (or direct IPC), so that other users
+     * are forced to go through the CIB and have ACLs applied.
+     */
+    bool allowed = pcmk_is_set(request->ipc_client->flags, pcmk__client_privileged);
+
+    pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
+                       "ack", NULL, CRM_EX_INDETERMINATE);
+
+    shutdown = create_xml_node(NULL, XML_CIB_ATTR_SHUTDOWN);
+
+    if (allowed) {
+        crm_notice("Shutting down in response to IPC request %s from %s",
+                   crm_element_value(msg, F_CRM_REFERENCE),
+                   crm_element_value(msg, F_CRM_ORIGIN));
+        crm_xml_add_int(shutdown, XML_LRM_ATTR_OPSTATUS, CRM_EX_OK);
+    } else {
+        crm_warn("Ignoring shutdown request from unprivileged client %s",
+                 pcmk__client_name(request->ipc_client));
+        crm_xml_add_int(shutdown, XML_LRM_ATTR_OPSTATUS, CRM_EX_INSUFFICIENT_PRIV);
+    }
+
+    reply = create_reply(msg, shutdown);
+    free_xml(shutdown);
+
+    if (reply == NULL) {
+        pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
+                            "Failed building shutdown reply for client %s",
+                            pcmk__client_name(request->ipc_client));
+    } else {
+        pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    }
+
+    if (allowed) {
+        pcmk_shutdown(15);
+    }
+
+    return reply;
+}
+
+static xmlNode *
+handle_unknown_request(pcmk__request_t *request)
+{
+    pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
+                       "ack", NULL, CRM_EX_INVALID_PARAM);
+
+    pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
+                        "Unknown IPC request type '%s' (bug?)",
+                        pcmk__client_name(request->ipc_client));
+    return NULL;
+}
+
+static void
+pcmkd_register_handlers(void)
+{
+    pcmk__server_command_t handlers[] = {
+        { CRM_OP_RM_NODE_CACHE, handle_node_cache_request },
+        { CRM_OP_PING, handle_ping_request },
+        { CRM_OP_QUIT, handle_shutdown_request },
+        { NULL, handle_unknown_request },
+    };
+
+    pcmkd_handlers = pcmk__register_handlers(handlers);
+}
+
+static int32_t
+pcmk_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
+{
+    crm_trace("Connection %p", c);
+    if (pcmk__new_client(c, uid, gid) == NULL) {
+        return -EIO;
+    }
+    return 0;
+}
+
+/* Error code means? */
+static int32_t
+pcmk_ipc_closed(qb_ipcs_connection_t * c)
+{
+    pcmk__client_t *client = pcmk__find_client(c);
+
+    if (client == NULL) {
+        return 0;
+    }
+    crm_trace("Connection %p", c);
+    if (shutdown_complete_state_reported_to == client->pid) {
+        shutdown_complete_state_reported_client_closed = TRUE;
+        if (shutdown_trigger) {
+            mainloop_set_trigger(shutdown_trigger);
+        }
+    }
+    pcmk__free_client(client);
+    return 0;
+}
+
+static void
+pcmk_ipc_destroy(qb_ipcs_connection_t * c)
+{
+    crm_trace("Connection %p", c);
+    pcmk_ipc_closed(c);
+}
+
+/* Exit code means? */
+static int32_t
+pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
+{
+    uint32_t id = 0;
+    uint32_t flags = 0;
+    xmlNode *msg = NULL;
+    pcmk__client_t *c = pcmk__find_client(qbc);
+
+    CRM_CHECK(c != NULL, return 0);
+
+    if (pcmkd_handlers == NULL) {
+        pcmkd_register_handlers();
+    }
+
+    msg = pcmk__client_data2xml(c, data, &id, &flags);
+    if (msg == NULL) {
+        pcmk__ipc_send_ack(c, id, flags, "ack", NULL, CRM_EX_PROTOCOL);
+        return 0;
+
+    } else {
+        char *log_msg = NULL;
+        const char *reason = NULL;
+        xmlNode *reply = NULL;
+
+        pcmk__request_t request = {
+            .ipc_client     = c,
+            .ipc_id         = id,
+            .ipc_flags      = flags,
+            .peer           = NULL,
+            .xml            = msg,
+            .call_options   = 0,
+            .result         = PCMK__UNKNOWN_RESULT,
+        };
+
+        request.op = crm_element_value_copy(request.xml, F_CRM_TASK);
+        CRM_CHECK(request.op != NULL, return 0);
+
+        reply = pcmk__process_request(&request, pcmkd_handlers);
+
+        if (reply != NULL) {
+            pcmk__ipc_send_xml(c, id, reply, crm_ipc_server_event);
+            free_xml(reply);
+        }
+
+        reason = request.result.exit_reason;
+
+        log_msg = crm_strdup_printf("Processed %s request from %s %s: %s%s%s%s",
+                                    request.op, pcmk__request_origin_type(&request),
+                                    pcmk__request_origin(&request),
+                                    pcmk_exec_status_str(request.result.execution_status),
+                                    (reason == NULL)? "" : " (",
+                                    (reason == NULL)? "" : reason,
+                                    (reason == NULL)? "" : ")");
+
+        if (!pcmk__result_ok(&request.result)) {
+            crm_warn("%s", log_msg);
+        } else {
+            crm_debug("%s", log_msg);
+        }
+
+        free(log_msg);
+        pcmk__reset_request(&request);
+    }
+
+    free_xml(msg);
+    return 0;
+}
+
+struct qb_ipcs_service_handlers mcp_ipc_callbacks = {
+    .connection_accept = pcmk_ipc_accept,
+    .connection_created = NULL,
+    .msg_process = pcmk_ipc_dispatch,
+    .connection_closed = pcmk_ipc_closed,
+    .connection_destroyed = pcmk_ipc_destroy
+};
diff --git a/daemons/pacemakerd/pcmkd_subdaemons.c b/daemons/pacemakerd/pcmkd_subdaemons.c
new file mode 100644
index 0000000..3b08ecc
--- /dev/null
+++ b/daemons/pacemakerd/pcmkd_subdaemons.c
@@ -0,0 +1,888 @@
+/*
+ * Copyright 2010-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+#include "pacemakerd.h"
+
+#include <errno.h>
+#include <grp.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+
+#include <crm/cluster.h>
+#include <crm/msg_xml.h>
+
+typedef struct pcmk_child_s {
+    pid_t pid;
+    int respawn_count;
+    bool respawn;
+    const char *name;
+    const char *uid;
+    const char *command;
+    const char *endpoint;  /* IPC server name */
+    bool needs_cluster;
+    int check_count;
+
+    /* Anything below here will be dynamically initialized */
+    bool needs_retry;
+    bool active_before_startup;
+} pcmk_child_t;
+
+#define PCMK_PROCESS_CHECK_INTERVAL 1
+#define PCMK_PROCESS_CHECK_RETRIES  5
+#define SHUTDOWN_ESCALATION_PERIOD  180000  /* 3m */
+
+/* Index into the array below */
+#define PCMK_CHILD_CONTROLD  5
+
+static pcmk_child_t pcmk_children[] = {
+    {
+        0, 0, true,  "pacemaker-based", CRM_DAEMON_USER,
+        CRM_DAEMON_DIR "/pacemaker-based", PCMK__SERVER_BASED_RO,
+        true
+    },
+    {
+        0, 0, true, "pacemaker-fenced", NULL,
+        CRM_DAEMON_DIR "/pacemaker-fenced", "stonith-ng",
+        true
+    },
+    {
+        0, 0, true,  "pacemaker-execd", NULL,
+        CRM_DAEMON_DIR "/pacemaker-execd", CRM_SYSTEM_LRMD,
+        false
+    },
+    {
+        0, 0, true, "pacemaker-attrd", CRM_DAEMON_USER,
+        CRM_DAEMON_DIR "/pacemaker-attrd", T_ATTRD,
+        true
+    },
+    {
+        0, 0, true, "pacemaker-schedulerd", CRM_DAEMON_USER,
+        CRM_DAEMON_DIR "/pacemaker-schedulerd", CRM_SYSTEM_PENGINE,
+        false
+    },
+    {
+        0, 0, true, "pacemaker-controld", CRM_DAEMON_USER,
+        CRM_DAEMON_DIR "/pacemaker-controld", CRM_SYSTEM_CRMD,
+        true
+    },
+};
+
+static char *opts_default[] = { NULL, NULL };
+static char *opts_vgrind[] = { NULL, NULL, NULL, NULL, NULL };
+
+crm_trigger_t *shutdown_trigger = NULL;
+crm_trigger_t *startup_trigger = NULL;
+time_t subdaemon_check_progress = 0;
+
+// Whether we need root group access to talk to cluster layer
+static bool need_root_group = true;
+
+/* When contacted via pacemakerd-api by a client having sbd in
+ * the name we assume it is sbd-daemon which wants to know
+ * if pacemakerd shutdown gracefully.
+ * Thus when everything is shutdown properly pacemakerd
+ * waits till it has reported the graceful completion of
+ * shutdown to sbd and just when sbd-client closes the
+ * connection we can assume that the report has arrived
+ * properly so that pacemakerd can finally exit.
+ * Following two variables are used to track that handshake.
+ */
+unsigned int shutdown_complete_state_reported_to = 0;
+gboolean shutdown_complete_state_reported_client_closed = FALSE;
+
+/* state we report when asked via pacemakerd-api status-ping */
+const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT;
+gboolean running_with_sbd = FALSE; /* local copy */
+
+GMainLoop *mainloop = NULL;
+
+static gboolean fatal_error = FALSE;
+
+static int child_liveness(pcmk_child_t *child);
+static gboolean escalate_shutdown(gpointer data);
+static int start_child(pcmk_child_t * child);
+static void pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode);
+static void pcmk_process_exit(pcmk_child_t * child);
+static gboolean pcmk_shutdown_worker(gpointer user_data);
+static gboolean stop_child(pcmk_child_t * child, int signal);
+
+static bool
+pcmkd_cluster_connected(void)
+{
+#if SUPPORT_COROSYNC
+    return pcmkd_corosync_connected();
+#else
+    return true;
+#endif
+}
+
+static gboolean
+check_next_subdaemon(gpointer user_data)
+{
+    static int next_child = 0;
+    int rc = child_liveness(&pcmk_children[next_child]);
+
+    crm_trace("Checked %s[%lld]: %s (%d)",
+              pcmk_children[next_child].name,
+              (long long) PCMK__SPECIAL_PID_AS_0(pcmk_children[next_child].pid),
+              pcmk_rc_str(rc), rc);
+
+    switch (rc) {
+        case pcmk_rc_ok:
+            pcmk_children[next_child].check_count = 0;
+            subdaemon_check_progress = time(NULL);
+            break;
+        case pcmk_rc_ipc_pid_only: // This case: it was previously OK
+            pcmk_children[next_child].check_count++;
+            if (pcmk_children[next_child].check_count >= PCMK_PROCESS_CHECK_RETRIES) {
+                crm_err("%s[%lld] is unresponsive to ipc after %d tries but "
+                        "we found the pid so have it killed that we can restart",
+                        pcmk_children[next_child].name,
+                        (long long) PCMK__SPECIAL_PID_AS_0(
+                            pcmk_children[next_child].pid),
+                        pcmk_children[next_child].check_count);
+                stop_child(&pcmk_children[next_child], SIGKILL);
+                if (pcmk_children[next_child].respawn) {
+                    /* as long as the respawn-limit isn't reached
+                       give it another round of check retries
+                     */
+                    pcmk_children[next_child].check_count = 0;
+                }
+            } else {
+                crm_notice("%s[%lld] is unresponsive to ipc after %d tries",
+                        pcmk_children[next_child].name,
+                        (long long) PCMK__SPECIAL_PID_AS_0(
+                            pcmk_children[next_child].pid),
+                        pcmk_children[next_child].check_count);
+                if (pcmk_children[next_child].respawn) {
+                    /* as long as the respawn-limit isn't reached
+                       and we haven't run out of connect retries
+                       we account this as progress we are willing
+                       to tell to sbd
+                     */
+                    subdaemon_check_progress = time(NULL);
+                }
+            }
+            /* go to the next child and see if
+               we can make progress there
+             */
+            break;
+        case pcmk_rc_ipc_unresponsive:
+            if (!pcmk_children[next_child].respawn) {
+                /* if a subdaemon is down and we don't want it
+                   to be restarted this is a success during
+                   shutdown. if it isn't restarted anymore
+                   due to MAX_RESPAWN it is
+                   rather no success.
+                 */
+                if (pcmk_children[next_child].respawn_count <= MAX_RESPAWN) {
+                    subdaemon_check_progress = time(NULL);
+                }
+            }
+            if (!pcmk_children[next_child].active_before_startup) {
+                crm_trace("found %s[%lld] missing - signal-handler "
+                          "will take care of it",
+                           pcmk_children[next_child].name,
+                           (long long) PCMK__SPECIAL_PID_AS_0(
+                            pcmk_children[next_child].pid));
+                break;
+            }
+            if (pcmk_children[next_child].respawn) {
+                crm_err("%s[%lld] terminated",
+                        pcmk_children[next_child].name,
+                        (long long) PCMK__SPECIAL_PID_AS_0(
+                            pcmk_children[next_child].pid));
+            } else {
+                /* orderly shutdown */
+                crm_notice("%s[%lld] terminated",
+                           pcmk_children[next_child].name,
+                           (long long) PCMK__SPECIAL_PID_AS_0(
+                                pcmk_children[next_child].pid));
+            }
+            pcmk_process_exit(&(pcmk_children[next_child]));
+            break;
+        default:
+            crm_exit(CRM_EX_FATAL);
+            break;  /* static analysis/noreturn */
+    }
+
+    next_child++;
+    if (next_child >= PCMK__NELEM(pcmk_children)) {
+        next_child = 0;
+    }
+
+    return G_SOURCE_CONTINUE;
+}
+
+static gboolean
+escalate_shutdown(gpointer data)
+{
+    pcmk_child_t *child = data;
+
+    if (child->pid == PCMK__SPECIAL_PID) {
+        pcmk_process_exit(child);
+
+    } else if (child->pid != 0) {
+        /* Use SIGSEGV instead of SIGKILL to create a core so we can see what it was up to */
+        crm_err("Child %s not terminating in a timely manner, forcing", child->name);
+        stop_child(child, SIGSEGV);
+    }
+    return FALSE;
+}
+
+static void
+pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
+{
+    pcmk_child_t *child = mainloop_child_userdata(p);
+    const char *name = mainloop_child_name(p);
+
+    if (signo) {
+        do_crm_log(((signo == SIGKILL)? LOG_WARNING : LOG_ERR),
+                   "%s[%d] terminated with signal %d (%s)%s",
+                   name, pid, signo, strsignal(signo),
+                   (core? " and dumped core" : ""));
+
+    } else {
+        switch(exitcode) {
+            case CRM_EX_OK:
+                crm_info("%s[%d] exited with status %d (%s)",
+                         name, pid, exitcode, crm_exit_str(exitcode));
+                break;
+
+            case CRM_EX_FATAL:
+                crm_warn("Shutting cluster down because %s[%d] had fatal failure",
+                         name, pid);
+                child->respawn = false;
+                fatal_error = TRUE;
+                pcmk_shutdown(SIGTERM);
+                break;
+
+            case CRM_EX_PANIC:
+                crm_emerg("%s[%d] instructed the machine to reset", name, pid);
+                child->respawn = false;
+                fatal_error = TRUE;
+                pcmk__panic(__func__);
+                pcmk_shutdown(SIGTERM);
+                break;
+
+            default:
+                crm_err("%s[%d] exited with status %d (%s)",
+                        name, pid, exitcode, crm_exit_str(exitcode));
+                break;
+        }
+    }
+
+    pcmk_process_exit(child);
+}
+
+static void
+pcmk_process_exit(pcmk_child_t * child)
+{
+    child->pid = 0;
+    child->active_before_startup = false;
+    child->check_count = 0;
+
+    child->respawn_count += 1;
+    if (child->respawn_count > MAX_RESPAWN) {
+        crm_err("Child respawn count exceeded by %s", child->name);
+        child->respawn = false;
+    }
+
+    if (shutdown_trigger) {
+        /* resume step-wise shutdown (returned TRUE yields no parallelizing) */
+        mainloop_set_trigger(shutdown_trigger);
+
+    } else if (!child->respawn) {
+        /* nothing to do */
+
+    } else if (crm_is_true(getenv("PCMK_fail_fast"))) {
+        crm_err("Rebooting system because of %s", child->name);
+        pcmk__panic(__func__);
+
+    } else if (child_liveness(child) == pcmk_rc_ok) {
+        crm_warn("One-off suppressing strict respawning of a child process %s,"
+                 " appears alright per %s IPC end-point",
+                 child->name, child->endpoint);
+
+    } else if (child->needs_cluster && !pcmkd_cluster_connected()) {
+        crm_notice("Not respawning %s subdaemon until cluster returns",
+                   child->name);
+        child->needs_retry = true;
+
+    } else {
+        crm_notice("Respawning %s subdaemon after unexpected exit",
+                   child->name);
+        start_child(child);
+    }
+}
+
+static gboolean
+pcmk_shutdown_worker(gpointer user_data)
+{
+    static int phase = PCMK__NELEM(pcmk_children) - 1;
+    static time_t next_log = 0;
+
+    if (phase == PCMK__NELEM(pcmk_children) - 1) {
+        crm_notice("Shutting down Pacemaker");
+        pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN;
+    }
+
+    for (; phase >= 0; phase--) {
+        pcmk_child_t *child = &(pcmk_children[phase]);
+
+        if (child->pid != 0) {
+            time_t now = time(NULL);
+
+            if (child->respawn) {
+                if (child->pid == PCMK__SPECIAL_PID) {
+                    crm_warn("The process behind %s IPC cannot be"
+                             " terminated, so either wait the graceful"
+                             " period of %ld s for its native termination"
+                             " if it vitally depends on some other daemons"
+                             " going down in a controlled way already,"
+                             " or locate and kill the correct %s process"
+                             " on your own; set PCMK_fail_fast=1 to avoid"
+                             " this altogether next time around",
+                             child->name, (long) SHUTDOWN_ESCALATION_PERIOD,
+                             child->command);
+                }
+                next_log = now + 30;
+                child->respawn = false;
+                stop_child(child, SIGTERM);
+                if (phase < PCMK_CHILD_CONTROLD) {
+                    g_timeout_add(SHUTDOWN_ESCALATION_PERIOD,
+                                  escalate_shutdown, child);
+                }
+
+            } else if (now >= next_log) {
+                next_log = now + 30;
+                crm_notice("Still waiting for %s to terminate "
+                           CRM_XS " pid=%lld",
+                           child->name, (long long) child->pid);
+            }
+            return TRUE;
+        }
+
+        /* cleanup */
+        crm_debug("%s confirmed stopped", child->name);
+        child->pid = 0;
+    }
+
+    crm_notice("Shutdown complete");
+    pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE;
+    if (!fatal_error && running_with_sbd &&
+        pcmk__get_sbd_sync_resource_startup() &&
+        !shutdown_complete_state_reported_client_closed) {
+        crm_notice("Waiting for SBD to pick up shutdown-complete-state.");
+        return TRUE;
+    }
+
+    {
+        const char *delay = pcmk__env_option(PCMK__ENV_SHUTDOWN_DELAY);
+        if(delay) {
+            sync();
+            pcmk__sleep_ms(crm_get_msec(delay));
+        }
+    }
+
+    g_main_loop_quit(mainloop);
+
+    if (fatal_error) {
+        crm_notice("Shutting down and staying down after fatal error");
+#ifdef SUPPORT_COROSYNC
+        pcmkd_shutdown_corosync();
+#endif
+        crm_exit(CRM_EX_FATAL);
+    }
+
+    return TRUE;
+}
+
+/* TODO once libqb is taught to juggle with IPC end-points carried over as
+        bare file descriptor (https://github.com/ClusterLabs/libqb/issues/325)
+        it shall hand over these descriptors here if/once they are successfully
+        pre-opened in (presumably) child_liveness(), to avoid any remaining
+        room for races */
+ // \return Standard Pacemaker return code
+static int
+start_child(pcmk_child_t * child)
+{
+    uid_t uid = 0;
+    gid_t gid = 0;
+    gboolean use_valgrind = FALSE;
+    gboolean use_callgrind = FALSE;
+    const char *env_valgrind = getenv("PCMK_valgrind_enabled");
+    const char *env_callgrind = getenv("PCMK_callgrind_enabled");
+
+    child->active_before_startup = false;
+    child->check_count = 0;
+
+    if (child->command == NULL) {
+        crm_info("Nothing to do for child \"%s\"", child->name);
+        return pcmk_rc_ok;
+    }
+
+    if (env_callgrind != NULL && crm_is_true(env_callgrind)) {
+        use_callgrind = TRUE;
+        use_valgrind = TRUE;
+
+    } else if (env_callgrind != NULL && strstr(env_callgrind, child->name)) {
+        use_callgrind = TRUE;
+        use_valgrind = TRUE;
+
+    } else if (env_valgrind != NULL && crm_is_true(env_valgrind)) {
+        use_valgrind = TRUE;
+
+    } else if (env_valgrind != NULL && strstr(env_valgrind, child->name)) {
+        use_valgrind = TRUE;
+    }
+
+    if (use_valgrind && strlen(VALGRIND_BIN) == 0) {
+        crm_warn("Cannot enable valgrind for %s:"
+                 " The location of the valgrind binary is unknown", child->name);
+        use_valgrind = FALSE;
+    }
+
+    if (child->uid) {
+        if (crm_user_lookup(child->uid, &uid, &gid) < 0) {
+            crm_err("Invalid user (%s) for %s: not found", child->uid, child->name);
+            return EACCES;
+        }
+        crm_info("Using uid=%u and group=%u for process %s", uid, gid, child->name);
+    }
+
+    child->pid = fork();
+    CRM_ASSERT(child->pid != -1);
+
+    if (child->pid > 0) {
+        /* parent */
+        mainloop_child_add(child->pid, 0, child->name, child, pcmk_child_exit);
+
+        crm_info("Forked child %lld for process %s%s",
+                 (long long) child->pid, child->name,
+                 use_valgrind ? " (valgrind enabled: " VALGRIND_BIN ")" : "");
+        return pcmk_rc_ok;
+
+    } else {
+        /* Start a new session */
+        (void)setsid();
+
+        /* Setup the two alternate arg arrays */
+        opts_vgrind[0] = strdup(VALGRIND_BIN);
+        if (use_callgrind) {
+            opts_vgrind[1] = strdup("--tool=callgrind");
+            opts_vgrind[2] = strdup("--callgrind-out-file=" CRM_STATE_DIR "/callgrind.out.%p");
+            opts_vgrind[3] = strdup(child->command);
+            opts_vgrind[4] = NULL;
+        } else {
+            opts_vgrind[1] = strdup(child->command);
+            opts_vgrind[2] = NULL;
+            opts_vgrind[3] = NULL;
+            opts_vgrind[4] = NULL;
+        }
+        opts_default[0] = strdup(child->command);
+
+        if(gid) {
+            // Drop root group access if not needed
+            if (!need_root_group && (setgid(gid) < 0)) {
+                crm_warn("Could not set group to %d: %s", gid, strerror(errno));
+            }
+
+            /* Initialize supplementary groups to only those always granted to
+             * the user, plus haclient (so we can access IPC).
+             */
+            if (initgroups(child->uid, gid) < 0) {
+                crm_err("Cannot initialize groups for %s: %s (%d)",
+                        child->uid, pcmk_rc_str(errno), errno);
+            }
+        }
+
+        if (uid && setuid(uid) < 0) {
+            crm_warn("Could not set user to %s (id %d): %s",
+                     child->uid, uid, strerror(errno));
+        }
+
+        pcmk__close_fds_in_child(true);
+
+        pcmk__open_devnull(O_RDONLY);   // stdin (fd 0)
+        pcmk__open_devnull(O_WRONLY);   // stdout (fd 1)
+        pcmk__open_devnull(O_WRONLY);   // stderr (fd 2)
+
+        if (use_valgrind) {
+            (void)execvp(VALGRIND_BIN, opts_vgrind);
+        } else {
+            (void)execvp(child->command, opts_default);
+        }
+        crm_crit("Could not execute %s: %s", child->command, strerror(errno));
+        crm_exit(CRM_EX_FATAL);
+    }
+    return pcmk_rc_ok;          /* never reached */
+}
+
+/*!
+ * \internal
+ * \brief Check the liveness of the child based on IPC name and PID if tracked
+ *
+ * \param[in,out] child  Child tracked data
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note Return codes of particular interest include pcmk_rc_ipc_unresponsive
+ *       indicating that no trace of IPC liveness was detected,
+ *       pcmk_rc_ipc_unauthorized indicating that the IPC endpoint is blocked by
+ *       an unauthorized process, and pcmk_rc_ipc_pid_only indicating that
+ *       the child is up by PID but not IPC end-point (possibly starting).
+ * \note This function doesn't modify any of \p child members but \c pid,
+ *       and is not actively toying with processes as such but invoking
+ *       \c stop_child in one particular case (there's for some reason
+ *       a different authentic holder of the IPC end-point).
+ */
+static int
+child_liveness(pcmk_child_t *child)
+{
+    uid_t cl_uid = 0;
+    gid_t cl_gid = 0;
+    const uid_t root_uid = 0;
+    const gid_t root_gid = 0;
+    const uid_t *ref_uid;
+    const gid_t *ref_gid;
+    int rc = pcmk_rc_ipc_unresponsive;
+    pid_t ipc_pid = 0;
+
+    if (child->endpoint == NULL
+            && (child->pid <= 0 || child->pid == PCMK__SPECIAL_PID)) {
+        crm_err("Cannot track child %s for missing both API end-point and PID",
+                child->name);
+        rc = EINVAL; // Misuse of function when child is not trackable
+
+    } else if (child->endpoint != NULL) {
+        int legacy_rc = pcmk_ok;
+
+        if (child->uid == NULL) {
+            ref_uid = &root_uid;
+            ref_gid = &root_gid;
+        } else {
+            ref_uid = &cl_uid;
+            ref_gid = &cl_gid;
+            legacy_rc = pcmk_daemon_user(&cl_uid, &cl_gid);
+        }
+
+        if (legacy_rc < 0) {
+            rc = pcmk_legacy2rc(legacy_rc);
+            crm_err("Could not find user and group IDs for user %s: %s "
+                    CRM_XS " rc=%d", CRM_DAEMON_USER, pcmk_rc_str(rc), rc);
+        } else {
+            rc = pcmk__ipc_is_authentic_process_active(child->endpoint,
+                                                       *ref_uid, *ref_gid,
+                                                       &ipc_pid);
+            if ((rc == pcmk_rc_ok) || (rc == pcmk_rc_ipc_unresponsive)) {
+                if (child->pid <= 0) {
+                    /* If rc is pcmk_rc_ok, ipc_pid is nonzero and this
+                     * initializes a new child. If rc is
+                     * pcmk_rc_ipc_unresponsive, ipc_pid is zero, and we will
+                     * investigate further.
+                     */
+                    child->pid = ipc_pid;
+                } else if ((ipc_pid != 0) && (child->pid != ipc_pid)) {
+                    /* An unexpected (but authorized) process is responding to
+                     * IPC. Investigate further.
+                     */
+                    rc = pcmk_rc_ipc_unresponsive;
+                }
+            }
+        }
+    }
+
+    if (rc == pcmk_rc_ipc_unresponsive) {
+        /* If we get here, a child without IPC is being tracked, no IPC liveness
+         * has been detected, or IPC liveness has been detected with an
+         * unexpected (but authorized) process. This is safe on FreeBSD since
+         * the only change possible from a proper child's PID into "special" PID
+         * of 1 behind more loosely related process.
+         */
+        int ret = pcmk__pid_active(child->pid, child->name);
+
+        if (ipc_pid && ((ret != pcmk_rc_ok)
+                        || ipc_pid == PCMK__SPECIAL_PID
+                        || (pcmk__pid_active(ipc_pid,
+                                             child->name) == pcmk_rc_ok))) {
+            /* An unexpected (but authorized) process was detected at the IPC
+             * endpoint, and either it is active, or the child we're tracking is
+             * not.
+             */
+
+            if (ret == pcmk_rc_ok) {
+                /* The child we're tracking is active. Kill it, and adopt the
+                 * detected process. This assumes that our children don't fork
+                 * (thus getting a different PID owning the IPC), but rather the
+                 * tracking got out of sync because of some means external to
+                 * Pacemaker, and adopting the detected process is better than
+                 * killing it and possibly having to spawn a new child.
+                 */
+                /* not possessing IPC, afterall (what about corosync CPG?) */
+                stop_child(child, SIGKILL);
+            }
+            rc = pcmk_rc_ok;
+            child->pid = ipc_pid;
+        } else if (ret == pcmk_rc_ok) {
+            // Our tracked child's PID was found active, but not its IPC
+            rc = pcmk_rc_ipc_pid_only;
+        } else if ((child->pid == 0) && (ret == EINVAL)) {
+            // FreeBSD can return EINVAL
+            rc = pcmk_rc_ipc_unresponsive;
+        } else {
+            switch (ret) {
+                case EACCES:
+                    rc = pcmk_rc_ipc_unauthorized;
+                    break;
+                case ESRCH:
+                    rc = pcmk_rc_ipc_unresponsive;
+                    break;
+                default:
+                    rc = ret;
+                    break;
+            }
+        }
+    }
+    return rc;
+}
+
+/*!
+ * \internal
+ * \brief Initial one-off check of the pre-existing "child" processes
+ *
+ * With "child" process, we mean the subdaemon that defines an API end-point
+ * (all of them do as of the comment) -- the possible complement is skipped
+ * as it is deemed it has no such shared resources to cause conflicts about,
+ * hence it can presumably be started anew without hesitation.
+ * If that won't hold true in the future, the concept of a shared resource
+ * will have to be generalized beyond the API end-point.
+ *
+ * For boundary cases that the "child" is still starting (IPC end-point is yet
+ * to be witnessed), or more rarely (practically FreeBSD only), when there's
+ * a pre-existing "untrackable" authentic process, we give the situation some
+ * time to possibly unfold in the right direction, meaning that said socket
+ * will appear or the unattainable process will disappear per the observable
+ * IPC, respectively.
+ *
+ * \return Standard Pacemaker return code
+ *
+ * \note Since this gets run at the very start, \c respawn_count fields
+ *       for particular children get temporarily overloaded with "rounds
+ *       of waiting" tracking, restored once we are about to finish with
+ *       success (i.e. returning value >=0) and will remain unrestored
+ *       otherwise.  One way to suppress liveness detection logic for
+ *       particular child is to set the said value to a negative number.
+ */
+#define WAIT_TRIES 4  /* together with interleaved sleeps, worst case ~ 1s */
+int
+find_and_track_existing_processes(void)
+{
+    bool wait_in_progress;
+    int rc;
+    size_t i, rounds;
+
+    for (rounds = 1; rounds <= WAIT_TRIES; rounds++) {
+        wait_in_progress = false;
+        for (i = 0; i < PCMK__NELEM(pcmk_children); i++) {
+
+            if ((pcmk_children[i].endpoint == NULL)
+                || (pcmk_children[i].respawn_count < 0)) {
+                continue;
+            }
+
+            rc = child_liveness(&pcmk_children[i]);
+            if (rc == pcmk_rc_ipc_unresponsive) {
+                /* As a speculation, don't give up if there are more rounds to
+                 * come for other reasons, but don't artificially wait just
+                 * because of this, since we would preferably start ASAP.
+                 */
+                continue;
+            }
+
+            pcmk_children[i].respawn_count = rounds;
+            switch (rc) {
+                case pcmk_rc_ok:
+                    if (pcmk_children[i].pid == PCMK__SPECIAL_PID) {
+                        if (crm_is_true(getenv("PCMK_fail_fast"))) {
+                            crm_crit("Cannot reliably track pre-existing"
+                                     " authentic process behind %s IPC on this"
+                                     " platform and PCMK_fail_fast requested",
+                                     pcmk_children[i].endpoint);
+                            return EOPNOTSUPP;
+                        } else if (pcmk_children[i].respawn_count == WAIT_TRIES) {
+                            crm_notice("Assuming pre-existing authentic, though"
+                                       " on this platform untrackable, process"
+                                       " behind %s IPC is stable (was in %d"
+                                       " previous samples) so rather than"
+                                       " bailing out (PCMK_fail_fast not"
+                                       " requested), we just switch to a less"
+                                       " optimal IPC liveness monitoring"
+                                       " (not very suitable for heavy load)",
+                                       pcmk_children[i].name, WAIT_TRIES - 1);
+                            crm_warn("The process behind %s IPC cannot be"
+                                     " terminated, so the overall shutdown"
+                                     " will get delayed implicitly (%ld s),"
+                                     " which serves as a graceful period for"
+                                     " its native termination if it vitally"
+                                     " depends on some other daemons going"
+                                     " down in a controlled way already",
+                                     pcmk_children[i].name,
+                                     (long) SHUTDOWN_ESCALATION_PERIOD);
+                        } else {
+                            wait_in_progress = true;
+                            crm_warn("Cannot reliably track pre-existing"
+                                     " authentic process behind %s IPC on this"
+                                     " platform, can still disappear in %d"
+                                     " attempt(s)", pcmk_children[i].endpoint,
+                                     WAIT_TRIES - pcmk_children[i].respawn_count);
+                            continue;
+                        }
+                    }
+                    crm_notice("Tracking existing %s process (pid=%lld)",
+                               pcmk_children[i].name,
+                               (long long) PCMK__SPECIAL_PID_AS_0(
+                                               pcmk_children[i].pid));
+                    pcmk_children[i].respawn_count = -1;  /* 0~keep watching */
+                    pcmk_children[i].active_before_startup = true;
+                    break;
+                case pcmk_rc_ipc_pid_only:
+                    if (pcmk_children[i].respawn_count == WAIT_TRIES) {
+                        crm_crit("%s IPC end-point for existing authentic"
+                                 " process %lld did not (re)appear",
+                                 pcmk_children[i].endpoint,
+                                 (long long) PCMK__SPECIAL_PID_AS_0(
+                                                 pcmk_children[i].pid));
+                        return rc;
+                    }
+                    wait_in_progress = true;
+                    crm_warn("Cannot find %s IPC end-point for existing"
+                             " authentic process %lld, can still (re)appear"
+                             " in %d attempts (?)",
+                             pcmk_children[i].endpoint,
+                             (long long) PCMK__SPECIAL_PID_AS_0(
+                                             pcmk_children[i].pid),
+                             WAIT_TRIES - pcmk_children[i].respawn_count);
+                    continue;
+                default:
+                    crm_crit("Checked liveness of %s: %s " CRM_XS " rc=%d",
+                             pcmk_children[i].name, pcmk_rc_str(rc), rc);
+                    return rc;
+            }
+        }
+        if (!wait_in_progress) {
+            break;
+        }
+        pcmk__sleep_ms(250); // Wait a bit for changes to possibly happen
+    }
+    for (i = 0; i < PCMK__NELEM(pcmk_children); i++) {
+        pcmk_children[i].respawn_count = 0;  /* restore pristine state */
+    }
+
+    g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL, check_next_subdaemon,
+                          NULL);
+    return pcmk_rc_ok;
+}
+
+gboolean
+init_children_processes(void *user_data)
+{
+    if (is_corosync_cluster()) {
+        /* Corosync clusters can drop root group access, because we set
+         * uidgid.gid.${gid}=1 via CMAP, which allows these processes to connect
+         * to corosync.
+         */
+        need_root_group = false;
+    }
+
+    /* start any children that have not been detected */
+    for (int i = 0; i < PCMK__NELEM(pcmk_children); i++) {
+        if (pcmk_children[i].pid != 0) {
+            /* we are already tracking it */
+            continue;
+        }
+
+        start_child(&(pcmk_children[i]));
+    }
+
+    /* From this point on, any daemons being started will be due to
+     * respawning rather than node start.
+     *
+     * This may be useful for the daemons to know
+     */
+    setenv("PCMK_respawned", "true", 1);
+    pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_RUNNING;
+    return TRUE;
+}
+
+void
+pcmk_shutdown(int nsig)
+{
+    if (shutdown_trigger == NULL) {
+        shutdown_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, pcmk_shutdown_worker, NULL);
+    }
+    mainloop_set_trigger(shutdown_trigger);
+}
+
+void
+restart_cluster_subdaemons(void)
+{
+    for (int i = 0; i < PCMK__NELEM(pcmk_children); i++) {
+        if (!pcmk_children[i].needs_retry || pcmk_children[i].pid != 0) {
+            continue;
+        }
+
+        crm_notice("Respawning cluster-based subdaemon: %s", pcmk_children[i].name);
+        if (start_child(&pcmk_children[i])) {
+            pcmk_children[i].needs_retry = false;
+        }
+    }
+}
+
+static gboolean
+stop_child(pcmk_child_t * child, int signal)
+{
+    if (signal == 0) {
+        signal = SIGTERM;
+    }
+
+    /* why to skip PID of 1?
+       - FreeBSD ~ how untrackable process behind IPC is masqueraded as
+       - elsewhere: how "init" task is designated; in particular, in systemd
+         arrangement of socket-based activation, this is pretty real */
+    if (child->command == NULL || child->pid == PCMK__SPECIAL_PID) {
+        crm_debug("Nothing to do for child \"%s\" (process %lld)",
+                  child->name, (long long) PCMK__SPECIAL_PID_AS_0(child->pid));
+        return TRUE;
+    }
+
+    if (child->pid <= 0) {
+        crm_trace("Client %s not running", child->name);
+        return TRUE;
+    }
+
+    errno = 0;
+    if (kill(child->pid, signal) == 0) {
+        crm_notice("Stopping %s "CRM_XS" sent signal %d to process %lld",
+                   child->name, signal, (long long) child->pid);
+
+    } else {
+        crm_err("Could not stop %s (process %lld) with signal %d: %s",
+                child->name, (long long) child->pid, signal, strerror(errno));
+    }
+
+    return TRUE;
+}
+
diff --git a/daemons/schedulerd/Makefile.am b/daemons/schedulerd/Makefile.am
new file mode 100644
index 0000000..57e819b
--- /dev/null
+++ b/daemons/schedulerd/Makefile.am
@@ -0,0 +1,53 @@
+#
+# Copyright 2004-2021 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+include $(top_srcdir)/mk/common.mk
+include $(top_srcdir)/mk/man.mk
+
+AM_CPPFLAGS	+= -I$(top_builddir) -I$(top_srcdir)
+
+halibdir	= $(CRM_DAEMON_DIR)
+
+## binary progs
+
+halib_PROGRAMS	= pacemaker-schedulerd
+
+if BUILD_XML_HELP
+man7_MANS =	pacemaker-schedulerd.7
+endif
+
+## SOURCES
+
+noinst_HEADERS 	= pacemaker-schedulerd.h
+
+pacemaker_schedulerd_CFLAGS	= $(CFLAGS_HARDENED_EXE)
+pacemaker_schedulerd_LDFLAGS	= $(LDFLAGS_HARDENED_EXE)
+pacemaker_schedulerd_LDADD	= $(top_builddir)/lib/common/libcrmcommon.la \
+							  $(top_builddir)/lib/pengine/libpe_status.la \
+							  $(top_builddir)/lib/pacemaker/libpacemaker.la
+# libcib for get_object_root()
+pacemaker_schedulerd_SOURCES	= pacemaker-schedulerd.c
+pacemaker_schedulerd_SOURCES	+= schedulerd_messages.c
+
+install-exec-local:
+	$(INSTALL) -d -m 750 $(DESTDIR)/$(PE_STATE_DIR)
+	-chown $(CRM_DAEMON_USER):$(CRM_DAEMON_GROUP) $(DESTDIR)/$(PE_STATE_DIR)
+
+if BUILD_LEGACY_LINKS
+install-exec-hook:
+	cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f pengine && $(LN_S) pacemaker-schedulerd pengine
+
+uninstall-hook:
+	cd $(DESTDIR)$(CRM_DAEMON_DIR) && rm -f pengine
+endif
+
+uninstall-local:
+	-rmdir $(DESTDIR)/$(PE_STATE_DIR)
+
+CLEANFILES = $(man7_MANS)
diff --git a/daemons/schedulerd/pacemaker-schedulerd.c b/daemons/schedulerd/pacemaker-schedulerd.c
new file mode 100644
index 0000000..3f2a3e8
--- /dev/null
+++ b/daemons/schedulerd/pacemaker-schedulerd.c
@@ -0,0 +1,181 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm/crm.h>
+#include <stdio.h>
+#include <stdbool.h>
+
+#include <stdlib.h>
+#include <errno.h>
+
+#include <crm/common/cmdline_internal.h>
+#include <crm/common/ipc_internal.h>
+#include <crm/common/mainloop.h>
+#include <crm/pengine/internal.h>
+#include <pacemaker-internal.h>
+
+#include "pacemaker-schedulerd.h"
+
+#define SUMMARY "pacemaker-schedulerd - daemon for calculating a Pacemaker cluster's response to events"
+
+struct {
+    gchar **remainder;
+} options;
+
+pcmk__output_t *logger_out = NULL;
+pcmk__output_t *out = NULL;
+
+static GMainLoop *mainloop = NULL;
+static qb_ipcs_service_t *ipcs = NULL;
+static crm_exit_t exit_code = CRM_EX_OK;
+
+pcmk__supported_format_t formats[] = {
+    PCMK__SUPPORTED_FORMAT_NONE,
+    PCMK__SUPPORTED_FORMAT_TEXT,
+    PCMK__SUPPORTED_FORMAT_XML,
+    { NULL, NULL, NULL }
+};
+
+void pengine_shutdown(int nsig);
+
+static GOptionContext *
+build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+    GOptionContext *context = NULL;
+
+    GOptionEntry extra_prog_entries[] = {
+        { G_OPTION_REMAINING, 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING_ARRAY, &options.remainder,
+          NULL,
+          NULL },
+
+        { NULL }
+    };
+
+    context = pcmk__build_arg_context(args, "text (default), xml", group,
+                                      "[metadata]");
+    pcmk__add_main_args(context, extra_prog_entries);
+    return context;
+}
+
+int
+main(int argc, char **argv)
+{
+    GError *error = NULL;
+    int rc = pcmk_rc_ok;
+
+    GOptionGroup *output_group = NULL;
+    pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+    gchar **processed_args = pcmk__cmdline_preproc(argv, NULL);
+    GOptionContext *context = build_arg_context(args, &output_group);
+
+    crm_log_preinit(NULL, argc, argv);
+    mainloop_add_signal(SIGTERM, pengine_shutdown);
+
+    pcmk__register_formats(output_group, formats);
+    if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+        exit_code = CRM_EX_USAGE;
+        goto done;
+    }
+
+    rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
+    if ((rc != pcmk_rc_ok) || (out == NULL)) {
+        exit_code = CRM_EX_FATAL;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
+                    args->output_ty, pcmk_rc_str(rc));
+        goto done;
+    }
+
+    pe__register_messages(out);
+    pcmk__register_lib_messages(out);
+
+    if (options.remainder) {
+        if (g_strv_length(options.remainder) == 1 &&
+            pcmk__str_eq("metadata", options.remainder[0], pcmk__str_casei)) {
+            pe_metadata(out);
+            goto done;
+        } else {
+            exit_code = CRM_EX_USAGE;
+            g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                        "Unsupported extra command line parameters");
+            goto done;
+        }
+    }
+
+    if (args->version) {
+        out->version(out, false);
+        goto done;
+    }
+
+    pcmk__cli_init_logging("pacemaker-schedulerd", args->verbosity);
+    crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
+    crm_notice("Starting Pacemaker scheduler");
+
+    if (pcmk__daemon_can_write(PE_STATE_DIR, NULL) == FALSE) {
+        crm_err("Terminating due to bad permissions on " PE_STATE_DIR);
+        exit_code = CRM_EX_FATAL;
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "ERROR: Bad permissions on %s (see logs for details)", PE_STATE_DIR);
+        goto done;
+    }
+
+    ipcs = pcmk__serve_schedulerd_ipc(&ipc_callbacks);
+    if (ipcs == NULL) {
+        g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
+                    "Failed to create pacemaker-schedulerd server: exiting and inhibiting respawn");
+        exit_code = CRM_EX_FATAL;
+        goto done;
+    }
+
+    if (pcmk__log_output_new(&logger_out) != pcmk_rc_ok) {
+        exit_code = CRM_EX_FATAL;
+        goto done;
+    }
+    pe__register_messages(logger_out);
+    pcmk__register_lib_messages(logger_out);
+    pcmk__output_set_log_level(logger_out, LOG_TRACE);
+
+    /* Create the mainloop and run it... */
+    mainloop = g_main_loop_new(NULL, FALSE);
+    crm_notice("Pacemaker scheduler successfully started and accepting connections");
+    g_main_loop_run(mainloop);
+
+done:
+    g_strfreev(options.remainder);
+    g_strfreev(processed_args);
+    pcmk__free_arg_context(context);
+
+    pcmk__output_and_clear_error(&error, out);
+    pengine_shutdown(0);
+}
+
+void
+pengine_shutdown(int nsig)
+{
+    if (ipcs != NULL) {
+        crm_trace("Closing IPC server");
+        mainloop_del_ipc_server(ipcs);
+        ipcs = NULL;
+    }
+
+    if (logger_out != NULL) {
+        logger_out->finish(logger_out, exit_code, true, NULL);
+        pcmk__output_free(logger_out);
+        logger_out = NULL;
+    }
+
+    if (out != NULL) {
+        out->finish(out, exit_code, true, NULL);
+        pcmk__output_free(out);
+        out = NULL;
+    }
+
+    pcmk__unregister_formats();
+    crm_exit(exit_code);
+}
diff --git a/daemons/schedulerd/pacemaker-schedulerd.h b/daemons/schedulerd/pacemaker-schedulerd.h
new file mode 100644
index 0000000..cbb07e1
--- /dev/null
+++ b/daemons/schedulerd/pacemaker-schedulerd.h
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef PCMK__PACEMAKER_SCHEDULERD__H
+#define PCMK__PACEMAKER_SCHEDULERD__H
+
+#include <crm_internal.h>
+#include <crm/pengine/pe_types.h>
+
+extern pcmk__output_t *logger_out;
+extern pcmk__output_t *out;
+extern struct qb_ipcs_service_handlers ipc_callbacks;
+
+#endif
diff --git a/daemons/schedulerd/schedulerd_messages.c b/daemons/schedulerd/schedulerd_messages.c
new file mode 100644
index 0000000..1c124d2
--- /dev/null
+++ b/daemons/schedulerd/schedulerd_messages.c
@@ -0,0 +1,335 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm/crm.h>
+#include <crm/msg_xml.h>
+#include <pacemaker-internal.h>
+
+#include <stdbool.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "pacemaker-schedulerd.h"
+
+static GHashTable *schedulerd_handlers = NULL;
+
+static pe_working_set_t *
+init_working_set(void)
+{
+    pe_working_set_t *data_set = pe_new_working_set();
+
+    CRM_ASSERT(data_set != NULL);
+
+    crm_config_error = FALSE;
+    crm_config_warning = FALSE;
+
+    was_processing_error = FALSE;
+    was_processing_warning = FALSE;
+
+    data_set->priv = logger_out;
+    return data_set;
+}
+
+static xmlNode *
+handle_pecalc_request(pcmk__request_t *request)
+{
+    static struct series_s {
+        const char *name;
+        const char *param;
+
+        /* Maximum number of inputs of this kind to save to disk.
+         * If -1, save all; if 0, save none.
+         */
+        int wrap;
+    } series[] = {
+        { "pe-error", "pe-error-series-max", -1 },
+        { "pe-warn",  "pe-warn-series-max",  5000 },
+        { "pe-input", "pe-input-series-max", 4000 },
+    };
+
+    xmlNode *msg = request->xml;
+    xmlNode *xml_data = get_message_xml(msg, F_CRM_DATA);
+
+    static char *last_digest = NULL;
+    static char *filename = NULL;
+
+    unsigned int seq;
+    int series_id = 0;
+    int series_wrap = 0;
+    char *digest = NULL;
+    const char *value = NULL;
+    time_t execution_date = time(NULL);
+    xmlNode *converted = NULL;
+    xmlNode *reply = NULL;
+    bool is_repoke = false;
+    bool process = true;
+    pe_working_set_t *data_set = init_working_set();
+
+    pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
+                       "ack", NULL, CRM_EX_INDETERMINATE);
+
+    digest = calculate_xml_versioned_digest(xml_data, FALSE, FALSE,
+                                            CRM_FEATURE_SET);
+    converted = copy_xml(xml_data);
+    if (!cli_config_update(&converted, NULL, TRUE)) {
+        data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);
+        crm_xml_add_int(data_set->graph, "transition_id", 0);
+        crm_xml_add_int(data_set->graph, "cluster-delay", 0);
+        process = false;
+        free(digest);
+
+    } else if (pcmk__str_eq(digest, last_digest, pcmk__str_casei)) {
+        is_repoke = true;
+        free(digest);
+
+    } else {
+        free(last_digest);
+        last_digest = digest;
+    }
+
+    if (process) {
+        pcmk__schedule_actions(converted,
+                               pe_flag_no_counts
+                               |pe_flag_no_compat
+                               |pe_flag_show_utilization, data_set);
+    }
+
+    // Get appropriate index into series[] array
+    if (was_processing_error) {
+        series_id = 0;
+    } else if (was_processing_warning) {
+        series_id = 1;
+    } else {
+        series_id = 2;
+    }
+
+    value = pe_pref(data_set->config_hash, series[series_id].param);
+    if ((value == NULL)
+        || (pcmk__scan_min_int(value, &series_wrap, -1) != pcmk_rc_ok)) {
+        series_wrap = series[series_id].wrap;
+    }
+
+    if (pcmk__read_series_sequence(PE_STATE_DIR, series[series_id].name,
+                                   &seq) != pcmk_rc_ok) {
+        // @TODO maybe handle errors better ...
+        seq = 0;
+    }
+    crm_trace("Series %s: wrap=%d, seq=%u, pref=%s",
+              series[series_id].name, series_wrap, seq, value);
+
+    data_set->input = NULL;
+    reply = create_reply(msg, data_set->graph);
+
+    if (reply == NULL) {
+        pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
+                            "Failed building ping reply for client %s",
+                            pcmk__client_name(request->ipc_client));
+        goto done;
+    }
+
+    if (series_wrap == 0) { // Don't save any inputs of this kind
+        free(filename);
+        filename = NULL;
+
+    } else if (!is_repoke) { // Input changed, save to disk
+        free(filename);
+        filename = pcmk__series_filename(PE_STATE_DIR,
+                                         series[series_id].name, seq, true);
+    }
+
+    crm_xml_add(reply, F_CRM_TGRAPH_INPUT, filename);
+    crm_xml_add_int(reply, PCMK__XA_GRAPH_ERRORS, was_processing_error);
+    crm_xml_add_int(reply, PCMK__XA_GRAPH_WARNINGS, was_processing_warning);
+    crm_xml_add_int(reply, PCMK__XA_CONFIG_ERRORS, crm_config_error);
+    crm_xml_add_int(reply, PCMK__XA_CONFIG_WARNINGS, crm_config_warning);
+
+    pcmk__log_transition_summary(filename);
+
+    if (series_wrap == 0) {
+        crm_debug("Not saving input to disk (disabled by configuration)");
+
+    } else if (is_repoke) {
+        crm_info("Input has not changed since last time, not saving to disk");
+
+    } else {
+        unlink(filename);
+        crm_xml_add_ll(xml_data, "execution-date", (long long) execution_date);
+        write_xml_file(xml_data, filename, TRUE);
+        pcmk__write_series_sequence(PE_STATE_DIR, series[series_id].name,
+                                    ++seq, series_wrap);
+    }
+
+    pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+
+done:
+    free_xml(converted);
+    pe_free_working_set(data_set);
+
+    return reply;
+}
+
+static xmlNode *
+handle_unknown_request(pcmk__request_t *request)
+{
+    pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
+                       "ack", NULL, CRM_EX_INVALID_PARAM);
+
+    pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
+                        "Unknown IPC request type '%s' (bug?)",
+                        pcmk__client_name(request->ipc_client));
+    return NULL;
+}
+
+static xmlNode *
+handle_hello_request(pcmk__request_t *request)
+{
+    pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
+                       "ack", NULL, CRM_EX_INDETERMINATE);
+
+    crm_trace("Received IPC hello from %s", pcmk__client_name(request->ipc_client));
+
+    pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
+    return NULL;
+}
+
+static void
+schedulerd_register_handlers(void)
+{
+    pcmk__server_command_t handlers[] = {
+        { CRM_OP_HELLO, handle_hello_request },
+        { CRM_OP_PECALC, handle_pecalc_request },
+        { NULL, handle_unknown_request },
+    };
+
+    schedulerd_handlers = pcmk__register_handlers(handlers);
+}
+
+static int32_t
+pe_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
+{
+    crm_trace("Connection %p", c);
+    if (pcmk__new_client(c, uid, gid) == NULL) {
+        return -EIO;
+    }
+    return 0;
+}
+
+static int32_t
+pe_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
+{
+    uint32_t id = 0;
+    uint32_t flags = 0;
+    xmlNode *msg = NULL;
+    pcmk__client_t *c = pcmk__find_client(qbc);
+    const char *sys_to = NULL;
+
+    CRM_CHECK(c != NULL, return 0);
+
+    if (schedulerd_handlers == NULL) {
+        schedulerd_register_handlers();
+    }
+
+    msg = pcmk__client_data2xml(c, data, &id, &flags);
+    if (msg == NULL) {
+        pcmk__ipc_send_ack(c, id, flags, "ack", NULL, CRM_EX_PROTOCOL);
+        return 0;
+    }
+
+    sys_to = crm_element_value(msg, F_CRM_SYS_TO);
+
+    if (pcmk__str_eq(crm_element_value(msg, F_CRM_MSG_TYPE),
+                            XML_ATTR_RESPONSE, pcmk__str_none)) {
+        pcmk__ipc_send_ack(c, id, flags, "ack", NULL, CRM_EX_INDETERMINATE);
+        crm_info("Ignoring IPC reply from %s", pcmk__client_name(c));
+
+    } else if (!pcmk__str_eq(sys_to, CRM_SYSTEM_PENGINE, pcmk__str_none)) {
+        pcmk__ipc_send_ack(c, id, flags, "ack", NULL, CRM_EX_INDETERMINATE);
+        crm_info("Ignoring invalid IPC message: to '%s' not "
+                 CRM_SYSTEM_PENGINE, pcmk__s(sys_to, ""));
+
+    } else {
+        char *log_msg = NULL;
+        const char *reason = NULL;
+        xmlNode *reply = NULL;
+
+        pcmk__request_t request = {
+            .ipc_client     = c,
+            .ipc_id         = id,
+            .ipc_flags      = flags,
+            .peer           = NULL,
+            .xml            = msg,
+            .call_options   = 0,
+            .result         = PCMK__UNKNOWN_RESULT,
+        };
+
+        request.op = crm_element_value_copy(request.xml, F_CRM_TASK);
+        CRM_CHECK(request.op != NULL, return 0);
+
+        reply = pcmk__process_request(&request, schedulerd_handlers);
+
+        if (reply != NULL) {
+            pcmk__ipc_send_xml(c, id, reply, crm_ipc_server_event);
+            free_xml(reply);
+        }
+
+        reason = request.result.exit_reason;
+
+        log_msg = crm_strdup_printf("Processed %s request from %s %s: %s%s%s%s",
+                                    request.op, pcmk__request_origin_type(&request),
+                                    pcmk__request_origin(&request),
+                                    pcmk_exec_status_str(request.result.execution_status),
+                                    (reason == NULL)? "" : " (",
+                                    (reason == NULL)? "" : reason,
+                                    (reason == NULL)? "" : ")");
+
+        if (!pcmk__result_ok(&request.result)) {
+            crm_warn("%s", log_msg);
+        } else {
+            crm_debug("%s", log_msg);
+        }
+
+        free(log_msg);
+        pcmk__reset_request(&request);
+    }
+
+    free_xml(msg);
+    return 0;
+}
+
+/* Error code means? */
+static int32_t
+pe_ipc_closed(qb_ipcs_connection_t * c)
+{
+    pcmk__client_t *client = pcmk__find_client(c);
+
+    if (client == NULL) {
+        return 0;
+    }
+    crm_trace("Connection %p", c);
+    pcmk__free_client(client);
+    return 0;
+}
+
+static void
+pe_ipc_destroy(qb_ipcs_connection_t * c)
+{
+    crm_trace("Connection %p", c);
+    pe_ipc_closed(c);
+}
+
+struct qb_ipcs_service_handlers ipc_callbacks = {
+    .connection_accept = pe_ipc_accept,
+    .connection_created = NULL,
+    .msg_process = pe_ipc_dispatch,
+    .connection_closed = pe_ipc_closed,
+    .connection_destroyed = pe_ipc_destroy
+};