Adding upstream version 2.1.6.upstream/2.1.6

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 741 insertions, 0 deletions

diff --git a/daemons/controld/controld_fsa.c b/daemons/controld/controld_fsa.c
new file mode 100644
index 0000000..622d1c8
--- /dev/null
+++ b/daemons/controld/controld_fsa.c
@@ -0,0 +1,741 @@
+/*
+ * Copyright 2004-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <stdint.h>                 // uint64_t
+#include <string.h>
+#include <time.h>
+
+#include <crm/crm.h>
+#include <crm/lrmd.h>
+#include <crm/cib.h>
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/cluster/election_internal.h>
+#include <crm/cluster.h>
+
+#include <pacemaker-controld.h>
+
+//! Triggers an FSA invocation
+static crm_trigger_t *fsa_trigger = NULL;
+
+#define DOT_PREFIX "actions:trace: "
+#define do_dot_log(fmt, args...)     crm_trace( fmt, ##args)
+
+static void do_state_transition(enum crmd_fsa_state cur_state,
+                                enum crmd_fsa_state next_state,
+                                fsa_data_t *msg_data);
+
+void s_crmd_fsa_actions(fsa_data_t * fsa_data);
+void log_fsa_input(fsa_data_t * stored_msg);
+void init_dotfile(void);
+
+void
+init_dotfile(void)
+{
+    do_dot_log(DOT_PREFIX "digraph \"g\" {");
+    do_dot_log(DOT_PREFIX "	size = \"30,30\"");
+    do_dot_log(DOT_PREFIX "	graph [");
+    do_dot_log(DOT_PREFIX "		fontsize = \"12\"");
+    do_dot_log(DOT_PREFIX "		fontname = \"Times-Roman\"");
+    do_dot_log(DOT_PREFIX "		fontcolor = \"black\"");
+    do_dot_log(DOT_PREFIX "		bb = \"0,0,398.922306,478.927856\"");
+    do_dot_log(DOT_PREFIX "		color = \"black\"");
+    do_dot_log(DOT_PREFIX "	]");
+    do_dot_log(DOT_PREFIX "	node [");
+    do_dot_log(DOT_PREFIX "		fontsize = \"12\"");
+    do_dot_log(DOT_PREFIX "		fontname = \"Times-Roman\"");
+    do_dot_log(DOT_PREFIX "		fontcolor = \"black\"");
+    do_dot_log(DOT_PREFIX "		shape = \"ellipse\"");
+    do_dot_log(DOT_PREFIX "		color = \"black\"");
+    do_dot_log(DOT_PREFIX "	]");
+    do_dot_log(DOT_PREFIX "	edge [");
+    do_dot_log(DOT_PREFIX "		fontsize = \"12\"");
+    do_dot_log(DOT_PREFIX "		fontname = \"Times-Roman\"");
+    do_dot_log(DOT_PREFIX "		fontcolor = \"black\"");
+    do_dot_log(DOT_PREFIX "		color = \"black\"");
+    do_dot_log(DOT_PREFIX "	]");
+    do_dot_log(DOT_PREFIX "// special nodes");
+    do_dot_log(DOT_PREFIX "	\"S_PENDING\" ");
+    do_dot_log(DOT_PREFIX "	[");
+    do_dot_log(DOT_PREFIX "	 color = \"blue\"");
+    do_dot_log(DOT_PREFIX "	 fontcolor = \"blue\"");
+    do_dot_log(DOT_PREFIX "	 ]");
+    do_dot_log(DOT_PREFIX "	\"S_TERMINATE\" ");
+    do_dot_log(DOT_PREFIX "	[");
+    do_dot_log(DOT_PREFIX "	 color = \"red\"");
+    do_dot_log(DOT_PREFIX "	 fontcolor = \"red\"");
+    do_dot_log(DOT_PREFIX "	 ]");
+    do_dot_log(DOT_PREFIX "// DC only nodes");
+    do_dot_log(DOT_PREFIX "	\"S_INTEGRATION\" [ fontcolor = \"green\" ]");
+    do_dot_log(DOT_PREFIX "	\"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]");
+    do_dot_log(DOT_PREFIX "	\"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]");
+    do_dot_log(DOT_PREFIX "	\"S_RELEASE_DC\" [ fontcolor = \"green\" ]");
+    do_dot_log(DOT_PREFIX "	\"S_IDLE\" [ fontcolor = \"green\" ]");
+}
+
+static void
+do_fsa_action(fsa_data_t * fsa_data, long long an_action,
+              void (*function) (long long action,
+                                enum crmd_fsa_cause cause,
+                                enum crmd_fsa_state cur_state,
+                                enum crmd_fsa_input cur_input, fsa_data_t * msg_data))
+{
+    controld_clear_fsa_action_flags(an_action);
+    crm_trace(DOT_PREFIX "\t// %s", fsa_action2string(an_action));
+    function(an_action, fsa_data->fsa_cause, controld_globals.fsa_state,
+             fsa_data->fsa_input, fsa_data);
+}
+
+static const uint64_t startup_actions =
+    A_STARTUP | A_CIB_START | A_LRM_CONNECT | A_HA_CONNECT | A_READCONFIG |
+    A_STARTED | A_CL_JOIN_QUERY;
+
+// A_LOG, A_WARN, A_ERROR
+void
+do_log(long long action, enum crmd_fsa_cause cause,
+       enum crmd_fsa_state cur_state,
+       enum crmd_fsa_input current_input, fsa_data_t *msg_data)
+{
+    unsigned log_type = LOG_TRACE;
+
+    if (action & A_LOG) {
+        log_type = LOG_INFO;
+    } else if (action & A_WARN) {
+        log_type = LOG_WARNING;
+    } else if (action & A_ERROR) {
+        log_type = LOG_ERR;
+    }
+
+    do_crm_log(log_type, "Input %s received in state %s from %s",
+               fsa_input2string(msg_data->fsa_input),
+               fsa_state2string(cur_state), msg_data->origin);
+
+    if (msg_data->data_type == fsa_dt_ha_msg) {
+        ha_msg_input_t *input = fsa_typed_data(msg_data->data_type);
+
+        crm_log_xml_debug(input->msg, __func__);
+
+    } else if (msg_data->data_type == fsa_dt_xml) {
+        xmlNode *input = fsa_typed_data(msg_data->data_type);
+
+        crm_log_xml_debug(input, __func__);
+
+    } else if (msg_data->data_type == fsa_dt_lrm) {
+        lrmd_event_data_t *input = fsa_typed_data(msg_data->data_type);
+
+        do_crm_log(log_type,
+                   "Resource %s: Call ID %d returned %d (%d)."
+                   "  New status if rc=0: %s",
+                   input->rsc_id, input->call_id, input->rc,
+                   input->op_status, (char *)input->user_data);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Initialize the FSA trigger
+ */
+void
+controld_init_fsa_trigger(void)
+{
+    fsa_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL);
+}
+
+/*!
+ * \internal
+ * \brief Destroy the FSA trigger
+ */
+void
+controld_destroy_fsa_trigger(void)
+{
+    // This basically will not work, since mainloop has a reference to it
+    mainloop_destroy_trigger(fsa_trigger);
+    fsa_trigger = NULL;
+}
+
+/*!
+ * \internal
+ * \brief Trigger an FSA invocation
+ *
+ * \param[in] fn    Calling function name
+ * \param[in] line  Line number where call occurred
+ */
+void
+controld_trigger_fsa_as(const char *fn, int line)
+{
+    if (fsa_trigger != NULL) {
+        crm_trace("%s:%d - Triggered FSA invocation", fn, line);
+        mainloop_set_trigger(fsa_trigger);
+    }
+}
+
+enum crmd_fsa_state
+s_crmd_fsa(enum crmd_fsa_cause cause)
+{
+    controld_globals_t *globals = &controld_globals;
+    fsa_data_t *fsa_data = NULL;
+    uint64_t register_copy = controld_globals.fsa_input_register;
+    uint64_t new_actions = A_NOTHING;
+    enum crmd_fsa_state last_state;
+
+    crm_trace("FSA invoked with Cause: %s\tState: %s",
+              fsa_cause2string(cause),
+              fsa_state2string(globals->fsa_state));
+
+    fsa_dump_actions(controld_globals.fsa_actions, "Initial");
+
+    controld_clear_global_flags(controld_fsa_is_stalled);
+    if ((controld_globals.fsa_message_queue == NULL)
+        && (controld_globals.fsa_actions != A_NOTHING)) {
+        /* fake the first message so we can get into the loop */
+        fsa_data = calloc(1, sizeof(fsa_data_t));
+        fsa_data->fsa_input = I_NULL;
+        fsa_data->fsa_cause = C_FSA_INTERNAL;
+        fsa_data->origin = __func__;
+        fsa_data->data_type = fsa_dt_none;
+        controld_globals.fsa_message_queue
+            = g_list_append(controld_globals.fsa_message_queue, fsa_data);
+        fsa_data = NULL;
+    }
+    while ((controld_globals.fsa_message_queue != NULL)
+           && !pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
+        crm_trace("Checking messages (%d remaining)",
+                  g_list_length(controld_globals.fsa_message_queue));
+
+        fsa_data = get_message();
+        if(fsa_data == NULL) {
+            continue;
+        }
+
+        log_fsa_input(fsa_data);
+
+        /* add any actions back to the queue */
+        controld_set_fsa_action_flags(fsa_data->actions);
+        fsa_dump_actions(fsa_data->actions, "Restored actions");
+
+        /* get the next batch of actions */
+        new_actions = controld_fsa_get_action(fsa_data->fsa_input);
+        controld_set_fsa_action_flags(new_actions);
+        fsa_dump_actions(new_actions, "New actions");
+
+        if (fsa_data->fsa_input != I_NULL && fsa_data->fsa_input != I_ROUTER) {
+            crm_debug("Processing %s: [ state=%s cause=%s origin=%s ]",
+                      fsa_input2string(fsa_data->fsa_input),
+                      fsa_state2string(globals->fsa_state),
+                      fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
+        }
+
+        /* logging : *before* the state is changed */
+        if (pcmk_is_set(controld_globals.fsa_actions, A_ERROR)) {
+            do_fsa_action(fsa_data, A_ERROR, do_log);
+        }
+        if (pcmk_is_set(controld_globals.fsa_actions, A_WARN)) {
+            do_fsa_action(fsa_data, A_WARN, do_log);
+        }
+        if (pcmk_is_set(controld_globals.fsa_actions, A_LOG)) {
+            do_fsa_action(fsa_data, A_LOG, do_log);
+        }
+
+        /* update state variables */
+        last_state = globals->fsa_state;
+        globals->fsa_state = controld_fsa_get_next_state(fsa_data->fsa_input);
+
+        /*
+         * Remove certain actions during shutdown
+         */
+        if ((globals->fsa_state == S_STOPPING)
+            || pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+            controld_clear_fsa_action_flags(startup_actions);
+        }
+
+        /*
+         * Hook for change of state.
+         * Allows actions to be added or removed when entering a state
+         */
+        if (last_state != globals->fsa_state) {
+            do_state_transition(last_state, globals->fsa_state, fsa_data);
+        } else {
+            do_dot_log(DOT_PREFIX "\t// FSA input: State=%s \tCause=%s"
+                       " \tInput=%s \tOrigin=%s() \tid=%d",
+                       fsa_state2string(globals->fsa_state),
+                       fsa_cause2string(fsa_data->fsa_cause),
+                       fsa_input2string(fsa_data->fsa_input), fsa_data->origin, fsa_data->id);
+        }
+
+        /* start doing things... */
+        s_crmd_fsa_actions(fsa_data);
+        delete_fsa_input(fsa_data);
+        fsa_data = NULL;
+    }
+
+    if ((controld_globals.fsa_message_queue != NULL)
+        || (controld_globals.fsa_actions != A_NOTHING)
+        || pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
+
+        crm_debug("Exiting the FSA: queue=%d, fsa_actions=%#llx, stalled=%s",
+                  g_list_length(controld_globals.fsa_message_queue),
+                  (unsigned long long) controld_globals.fsa_actions,
+                  pcmk__btoa(pcmk_is_set(controld_globals.flags,
+                                         controld_fsa_is_stalled)));
+    } else {
+        crm_trace("Exiting the FSA");
+    }
+
+    /* cleanup inputs? */
+    if (register_copy != controld_globals.fsa_input_register) {
+        uint64_t same = register_copy & controld_globals.fsa_input_register;
+
+        fsa_dump_inputs(LOG_DEBUG, "Added",
+                        controld_globals.fsa_input_register ^ same);
+        fsa_dump_inputs(LOG_DEBUG, "Removed", register_copy ^ same);
+    }
+
+    fsa_dump_actions(controld_globals.fsa_actions, "Remaining");
+    fsa_dump_queue(LOG_DEBUG);
+
+    return globals->fsa_state;
+}
+
+void
+s_crmd_fsa_actions(fsa_data_t * fsa_data)
+{
+    /*
+     * Process actions in order of priority but do only one
+     * action at a time to avoid complicating the ordering.
+     */
+    CRM_CHECK(fsa_data != NULL, return);
+    while ((controld_globals.fsa_actions != A_NOTHING)
+           && !pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
+
+        /* regular action processing in order of action priority
+         *
+         * Make sure all actions that connect to required systems
+         * are performed first
+         */
+        if (pcmk_is_set(controld_globals.fsa_actions, A_ERROR)) {
+            do_fsa_action(fsa_data, A_ERROR, do_log);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_WARN)) {
+            do_fsa_action(fsa_data, A_WARN, do_log);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_LOG)) {
+            do_fsa_action(fsa_data, A_LOG, do_log);
+
+            /* get out of here NOW! before anything worse happens */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_EXIT_1)) {
+            do_fsa_action(fsa_data, A_EXIT_1, do_exit);
+
+            /* sub-system restart */
+        } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
+                                      O_LRM_RECONNECT)) {
+            do_fsa_action(fsa_data, O_LRM_RECONNECT, do_lrm_control);
+
+        } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
+                                      O_CIB_RESTART)) {
+            do_fsa_action(fsa_data, O_CIB_RESTART, do_cib_control);
+
+        } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
+                                      O_PE_RESTART)) {
+            do_fsa_action(fsa_data, O_PE_RESTART, do_pe_control);
+
+        } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
+                                      O_TE_RESTART)) {
+            do_fsa_action(fsa_data, O_TE_RESTART, do_te_control);
+
+            /* essential start tasks */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_STARTUP)) {
+            do_fsa_action(fsa_data, A_STARTUP, do_startup);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_CIB_START)) {
+            do_fsa_action(fsa_data, A_CIB_START, do_cib_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_HA_CONNECT)) {
+            do_fsa_action(fsa_data, A_HA_CONNECT, do_ha_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_READCONFIG)) {
+            do_fsa_action(fsa_data, A_READCONFIG, do_read_config);
+
+            /* sub-system start/connect */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_CONNECT)) {
+            do_fsa_action(fsa_data, A_LRM_CONNECT, do_lrm_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_START)) {
+            do_fsa_action(fsa_data, A_TE_START, do_te_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_START)) {
+            do_fsa_action(fsa_data, A_PE_START, do_pe_control);
+
+            /* Timers */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_TIMER_STOP)) {
+            do_fsa_action(fsa_data, A_DC_TIMER_STOP, do_timer_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_INTEGRATE_TIMER_STOP)) {
+            do_fsa_action(fsa_data, A_INTEGRATE_TIMER_STOP, do_timer_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_INTEGRATE_TIMER_START)) {
+            do_fsa_action(fsa_data, A_INTEGRATE_TIMER_START, do_timer_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_FINALIZE_TIMER_STOP)) {
+            do_fsa_action(fsa_data, A_FINALIZE_TIMER_STOP, do_timer_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_FINALIZE_TIMER_START)) {
+            do_fsa_action(fsa_data, A_FINALIZE_TIMER_START, do_timer_control);
+
+            /*
+             * Highest priority actions
+             */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_MSG_ROUTE)) {
+            do_fsa_action(fsa_data, A_MSG_ROUTE, do_msg_route);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_RECOVER)) {
+            do_fsa_action(fsa_data, A_RECOVER, do_recover);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_CL_JOIN_RESULT)) {
+            do_fsa_action(fsa_data, A_CL_JOIN_RESULT,
+                          do_cl_join_finalize_respond);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_CL_JOIN_REQUEST)) {
+            do_fsa_action(fsa_data, A_CL_JOIN_REQUEST,
+                          do_cl_join_offer_respond);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_SHUTDOWN_REQ)) {
+            do_fsa_action(fsa_data, A_SHUTDOWN_REQ, do_shutdown_req);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_ELECTION_VOTE)) {
+            do_fsa_action(fsa_data, A_ELECTION_VOTE, do_election_vote);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_ELECTION_COUNT)) {
+            do_fsa_action(fsa_data, A_ELECTION_COUNT, do_election_count_vote);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_EVENT)) {
+            do_fsa_action(fsa_data, A_LRM_EVENT, do_lrm_event);
+
+            /*
+             * High priority actions
+             */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_STARTED)) {
+            do_fsa_action(fsa_data, A_STARTED, do_started);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_CL_JOIN_QUERY)) {
+            do_fsa_action(fsa_data, A_CL_JOIN_QUERY, do_cl_join_query);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_DC_TIMER_START)) {
+            do_fsa_action(fsa_data, A_DC_TIMER_START, do_timer_control);
+
+            /*
+             * Medium priority actions
+             * - Membership
+             */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_TAKEOVER)) {
+            do_fsa_action(fsa_data, A_DC_TAKEOVER, do_dc_takeover);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_RELEASE)) {
+            do_fsa_action(fsa_data, A_DC_RELEASE, do_dc_release);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_JOIN_FINAL)) {
+            do_fsa_action(fsa_data, A_DC_JOIN_FINAL, do_dc_join_final);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_ELECTION_CHECK)) {
+            do_fsa_action(fsa_data, A_ELECTION_CHECK, do_election_check);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_ELECTION_START)) {
+            do_fsa_action(fsa_data, A_ELECTION_START, do_election_vote);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_DC_JOIN_OFFER_ALL)) {
+            do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_DC_JOIN_OFFER_ONE)) {
+            do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ONE, do_dc_join_offer_one);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_DC_JOIN_PROCESS_REQ)) {
+            do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_REQ,
+                          do_dc_join_filter_offer);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_DC_JOIN_PROCESS_ACK)) {
+            do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_ACK, do_dc_join_ack);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_DC_JOIN_FINALIZE)) {
+            do_fsa_action(fsa_data, A_DC_JOIN_FINALIZE, do_dc_join_finalize);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_CL_JOIN_ANNOUNCE)) {
+            do_fsa_action(fsa_data, A_CL_JOIN_ANNOUNCE, do_cl_join_announce);
+
+            /*
+             * Low(er) priority actions
+             * Make sure the CIB is always updated before invoking the
+             * scheduler, and the scheduler before the transition engine.
+             */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_HALT)) {
+            do_fsa_action(fsa_data, A_TE_HALT, do_te_invoke);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_CANCEL)) {
+            do_fsa_action(fsa_data, A_TE_CANCEL, do_te_invoke);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_INVOKE)) {
+            do_fsa_action(fsa_data, A_LRM_INVOKE, do_lrm_invoke);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_INVOKE)) {
+            do_fsa_action(fsa_data, A_PE_INVOKE, do_pe_invoke);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_INVOKE)) {
+            do_fsa_action(fsa_data, A_TE_INVOKE, do_te_invoke);
+
+            /* Shutdown actions */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_RELEASED)) {
+            do_fsa_action(fsa_data, A_DC_RELEASED, do_dc_release);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_STOP)) {
+            do_fsa_action(fsa_data, A_PE_STOP, do_pe_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_STOP)) {
+            do_fsa_action(fsa_data, A_TE_STOP, do_te_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_SHUTDOWN)) {
+            do_fsa_action(fsa_data, A_SHUTDOWN, do_shutdown);
+        } else if (pcmk_is_set(controld_globals.fsa_actions,
+                               A_LRM_DISCONNECT)) {
+            do_fsa_action(fsa_data, A_LRM_DISCONNECT, do_lrm_control);
+
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_HA_DISCONNECT)) {
+            do_fsa_action(fsa_data, A_HA_DISCONNECT, do_ha_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_CIB_STOP)) {
+            do_fsa_action(fsa_data, A_CIB_STOP, do_cib_control);
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_STOP)) {
+            do_fsa_action(fsa_data, A_STOP, do_stop);
+
+            /* exit gracefully */
+        } else if (pcmk_is_set(controld_globals.fsa_actions, A_EXIT_0)) {
+            do_fsa_action(fsa_data, A_EXIT_0, do_exit);
+
+            /* Error checking and reporting */
+        } else {
+            crm_err("Action %s not supported "CRM_XS" %#llx",
+                    fsa_action2string(controld_globals.fsa_actions),
+                    (unsigned long long) controld_globals.fsa_actions);
+            register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, fsa_data, NULL,
+                                   __func__);
+        }
+    }
+}
+
+void
+log_fsa_input(fsa_data_t * stored_msg)
+{
+    CRM_ASSERT(stored_msg);
+    crm_trace("Processing queued input %d", stored_msg->id);
+    if (stored_msg->fsa_cause == C_LRM_OP_CALLBACK) {
+        crm_trace("FSA processing LRM callback from %s", stored_msg->origin);
+
+    } else if (stored_msg->data == NULL) {
+        crm_trace("FSA processing input from %s", stored_msg->origin);
+
+    } else {
+        ha_msg_input_t *ha_input = fsa_typed_data_adv(stored_msg, fsa_dt_ha_msg,
+                                                      __func__);
+
+        crm_trace("FSA processing XML message from %s", stored_msg->origin);
+        crm_log_xml_trace(ha_input->xml, "FSA message data");
+    }
+}
+
+static void
+check_join_counts(fsa_data_t *msg_data)
+{
+    int count;
+    guint npeers;
+
+    count = crmd_join_phase_count(crm_join_finalized);
+    if (count > 0) {
+        crm_err("%d cluster node%s failed to confirm join",
+                count, pcmk__plural_s(count));
+        crmd_join_phase_log(LOG_NOTICE);
+        return;
+    }
+
+    npeers = crm_active_peers();
+    count = crmd_join_phase_count(crm_join_confirmed);
+    if (count == npeers) {
+        if (npeers == 1) {
+            crm_debug("Sole active cluster node is fully joined");
+        } else {
+            crm_debug("All %d active cluster nodes are fully joined", count);
+        }
+
+    } else if (count > npeers) {
+        crm_err("New election needed because more nodes confirmed join "
+                "than are in membership (%d > %u)", count, npeers);
+        register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
+
+    } else if (controld_globals.membership_id != crm_peer_seq) {
+        crm_info("New join needed because membership changed (%llu -> %llu)",
+                 controld_globals.membership_id, crm_peer_seq);
+        register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
+
+    } else {
+        crm_warn("Only %d of %u active cluster nodes fully joined "
+                 "(%d did not respond to offer)",
+                 count, npeers, crmd_join_phase_count(crm_join_welcomed));
+    }
+}
+
+static void
+do_state_transition(enum crmd_fsa_state cur_state,
+                    enum crmd_fsa_state next_state, fsa_data_t *msg_data)
+{
+    int level = LOG_INFO;
+    int count = 0;
+    gboolean clear_recovery_bit = TRUE;
+#if 0
+    uint64_t original_fsa_actions = controld_globals.fsa_actions;
+#endif
+
+    enum crmd_fsa_cause cause = msg_data->fsa_cause;
+    enum crmd_fsa_input current_input = msg_data->fsa_input;
+
+    const char *state_from = fsa_state2string(cur_state);
+    const char *state_to = fsa_state2string(next_state);
+    const char *input = fsa_input2string(current_input);
+
+    CRM_LOG_ASSERT(cur_state != next_state);
+
+    do_dot_log(DOT_PREFIX "\t%s -> %s [ label=%s cause=%s origin=%s ]",
+               state_from, state_to, input, fsa_cause2string(cause), msg_data->origin);
+
+    if (cur_state == S_IDLE || next_state == S_IDLE) {
+        level = LOG_NOTICE;
+    } else if (cur_state == S_NOT_DC || next_state == S_NOT_DC) {
+        level = LOG_NOTICE;
+    } else if (cur_state == S_ELECTION) {
+        level = LOG_NOTICE;
+    } else if (cur_state == S_STARTING) {
+        level = LOG_NOTICE;
+    } else if (next_state == S_RECOVERY) {
+        level = LOG_WARNING;
+    }
+
+    do_crm_log(level, "State transition %s -> %s "
+               CRM_XS " input=%s cause=%s origin=%s",
+               state_from, state_to, input, fsa_cause2string(cause),
+               msg_data->origin);
+
+    if (next_state != S_ELECTION && cur_state != S_RELEASE_DC) {
+        controld_stop_current_election_timeout();
+    }
+#if 0
+    if ((controld_globals.fsa_input_register & R_SHUTDOWN)) {
+        controld_set_fsa_action_flags(A_DC_TIMER_STOP);
+    }
+#endif
+    if (next_state == S_INTEGRATION) {
+        controld_set_fsa_action_flags(A_INTEGRATE_TIMER_START);
+    } else {
+        controld_set_fsa_action_flags(A_INTEGRATE_TIMER_STOP);
+    }
+
+    if (next_state == S_FINALIZE_JOIN) {
+        controld_set_fsa_action_flags(A_FINALIZE_TIMER_START);
+    } else {
+        controld_set_fsa_action_flags(A_FINALIZE_TIMER_STOP);
+    }
+
+    if (next_state != S_PENDING) {
+        controld_set_fsa_action_flags(A_DC_TIMER_STOP);
+    }
+    if (next_state != S_IDLE) {
+        controld_stop_recheck_timer();
+    }
+
+    if (cur_state == S_FINALIZE_JOIN && next_state == S_POLICY_ENGINE) {
+        populate_cib_nodes(node_update_quick|node_update_all, __func__);
+    }
+
+    switch (next_state) {
+        case S_PENDING:
+            {
+                cib_t *cib_conn = controld_globals.cib_conn;
+                cib_conn->cmds->set_secondary(cib_conn, cib_scope_local);
+            }
+            update_dc(NULL);
+            break;
+
+        case S_ELECTION:
+            update_dc(NULL);
+            break;
+
+        case S_NOT_DC:
+            controld_reset_counter_election_timer();
+            purge_stonith_cleanup();
+
+            if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+                crm_info("(Re)Issuing shutdown request now" " that we have a new DC");
+                controld_set_fsa_action_flags(A_SHUTDOWN_REQ);
+            }
+            CRM_LOG_ASSERT(controld_globals.dc_name != NULL);
+            if (controld_globals.dc_name == NULL) {
+                crm_err("Reached S_NOT_DC without a DC" " being recorded");
+            }
+            break;
+
+        case S_RECOVERY:
+            clear_recovery_bit = FALSE;
+            break;
+
+        case S_FINALIZE_JOIN:
+            CRM_LOG_ASSERT(AM_I_DC);
+            if (cause == C_TIMER_POPPED) {
+                crm_warn("Progressed to state %s after %s",
+                         fsa_state2string(next_state), fsa_cause2string(cause));
+            }
+            count = crmd_join_phase_count(crm_join_welcomed);
+            if (count > 0) {
+                crm_warn("%d cluster node%s failed to respond to join offer",
+                         count, pcmk__plural_s(count));
+                crmd_join_phase_log(LOG_NOTICE);
+
+            } else {
+                crm_debug("All cluster nodes (%d) responded to join offer",
+                          crmd_join_phase_count(crm_join_integrated));
+            }
+            break;
+
+        case S_POLICY_ENGINE:
+            controld_reset_counter_election_timer();
+            CRM_LOG_ASSERT(AM_I_DC);
+            if (cause == C_TIMER_POPPED) {
+                crm_info("Progressed to state %s after %s",
+                         fsa_state2string(next_state), fsa_cause2string(cause));
+            }
+            check_join_counts(msg_data);
+            break;
+
+        case S_STOPPING:
+        case S_TERMINATE:
+            /* possibly redundant */
+            controld_set_fsa_input_flags(R_SHUTDOWN);
+            break;
+
+        case S_IDLE:
+            CRM_LOG_ASSERT(AM_I_DC);
+            if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
+                crm_info("(Re)Issuing shutdown request now" " that we are the DC");
+                controld_set_fsa_action_flags(A_SHUTDOWN_REQ);
+            }
+            controld_start_recheck_timer();
+            break;
+
+        default:
+            break;
+    }
+
+    if (clear_recovery_bit && next_state != S_PENDING) {
+        controld_clear_fsa_action_flags(A_RECOVER);
+    } else if (clear_recovery_bit == FALSE) {
+        controld_set_fsa_action_flags(A_RECOVER);
+    }
+
+#if 0
+    if (original_fsa_actions != controld_globals.fsa_actions) {
+        fsa_dump_actions(original_fsa_actions ^ controld_globals.fsa_actions,
+                         "New actions");
+    }
+#endif
+}