Adding upstream version 2.1.6.upstream/2.1.6

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 987 insertions, 0 deletions

diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c
new file mode 100644
index 0000000..f82b132
--- /dev/null
+++ b/daemons/controld/controld_join_dc.c
@@ -0,0 +1,987 @@
+/*
+ * Copyright 2004-2023 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU General Public License version 2
+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <crm/crm.h>
+
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+#include <crm/cluster.h>
+
+#include <pacemaker-controld.h>
+
+static char *max_generation_from = NULL;
+static xmlNodePtr max_generation_xml = NULL;
+
+/*!
+ * \internal
+ * \brief Nodes from which a CIB sync has failed since the peer joined
+ *
+ * This table is of the form (<tt>node_name -> join_id</tt>). \p node_name is
+ * the name of a client node from which a CIB \p sync_from() call has failed in
+ * \p do_dc_join_finalize() since the client joined the cluster as a peer.
+ * \p join_id is the ID of the join round in which the \p sync_from() failed,
+ * and is intended for use in nack log messages.
+ */
+static GHashTable *failed_sync_nodes = NULL;
+
+void finalize_join_for(gpointer key, gpointer value, gpointer user_data);
+void finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
+gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
+
+/* Numeric counter used to identify join rounds (an unsigned int would be
+ * appropriate, except we get and set it in XML as int)
+ */
+static int current_join_id = 0;
+
+/*!
+ * \internal
+ * \brief Destroy the hash table containing failed sync nodes
+ */
+void
+controld_destroy_failed_sync_table(void)
+{
+    if (failed_sync_nodes != NULL) {
+        g_hash_table_destroy(failed_sync_nodes);
+        failed_sync_nodes = NULL;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Remove a node from the failed sync nodes table if present
+ *
+ * \param[in] node_name  Node name to remove
+ */
+void
+controld_remove_failed_sync_node(const char *node_name)
+{
+    if (failed_sync_nodes != NULL) {
+        g_hash_table_remove(failed_sync_nodes, (gchar *) node_name);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Add to a hash table a node whose CIB failed to sync
+ *
+ * \param[in] node_name  Name of node whose CIB failed to sync
+ * \param[in] join_id    Join round when the failure occurred
+ */
+static void
+record_failed_sync_node(const char *node_name, gint join_id)
+{
+    if (failed_sync_nodes == NULL) {
+        failed_sync_nodes = pcmk__strikey_table(g_free, NULL);
+    }
+
+    /* If the node is already in the table then we failed to nack it during the
+     * filter offer step
+     */
+    CRM_LOG_ASSERT(g_hash_table_insert(failed_sync_nodes, g_strdup(node_name),
+                                       GINT_TO_POINTER(join_id)));
+}
+
+/*!
+ * \internal
+ * \brief Look up a node name in the failed sync table
+ *
+ * \param[in]  node_name  Name of node to look up
+ * \param[out] join_id    Where to store the join ID of when the sync failed
+ *
+ * \return Standard Pacemaker return code. Specifically, \p pcmk_rc_ok if the
+ *         node name was found, or \p pcmk_rc_node_unknown otherwise.
+ * \note \p *join_id is set to -1 if the node is not found.
+ */
+static int
+lookup_failed_sync_node(const char *node_name, gint *join_id)
+{
+    *join_id = -1;
+
+    if (failed_sync_nodes != NULL) {
+        gpointer result = g_hash_table_lookup(failed_sync_nodes,
+                                              (gchar *) node_name);
+        if (result != NULL) {
+            *join_id = GPOINTER_TO_INT(result);
+            return pcmk_rc_ok;
+        }
+    }
+    return pcmk_rc_node_unknown;
+}
+
+void
+crm_update_peer_join(const char *source, crm_node_t * node, enum crm_join_phase phase)
+{
+    enum crm_join_phase last = 0;
+
+    CRM_CHECK(node != NULL, return);
+
+    /* Remote nodes do not participate in joins */
+    if (pcmk_is_set(node->flags, crm_remote_node)) {
+        return;
+    }
+
+    last = node->join;
+
+    if(phase == last) {
+        crm_trace("Node %s join-%d phase is still %s "
+                  CRM_XS " nodeid=%u source=%s",
+                  node->uname, current_join_id, crm_join_phase_str(last),
+                  node->id, source);
+
+    } else if ((phase <= crm_join_none) || (phase == (last + 1))) {
+        node->join = phase;
+        crm_trace("Node %s join-%d phase is now %s (was %s) "
+                  CRM_XS " nodeid=%u source=%s",
+                 node->uname, current_join_id, crm_join_phase_str(phase),
+                 crm_join_phase_str(last), node->id, source);
+
+    } else {
+        crm_warn("Rejecting join-%d phase update for node %s because "
+                 "can't go from %s to %s " CRM_XS " nodeid=%u source=%s",
+                 current_join_id, node->uname, crm_join_phase_str(last),
+                 crm_join_phase_str(phase), node->id, source);
+    }
+}
+
+static void
+start_join_round(void)
+{
+    GHashTableIter iter;
+    crm_node_t *peer = NULL;
+
+    crm_debug("Starting new join round join-%d", current_join_id);
+
+    g_hash_table_iter_init(&iter, crm_peer_cache);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
+        crm_update_peer_join(__func__, peer, crm_join_none);
+    }
+    if (max_generation_from != NULL) {
+        free(max_generation_from);
+        max_generation_from = NULL;
+    }
+    if (max_generation_xml != NULL) {
+        free_xml(max_generation_xml);
+        max_generation_xml = NULL;
+    }
+    controld_clear_fsa_input_flags(R_HAVE_CIB);
+    controld_forget_all_cib_replace_calls();
+}
+
+/*!
+ * \internal
+ * \brief Create a join message from the DC
+ *
+ * \param[in] join_op  Join operation name
+ * \param[in] host_to  Recipient of message
+ */
+static xmlNode *
+create_dc_message(const char *join_op, const char *host_to)
+{
+    xmlNode *msg = create_request(join_op, NULL, host_to, CRM_SYSTEM_CRMD,
+                                  CRM_SYSTEM_DC, NULL);
+
+    /* Identify which election this is a part of */
+    crm_xml_add_int(msg, F_CRM_JOIN_ID, current_join_id);
+
+    /* Add a field specifying whether the DC is shutting down. This keeps the
+     * joining node from fencing the old DC if it becomes the new DC.
+     */
+    pcmk__xe_set_bool_attr(msg, F_CRM_DC_LEAVING,
+                           pcmk_is_set(controld_globals.fsa_input_register,
+                                       R_SHUTDOWN));
+    return msg;
+}
+
+static void
+join_make_offer(gpointer key, gpointer value, gpointer user_data)
+{
+    xmlNode *offer = NULL;
+    crm_node_t *member = (crm_node_t *)value;
+
+    CRM_ASSERT(member != NULL);
+    if (crm_is_peer_active(member) == FALSE) {
+        crm_info("Not making join-%d offer to inactive node %s",
+                 current_join_id,
+                 (member->uname? member->uname : "with unknown name"));
+        if(member->expected == NULL && pcmk__str_eq(member->state, CRM_NODE_LOST, pcmk__str_casei)) {
+            /* You would think this unsafe, but in fact this plus an
+             * active resource is what causes it to be fenced.
+             *
+             * Yes, this does mean that any node that dies at the same
+             * time as the old DC and is not running resource (still)
+             * won't be fenced.
+             *
+             * I'm not happy about this either.
+             */
+            pcmk__update_peer_expected(__func__, member, CRMD_JOINSTATE_DOWN);
+        }
+        return;
+    }
+
+    if (member->uname == NULL) {
+        crm_info("Not making join-%d offer to node uuid %s with unknown name",
+                 current_join_id, member->uuid);
+        return;
+    }
+
+    if (controld_globals.membership_id != crm_peer_seq) {
+        controld_globals.membership_id = crm_peer_seq;
+        crm_info("Making join-%d offers based on membership event %llu",
+                 current_join_id, crm_peer_seq);
+    }
+
+    if(user_data && member->join > crm_join_none) {
+        crm_info("Not making join-%d offer to already known node %s (%s)",
+                 current_join_id, member->uname,
+                 crm_join_phase_str(member->join));
+        return;
+    }
+
+    crm_update_peer_join(__func__, (crm_node_t*)member, crm_join_none);
+
+    offer = create_dc_message(CRM_OP_JOIN_OFFER, member->uname);
+
+    // Advertise our feature set so the joining node can bail if not compatible
+    crm_xml_add(offer, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
+
+    crm_info("Sending join-%d offer to %s", current_join_id, member->uname);
+    send_cluster_message(member, crm_msg_crmd, offer, TRUE);
+    free_xml(offer);
+
+    crm_update_peer_join(__func__, member, crm_join_welcomed);
+}
+
+/*	 A_DC_JOIN_OFFER_ALL	*/
+void
+do_dc_join_offer_all(long long action,
+                     enum crmd_fsa_cause cause,
+                     enum crmd_fsa_state cur_state,
+                     enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    int count;
+
+    /* Reset everyone's status back to down or in_ccm in the CIB.
+     * Any nodes that are active in the CIB but not in the cluster membership
+     * will be seen as offline by the scheduler anyway.
+     */
+    current_join_id++;
+    start_join_round();
+
+    update_dc(NULL);
+    if (cause == C_HA_MESSAGE && current_input == I_NODE_JOIN) {
+        crm_info("A new node joined the cluster");
+    }
+    g_hash_table_foreach(crm_peer_cache, join_make_offer, NULL);
+
+    count = crmd_join_phase_count(crm_join_welcomed);
+    crm_info("Waiting on join-%d requests from %d outstanding node%s",
+             current_join_id, count, pcmk__plural_s(count));
+
+    // Don't waste time by invoking the scheduler yet
+}
+
+/*	 A_DC_JOIN_OFFER_ONE	*/
+void
+do_dc_join_offer_one(long long action,
+                     enum crmd_fsa_cause cause,
+                     enum crmd_fsa_state cur_state,
+                     enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    crm_node_t *member;
+    ha_msg_input_t *welcome = NULL;
+    int count;
+    const char *join_to = NULL;
+
+    if (msg_data->data == NULL) {
+        crm_info("Making join-%d offers to any unconfirmed nodes "
+                 "because an unknown node joined", current_join_id);
+        g_hash_table_foreach(crm_peer_cache, join_make_offer, &member);
+        check_join_state(cur_state, __func__);
+        return;
+    }
+
+    welcome = fsa_typed_data(fsa_dt_ha_msg);
+    if (welcome == NULL) {
+        // fsa_typed_data() already logged an error
+        return;
+    }
+
+    join_to = crm_element_value(welcome->msg, F_CRM_HOST_FROM);
+    if (join_to == NULL) {
+        crm_err("Can't make join-%d offer to unknown node", current_join_id);
+        return;
+    }
+    member = crm_get_peer(0, join_to);
+
+    /* It is possible that a node will have been sick or starting up when the
+     * original offer was made. However, it will either re-announce itself in
+     * due course, or we can re-store the original offer on the client.
+     */
+
+    crm_update_peer_join(__func__, member, crm_join_none);
+    join_make_offer(NULL, member, NULL);
+
+    /* If the offer isn't to the local node, make an offer to the local node as
+     * well, to ensure the correct value for max_generation_from.
+     */
+    if (strcasecmp(join_to, controld_globals.our_nodename) != 0) {
+        member = crm_get_peer(0, controld_globals.our_nodename);
+        join_make_offer(NULL, member, NULL);
+    }
+
+    /* This was a genuine join request; cancel any existing transition and
+     * invoke the scheduler.
+     */
+    abort_transition(INFINITY, pcmk__graph_restart, "Node join", NULL);
+
+    count = crmd_join_phase_count(crm_join_welcomed);
+    crm_info("Waiting on join-%d requests from %d outstanding node%s",
+             current_join_id, count, pcmk__plural_s(count));
+
+    // Don't waste time by invoking the scheduler yet
+}
+
+static int
+compare_int_fields(xmlNode * left, xmlNode * right, const char *field)
+{
+    const char *elem_l = crm_element_value(left, field);
+    const char *elem_r = crm_element_value(right, field);
+
+    long long int_elem_l;
+    long long int_elem_r;
+
+    pcmk__scan_ll(elem_l, &int_elem_l, -1LL);
+    pcmk__scan_ll(elem_r, &int_elem_r, -1LL);
+
+    if (int_elem_l < int_elem_r) {
+        return -1;
+
+    } else if (int_elem_l > int_elem_r) {
+        return 1;
+    }
+
+    return 0;
+}
+
+/*	 A_DC_JOIN_PROCESS_REQ	*/
+void
+do_dc_join_filter_offer(long long action,
+                        enum crmd_fsa_cause cause,
+                        enum crmd_fsa_state cur_state,
+                        enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    xmlNode *generation = NULL;
+
+    int cmp = 0;
+    int join_id = -1;
+    int count = 0;
+    gint value = 0;
+    gboolean ack_nack_bool = TRUE;
+    ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);
+
+    const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
+    const char *ref = crm_element_value(join_ack->msg, F_CRM_REFERENCE);
+    const char *join_version = crm_element_value(join_ack->msg,
+                                                 XML_ATTR_CRM_VERSION);
+    crm_node_t *join_node = NULL;
+
+    if (join_from == NULL) {
+        crm_err("Ignoring invalid join request without node name");
+        return;
+    }
+    join_node = crm_get_peer(0, join_from);
+
+    crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id);
+    if (join_id != current_join_id) {
+        crm_debug("Ignoring join-%d request from %s because we are on join-%d",
+                  join_id, join_from, current_join_id);
+        check_join_state(cur_state, __func__);
+        return;
+    }
+
+    generation = join_ack->xml;
+    if (max_generation_xml != NULL && generation != NULL) {
+        int lpc = 0;
+
+        const char *attributes[] = {
+            XML_ATTR_GENERATION_ADMIN,
+            XML_ATTR_GENERATION,
+            XML_ATTR_NUMUPDATES,
+        };
+
+        for (lpc = 0; cmp == 0 && lpc < PCMK__NELEM(attributes); lpc++) {
+            cmp = compare_int_fields(max_generation_xml, generation, attributes[lpc]);
+        }
+    }
+
+    if (ref == NULL) {
+        ref = "none"; // for logging only
+    }
+
+    if (lookup_failed_sync_node(join_from, &value) == pcmk_rc_ok) {
+        crm_err("Rejecting join-%d request from node %s because we failed to "
+                "sync its CIB in join-%d " CRM_XS " ref=%s",
+                join_id, join_from, value, ref);
+        ack_nack_bool = FALSE;
+
+    } else if (!crm_is_peer_active(join_node)) {
+        if (match_down_event(join_from) != NULL) {
+            /* The join request was received after the node was fenced or
+             * otherwise shutdown in a way that we're aware of. No need to log
+             * an error in this rare occurrence; we know the client was recently
+             * shut down, and receiving a lingering in-flight request is not
+             * cause for alarm.
+             */
+            crm_debug("Rejecting join-%d request from inactive node %s "
+                      CRM_XS " ref=%s", join_id, join_from, ref);
+        } else {
+            crm_err("Rejecting join-%d request from inactive node %s "
+                    CRM_XS " ref=%s", join_id, join_from, ref);
+        }
+        ack_nack_bool = FALSE;
+
+    } else if (generation == NULL) {
+        crm_err("Rejecting invalid join-%d request from node %s "
+                "missing CIB generation " CRM_XS " ref=%s",
+                join_id, join_from, ref);
+        ack_nack_bool = FALSE;
+
+    } else if ((join_version == NULL)
+               || !feature_set_compatible(CRM_FEATURE_SET, join_version)) {
+        crm_err("Rejecting join-%d request from node %s because feature set %s"
+                " is incompatible with ours (%s) " CRM_XS " ref=%s",
+                join_id, join_from, (join_version? join_version : "pre-3.1.0"),
+                CRM_FEATURE_SET, ref);
+        ack_nack_bool = FALSE;
+
+    } else if (max_generation_xml == NULL) {
+        const char *validation = crm_element_value(generation,
+                                                   XML_ATTR_VALIDATION);
+
+        if (get_schema_version(validation) < 0) {
+            crm_err("Rejecting join-%d request from %s (with first CIB "
+                    "generation) due to unknown schema version %s "
+                    CRM_XS " ref=%s",
+                    join_id, join_from, validation, ref);
+            ack_nack_bool = FALSE;
+
+        } else {
+            crm_debug("Accepting join-%d request from %s (with first CIB "
+                      "generation) " CRM_XS " ref=%s",
+                      join_id, join_from, ref);
+            max_generation_xml = copy_xml(generation);
+            pcmk__str_update(&max_generation_from, join_from);
+        }
+
+    } else if ((cmp < 0)
+               || ((cmp == 0)
+                   && pcmk__str_eq(join_from, controld_globals.our_nodename,
+                                   pcmk__str_casei))) {
+        const char *validation = crm_element_value(generation,
+                                                   XML_ATTR_VALIDATION);
+
+        if (get_schema_version(validation) < 0) {
+            crm_err("Rejecting join-%d request from %s (with better CIB "
+                    "generation than current best from %s) due to unknown "
+                    "schema version %s " CRM_XS " ref=%s",
+                    join_id, join_from, max_generation_from, validation, ref);
+            ack_nack_bool = FALSE;
+
+        } else {
+            crm_debug("Accepting join-%d request from %s (with better CIB "
+                      "generation than current best from %s) " CRM_XS " ref=%s",
+                      join_id, join_from, max_generation_from, ref);
+            crm_log_xml_debug(max_generation_xml, "Old max generation");
+            crm_log_xml_debug(generation, "New max generation");
+
+            free_xml(max_generation_xml);
+            max_generation_xml = copy_xml(join_ack->xml);
+            pcmk__str_update(&max_generation_from, join_from);
+        }
+
+    } else {
+        crm_debug("Accepting join-%d request from %s " CRM_XS " ref=%s",
+                  join_id, join_from, ref);
+    }
+
+    if (!ack_nack_bool) {
+        if (compare_version(join_version, "3.17.0") < 0) {
+            /* Clients with CRM_FEATURE_SET < 3.17.0 may respawn infinitely
+             * after a nack message, don't send one
+             */
+            crm_update_peer_join(__func__, join_node, crm_join_nack_quiet);
+        } else {
+            crm_update_peer_join(__func__, join_node, crm_join_nack);
+        }
+        pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_NACK);
+
+    } else {
+        crm_update_peer_join(__func__, join_node, crm_join_integrated);
+        pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
+    }
+
+    count = crmd_join_phase_count(crm_join_integrated);
+    crm_debug("%d node%s currently integrated in join-%d",
+              count, pcmk__plural_s(count), join_id);
+
+    if (check_join_state(cur_state, __func__) == FALSE) {
+        // Don't waste time by invoking the scheduler yet
+        count = crmd_join_phase_count(crm_join_welcomed);
+        crm_debug("Waiting on join-%d requests from %d outstanding node%s",
+                  join_id, count, pcmk__plural_s(count));
+    }
+}
+
+/*	A_DC_JOIN_FINALIZE	*/
+void
+do_dc_join_finalize(long long action,
+                    enum crmd_fsa_cause cause,
+                    enum crmd_fsa_state cur_state,
+                    enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    char *sync_from = NULL;
+    int rc = pcmk_ok;
+    int count_welcomed = crmd_join_phase_count(crm_join_welcomed);
+    int count_finalizable = crmd_join_phase_count(crm_join_integrated)
+                            + crmd_join_phase_count(crm_join_nack)
+                            + crmd_join_phase_count(crm_join_nack_quiet);
+
+    /* This we can do straight away and avoid clients timing us out
+     *  while we compute the latest CIB
+     */
+    if (count_welcomed != 0) {
+        crm_debug("Waiting on join-%d requests from %d outstanding node%s "
+                  "before finalizing join", current_join_id, count_welcomed,
+                  pcmk__plural_s(count_welcomed));
+        crmd_join_phase_log(LOG_DEBUG);
+        /* crmd_fsa_stall(FALSE); Needed? */
+        return;
+
+    } else if (count_finalizable == 0) {
+        crm_debug("Finalization not needed for join-%d at the current time",
+                  current_join_id);
+        crmd_join_phase_log(LOG_DEBUG);
+        check_join_state(controld_globals.fsa_state, __func__);
+        return;
+    }
+
+    controld_clear_fsa_input_flags(R_HAVE_CIB);
+    if (pcmk__str_eq(max_generation_from, controld_globals.our_nodename,
+                     pcmk__str_null_matches|pcmk__str_casei)) {
+        controld_set_fsa_input_flags(R_HAVE_CIB);
+    }
+
+    if (!controld_globals.transition_graph->complete) {
+        crm_warn("Delaying join-%d finalization while transition in progress",
+                 current_join_id);
+        crmd_join_phase_log(LOG_DEBUG);
+        crmd_fsa_stall(FALSE);
+        return;
+    }
+
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
+        // Send our CIB out to everyone
+        pcmk__str_update(&sync_from, controld_globals.our_nodename);
+        crm_debug("Finalizing join-%d for %d node%s (sync'ing from local CIB)",
+                  current_join_id, count_finalizable,
+                  pcmk__plural_s(count_finalizable));
+        crm_log_xml_debug(max_generation_xml, "Requested CIB version");
+
+    } else {
+        // Ask for the agreed best CIB
+        pcmk__str_update(&sync_from, max_generation_from);
+        crm_notice("Finalizing join-%d for %d node%s (sync'ing CIB from %s)",
+                   current_join_id, count_finalizable,
+                   pcmk__plural_s(count_finalizable), sync_from);
+        crm_log_xml_notice(max_generation_xml, "Requested CIB version");
+    }
+    crmd_join_phase_log(LOG_DEBUG);
+
+    rc = controld_globals.cib_conn->cmds->sync_from(controld_globals.cib_conn,
+                                                    sync_from, NULL, cib_none);
+
+    if (pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
+        controld_record_cib_replace_call(rc);
+    }
+    fsa_register_cib_callback(rc, sync_from, finalize_sync_callback);
+}
+
+void
+free_max_generation(void)
+{
+    free(max_generation_from);
+    max_generation_from = NULL;
+
+    free_xml(max_generation_xml);
+    max_generation_xml = NULL;
+}
+
+void
+finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    CRM_LOG_ASSERT(-EPERM != rc);
+
+    controld_forget_cib_replace_call(call_id);
+
+    if (rc != pcmk_ok) {
+        const char *sync_from = (const char *) user_data;
+
+        do_crm_log(((rc == -pcmk_err_old_data)? LOG_WARNING : LOG_ERR),
+                   "Could not sync CIB from %s in join-%d: %s",
+                   sync_from, current_join_id, pcmk_strerror(rc));
+
+        if (rc != -pcmk_err_old_data) {
+            record_failed_sync_node(sync_from, current_join_id);
+        }
+
+        /* restart the whole join process */
+        register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION_DC, NULL, NULL,
+                               __func__);
+
+    } else if (!AM_I_DC) {
+        crm_debug("Sync'ed CIB for join-%d but no longer DC", current_join_id);
+
+    } else if (controld_globals.fsa_state != S_FINALIZE_JOIN) {
+        crm_debug("Sync'ed CIB for join-%d but no longer in S_FINALIZE_JOIN "
+                  "(%s)", current_join_id,
+                  fsa_state2string(controld_globals.fsa_state));
+
+    } else {
+        controld_set_fsa_input_flags(R_HAVE_CIB);
+
+        /* make sure dc_uuid is re-set to us */
+        if (!check_join_state(controld_globals.fsa_state, __func__)) {
+            int count_finalizable = 0;
+
+            count_finalizable = crmd_join_phase_count(crm_join_integrated)
+                                + crmd_join_phase_count(crm_join_nack)
+                                + crmd_join_phase_count(crm_join_nack_quiet);
+
+            crm_debug("Notifying %d node%s of join-%d results",
+                      count_finalizable, pcmk__plural_s(count_finalizable),
+                      current_join_id);
+            g_hash_table_foreach(crm_peer_cache, finalize_join_for, NULL);
+        }
+    }
+}
+
+static void
+join_update_complete_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
+{
+    fsa_data_t *msg_data = NULL;
+
+    if (rc == pcmk_ok) {
+        crm_debug("join-%d node history update (via CIB call %d) complete",
+                  current_join_id, call_id);
+        check_join_state(controld_globals.fsa_state, __func__);
+
+    } else {
+        crm_err("join-%d node history update (via CIB call %d) failed: %s "
+                "(next transition may determine resource status incorrectly)",
+                current_join_id, call_id, pcmk_strerror(rc));
+        crm_log_xml_debug(msg, "failed");
+        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
+    }
+}
+
+/*	A_DC_JOIN_PROCESS_ACK	*/
+void
+do_dc_join_ack(long long action,
+               enum crmd_fsa_cause cause,
+               enum crmd_fsa_state cur_state,
+               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    int join_id = -1;
+    ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);
+    enum controld_section_e section = controld_section_lrm;
+    const int cib_opts = cib_scope_local|cib_can_create;
+
+    const char *op = crm_element_value(join_ack->msg, F_CRM_TASK);
+    const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
+    crm_node_t *peer = NULL;
+
+    // Sanity checks
+    if (join_from == NULL) {
+        crm_warn("Ignoring message received without node identification");
+        return;
+    }
+    if (op == NULL) {
+        crm_warn("Ignoring message received from %s without task", join_from);
+        return;
+    }
+
+    if (strcmp(op, CRM_OP_JOIN_CONFIRM)) {
+        crm_debug("Ignoring '%s' message from %s while waiting for '%s'",
+                  op, join_from, CRM_OP_JOIN_CONFIRM);
+        return;
+    }
+
+    if (crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id) != 0) {
+        crm_warn("Ignoring join confirmation from %s without valid join ID",
+                 join_from);
+        return;
+    }
+
+    peer = crm_get_peer(0, join_from);
+    if (peer->join != crm_join_finalized) {
+        crm_info("Ignoring out-of-sequence join-%d confirmation from %s "
+                 "(currently %s not %s)",
+                 join_id, join_from, crm_join_phase_str(peer->join),
+                 crm_join_phase_str(crm_join_finalized));
+        return;
+    }
+
+    if (join_id != current_join_id) {
+        crm_err("Rejecting join-%d confirmation from %s "
+                "because currently on join-%d",
+                join_id, join_from, current_join_id);
+        crm_update_peer_join(__func__, peer, crm_join_nack);
+        return;
+    }
+
+    crm_update_peer_join(__func__, peer, crm_join_confirmed);
+
+    /* Update CIB with node's current executor state. A new transition will be
+     * triggered later, when the CIB notifies us of the change.
+     */
+    if (pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
+        section = controld_section_lrm_unlocked;
+    }
+    controld_delete_node_state(join_from, section, cib_scope_local);
+    if (pcmk__str_eq(join_from, controld_globals.our_nodename,
+                     pcmk__str_casei)) {
+        xmlNode *now_dc_lrmd_state = controld_query_executor_state();
+
+        if (now_dc_lrmd_state != NULL) {
+            crm_debug("Updating local node history for join-%d "
+                      "from query result", join_id);
+            controld_update_cib(XML_CIB_TAG_STATUS, now_dc_lrmd_state, cib_opts,
+                                join_update_complete_callback);
+            free_xml(now_dc_lrmd_state);
+        } else {
+            crm_warn("Updating local node history from join-%d confirmation "
+                     "because query failed", join_id);
+            controld_update_cib(XML_CIB_TAG_STATUS, join_ack->xml, cib_opts,
+                                join_update_complete_callback);
+        }
+    } else {
+        crm_debug("Updating node history for %s from join-%d confirmation",
+                  join_from, join_id);
+        controld_update_cib(XML_CIB_TAG_STATUS, join_ack->xml, cib_opts,
+                            join_update_complete_callback);
+    }
+}
+
+void
+finalize_join_for(gpointer key, gpointer value, gpointer user_data)
+{
+    xmlNode *acknak = NULL;
+    xmlNode *tmp1 = NULL;
+    crm_node_t *join_node = value;
+    const char *join_to = join_node->uname;
+    bool integrated = false;
+
+    switch (join_node->join) {
+        case crm_join_integrated:
+            integrated = true;
+            break;
+        case crm_join_nack:
+        case crm_join_nack_quiet:
+            break;
+        default:
+            crm_trace("Not updating non-integrated and non-nacked node %s (%s) "
+                      "for join-%d", join_to,
+                      crm_join_phase_str(join_node->join), current_join_id);
+            return;
+    }
+
+    /* Update the <node> element with the node's name and UUID, in case they
+     * weren't known before
+     */
+    crm_trace("Updating node name and UUID in CIB for %s", join_to);
+    tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE);
+    set_uuid(tmp1, XML_ATTR_ID, join_node);
+    crm_xml_add(tmp1, XML_ATTR_UNAME, join_to);
+    fsa_cib_anon_update(XML_CIB_TAG_NODES, tmp1);
+    free_xml(tmp1);
+
+    if (join_node->join == crm_join_nack_quiet) {
+        crm_trace("Not sending nack message to node %s with feature set older "
+                  "than 3.17.0", join_to);
+        return;
+    }
+
+    join_node = crm_get_peer(0, join_to);
+    if (!crm_is_peer_active(join_node)) {
+        /*
+         * NACK'ing nodes that the membership layer doesn't know about yet
+         * simply creates more churn
+         *
+         * Better to leave them waiting and let the join restart when
+         * the new membership event comes in
+         *
+         * All other NACKs (due to versions etc) should still be processed
+         */
+        pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_PENDING);
+        return;
+    }
+
+    // Acknowledge or nack node's join request
+    crm_debug("%sing join-%d request from %s",
+              integrated? "Acknowledg" : "Nack", current_join_id, join_to);
+    acknak = create_dc_message(CRM_OP_JOIN_ACKNAK, join_to);
+    pcmk__xe_set_bool_attr(acknak, CRM_OP_JOIN_ACKNAK, integrated);
+
+    if (integrated) {
+        // No change needed for a nacked node
+        crm_update_peer_join(__func__, join_node, crm_join_finalized);
+        pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
+
+        /* Iterate through the remote peer cache and add information on which
+         * node hosts each to the ACK message.  This keeps new controllers in
+         * sync with what has already happened.
+         */
+        if (crm_remote_peer_cache_size() != 0) {
+            GHashTableIter iter;
+            crm_node_t *node = NULL;
+            xmlNode *remotes = create_xml_node(acknak, XML_CIB_TAG_NODES);
+
+            g_hash_table_iter_init(&iter, crm_remote_peer_cache);
+            while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
+                xmlNode *remote = NULL;
+
+                if (!node->conn_host) {
+                    continue;
+                }
+
+                remote = create_xml_node(remotes, XML_CIB_TAG_NODE);
+                pcmk__xe_set_props(remote,
+                                   XML_ATTR_ID, node->uname,
+                                   XML_CIB_TAG_STATE, node->state,
+                                   PCMK__XA_CONN_HOST, node->conn_host,
+                                   NULL);
+            }
+        }
+    }
+    send_cluster_message(join_node, crm_msg_crmd, acknak, TRUE);
+    free_xml(acknak);
+    return;
+}
+
+gboolean
+check_join_state(enum crmd_fsa_state cur_state, const char *source)
+{
+    static unsigned long long highest_seq = 0;
+
+    if (controld_globals.membership_id != crm_peer_seq) {
+        crm_debug("join-%d: Membership changed from %llu to %llu "
+                  CRM_XS " highest=%llu state=%s for=%s",
+                  current_join_id, controld_globals.membership_id, crm_peer_seq,
+                  highest_seq, fsa_state2string(cur_state), source);
+        if(highest_seq < crm_peer_seq) {
+            /* Don't spam the FSA with duplicates */
+            highest_seq = crm_peer_seq;
+            register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
+        }
+
+    } else if (cur_state == S_INTEGRATION) {
+        if (crmd_join_phase_count(crm_join_welcomed) == 0) {
+            int count = crmd_join_phase_count(crm_join_integrated);
+
+            crm_debug("join-%d: Integration of %d peer%s complete "
+                      CRM_XS " state=%s for=%s",
+                      current_join_id, count, pcmk__plural_s(count),
+                      fsa_state2string(cur_state), source);
+            register_fsa_input_before(C_FSA_INTERNAL, I_INTEGRATED, NULL);
+            return TRUE;
+        }
+
+    } else if (cur_state == S_FINALIZE_JOIN) {
+        if (!pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
+            crm_debug("join-%d: Delaying finalization until we have CIB "
+                      CRM_XS " state=%s for=%s",
+                      current_join_id, fsa_state2string(cur_state), source);
+            return TRUE;
+
+        } else if (crmd_join_phase_count(crm_join_welcomed) != 0) {
+            int count = crmd_join_phase_count(crm_join_welcomed);
+
+            crm_debug("join-%d: Still waiting on %d welcomed node%s "
+                      CRM_XS " state=%s for=%s",
+                      current_join_id, count, pcmk__plural_s(count),
+                      fsa_state2string(cur_state), source);
+            crmd_join_phase_log(LOG_DEBUG);
+
+        } else if (crmd_join_phase_count(crm_join_integrated) != 0) {
+            int count = crmd_join_phase_count(crm_join_integrated);
+
+            crm_debug("join-%d: Still waiting on %d integrated node%s "
+                      CRM_XS " state=%s for=%s",
+                      current_join_id, count, pcmk__plural_s(count),
+                      fsa_state2string(cur_state), source);
+            crmd_join_phase_log(LOG_DEBUG);
+
+        } else if (crmd_join_phase_count(crm_join_finalized) != 0) {
+            int count = crmd_join_phase_count(crm_join_finalized);
+
+            crm_debug("join-%d: Still waiting on %d finalized node%s "
+                      CRM_XS " state=%s for=%s",
+                      current_join_id, count, pcmk__plural_s(count),
+                      fsa_state2string(cur_state), source);
+            crmd_join_phase_log(LOG_DEBUG);
+
+        } else {
+            crm_debug("join-%d: Complete " CRM_XS " state=%s for=%s",
+                      current_join_id, fsa_state2string(cur_state), source);
+            register_fsa_input_later(C_FSA_INTERNAL, I_FINALIZED, NULL);
+            return TRUE;
+        }
+    }
+
+    return FALSE;
+}
+
+void
+do_dc_join_final(long long action,
+                 enum crmd_fsa_cause cause,
+                 enum crmd_fsa_state cur_state,
+                 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
+{
+    crm_debug("Ensuring DC, quorum and node attributes are up-to-date");
+    crm_update_quorum(crm_have_quorum, TRUE);
+}
+
+int crmd_join_phase_count(enum crm_join_phase phase)
+{
+    int count = 0;
+    crm_node_t *peer;
+    GHashTableIter iter;
+
+    g_hash_table_iter_init(&iter, crm_peer_cache);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
+        if(peer->join == phase) {
+            count++;
+        }
+    }
+    return count;
+}
+
+void crmd_join_phase_log(int level)
+{
+    crm_node_t *peer;
+    GHashTableIter iter;
+
+    g_hash_table_iter_init(&iter, crm_peer_cache);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
+        do_crm_log(level, "join-%d: %s=%s", current_join_id, peer->uname,
+                   crm_join_phase_str(peer->join));
+    }
+}