path: root/daemons/fenced/fenced_history.c

/*
 * Copyright 2009-2022 the Pacemaker project contributors
 *
 * The version control history for this file may have further details.
 *
 * This source code is licensed under the GNU General Public License version 2
 * or later (GPLv2+) WITHOUT ANY WARRANTY.
 */

#include <crm_internal.h>

#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>

#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/ipc.h>
#include <crm/common/ipc_internal.h>
#include <crm/cluster/internal.h>

#include <crm/stonith-ng.h>
#include <crm/fencing/internal.h>
#include <crm/common/xml.h>
#include <crm/common/xml_internal.h>

#include <pacemaker-fenced.h>

#define MAX_STONITH_HISTORY 500

/*!
 * \internal
 * \brief Send a broadcast to all nodes to trigger cleanup or
 *        history synchronisation
 *
 * \param[in] history   Optional history to be attached
 * \param[in] callopts  We control cleanup via a flag in the callopts
 * \param[in] target    Cleanup can be limited to certain fence-targets
 */
static void
stonith_send_broadcast_history(xmlNode *history,
                               int callopts,
                               const char *target)
{
    xmlNode *bcast = create_xml_node(NULL, "stonith_command");
    xmlNode *data = create_xml_node(NULL, __func__);

    if (target) {
        crm_xml_add(data, F_STONITH_TARGET, target);
    }
    crm_xml_add(bcast, F_TYPE, T_STONITH_NG);
    crm_xml_add(bcast, F_SUBTYPE, "broadcast");
    crm_xml_add(bcast, F_STONITH_OPERATION, STONITH_OP_FENCE_HISTORY);
    crm_xml_add_int(bcast, F_STONITH_CALLOPTS, callopts);
    if (history) {
        add_node_copy(data, history);
    }
    add_message_xml(bcast, F_STONITH_CALLDATA, data);
    send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE);

    free_xml(data);
    free_xml(bcast);
}

static gboolean
stonith_remove_history_entry (gpointer key,
                              gpointer value,
                              gpointer user_data)
{
    remote_fencing_op_t *op = value;
    const char *target = (const char *) user_data;

    if ((op->state == st_failed) || (op->state == st_done)) {
        if ((target) && (strcmp(op->target, target) != 0)) {
            return FALSE;
        }
        return TRUE;
    }

    return FALSE; /* don't clean pending operations */
}

/*!
 * \internal
 * \brief Send out a cleanup broadcast or do a local history-cleanup
 *
 * \param[in] target    Cleanup can be limited to certain fence-targets
 * \param[in] broadcast Send out a cleanup broadcast
 */
static void
stonith_fence_history_cleanup(const char *target,
                              gboolean broadcast)
{
    if (broadcast) {
        stonith_send_broadcast_history(NULL,
                                       st_opt_cleanup | st_opt_discard_reply,
                                       target);
        /* we'll do the local clean when we receive back our own broadcast */
    } else if (stonith_remote_op_list) {
        g_hash_table_foreach_remove(stonith_remote_op_list,
                             stonith_remove_history_entry,
                             (gpointer) target);
        fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
    }
}

/* keeping the length of fence-history within bounds
 * =================================================
 *
 * If things are really running wild a lot of fencing-attempts
 * might fill up the hash-map, eventually using up a lot
 * of memory and creating huge history-sync messages.
 * Before the history being synced across nodes at least
 * the reboot of a cluster-node helped keeping the
 * history within bounds even though not in a reliable
 * manner.
 *
 * stonith_remote_op_list isn't sorted for time-stamps
 * thus it would be kind of expensive to delete e.g.
 * the oldest entry if it would grow past MAX_STONITH_HISTORY
 * entries.
 * It is more efficient to purge MAX_STONITH_HISTORY/2
 * entries whenever the list grows beyond MAX_STONITH_HISTORY.
 * (sort for age + purge the MAX_STONITH_HISTORY/2 oldest)
 * That done on a per-node-base might raise the
 * probability of large syncs to occur.
 * Things like introducing a broadcast to purge
 * MAX_STONITH_HISTORY/2 entries or not sync above a certain
 * threshold coming to mind ...
 * Simplest thing though is to purge the full history
 * throughout the cluster once MAX_STONITH_HISTORY is reached.
 * On the other hand this leads to purging the history in
 * situations where it would be handy to have it probably.
 */


static int
op_time_sort(const void *a_voidp, const void *b_voidp)
{
    const remote_fencing_op_t **a = (const remote_fencing_op_t **) a_voidp;
    const remote_fencing_op_t **b = (const remote_fencing_op_t **) b_voidp;
    gboolean a_pending = ((*a)->state != st_failed) && ((*a)->state != st_done);
    gboolean b_pending = ((*b)->state != st_failed) && ((*b)->state != st_done);

    if (a_pending && b_pending) {
        return 0;
    } else if (a_pending) {
        return -1;
    } else if (b_pending) {
        return 1;
    } else if ((*b)->completed == (*a)->completed) {
        if ((*b)->completed_nsec > (*a)->completed_nsec) {
            return 1;
        } else if ((*b)->completed_nsec == (*a)->completed_nsec) {
            return 0;
        }
    } else if ((*b)->completed > (*a)->completed) {
        return 1;
    }

    return -1;
}


/*!
 * \internal
 * \brief Do a local history-trim to MAX_STONITH_HISTORY / 2 entries
 *        once over MAX_STONITH_HISTORY
 */
void
stonith_fence_history_trim(void)
{
    guint num_ops;

    if (!stonith_remote_op_list) {
        return;
    }
    num_ops = g_hash_table_size(stonith_remote_op_list);
    if (num_ops > MAX_STONITH_HISTORY) {
        remote_fencing_op_t *ops[num_ops];
        remote_fencing_op_t *op = NULL;
        GHashTableIter iter;
        int i;

        crm_trace("Fencing History growing beyond limit of %d so purge "
                  "half of failed/successful attempts", MAX_STONITH_HISTORY);

        /* write all ops into an array */
        i = 0;
        g_hash_table_iter_init(&iter, stonith_remote_op_list);
        while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
            ops[i++] = op;
        }
        /* run quicksort over the array so that we get pending ops
         * first and then sorted most recent to oldest
         */
        qsort(ops, num_ops, sizeof(remote_fencing_op_t *), op_time_sort);
        /* purgest oldest half of the history entries */
        for (i = MAX_STONITH_HISTORY / 2; i < num_ops; i++) {
            /* keep pending ops even if they shouldn't fill more than
             * half of our buffer
             */
            if ((ops[i]->state == st_failed) || (ops[i]->state == st_done)) {
                g_hash_table_remove(stonith_remote_op_list, ops[i]->id);
            }
        }
        /* we've just purged valid data from the list so there is no need
         * to create a notification - if displayed it can stay
         */
    }
}

/*!
 * \internal
 * \brief Convert xml fence-history to a hash-table like stonith_remote_op_list
 *
 * \param[in] history   Fence-history in xml
 *
 * \return Fence-history as hash-table
 */
static GHashTable *
stonith_xml_history_to_list(const xmlNode *history)
{
    xmlNode *xml_op = NULL;
    GHashTable *rv = NULL;

    init_stonith_remote_op_hash_table(&rv);

    CRM_LOG_ASSERT(rv != NULL);

    for (xml_op = pcmk__xml_first_child(history); xml_op != NULL;
         xml_op = pcmk__xml_next(xml_op)) {
        remote_fencing_op_t *op = NULL;
        char *id = crm_element_value_copy(xml_op, F_STONITH_REMOTE_OP_ID);
        int state;
        int exit_status = CRM_EX_OK;
        int execution_status = PCMK_EXEC_DONE;
        long long completed;
        long long completed_nsec = 0L;

        if (!id) {
            crm_warn("Malformed fencing history received from peer");
            continue;
        }

        crm_trace("Attaching op %s to hashtable", id);

        op = calloc(1, sizeof(remote_fencing_op_t));

        op->id = id;
        op->target = crm_element_value_copy(xml_op, F_STONITH_TARGET);
        op->action = crm_element_value_copy(xml_op, F_STONITH_ACTION);
        op->originator = crm_element_value_copy(xml_op, F_STONITH_ORIGIN);
        op->delegate = crm_element_value_copy(xml_op, F_STONITH_DELEGATE);
        op->client_name = crm_element_value_copy(xml_op, F_STONITH_CLIENTNAME);
        crm_element_value_ll(xml_op, F_STONITH_DATE, &completed);
        op->completed = (time_t) completed;
        crm_element_value_ll(xml_op, F_STONITH_DATE_NSEC, &completed_nsec);
        op->completed_nsec = completed_nsec;
        crm_element_value_int(xml_op, F_STONITH_STATE, &state);
        op->state = (enum op_state) state;

        /* @COMPAT We can't use stonith__xe_get_result() here because
         * fencers <2.1.3 didn't include results, leading it to assume an error
         * status. Instead, set an unknown status in that case.
         */
        if ((crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &exit_status) < 0)
            || (crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS,
                                      &execution_status) < 0)) {
            exit_status = CRM_EX_INDETERMINATE;
            execution_status = PCMK_EXEC_UNKNOWN;
        }
        pcmk__set_result(&op->result, exit_status, execution_status,
                         crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON));
        pcmk__set_result_output(&op->result,
                                crm_element_value_copy(xml_op, F_STONITH_OUTPUT),
                                NULL);


        g_hash_table_replace(rv, id, op);
        CRM_LOG_ASSERT(g_hash_table_lookup(rv, id) != NULL);
    }

    return rv;
}

/*!
 * \internal
 * \brief Craft xml difference between local fence-history and a history
 *        coming from remote, and merge the remote history into the local
 *
 * \param[in,out] remote_history  Fence-history as hash-table (may be NULL)
 * \param[in]     add_id          If crafting the answer for an API
 *                                history-request there is no need for the id
 * \param[in]     target          Optionally limit to certain fence-target
 *
 * \return The fence-history as xml
 */
static xmlNode *
stonith_local_history_diff_and_merge(GHashTable *remote_history,
                                     gboolean add_id, const char *target)
{
    xmlNode *history = NULL;
    GHashTableIter iter;
    remote_fencing_op_t *op = NULL;
    gboolean updated = FALSE;
    int cnt = 0;

    if (stonith_remote_op_list) {
            char *id = NULL;

            history = create_xml_node(NULL, F_STONITH_HISTORY_LIST);

            g_hash_table_iter_init(&iter, stonith_remote_op_list);
            while (g_hash_table_iter_next(&iter, (void **)&id, (void **)&op)) {
                xmlNode *entry = NULL;

                if (remote_history) {
                    remote_fencing_op_t *remote_op =
                        g_hash_table_lookup(remote_history, op->id);

                    if (remote_op) {
                        if (stonith__op_state_pending(op->state)
                            && !stonith__op_state_pending(remote_op->state)) {

                            crm_debug("Updating outdated pending operation %.8s "
                                      "(state=%s) according to the one (state=%s) from "
                                      "remote peer history",
                                      op->id, stonith_op_state_str(op->state),
                                      stonith_op_state_str(remote_op->state));

                            g_hash_table_steal(remote_history, op->id);
                            op->id = remote_op->id;
                            remote_op->id = id;
                            g_hash_table_iter_replace(&iter, remote_op);

                            updated = TRUE;
                            continue; /* skip outdated entries */

                        } else if (!stonith__op_state_pending(op->state)
                                   && stonith__op_state_pending(remote_op->state)) {

                            crm_debug("Broadcasting operation %.8s (state=%s) to "
                                      "update the outdated pending one "
                                      "(state=%s) in remote peer history",
                                      op->id, stonith_op_state_str(op->state),
                                      stonith_op_state_str(remote_op->state));

                            g_hash_table_remove(remote_history, op->id);

                        } else {
                            g_hash_table_remove(remote_history, op->id);
                            continue; /* skip entries broadcasted already */
                        }
                    }
                }

                if (!pcmk__str_eq(target, op->target, pcmk__str_null_matches)) {
                    continue;
                }

                cnt++;
                crm_trace("Attaching op %s", op->id);
                entry = create_xml_node(history, STONITH_OP_EXEC);
                if (add_id) {
                    crm_xml_add(entry, F_STONITH_REMOTE_OP_ID, op->id);
                }
                crm_xml_add(entry, F_STONITH_TARGET, op->target);
                crm_xml_add(entry, F_STONITH_ACTION, op->action);
                crm_xml_add(entry, F_STONITH_ORIGIN, op->originator);
                crm_xml_add(entry, F_STONITH_DELEGATE, op->delegate);
                crm_xml_add(entry, F_STONITH_CLIENTNAME, op->client_name);
                crm_xml_add_ll(entry, F_STONITH_DATE, op->completed);
                crm_xml_add_ll(entry, F_STONITH_DATE_NSEC, op->completed_nsec);
                crm_xml_add_int(entry, F_STONITH_STATE, op->state);
                stonith__xe_set_result(entry, &op->result);
            }
    }

    if (remote_history) {
        init_stonith_remote_op_hash_table(&stonith_remote_op_list);

        updated |= g_hash_table_size(remote_history);

        g_hash_table_iter_init(&iter, remote_history);
        while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
            if (stonith__op_state_pending(op->state) &&
                pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {

                crm_warn("Failing pending operation %.8s originated by us but "
                         "known only from peer history", op->id);
                op->state = st_failed;
                set_fencing_completed(op);

                /* CRM_EX_EXPIRED + PCMK_EXEC_INVALID prevents finalize_op()
                 * from setting a delegate
                 */
                pcmk__set_result(&op->result, CRM_EX_EXPIRED, PCMK_EXEC_INVALID,
                                 "Initiated by earlier fencer "
                                 "process and presumed failed");
                fenced_broadcast_op_result(op, false);
            }

            g_hash_table_iter_steal(&iter);
            g_hash_table_replace(stonith_remote_op_list, op->id, op);
            /* we could trim the history here but if we bail
             * out after trim we might miss more recent entries
             * of those that might still be in the list
             * if we don't bail out trimming once is more
             * efficient and memory overhead is minimal as
             * we are just moving pointers from one hash to
             * another
             */
        }

        g_hash_table_destroy(remote_history); /* remove what is left */
    }

    if (updated) {
        stonith_fence_history_trim();
        fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
    }

    if (cnt == 0) {
        free_xml(history);
        return NULL;
    } else {
        return history;
    }
}

/*!
 * \internal
 * \brief Craft xml from the local fence-history
 *
 * \param[in] add_id            If crafting the answer for an API
 *                              history-request there is no need for the id
 * \param[in] target            Optionally limit to certain fence-target
 *
 * \return The fence-history as xml
 */
static xmlNode *
stonith_local_history(gboolean add_id, const char *target)
{
    return stonith_local_history_diff_and_merge(NULL, add_id, target);
}

/*!
 * \internal
 * \brief Handle fence-history messages (from API or coming in as broadcasts)
 *
 * \param[in,out] msg          Request XML
 * \param[out]    output       Where to set local history, if requested
 * \param[in]     remote_peer  If broadcast, peer that sent it
 * \param[in]     options      Call options from the request
 */
void
stonith_fence_history(xmlNode *msg, xmlNode **output,
                      const char *remote_peer, int options)
{
    const char *target = NULL;
    xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_NEVER);
    xmlNode *out_history = NULL;

    if (dev) {
        target = crm_element_value(dev, F_STONITH_TARGET);
        if (target && (options & st_opt_cs_nodeid)) {
            int nodeid;
            crm_node_t *node;

            pcmk__scan_min_int(target, &nodeid, 0);
            node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY);
            if (node) {
                target = node->uname;
            }
        }
    }

    if (options & st_opt_cleanup) {
        crm_trace("Cleaning up operations on %s in %p", target,
                  stonith_remote_op_list);

        stonith_fence_history_cleanup(target,
            crm_element_value(msg, F_STONITH_CALLID) != NULL);
    } else if (options & st_opt_broadcast) {
        /* there is no clear sign atm for when a history sync
           is done so send a notification for anything
           that smells like history-sync
         */
        fenced_send_notification(T_STONITH_NOTIFY_HISTORY_SYNCED, NULL, NULL);
        if (crm_element_value(msg, F_STONITH_CALLID)) {
            /* this is coming from the stonith-API
            *
            * craft a broadcast with node's history
            * so that every node can merge and broadcast
            * what it has on top
            */
            out_history = stonith_local_history(TRUE, NULL);
            crm_trace("Broadcasting history to peers");
            stonith_send_broadcast_history(out_history,
                                        st_opt_broadcast | st_opt_discard_reply,
                                        NULL);
        } else if (remote_peer &&
                   !pcmk__str_eq(remote_peer, stonith_our_uname, pcmk__str_casei)) {
            xmlNode *history = get_xpath_object("//" F_STONITH_HISTORY_LIST,
                                                msg, LOG_NEVER);

            /* either a broadcast created directly upon stonith-API request
            * or a diff as response to such a thing
            *
            * in both cases it may have a history or not
            * if we have differential data
            * merge in what we've received and stop
            * otherwise broadcast what we have on top
            * marking as differential and merge in afterwards
            */
            if (!history || !pcmk__xe_attr_is_true(history, F_STONITH_DIFFERENTIAL)) {
                GHashTable *received_history = NULL;

                if (history != NULL) {
                    received_history = stonith_xml_history_to_list(history);
                }
                out_history =
                    stonith_local_history_diff_and_merge(received_history, TRUE, NULL);
                if (out_history) {
                    crm_trace("Broadcasting history-diff to peers");
                    pcmk__xe_set_bool_attr(out_history, F_STONITH_DIFFERENTIAL, true);
                    stonith_send_broadcast_history(out_history,
                        st_opt_broadcast | st_opt_discard_reply,
                        NULL);
                } else {
                    crm_trace("History-diff is empty - skip broadcast");
                }
            }
        } else {
            crm_trace("Skipping history-query-broadcast (%s%s)"
                      " we sent ourselves",
                      remote_peer?"remote-peer=":"local-ipc",
                      remote_peer?remote_peer:"");
        }
    } else {
        /* plain history request */
        crm_trace("Looking for operations on %s in %p", target,
                  stonith_remote_op_list);
        *output = stonith_local_history(FALSE, target);
    }
    free_xml(out_history);
}