Adding upstream version 2.1.6.upstream/2.1.6

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1417 insertions, 0 deletions

diff --git a/lib/services/services.c b/lib/services/services.c
new file mode 100644
index 0000000..b60d8bd
--- /dev/null
+++ b/lib/services/services.c
@@ -0,0 +1,1417 @@
+/*
+ * Copyright 2010-2022 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#ifndef _GNU_SOURCE
+#  define _GNU_SOURCE
+#endif
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <stdio.h>
+#include <errno.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <fcntl.h>
+
+#include <crm/crm.h>
+#include <crm/common/mainloop.h>
+#include <crm/services.h>
+#include <crm/services_internal.h>
+#include <crm/stonith-ng.h>
+#include <crm/msg_xml.h>
+#include "services_private.h"
+#include "services_ocf.h"
+#include "services_lsb.h"
+
+#if SUPPORT_UPSTART
+#  include <upstart.h>
+#endif
+
+#if SUPPORT_SYSTEMD
+#  include <systemd.h>
+#endif
+
+#if SUPPORT_NAGIOS
+#  include <services_nagios.h>
+#endif
+
+/* TODO: Develop a rollover strategy */
+
+static int operations = 0;
+static GHashTable *recurring_actions = NULL;
+
+/* ops waiting to run async because of conflicting active
+ * pending ops */
+static GList *blocked_ops = NULL;
+
+/* ops currently active (in-flight) */
+static GList *inflight_ops = NULL;
+
+static void handle_blocked_ops(void);
+
+/*!
+ * \brief Find first service class that can provide a specified agent
+ *
+ * \param[in] agent  Name of agent to search for
+ *
+ * \return Service class if found, NULL otherwise
+ *
+ * \note The priority is LSB, then systemd, then upstart. It would be preferable
+ *       to put systemd first, but LSB merely requires a file existence check,
+ *       while systemd requires contacting D-Bus.
+ */
+const char *
+resources_find_service_class(const char *agent)
+{
+    if (services__lsb_agent_exists(agent)) {
+        return PCMK_RESOURCE_CLASS_LSB;
+    }
+
+#if SUPPORT_SYSTEMD
+    if (systemd_unit_exists(agent)) {
+        return PCMK_RESOURCE_CLASS_SYSTEMD;
+    }
+#endif
+
+#if SUPPORT_UPSTART
+    if (upstart_job_exists(agent)) {
+        return PCMK_RESOURCE_CLASS_UPSTART;
+    }
+#endif
+    return NULL;
+}
+
+static inline void
+init_recurring_actions(void)
+{
+    if (recurring_actions == NULL) {
+        recurring_actions = pcmk__strkey_table(NULL, NULL);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Check whether op is in-flight systemd or upstart op
+ *
+ * \param[in] op  Operation to check
+ *
+ * \return TRUE if op is in-flight systemd or upstart op
+ */
+static inline gboolean
+inflight_systemd_or_upstart(const svc_action_t *op)
+{
+    return pcmk__strcase_any_of(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD,
+                           PCMK_RESOURCE_CLASS_UPSTART, NULL) &&
+           g_list_find(inflight_ops, op) != NULL;
+}
+
+/*!
+ * \internal
+ * \brief Expand "service" alias to an actual resource class
+ *
+ * \param[in] rsc       Resource name (for logging only)
+ * \param[in] standard  Resource class as configured
+ * \param[in] agent     Agent name to look for
+ *
+ * \return Newly allocated string with actual resource class
+ *
+ * \note The caller is responsible for calling free() on the result.
+ */
+static char *
+expand_resource_class(const char *rsc, const char *standard, const char *agent)
+{
+    char *expanded_class = NULL;
+
+    if (strcasecmp(standard, PCMK_RESOURCE_CLASS_SERVICE) == 0) {
+        const char *found_class = resources_find_service_class(agent);
+
+        if (found_class) {
+            crm_debug("Found %s agent %s for %s", found_class, agent, rsc);
+            expanded_class = strdup(found_class);
+        } else {
+            crm_info("Assuming resource class lsb for agent %s for %s",
+                     agent, rsc);
+            expanded_class = strdup(PCMK_RESOURCE_CLASS_LSB);
+        }
+    } else {
+        expanded_class = strdup(standard);
+    }
+    CRM_ASSERT(expanded_class);
+    return expanded_class;
+}
+
+/*!
+ * \internal
+ * \brief Create a simple svc_action_t instance
+ *
+ * \return Newly allocated instance (or NULL if not enough memory)
+ */
+static svc_action_t *
+new_action(void)
+{
+    svc_action_t *op = calloc(1, sizeof(svc_action_t));
+
+    if (op == NULL) {
+        return NULL;
+    }
+
+    op->opaque = calloc(1, sizeof(svc_action_private_t));
+    if (op->opaque == NULL) {
+        free(op);
+        return NULL;
+    }
+
+    // Initialize result
+    services__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_UNKNOWN, NULL);
+    return op;
+}
+
+static bool
+required_argument_missing(uint32_t ra_caps, const char *name,
+                          const char *standard, const char *provider,
+                          const char *agent, const char *action)
+{
+    if (pcmk__str_empty(name)) {
+        crm_info("Cannot create operation without resource name (bug?)");
+        return true;
+    }
+
+    if (pcmk__str_empty(standard)) {
+        crm_info("Cannot create operation for %s without resource class (bug?)",
+                 name);
+        return true;
+    }
+
+    if (pcmk_is_set(ra_caps, pcmk_ra_cap_provider)
+        && pcmk__str_empty(provider)) {
+        crm_info("Cannot create operation for %s resource %s "
+                 "without provider (bug?)", standard, name);
+        return true;
+    }
+
+    if (pcmk__str_empty(agent)) {
+        crm_info("Cannot create operation for %s without agent name (bug?)",
+                 name);
+        return true;
+    }
+
+    if (pcmk__str_empty(action)) {
+        crm_info("Cannot create operation for %s without action name (bug?)",
+                 name);
+        return true;
+    }
+    return false;
+}
+
+// \return Standard Pacemaker return code (pcmk_rc_ok or ENOMEM)
+static int
+copy_action_arguments(svc_action_t *op, uint32_t ra_caps, const char *name,
+                      const char *standard, const char *provider,
+                      const char *agent, const char *action)
+{
+    op->rsc = strdup(name);
+    if (op->rsc == NULL) {
+        return ENOMEM;
+    }
+
+    op->agent = strdup(agent);
+    if (op->agent == NULL) {
+        return ENOMEM;
+    }
+
+    op->standard = expand_resource_class(name, standard, agent);
+    if (op->standard == NULL) {
+        return ENOMEM;
+    }
+
+    if (pcmk_is_set(ra_caps, pcmk_ra_cap_status)
+        && pcmk__str_eq(action, "monitor", pcmk__str_casei)) {
+        action = "status";
+    }
+    op->action = strdup(action);
+    if (op->action == NULL) {
+        return ENOMEM;
+    }
+
+    if (pcmk_is_set(ra_caps, pcmk_ra_cap_provider)) {
+        op->provider = strdup(provider);
+        if (op->provider == NULL) {
+            return ENOMEM;
+        }
+    }
+    return pcmk_rc_ok;
+}
+
+svc_action_t *
+services__create_resource_action(const char *name, const char *standard,
+                        const char *provider, const char *agent,
+                        const char *action, guint interval_ms, int timeout,
+                        GHashTable *params, enum svc_action_flags flags)
+{
+    svc_action_t *op = NULL;
+    uint32_t ra_caps = pcmk_get_ra_caps(standard);
+    int rc = pcmk_rc_ok;
+
+    op = new_action();
+    if (op == NULL) {
+        crm_crit("Cannot prepare action: %s", strerror(ENOMEM));
+        if (params != NULL) {
+            g_hash_table_destroy(params);
+        }
+        return NULL;
+    }
+
+    op->interval_ms = interval_ms;
+    op->timeout = timeout;
+    op->flags = flags;
+    op->sequence = ++operations;
+
+    // Take ownership of params
+    if (pcmk_is_set(ra_caps, pcmk_ra_cap_params)) {
+        op->params = params;
+    } else if (params != NULL) {
+        g_hash_table_destroy(params);
+        params = NULL;
+    }
+
+    if (required_argument_missing(ra_caps, name, standard, provider, agent,
+                                  action)) {
+        services__set_result(op, services__generic_error(op),
+                             PCMK_EXEC_ERROR_FATAL,
+                             "Required agent or action information missing");
+        return op;
+    }
+
+    op->id = pcmk__op_key(name, action, interval_ms);
+
+    if (copy_action_arguments(op, ra_caps, name, standard, provider, agent,
+                              action) != pcmk_rc_ok) {
+        crm_crit("Cannot prepare %s action for %s: %s",
+                 action, name, strerror(ENOMEM));
+        services__handle_exec_error(op, ENOMEM);
+        return op;
+    }
+
+    if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_OCF) == 0) {
+        rc = services__ocf_prepare(op);
+
+    } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_LSB) == 0) {
+        rc = services__lsb_prepare(op);
+
+#if SUPPORT_SYSTEMD
+    } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD) == 0) {
+        rc = services__systemd_prepare(op);
+#endif
+#if SUPPORT_UPSTART
+    } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_UPSTART) == 0) {
+        rc = services__upstart_prepare(op);
+#endif
+#if SUPPORT_NAGIOS
+    } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_NAGIOS) == 0) {
+        rc = services__nagios_prepare(op);
+#endif
+    } else {
+        crm_info("Unknown resource standard: %s", op->standard);
+        rc = ENOENT;
+    }
+
+    if (rc != pcmk_rc_ok) {
+        crm_info("Cannot prepare %s operation for %s: %s",
+                 action, name, strerror(rc));
+        services__handle_exec_error(op, rc);
+    }
+    return op;
+}
+
+svc_action_t *
+resources_action_create(const char *name, const char *standard,
+                        const char *provider, const char *agent,
+                        const char *action, guint interval_ms, int timeout,
+                        GHashTable *params, enum svc_action_flags flags)
+{
+    svc_action_t *op = services__create_resource_action(name, standard,
+                            provider, agent, action, interval_ms, timeout,
+                            params, flags);
+    if (op == NULL || op->rc != 0) {
+        services_action_free(op);
+        return NULL;
+    } else {
+        // Preserve public API backward compatibility
+        op->rc = PCMK_OCF_OK;
+        op->status = PCMK_EXEC_DONE;
+
+        return op;
+    }
+}
+
+svc_action_t *
+services_action_create_generic(const char *exec, const char *args[])
+{
+    svc_action_t *op = new_action();
+
+    CRM_ASSERT(op != NULL);
+
+    op->opaque->exec = strdup(exec);
+    op->opaque->args[0] = strdup(exec);
+    if ((op->opaque->exec == NULL) || (op->opaque->args[0] == NULL)) {
+        crm_crit("Cannot prepare action for '%s': %s", exec, strerror(ENOMEM));
+        services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
+                             strerror(ENOMEM));
+        return op;
+    }
+
+    if (args == NULL) {
+        return op;
+    }
+
+    for (int cur_arg = 1; args[cur_arg - 1] != NULL; cur_arg++) {
+
+        if (cur_arg == PCMK__NELEM(op->opaque->args)) {
+            crm_info("Cannot prepare action for '%s': Too many arguments",
+                     exec);
+            services__set_result(op, PCMK_OCF_UNKNOWN_ERROR,
+                                 PCMK_EXEC_ERROR_HARD, "Too many arguments");
+            break;
+        }
+
+        op->opaque->args[cur_arg] = strdup(args[cur_arg - 1]);
+        if (op->opaque->args[cur_arg] == NULL) {
+            crm_crit("Cannot prepare action for '%s': %s",
+                     exec, strerror(ENOMEM));
+            services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
+                                 strerror(ENOMEM));
+            break;
+        }
+    }
+
+    return op;
+}
+
+/*!
+ * \brief Create an alert agent action
+ *
+ * \param[in] id        Alert ID
+ * \param[in] exec      Path to alert agent executable
+ * \param[in] timeout   Action timeout
+ * \param[in] params    Parameters to use with action
+ * \param[in] sequence  Action sequence number
+ * \param[in] cb_data   Data to pass to callback function
+ *
+ * \return New action on success, NULL on error
+ * \note It is the caller's responsibility to free cb_data.
+ *       The caller should not free params explicitly.
+ */
+svc_action_t *
+services_alert_create(const char *id, const char *exec, int timeout,
+                      GHashTable *params, int sequence, void *cb_data)
+{
+    svc_action_t *action = services_action_create_generic(exec, NULL);
+
+    action->id = strdup(id);
+    action->standard = strdup(PCMK_RESOURCE_CLASS_ALERT);
+    CRM_ASSERT((action->id != NULL) && (action->standard != NULL));
+
+    action->timeout = timeout;
+    action->params = params;
+    action->sequence = sequence;
+    action->cb_data = cb_data;
+    return action;
+}
+
+/*!
+ * \brief Set the user and group that an action will execute as
+ *
+ * \param[in,out] op      Action to modify
+ * \param[in]     user    Name of user to execute action as
+ * \param[in]     group   Name of group to execute action as
+ *
+ * \return pcmk_ok on success, -errno otherwise
+ *
+ * \note This will have no effect unless the process executing the action runs
+ *       as root, and the action is not a systemd or upstart action.
+ *       We could implement this for systemd by adding User= and Group= to
+ *       [Service] in the override file, but that seems more likely to cause
+ *       problems than be useful.
+ */
+int
+services_action_user(svc_action_t *op, const char *user)
+{
+    CRM_CHECK((op != NULL) && (user != NULL), return -EINVAL);
+    return crm_user_lookup(user, &(op->opaque->uid), &(op->opaque->gid));
+}
+
+/*!
+ * \brief Execute an alert agent action
+ *
+ * \param[in,out] action  Action to execute
+ * \param[in]     cb      Function to call when action completes
+ *
+ * \return TRUE if the library will free action, FALSE otherwise
+ *
+ * \note If this function returns FALSE, it is the caller's responsibility to
+ *       free the action with services_action_free(). However, unless someone
+ *       intentionally creates a recurring alert action, this will never return
+ *       FALSE.
+ */
+gboolean
+services_alert_async(svc_action_t *action, void (*cb)(svc_action_t *op))
+{
+    action->synchronous = false;
+    action->opaque->callback = cb;
+    return services__execute_file(action) == pcmk_rc_ok;
+}
+
+#if HAVE_DBUS
+/*!
+ * \internal
+ * \brief Update operation's pending DBus call, unreferencing old one if needed
+ *
+ * \param[in,out] op       Operation to modify
+ * \param[in]     pending  Pending call to set
+ */
+void
+services_set_op_pending(svc_action_t *op, DBusPendingCall *pending)
+{
+    if (op->opaque->pending && (op->opaque->pending != pending)) {
+        if (pending) {
+            crm_info("Lost pending %s DBus call (%p)", op->id, op->opaque->pending);
+        } else {
+            crm_trace("Done with pending %s DBus call (%p)", op->id, op->opaque->pending);
+        }
+        dbus_pending_call_unref(op->opaque->pending);
+    }
+    op->opaque->pending = pending;
+    if (pending) {
+        crm_trace("Updated pending %s DBus call (%p)", op->id, pending);
+    } else {
+        crm_trace("Cleared pending %s DBus call", op->id);
+    }
+}
+#endif
+
+void
+services_action_cleanup(svc_action_t * op)
+{
+    if ((op == NULL) || (op->opaque == NULL)) {
+        return;
+    }
+
+#if HAVE_DBUS
+    if(op->opaque->timerid != 0) {
+        crm_trace("Removing timer for call %s to %s", op->action, op->rsc);
+        g_source_remove(op->opaque->timerid);
+        op->opaque->timerid = 0;
+    }
+
+    if(op->opaque->pending) {
+        if (dbus_pending_call_get_completed(op->opaque->pending)) {
+            // This should never be the case
+            crm_warn("Result of %s op %s was unhandled",
+                     op->standard, op->id);
+        } else {
+            crm_debug("Will ignore any result of canceled %s op %s",
+                      op->standard, op->id);
+        }
+        dbus_pending_call_cancel(op->opaque->pending);
+        services_set_op_pending(op, NULL);
+    }
+#endif
+
+    if (op->opaque->stderr_gsource) {
+        mainloop_del_fd(op->opaque->stderr_gsource);
+        op->opaque->stderr_gsource = NULL;
+    }
+
+    if (op->opaque->stdout_gsource) {
+        mainloop_del_fd(op->opaque->stdout_gsource);
+        op->opaque->stdout_gsource = NULL;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Map an actual resource action result to a standard OCF result
+ *
+ * \param[in] standard     Agent standard (must not be "service")
+ * \param[in] action       Action that result is for
+ * \param[in] exit_status  Actual agent exit status
+ *
+ * \return Standard OCF result
+ */
+enum ocf_exitcode
+services_result2ocf(const char *standard, const char *action, int exit_status)
+{
+    if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
+        return services__ocf2ocf(exit_status);
+
+#if SUPPORT_SYSTEMD
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SYSTEMD,
+                            pcmk__str_casei)) {
+        return services__systemd2ocf(exit_status);
+#endif
+
+#if SUPPORT_UPSTART
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_UPSTART,
+                            pcmk__str_casei)) {
+        return services__upstart2ocf(exit_status);
+#endif
+
+#if SUPPORT_NAGIOS
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_NAGIOS,
+                            pcmk__str_casei)) {
+        return services__nagios2ocf(exit_status);
+#endif
+
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_LSB,
+                            pcmk__str_casei)) {
+        return services__lsb2ocf(action, exit_status);
+
+    } else {
+        crm_warn("Treating result from unknown standard '%s' as OCF",
+                 ((standard == NULL)? "unspecified" : standard));
+        return services__ocf2ocf(exit_status);
+    }
+}
+
+void
+services_action_free(svc_action_t * op)
+{
+    unsigned int i;
+
+    if (op == NULL) {
+        return;
+    }
+
+    /* The operation should be removed from all tracking lists by this point.
+     * If it's not, we have a bug somewhere, so bail. That may lead to a
+     * memory leak, but it's better than a use-after-free segmentation fault.
+     */
+    CRM_CHECK(g_list_find(inflight_ops, op) == NULL, return);
+    CRM_CHECK(g_list_find(blocked_ops, op) == NULL, return);
+    CRM_CHECK((recurring_actions == NULL)
+              || (g_hash_table_lookup(recurring_actions, op->id) == NULL),
+              return);
+
+    services_action_cleanup(op);
+
+    if (op->opaque->repeat_timer) {
+        g_source_remove(op->opaque->repeat_timer);
+        op->opaque->repeat_timer = 0;
+    }
+
+    free(op->id);
+    free(op->opaque->exec);
+
+    for (i = 0; i < PCMK__NELEM(op->opaque->args); i++) {
+        free(op->opaque->args[i]);
+    }
+
+    free(op->opaque->exit_reason);
+    free(op->opaque);
+    free(op->rsc);
+    free(op->action);
+
+    free(op->standard);
+    free(op->agent);
+    free(op->provider);
+
+    free(op->stdout_data);
+    free(op->stderr_data);
+
+    if (op->params) {
+        g_hash_table_destroy(op->params);
+        op->params = NULL;
+    }
+
+    free(op);
+}
+
+gboolean
+cancel_recurring_action(svc_action_t * op)
+{
+    crm_info("Cancelling %s operation %s", op->standard, op->id);
+
+    if (recurring_actions) {
+        g_hash_table_remove(recurring_actions, op->id);
+    }
+
+    if (op->opaque->repeat_timer) {
+        g_source_remove(op->opaque->repeat_timer);
+        op->opaque->repeat_timer = 0;
+    }
+
+    return TRUE;
+}
+
+/*!
+ * \brief Cancel a recurring action
+ *
+ * \param[in] name         Name of resource that operation is for
+ * \param[in] action       Name of operation to cancel
+ * \param[in] interval_ms  Interval of operation to cancel
+ *
+ * \return TRUE if action was successfully cancelled, FALSE otherwise
+ */
+gboolean
+services_action_cancel(const char *name, const char *action, guint interval_ms)
+{
+    gboolean cancelled = FALSE;
+    char *id = pcmk__op_key(name, action, interval_ms);
+    svc_action_t *op = NULL;
+
+    /* We can only cancel a recurring action */
+    init_recurring_actions();
+    op = g_hash_table_lookup(recurring_actions, id);
+    if (op == NULL) {
+        goto done;
+    }
+
+    // Tell services__finalize_async_op() not to reschedule the operation
+    op->cancel = TRUE;
+
+    /* Stop tracking it as a recurring operation, and stop its repeat timer */
+    cancel_recurring_action(op);
+
+    /* If the op has a PID, it's an in-flight child process, so kill it.
+     *
+     * Whether the kill succeeds or fails, the main loop will send the op to
+     * async_action_complete() (and thus services__finalize_async_op()) when the
+     * process goes away.
+     */
+    if (op->pid != 0) {
+        crm_info("Terminating in-flight op %s[%d] early because it was cancelled",
+                 id, op->pid);
+        cancelled = mainloop_child_kill(op->pid);
+        if (cancelled == FALSE) {
+            crm_err("Termination of %s[%d] failed", id, op->pid);
+        }
+        goto done;
+    }
+
+#if HAVE_DBUS
+    // In-flight systemd and upstart ops don't have a pid
+    if (inflight_systemd_or_upstart(op)) {
+        inflight_ops = g_list_remove(inflight_ops, op);
+
+        /* This will cause any result that comes in later to be discarded, so we
+         * don't call the callback and free the operation twice.
+         */
+        services_action_cleanup(op);
+    }
+#endif
+
+    /* The rest of this is essentially equivalent to
+     * services__finalize_async_op(), minus the handle_blocked_ops() call.
+     */
+
+    // Report operation as cancelled
+    services__set_cancelled(op);
+    if (op->opaque->callback) {
+        op->opaque->callback(op);
+    }
+
+    blocked_ops = g_list_remove(blocked_ops, op);
+    services_action_free(op);
+    cancelled = TRUE;
+    // @TODO Initiate handle_blocked_ops() asynchronously
+
+done:
+    free(id);
+    return cancelled;
+}
+
+gboolean
+services_action_kick(const char *name, const char *action, guint interval_ms)
+{
+    svc_action_t * op = NULL;
+    char *id = pcmk__op_key(name, action, interval_ms);
+
+    init_recurring_actions();
+    op = g_hash_table_lookup(recurring_actions, id);
+    free(id);
+
+    if (op == NULL) {
+        return FALSE;
+    }
+
+
+    if (op->pid || inflight_systemd_or_upstart(op)) {
+        return TRUE;
+    } else {
+        if (op->opaque->repeat_timer) {
+            g_source_remove(op->opaque->repeat_timer);
+            op->opaque->repeat_timer = 0;
+        }
+        recurring_action_timer(op);
+        return TRUE;
+    }
+
+}
+
+/*!
+ * \internal
+ * \brief Add a new recurring operation, checking for duplicates
+ *
+ * \param[in,out] op  Operation to add
+ *
+ * \return TRUE if duplicate found (and reschedule), FALSE otherwise
+ */
+static gboolean
+handle_duplicate_recurring(svc_action_t *op)
+{
+    svc_action_t * dup = NULL;
+
+    /* check for duplicates */
+    dup = g_hash_table_lookup(recurring_actions, op->id);
+
+    if (dup && (dup != op)) {
+        /* update user data */
+        if (op->opaque->callback) {
+            dup->opaque->callback = op->opaque->callback;
+            dup->cb_data = op->cb_data;
+            op->cb_data = NULL;
+        }
+        /* immediately execute the next interval */
+        if (dup->pid != 0) {
+            if (op->opaque->repeat_timer) {
+                g_source_remove(op->opaque->repeat_timer);
+                op->opaque->repeat_timer = 0;
+            }
+            recurring_action_timer(dup);
+        }
+        /* free the duplicate */
+        services_action_free(op);
+        return TRUE;
+    }
+
+    return FALSE;
+}
+
+/*!
+ * \internal
+ * \brief Execute an action appropriately according to its standard
+ *
+ * \param[in,out] op  Action to execute
+ *
+ * \return Standard Pacemaker return code
+ * \retval EBUSY          Recurring operation could not be initiated
+ * \retval pcmk_rc_error  Synchronous action failed
+ * \retval pcmk_rc_ok     Synchronous action succeeded, or asynchronous action
+ *                        should not be freed (because it's pending or because
+ *                        it failed to execute and was already freed)
+ *
+ * \note If the return value for an asynchronous action is not pcmk_rc_ok, the
+ *       caller is responsible for freeing the action.
+ */
+static int
+execute_action(svc_action_t *op)
+{
+#if SUPPORT_UPSTART
+    if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_UPSTART,
+                     pcmk__str_casei)) {
+        return services__execute_upstart(op);
+    }
+#endif
+
+#if SUPPORT_SYSTEMD
+    if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD,
+                     pcmk__str_casei)) {
+        return services__execute_systemd(op);
+    }
+#endif
+
+    return services__execute_file(op);
+}
+
+void
+services_add_inflight_op(svc_action_t * op)
+{
+    if (op == NULL) {
+        return;
+    }
+
+    CRM_ASSERT(op->synchronous == FALSE);
+
+    /* keep track of ops that are in-flight to avoid collisions in the same namespace */
+    if (op->rsc) {
+        inflight_ops = g_list_append(inflight_ops, op);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Stop tracking an operation that completed
+ *
+ * \param[in] op  Operation to stop tracking
+ */
+void
+services_untrack_op(const svc_action_t *op)
+{
+    /* Op is no longer in-flight or blocked */
+    inflight_ops = g_list_remove(inflight_ops, op);
+    blocked_ops = g_list_remove(blocked_ops, op);
+
+    /* Op is no longer blocking other ops, so check if any need to run */
+    handle_blocked_ops();
+}
+
+gboolean
+services_action_async_fork_notify(svc_action_t * op,
+                                  void (*action_callback) (svc_action_t *),
+                                  void (*action_fork_callback) (svc_action_t *))
+{
+    CRM_CHECK(op != NULL, return TRUE);
+
+    op->synchronous = false;
+    if (action_callback != NULL) {
+        op->opaque->callback = action_callback;
+    }
+    if (action_fork_callback != NULL) {
+        op->opaque->fork_callback = action_fork_callback;
+    }
+
+    if (op->interval_ms > 0) {
+        init_recurring_actions();
+        if (handle_duplicate_recurring(op)) {
+            /* entry rescheduled, dup freed */
+            /* exit early */
+            return TRUE;
+        }
+        g_hash_table_replace(recurring_actions, op->id, op);
+    }
+
+    if (!pcmk_is_set(op->flags, SVC_ACTION_NON_BLOCKED)
+        && op->rsc && is_op_blocked(op->rsc)) {
+        blocked_ops = g_list_append(blocked_ops, op);
+        return TRUE;
+    }
+
+    return execute_action(op) == pcmk_rc_ok;
+}
+
+gboolean
+services_action_async(svc_action_t * op,
+                      void (*action_callback) (svc_action_t *))
+{
+    return services_action_async_fork_notify(op, action_callback, NULL);
+}
+
+static gboolean processing_blocked_ops = FALSE;
+
+gboolean
+is_op_blocked(const char *rsc)
+{
+    GList *gIter = NULL;
+    svc_action_t *op = NULL;
+
+    for (gIter = inflight_ops; gIter != NULL; gIter = gIter->next) {
+        op = gIter->data;
+        if (pcmk__str_eq(op->rsc, rsc, pcmk__str_casei)) {
+            return TRUE;
+        }
+    }
+
+    return FALSE;
+}
+
+static void
+handle_blocked_ops(void)
+{
+    GList *executed_ops = NULL;
+    GList *gIter = NULL;
+    svc_action_t *op = NULL;
+
+    if (processing_blocked_ops) {
+        /* avoid nested calling of this function */
+        return;
+    }
+
+    processing_blocked_ops = TRUE;
+
+    /* n^2 operation here, but blocked ops are incredibly rare. this list
+     * will be empty 99% of the time. */
+    for (gIter = blocked_ops; gIter != NULL; gIter = gIter->next) {
+        op = gIter->data;
+        if (is_op_blocked(op->rsc)) {
+            continue;
+        }
+        executed_ops = g_list_append(executed_ops, op);
+        if (execute_action(op) != pcmk_rc_ok) {
+            /* this can cause this function to be called recursively
+             * which is why we have processing_blocked_ops static variable */
+            services__finalize_async_op(op);
+        }
+    }
+
+    for (gIter = executed_ops; gIter != NULL; gIter = gIter->next) {
+        op = gIter->data;
+        blocked_ops = g_list_remove(blocked_ops, op);
+    }
+    g_list_free(executed_ops);
+
+    processing_blocked_ops = FALSE;
+}
+
+/*!
+ * \internal
+ * \brief Execute a meta-data action appropriately to standard
+ *
+ * \param[in,out] op  Meta-data action to execute
+ *
+ * \return Standard Pacemaker return code
+ */
+static int
+execute_metadata_action(svc_action_t *op)
+{
+    const char *class = op->standard;
+
+    if (op->agent == NULL) {
+        crm_info("Meta-data requested without specifying agent");
+        services__set_result(op, services__generic_error(op),
+                             PCMK_EXEC_ERROR_FATAL, "Agent not specified");
+        return EINVAL;
+    }
+
+    if (class == NULL) {
+        crm_info("Meta-data requested for agent %s without specifying class",
+                op->agent);
+        services__set_result(op, services__generic_error(op),
+                             PCMK_EXEC_ERROR_FATAL,
+                             "Agent standard not specified");
+        return EINVAL;
+    }
+
+    if (!strcmp(class, PCMK_RESOURCE_CLASS_SERVICE)) {
+        class = resources_find_service_class(op->agent);
+    }
+    if (class == NULL) {
+        crm_info("Meta-data requested for %s, but could not determine class",
+                 op->agent);
+        services__set_result(op, services__generic_error(op),
+                             PCMK_EXEC_ERROR_HARD,
+                             "Agent standard could not be determined");
+        return EINVAL;
+    }
+
+    if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)) {
+        return pcmk_legacy2rc(services__get_lsb_metadata(op->agent,
+                                                         &op->stdout_data));
+    }
+
+#if SUPPORT_NAGIOS
+    if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
+        return pcmk_legacy2rc(services__get_nagios_metadata(op->agent,
+                                                            &op->stdout_data));
+    }
+#endif
+
+    return execute_action(op);
+}
+
+gboolean
+services_action_sync(svc_action_t * op)
+{
+    gboolean rc = TRUE;
+
+    if (op == NULL) {
+        crm_trace("No operation to execute");
+        return FALSE;
+    }
+
+    op->synchronous = true;
+
+    if (pcmk__str_eq(op->action, "meta-data", pcmk__str_casei)) {
+        /* Synchronous meta-data operations are handled specially. Since most
+         * resource classes don't provide any meta-data, it has to be
+         * synthesized from available information about the agent.
+         *
+         * services_action_async() doesn't treat meta-data actions specially, so
+         * it will result in an error for classes that don't support the action.
+         */
+        rc = (execute_metadata_action(op) == pcmk_rc_ok);
+    } else {
+        rc = (execute_action(op) == pcmk_rc_ok);
+    }
+    crm_trace(" > " PCMK__OP_FMT ": %s = %d",
+              op->rsc, op->action, op->interval_ms, op->opaque->exec, op->rc);
+    if (op->stdout_data) {
+        crm_trace(" >  stdout: %s", op->stdout_data);
+    }
+    if (op->stderr_data) {
+        crm_trace(" >  stderr: %s", op->stderr_data);
+    }
+    return rc;
+}
+
+GList *
+get_directory_list(const char *root, gboolean files, gboolean executable)
+{
+    return services_os_get_directory_list(root, files, executable);
+}
+
+GList *
+resources_list_standards(void)
+{
+    GList *standards = NULL;
+
+    standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_OCF));
+    standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_LSB));
+    standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_SERVICE));
+
+#if SUPPORT_SYSTEMD
+    {
+        GList *agents = systemd_unit_listall();
+
+        if (agents != NULL) {
+            standards = g_list_append(standards,
+                                      strdup(PCMK_RESOURCE_CLASS_SYSTEMD));
+            g_list_free_full(agents, free);
+        }
+    }
+#endif
+
+#if SUPPORT_UPSTART
+    {
+        GList *agents = upstart_job_listall();
+
+        if (agents != NULL) {
+            standards = g_list_append(standards,
+                                      strdup(PCMK_RESOURCE_CLASS_UPSTART));
+            g_list_free_full(agents, free);
+        }
+    }
+#endif
+
+#if SUPPORT_NAGIOS
+    {
+        GList *agents = services__list_nagios_agents();
+
+        if (agents != NULL) {
+            standards = g_list_append(standards,
+                                      strdup(PCMK_RESOURCE_CLASS_NAGIOS));
+            g_list_free_full(agents, free);
+        }
+    }
+#endif
+
+    return standards;
+}
+
+GList *
+resources_list_providers(const char *standard)
+{
+    if (pcmk_is_set(pcmk_get_ra_caps(standard), pcmk_ra_cap_provider)) {
+        return resources_os_list_ocf_providers();
+    }
+
+    return NULL;
+}
+
+GList *
+resources_list_agents(const char *standard, const char *provider)
+{
+    if ((standard == NULL)
+        || (strcasecmp(standard, PCMK_RESOURCE_CLASS_SERVICE) == 0)) {
+
+        GList *tmp1;
+        GList *tmp2;
+        GList *result = services__list_lsb_agents();
+
+        if (standard == NULL) {
+            tmp1 = result;
+            tmp2 = resources_os_list_ocf_agents(NULL);
+            if (tmp2) {
+                result = g_list_concat(tmp1, tmp2);
+            }
+        }
+#if SUPPORT_SYSTEMD
+        tmp1 = result;
+        tmp2 = systemd_unit_listall();
+        if (tmp2) {
+            result = g_list_concat(tmp1, tmp2);
+        }
+#endif
+
+#if SUPPORT_UPSTART
+        tmp1 = result;
+        tmp2 = upstart_job_listall();
+        if (tmp2) {
+            result = g_list_concat(tmp1, tmp2);
+        }
+#endif
+
+        return result;
+
+    } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_OCF) == 0) {
+        return resources_os_list_ocf_agents(provider);
+    } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_LSB) == 0) {
+        return services__list_lsb_agents();
+#if SUPPORT_SYSTEMD
+    } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_SYSTEMD) == 0) {
+        return systemd_unit_listall();
+#endif
+#if SUPPORT_UPSTART
+    } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_UPSTART) == 0) {
+        return upstart_job_listall();
+#endif
+#if SUPPORT_NAGIOS
+    } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_NAGIOS) == 0) {
+        return services__list_nagios_agents();
+#endif
+    }
+
+    return NULL;
+}
+
+gboolean
+resources_agent_exists(const char *standard, const char *provider, const char *agent)
+{
+    GList *standards = NULL;
+    GList *providers = NULL;
+    GList *iter = NULL;
+    gboolean rc = FALSE;
+    gboolean has_providers = FALSE;
+
+    standards = resources_list_standards();
+    for (iter = standards; iter != NULL; iter = iter->next) {
+        if (pcmk__str_eq(iter->data, standard, pcmk__str_none)) {
+            rc = TRUE;
+            break;
+        }
+    }
+
+    if (rc == FALSE) {
+        goto done;
+    }
+
+    rc = FALSE;
+
+    has_providers = pcmk_is_set(pcmk_get_ra_caps(standard), pcmk_ra_cap_provider);
+    if (has_providers == TRUE && provider != NULL) {
+        providers = resources_list_providers(standard);
+        for (iter = providers; iter != NULL; iter = iter->next) {
+            if (pcmk__str_eq(iter->data, provider, pcmk__str_none)) {
+                rc = TRUE;
+                break;
+            }
+        }
+    } else if (has_providers == FALSE && provider == NULL) {
+        rc = TRUE;
+    }
+
+    if (rc == FALSE) {
+        goto done;
+    }
+
+    if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
+        if (services__lsb_agent_exists(agent)) {
+            rc = TRUE;
+#if SUPPORT_SYSTEMD
+        } else if (systemd_unit_exists(agent)) {
+            rc = TRUE;
+#endif
+
+#if SUPPORT_UPSTART
+        } else if (upstart_job_exists(agent)) {
+            rc = TRUE;
+#endif
+        } else {
+            rc = FALSE;
+        }
+
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
+        rc = services__ocf_agent_exists(provider, agent);
+
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)) {
+        rc = services__lsb_agent_exists(agent);
+
+#if SUPPORT_SYSTEMD
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SYSTEMD, pcmk__str_casei)) {
+        rc = systemd_unit_exists(agent);
+#endif
+
+#if SUPPORT_UPSTART
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_casei)) {
+        rc = upstart_job_exists(agent);
+#endif
+
+#if SUPPORT_NAGIOS
+    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
+        rc = services__nagios_agent_exists(agent);
+#endif
+
+    } else {
+        rc = FALSE;
+    }
+
+done:
+    g_list_free(standards);
+    g_list_free(providers);
+    return rc;
+}
+
+/*!
+ * \internal
+ * \brief Set the result of an action
+ *
+ * \param[out] action        Where to set action result
+ * \param[in]  agent_status  Exit status to set
+ * \param[in]  exec_status   Execution status to set
+ * \param[in]  reason        Human-friendly description of event to set
+ */
+void
+services__set_result(svc_action_t *action, int agent_status,
+                     enum pcmk_exec_status exec_status, const char *reason)
+{
+    if (action == NULL) {
+        return;
+    }
+
+    action->rc = agent_status;
+    action->status = exec_status;
+
+    if (!pcmk__str_eq(action->opaque->exit_reason, reason,
+                      pcmk__str_none)) {
+        free(action->opaque->exit_reason);
+        action->opaque->exit_reason = (reason == NULL)? NULL : strdup(reason);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Set the result of an action, with a formatted exit reason
+ *
+ * \param[out] action        Where to set action result
+ * \param[in]  agent_status  Exit status to set
+ * \param[in]  exec_status   Execution status to set
+ * \param[in]  format        printf-style format for a human-friendly
+ *                           description of reason for result
+ * \param[in]  ...           arguments for \p format
+ */
+void
+services__format_result(svc_action_t *action, int agent_status,
+                        enum pcmk_exec_status exec_status,
+                        const char *format, ...)
+{
+    va_list ap;
+    int len = 0;
+    char *reason = NULL;
+
+    if (action == NULL) {
+        return;
+    }
+
+    action->rc = agent_status;
+    action->status = exec_status;
+
+    if (format != NULL) {
+        va_start(ap, format);
+        len = vasprintf(&reason, format, ap);
+        CRM_ASSERT(len > 0);
+        va_end(ap);
+    }
+    free(action->opaque->exit_reason);
+    action->opaque->exit_reason = reason;
+}
+
+/*!
+ * \internal
+ * \brief Set the result of an action to cancelled
+ *
+ * \param[out] action        Where to set action result
+ *
+ * \note This sets execution status but leaves the exit status unchanged
+ */
+void
+services__set_cancelled(svc_action_t *action)
+{
+    if (action != NULL) {
+        action->status = PCMK_EXEC_CANCELLED;
+        free(action->opaque->exit_reason);
+        action->opaque->exit_reason = NULL;
+    }
+}
+
+/*!
+ * \internal
+ * \brief Get a readable description of what an action is for
+ *
+ * \param[in] action  Action to check
+ *
+ * \return Readable name for the kind of \p action
+ */
+const char *
+services__action_kind(const svc_action_t *action)
+{
+    if ((action == NULL) || (action->standard == NULL)) {
+        return "Process";
+    } else if (pcmk__str_eq(action->standard, PCMK_RESOURCE_CLASS_STONITH,
+                            pcmk__str_none)) {
+        return "Fence agent";
+    } else if (pcmk__str_eq(action->standard, PCMK_RESOURCE_CLASS_ALERT,
+                            pcmk__str_none)) {
+        return "Alert agent";
+    } else {
+        return "Resource agent";
+    }
+}
+
+/*!
+ * \internal
+ * \brief Get the exit reason of an action
+ *
+ * \param[in] action  Action to check
+ *
+ * \return Action's exit reason (or NULL if none)
+ */
+const char *
+services__exit_reason(const svc_action_t *action)
+{
+    return action->opaque->exit_reason;
+}
+
+/*!
+ * \internal
+ * \brief Steal stdout from an action
+ *
+ * \param[in,out] action  Action whose stdout is desired
+ *
+ * \return Action's stdout (which may be NULL)
+ * \note Upon return, \p action will no longer track the output, so it is the
+ *       caller's responsibility to free the return value.
+ */
+char *
+services__grab_stdout(svc_action_t *action)
+{
+    char *output = action->stdout_data;
+
+    action->stdout_data = NULL;
+    return output;
+}
+
+/*!
+ * \internal
+ * \brief Steal stderr from an action
+ *
+ * \param[in,out] action  Action whose stderr is desired
+ *
+ * \return Action's stderr (which may be NULL)
+ * \note Upon return, \p action will no longer track the output, so it is the
+ *       caller's responsibility to free the return value.
+ */
+char *
+services__grab_stderr(svc_action_t *action)
+{
+    char *output = action->stderr_data;
+
+    action->stderr_data = NULL;
+    return output;
+}