path: root/lib/services/services.c

/*
 * Copyright 2010-2023 the Pacemaker project contributors
 *
 * The version control history for this file may have further details.
 *
 * This source code is licensed under the GNU Lesser General Public License
 * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
 */

#include <crm_internal.h>

#ifndef _GNU_SOURCE
#  define _GNU_SOURCE
#endif

#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <dirent.h>
#include <fcntl.h>

#include <crm/crm.h>
#include <crm/common/mainloop.h>
#include <crm/services.h>
#include <crm/services_internal.h>
#include <crm/stonith-ng.h>
#include <crm/msg_xml.h>
#include "services_private.h"
#include "services_ocf.h"
#include "services_lsb.h"

#if SUPPORT_UPSTART
#  include <upstart.h>
#endif

#if SUPPORT_SYSTEMD
#  include <systemd.h>
#endif

#if SUPPORT_NAGIOS
#  include <services_nagios.h>
#endif

/* TODO: Develop a rollover strategy */

static int operations = 0;
static GHashTable *recurring_actions = NULL;

/* ops waiting to run async because of conflicting active
 * pending ops */
static GList *blocked_ops = NULL;

/* ops currently active (in-flight) */
static GList *inflight_ops = NULL;

static void handle_blocked_ops(void);

/*!
 * \brief Find first service class that can provide a specified agent
 *
 * \param[in] agent  Name of agent to search for
 *
 * \return Service class if found, NULL otherwise
 *
 * \note The priority is LSB, then systemd, then upstart. It would be preferable
 *       to put systemd first, but LSB merely requires a file existence check,
 *       while systemd requires contacting D-Bus.
 */
const char *
resources_find_service_class(const char *agent)
{
    if (services__lsb_agent_exists(agent)) {
        return PCMK_RESOURCE_CLASS_LSB;
    }

#if SUPPORT_SYSTEMD
    if (systemd_unit_exists(agent)) {
        return PCMK_RESOURCE_CLASS_SYSTEMD;
    }
#endif

#if SUPPORT_UPSTART
    if (upstart_job_exists(agent)) {
        return PCMK_RESOURCE_CLASS_UPSTART;
    }
#endif
    return NULL;
}

static inline void
init_recurring_actions(void)
{
    if (recurring_actions == NULL) {
        recurring_actions = pcmk__strkey_table(NULL, NULL);
    }
}

/*!
 * \internal
 * \brief Check whether op is in-flight systemd or upstart op
 *
 * \param[in] op  Operation to check
 *
 * \return TRUE if op is in-flight systemd or upstart op
 */
static inline gboolean
inflight_systemd_or_upstart(const svc_action_t *op)
{
    return pcmk__strcase_any_of(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD,
                           PCMK_RESOURCE_CLASS_UPSTART, NULL) &&
           g_list_find(inflight_ops, op) != NULL;
}

/*!
 * \internal
 * \brief Expand "service" alias to an actual resource class
 *
 * \param[in] rsc       Resource name (for logging only)
 * \param[in] standard  Resource class as configured
 * \param[in] agent     Agent name to look for
 *
 * \return Newly allocated string with actual resource class
 *
 * \note The caller is responsible for calling free() on the result.
 */
static char *
expand_resource_class(const char *rsc, const char *standard, const char *agent)
{
    char *expanded_class = NULL;

    if (strcasecmp(standard, PCMK_RESOURCE_CLASS_SERVICE) == 0) {
        const char *found_class = resources_find_service_class(agent);

        if (found_class) {
            crm_debug("Found %s agent %s for %s", found_class, agent, rsc);
            expanded_class = strdup(found_class);
        } else {
            crm_info("Assuming resource class lsb for agent %s for %s",
                     agent, rsc);
            expanded_class = strdup(PCMK_RESOURCE_CLASS_LSB);
        }
    } else {
        expanded_class = strdup(standard);
    }
    CRM_ASSERT(expanded_class);
    return expanded_class;
}

/*!
 * \internal
 * \brief Create a simple svc_action_t instance
 *
 * \return Newly allocated instance (or NULL if not enough memory)
 */
static svc_action_t *
new_action(void)
{
    svc_action_t *op = calloc(1, sizeof(svc_action_t));

    if (op == NULL) {
        return NULL;
    }

    op->opaque = calloc(1, sizeof(svc_action_private_t));
    if (op->opaque == NULL) {
        free(op);
        return NULL;
    }

    // Initialize result
    services__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_UNKNOWN, NULL);
    return op;
}

static bool
required_argument_missing(uint32_t ra_caps, const char *name,
                          const char *standard, const char *provider,
                          const char *agent, const char *action)
{
    if (pcmk__str_empty(name)) {
        crm_info("Cannot create operation without resource name (bug?)");
        return true;
    }

    if (pcmk__str_empty(standard)) {
        crm_info("Cannot create operation for %s without resource class (bug?)",
                 name);
        return true;
    }

    if (pcmk_is_set(ra_caps, pcmk_ra_cap_provider)
        && pcmk__str_empty(provider)) {
        crm_info("Cannot create operation for %s resource %s "
                 "without provider (bug?)", standard, name);
        return true;
    }

    if (pcmk__str_empty(agent)) {
        crm_info("Cannot create operation for %s without agent name (bug?)",
                 name);
        return true;
    }

    if (pcmk__str_empty(action)) {
        crm_info("Cannot create operation for %s without action name (bug?)",
                 name);
        return true;
    }
    return false;
}

// \return Standard Pacemaker return code (pcmk_rc_ok or ENOMEM)
static int
copy_action_arguments(svc_action_t *op, uint32_t ra_caps, const char *name,
                      const char *standard, const char *provider,
                      const char *agent, const char *action)
{
    op->rsc = strdup(name);
    if (op->rsc == NULL) {
        return ENOMEM;
    }

    op->agent = strdup(agent);
    if (op->agent == NULL) {
        return ENOMEM;
    }

    op->standard = expand_resource_class(name, standard, agent);
    if (op->standard == NULL) {
        return ENOMEM;
    }

    if (pcmk_is_set(ra_caps, pcmk_ra_cap_status)
        && pcmk__str_eq(action, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
        action = PCMK_ACTION_STATUS;
    }
    op->action = strdup(action);
    if (op->action == NULL) {
        return ENOMEM;
    }

    if (pcmk_is_set(ra_caps, pcmk_ra_cap_provider)) {
        op->provider = strdup(provider);
        if (op->provider == NULL) {
            return ENOMEM;
        }
    }
    return pcmk_rc_ok;
}

svc_action_t *
services__create_resource_action(const char *name, const char *standard,
                        const char *provider, const char *agent,
                        const char *action, guint interval_ms, int timeout,
                        GHashTable *params, enum svc_action_flags flags)
{
    svc_action_t *op = NULL;
    uint32_t ra_caps = pcmk_get_ra_caps(standard);
    int rc = pcmk_rc_ok;

    op = new_action();
    if (op == NULL) {
        crm_crit("Cannot prepare action: %s", strerror(ENOMEM));
        if (params != NULL) {
            g_hash_table_destroy(params);
        }
        return NULL;
    }

    op->interval_ms = interval_ms;
    op->timeout = timeout;
    op->flags = flags;
    op->sequence = ++operations;

    // Take ownership of params
    if (pcmk_is_set(ra_caps, pcmk_ra_cap_params)) {
        op->params = params;
    } else if (params != NULL) {
        g_hash_table_destroy(params);
        params = NULL;
    }

    if (required_argument_missing(ra_caps, name, standard, provider, agent,
                                  action)) {
        services__set_result(op, services__generic_error(op),
                             PCMK_EXEC_ERROR_FATAL,
                             "Required agent or action information missing");
        return op;
    }

    op->id = pcmk__op_key(name, action, interval_ms);

    if (copy_action_arguments(op, ra_caps, name, standard, provider, agent,
                              action) != pcmk_rc_ok) {
        crm_crit("Cannot prepare %s action for %s: %s",
                 action, name, strerror(ENOMEM));
        services__handle_exec_error(op, ENOMEM);
        return op;
    }

    if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_OCF) == 0) {
        rc = services__ocf_prepare(op);

    } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_LSB) == 0) {
        rc = services__lsb_prepare(op);

#if SUPPORT_SYSTEMD
    } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD) == 0) {
        rc = services__systemd_prepare(op);
#endif
#if SUPPORT_UPSTART
    } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_UPSTART) == 0) {
        rc = services__upstart_prepare(op);
#endif
#if SUPPORT_NAGIOS
    } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_NAGIOS) == 0) {
        rc = services__nagios_prepare(op);
#endif
    } else {
        crm_info("Unknown resource standard: %s", op->standard);
        rc = ENOENT;
    }

    if (rc != pcmk_rc_ok) {
        crm_info("Cannot prepare %s operation for %s: %s",
                 action, name, strerror(rc));
        services__handle_exec_error(op, rc);
    }
    return op;
}

svc_action_t *
resources_action_create(const char *name, const char *standard,
                        const char *provider, const char *agent,
                        const char *action, guint interval_ms, int timeout,
                        GHashTable *params, enum svc_action_flags flags)
{
    svc_action_t *op = services__create_resource_action(name, standard,
                            provider, agent, action, interval_ms, timeout,
                            params, flags);
    if (op == NULL || op->rc != 0) {
        services_action_free(op);
        return NULL;
    } else {
        // Preserve public API backward compatibility
        op->rc = PCMK_OCF_OK;
        op->status = PCMK_EXEC_DONE;

        return op;
    }
}

svc_action_t *
services_action_create_generic(const char *exec, const char *args[])
{
    svc_action_t *op = new_action();

    CRM_ASSERT(op != NULL);

    op->opaque->exec = strdup(exec);
    op->opaque->args[0] = strdup(exec);
    if ((op->opaque->exec == NULL) || (op->opaque->args[0] == NULL)) {
        crm_crit("Cannot prepare action for '%s': %s", exec, strerror(ENOMEM));
        services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                             strerror(ENOMEM));
        return op;
    }

    if (args == NULL) {
        return op;
    }

    for (int cur_arg = 1; args[cur_arg - 1] != NULL; cur_arg++) {

        if (cur_arg == PCMK__NELEM(op->opaque->args)) {
            crm_info("Cannot prepare action for '%s': Too many arguments",
                     exec);
            services__set_result(op, PCMK_OCF_UNKNOWN_ERROR,
                                 PCMK_EXEC_ERROR_HARD, "Too many arguments");
            break;
        }

        op->opaque->args[cur_arg] = strdup(args[cur_arg - 1]);
        if (op->opaque->args[cur_arg] == NULL) {
            crm_crit("Cannot prepare action for '%s': %s",
                     exec, strerror(ENOMEM));
            services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                                 strerror(ENOMEM));
            break;
        }
    }

    return op;
}

/*!
 * \brief Create an alert agent action
 *
 * \param[in] id        Alert ID
 * \param[in] exec      Path to alert agent executable
 * \param[in] timeout   Action timeout
 * \param[in] params    Parameters to use with action
 * \param[in] sequence  Action sequence number
 * \param[in] cb_data   Data to pass to callback function
 *
 * \return New action on success, NULL on error
 * \note It is the caller's responsibility to free cb_data.
 *       The caller should not free params explicitly.
 */
svc_action_t *
services_alert_create(const char *id, const char *exec, int timeout,
                      GHashTable *params, int sequence, void *cb_data)
{
    svc_action_t *action = services_action_create_generic(exec, NULL);

    action->id = strdup(id);
    action->standard = strdup(PCMK_RESOURCE_CLASS_ALERT);
    CRM_ASSERT((action->id != NULL) && (action->standard != NULL));

    action->timeout = timeout;
    action->params = params;
    action->sequence = sequence;
    action->cb_data = cb_data;
    return action;
}

/*!
 * \brief Set the user and group that an action will execute as
 *
 * \param[in,out] op      Action to modify
 * \param[in]     user    Name of user to execute action as
 * \param[in]     group   Name of group to execute action as
 *
 * \return pcmk_ok on success, -errno otherwise
 *
 * \note This will have no effect unless the process executing the action runs
 *       as root, and the action is not a systemd or upstart action.
 *       We could implement this for systemd by adding User= and Group= to
 *       [Service] in the override file, but that seems more likely to cause
 *       problems than be useful.
 */
int
services_action_user(svc_action_t *op, const char *user)
{
    CRM_CHECK((op != NULL) && (user != NULL), return -EINVAL);
    return crm_user_lookup(user, &(op->opaque->uid), &(op->opaque->gid));
}

/*!
 * \brief Execute an alert agent action
 *
 * \param[in,out] action  Action to execute
 * \param[in]     cb      Function to call when action completes
 *
 * \return TRUE if the library will free action, FALSE otherwise
 *
 * \note If this function returns FALSE, it is the caller's responsibility to
 *       free the action with services_action_free(). However, unless someone
 *       intentionally creates a recurring alert action, this will never return
 *       FALSE.
 */
gboolean
services_alert_async(svc_action_t *action, void (*cb)(svc_action_t *op))
{
    action->synchronous = false;
    action->opaque->callback = cb;
    return services__execute_file(action) == pcmk_rc_ok;
}

#if HAVE_DBUS
/*!
 * \internal
 * \brief Update operation's pending DBus call, unreferencing old one if needed
 *
 * \param[in,out] op       Operation to modify
 * \param[in]     pending  Pending call to set
 */
void
services_set_op_pending(svc_action_t *op, DBusPendingCall *pending)
{
    if (op->opaque->pending && (op->opaque->pending != pending)) {
        if (pending) {
            crm_info("Lost pending %s DBus call (%p)", op->id, op->opaque->pending);
        } else {
            crm_trace("Done with pending %s DBus call (%p)", op->id, op->opaque->pending);
        }
        dbus_pending_call_unref(op->opaque->pending);
    }
    op->opaque->pending = pending;
    if (pending) {
        crm_trace("Updated pending %s DBus call (%p)", op->id, pending);
    } else {
        crm_trace("Cleared pending %s DBus call", op->id);
    }
}
#endif

void
services_action_cleanup(svc_action_t * op)
{
    if ((op == NULL) || (op->opaque == NULL)) {
        return;
    }

#if HAVE_DBUS
    if(op->opaque->timerid != 0) {
        crm_trace("Removing timer for call %s to %s", op->action, op->rsc);
        g_source_remove(op->opaque->timerid);
        op->opaque->timerid = 0;
    }

    if(op->opaque->pending) {
        if (dbus_pending_call_get_completed(op->opaque->pending)) {
            // This should never be the case
            crm_warn("Result of %s op %s was unhandled",
                     op->standard, op->id);
        } else {
            crm_debug("Will ignore any result of canceled %s op %s",
                      op->standard, op->id);
        }
        dbus_pending_call_cancel(op->opaque->pending);
        services_set_op_pending(op, NULL);
    }
#endif

    if (op->opaque->stderr_gsource) {
        mainloop_del_fd(op->opaque->stderr_gsource);
        op->opaque->stderr_gsource = NULL;
    }

    if (op->opaque->stdout_gsource) {
        mainloop_del_fd(op->opaque->stdout_gsource);
        op->opaque->stdout_gsource = NULL;
    }
}

/*!
 * \internal
 * \brief Map an actual resource action result to a standard OCF result
 *
 * \param[in] standard     Agent standard (must not be "service")
 * \param[in] action       Action that result is for
 * \param[in] exit_status  Actual agent exit status
 *
 * \return Standard OCF result
 */
enum ocf_exitcode
services_result2ocf(const char *standard, const char *action, int exit_status)
{
    if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
        return services__ocf2ocf(exit_status);

#if SUPPORT_SYSTEMD
    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SYSTEMD,
                            pcmk__str_casei)) {
        return services__systemd2ocf(exit_status);
#endif

#if SUPPORT_UPSTART
    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_UPSTART,
                            pcmk__str_casei)) {
        return services__upstart2ocf(exit_status);
#endif

#if SUPPORT_NAGIOS
    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_NAGIOS,
                            pcmk__str_casei)) {
        return services__nagios2ocf(exit_status);
#endif

    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_LSB,
                            pcmk__str_casei)) {
        return services__lsb2ocf(action, exit_status);

    } else {
        crm_warn("Treating result from unknown standard '%s' as OCF",
                 ((standard == NULL)? "unspecified" : standard));
        return services__ocf2ocf(exit_status);
    }
}

void
services_action_free(svc_action_t * op)
{
    unsigned int i;

    if (op == NULL) {
        return;
    }

    /* The operation should be removed from all tracking lists by this point.
     * If it's not, we have a bug somewhere, so bail. That may lead to a
     * memory leak, but it's better than a use-after-free segmentation fault.
     */
    CRM_CHECK(g_list_find(inflight_ops, op) == NULL, return);
    CRM_CHECK(g_list_find(blocked_ops, op) == NULL, return);
    CRM_CHECK((recurring_actions == NULL)
              || (g_hash_table_lookup(recurring_actions, op->id) == NULL),
              return);

    services_action_cleanup(op);

    if (op->opaque->repeat_timer) {
        g_source_remove(op->opaque->repeat_timer);
        op->opaque->repeat_timer = 0;
    }

    free(op->id);
    free(op->opaque->exec);

    for (i = 0; i < PCMK__NELEM(op->opaque->args); i++) {
        free(op->opaque->args[i]);
    }

    free(op->opaque->exit_reason);
    free(op->opaque);
    free(op->rsc);
    free(op->action);

    free(op->standard);
    free(op->agent);
    free(op->provider);

    free(op->stdout_data);
    free(op->stderr_data);

    if (op->params) {
        g_hash_table_destroy(op->params);
        op->params = NULL;
    }

    free(op);
}

gboolean
cancel_recurring_action(svc_action_t * op)
{
    crm_info("Cancelling %s operation %s", op->standard, op->id);

    if (recurring_actions) {
        g_hash_table_remove(recurring_actions, op->id);
    }

    if (op->opaque->repeat_timer) {
        g_source_remove(op->opaque->repeat_timer);
        op->opaque->repeat_timer = 0;
    }

    return TRUE;
}

/*!
 * \brief Cancel a recurring action
 *
 * \param[in] name         Name of resource that operation is for
 * \param[in] action       Name of operation to cancel
 * \param[in] interval_ms  Interval of operation to cancel
 *
 * \return TRUE if action was successfully cancelled, FALSE otherwise
 */
gboolean
services_action_cancel(const char *name, const char *action, guint interval_ms)
{
    gboolean cancelled = FALSE;
    char *id = pcmk__op_key(name, action, interval_ms);
    svc_action_t *op = NULL;

    /* We can only cancel a recurring action */
    init_recurring_actions();
    op = g_hash_table_lookup(recurring_actions, id);
    if (op == NULL) {
        goto done;
    }

    // Tell services__finalize_async_op() not to reschedule the operation
    op->cancel = TRUE;

    /* Stop tracking it as a recurring operation, and stop its repeat timer */
    cancel_recurring_action(op);

    /* If the op has a PID, it's an in-flight child process, so kill it.
     *
     * Whether the kill succeeds or fails, the main loop will send the op to
     * async_action_complete() (and thus services__finalize_async_op()) when the
     * process goes away.
     */
    if (op->pid != 0) {
        crm_info("Terminating in-flight op %s[%d] early because it was cancelled",
                 id, op->pid);
        cancelled = mainloop_child_kill(op->pid);
        if (cancelled == FALSE) {
            crm_err("Termination of %s[%d] failed", id, op->pid);
        }
        goto done;
    }

#if HAVE_DBUS
    // In-flight systemd and upstart ops don't have a pid
    if (inflight_systemd_or_upstart(op)) {
        inflight_ops = g_list_remove(inflight_ops, op);

        /* This will cause any result that comes in later to be discarded, so we
         * don't call the callback and free the operation twice.
         */
        services_action_cleanup(op);
    }
#endif

    /* The rest of this is essentially equivalent to
     * services__finalize_async_op(), minus the handle_blocked_ops() call.
     */

    // Report operation as cancelled
    services__set_cancelled(op);
    if (op->opaque->callback) {
        op->opaque->callback(op);
    }

    blocked_ops = g_list_remove(blocked_ops, op);
    services_action_free(op);
    cancelled = TRUE;
    // @TODO Initiate handle_blocked_ops() asynchronously

done:
    free(id);
    return cancelled;
}

gboolean
services_action_kick(const char *name, const char *action, guint interval_ms)
{
    svc_action_t * op = NULL;
    char *id = pcmk__op_key(name, action, interval_ms);

    init_recurring_actions();
    op = g_hash_table_lookup(recurring_actions, id);
    free(id);

    if (op == NULL) {
        return FALSE;
    }


    if (op->pid || inflight_systemd_or_upstart(op)) {
        return TRUE;
    } else {
        if (op->opaque->repeat_timer) {
            g_source_remove(op->opaque->repeat_timer);
            op->opaque->repeat_timer = 0;
        }
        recurring_action_timer(op);
        return TRUE;
    }

}

/*!
 * \internal
 * \brief Add a new recurring operation, checking for duplicates
 *
 * \param[in,out] op  Operation to add
 *
 * \return TRUE if duplicate found (and reschedule), FALSE otherwise
 */
static gboolean
handle_duplicate_recurring(svc_action_t *op)
{
    svc_action_t * dup = NULL;

    /* check for duplicates */
    dup = g_hash_table_lookup(recurring_actions, op->id);

    if (dup && (dup != op)) {
        /* update user data */
        if (op->opaque->callback) {
            dup->opaque->callback = op->opaque->callback;
            dup->cb_data = op->cb_data;
            op->cb_data = NULL;
        }
        /* immediately execute the next interval */
        if (dup->pid != 0) {
            if (op->opaque->repeat_timer) {
                g_source_remove(op->opaque->repeat_timer);
                op->opaque->repeat_timer = 0;
            }
            recurring_action_timer(dup);
        }
        /* free the duplicate */
        services_action_free(op);
        return TRUE;
    }

    return FALSE;
}

/*!
 * \internal
 * \brief Execute an action appropriately according to its standard
 *
 * \param[in,out] op  Action to execute
 *
 * \return Standard Pacemaker return code
 * \retval EBUSY          Recurring operation could not be initiated
 * \retval pcmk_rc_error  Synchronous action failed
 * \retval pcmk_rc_ok     Synchronous action succeeded, or asynchronous action
 *                        should not be freed (because it's pending or because
 *                        it failed to execute and was already freed)
 *
 * \note If the return value for an asynchronous action is not pcmk_rc_ok, the
 *       caller is responsible for freeing the action.
 */
static int
execute_action(svc_action_t *op)
{
#if SUPPORT_UPSTART
    if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_UPSTART,
                     pcmk__str_casei)) {
        return services__execute_upstart(op);
    }
#endif

#if SUPPORT_SYSTEMD
    if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD,
                     pcmk__str_casei)) {
        return services__execute_systemd(op);
    }
#endif

    return services__execute_file(op);
}

void
services_add_inflight_op(svc_action_t * op)
{
    if (op == NULL) {
        return;
    }

    CRM_ASSERT(op->synchronous == FALSE);

    /* keep track of ops that are in-flight to avoid collisions in the same namespace */
    if (op->rsc) {
        inflight_ops = g_list_append(inflight_ops, op);
    }
}

/*!
 * \internal
 * \brief Stop tracking an operation that completed
 *
 * \param[in] op  Operation to stop tracking
 */
void
services_untrack_op(const svc_action_t *op)
{
    /* Op is no longer in-flight or blocked */
    inflight_ops = g_list_remove(inflight_ops, op);
    blocked_ops = g_list_remove(blocked_ops, op);

    /* Op is no longer blocking other ops, so check if any need to run */
    handle_blocked_ops();
}

gboolean
services_action_async_fork_notify(svc_action_t * op,
                                  void (*action_callback) (svc_action_t *),
                                  void (*action_fork_callback) (svc_action_t *))
{
    CRM_CHECK(op != NULL, return TRUE);

    op->synchronous = false;
    if (action_callback != NULL) {
        op->opaque->callback = action_callback;
    }
    if (action_fork_callback != NULL) {
        op->opaque->fork_callback = action_fork_callback;
    }

    if (op->interval_ms > 0) {
        init_recurring_actions();
        if (handle_duplicate_recurring(op)) {
            /* entry rescheduled, dup freed */
            /* exit early */
            return TRUE;
        }
        g_hash_table_replace(recurring_actions, op->id, op);
    }

    if (!pcmk_is_set(op->flags, SVC_ACTION_NON_BLOCKED)
        && op->rsc && is_op_blocked(op->rsc)) {
        blocked_ops = g_list_append(blocked_ops, op);
        return TRUE;
    }

    return execute_action(op) == pcmk_rc_ok;
}

gboolean
services_action_async(svc_action_t * op,
                      void (*action_callback) (svc_action_t *))
{
    return services_action_async_fork_notify(op, action_callback, NULL);
}

static gboolean processing_blocked_ops = FALSE;

gboolean
is_op_blocked(const char *rsc)
{
    GList *gIter = NULL;
    svc_action_t *op = NULL;

    for (gIter = inflight_ops; gIter != NULL; gIter = gIter->next) {
        op = gIter->data;
        if (pcmk__str_eq(op->rsc, rsc, pcmk__str_casei)) {
            return TRUE;
        }
    }

    return FALSE;
}

static void
handle_blocked_ops(void)
{
    GList *executed_ops = NULL;
    GList *gIter = NULL;
    svc_action_t *op = NULL;

    if (processing_blocked_ops) {
        /* avoid nested calling of this function */
        return;
    }

    processing_blocked_ops = TRUE;

    /* n^2 operation here, but blocked ops are incredibly rare. this list
     * will be empty 99% of the time. */
    for (gIter = blocked_ops; gIter != NULL; gIter = gIter->next) {
        op = gIter->data;
        if (is_op_blocked(op->rsc)) {
            continue;
        }
        executed_ops = g_list_append(executed_ops, op);
        if (execute_action(op) != pcmk_rc_ok) {
            /* this can cause this function to be called recursively
             * which is why we have processing_blocked_ops static variable */
            services__finalize_async_op(op);
        }
    }

    for (gIter = executed_ops; gIter != NULL; gIter = gIter->next) {
        op = gIter->data;
        blocked_ops = g_list_remove(blocked_ops, op);
    }
    g_list_free(executed_ops);

    processing_blocked_ops = FALSE;
}

/*!
 * \internal
 * \brief Execute a meta-data action appropriately to standard
 *
 * \param[in,out] op  Meta-data action to execute
 *
 * \return Standard Pacemaker return code
 */
static int
execute_metadata_action(svc_action_t *op)
{
    const char *class = op->standard;

    if (op->agent == NULL) {
        crm_info("Meta-data requested without specifying agent");
        services__set_result(op, services__generic_error(op),
                             PCMK_EXEC_ERROR_FATAL, "Agent not specified");
        return EINVAL;
    }

    if (class == NULL) {
        crm_info("Meta-data requested for agent %s without specifying class",
                op->agent);
        services__set_result(op, services__generic_error(op),
                             PCMK_EXEC_ERROR_FATAL,
                             "Agent standard not specified");
        return EINVAL;
    }

    if (!strcmp(class, PCMK_RESOURCE_CLASS_SERVICE)) {
        class = resources_find_service_class(op->agent);
    }
    if (class == NULL) {
        crm_info("Meta-data requested for %s, but could not determine class",
                 op->agent);
        services__set_result(op, services__generic_error(op),
                             PCMK_EXEC_ERROR_HARD,
                             "Agent standard could not be determined");
        return EINVAL;
    }

    if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)) {
        return pcmk_legacy2rc(services__get_lsb_metadata(op->agent,
                                                         &op->stdout_data));
    }

#if SUPPORT_NAGIOS
    if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
        return pcmk_legacy2rc(services__get_nagios_metadata(op->agent,
                                                            &op->stdout_data));
    }
#endif

    return execute_action(op);
}

gboolean
services_action_sync(svc_action_t * op)
{
    gboolean rc = TRUE;

    if (op == NULL) {
        crm_trace("No operation to execute");
        return FALSE;
    }

    op->synchronous = true;

    if (pcmk__str_eq(op->action, PCMK_ACTION_META_DATA, pcmk__str_casei)) {
        /* Synchronous meta-data operations are handled specially. Since most
         * resource classes don't provide any meta-data, it has to be
         * synthesized from available information about the agent.
         *
         * services_action_async() doesn't treat meta-data actions specially, so
         * it will result in an error for classes that don't support the action.
         */
        rc = (execute_metadata_action(op) == pcmk_rc_ok);
    } else {
        rc = (execute_action(op) == pcmk_rc_ok);
    }
    crm_trace(" > " PCMK__OP_FMT ": %s = %d",
              op->rsc, op->action, op->interval_ms, op->opaque->exec, op->rc);
    if (op->stdout_data) {
        crm_trace(" >  stdout: %s", op->stdout_data);
    }
    if (op->stderr_data) {
        crm_trace(" >  stderr: %s", op->stderr_data);
    }
    return rc;
}

GList *
get_directory_list(const char *root, gboolean files, gboolean executable)
{
    return services_os_get_directory_list(root, files, executable);
}

GList *
resources_list_standards(void)
{
    GList *standards = NULL;

    standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_OCF));
    standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_LSB));
    standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_SERVICE));

#if SUPPORT_SYSTEMD
    {
        GList *agents = systemd_unit_listall();

        if (agents != NULL) {
            standards = g_list_append(standards,
                                      strdup(PCMK_RESOURCE_CLASS_SYSTEMD));
            g_list_free_full(agents, free);
        }
    }
#endif

#if SUPPORT_UPSTART
    {
        GList *agents = upstart_job_listall();

        if (agents != NULL) {
            standards = g_list_append(standards,
                                      strdup(PCMK_RESOURCE_CLASS_UPSTART));
            g_list_free_full(agents, free);
        }
    }
#endif

#if SUPPORT_NAGIOS
    {
        GList *agents = services__list_nagios_agents();

        if (agents != NULL) {
            standards = g_list_append(standards,
                                      strdup(PCMK_RESOURCE_CLASS_NAGIOS));
            g_list_free_full(agents, free);
        }
    }
#endif

    return standards;
}

GList *
resources_list_providers(const char *standard)
{
    if (pcmk_is_set(pcmk_get_ra_caps(standard), pcmk_ra_cap_provider)) {
        return resources_os_list_ocf_providers();
    }

    return NULL;
}

GList *
resources_list_agents(const char *standard, const char *provider)
{
    if ((standard == NULL)
        || (strcasecmp(standard, PCMK_RESOURCE_CLASS_SERVICE) == 0)) {

        GList *tmp1;
        GList *tmp2;
        GList *result = services__list_lsb_agents();

        if (standard == NULL) {
            tmp1 = result;
            tmp2 = resources_os_list_ocf_agents(NULL);
            if (tmp2) {
                result = g_list_concat(tmp1, tmp2);
            }
        }
#if SUPPORT_SYSTEMD
        tmp1 = result;
        tmp2 = systemd_unit_listall();
        if (tmp2) {
            result = g_list_concat(tmp1, tmp2);
        }
#endif

#if SUPPORT_UPSTART
        tmp1 = result;
        tmp2 = upstart_job_listall();
        if (tmp2) {
            result = g_list_concat(tmp1, tmp2);
        }
#endif

        return result;

    } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_OCF) == 0) {
        return resources_os_list_ocf_agents(provider);
    } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_LSB) == 0) {
        return services__list_lsb_agents();
#if SUPPORT_SYSTEMD
    } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_SYSTEMD) == 0) {
        return systemd_unit_listall();
#endif
#if SUPPORT_UPSTART
    } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_UPSTART) == 0) {
        return upstart_job_listall();
#endif
#if SUPPORT_NAGIOS
    } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_NAGIOS) == 0) {
        return services__list_nagios_agents();
#endif
    }

    return NULL;
}

gboolean
resources_agent_exists(const char *standard, const char *provider, const char *agent)
{
    GList *standards = NULL;
    GList *providers = NULL;
    GList *iter = NULL;
    gboolean rc = FALSE;
    gboolean has_providers = FALSE;

    standards = resources_list_standards();
    for (iter = standards; iter != NULL; iter = iter->next) {
        if (pcmk__str_eq(iter->data, standard, pcmk__str_none)) {
            rc = TRUE;
            break;
        }
    }

    if (rc == FALSE) {
        goto done;
    }

    rc = FALSE;

    has_providers = pcmk_is_set(pcmk_get_ra_caps(standard), pcmk_ra_cap_provider);
    if (has_providers == TRUE && provider != NULL) {
        providers = resources_list_providers(standard);
        for (iter = providers; iter != NULL; iter = iter->next) {
            if (pcmk__str_eq(iter->data, provider, pcmk__str_none)) {
                rc = TRUE;
                break;
            }
        }
    } else if (has_providers == FALSE && provider == NULL) {
        rc = TRUE;
    }

    if (rc == FALSE) {
        goto done;
    }

    if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
        if (services__lsb_agent_exists(agent)) {
            rc = TRUE;
#if SUPPORT_SYSTEMD
        } else if (systemd_unit_exists(agent)) {
            rc = TRUE;
#endif

#if SUPPORT_UPSTART
        } else if (upstart_job_exists(agent)) {
            rc = TRUE;
#endif
        } else {
            rc = FALSE;
        }

    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
        rc = services__ocf_agent_exists(provider, agent);

    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)) {
        rc = services__lsb_agent_exists(agent);

#if SUPPORT_SYSTEMD
    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SYSTEMD, pcmk__str_casei)) {
        rc = systemd_unit_exists(agent);
#endif

#if SUPPORT_UPSTART
    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_casei)) {
        rc = upstart_job_exists(agent);
#endif

#if SUPPORT_NAGIOS
    } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
        rc = services__nagios_agent_exists(agent);
#endif

    } else {
        rc = FALSE;
    }

done:
    g_list_free(standards);
    g_list_free(providers);
    return rc;
}

/*!
 * \internal
 * \brief Set the result of an action
 *
 * \param[out] action        Where to set action result
 * \param[in]  agent_status  Exit status to set
 * \param[in]  exec_status   Execution status to set
 * \param[in]  reason        Human-friendly description of event to set
 */
void
services__set_result(svc_action_t *action, int agent_status,
                     enum pcmk_exec_status exec_status, const char *reason)
{
    if (action == NULL) {
        return;
    }

    action->rc = agent_status;
    action->status = exec_status;

    if (!pcmk__str_eq(action->opaque->exit_reason, reason,
                      pcmk__str_none)) {
        free(action->opaque->exit_reason);
        action->opaque->exit_reason = (reason == NULL)? NULL : strdup(reason);
    }
}

/*!
 * \internal
 * \brief Set the result of an action, with a formatted exit reason
 *
 * \param[out] action        Where to set action result
 * \param[in]  agent_status  Exit status to set
 * \param[in]  exec_status   Execution status to set
 * \param[in]  format        printf-style format for a human-friendly
 *                           description of reason for result
 * \param[in]  ...           arguments for \p format
 */
void
services__format_result(svc_action_t *action, int agent_status,
                        enum pcmk_exec_status exec_status,
                        const char *format, ...)
{
    va_list ap;
    int len = 0;
    char *reason = NULL;

    if (action == NULL) {
        return;
    }

    action->rc = agent_status;
    action->status = exec_status;

    if (format != NULL) {
        va_start(ap, format);
        len = vasprintf(&reason, format, ap);
        CRM_ASSERT(len > 0);
        va_end(ap);
    }
    free(action->opaque->exit_reason);
    action->opaque->exit_reason = reason;
}

/*!
 * \internal
 * \brief Set the result of an action to cancelled
 *
 * \param[out] action        Where to set action result
 *
 * \note This sets execution status but leaves the exit status unchanged
 */
void
services__set_cancelled(svc_action_t *action)
{
    if (action != NULL) {
        action->status = PCMK_EXEC_CANCELLED;
        free(action->opaque->exit_reason);
        action->opaque->exit_reason = NULL;
    }
}

/*!
 * \internal
 * \brief Get a readable description of what an action is for
 *
 * \param[in] action  Action to check
 *
 * \return Readable name for the kind of \p action
 */
const char *
services__action_kind(const svc_action_t *action)
{
    if ((action == NULL) || (action->standard == NULL)) {
        return "Process";
    } else if (pcmk__str_eq(action->standard, PCMK_RESOURCE_CLASS_STONITH,
                            pcmk__str_none)) {
        return "Fence agent";
    } else if (pcmk__str_eq(action->standard, PCMK_RESOURCE_CLASS_ALERT,
                            pcmk__str_none)) {
        return "Alert agent";
    } else {
        return "Resource agent";
    }
}

/*!
 * \internal
 * \brief Get the exit reason of an action
 *
 * \param[in] action  Action to check
 *
 * \return Action's exit reason (or NULL if none)
 */
const char *
services__exit_reason(const svc_action_t *action)
{
    return action->opaque->exit_reason;
}

/*!
 * \internal
 * \brief Steal stdout from an action
 *
 * \param[in,out] action  Action whose stdout is desired
 *
 * \return Action's stdout (which may be NULL)
 * \note Upon return, \p action will no longer track the output, so it is the
 *       caller's responsibility to free the return value.
 */
char *
services__grab_stdout(svc_action_t *action)
{
    char *output = action->stdout_data;

    action->stdout_data = NULL;
    return output;
}

/*!
 * \internal
 * \brief Steal stderr from an action
 *
 * \param[in,out] action  Action whose stderr is desired
 *
 * \return Action's stderr (which may be NULL)
 * \note Upon return, \p action will no longer track the output, so it is the
 *       caller's responsibility to free the return value.
 */
char *
services__grab_stderr(svc_action_t *action)
{
    char *output = action->stderr_data;

    action->stderr_data = NULL;
    return output;
}