Adding upstream version 2.1.7.upstream/2.1.7

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

9 files changed, 599 insertions, 1028 deletions

diff --git a/include/crm/pengine/Makefile.am b/include/crm/pengine/Makefile.am
index fac6031..3560d24 100644
--- a/include/crm/pengine/Makefile.am
+++ b/include/crm/pengine/Makefile.am
@@ -1,5 +1,5 @@
 #
-# Copyright 2006-2021 the Pacemaker project contributors
+# Copyright 2006-2023 the Pacemaker project contributors
 #
 # The version control history for this file may have further details.
 #
@@ -10,8 +10,13 @@ MAINTAINERCLEANFILES    = Makefile.in
 
 headerdir=$(pkgincludedir)/crm/pengine
 
-noinst_HEADERS		= internal.h remote_internal.h rules_internal.h
-header_HEADERS		= common.h complex.h pe_types.h rules.h status.h \
+noinst_HEADERS		= internal.h 			\
+			  $(wildcard *_internal.h)
+header_HEADERS		= common.h 		\
+			  complex.h 		\
+			  pe_types.h 		\
+			  rules.h 		\
+			  status.h 		\
 			  common_compat.h	\
 			  pe_types_compat.h	\
 			  rules_compat.h
diff --git a/include/crm/pengine/common.h b/include/crm/pengine/common.h
index 9fe05bd..2feac8a 100644
--- a/include/crm/pengine/common.h
+++ b/include/crm/pengine/common.h
@@ -13,6 +13,7 @@
 #  include <glib.h>
 #  include <regex.h>
 #  include <crm/common/iso8601.h>
+#  include <crm/common/scheduler.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -21,120 +22,6 @@ extern "C" {
 extern gboolean was_processing_error;
 extern gboolean was_processing_warning;
 
-/* The order is (partially) significant here; the values from action_fail_ignore
- * through action_fail_fence are in order of increasing severity.
- *
- * @COMPAT The values should be ordered and numbered per the "TODO" comments
- *         below, so all values are in order of severity and there is room for
- *         future additions, but that would break API compatibility.
- * @TODO   For now, we just use a function to compare the values specially, but
- *         at the next compatibility break, we should arrange things properly.
- */
-enum action_fail_response {
-    action_fail_ignore,     // @TODO = 10
-    // @TODO action_fail_demote = 20,
-    action_fail_recover,    // @TODO = 30
-    // @TODO action_fail_reset_remote = 40,
-    // @TODO action_fail_restart_container = 50,
-    action_fail_migrate,    // @TODO = 60
-    action_fail_block,      // @TODO = 70
-    action_fail_stop,       // @TODO = 80
-    action_fail_standby,    // @TODO = 90
-    action_fail_fence,      // @TODO = 100
-
-    // @COMPAT Values below here are out of order for API compatibility
-
-    action_fail_restart_container,
-
-    /* This is reserved for internal use for remote node connection resources.
-     * Fence the remote node if stonith is enabled, otherwise attempt to recover
-     * the connection resource. This allows us to specify types of connection
-     * resource failures that should result in fencing the remote node
-     * (for example, recurring monitor failures).
-     */
-    action_fail_reset_remote,
-
-    action_fail_demote,
-};
-
-/* the "done" action must be the "pre" action +1 */
-enum action_tasks {
-    no_action,
-    monitor_rsc,
-    stop_rsc,
-    stopped_rsc,
-    start_rsc,
-    started_rsc,
-    action_notify,
-    action_notified,
-    action_promote,
-    action_promoted,
-    action_demote,
-    action_demoted,
-    shutdown_crm,
-    stonith_node
-};
-
-enum rsc_recovery_type {
-    recovery_stop_start,
-    recovery_stop_only,
-    recovery_block,
-    recovery_stop_unexpected,
-};
-
-enum rsc_start_requirement {
-    rsc_req_nothing,            /* Allowed by custom_action() */
-    rsc_req_quorum,             /* Enforced by custom_action() */
-    rsc_req_stonith             /* Enforced by native_start_constraints() */
-};
-
-//! Possible roles that a resource can be in
-enum rsc_role_e {
-    RSC_ROLE_UNKNOWN    = 0,
-    RSC_ROLE_STOPPED    = 1,
-    RSC_ROLE_STARTED    = 2,
-    RSC_ROLE_UNPROMOTED = 3,
-    RSC_ROLE_PROMOTED   = 4,
-
-#if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
-    //! \deprecated Use RSC_ROLE_UNPROMOTED instead
-    RSC_ROLE_SLAVE      = RSC_ROLE_UNPROMOTED,
-
-    //! \deprecated Use RSC_ROLE_PROMOTED instead
-    RSC_ROLE_MASTER     = RSC_ROLE_PROMOTED,
-#endif
-};
-
-#  define RSC_ROLE_MAX  (RSC_ROLE_PROMOTED + 1)
-
-#  define RSC_ROLE_UNKNOWN_S "Unknown"
-#  define RSC_ROLE_STOPPED_S "Stopped"
-#  define RSC_ROLE_STARTED_S "Started"
-#  define RSC_ROLE_UNPROMOTED_S         "Unpromoted"
-#  define RSC_ROLE_PROMOTED_S           "Promoted"
-#  define RSC_ROLE_UNPROMOTED_LEGACY_S  "Slave"
-#  define RSC_ROLE_PROMOTED_LEGACY_S    "Master"
-
-//! Deprecated
-enum pe_print_options {
-    pe_print_log            = (1 << 0),
-    pe_print_html           = (1 << 1),
-    pe_print_ncurses        = (1 << 2),
-    pe_print_printf         = (1 << 3),
-    pe_print_dev            = (1 << 4),  //! Ignored
-    pe_print_details        = (1 << 5),  //! Ignored
-    pe_print_max_details    = (1 << 6),  //! Ignored
-    pe_print_rsconly        = (1 << 7),
-    pe_print_ops            = (1 << 8),
-    pe_print_suppres_nl     = (1 << 9),
-    pe_print_xml            = (1 << 10),
-    pe_print_brief          = (1 << 11),
-    pe_print_pending        = (1 << 12),
-    pe_print_clone_details  = (1 << 13),
-    pe_print_clone_active   = (1 << 14), // Print clone instances only if active
-    pe_print_implicit       = (1 << 15)  // Print implicitly created resources
-};
-
 const char *task2text(enum action_tasks task);
 enum action_tasks text2task(const char *task);
 enum rsc_role_e text2role(const char *role);
@@ -154,13 +41,13 @@ static inline const char *
 recovery2text(enum rsc_recovery_type type)
 {
     switch (type) {
-        case recovery_stop_only:
+        case pcmk_multiply_active_stop:
             return "shutting it down";
-        case recovery_stop_start:
+        case pcmk_multiply_active_restart:
             return "attempting recovery";
-        case recovery_block:
+        case pcmk_multiply_active_block:
             return "waiting for an administrator";
-        case recovery_stop_unexpected:
+        case pcmk_multiply_active_unexpected:
             return "stopping unexpected instances";
     }
     return "Unknown";
diff --git a/include/crm/pengine/common_compat.h b/include/crm/pengine/common_compat.h
index 773bb3d..4330ccf 100644
--- a/include/crm/pengine/common_compat.h
+++ b/include/crm/pengine/common_compat.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2004-2021 the Pacemaker project contributors
+ * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
@@ -10,6 +10,8 @@
 #ifndef PCMK__CRM_PENGINE_COMMON_COMPAT__H
 #  define PCMK__CRM_PENGINE_COMMON_COMPAT__H
 
+#include <crm/common/scheduler.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -23,12 +25,35 @@ extern "C" {
  *             release.
  */
 
-//! \deprecated Use RSC_ROLE_UNPROMOTED_LEGACY_S instead
-#  define RSC_ROLE_SLAVE_S   RSC_ROLE_UNPROMOTED_LEGACY_S
+//! \deprecated Use (pcmk_role_promoted + 1) instead
+#define RSC_ROLE_MAX    (pcmk_role_promoted + 1)
+
+//! \deprecated Use role2text(pcmk_role_unknown) instead
+#define RSC_ROLE_UNKNOWN_S      role2text(pcmk_role_unknown)
+
+//! \deprecated Use role2text(pcmk_role_stopped) instead
+#define RSC_ROLE_STOPPED_S      role2text(pcmk_role_stopped)
+
+//! \deprecated Use role2text(pcmk_role_started) instead
+#define RSC_ROLE_STARTED_S      role2text(pcmk_role_started)
+
+//! \deprecated Use role2text(pcmk_role_unpromoted) instead
+#define RSC_ROLE_UNPROMOTED_S   role2text(pcmk_role_unpromoted)
+
+//! \deprecated Use role2text(pcmk_role_promoted) instead
+#define RSC_ROLE_PROMOTED_S     role2text(pcmk_role_promoted)
+
+//! \deprecated Do not use
+#define RSC_ROLE_UNPROMOTED_LEGACY_S    "Slave"
+
+//! \deprecated Do not use
+#define RSC_ROLE_SLAVE_S                RSC_ROLE_UNPROMOTED_LEGACY_S
 
-//! \deprecated Use RSC_ROLE_PROMOTED_LEGACY_S instead
-#  define RSC_ROLE_MASTER_S  RSC_ROLE_PROMOTED_LEGACY_S
+//! \deprecated Do not use
+#define RSC_ROLE_PROMOTED_LEGACY_S      "Master"
 
+//! \deprecated Do not use
+#define RSC_ROLE_MASTER_S               RSC_ROLE_PROMOTED_LEGACY_S
 
 #ifdef __cplusplus
 }
diff --git a/include/crm/pengine/complex.h b/include/crm/pengine/complex.h
index 929e4da..9b6ad1b 100644
--- a/include/crm/pengine/complex.h
+++ b/include/crm/pengine/complex.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2004-2022 the Pacemaker project contributors
+ * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
@@ -12,23 +12,23 @@
 
 #include <glib.h>                   // gboolean, GHashTable
 #include <libxml/tree.h>            // xmlNode
-#include <crm/pengine/pe_types.h>   // pe_node_t, pe_resource_t, etc.
+#include <crm/pengine/pe_types.h>   // pcmk_node_t, pcmk_resource_t, etc.
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-extern resource_object_functions_t resource_class_functions[];
+extern pcmk_rsc_methods_t resource_class_functions[];
 
-GHashTable *pe_rsc_params(pe_resource_t *rsc, const pe_node_t *node,
-                          pe_working_set_t *data_set);
-void get_meta_attributes(GHashTable * meta_hash, pe_resource_t *rsc,
-                         pe_node_t *node, pe_working_set_t *data_set);
-void get_rsc_attributes(GHashTable *meta_hash, const pe_resource_t *rsc,
-                        const pe_node_t *node, pe_working_set_t *data_set);
+GHashTable *pe_rsc_params(pcmk_resource_t *rsc, const pcmk_node_t *node,
+                          pcmk_scheduler_t *scheduler);
+void get_meta_attributes(GHashTable * meta_hash, pcmk_resource_t *rsc,
+                         pcmk_node_t *node, pcmk_scheduler_t *scheduler);
+void get_rsc_attributes(GHashTable *meta_hash, const pcmk_resource_t *rsc,
+                        const pcmk_node_t *node, pcmk_scheduler_t *scheduler);
 
-gboolean is_parent(pe_resource_t *child, pe_resource_t *rsc);
-pe_resource_t *uber_parent(pe_resource_t *rsc);
+gboolean is_parent(pcmk_resource_t *child, pcmk_resource_t *rsc);
+pcmk_resource_t *uber_parent(pcmk_resource_t *rsc);
 
 #ifdef __cplusplus
 }
diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
index 1b5f6f1..9c8068f 100644
--- a/include/crm/pengine/internal.h
+++ b/include/crm/pengine/internal.h
@@ -10,6 +10,7 @@
 #ifndef PE_INTERNAL__H
 #  define PE_INTERNAL__H
 
+#  include <stdbool.h>
 #  include <stdint.h>
 #  include <string.h>
 #  include <crm/msg_xml.h>
@@ -18,31 +19,17 @@
 #  include <crm/common/internal.h>
 #  include <crm/common/options_internal.h>
 #  include <crm/common/output_internal.h>
+#  include <crm/common/scheduler_internal.h>
 
-const char *pe__resource_description(const pe_resource_t *rsc, uint32_t show_opts);
+const char *pe__resource_description(const pcmk_resource_t *rsc,
+                                     uint32_t show_opts);
 
-enum pe__clone_flags {
-    // Whether instances should be started sequentially
-    pe__clone_ordered               = (1 << 0),
+bool pe__clone_is_ordered(const pcmk_resource_t *clone);
+int pe__set_clone_flag(pcmk_resource_t *clone, enum pcmk__clone_flags flag);
+bool pe__clone_flag_is_set(const pcmk_resource_t *clone, uint32_t flags);
 
-    // Whether promotion scores have been added
-    pe__clone_promotion_added       = (1 << 1),
-
-    // Whether promotion constraints have been added
-    pe__clone_promotion_constrained = (1 << 2),
-};
-
-bool pe__clone_is_ordered(const pe_resource_t *clone);
-int pe__set_clone_flag(pe_resource_t *clone, enum pe__clone_flags flag);
-
-
-enum pe__group_flags {
-    pe__group_ordered       = (1 << 0), // Members start sequentially
-    pe__group_colocated     = (1 << 1), // Members must be on same node
-};
-
-bool pe__group_flag_is_set(const pe_resource_t *group, uint32_t flags);
-pe_resource_t *pe__last_group_member(const pe_resource_t *group);
+bool pe__group_flag_is_set(const pcmk_resource_t *group, uint32_t flags);
+pcmk_resource_t *pe__last_group_member(const pcmk_resource_t *group);
 
 
 #  define pe_rsc_info(rsc, fmt, args...)  crm_log_tag(LOG_INFO,  rsc ? rsc->id : "<NULL>", fmt, ##args)
@@ -62,16 +49,16 @@ pe_resource_t *pe__last_group_member(const pe_resource_t *group);
 #  define pe_proc_err(fmt...) { was_processing_error = TRUE; crm_err(fmt); }
 #  define pe_proc_warn(fmt...) { was_processing_warning = TRUE; crm_warn(fmt); }
 
-#define pe__set_working_set_flags(working_set, flags_to_set) do {           \
-        (working_set)->flags = pcmk__set_flags_as(__func__, __LINE__,       \
-            LOG_TRACE, "Working set", crm_system_name,                      \
-            (working_set)->flags, (flags_to_set), #flags_to_set);           \
+#define pe__set_working_set_flags(scheduler, flags_to_set) do {             \
+        (scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__,         \
+            LOG_TRACE, "Scheduler", crm_system_name,                        \
+            (scheduler)->flags, (flags_to_set), #flags_to_set);             \
     } while (0)
 
-#define pe__clear_working_set_flags(working_set, flags_to_clear) do {       \
-        (working_set)->flags = pcmk__clear_flags_as(__func__, __LINE__,     \
-            LOG_TRACE, "Working set", crm_system_name,                      \
-            (working_set)->flags, (flags_to_clear), #flags_to_clear);       \
+#define pe__clear_working_set_flags(scheduler, flags_to_clear) do {         \
+        (scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__,       \
+            LOG_TRACE, "Scheduler", crm_system_name,                        \
+            (scheduler)->flags, (flags_to_clear), #flags_to_clear);         \
     } while (0)
 
 #define pe__set_resource_flags(resource, flags_to_set) do {                 \
@@ -152,144 +139,127 @@ pe_resource_t *pe__last_group_member(const pe_resource_t *group);
                                            #flags_to_clear);                  \
     } while (0)
 
-// Some warnings we don't want to print every transition
-
-enum pe_warn_once_e {
-    pe_wo_blind         = (1 << 0),
-    pe_wo_restart_type  = (1 << 1),
-    pe_wo_role_after    = (1 << 2),
-    pe_wo_poweroff      = (1 << 3),
-    pe_wo_require_all   = (1 << 4),
-    pe_wo_order_score   = (1 << 5),
-    pe_wo_neg_threshold = (1 << 6),
-    pe_wo_remove_after  = (1 << 7),
-    pe_wo_ping_node     = (1 << 8),
-    pe_wo_order_inst    = (1 << 9),
-    pe_wo_coloc_inst    = (1 << 10),
-    pe_wo_group_order   = (1 << 11),
-    pe_wo_group_coloc   = (1 << 12),
-    pe_wo_upstart       = (1 << 13),
-    pe_wo_nagios        = (1 << 14),
-};
-
-extern uint32_t pe_wo;
-
 #define pe_warn_once(pe_wo_bit, fmt...) do {    \
-        if (!pcmk_is_set(pe_wo, pe_wo_bit)) {  \
-            if (pe_wo_bit == pe_wo_blind) {     \
+        if (!pcmk_is_set(pcmk__warnings, pe_wo_bit)) {  \
+            if (pe_wo_bit == pcmk__wo_blind) {  \
                 crm_warn(fmt);                  \
             } else {                            \
                 pe_warn(fmt);                   \
             }                                   \
-            pe_wo = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,       \
-                                      "Warn-once", "logging", pe_wo,        \
-                                      (pe_wo_bit), #pe_wo_bit);             \
-        }                                       \
+            pcmk__warnings = pcmk__set_flags_as(__func__, __LINE__,         \
+                                               LOG_TRACE,                   \
+                                               "Warn-once", "logging",      \
+                                               pcmk__warnings,              \
+                                               (pe_wo_bit), #pe_wo_bit);    \
+        }                                                                   \
     } while (0);
 
 
 typedef struct pe__location_constraint_s {
     char *id;                           // Constraint XML ID
-    pe_resource_t *rsc_lh;              // Resource being located
+    pcmk_resource_t *rsc_lh;            // Resource being located
     enum rsc_role_e role_filter;        // Role to locate
     enum pe_discover_e discover_mode;   // Resource discovery
-    GList *node_list_rh;              // List of pe_node_t*
+    GList *node_list_rh;                // List of pcmk_node_t*
 } pe__location_t;
 
 typedef struct pe__order_constraint_s {
     int id;
-    uint32_t flags; // Group of enum pe_ordering flags
+    uint32_t flags; // Group of enum pcmk__action_relation_flags
 
     void *lh_opaque;
-    pe_resource_t *lh_rsc;
-    pe_action_t *lh_action;
+    pcmk_resource_t *lh_rsc;
+    pcmk_action_t *lh_action;
     char *lh_action_task;
 
     void *rh_opaque;
-    pe_resource_t *rh_rsc;
-    pe_action_t *rh_action;
+    pcmk_resource_t *rh_rsc;
+    pcmk_action_t *rh_action;
     char *rh_action_task;
 } pe__ordering_t;
 
-const pe_resource_t *pe__const_top_resource(const pe_resource_t *rsc,
-                                            bool include_bundle);
+const pcmk_resource_t *pe__const_top_resource(const pcmk_resource_t *rsc,
+                                              bool include_bundle);
 
-int pe__clone_max(const pe_resource_t *clone);
-int pe__clone_node_max(const pe_resource_t *clone);
-int pe__clone_promoted_max(const pe_resource_t *clone);
-int pe__clone_promoted_node_max(const pe_resource_t *clone);
-void pe__create_clone_notifications(pe_resource_t *clone);
-void pe__free_clone_notification_data(pe_resource_t *clone);
-void pe__create_clone_notif_pseudo_ops(pe_resource_t *clone,
-                                       pe_action_t *start, pe_action_t *started,
-                                       pe_action_t *stop, pe_action_t *stopped);
+int pe__clone_max(const pcmk_resource_t *clone);
+int pe__clone_node_max(const pcmk_resource_t *clone);
+int pe__clone_promoted_max(const pcmk_resource_t *clone);
+int pe__clone_promoted_node_max(const pcmk_resource_t *clone);
+void pe__create_clone_notifications(pcmk_resource_t *clone);
+void pe__free_clone_notification_data(pcmk_resource_t *clone);
+void pe__create_clone_notif_pseudo_ops(pcmk_resource_t *clone,
+                                       pcmk_action_t *start,
+                                       pcmk_action_t *started,
+                                       pcmk_action_t *stop,
+                                       pcmk_action_t *stopped);
 
+pcmk_action_t *pe__new_rsc_pseudo_action(pcmk_resource_t *rsc, const char *task,
+                                         bool optional, bool runnable);
 
-pe_action_t *pe__new_rsc_pseudo_action(pe_resource_t *rsc, const char *task,
-                                       bool optional, bool runnable);
+void pe__create_promotable_pseudo_ops(pcmk_resource_t *clone,
+                                      bool any_promoting, bool any_demoting);
 
-void pe__create_promotable_pseudo_ops(pe_resource_t *clone, bool any_promoting,
-                                      bool any_demoting);
-
-bool pe_can_fence(const pe_working_set_t *data_set, const pe_node_t *node);
+bool pe_can_fence(const pcmk_scheduler_t *scheduler, const pcmk_node_t *node);
 
 void add_hash_param(GHashTable * hash, const char *name, const char *value);
 
-char *native_parameter(pe_resource_t * rsc, pe_node_t * node, gboolean create, const char *name,
-                       pe_working_set_t * data_set);
-pe_node_t *native_location(const pe_resource_t *rsc, GList **list, int current);
+char *native_parameter(pcmk_resource_t *rsc, pcmk_node_t *node, gboolean create,
+                       const char *name, pcmk_scheduler_t *scheduler);
+pcmk_node_t *native_location(const pcmk_resource_t *rsc, GList **list,
+                             int current);
 
 void pe_metadata(pcmk__output_t *out);
 void verify_pe_options(GHashTable * options);
 
-void native_add_running(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set, gboolean failed);
+void native_add_running(pcmk_resource_t *rsc, pcmk_node_t *node,
+                        pcmk_scheduler_t *scheduler, gboolean failed);
 
-gboolean native_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
-gboolean group_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
-gboolean clone_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
-gboolean pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set);
+gboolean native_unpack(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler);
+gboolean group_unpack(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler);
+gboolean clone_unpack(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler);
+gboolean pe__unpack_bundle(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler);
 
-pe_resource_t *native_find_rsc(pe_resource_t *rsc, const char *id, const pe_node_t *node,
-                               int flags);
+pcmk_resource_t *native_find_rsc(pcmk_resource_t *rsc, const char *id,
+                                 const pcmk_node_t *node, int flags);
 
-gboolean native_active(pe_resource_t * rsc, gboolean all);
-gboolean group_active(pe_resource_t * rsc, gboolean all);
-gboolean clone_active(pe_resource_t * rsc, gboolean all);
-gboolean pe__bundle_active(pe_resource_t *rsc, gboolean all);
+gboolean native_active(pcmk_resource_t *rsc, gboolean all);
+gboolean group_active(pcmk_resource_t *rsc, gboolean all);
+gboolean clone_active(pcmk_resource_t *rsc, gboolean all);
+gboolean pe__bundle_active(pcmk_resource_t *rsc, gboolean all);
 
 //! \deprecated This function will be removed in a future release
-void native_print(pe_resource_t *rsc, const char *pre_text, long options,
+void native_print(pcmk_resource_t *rsc, const char *pre_text, long options,
                   void *print_data);
 
 //! \deprecated This function will be removed in a future release
-void group_print(pe_resource_t *rsc, const char *pre_text, long options,
+void group_print(pcmk_resource_t *rsc, const char *pre_text, long options,
                  void *print_data);
 
 //! \deprecated This function will be removed in a future release
-void clone_print(pe_resource_t *rsc, const char *pre_text, long options,
+void clone_print(pcmk_resource_t *rsc, const char *pre_text, long options,
                  void *print_data);
 
 //! \deprecated This function will be removed in a future release
-void pe__print_bundle(pe_resource_t *rsc, const char *pre_text, long options,
+void pe__print_bundle(pcmk_resource_t *rsc, const char *pre_text, long options,
                       void *print_data);
 
-gchar *pcmk__native_output_string(const pe_resource_t *rsc, const char *name,
-                                  const pe_node_t *node, uint32_t show_opts,
+gchar *pcmk__native_output_string(const pcmk_resource_t *rsc, const char *name,
+                                  const pcmk_node_t *node, uint32_t show_opts,
                                   const char *target_role, bool show_nodes);
 
 int pe__name_and_nvpairs_xml(pcmk__output_t *out, bool is_list, const char *tag_name
                          , size_t pairs_count, ...);
-char *pe__node_display_name(pe_node_t *node, bool print_detail);
+char *pe__node_display_name(pcmk_node_t *node, bool print_detail);
 
 
 // Clone notifications (pe_notif.c)
-void pe__order_notifs_after_fencing(const pe_action_t *action,
-                                    pe_resource_t *rsc,
-                                    pe_action_t *stonith_op);
+void pe__order_notifs_after_fencing(const pcmk_action_t *action,
+                                    pcmk_resource_t *rsc,
+                                    pcmk_action_t *stonith_op);
 
 
 static inline const char *
-pe__rsc_bool_str(const pe_resource_t *rsc, uint64_t rsc_flag)
+pe__rsc_bool_str(const pcmk_resource_t *rsc, uint64_t rsc_flag)
 {
     return pcmk__btoa(pcmk_is_set(rsc->flags, rsc_flag));
 }
@@ -308,167 +278,156 @@ int pe__resource_xml(pcmk__output_t *out, va_list args);
 int pe__resource_html(pcmk__output_t *out, va_list args);
 int pe__resource_text(pcmk__output_t *out, va_list args);
 
-void native_free(pe_resource_t * rsc);
-void group_free(pe_resource_t * rsc);
-void clone_free(pe_resource_t * rsc);
-void pe__free_bundle(pe_resource_t *rsc);
-
-enum rsc_role_e native_resource_state(const pe_resource_t * rsc, gboolean current);
-enum rsc_role_e group_resource_state(const pe_resource_t * rsc, gboolean current);
-enum rsc_role_e clone_resource_state(const pe_resource_t * rsc, gboolean current);
-enum rsc_role_e pe__bundle_resource_state(const pe_resource_t *rsc,
+void native_free(pcmk_resource_t *rsc);
+void group_free(pcmk_resource_t *rsc);
+void clone_free(pcmk_resource_t *rsc);
+void pe__free_bundle(pcmk_resource_t *rsc);
+
+enum rsc_role_e native_resource_state(const pcmk_resource_t *rsc,
+                                      gboolean current);
+enum rsc_role_e group_resource_state(const pcmk_resource_t *rsc,
+                                     gboolean current);
+enum rsc_role_e clone_resource_state(const pcmk_resource_t *rsc,
+                                     gboolean current);
+enum rsc_role_e pe__bundle_resource_state(const pcmk_resource_t *rsc,
                                           gboolean current);
 
-void pe__count_common(pe_resource_t *rsc);
-void pe__count_bundle(pe_resource_t *rsc);
+void pe__count_common(pcmk_resource_t *rsc);
+void pe__count_bundle(pcmk_resource_t *rsc);
 
-void common_free(pe_resource_t * rsc);
+void common_free(pcmk_resource_t *rsc);
 
-pe_node_t *pe__copy_node(const pe_node_t *this_node);
-extern time_t get_effective_time(pe_working_set_t * data_set);
+pcmk_node_t *pe__copy_node(const pcmk_node_t *this_node);
+time_t get_effective_time(pcmk_scheduler_t *scheduler);
 
 /* Failure handling utilities (from failcounts.c) */
 
-// bit flags for fail count handling options
-enum pe_fc_flags_e {
-    pe_fc_default   = (1 << 0),
-    pe_fc_effective = (1 << 1), // don't count expired failures
-    pe_fc_fillers   = (1 << 2), // if container, include filler failures in count
-};
-
-int pe_get_failcount(const pe_node_t *node, pe_resource_t *rsc,
+int pe_get_failcount(const pcmk_node_t *node, pcmk_resource_t *rsc,
                      time_t *last_failure, uint32_t flags,
                      const xmlNode *xml_op);
 
-pe_action_t *pe__clear_failcount(pe_resource_t *rsc, const pe_node_t *node,
-                                 const char *reason,
-                                 pe_working_set_t *data_set);
+pcmk_action_t *pe__clear_failcount(pcmk_resource_t *rsc,
+                                   const pcmk_node_t *node, const char *reason,
+                                   pcmk_scheduler_t *scheduler);
 
 /* Functions for finding/counting a resource's active nodes */
 
-bool pe__count_active_node(const pe_resource_t *rsc, pe_node_t *node,
-                           pe_node_t **active, unsigned int *count_all,
+bool pe__count_active_node(const pcmk_resource_t *rsc, pcmk_node_t *node,
+                           pcmk_node_t **active, unsigned int *count_all,
                            unsigned int *count_clean);
 
-pe_node_t *pe__find_active_requires(const pe_resource_t *rsc,
+pcmk_node_t *pe__find_active_requires(const pcmk_resource_t *rsc,
                                     unsigned int *count);
 
-static inline pe_node_t *
-pe__current_node(const pe_resource_t *rsc)
+static inline pcmk_node_t *
+pe__current_node(const pcmk_resource_t *rsc)
 {
     return (rsc == NULL)? NULL : rsc->fns->active_node(rsc, NULL, NULL);
 }
 
 
 /* Binary like operators for lists of nodes */
-extern void node_list_exclude(GHashTable * list, GList *list2, gboolean merge_scores);
-
 GHashTable *pe__node_list2table(const GList *list);
 
-static inline gpointer
-pe_hash_table_lookup(GHashTable * hash, gconstpointer key)
-{
-    if (hash) {
-        return g_hash_table_lookup(hash, key);
-    }
-    return NULL;
-}
-
-extern pe_action_t *get_pseudo_op(const char *name, pe_working_set_t * data_set);
-extern gboolean order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order);
-
-void pe__show_node_weights_as(const char *file, const char *function,
-                              int line, bool to_log, const pe_resource_t *rsc,
-                              const char *comment, GHashTable *nodes,
-                              pe_working_set_t *data_set);
-
-#define pe__show_node_weights(level, rsc, text, nodes, data_set)    \
-        pe__show_node_weights_as(__FILE__, __func__, __LINE__,      \
-                                 (level), (rsc), (text), (nodes), (data_set))
-
-xmlNode *find_rsc_op_entry(const pe_resource_t *rsc, const char *key);
-
-pe_action_t *custom_action(pe_resource_t *rsc, char *key, const char *task,
-                           const pe_node_t *on_node, gboolean optional,
-                           gboolean foo, pe_working_set_t *data_set);
-
-#  define delete_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DELETE, 0)
+pcmk_action_t *get_pseudo_op(const char *name, pcmk_scheduler_t *scheduler);
+gboolean order_actions(pcmk_action_t *lh_action, pcmk_action_t *rh_action,
+                       uint32_t flags);
+
+void pe__show_node_scores_as(const char *file, const char *function,
+                             int line, bool to_log, const pcmk_resource_t *rsc,
+                             const char *comment, GHashTable *nodes,
+                             pcmk_scheduler_t *scheduler);
+
+#define pe__show_node_scores(level, rsc, text, nodes, scheduler)    \
+        pe__show_node_scores_as(__FILE__, __func__, __LINE__,      \
+                                (level), (rsc), (text), (nodes), (scheduler))
+
+GHashTable *pcmk__unpack_action_meta(pcmk_resource_t *rsc,
+                                     const pcmk_node_t *node,
+                                     const char *action_name, guint interval_ms,
+                                     const xmlNode *action_config);
+GHashTable *pcmk__unpack_action_rsc_params(const xmlNode *action_xml,
+                                           GHashTable *node_attrs,
+                                           pcmk_scheduler_t *data_set);
+xmlNode *pcmk__find_action_config(const pcmk_resource_t *rsc,
+                                  const char *action_name, guint interval_ms,
+                                  bool include_disabled);
+
+enum rsc_start_requirement pcmk__action_requires(const pcmk_resource_t *rsc,
+                                                 const char *action_name);
+
+enum action_fail_response pcmk__parse_on_fail(const pcmk_resource_t *rsc,
+                                              const char *action_name,
+                                              guint interval_ms,
+                                              const char *value);
+
+enum rsc_role_e pcmk__role_after_failure(const pcmk_resource_t *rsc,
+                                         const char *action_name,
+                                         enum action_fail_response on_fail,
+                                         GHashTable *meta);
+
+pcmk_action_t *custom_action(pcmk_resource_t *rsc, char *key, const char *task,
+                             const pcmk_node_t *on_node, gboolean optional,
+                             pcmk_scheduler_t *scheduler);
+
+#  define delete_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_DELETE, 0)
 #  define delete_action(rsc, node, optional) custom_action(		\
-		rsc, delete_key(rsc), CRMD_ACTION_DELETE, node,		\
-		optional, TRUE, rsc->cluster);
-
-#  define stopped_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOPPED, 0)
-#  define stopped_action(rsc, node, optional) custom_action(		\
-		rsc, stopped_key(rsc), CRMD_ACTION_STOPPED, node,	\
-		optional, TRUE, rsc->cluster);
+		rsc, delete_key(rsc), PCMK_ACTION_DELETE, node, \
+		optional, rsc->cluster);
 
-#  define stop_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOP, 0)
+#  define stop_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0)
 #  define stop_action(rsc, node, optional) custom_action(			\
-		rsc, stop_key(rsc), CRMD_ACTION_STOP, node,		\
-		optional, TRUE, rsc->cluster);
+		rsc, stop_key(rsc), PCMK_ACTION_STOP, node,		\
+		optional, rsc->cluster);
 
-#  define reload_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_RELOAD_AGENT, 0)
-#  define start_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_START, 0)
+#  define reload_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_RELOAD_AGENT, 0)
+#  define start_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_START, 0)
 #  define start_action(rsc, node, optional) custom_action(		\
-		rsc, start_key(rsc), CRMD_ACTION_START, node,		\
-		optional, TRUE, rsc->cluster)
+		rsc, start_key(rsc), PCMK_ACTION_START, node,           \
+		optional, rsc->cluster)
 
-#  define started_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STARTED, 0)
-#  define started_action(rsc, node, optional) custom_action(		\
-		rsc, started_key(rsc), CRMD_ACTION_STARTED, node,	\
-		optional, TRUE, rsc->cluster)
-
-#  define promote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTE, 0)
+#  define promote_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_PROMOTE, 0)
 #  define promote_action(rsc, node, optional) custom_action(		\
-		rsc, promote_key(rsc), CRMD_ACTION_PROMOTE, node,	\
-		optional, TRUE, rsc->cluster)
-
-#  define promoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTED, 0)
-#  define promoted_action(rsc, node, optional) custom_action(		\
-		rsc, promoted_key(rsc), CRMD_ACTION_PROMOTED, node,	\
-		optional, TRUE, rsc->cluster)
+		rsc, promote_key(rsc), PCMK_ACTION_PROMOTE, node,	\
+		optional, rsc->cluster)
 
-#  define demote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTE, 0)
+#  define demote_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_DEMOTE, 0)
 #  define demote_action(rsc, node, optional) custom_action(		\
-		rsc, demote_key(rsc), CRMD_ACTION_DEMOTE, node,		\
-		optional, TRUE, rsc->cluster)
+		rsc, demote_key(rsc), PCMK_ACTION_DEMOTE, node, \
+		optional, rsc->cluster)
 
-#  define demoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTED, 0)
-#  define demoted_action(rsc, node, optional) custom_action(		\
-		rsc, demoted_key(rsc), CRMD_ACTION_DEMOTED, node,	\
-		optional, TRUE, rsc->cluster)
+extern int pe_get_configured_timeout(pcmk_resource_t *rsc, const char *action,
+                                     pcmk_scheduler_t *scheduler);
 
-extern int pe_get_configured_timeout(pe_resource_t *rsc, const char *action,
-                                     pe_working_set_t *data_set);
+pcmk_action_t *find_first_action(const GList *input, const char *uuid,
+                                 const char *task, const pcmk_node_t *on_node);
 
-pe_action_t *find_first_action(const GList *input, const char *uuid,
-                               const char *task, const pe_node_t *on_node);
+enum action_tasks get_complex_task(const pcmk_resource_t *rsc,
+                                   const char *name);
 
-enum action_tasks get_complex_task(const pe_resource_t *rsc, const char *name);
-
-extern GList *find_actions(GList *input, const char *key, const pe_node_t *on_node);
+GList *find_actions(GList *input, const char *key, const pcmk_node_t *on_node);
 GList *find_actions_exact(GList *input, const char *key,
-                          const pe_node_t *on_node);
-GList *pe__resource_actions(const pe_resource_t *rsc, const pe_node_t *node,
+                          const pcmk_node_t *on_node);
+GList *pe__resource_actions(const pcmk_resource_t *rsc, const pcmk_node_t *node,
                             const char *task, bool require_node);
 
-extern void pe_free_action(pe_action_t * action);
+extern void pe_free_action(pcmk_action_t *action);
 
-void resource_location(pe_resource_t *rsc, const pe_node_t *node, int score,
-                       const char *tag, pe_working_set_t *data_set);
+void resource_location(pcmk_resource_t *rsc, const pcmk_node_t *node, int score,
+                       const char *tag, pcmk_scheduler_t *scheduler);
 
 extern int pe__is_newer_op(const xmlNode *xml_a, const xmlNode *xml_b,
                            bool same_node_default);
 extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
-gboolean get_target_role(const pe_resource_t *rsc, enum rsc_role_e *role);
-void pe__set_next_role(pe_resource_t *rsc, enum rsc_role_e role,
+gboolean get_target_role(const pcmk_resource_t *rsc, enum rsc_role_e *role);
+void pe__set_next_role(pcmk_resource_t *rsc, enum rsc_role_e role,
                        const char *why);
 
-pe_resource_t *find_clone_instance(const pe_resource_t *rsc,
-                                   const char *sub_id);
+pcmk_resource_t *find_clone_instance(const pcmk_resource_t *rsc,
+                                     const char *sub_id);
 
 extern void destroy_ticket(gpointer data);
-extern pe_ticket_t *ticket_new(const char *ticket_id, pe_working_set_t * data_set);
+pcmk_ticket_t *ticket_new(const char *ticket_id, pcmk_scheduler_t *scheduler);
 
 // Resources for manipulating resource names
 const char *pe_base_name_end(const char *id);
@@ -476,7 +435,7 @@ char *clone_strip(const char *last_rsc_id);
 char *clone_zero(const char *last_rsc_id);
 
 static inline bool
-pe_base_name_eq(const pe_resource_t *rsc, const char *id)
+pe_base_name_eq(const pcmk_resource_t *rsc, const char *id)
 {
     if (id && rsc && rsc->id) {
         // Number of characters in rsc->id before any clone suffix
@@ -490,22 +449,10 @@ pe_base_name_eq(const pe_resource_t *rsc, const char *id)
 int pe__target_rc_from_xml(const xmlNode *xml_op);
 
 gint pe__cmp_node_name(gconstpointer a, gconstpointer b);
-bool is_set_recursive(const pe_resource_t *rsc, long long flag, bool any);
-
-enum rsc_digest_cmp_val {
-    /*! Digests are the same */
-    RSC_DIGEST_MATCH = 0,
-    /*! Params that require a restart changed */
-    RSC_DIGEST_RESTART,
-    /*! Some parameter changed.  */
-    RSC_DIGEST_ALL,
-    /*! rsc op didn't have a digest associated with it, so
-     *  it is unknown if parameters changed or not. */
-    RSC_DIGEST_UNKNOWN,
-};
+bool is_set_recursive(const pcmk_resource_t *rsc, long long flag, bool any);
 
 typedef struct op_digest_cache_s {
-    enum rsc_digest_cmp_val rc;
+    enum pcmk__digest_result rc;
     xmlNode *params_all;
     xmlNode *params_secure;
     xmlNode *params_restart;
@@ -514,35 +461,37 @@ typedef struct op_digest_cache_s {
     char *digest_restart_calc;
 } op_digest_cache_t;
 
-op_digest_cache_t *pe__calculate_digests(pe_resource_t *rsc, const char *task,
+op_digest_cache_t *pe__calculate_digests(pcmk_resource_t *rsc, const char *task,
                                          guint *interval_ms,
-                                         const pe_node_t *node,
+                                         const pcmk_node_t *node,
                                          const xmlNode *xml_op,
                                          GHashTable *overrides,
                                          bool calc_secure,
-                                         pe_working_set_t *data_set);
+                                         pcmk_scheduler_t *scheduler);
 
 void pe__free_digests(gpointer ptr);
 
-op_digest_cache_t *rsc_action_digest_cmp(pe_resource_t *rsc,
+op_digest_cache_t *rsc_action_digest_cmp(pcmk_resource_t *rsc,
                                          const xmlNode *xml_op,
-                                         pe_node_t *node,
-                                         pe_working_set_t *data_set);
-
-pe_action_t *pe_fence_op(pe_node_t *node, const char *op, bool optional,
-                         const char *reason, bool priority_delay,
-                         pe_working_set_t *data_set);
-void trigger_unfencing(pe_resource_t *rsc, pe_node_t *node,
-                       const char *reason, pe_action_t *dependency,
-                       pe_working_set_t *data_set);
-
-char *pe__action2reason(const pe_action_t *action, enum pe_action_flags flag);
-void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrite);
-void pe__add_action_expected_result(pe_action_t *action, int expected_result);
-
-void pe__set_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags);
-void pe__clear_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags);
-void pe__clear_resource_flags_on_all(pe_working_set_t *data_set, uint64_t flag);
+                                         pcmk_node_t *node,
+                                         pcmk_scheduler_t *scheduler);
+
+pcmk_action_t *pe_fence_op(pcmk_node_t *node, const char *op, bool optional,
+                           const char *reason, bool priority_delay,
+                           pcmk_scheduler_t *scheduler);
+void trigger_unfencing(pcmk_resource_t *rsc, pcmk_node_t *node,
+                       const char *reason, pcmk_action_t *dependency,
+                       pcmk_scheduler_t *scheduler);
+
+char *pe__action2reason(const pcmk_action_t *action, enum pe_action_flags flag);
+void pe_action_set_reason(pcmk_action_t *action, const char *reason,
+                          bool overwrite);
+void pe__add_action_expected_result(pcmk_action_t *action, int expected_result);
+
+void pe__set_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags);
+void pe__clear_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags);
+void pe__clear_resource_flags_on_all(pcmk_scheduler_t *scheduler,
+                                     uint64_t flag);
 
 gboolean add_tag_ref(GHashTable * tags, const char * tag_name,  const char * obj_ref);
 
@@ -550,49 +499,75 @@ gboolean add_tag_ref(GHashTable * tags, const char * tag_name,  const char * obj
 void print_rscs_brief(GList *rsc_list, const char * pre_text, long options,
                       void * print_data, gboolean print_all);
 int pe__rscs_brief_output(pcmk__output_t *out, GList *rsc_list, unsigned int options);
-void pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, const char *reason, bool priority_delay);
+void pe_fence_node(pcmk_scheduler_t *scheduler, pcmk_node_t *node,
+                   const char *reason, bool priority_delay);
 
-pe_node_t *pe_create_node(const char *id, const char *uname, const char *type,
-                          const char *score, pe_working_set_t * data_set);
+pcmk_node_t *pe_create_node(const char *id, const char *uname, const char *type,
+                            const char *score, pcmk_scheduler_t *scheduler);
 
 //! \deprecated This function will be removed in a future release
-void common_print(pe_resource_t *rsc, const char *pre_text, const char *name,
-                  const pe_node_t *node, long options, void *print_data);
-int pe__common_output_text(pcmk__output_t *out, const pe_resource_t *rsc,
-                           const char *name, const pe_node_t *node,
+void common_print(pcmk_resource_t *rsc, const char *pre_text, const char *name,
+                  const pcmk_node_t *node, long options, void *print_data);
+int pe__common_output_text(pcmk__output_t *out, const pcmk_resource_t *rsc,
+                           const char *name, const pcmk_node_t *node,
                            unsigned int options);
-int pe__common_output_html(pcmk__output_t *out, const pe_resource_t *rsc,
-                           const char *name, const pe_node_t *node,
+int pe__common_output_html(pcmk__output_t *out, const pcmk_resource_t *rsc,
+                           const char *name, const pcmk_node_t *node,
                            unsigned int options);
 
-GList *pe__bundle_containers(const pe_resource_t *bundle);
-
-int pe__bundle_max(const pe_resource_t *rsc);
-int pe__bundle_max_per_node(const pe_resource_t *rsc);
-
-pe_resource_t *pe__find_bundle_replica(const pe_resource_t *bundle,
-                                       const pe_node_t *node);
-bool pe__bundle_needs_remote_name(pe_resource_t *rsc);
-const char *pe__add_bundle_remote_name(pe_resource_t *rsc,
-                                       pe_working_set_t *data_set,
+//! A single instance of a bundle
+typedef struct {
+    int offset;                 //!< 0-origin index of this instance in bundle
+    char *ipaddr;               //!< IP address associated with this instance
+    pcmk_node_t *node;          //!< Node created for this instance
+    pcmk_resource_t *ip;        //!< IP address resource for ipaddr
+    pcmk_resource_t *child;     //!< Instance of bundled resource
+    pcmk_resource_t *container; //!< Container associated with this instance
+    pcmk_resource_t *remote;    //!< Pacemaker Remote connection into container
+} pe__bundle_replica_t;
+
+GList *pe__bundle_containers(const pcmk_resource_t *bundle);
+
+int pe__bundle_max(const pcmk_resource_t *rsc);
+bool pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
+                                 const pcmk_node_t *node);
+pcmk_resource_t *pe__bundled_resource(const pcmk_resource_t *rsc);
+const pcmk_resource_t *pe__get_rsc_in_container(const pcmk_resource_t *instance);
+pcmk_resource_t *pe__first_container(const pcmk_resource_t *bundle);
+void pe__foreach_bundle_replica(pcmk_resource_t *bundle,
+                                bool (*fn)(pe__bundle_replica_t *, void *),
+                                void *user_data);
+void pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
+                                      bool (*fn)(const pe__bundle_replica_t *,
+                                                 void *),
+                                      void *user_data);
+pcmk_resource_t *pe__find_bundle_replica(const pcmk_resource_t *bundle,
+                                         const pcmk_node_t *node);
+bool pe__bundle_needs_remote_name(pcmk_resource_t *rsc);
+const char *pe__add_bundle_remote_name(pcmk_resource_t *rsc,
+                                       pcmk_scheduler_t *scheduler,
                                        xmlNode *xml, const char *field);
-const char *pe_node_attribute_calculated(const pe_node_t *node,
-                                         const char *name,
-                                         const pe_resource_t *rsc);
-const char *pe_node_attribute_raw(const pe_node_t *node, const char *name);
-bool pe__is_universal_clone(const pe_resource_t *rsc,
-                            const pe_working_set_t *data_set);
-void pe__add_param_check(const xmlNode *rsc_op, pe_resource_t *rsc,
-                         pe_node_t *node, enum pe_check_parameters,
-                         pe_working_set_t *data_set);
-void pe__foreach_param_check(pe_working_set_t *data_set,
-                             void (*cb)(pe_resource_t*, pe_node_t*,
+
+const char *pe__node_attribute_calculated(const pcmk_node_t *node,
+                                          const char *name,
+                                          const pcmk_resource_t *rsc,
+                                          enum pcmk__rsc_node node_type,
+                                          bool force_host);
+const char *pe_node_attribute_raw(const pcmk_node_t *node, const char *name);
+bool pe__is_universal_clone(const pcmk_resource_t *rsc,
+                            const pcmk_scheduler_t *scheduler);
+void pe__add_param_check(const xmlNode *rsc_op, pcmk_resource_t *rsc,
+                         pcmk_node_t *node, enum pcmk__check_parameters,
+                         pcmk_scheduler_t *scheduler);
+void pe__foreach_param_check(pcmk_scheduler_t *scheduler,
+                             void (*cb)(pcmk_resource_t*, pcmk_node_t*,
                                         const xmlNode*,
-                                        enum pe_check_parameters));
-void pe__free_param_checks(pe_working_set_t *data_set);
+                                        enum pcmk__check_parameters));
+void pe__free_param_checks(pcmk_scheduler_t *scheduler);
 
-bool pe__shutdown_requested(const pe_node_t *node);
-void pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set);
+bool pe__shutdown_requested(const pcmk_node_t *node);
+void pe__update_recheck_time(time_t recheck, pcmk_scheduler_t *scheduler,
+                             const char *reason);
 
 /*!
  * \internal
@@ -605,53 +580,55 @@ void pe__register_messages(pcmk__output_t *out);
 void pe__unpack_dataset_nvpairs(const xmlNode *xml_obj, const char *set_name,
                                 const pe_rule_eval_data_t *rule_data,
                                 GHashTable *hash, const char *always_first,
-                                gboolean overwrite, pe_working_set_t *data_set);
-
-bool pe__resource_is_disabled(const pe_resource_t *rsc);
-pe_action_t *pe__clear_resource_history(pe_resource_t *rsc,
-                                        const pe_node_t *node,
-                                        pe_working_set_t *data_set);
-
-GList *pe__rscs_with_tag(pe_working_set_t *data_set, const char *tag_name);
-GList *pe__unames_with_tag(pe_working_set_t *data_set, const char *tag_name);
-bool pe__rsc_has_tag(pe_working_set_t *data_set, const char *rsc, const char *tag);
-bool pe__uname_has_tag(pe_working_set_t *data_set, const char *node, const char *tag);
-
-bool pe__rsc_running_on_only(const pe_resource_t *rsc, const pe_node_t *node);
-bool pe__rsc_running_on_any(pe_resource_t *rsc, GList *node_list);
+                                gboolean overwrite,
+                                pcmk_scheduler_t *scheduler);
+
+bool pe__resource_is_disabled(const pcmk_resource_t *rsc);
+void pe__clear_resource_history(pcmk_resource_t *rsc, const pcmk_node_t *node);
+
+GList *pe__rscs_with_tag(pcmk_scheduler_t *scheduler, const char *tag_name);
+GList *pe__unames_with_tag(pcmk_scheduler_t *scheduler, const char *tag_name);
+bool pe__rsc_has_tag(pcmk_scheduler_t *scheduler, const char *rsc,
+                     const char *tag);
+bool pe__uname_has_tag(pcmk_scheduler_t *scheduler, const char *node,
+                       const char *tag);
+
+bool pe__rsc_running_on_only(const pcmk_resource_t *rsc,
+                             const pcmk_node_t *node);
+bool pe__rsc_running_on_any(pcmk_resource_t *rsc, GList *node_list);
 GList *pe__filter_rsc_list(GList *rscs, GList *filter);
-GList * pe__build_node_name_list(pe_working_set_t *data_set, const char *s);
-GList * pe__build_rsc_list(pe_working_set_t *data_set, const char *s);
+GList * pe__build_node_name_list(pcmk_scheduler_t *scheduler, const char *s);
+GList * pe__build_rsc_list(pcmk_scheduler_t *scheduler, const char *s);
 
-bool pcmk__rsc_filtered_by_node(pe_resource_t *rsc, GList *only_node);
+bool pcmk__rsc_filtered_by_node(pcmk_resource_t *rsc, GList *only_node);
 
-gboolean pe__bundle_is_filtered(const pe_resource_t *rsc, GList *only_rsc,
+gboolean pe__bundle_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
                                 gboolean check_parent);
-gboolean pe__clone_is_filtered(const pe_resource_t *rsc, GList *only_rsc,
+gboolean pe__clone_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
                                gboolean check_parent);
-gboolean pe__group_is_filtered(const pe_resource_t *rsc, GList *only_rsc,
+gboolean pe__group_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
                                gboolean check_parent);
-gboolean pe__native_is_filtered(const pe_resource_t *rsc, GList *only_rsc,
+gboolean pe__native_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
                                 gboolean check_parent);
 
-xmlNode *pe__failed_probe_for_rsc(const pe_resource_t *rsc, const char *name);
+xmlNode *pe__failed_probe_for_rsc(const pcmk_resource_t *rsc, const char *name);
 
-const char *pe__clone_child_id(const pe_resource_t *rsc);
+const char *pe__clone_child_id(const pcmk_resource_t *rsc);
 
-int pe__sum_node_health_scores(const pe_node_t *node, int base_health);
-int pe__node_health(pe_node_t *node);
+int pe__sum_node_health_scores(const pcmk_node_t *node, int base_health);
+int pe__node_health(pcmk_node_t *node);
 
 static inline enum pcmk__health_strategy
-pe__health_strategy(pe_working_set_t *data_set)
+pe__health_strategy(pcmk_scheduler_t *scheduler)
 {
-    return pcmk__parse_health_strategy(pe_pref(data_set->config_hash,
+    return pcmk__parse_health_strategy(pe_pref(scheduler->config_hash,
                                                PCMK__OPT_NODE_HEALTH_STRATEGY));
 }
 
 static inline int
-pe__health_score(const char *option, pe_working_set_t *data_set)
+pe__health_score(const char *option, pcmk_scheduler_t *scheduler)
 {
-    return char2score(pe_pref(data_set->config_hash, option));
+    return char2score(pe_pref(scheduler->config_hash, option));
 }
 
 /*!
@@ -665,7 +642,7 @@ pe__health_score(const char *option, pe_working_set_t *data_set)
  *         if node has neither a name nor ID.
  */
 static inline const char *
-pe__node_name(const pe_node_t *node)
+pe__node_name(const pcmk_node_t *node)
 {
     if (node == NULL) {
         return "unspecified node";
@@ -691,7 +668,7 @@ pe__node_name(const pe_node_t *node)
  * \return true if \p node1 and \p node2 refer to the same node
  */
 static inline bool
-pe__same_node(const pe_node_t *node1, const pe_node_t *node2)
+pe__same_node(const pcmk_node_t *node1, const pcmk_node_t *node2)
 {
     return (node1 != NULL) && (node2 != NULL)
            && (node1->details == node2->details);
diff --git a/include/crm/pengine/pe_types.h b/include/crm/pengine/pe_types.h
index cc626c8..24355f8 100644
--- a/include/crm/pengine/pe_types.h
+++ b/include/crm/pengine/pe_types.h
@@ -16,6 +16,7 @@
 #  include <libxml/tree.h>          // xmlNode
 #  include <glib.h>                 // gboolean, guint, GList, GHashTable
 #  include <crm/common/iso8601.h>
+#  include <crm/common/scheduler.h>
 #  include <crm/pengine/common.h>
 
 #ifdef __cplusplus
@@ -28,535 +29,6 @@ extern "C" {
  * \ingroup pengine
  */
 
-typedef struct pe_node_s pe_node_t;
-typedef struct pe_action_s pe_action_t;
-typedef struct pe_resource_s pe_resource_t;
-typedef struct pe_working_set_s pe_working_set_t;
-
-enum pe_obj_types {
-    pe_unknown = -1,
-    pe_native = 0,
-    pe_group = 1,
-    pe_clone = 2,
-    pe_container = 3,
-};
-
-typedef struct resource_object_functions_s {
-    gboolean (*unpack) (pe_resource_t*, pe_working_set_t*);
-    pe_resource_t *(*find_rsc) (pe_resource_t *parent, const char *search,
-                                const pe_node_t *node, int flags);
-    /* parameter result must be free'd */
-    char *(*parameter) (pe_resource_t*, pe_node_t*, gboolean, const char*,
-                        pe_working_set_t*);
-    //! \deprecated will be removed in a future release
-    void (*print) (pe_resource_t*, const char*, long, void*);
-    gboolean (*active) (pe_resource_t*, gboolean);
-    enum rsc_role_e (*state) (const pe_resource_t*, gboolean);
-    pe_node_t *(*location) (const pe_resource_t*, GList**, int);
-    void (*free) (pe_resource_t*);
-    void (*count) (pe_resource_t*);
-    gboolean (*is_filtered) (const pe_resource_t*, GList *, gboolean);
-
-    /*!
-     * \brief
-     * \internal Find a node (and optionally count all) where resource is active
-     *
-     * \param[in]  rsc          Resource to check
-     * \param[out] count_all    If not NULL, set this to count of active nodes
-     * \param[out] count_clean  If not NULL, set this to count of clean nodes
-     *
-     * \return A node where the resource is active, preferring the source node
-     *         if the resource is involved in a partial migration or a clean,
-     *         online node if the resource's "requires" is "quorum" or
-     *         "nothing", or NULL if the resource is inactive.
-     */
-    pe_node_t *(*active_node)(const pe_resource_t *rsc, unsigned int *count_all,
-                              unsigned int *count_clean);
-} resource_object_functions_t;
-
-typedef struct resource_alloc_functions_s resource_alloc_functions_t;
-
-enum pe_quorum_policy {
-    no_quorum_freeze,
-    no_quorum_stop,
-    no_quorum_ignore,
-    no_quorum_suicide,
-    no_quorum_demote
-};
-
-enum node_type {
-    node_ping,      //! \deprecated Do not use
-    node_member,
-    node_remote
-};
-
-//! \deprecated will be removed in a future release
-enum pe_restart {
-    pe_restart_restart, //! \deprecated will be removed in a future release
-    pe_restart_ignore   //! \deprecated will be removed in a future release
-};
-
-//! Determine behavior of pe_find_resource_with_flags()
-enum pe_find {
-    pe_find_renamed  = 0x001, //!< match resource ID or LRM history ID
-    pe_find_anon     = 0x002, //!< match base name of anonymous clone instances
-    pe_find_clone    = 0x004, //!< match only clone instances
-    pe_find_current  = 0x008, //!< match resource active on specified node
-    pe_find_inactive = 0x010, //!< match resource not running anywhere
-    pe_find_any      = 0x020, //!< match base name of any clone instance
-};
-
-// @TODO Make these an enum
-
-#  define pe_flag_have_quorum           0x00000001ULL
-#  define pe_flag_symmetric_cluster     0x00000002ULL
-#  define pe_flag_maintenance_mode      0x00000008ULL
-
-#  define pe_flag_stonith_enabled       0x00000010ULL
-#  define pe_flag_have_stonith_resource 0x00000020ULL
-#  define pe_flag_enable_unfencing      0x00000040ULL
-#  define pe_flag_concurrent_fencing    0x00000080ULL
-
-#  define pe_flag_stop_rsc_orphans      0x00000100ULL
-#  define pe_flag_stop_action_orphans   0x00000200ULL
-#  define pe_flag_stop_everything       0x00000400ULL
-
-#  define pe_flag_start_failure_fatal   0x00001000ULL
-
-//! \deprecated
-#  define pe_flag_remove_after_stop     0x00002000ULL
-
-#  define pe_flag_startup_fencing       0x00004000ULL
-#  define pe_flag_shutdown_lock         0x00008000ULL
-
-#  define pe_flag_startup_probes        0x00010000ULL
-#  define pe_flag_have_status           0x00020000ULL
-#  define pe_flag_have_remote_nodes     0x00040000ULL
-
-#  define pe_flag_quick_location        0x00100000ULL
-#  define pe_flag_sanitized             0x00200000ULL
-
-//! \deprecated
-#  define pe_flag_stdout                0x00400000ULL
-
-//! Don't count total, disabled and blocked resource instances
-#  define pe_flag_no_counts             0x00800000ULL
-
-/*! Skip deprecated code that is kept solely for backward API compatibility.
- * (Internal code should always set this.)
- */
-#  define pe_flag_no_compat             0x01000000ULL
-
-#  define pe_flag_show_scores           0x02000000ULL
-#  define pe_flag_show_utilization      0x04000000ULL
-
-/*!
- * When scheduling, only unpack the CIB (including constraints), calculate
- * as much cluster status as possible, and apply node health.
- */
-#  define pe_flag_check_config          0x08000000ULL
-
-struct pe_working_set_s {
-    xmlNode *input;
-    crm_time_t *now;
-
-    /* options extracted from the input */
-    char *dc_uuid;
-    pe_node_t *dc_node;
-    const char *stonith_action;
-    const char *placement_strategy;
-
-    unsigned long long flags;
-
-    int stonith_timeout;
-    enum pe_quorum_policy no_quorum_policy;
-
-    GHashTable *config_hash;
-    GHashTable *tickets;
-
-    // Actions for which there can be only one (e.g. fence nodeX)
-    GHashTable *singletons;
-
-    GList *nodes;
-    GList *resources;
-    GList *placement_constraints;
-    GList *ordering_constraints;
-    GList *colocation_constraints;
-    GList *ticket_constraints;
-
-    GList *actions;
-    xmlNode *failed;
-    xmlNode *op_defaults;
-    xmlNode *rsc_defaults;
-
-    /* stats */
-    int num_synapse;
-    int max_valid_nodes;    //! Deprecated (will be removed in a future release)
-    int order_id;
-    int action_id;
-
-    /* final output */
-    xmlNode *graph;
-
-    GHashTable *template_rsc_sets;
-    const char *localhost;
-    GHashTable *tags;
-
-    int blocked_resources;
-    int disabled_resources;
-
-    GList *param_check; // History entries that need to be checked
-    GList *stop_needed; // Containers that need stop actions
-    time_t recheck_by;  // Hint to controller to re-run scheduler by this time
-    int ninstances;     // Total number of resource instances
-    guint shutdown_lock;// How long (seconds) to lock resources to shutdown node
-    int priority_fencing_delay; // Priority fencing delay
-
-    void *priv;
-};
-
-enum pe_check_parameters {
-    /* Clear fail count if parameters changed for un-expired start or monitor
-     * last_failure.
-     */
-    pe_check_last_failure,
-
-    /* Clear fail count if parameters changed for start, monitor, promote, or
-     * migrate_from actions for active resources.
-     */
-    pe_check_active,
-};
-
-struct pe_node_shared_s {
-    const char *id;
-    const char *uname;
-    enum node_type type;
-
-    /* @TODO convert these flags into a bitfield */
-    gboolean online;
-    gboolean standby;
-    gboolean standby_onfail;
-    gboolean pending;
-    gboolean unclean;
-    gboolean unseen;
-    gboolean shutdown;
-    gboolean expected_up;
-    gboolean is_dc;
-    gboolean maintenance;
-    gboolean rsc_discovery_enabled;
-    gboolean remote_requires_reset;
-    gboolean remote_was_fenced;
-    gboolean remote_maintenance; /* what the remote-rsc is thinking */
-    gboolean unpacked;
-
-    int num_resources;
-    pe_resource_t *remote_rsc;
-    GList *running_rsc;       /* pe_resource_t* */
-    GList *allocated_rsc;     /* pe_resource_t* */
-
-    GHashTable *attrs;          /* char* => char* */
-    GHashTable *utilization;
-    GHashTable *digest_cache;   //!< cache of calculated resource digests
-    int priority; // calculated based on the priority of resources running on the node
-    pe_working_set_t *data_set; //!< Cluster that this node is part of
-};
-
-struct pe_node_s {
-    int weight;
-    gboolean fixed; //!< \deprecated Will be removed in a future release
-    int count;
-    struct pe_node_shared_s *details;
-    int rsc_discover_mode;
-};
-
-#  define pe_rsc_orphan                     0x00000001ULL
-#  define pe_rsc_managed                    0x00000002ULL
-#  define pe_rsc_block                      0x00000004ULL
-#  define pe_rsc_orphan_container_filler    0x00000008ULL
-
-#  define pe_rsc_notify                     0x00000010ULL
-#  define pe_rsc_unique                     0x00000020ULL
-#  define pe_rsc_fence_device               0x00000040ULL
-#  define pe_rsc_promotable                 0x00000080ULL
-
-#  define pe_rsc_provisional                0x00000100ULL
-#  define pe_rsc_allocating                 0x00000200ULL
-#  define pe_rsc_merging                    0x00000400ULL
-#  define pe_rsc_restarting                 0x00000800ULL
-
-#  define pe_rsc_stop                       0x00001000ULL
-#  define pe_rsc_reload                     0x00002000ULL
-#  define pe_rsc_allow_remote_remotes       0x00004000ULL
-#  define pe_rsc_critical                   0x00008000ULL
-
-#  define pe_rsc_failed                     0x00010000ULL
-#  define pe_rsc_detect_loop                0x00020000ULL
-#  define pe_rsc_runnable                   0x00040000ULL
-#  define pe_rsc_start_pending              0x00080000ULL
-
-//!< \deprecated Do not use
-#  define pe_rsc_starting                   0x00100000ULL
-
-//!< \deprecated Do not use
-#  define pe_rsc_stopping                   0x00200000ULL
-
-#  define pe_rsc_stop_unexpected            0x00400000ULL
-#  define pe_rsc_allow_migrate              0x00800000ULL
-
-#  define pe_rsc_failure_ignored            0x01000000ULL
-#  define pe_rsc_replica_container          0x02000000ULL
-#  define pe_rsc_maintenance                0x04000000ULL
-#  define pe_rsc_is_container               0x08000000ULL
-
-#  define pe_rsc_needs_quorum               0x10000000ULL
-#  define pe_rsc_needs_fencing              0x20000000ULL
-#  define pe_rsc_needs_unfencing            0x40000000ULL
-
-/* *INDENT-OFF* */
-enum pe_action_flags {
-    pe_action_pseudo = 0x00001,
-    pe_action_runnable = 0x00002,
-    pe_action_optional = 0x00004,
-    pe_action_print_always = 0x00008,
-
-    pe_action_have_node_attrs = 0x00010,
-    pe_action_implied_by_stonith = 0x00040,
-    pe_action_migrate_runnable =   0x00080,
-
-    pe_action_dumped = 0x00100,
-    pe_action_processed = 0x00200,
-#if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
-    pe_action_clear = 0x00400, //! \deprecated Unused
-#endif
-    pe_action_dangle = 0x00800,
-
-    /* This action requires one or more of its dependencies to be runnable.
-     * We use this to clear the runnable flag before checking dependencies.
-     */
-    pe_action_requires_any = 0x01000,
-
-    pe_action_reschedule = 0x02000,
-    pe_action_tracking = 0x04000,
-    pe_action_dedup = 0x08000, //! Internal state tracking when creating graph
-
-    pe_action_dc = 0x10000,         //! Action may run on DC instead of target
-};
-/* *INDENT-ON* */
-
-struct pe_resource_s {
-    char *id;
-    char *clone_name;
-    xmlNode *xml;
-    xmlNode *orig_xml;
-    xmlNode *ops_xml;
-
-    pe_working_set_t *cluster;
-    pe_resource_t *parent;
-
-    enum pe_obj_types variant;
-    void *variant_opaque;
-    resource_object_functions_t *fns;
-    resource_alloc_functions_t *cmds;
-
-    enum rsc_recovery_type recovery_type;
-
-    enum pe_restart restart_type; //!< \deprecated will be removed in future release
-
-    int priority;
-    int stickiness;
-    int sort_index;
-    int failure_timeout;
-    int migration_threshold;
-    guint remote_reconnect_ms;
-    char *pending_task;
-
-    unsigned long long flags;
-
-    // @TODO merge these into flags
-    gboolean is_remote_node;
-    gboolean exclusive_discover;
-
-    /* Pay special attention to whether you want to use rsc_cons_lhs and
-     * rsc_cons directly, which include only colocations explicitly involving
-     * this resource, or call libpacemaker's pcmk__with_this_colocations() and
-     * pcmk__this_with_colocations() functions, which may return relevant
-     * colocations involving the resource's ancestors as well.
-     */
-
-    //!@{
-    //! This field should be treated as internal to Pacemaker
-    GList *rsc_cons_lhs;      // List of pcmk__colocation_t*
-    GList *rsc_cons;          // List of pcmk__colocation_t*
-    GList *rsc_location;      // List of pe__location_t*
-    GList *actions;           // List of pe_action_t*
-    GList *rsc_tickets;       // List of rsc_ticket*
-    //!@}
-
-    pe_node_t *allocated_to;
-    pe_node_t *partial_migration_target;
-    pe_node_t *partial_migration_source;
-    GList *running_on;        /* pe_node_t*   */
-    GHashTable *known_on;       /* pe_node_t*   */
-    GHashTable *allowed_nodes;  /* pe_node_t*   */
-
-    enum rsc_role_e role;
-    enum rsc_role_e next_role;
-
-    GHashTable *meta;
-    GHashTable *parameters; //! \deprecated Use pe_rsc_params() instead
-    GHashTable *utilization;
-
-    GList *children;          /* pe_resource_t*   */
-    GList *dangling_migrations;       /* pe_node_t*       */
-
-    pe_resource_t *container;
-    GList *fillers;
-
-    // @COMPAT These should be made const at next API compatibility break
-    pe_node_t *pending_node;    // Node on which pending_task is happening
-    pe_node_t *lock_node;       // Resource is shutdown-locked to this node
-
-    time_t lock_time;           // When shutdown lock started
-
-    /* Resource parameters may have node-attribute-based rules, which means the
-     * values can vary by node. This table is a cache of parameter name/value
-     * tables for each node (as needed). Use pe_rsc_params() to get the table
-     * for a given node.
-     */
-    GHashTable *parameter_cache; // Key = node name, value = parameters table
-};
-
-struct pe_action_s {
-    int id;
-    int priority;
-
-    pe_resource_t *rsc;
-    pe_node_t *node;
-    xmlNode *op_entry;
-
-    char *task;
-    char *uuid;
-    char *cancel_task;
-    char *reason;
-
-    enum pe_action_flags flags;
-    enum rsc_start_requirement needs;
-    enum action_fail_response on_fail;
-    enum rsc_role_e fail_role;
-
-    GHashTable *meta;
-    GHashTable *extra;
-
-    /* 
-     * These two varables are associated with the constraint logic
-     * that involves first having one or more actions runnable before
-     * then allowing this action to execute.
-     *
-     * These varables are used with features such as 'clone-min' which
-     * requires at minimum X number of cloned instances to be running
-     * before an order dependency can run. Another option that uses
-     * this is 'require-all=false' in ordering constrants. This option
-     * says "only require one instance of a resource to start before
-     * allowing dependencies to start" -- basically, require-all=false is
-     * the same as clone-min=1.
-     */
-
-    /* current number of known runnable actions in the before list. */
-    int runnable_before;
-    /* the number of "before" runnable actions required for this action
-     * to be considered runnable */ 
-    int required_runnable_before;
-
-    GList *actions_before;    /* pe_action_wrapper_t* */
-    GList *actions_after;     /* pe_action_wrapper_t* */
-
-    /* Some of the above fields could be moved to the details,
-     * except for API backward compatibility.
-     */
-    void *action_details; // varies by type of action
-};
-
-typedef struct pe_ticket_s {
-    char *id;
-    gboolean granted;
-    time_t last_granted;
-    gboolean standby;
-    GHashTable *state;
-} pe_ticket_t;
-
-typedef struct pe_tag_s {
-    char *id;
-    GList *refs;
-} pe_tag_t;
-
-//! Internal tracking for transition graph creation
-enum pe_link_state {
-    pe_link_not_dumped, //! Internal tracking for transition graph creation
-    pe_link_dumped,     //! Internal tracking for transition graph creation
-    pe_link_dup,        //! \deprecated No longer used by Pacemaker
-};
-
-enum pe_discover_e {
-    pe_discover_always = 0,
-    pe_discover_never,
-    pe_discover_exclusive,
-};
-
-/* *INDENT-OFF* */
-enum pe_ordering {
-    pe_order_none                  = 0x0,       /* deleted */
-    pe_order_optional              = 0x1,       /* pure ordering, nothing implied */
-    pe_order_apply_first_non_migratable = 0x2,  /* Only apply this constraint's ordering if first is not migratable. */
-
-    pe_order_implies_first         = 0x10,      /* If 'then' is required, ensure 'first' is too */
-    pe_order_implies_then          = 0x20,      /* If 'first' is required, ensure 'then' is too */
-    pe_order_promoted_implies_first = 0x40,     /* If 'then' is required and then's rsc is promoted, ensure 'first' becomes required too */
-
-    /* first requires then to be both runnable and migrate runnable. */
-    pe_order_implies_first_migratable  = 0x80,
-
-    pe_order_runnable_left         = 0x100,     /* 'then' requires 'first' to be runnable */
-
-    pe_order_pseudo_left           = 0x200,     /* 'then' can only be pseudo if 'first' is runnable */
-    pe_order_implies_then_on_node  = 0x400,     /* If 'first' is required on 'nodeX',
-                                                 * ensure instances of 'then' on 'nodeX' are too.
-                                                 * Only really useful if 'then' is a clone and 'first' is not
-                                                 */
-    pe_order_probe                 = 0x800,     /* If 'first->rsc' is
-                                                 *  - running but about to stop, ignore the constraint
-                                                 *  - otherwise, behave as runnable_left
-                                                 */
-
-    pe_order_restart               = 0x1000,    /* 'then' is runnable if 'first' is optional or runnable */
-    pe_order_stonith_stop          = 0x2000,    //<! \deprecated Will be removed in future release
-    pe_order_serialize_only        = 0x4000,    /* serialize */
-    pe_order_same_node             = 0x8000,    /* applies only if 'first' and 'then' are on same node */
-
-    pe_order_implies_first_printed = 0x10000,   /* Like ..implies_first but only ensures 'first' is printed, not mandatory */
-    pe_order_implies_then_printed  = 0x20000,   /* Like ..implies_then but only ensures 'then' is printed, not mandatory */
-
-    pe_order_asymmetrical          = 0x100000,  /* Indicates asymmetrical one way ordering constraint. */
-    pe_order_load                  = 0x200000,  /* Only relevant if... */
-    pe_order_one_or_more           = 0x400000,  /* 'then' is runnable only if one or more of its dependencies are too */
-    pe_order_anti_colocation       = 0x800000,
-
-    pe_order_preserve              = 0x1000000, /* Hack for breaking user ordering constraints with container resources */
-    pe_order_then_cancels_first    = 0x2000000, // if 'then' becomes required, 'first' becomes optional
-    pe_order_trace                 = 0x4000000, /* test marker */
-
-#if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
-    // \deprecated Use pe_order_promoted_implies_first instead
-    pe_order_implies_first_master  = pe_order_promoted_implies_first,
-#endif
-};
-/* *INDENT-ON* */
-
-typedef struct pe_action_wrapper_s {
-    enum pe_ordering type;
-    enum pe_link_state state;
-    pe_action_t *action;
-} pe_action_wrapper_t;
-
 #if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
 #include <crm/pengine/pe_types_compat.h>
 #endif
diff --git a/include/crm/pengine/pe_types_compat.h b/include/crm/pengine/pe_types_compat.h
index 6f174c4..1becd12 100644
--- a/include/crm/pengine/pe_types_compat.h
+++ b/include/crm/pengine/pe_types_compat.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2004-2022 the Pacemaker project contributors
+ * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
@@ -10,7 +10,7 @@
 #ifndef PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H
 #  define PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H
 
-#include <crm/pengine/pe_types.h>
+#include <crm/common/scheduler.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -25,6 +25,174 @@ extern "C" {
  *             release.
  */
 
+//! \deprecated Use pcmk_rsc_removed instead
+#define pe_rsc_orphan                   pcmk_rsc_removed
+
+//! \deprecated Use pcmk_rsc_managed instead
+#define pe_rsc_managed                  pcmk_rsc_managed
+
+//! \deprecated Use pcmk_rsc_blocked instead
+#define pe_rsc_block                    pcmk_rsc_blocked
+
+//! \deprecated Use pcmk_rsc_removed_filler instead
+#define pe_rsc_orphan_container_filler  pcmk_rsc_removed_filler
+
+//! \deprecated Use pcmk_rsc_notify instead
+#define pe_rsc_notify                   pcmk_rsc_notify
+
+//! \deprecated Use pcmk_rsc_unique instead
+#define pe_rsc_unique                   pcmk_rsc_unique
+
+//! \deprecated Use pcmk_rsc_fence_device instead
+#define pe_rsc_fence_device             pcmk_rsc_fence_device
+
+//! \deprecated Use pcmk_rsc_promotable instead
+#define pe_rsc_promotable               pcmk_rsc_promotable
+
+//! \deprecated Use pcmk_rsc_unassigned instead
+#define pe_rsc_provisional              pcmk_rsc_unassigned
+
+//! \deprecated Use pcmk_rsc_assigning instead
+#define pe_rsc_allocating               pcmk_rsc_assigning
+
+//! \deprecated Use pcmk_rsc_updating_nodes instead
+#define pe_rsc_merging                  pcmk_rsc_updating_nodes
+
+//! \deprecated Use pcmk_rsc_restarting instead
+#define pe_rsc_restarting               pcmk_rsc_restarting
+
+//! \deprecated Use pcmk_rsc_stop_if_failed instead
+#define pe_rsc_stop                     pcmk_rsc_stop_if_failed
+
+//! \deprecated Use pcmk_rsc_reload instead
+#define pe_rsc_reload                   pcmk_rsc_reload
+
+//! \deprecated Use pcmk_rsc_remote_nesting_allowed instead
+#define pe_rsc_allow_remote_remotes     pcmk_rsc_remote_nesting_allowed
+
+//! \deprecated Use pcmk_rsc_critical instead
+#define pe_rsc_critical                 pcmk_rsc_critical
+
+//! \deprecated Use pcmk_rsc_failed instead
+#define pe_rsc_failed                   pcmk_rsc_failed
+
+//! \deprecated Use pcmk_rsc_detect_loop instead
+#define pe_rsc_detect_loop              pcmk_rsc_detect_loop
+
+//! \deprecated Do not use
+#define pe_rsc_runnable                 pcmk_rsc_runnable
+
+//! \deprecated Use pcmk_rsc_start_pending instead
+#define pe_rsc_start_pending            pcmk_rsc_start_pending
+
+//!< \deprecated Do not use
+#define pe_rsc_starting                 pcmk_rsc_starting
+
+//!< \deprecated Do not use
+#define pe_rsc_stopping                 pcmk_rsc_stopping
+
+//! \deprecated Use pcmk_rsc_stop_unexpected instead
+#define pe_rsc_stop_unexpected          pcmk_rsc_stop_unexpected
+
+//! \deprecated Use pcmk_rsc_migratable instead
+#define pe_rsc_allow_migrate            pcmk_rsc_migratable
+
+//! \deprecated Use pcmk_rsc_ignore_failure instead
+#define pe_rsc_failure_ignored          pcmk_rsc_ignore_failure
+
+//! \deprecated Use pcmk_rsc_replica_container instead
+#define pe_rsc_replica_container        pcmk_rsc_replica_container
+
+//! \deprecated Use pcmk_rsc_maintenance instead
+#define pe_rsc_maintenance              pcmk_rsc_maintenance
+
+//! \deprecated Do not use
+#define pe_rsc_is_container             pcmk_rsc_has_filler
+
+//! \deprecated Use pcmk_rsc_needs_quorum instead
+#define pe_rsc_needs_quorum             pcmk_rsc_needs_quorum
+
+//! \deprecated Use pcmk_rsc_needs_fencing instead
+#define pe_rsc_needs_fencing            pcmk_rsc_needs_fencing
+
+//! \deprecated Use pcmk_rsc_needs_unfencing instead
+#define pe_rsc_needs_unfencing          pcmk_rsc_needs_unfencing
+
+//! \deprecated Use pcmk_sched_quorate instead
+#define pe_flag_have_quorum             pcmk_sched_quorate
+
+//! \deprecated Use pcmk_sched_symmetric_cluster instead
+#define pe_flag_symmetric_cluster       pcmk_sched_symmetric_cluster
+
+//! \deprecated Use pcmk_sched_in_maintenance instead
+#define pe_flag_maintenance_mode        pcmk_sched_in_maintenance
+
+//! \deprecated Use pcmk_sched_fencing_enabled instead
+#define pe_flag_stonith_enabled         pcmk_sched_fencing_enabled
+
+//! \deprecated Use pcmk_sched_have_fencing instead
+#define pe_flag_have_stonith_resource   pcmk_sched_have_fencing
+
+//! \deprecated Use pcmk_sched_enable_unfencing instead
+#define pe_flag_enable_unfencing        pcmk_sched_enable_unfencing
+
+//! \deprecated Use pcmk_sched_concurrent_fencing instead
+#define pe_flag_concurrent_fencing      pcmk_sched_concurrent_fencing
+
+//! \deprecated Use pcmk_sched_stop_removed_resources instead
+#define pe_flag_stop_rsc_orphans        pcmk_sched_stop_removed_resources
+
+//! \deprecated Use pcmk_sched_cancel_removed_actions instead
+#define pe_flag_stop_action_orphans     pcmk_sched_cancel_removed_actions
+
+//! \deprecated Use pcmk_sched_stop_all instead
+#define pe_flag_stop_everything         pcmk_sched_stop_all
+
+//! \deprecated Use pcmk_sched_start_failure_fatal instead
+#define pe_flag_start_failure_fatal     pcmk_sched_start_failure_fatal
+
+//! \deprecated Do not use
+#define pe_flag_remove_after_stop       pcmk_sched_remove_after_stop
+
+//! \deprecated Use pcmk_sched_startup_fencing instead
+#define pe_flag_startup_fencing         pcmk_sched_startup_fencing
+
+//! \deprecated Use pcmk_sched_shutdown_lock instead
+#define pe_flag_shutdown_lock           pcmk_sched_shutdown_lock
+
+//! \deprecated Use pcmk_sched_probe_resources instead
+#define pe_flag_startup_probes          pcmk_sched_probe_resources
+
+//! \deprecated Use pcmk_sched_have_status instead
+#define pe_flag_have_status             pcmk_sched_have_status
+
+//! \deprecated Use pcmk_sched_have_remote_nodes instead
+#define pe_flag_have_remote_nodes       pcmk_sched_have_remote_nodes
+
+//! \deprecated Use pcmk_sched_location_only instead
+#define pe_flag_quick_location          pcmk_sched_location_only
+
+//! \deprecated Use pcmk_sched_sanitized instead
+#define pe_flag_sanitized               pcmk_sched_sanitized
+
+//! \deprecated Do not use
+#define pe_flag_stdout                  (1ULL << 22)
+
+//! \deprecated Use pcmk_sched_no_counts instead
+#define pe_flag_no_counts               pcmk_sched_no_counts
+
+//! \deprecated Use pcmk_sched_no_compat instead
+#define pe_flag_no_compat               pcmk_sched_no_compat
+
+//! \deprecated Use pcmk_sched_output_scores instead
+#define pe_flag_show_scores             pcmk_sched_output_scores
+
+//! \deprecated Use pcmk_sched_show_utilization instead
+#define pe_flag_show_utilization        pcmk_sched_show_utilization
+
+//! \deprecated Use pcmk_sched_validate_only instead
+#define pe_flag_check_config            pcmk_sched_validate_only
+
 //!@{
 //! \deprecated Do not use (unused by Pacemaker)
 enum pe_graph_flags {
@@ -35,27 +203,62 @@ enum pe_graph_flags {
 };
 //!@}
 
-//!< \deprecated Use pe_action_t instead
+//!@{
+//! \deprecated Do not use
+enum pe_check_parameters {
+    pe_check_last_failure,
+    pe_check_active,
+};
+//!@}
+
+//! \deprecated Use pcmk_action_t instead
 typedef struct pe_action_s action_t;
 
-//!< \deprecated Use pe_action_wrapper_t instead
+//! \deprecated Use pcmk_action_t instead
+typedef struct pe_action_s pe_action_t;
+
+//! \deprecated Do not use
 typedef struct pe_action_wrapper_s action_wrapper_t;
 
-//!< \deprecated Use pe_node_t instead
+//! \deprecated Do not use
+typedef struct pe_action_wrapper_s pe_action_wrapper_t;
+
+//! \deprecated Use pcmk_node_t instead
 typedef struct pe_node_s node_t;
 
-//!< \deprecated Use enum pe_quorum_policy instead
+//! \deprecated Use pcmk_node_t instead
+typedef struct pe_node_s pe_node_t;
+
+//! \deprecated Use enum pe_quorum_policy instead
 typedef enum pe_quorum_policy no_quorum_policy_t;
 
-//!< \deprecated use pe_resource_t instead
+//! \deprecated use pcmk_resource_t instead
 typedef struct pe_resource_s resource_t;
 
-//!< \deprecated Use pe_tag_t instead
+//! \deprecated use pcmk_resource_t instead
+typedef struct pe_resource_s pe_resource_t;
+
+//! \deprecated Use pcmk_tag_t instead
 typedef struct pe_tag_s tag_t;
 
-//!< \deprecated Use pe_ticket_t instead
+//! \deprecated Use pcmk_tag_t instead
+typedef struct pe_tag_s pe_tag_t;
+
+//! \deprecated Use pcmk_ticket_t instead
 typedef struct pe_ticket_s ticket_t;
 
+//! \deprecated Use pcmk_ticket_t instead
+typedef struct pe_ticket_s pe_ticket_t;
+
+//! \deprecated Use pcmk_scheduler_t instead
+typedef struct pe_working_set_s pe_working_set_t;
+
+//! \deprecated This type should be treated as internal to Pacemaker
+typedef struct resource_alloc_functions_s resource_alloc_functions_t;
+
+//! \deprecated Use pcmk_rsc_methods_t instead
+typedef struct resource_object_functions_s resource_object_functions_t;
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/include/crm/pengine/remote_internal.h b/include/crm/pengine/remote_internal.h
index 46d58fc..0e7c044 100644
--- a/include/crm/pengine/remote_internal.h
+++ b/include/crm/pengine/remote_internal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2019 the Pacemaker project contributors
+ * Copyright 2013-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
@@ -19,16 +19,17 @@ extern "C" {
 #include <crm/pengine/status.h>
 
 bool xml_contains_remote_node(xmlNode *xml);
-bool pe__is_remote_node(const pe_node_t *node);
-bool pe__is_guest_node(const pe_node_t *node);
-bool pe__is_guest_or_remote_node(const pe_node_t *node);
-bool pe__is_bundle_node(const pe_node_t *node);
-bool pe__resource_is_remote_conn(const pe_resource_t *rsc,
-                                 const pe_working_set_t *data_set);
-pe_resource_t *pe__resource_contains_guest_node(const pe_working_set_t *data_set,
-                                                const pe_resource_t *rsc);
-void pe_foreach_guest_node(const pe_working_set_t *data_set, const pe_node_t *host,
-                           void (*helper)(const pe_node_t*, void*), void *user_data);
+bool pe__is_remote_node(const pcmk_node_t *node);
+bool pe__is_guest_node(const pcmk_node_t *node);
+bool pe__is_guest_or_remote_node(const pcmk_node_t *node);
+bool pe__is_bundle_node(const pcmk_node_t *node);
+bool pe__resource_is_remote_conn(const pcmk_resource_t *rsc);
+pcmk_resource_t *pe__resource_contains_guest_node(const pcmk_scheduler_t *scheduler,
+                                                  const pcmk_resource_t *rsc);
+void pe_foreach_guest_node(const pcmk_scheduler_t *scheduler,
+                           const pcmk_node_t *host,
+                           void (*helper)(const pcmk_node_t*, void*),
+                           void *user_data);
 xmlNode *pe_create_remote_xml(xmlNode *parent, const char *uname,
                               const char *container_id, const char *migrateable,
                               const char *is_managed, const char *start_timeout,
diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h
index 145a166..9c85425 100644
--- a/include/crm/pengine/status.h
+++ b/include/crm/pengine/status.h
@@ -15,7 +15,7 @@
 #  include <crm/common/util.h>      // pcmk_is_set()
 #  include <crm/common/iso8601.h>
 #  include <crm/pengine/common.h>
-#  include <crm/pengine/pe_types.h> // pe_node_t, pe_resource_t, etc.
+#  include <crm/pengine/pe_types.h> // pcmk_node_t, pcmk_resource_t, etc.
 #  include <crm/pengine/complex.h>
 
 #ifdef __cplusplus
@@ -28,24 +28,25 @@ extern "C" {
  * \ingroup pengine
  */
 
-const char *rsc_printable_id(const pe_resource_t *rsc);
-gboolean cluster_status(pe_working_set_t * data_set);
-pe_working_set_t *pe_new_working_set(void);
-void pe_free_working_set(pe_working_set_t *data_set);
-void set_working_set_defaults(pe_working_set_t * data_set);
-void cleanup_calculations(pe_working_set_t * data_set);
-void pe_reset_working_set(pe_working_set_t *data_set);
-pe_resource_t *pe_find_resource(GList *rsc_list, const char *id_rh);
-pe_resource_t *pe_find_resource_with_flags(GList *rsc_list, const char *id, enum pe_find flags);
-pe_node_t *pe_find_node(const GList *node_list, const char *node_name);
-pe_node_t *pe_find_node_id(const GList *node_list, const char *id);
-pe_node_t *pe_find_node_any(const GList *node_list, const char *id,
+const char *rsc_printable_id(const pcmk_resource_t *rsc);
+gboolean cluster_status(pcmk_scheduler_t *scheduler);
+pcmk_scheduler_t *pe_new_working_set(void);
+void pe_free_working_set(pcmk_scheduler_t *scheduler);
+void set_working_set_defaults(pcmk_scheduler_t *scheduler);
+void cleanup_calculations(pcmk_scheduler_t *scheduler);
+void pe_reset_working_set(pcmk_scheduler_t *scheduler);
+pcmk_resource_t *pe_find_resource(GList *rsc_list, const char *id_rh);
+pcmk_resource_t *pe_find_resource_with_flags(GList *rsc_list, const char *id,
+                                             enum pe_find flags);
+pcmk_node_t *pe_find_node(const GList *node_list, const char *node_name);
+pcmk_node_t *pe_find_node_id(const GList *node_list, const char *id);
+pcmk_node_t *pe_find_node_any(const GList *node_list, const char *id,
                             const char *node_name);
 GList *find_operations(const char *rsc, const char *node, gboolean active_filter,
-                         pe_working_set_t * data_set);
+                         pcmk_scheduler_t *scheduler);
 void calculate_active_ops(const GList *sorted_op_list, int *start_index,
                           int *stop_index);
-int pe_bundle_replicas(const pe_resource_t *rsc);
+int pe_bundle_replicas(const pcmk_resource_t *rsc);
 
 /*!
  * \brief Check whether a resource is any clone type
@@ -55,9 +56,9 @@ int pe_bundle_replicas(const pe_resource_t *rsc);
  * \return true if resource is clone, false otherwise
  */
 static inline bool
-pe_rsc_is_clone(const pe_resource_t *rsc)
+pe_rsc_is_clone(const pcmk_resource_t *rsc)
 {
-    return rsc && (rsc->variant == pe_clone);
+    return (rsc != NULL) && (rsc->variant == pcmk_rsc_variant_clone);
 }
 
 /*!
@@ -68,9 +69,9 @@ pe_rsc_is_clone(const pe_resource_t *rsc)
  * \return true if resource is unique clone, false otherwise
  */
 static inline bool
-pe_rsc_is_unique_clone(const pe_resource_t *rsc)
+pe_rsc_is_unique_clone(const pcmk_resource_t *rsc)
 {
-    return pe_rsc_is_clone(rsc) && pcmk_is_set(rsc->flags, pe_rsc_unique);
+    return pe_rsc_is_clone(rsc) && pcmk_is_set(rsc->flags, pcmk_rsc_unique);
 }
 
 /*!
@@ -81,9 +82,9 @@ pe_rsc_is_unique_clone(const pe_resource_t *rsc)
  * \return true if resource is anonymous clone, false otherwise
  */
 static inline bool
-pe_rsc_is_anon_clone(const pe_resource_t *rsc)
+pe_rsc_is_anon_clone(const pcmk_resource_t *rsc)
 {
-    return pe_rsc_is_clone(rsc) && !pcmk_is_set(rsc->flags, pe_rsc_unique);
+    return pe_rsc_is_clone(rsc) && !pcmk_is_set(rsc->flags, pcmk_rsc_unique);
 }
 
 /*!
@@ -94,7 +95,7 @@ pe_rsc_is_anon_clone(const pe_resource_t *rsc)
  * \return true if resource is part of a bundle, false otherwise
  */
 static inline bool
-pe_rsc_is_bundled(const pe_resource_t *rsc)
+pe_rsc_is_bundled(const pcmk_resource_t *rsc)
 {
     if (rsc == NULL) {
         return false;
@@ -102,7 +103,7 @@ pe_rsc_is_bundled(const pe_resource_t *rsc)
     while (rsc->parent != NULL) {
         rsc = rsc->parent;
     }
-    return rsc->variant == pe_container;
+    return rsc->variant == pcmk_rsc_variant_bundle;
 }
 
 #ifdef __cplusplus