Adding upstream version 2.1.6.upstream/2.1.6

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 963 insertions, 0 deletions

diff --git a/cts/cts-exec.in b/cts/cts-exec.in
new file mode 100644
index 0000000..a89eb0e
--- /dev/null
+++ b/cts/cts-exec.in
@@ -0,0 +1,963 @@
+#!@PYTHON@
+""" Regression tests for Pacemaker's pacemaker-execd
+"""
+
+__copyright__ = "Copyright 2012-2023 the Pacemaker project contributors"
+__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
+
+import argparse
+import io
+import os
+import psutil
+import stat
+import sys
+import subprocess
+import shutil
+import tempfile
+import time
+
+# Where to find test binaries
+# Prefer the source tree if available
+TEST_DIR = sys.path[0]
+
+# These imports allow running from a source checkout after running `make`.
+# Note that while this doesn't necessarily mean it will successfully run tests,
+# but being able to see --help output can be useful.
+if os.path.exists("@abs_top_srcdir@/python"):
+    sys.path.insert(0, "@abs_top_srcdir@/python")
+
+if os.path.exists("@abs_top_builddir@/python") and "@abs_top_builddir@" != "@abs_top_srcdir@":
+    sys.path.insert(0, "@abs_top_builddir@/python")
+
+from pacemaker.buildoptions import BuildOptions
+from pacemaker.exitstatus import ExitStatus
+from pacemaker._cts.errors import ExitCodeError, OutputFoundError, OutputNotFoundError
+from pacemaker._cts.process import killall, exit_if_proc_running, pipe_communicate, stdout_from_command
+from pacemaker._cts.test import Test, Tests
+
+# File permissions for executable scripts we create
+EXECMODE = stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
+
+def update_path():
+    """ Set the PATH environment variable appropriately for the tests """
+
+    new_path = os.environ['PATH']
+    if os.path.exists("%s/cts-exec.in" % TEST_DIR):
+        print("Running tests from the source tree: %s (%s)" % (BuildOptions._BUILD_DIR, TEST_DIR))
+        # For pacemaker-execd, cts-exec-helper, and pacemaker-remoted
+        new_path = "%s/daemons/execd:%s" % (BuildOptions._BUILD_DIR, new_path)
+        new_path = "%s/tools:%s" % (BuildOptions._BUILD_DIR, new_path)   # For crm_resource
+        # For pacemaker-fenced
+        new_path = "%s/daemons/fenced:%s" % (BuildOptions._BUILD_DIR, new_path)
+        # For cts-support
+        new_path = "%s/cts/support:%s" % (BuildOptions._BUILD_DIR, new_path)
+
+    else:
+        print("Running tests from the install tree: %s (not %s)" % (BuildOptions.DAEMON_DIR, TEST_DIR))
+        # For cts-exec-helper, cts-support, pacemaker-execd, pacemaker-fenced,
+        # and pacemaker-remoted
+        new_path = "%s:%s" % (BuildOptions.DAEMON_DIR, new_path)
+
+    print('Using PATH="%s"' % new_path)
+    os.environ['PATH'] = new_path
+
+
+class ExecTest(Test):
+    """ Executor for a single pacemaker-execd regression test """
+
+    def __init__(self, name, description, **kwargs):
+        Test.__init__(self, name, description, **kwargs)
+
+        self.tls = kwargs.get("tls", False)
+
+        if self.tls:
+            self._daemon_location = "pacemaker-remoted"
+        else:
+            self._daemon_location = "pacemaker-execd"
+
+        self._test_tool_location = "cts-exec-helper"
+
+        # We additionally need to keep track of a stonith process.
+        self._stonith_process = None
+
+    def _new_cmd(self, cmd, args, exitcode, **kwargs):
+        """ Add a command to be executed as part of this test """
+
+        if self.verbose and cmd == self._test_tool_location:
+            args += " -V "
+
+        if (cmd == self._test_tool_location) and self.tls:
+            args += " -S "
+
+        kwargs["validate"] = False
+        kwargs["check_rng"] = False
+        kwargs["check_stderr"] = False
+
+        Test._new_cmd(self, cmd, args, exitcode, **kwargs)
+
+    def _kill_daemons(self):
+        killall([
+            "pacemaker-fenced",
+            "lt-pacemaker-fenced",
+            "pacemaker-execd",
+            "lt-pacemaker-execd",
+            "cts-exec-helper",
+            "lt-cts-exec-helper",
+            "pacemaker-remoted",
+        ])
+
+    def _start_daemons(self):
+        if not self.tls:
+            self._stonith_process = subprocess.Popen(["pacemaker-fenced", "-s"])
+
+        cmd = [self._daemon_location, "-l", self.logpath]
+        if self.verbose:
+            cmd += ["-V"]
+
+        self._daemon_process = subprocess.Popen(cmd)
+
+    def clean_environment(self):
+        """ Clean up the host after running a test """
+
+        if self._daemon_process:
+            self._daemon_process.terminate()
+            self._daemon_process.wait()
+
+            if self.verbose:
+                print("Daemon Output Start")
+                logfile = io.open(self.logpath, 'rt', errors='replace')
+                for line in logfile:
+                    print(line.strip().encode('utf-8', 'replace'))
+                print("Daemon Output End")
+
+        if self._stonith_process:
+            self._stonith_process.terminate()
+            self._stonith_process.wait()
+
+        self._daemon_process = None
+        self._stonith_process = None
+
+    def add_cmd(self, args):
+        """ Add a cts-exec-helper command to be executed as part of this test """
+
+        self._new_cmd(self._test_tool_location, args, ExitStatus.OK)
+
+    def add_cmd_and_kill(self, args, kill_proc):
+        """ Add a cts-exec-helper command and system command to be executed as part of this test """
+
+        self._new_cmd(self._test_tool_location, args, ExitStatus.OK, kill=kill_proc)
+
+    def add_cmd_check_stdout(self, args, match, no_match=None):
+        """ Add a command with expected output to be executed as part of this test """
+
+        self._new_cmd(self._test_tool_location, args, ExitStatus.OK,
+                      stdout_match=match, stdout_negative_match=no_match)
+
+    def add_cmd_expected_fail(self, args, exitcode=ExitStatus.ERROR):
+        """ Add a cts-exec-helper command to be executed as part of this test and expected to fail """
+
+        self._new_cmd(self._test_tool_location, args, exitcode)
+
+    def add_sys_cmd(self, cmd, args):
+        """ Add a simple command to be executed as part of this test """
+
+        self._new_cmd(cmd, args, ExitStatus.OK)
+
+    def run(self):
+        """ Execute this test. """
+
+        if self.tls and self.name.count("stonith") != 0:
+            self._result_txt = "SKIPPED - '%s' - disabled when testing pacemaker_remote" % (self.name)
+            print(self._result_txt)
+            return
+
+        Test.run(self)
+
+class ExecTests(Tests):
+    """ Collection of all pacemaker-execd regression tests """
+
+    def __init__(self, **kwargs):
+        Tests.__init__(self, **kwargs)
+
+        self.tls = kwargs.get("tls", False)
+
+        self._action_timeout = " -t 9000 "
+        self._installed_files = []
+        self._rsc_classes = self._setup_rsc_classes()
+
+        print("Testing resource classes", repr(self._rsc_classes))
+
+        self._common_cmds = {
+            "ocf_reg_line"      : "-c register_rsc -r ocf_test_rsc "+self._action_timeout+" -C ocf -P pacemaker -T Dummy",
+            "ocf_reg_event"     : "-l \"NEW_EVENT event_type:register rsc_id:ocf_test_rsc action:none rc:ok op_status:complete\"",
+            "ocf_unreg_line"    : "-c unregister_rsc -r \"ocf_test_rsc\" "+self._action_timeout,
+            "ocf_unreg_event"   : "-l \"NEW_EVENT event_type:unregister rsc_id:ocf_test_rsc action:none rc:ok op_status:complete\"",
+            "ocf_start_line"    : "-c exec -r \"ocf_test_rsc\" -a \"start\" "+self._action_timeout,
+            "ocf_start_event"   : "-l \"NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:start rc:ok op_status:complete\" ",
+            "ocf_stop_line"     : "-c exec -r \"ocf_test_rsc\" -a \"stop\" "+self._action_timeout,
+            "ocf_stop_event"    : "-l \"NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:stop rc:ok op_status:complete\" ",
+            "ocf_monitor_line"  : '-c exec -r ocf_test_rsc -a monitor -i 2s ' + self._action_timeout,
+            "ocf_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout,
+            "ocf_cancel_line"   : '-c cancel -r ocf_test_rsc -a monitor -i 2s ' + self._action_timeout,
+            "ocf_cancel_event"  : "-l \"NEW_EVENT event_type:exec_complete rsc_id:ocf_test_rsc action:monitor rc:ok op_status:Cancelled\" ",
+
+            "systemd_reg_line"      : "-c register_rsc -r systemd_test_rsc " +
+                                      self._action_timeout +
+                                      " -C systemd -T pacemaker-cts-dummyd@3",
+            "systemd_reg_event"     : "-l \"NEW_EVENT event_type:register rsc_id:systemd_test_rsc action:none rc:ok op_status:complete\"",
+            "systemd_unreg_line"    : "-c unregister_rsc -r \"systemd_test_rsc\" "+self._action_timeout,
+            "systemd_unreg_event"   : "-l \"NEW_EVENT event_type:unregister rsc_id:systemd_test_rsc action:none rc:ok op_status:complete\"",
+            "systemd_start_line"    : "-c exec -r \"systemd_test_rsc\" -a \"start\" "+self._action_timeout,
+            "systemd_start_event"   : "-l \"NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:start rc:ok op_status:complete\" ",
+            "systemd_stop_line"     : "-c exec -r \"systemd_test_rsc\" -a \"stop\" "+self._action_timeout,
+            "systemd_stop_event"    : "-l \"NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:stop rc:ok op_status:complete\" ",
+            "systemd_monitor_line"  : '-c exec -r systemd_test_rsc -a monitor -i 2s ' + self._action_timeout,
+            "systemd_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:monitor rc:ok op_status:complete\" -t 15000 ",
+            "systemd_cancel_line"   : '-c cancel -r systemd_test_rsc -a monitor -i 2s ' + self._action_timeout,
+            "systemd_cancel_event"  : "-l \"NEW_EVENT event_type:exec_complete rsc_id:systemd_test_rsc action:monitor rc:ok op_status:Cancelled\" ",
+
+            "upstart_reg_line"      : "-c register_rsc -r upstart_test_rsc "+self._action_timeout+" -C upstart -T pacemaker-cts-dummyd",
+            "upstart_reg_event"     : "-l \"NEW_EVENT event_type:register rsc_id:upstart_test_rsc action:none rc:ok op_status:complete\"",
+            "upstart_unreg_line"    : "-c unregister_rsc -r \"upstart_test_rsc\" "+self._action_timeout,
+            "upstart_unreg_event"   : "-l \"NEW_EVENT event_type:unregister rsc_id:upstart_test_rsc action:none rc:ok op_status:complete\"",
+            "upstart_start_line"    : "-c exec -r \"upstart_test_rsc\" -a \"start\" "+self._action_timeout,
+            "upstart_start_event"   : "-l \"NEW_EVENT event_type:exec_complete rsc_id:upstart_test_rsc action:start rc:ok op_status:complete\" ",
+            "upstart_stop_line"     : "-c exec -r \"upstart_test_rsc\" -a \"stop\" "+self._action_timeout,
+            "upstart_stop_event"    : "-l \"NEW_EVENT event_type:exec_complete rsc_id:upstart_test_rsc action:stop rc:ok op_status:complete\" ",
+            "upstart_monitor_line"  : '-c exec -r upstart_test_rsc -a monitor -i 2s ' + self._action_timeout,
+            "upstart_monitor_event" : '-l "NEW_EVENT event_type:exec_complete rsc_id:upstart_test_rsc action:monitor rc:ok op_status:complete" -t 15000',
+            "upstart_cancel_line"   : '-c cancel -r upstart_test_rsc -a monitor -i 2s ' + self._action_timeout,
+            "upstart_cancel_event"  : "-l \"NEW_EVENT event_type:exec_complete rsc_id:upstart_test_rsc action:monitor rc:ok op_status:Cancelled\" ",
+
+            "service_reg_line"      : "-c register_rsc -r service_test_rsc "+self._action_timeout+" -C service -T LSBDummy",
+            "service_reg_event"     : "-l \"NEW_EVENT event_type:register rsc_id:service_test_rsc action:none rc:ok op_status:complete\"",
+            "service_unreg_line"    : "-c unregister_rsc -r \"service_test_rsc\" "+self._action_timeout,
+            "service_unreg_event"   : "-l \"NEW_EVENT event_type:unregister rsc_id:service_test_rsc action:none rc:ok op_status:complete\"",
+            "service_start_line"    : "-c exec -r \"service_test_rsc\" -a \"start\" "+self._action_timeout,
+            "service_start_event"   : "-l \"NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:start rc:ok op_status:complete\" ",
+            "service_stop_line"     : "-c exec -r \"service_test_rsc\" -a \"stop\" "+self._action_timeout,
+            "service_stop_event"    : "-l \"NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:stop rc:ok op_status:complete\" ",
+            "service_monitor_line"  : '-c exec -r service_test_rsc -a monitor -i 2s ' + self._action_timeout,
+            "service_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout,
+            "service_cancel_line"   : '-c cancel -r service_test_rsc -a monitor -i 2s ' + self._action_timeout,
+            "service_cancel_event"  : "-l \"NEW_EVENT event_type:exec_complete rsc_id:service_test_rsc action:monitor rc:ok op_status:Cancelled\" ",
+
+            "lsb_reg_line"      : "-c register_rsc -r lsb_test_rsc "+self._action_timeout+" -C lsb -T LSBDummy",
+            "lsb_reg_event"     : "-l \"NEW_EVENT event_type:register rsc_id:lsb_test_rsc action:none rc:ok op_status:complete\" ",
+            "lsb_unreg_line"    : "-c unregister_rsc -r \"lsb_test_rsc\" "+self._action_timeout,
+            "lsb_unreg_event"   : "-l \"NEW_EVENT event_type:unregister rsc_id:lsb_test_rsc action:none rc:ok op_status:complete\"",
+            "lsb_start_line"    : "-c exec -r \"lsb_test_rsc\" -a \"start\" "+self._action_timeout,
+            "lsb_start_event"   : "-l \"NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:start rc:ok op_status:complete\" ",
+            "lsb_stop_line"     : "-c exec -r \"lsb_test_rsc\" -a \"stop\" "+self._action_timeout,
+            "lsb_stop_event"    : "-l \"NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:stop rc:ok op_status:complete\" ",
+            "lsb_monitor_line"  : '-c exec -r lsb_test_rsc -a status -i 2s ' + self._action_timeout,
+            "lsb_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:status rc:ok op_status:complete\" "+self._action_timeout,
+            "lsb_cancel_line"   : '-c cancel -r lsb_test_rsc -a status -i 2s ' + self._action_timeout,
+            "lsb_cancel_event"  : "-l \"NEW_EVENT event_type:exec_complete rsc_id:lsb_test_rsc action:status rc:ok op_status:Cancelled\" ",
+
+            "stonith_reg_line"      : "-c register_rsc -r stonith_test_rsc " + self._action_timeout +
+				      " -C stonith -P pacemaker -T fence_dummy",
+            "stonith_reg_event"     : "-l \"NEW_EVENT event_type:register rsc_id:stonith_test_rsc action:none rc:ok op_status:complete\" ",
+            "stonith_unreg_line"    : "-c unregister_rsc -r \"stonith_test_rsc\" "+self._action_timeout,
+            "stonith_unreg_event"   : "-l \"NEW_EVENT event_type:unregister rsc_id:stonith_test_rsc action:none rc:ok op_status:complete\"",
+            "stonith_start_line"    : '-c exec -r stonith_test_rsc -a start ' + self._action_timeout,
+            "stonith_start_event"   : "-l \"NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:start rc:ok op_status:complete\" ",
+            "stonith_stop_line"     : "-c exec -r \"stonith_test_rsc\" -a \"stop\" "+self._action_timeout,
+            "stonith_stop_event"    : "-l \"NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:stop rc:ok op_status:complete\" ",
+            "stonith_monitor_line"  : '-c exec -r stonith_test_rsc -a monitor -i 2s ' + self._action_timeout,
+            "stonith_monitor_event" : "-l \"NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout,
+            "stonith_cancel_line"   : '-c cancel -r stonith_test_rsc -a monitor -i 2s ' + self._action_timeout,
+            "stonith_cancel_event"  : "-l \"NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:ok op_status:Cancelled\" ",
+        }
+
+    def _setup_rsc_classes(self):
+        classes = stdout_from_command(["crm_resource", "--list-standards"])
+        classes = classes[:-1] # Strip trailing empty line
+
+        if self.tls:
+            classes.remove("stonith")
+
+        if "nagios" in classes:
+            classes.remove("nagios")
+
+        if "systemd" in classes:
+            try:
+                # This code doesn't need this import, but pacemaker-cts-dummyd
+                # does, so ensure the dependency is available rather than cause
+                # all systemd tests to fail.
+                import systemd.daemon
+            except ImportError:
+                print("Python systemd bindings not found.")
+                print("The tests for systemd class are not going to be run.")
+                classes.remove("systemd")
+
+        return classes
+
+    def new_test(self, name, description):
+        """ Create a named test """
+
+        test = ExecTest(name, description, verbose=self.verbose, tls=self.tls,
+                        timeout=self.timeout, force_wait=self.force_wait,
+                        logdir=self.logdir)
+        self._tests.append(test)
+        return test
+
+    def setup_environment(self):
+        """ Prepare the host before executing any tests """
+
+        if BuildOptions.REMOTE_ENABLED:
+            os.system("service pacemaker_remote stop")
+        self.cleanup_environment()
+
+        if self.tls and not os.path.isfile("/etc/pacemaker/authkey"):
+            print("Installing /etc/pacemaker/authkey ...")
+            os.system("mkdir -p /etc/pacemaker")
+            os.system("dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1")
+            self._installed_files.append("/etc/pacemaker/authkey")
+
+        # If we're in build directory, install agents if not already installed
+        if os.path.exists("%s/cts/cts-exec.in" % BuildOptions._BUILD_DIR):
+
+            if not os.path.exists("%s/pacemaker" % BuildOptions.OCF_RA_INSTALL_DIR):
+                # @TODO remember which components were created and remove them
+                os.makedirs("%s/pacemaker" % BuildOptions.OCF_RA_INSTALL_DIR, 0o755)
+
+            for agent in ["Dummy", "Stateful", "ping"]:
+                agent_source = "%s/extra/resources/%s" % (BuildOptions._BUILD_DIR, agent)
+                agent_dest = "%s/pacemaker/%s" % (BuildOptions.OCF_RA_INSTALL_DIR, agent)
+                if not os.path.exists(agent_dest):
+                    print("Installing %s ..." % (agent_dest))
+                    shutil.copyfile(agent_source, agent_dest)
+                    os.chmod(agent_dest, EXECMODE)
+                    self._installed_files.append(agent_dest)
+
+        subprocess.call(["cts-support", "install"])
+
+    def cleanup_environment(self):
+        """ Clean up the host after executing desired tests """
+
+        for installed_file in self._installed_files:
+            print("Removing %s ..." % (installed_file))
+            os.remove(installed_file)
+
+        subprocess.call(["cts-support", "uninstall"])
+
+    def build_generic_tests(self):
+        """ Register tests that apply to all resource classes """
+
+        common_cmds = self._common_cmds
+
+        ### register/unregister tests ###
+        for rsc in self._rsc_classes:
+            test = self.new_test("generic_registration_%s" % (rsc),
+                                 "Simple resource registration test for %s standard" % (rsc))
+            test.add_cmd(common_cmds["%s_reg_line" % (rsc)] + " " + common_cmds["%s_reg_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])
+
+        ### start/stop tests  ###
+        for rsc in self._rsc_classes:
+            test = self.new_test("generic_start_stop_%s" % (rsc), "Simple start and stop test for %s standard" % (rsc))
+            test.add_cmd(common_cmds["%s_reg_line" % (rsc)]   + " " + common_cmds["%s_reg_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_stop_line" % (rsc)]  + " " + common_cmds["%s_stop_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])
+
+        ### monitor cancel test ###
+        for rsc in self._rsc_classes:
+            test = self.new_test("generic_monitor_cancel_%s" % (rsc),
+                                 "Simple monitor cancel test for %s standard" % (rsc))
+            test.add_cmd(common_cmds["%s_reg_line" % (rsc)]   + " " + common_cmds["%s_reg_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
+            ### If this fails, that means the monitor may not be getting rescheduled ####
+            test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
+            ### If this fails, that means the monitor may not be getting rescheduled ####
+            test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_cancel_line" % (rsc)] + " " + common_cmds["%s_cancel_event" % (rsc)])
+            ### If this happens the monitor did not actually cancel correctly. ###
+            test.add_cmd_expected_fail(common_cmds["%s_monitor_event" % (rsc)], ExitStatus.TIMEOUT)
+            ### If this happens the monitor did not actually cancel correctly. ###
+            test.add_cmd_expected_fail(common_cmds["%s_monitor_event" % (rsc)], ExitStatus.TIMEOUT)
+            test.add_cmd(common_cmds["%s_stop_line" % (rsc)]  + " " + common_cmds["%s_stop_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])
+
+        ### monitor duplicate test ###
+        for rsc in self._rsc_classes:
+            test = self.new_test("generic_monitor_duplicate_%s" % (rsc),
+                                 "Test creation and canceling of duplicate monitors for %s standard" % (rsc))
+            test.add_cmd(common_cmds["%s_reg_line" % (rsc)]   + " " + common_cmds["%s_reg_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
+            ### If this fails, that means the monitor may not be getting rescheduled ####
+            test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
+            ### If this fails, that means the monitor may not be getting rescheduled ####
+            test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
+
+            # Add the duplicate monitors
+            test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
+            # verify we still get update events
+            ### If this fails, that means the monitor may not be getting rescheduled ####
+            test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
+
+            # cancel the monitor, if the duplicate merged with the original, we should no longer see monitor updates
+            test.add_cmd(common_cmds["%s_cancel_line" % (rsc)] + " " + common_cmds["%s_cancel_event" % (rsc)])
+            ### If this happens the monitor did not actually cancel correctly. ###
+            test.add_cmd_expected_fail(common_cmds["%s_monitor_event" % (rsc)], ExitStatus.TIMEOUT)
+            ### If this happens the monitor did not actually cancel correctly. ###
+            test.add_cmd_expected_fail(common_cmds["%s_monitor_event" % (rsc)], ExitStatus.TIMEOUT)
+            test.add_cmd(common_cmds["%s_stop_line" % (rsc)]  + " " + common_cmds["%s_stop_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])
+
+        ### stop implies cancel test ###
+        for rsc in self._rsc_classes:
+            test = self.new_test("generic_stop_implies_cancel_%s" % (rsc),
+                                 "Verify stopping a resource implies cancel of recurring ops for %s standard" % (rsc))
+            test.add_cmd(common_cmds["%s_reg_line" % (rsc)]   + " " + common_cmds["%s_reg_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
+            ### If this fails, that means the monitor may not be getting rescheduled ####
+            test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
+            ### If this fails, that means the monitor may not be getting rescheduled ####
+            test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
+            test.add_cmd(common_cmds["%s_stop_line" % (rsc)]  + " " + common_cmds["%s_stop_event" % (rsc)])
+            ### If this happens the monitor did not actually cancel correctly. ###
+            test.add_cmd_expected_fail(common_cmds["%s_monitor_event" % (rsc)], ExitStatus.TIMEOUT)
+            ### If this happens the monitor did not actually cancel correctly. ###
+            test.add_cmd_expected_fail(common_cmds["%s_monitor_event" % (rsc)], ExitStatus.TIMEOUT)
+            test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])
+
+
+    def build_multi_rsc_tests(self):
+        """ Register complex tests that involve managing multiple resouces of different types """
+
+        common_cmds = self._common_cmds
+        # do not use service and systemd at the same time, it is the same resource.
+
+        ### register start monitor stop unregister resources of each type at the same time. ###
+        test = self.new_test("multi_rsc_start_stop_all",
+                             "Start, monitor, and stop resources of multiple types and classes")
+        for rsc in self._rsc_classes:
+            test.add_cmd(common_cmds["%s_reg_line" % (rsc)]   + " " + common_cmds["%s_reg_event" % (rsc)])
+        for rsc in self._rsc_classes:
+            test.add_cmd(common_cmds["%s_start_line" % (rsc)] + " " + common_cmds["%s_start_event" % (rsc)])
+        for rsc in self._rsc_classes:
+            test.add_cmd(common_cmds["%s_monitor_line" % (rsc)] + " " + common_cmds["%s_monitor_event" % (rsc)])
+        for rsc in self._rsc_classes:
+            ### If this fails, that means the monitor is not being rescheduled ####
+            test.add_cmd(common_cmds["%s_monitor_event" % (rsc)])
+        for rsc in self._rsc_classes:
+            test.add_cmd(common_cmds["%s_cancel_line" % (rsc)] + " " + common_cmds["%s_cancel_event" % (rsc)])
+        for rsc in self._rsc_classes:
+            test.add_cmd(common_cmds["%s_stop_line" % (rsc)]  + " " + common_cmds["%s_stop_event" % (rsc)])
+        for rsc in self._rsc_classes:
+            test.add_cmd(common_cmds["%s_unreg_line" % (rsc)] + " " + common_cmds["%s_unreg_event" % (rsc)])
+
+    def build_negative_tests(self):
+        """ Register tests related to how pacemaker-execd handles failures """
+
+        ### ocf start timeout test  ###
+        test = self.new_test("ocf_start_timeout", "Force start timeout to occur, verify start failure.")
+        test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "
+                     + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+        # -t must be less than self._action_timeout
+        test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" -k \"op_sleep\" -v \"5\" -t 1000 -w")
+        test.add_cmd('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:error op_status:Timed Out" '
+                     + self._action_timeout)
+        test.add_cmd("-c exec -r test_rsc -a stop " + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\" ")
+        test.add_cmd("-c unregister_rsc -r test_rsc " + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### stonith start timeout test  ###
+        test = self.new_test("stonith_start_timeout", "Force start timeout to occur, verify start failure.")
+        test.add_cmd('-c register_rsc -r test_rsc ' +
+                     '-C stonith -P pacemaker -T fence_dummy ' +
+                     self._action_timeout +
+                     '-l "NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete"')
+        test.add_cmd('-c exec -r test_rsc -a start -k monitor_delay -v 30 ' +
+                     '-t 1000 -w') # -t must be less than self._action_timeout
+        test.add_cmd('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:error op_status:Timed Out" '
+                     + self._action_timeout)
+        test.add_cmd("-c exec -r test_rsc -a stop " + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\" ")
+        test.add_cmd("-c unregister_rsc -r test_rsc " + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### stonith component fail ###
+        common_cmds = self._common_cmds
+        test = self.new_test("stonith_component_fail", "Kill stonith component after pacemaker-execd connects")
+        test.add_cmd(common_cmds["stonith_reg_line"]   + " " + common_cmds["stonith_reg_event"])
+        test.add_cmd(common_cmds["stonith_start_line"] + " " + common_cmds["stonith_start_event"])
+
+        test.add_cmd('-c exec -r stonith_test_rsc -a monitor -i 600s '
+                     '-l "NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:ok op_status:complete" '
+                     + self._action_timeout)
+
+        test.add_cmd_and_kill('-l "NEW_EVENT event_type:exec_complete rsc_id:stonith_test_rsc action:monitor rc:error op_status:error" -t 15000',
+                              "killall -9 -q pacemaker-fenced lt-pacemaker-fenced")
+        test.add_cmd(common_cmds["stonith_unreg_line"] + " " + common_cmds["stonith_unreg_event"])
+
+
+        ### monitor fail for ocf resources ###
+        test = self.new_test("monitor_fail_ocf", "Force ocf monitor to fail, verify failure is reported.")
+        test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "
+                     + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+        test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" " + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
+        test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" " + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
+        test.add_cmd('-c exec -r test_rsc -a monitor -i 1s '
+                     + self._action_timeout +
+                     '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"')
+        test.add_cmd('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"'
+                     + self._action_timeout)
+        test.add_cmd('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"'
+                     + self._action_timeout)
+        test.add_cmd_and_kill('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete" ' + self._action_timeout,
+                              "rm -f %s/run/Dummy-test_rsc.state" % BuildOptions.LOCAL_STATE_DIR)
+        test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s ' + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
+        test.add_cmd_expected_fail("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete\" "
+                                   + self._action_timeout, ExitStatus.TIMEOUT)
+        test.add_cmd_expected_fail("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "
+                                   + self._action_timeout, ExitStatus.TIMEOUT)
+        test.add_cmd("-c unregister_rsc -r \"test_rsc\" "
+                     + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### verify notify changes only for monitor operation.  ###
+        test = self.new_test("monitor_changes_only", "Verify when flag is set, only monitor changes are notified.")
+        test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+        test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+" -o "
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
+        test.add_cmd('-c exec -r test_rsc -a monitor -i 1s '
+                     + self._action_timeout +
+                     ' -o -l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete" ')
+        test.add_cmd_expected_fail("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout, ExitStatus.TIMEOUT)
+        test.add_cmd_and_kill('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete"' + self._action_timeout,
+                              'rm -f %s/run/Dummy-test_rsc.state' % BuildOptions.LOCAL_STATE_DIR)
+        test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s'
+                     + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
+        test.add_cmd_expected_fail("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete\" "+self._action_timeout, ExitStatus.TIMEOUT)
+        test.add_cmd_expected_fail("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout, ExitStatus.TIMEOUT)
+        test.add_cmd('-c unregister_rsc -r "test_rsc" ' + self._action_timeout +
+                     '-l "NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete"')
+
+        ### monitor fail for systemd resource ###
+        if "systemd" in self._rsc_classes:
+            test = self.new_test("monitor_fail_systemd", "Force systemd monitor to fail, verify failure is reported..")
+            test.add_cmd("-c register_rsc -r \"test_rsc\" -C systemd -T pacemaker-cts-dummyd@3 " +
+                         self._action_timeout +
+                         "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+            test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
+            test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
+            test.add_cmd('-c exec -r test_rsc -a monitor -i 1s '
+                         + self._action_timeout +
+                         "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" ")
+            test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout)
+            test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout)
+            test.add_cmd_and_kill('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete"' + self._action_timeout,
+                                  "pkill -9 -f pacemaker-cts-dummyd")
+            test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s' + self._action_timeout +
+                         "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
+            test.add_cmd_expected_fail("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete\" "+self._action_timeout, ExitStatus.TIMEOUT)
+            test.add_cmd_expected_fail("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout, ExitStatus.TIMEOUT)
+            test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### monitor fail for upstart resource ###
+        if "upstart" in self._rsc_classes:
+            test = self.new_test("monitor_fail_upstart", "Force upstart monitor to fail, verify failure is reported..")
+            test.add_cmd("-c register_rsc -r \"test_rsc\" -C upstart -T pacemaker-cts-dummyd "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+            test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
+            test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
+            test.add_cmd('-c exec -r test_rsc -a monitor -i 1s ' + self._action_timeout +
+                         "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" ")
+            test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout)
+            test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout)
+            test.add_cmd_and_kill('-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete"' + self._action_timeout,
+                                  'killall -9 -q dd')
+            test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s'
+                         + self._action_timeout +
+                         "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
+            test.add_cmd_expected_fail("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:complete\" "+self._action_timeout, ExitStatus.TIMEOUT)
+            test.add_cmd_expected_fail("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout, ExitStatus.TIMEOUT)
+            test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### Cancel non-existent operation on a resource ###
+        test = self.new_test("cancel_non_existent_op", "Attempt to cancel the wrong monitor operation, verify expected failure")
+        test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+        test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
+        test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
+        test.add_cmd('-c exec -r test_rsc -a monitor -i 1s '
+                     + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" ")
+        test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout)
+        ### interval is wrong, should fail
+        test.add_cmd_expected_fail('-c cancel -r test_rsc -a monitor -i 2s' + self._action_timeout +
+                                   "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
+        ### action name is wrong, should fail
+        test.add_cmd_expected_fail('-c cancel -r test_rsc -a stop -i 1s' + self._action_timeout +
+                                   "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:not running op_status:Cancelled\" ")
+        test.add_cmd("-c unregister_rsc -r \"test_rsc\" " + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### Attempt to invoke non-existent rsc id ###
+        test = self.new_test("invoke_non_existent_rsc", "Attempt to perform operations on a non-existent rsc id.")
+        test.add_cmd_expected_fail("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                                   "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:error op_status:complete\" ")
+        test.add_cmd_expected_fail("-c exec -r test_rsc -a stop "+self._action_timeout+
+                                   "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\" ")
+        test.add_cmd_expected_fail('-c exec -r test_rsc -a monitor -i 6s '
+                                   + self._action_timeout +
+                                   "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" ")
+        test.add_cmd_expected_fail("-c cancel -r test_rsc -a start "+self._action_timeout+
+                                   "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Cancelled\" ")
+        test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### Register and start a resource that doesn't exist, systemd  ###
+        if "systemd" in self._rsc_classes:
+            test = self.new_test("start_uninstalled_systemd", "Register uninstalled systemd agent, try to start, verify expected failure")
+            test.add_cmd("-c register_rsc -r \"test_rsc\" -C systemd -T this_is_fake1234 "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+            test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed\" ")
+            test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        if "upstart" in self._rsc_classes:
+            test = self.new_test("start_uninstalled_upstart", "Register uninstalled upstart agent, try to start, verify expected failure")
+            test.add_cmd("-c register_rsc -r \"test_rsc\" -C upstart -T this_is_fake1234 "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+            test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed\" ")
+            test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self._action_timeout+
+                         "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### Register and start a resource that doesn't exist, ocf ###
+        test = self.new_test("start_uninstalled_ocf", "Register uninstalled ocf agent, try to start, verify expected failure.")
+        test.add_cmd("-c register_rsc -r \"test_rsc\" -C ocf -P pacemaker -T this_is_fake1234 "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+        test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed\" ")
+        test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### Register ocf with non-existent provider  ###
+        test = self.new_test("start_ocf_bad_provider", "Register ocf agent with a non-existent provider, verify expected failure.")
+        test.add_cmd("-c register_rsc -r \"test_rsc\" -C ocf -P pancakes -T Dummy "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+        test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:not installed op_status:Not installed\" ")
+        test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### Register ocf with empty provider field  ###
+        test = self.new_test("start_ocf_no_provider", "Register ocf agent with a no provider, verify expected failure.")
+        test.add_cmd_expected_fail("-c register_rsc -r \"test_rsc\" -C ocf -T Dummy "+self._action_timeout+
+                                   "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+        test.add_cmd_expected_fail("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                                   "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Error\" ")
+        test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+    def build_stress_tests(self):
+        """ Register stress tests """
+
+        timeout = "-t 20000"
+
+        iterations = 25
+        test = self.new_test("ocf_stress", "Verify OCF agent handling works under load")
+        for i in range(iterations):
+            test.add_cmd("-c register_rsc -r rsc_%s %s -C ocf -P heartbeat -T Dummy -l \"NEW_EVENT event_type:register rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
+            test.add_cmd("-c exec -r rsc_%s -a start %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:start rc:ok op_status:complete\"" % (i, timeout, i))
+            test.add_cmd('-c exec -r rsc_%s -a monitor %s -i 1s '
+                         '-l "NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:monitor rc:ok op_status:complete"' % (i, timeout, i))
+        for i in range(iterations):
+            test.add_cmd("-c exec -r rsc_%s -a stop %s  -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:stop rc:ok op_status:complete\"" % (i, timeout, i))
+            test.add_cmd("-c unregister_rsc -r rsc_%s %s -l \"NEW_EVENT event_type:unregister rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
+
+
+        if "systemd" in self._rsc_classes:
+            test = self.new_test("systemd_stress", "Verify systemd dbus connection works under load")
+            for i in range(iterations):
+                test.add_cmd("-c register_rsc -r rsc_%s %s -C systemd -T pacemaker-cts-dummyd@3 -l \"NEW_EVENT event_type:register rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
+                test.add_cmd("-c exec -r rsc_%s -a start %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:start rc:ok op_status:complete\"" % (i, timeout, i))
+                test.add_cmd('-c exec -r rsc_%s -a monitor %s -i 1s '
+                             '-l "NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:monitor rc:ok op_status:complete"' % (i, timeout, i))
+
+            for i in range(iterations):
+                test.add_cmd("-c exec -r rsc_%s -a stop %s -l \"NEW_EVENT event_type:exec_complete rsc_id:rsc_%s action:stop rc:ok op_status:complete\"" % (i, timeout, i))
+                test.add_cmd("-c unregister_rsc -r rsc_%s %s -l \"NEW_EVENT event_type:unregister rsc_id:rsc_%s action:none rc:ok op_status:complete\"" % (i, timeout, i))
+
+        iterations = 9
+        timeout = "-t 30000"
+        ### Verify recurring op in-flight collision is handled in series properly
+        test = self.new_test("rsc_inflight_collision", "Verify recurring ops do not collide with other operations for the same rsc.")
+        test.add_cmd("-c register_rsc -r test_rsc -P pacemaker -C ocf -T Dummy "
+                     "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" "+self._action_timeout)
+        test.add_cmd("-c exec -r test_rsc -a start %s -k op_sleep -v 1 -l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\"" % (timeout))
+        for i in range(iterations):
+            test.add_cmd('-c exec -r test_rsc -a monitor %s -i 100%dms '
+                         '-k op_sleep -v 2 '
+                         '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"' % (timeout, i))
+
+        test.add_cmd("-c exec -r test_rsc -a stop %s -l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\"" % (timeout))
+        test.add_cmd("-c unregister_rsc -r test_rsc %s -l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\"" % (timeout))
+
+    def build_custom_tests(self):
+        """ Register tests that target specific cases """
+
+        ### verify resource temporary folder is created and used by OCF agents.  ###
+        test = self.new_test("rsc_tmp_dir", "Verify creation and use of rsc temporary state directory")
+        test.add_sys_cmd("ls", "-al %s" % BuildOptions.RSC_TMP_DIR)
+        test.add_cmd("-c register_rsc -r test_rsc -P heartbeat -C ocf -T Dummy "
+                     "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" "+self._action_timeout)
+        test.add_cmd("-c exec -r test_rsc -a start -t 4000")
+        test.add_sys_cmd("ls", "-al %s" % BuildOptions.RSC_TMP_DIR)
+        test.add_sys_cmd("ls", "%s/Dummy-test_rsc.state" % BuildOptions.RSC_TMP_DIR)
+        test.add_cmd("-c exec -r test_rsc -a stop -t 4000")
+        test.add_cmd("-c unregister_rsc -r test_rsc "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### start delay then stop test ###
+        test = self.new_test("start_delay", "Verify start delay works as expected.")
+        test.add_cmd("-c register_rsc -r test_rsc -P pacemaker -C ocf -T Dummy "
+                     "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" "+self._action_timeout)
+        test.add_cmd("-c exec -r test_rsc -s 6000 -a start -w -t 6000")
+        test.add_cmd_expected_fail("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" -t 2000", ExitStatus.TIMEOUT)
+        test.add_cmd("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" -t 6000")
+        test.add_cmd("-c exec -r test_rsc -a stop " + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:stop rc:ok op_status:complete\" ")
+        test.add_cmd("-c unregister_rsc -r test_rsc " + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### start delay, but cancel before it gets a chance to start.  ###
+        test = self.new_test("start_delay_cancel", "Using start_delay, start a rsc, but cancel the start op before execution.")
+        test.add_cmd("-c register_rsc -r test_rsc -P pacemaker -C ocf -T Dummy "
+                     "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" "+self._action_timeout)
+        test.add_cmd("-c exec -r test_rsc -s 5000 -a start -w -t 4000")
+        test.add_cmd("-c cancel -r test_rsc -a start " + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:Cancelled\" ")
+        test.add_cmd_expected_fail("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" -t 5000", ExitStatus.TIMEOUT)
+        test.add_cmd("-c unregister_rsc -r test_rsc " + self._action_timeout +
+                     "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### Register a bunch of resources, verify we can get info on them ###
+        test = self.new_test("verify_get_rsc_info", "Register multiple resources, verify retrieval of rsc info.")
+        if "systemd" in self._rsc_classes:
+            test.add_cmd("-c register_rsc -r rsc1 -C systemd -T pacemaker-cts-dummyd@3 "+self._action_timeout)
+            test.add_cmd("-c get_rsc_info -r rsc1 ")
+            test.add_cmd("-c unregister_rsc -r rsc1 "+self._action_timeout)
+            test.add_cmd_expected_fail("-c get_rsc_info -r rsc1 ")
+
+        if "upstart" in self._rsc_classes:
+            test.add_cmd("-c register_rsc -r rsc1 -C upstart -T pacemaker-cts-dummyd "+self._action_timeout)
+            test.add_cmd("-c get_rsc_info -r rsc1 ")
+            test.add_cmd("-c unregister_rsc -r rsc1 "+self._action_timeout)
+            test.add_cmd_expected_fail("-c get_rsc_info -r rsc1 ")
+
+        test.add_cmd("-c register_rsc -r rsc2 -C ocf -T Dummy -P pacemaker "+self._action_timeout)
+        test.add_cmd("-c get_rsc_info -r rsc2 ")
+        test.add_cmd("-c unregister_rsc -r rsc2 "+self._action_timeout)
+        test.add_cmd_expected_fail("-c get_rsc_info -r rsc2 ")
+
+        ### Register duplicate, verify only one entry exists and can still be removed.
+        test = self.new_test("duplicate_registration", "Register resource multiple times, verify only one entry exists and can be removed.")
+        test.add_cmd("-c register_rsc -r rsc2 -C ocf -T Dummy -P pacemaker "+self._action_timeout)
+        test.add_cmd_check_stdout("-c get_rsc_info -r rsc2 ", "id:rsc2 class:ocf provider:pacemaker type:Dummy")
+        test.add_cmd("-c register_rsc -r rsc2 -C ocf -T Dummy -P pacemaker "+self._action_timeout)
+        test.add_cmd_check_stdout("-c get_rsc_info -r rsc2 ", "id:rsc2 class:ocf provider:pacemaker type:Dummy")
+        test.add_cmd("-c register_rsc -r rsc2 -C ocf -T Stateful -P pacemaker "+self._action_timeout)
+        test.add_cmd_check_stdout("-c get_rsc_info -r rsc2 ", "id:rsc2 class:ocf provider:pacemaker type:Stateful")
+        test.add_cmd("-c unregister_rsc -r rsc2 "+self._action_timeout)
+        test.add_cmd_expected_fail("-c get_rsc_info -r rsc2 ")
+
+        ### verify the option to only send notification to the original client. ###
+        test = self.new_test("notify_orig_client_only", "Verify option to only send notifications to the client originating the action.")
+        test.add_cmd("-c register_rsc -r \"test_rsc\" -C \"ocf\" -P \"pacemaker\" -T \"Dummy\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:register rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+        test.add_cmd("-c exec -r \"test_rsc\" -a \"start\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:start rc:ok op_status:complete\" ")
+        test.add_cmd('-c exec -r \"test_rsc\" -a \"monitor\" -i 1s '
+                     + self._action_timeout + ' -n '
+                     '-l "NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete"')
+        # this will fail because the monitor notifications should only go to the original caller, which no longer exists.
+        test.add_cmd_expected_fail("-l \"NEW_EVENT event_type:exec_complete rsc_id:test_rsc action:monitor rc:ok op_status:complete\" "+self._action_timeout, ExitStatus.TIMEOUT)
+        test.add_cmd('-c cancel -r test_rsc -a monitor -i 1s -t 6000 ')
+        test.add_cmd("-c unregister_rsc -r \"test_rsc\" "+self._action_timeout+
+                     "-l \"NEW_EVENT event_type:unregister rsc_id:test_rsc action:none rc:ok op_status:complete\" ")
+
+        ### get metadata ###
+        test = self.new_test("get_ocf_metadata", "Retrieve metadata for a resource")
+        test.add_cmd_check_stdout("-c metadata -C \"ocf\" -P \"pacemaker\" -T \"Dummy\"",
+                                  "resource-agent name=\"Dummy\"")
+        test.add_cmd("-c metadata -C \"ocf\" -P \"pacemaker\" -T \"Stateful\"")
+        test.add_cmd_expected_fail("-c metadata -P \"pacemaker\" -T \"Stateful\"")
+        test.add_cmd_expected_fail("-c metadata -C \"ocf\" -P \"pacemaker\" -T \"fake_agent\"")
+
+        ### get metadata ###
+        test = self.new_test("get_lsb_metadata", "Retrieve metadata for a resource")
+        test.add_cmd_check_stdout("-c metadata -C \"lsb\" -T \"LSBDummy\"",
+                                  "resource-agent name='LSBDummy'")
+
+        ### get stonith metadata ###
+        test = self.new_test("get_stonith_metadata", "Retrieve stonith metadata for a resource")
+        test.add_cmd_check_stdout("-c metadata -C \"stonith\" -P \"pacemaker\" -T \"fence_dummy\"",
+                                  "resource-agent name=\"fence_dummy\"")
+
+        ### get metadata ###
+        if "systemd" in self._rsc_classes:
+            test = self.new_test("get_systemd_metadata", "Retrieve metadata for a resource")
+            test.add_cmd_check_stdout("-c metadata -C \"systemd\" -T \"pacemaker-cts-dummyd@\"",
+                                      "resource-agent name=\"pacemaker-cts-dummyd@\"")
+
+        ### get metadata ###
+        if "upstart" in self._rsc_classes:
+            test = self.new_test("get_upstart_metadata", "Retrieve metadata for a resource")
+            test.add_cmd_check_stdout("-c metadata -C \"upstart\" -T \"pacemaker-cts-dummyd\"",
+                                      "resource-agent name=\"pacemaker-cts-dummyd\"")
+
+        ### get ocf providers  ###
+        test = self.new_test("list_ocf_providers",
+                             "Retrieve list of available resource providers, verifies pacemaker is a provider.")
+        test.add_cmd_check_stdout("-c list_ocf_providers ", "pacemaker")
+        test.add_cmd_check_stdout("-c list_ocf_providers -T ping", "pacemaker")
+
+        ### Verify agents only exist in their lists ###
+        test = self.new_test("verify_agent_lists", "Verify the agent lists contain the right data.")
+        test.add_cmd_check_stdout("-c list_agents ", "Stateful")                                  ### ocf ###
+        test.add_cmd_check_stdout("-c list_agents -C ocf", "Stateful")
+        test.add_cmd_check_stdout("-c list_agents -C lsb", "", "Stateful")                        ### should not exist
+        test.add_cmd_check_stdout("-c list_agents -C service", "", "Stateful")                    ### should not exist
+        test.add_cmd_check_stdout("-c list_agents ", "LSBDummy")                                  ### init.d ###
+        test.add_cmd_check_stdout("-c list_agents -C lsb", "LSBDummy")
+        test.add_cmd_check_stdout("-c list_agents -C service", "LSBDummy")
+        test.add_cmd_check_stdout("-c list_agents -C ocf", "", "pacemaker-cts-dummyd@")           ### should not exist
+
+        test.add_cmd_check_stdout("-c list_agents -C ocf", "", "pacemaker-cts-dummyd@")           ### should not exist
+        test.add_cmd_check_stdout("-c list_agents -C lsb", "", "fence_dummy")                     ### should not exist
+        test.add_cmd_check_stdout("-c list_agents -C service", "", "fence_dummy")                 ### should not exist
+        test.add_cmd_check_stdout("-c list_agents -C ocf", "", "fence_dummy")                     ### should not exist
+
+        if "systemd" in self._rsc_classes:
+            test.add_cmd_check_stdout("-c list_agents ", "pacemaker-cts-dummyd@")             ### systemd ###
+            test.add_cmd_check_stdout("-c list_agents -C service", "LSBDummy")
+            test.add_cmd_check_stdout("-c list_agents -C systemd", "", "Stateful")            ### should not exist
+            test.add_cmd_check_stdout("-c list_agents -C systemd", "pacemaker-cts-dummyd@")
+            test.add_cmd_check_stdout("-c list_agents -C systemd", "", "fence_dummy")         ### should not exist
+
+        if "upstart" in self._rsc_classes:
+            test.add_cmd_check_stdout("-c list_agents ", "pacemaker-cts-dummyd")              ### upstart ###
+            test.add_cmd_check_stdout("-c list_agents -C service", "LSBDummy")
+            test.add_cmd_check_stdout("-c list_agents -C upstart", "", "Stateful")            ### should not exist
+            test.add_cmd_check_stdout("-c list_agents -C upstart", "pacemaker-cts-dummyd")
+            test.add_cmd_check_stdout("-c list_agents -C upstart", "", "fence_dummy")         ### should not exist
+
+        if "stonith" in self._rsc_classes:
+            test.add_cmd_check_stdout("-c list_agents -C stonith", "fence_dummy")             ### stonith ###
+            test.add_cmd_check_stdout("-c list_agents -C stonith", "", "pacemaker-cts-dummyd@") ### should not exist
+            test.add_cmd_check_stdout("-c list_agents -C stonith", "", "Stateful")            ### should not exist
+            test.add_cmd_check_stdout("-c list_agents ", "fence_dummy")
+
+def build_options():
+    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
+                                     description="Run pacemaker-execd regression tests",
+                                     epilog="Example: Run only the test 'start_stop'\n"
+                                            "\t " + sys.argv[0] + " --run-only start_stop\n\n"
+                                            "Example: Run only the tests with the string 'systemd' present in them\n"
+                                            "\t " + sys.argv[0] + " --run-only-pattern systemd")
+    parser.add_argument("-l", "--list-tests", action="store_true",
+                        help="Print out all registered tests")
+    parser.add_argument("-p", "--run-only-pattern", metavar='PATTERN',
+                        help="Run only tests matching the given pattern")
+    parser.add_argument("-r", "--run-only", metavar='TEST',
+                        help="Run a specific test")
+    parser.add_argument("-t", "--timeout", type=float, default=2,
+                        help="Up to how many seconds each test case waits for the daemon to "
+                             "be initialized.  Defaults to 2.  The value 0 means no limit.")
+    parser.add_argument("-w", "--force-wait", action="store_true",
+                        help="Each test case waits the default/specified --timeout for the "
+                             "daemon without tracking the log")
+    if BuildOptions.REMOTE_ENABLED:
+        parser.add_argument("-R", "--pacemaker-remote", action="store_true",
+                            help="Test pacemaker-remoted binary instead of pacemaker-execd")
+    parser.add_argument("-V", "--verbose", action="store_true",
+                        help="Verbose output")
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    """ Run pacemaker-execd regression tests as specified by arguments """
+
+    update_path()
+
+    # Ensure all command output is in portable locale for comparison
+    os.environ['LC_ALL'] = "C"
+
+    opts = build_options()
+
+    if opts.pacemaker_remote:
+        daemon_name = "pacemaker-remoted"
+    else:
+        daemon_name = "pacemaker-execd"
+
+    exit_if_proc_running(daemon_name)
+
+    # Create a temporary directory for log files (the directory will
+    # automatically be erased when done)
+    with tempfile.TemporaryDirectory(prefix="cts-exec-") as logdir:
+        tests = ExecTests(verbose=opts.verbose, tls=opts.pacemaker_remote,
+                          timeout=opts.timeout, force_wait=opts.force_wait,
+                          logdir=logdir)
+
+        tests.build_generic_tests()
+        tests.build_multi_rsc_tests()
+        tests.build_negative_tests()
+        tests.build_custom_tests()
+        tests.build_stress_tests()
+
+        if opts.list_tests:
+            tests.print_list()
+            sys.exit(ExitStatus.OK)
+
+        print("Starting ...")
+
+        tests.setup_environment()
+
+        if opts.run_only_pattern:
+            tests.run_tests_matching(opts.run_only_pattern)
+            tests.print_results()
+        elif opts.run_only:
+            tests.run_single(opts.run_only)
+            tests.print_results()
+        else:
+            tests.run_tests()
+            tests.print_results()
+
+        tests.cleanup_environment()
+
+    tests.exit()
+
+
+if __name__ == "__main__":
+    main()