diff options
Diffstat (limited to 'cts/cts-fencing.in')
| -rw-r--r-- | cts/cts-fencing.in | 1114 |
1 files changed, 1114 insertions, 0 deletions
diff --git a/cts/cts-fencing.in b/cts/cts-fencing.in new file mode 100644 index 0000000..c2ed29a --- /dev/null +++ b/cts/cts-fencing.in @@ -0,0 +1,1114 @@ +#!@PYTHON@ +""" Regression tests for Pacemaker's fencer +""" + +__copyright__ = "Copyright 2012-2023 the Pacemaker project contributors" +__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY" + +import argparse +import os +import sys +import subprocess +import tempfile + +# These imports allow running from a source checkout after running `make`. +# Note that while this doesn't necessarily mean it will successfully run tests, +# but being able to see --help output can be useful. +if os.path.exists("@abs_top_srcdir@/python"): + sys.path.insert(0, "@abs_top_srcdir@/python") + +if os.path.exists("@abs_top_builddir@/python") and "@abs_top_builddir@" != "@abs_top_srcdir@": + sys.path.insert(0, "@abs_top_builddir@/python") + +from pacemaker.buildoptions import BuildOptions +from pacemaker.exitstatus import ExitStatus +from pacemaker._cts.corosync import Corosync, localname +from pacemaker._cts.errors import ExitCodeError, OutputFoundError, OutputNotFoundError, XmlValidationError +from pacemaker._cts.process import killall, exit_if_proc_running +from pacemaker._cts.test import Test, Tests + +TEST_DIR = sys.path[0] + +def update_path(): + """ Set the PATH environment variable appropriately for the tests """ + + new_path = os.environ['PATH'] + if os.path.exists("%s/cts-fencing.in" % TEST_DIR): + print("Running tests from the source tree: %s (%s)" % (BuildOptions._BUILD_DIR, TEST_DIR)) + # For pacemaker-fenced and cts-fence-helper + new_path = "%s/daemons/fenced:%s" % (BuildOptions._BUILD_DIR, new_path) + new_path = "%s/tools:%s" % (BuildOptions._BUILD_DIR, new_path) # For stonith_admin + new_path = "%s/cts/support:%s" % (BuildOptions._BUILD_DIR, new_path) # For cts-support + + else: + print("Running tests from the install tree: %s (not %s)" % (BuildOptions.DAEMON_DIR, TEST_DIR)) + # For pacemaker-fenced, cts-fence-helper, and cts-support + new_path = "%s:%s" % (BuildOptions.DAEMON_DIR, new_path) + + print('Using PATH="%s"' % new_path) + os.environ['PATH'] = new_path + + +class FenceTest(Test): + """ Executor for a single test """ + + def __init__(self, name, description, **kwargs): + Test.__init__(self, name, description, **kwargs) + + if kwargs.get("with_cpg", False): + self._enable_corosync = True + self._daemon_options = ["-c"] + else: + self._enable_corosync = False + self._daemon_options = ["-s"] + + self._daemon_location = "pacemaker-fenced" + + def _kill_daemons(self): + killall(["pacemakerd", "pacemaker-fenced"]) + + def _start_daemons(self): + if self.verbose: + self._daemon_options += ["-V"] + print("Starting %s with %s" % (self._daemon_location, self._daemon_options)) + + cmd = ["pacemaker-fenced", "-l", self.logpath] + self._daemon_options + self._daemon_process = subprocess.Popen(cmd) + +class FenceTests(Tests): + """ Collection of all fencing regression tests """ + + def __init__(self, **kwargs): + Tests.__init__(self, **kwargs) + + self._corosync = Corosync(self.verbose, self.logdir, "cts-fencing") + + def new_test(self, name, description, with_cpg=False): + """ Create a named test """ + + test = FenceTest(name, description, verbose=self.verbose, with_cpg=with_cpg, + timeout=self.timeout, force_wait=self.force_wait, + logdir=self.logdir) + self._tests.append(test) + return test + + def run_cpg_only(self): + """ Run all corosync-enabled tests """ + + for test in self._tests: + if test._enable_corosync: + test.run() + + def run_no_cpg(self): + """ Run all standalone tests """ + + for test in self._tests: + if not test._enable_corosync: + test.run() + + def build_api_sanity_tests(self): + """ Register tests to verify basic API usage """ + + verbose_arg = "" + if self.verbose: + verbose_arg = "-V" + + test = self.new_test("standalone_low_level_api_test", "Sanity test client api in standalone mode.") + test.add_cmd("cts-fence-helper", "-t %s" % (verbose_arg), validate=False) + + test = self.new_test("cpg_low_level_api_test", "Sanity test client api using mainloop and cpg.", True) + test.add_cmd("cts-fence-helper", "-m %s" % (verbose_arg), validate=False) + + def build_custom_timeout_tests(self): + """ Register tests to verify custom timeout usage """ + + # custom timeout without topology + test = self.new_test("cpg_custom_timeout_1", + "Verify per device timeouts work as expected without using topology.", True) + test.add_cmd('stonith_admin', + '--output-as=xml -R false1 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node1 node2 node3"') + test.add_cmd('stonith_admin', + '--output-as=xml -R true1 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node3" -o "pcmk_off_timeout=1"') + test.add_cmd('stonith_admin', + '--output-as=xml -R false2 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node3" -o "pcmk_off_timeout=4"') + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5") + # timeout is 5+1+4 = 10 + test.add_log_pattern("Total timeout set to 12") + + # custom timeout _WITH_ topology + test = self.new_test("cpg_custom_timeout_2", + "Verify per device timeouts work as expected _WITH_ topology.", True) + test.add_cmd('stonith_admin', + '--output-as=xml -R false1 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node1 node2 node3"') + test.add_cmd('stonith_admin', + '--output-as=xml -R true1 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node3" -o "pcmk_off_timeout=1"') + test.add_cmd('stonith_admin', + '--output-as=xml -R false2 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node3" -o "pcmk_off_timeout=4000"') + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true1") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v false2") + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5") + # timeout is 5+1+4000 = 4006 + test.add_log_pattern("Total timeout set to 4807") + + def build_fence_merge_tests(self): + """ Register tests to verify when fence operations should be merged """ + + ### Simple test that overlapping fencing operations get merged + test = self.new_test("cpg_custom_merge_single", + "Verify overlapping identical fencing operations are merged, no fencing levels used.", True) + test.add_cmd("stonith_admin", "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"") + test.add_cmd("stonith_admin", "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\" ") + test.add_cmd("stonith_admin", "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"") + test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10") + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 10") + ### one merger will happen + test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") + ### the pattern below signifies that both the original and duplicate operation completed + test.add_log_pattern("Operation 'off' targeting node3 by ") + test.add_log_pattern("Operation 'off' targeting node3 by ") + + ### Test that multiple mergers occur + test = self.new_test("cpg_custom_merge_multiple", + "Verify multiple overlapping identical fencing operations are merged", True) + test.add_cmd("stonith_admin", "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"delay=2\" -o \"pcmk_host_list=node3\" ") + test.add_cmd("stonith_admin", "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"") + test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10") + test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10") + test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10") + test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10") + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 10") + ### 4 mergers should occur + test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") + test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") + test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") + test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") + ### the pattern below signifies that both the original and duplicate operation completed + test.add_log_pattern("Operation 'off' targeting node3 by ") + test.add_log_pattern("Operation 'off' targeting node3 by ") + test.add_log_pattern("Operation 'off' targeting node3 by ") + test.add_log_pattern("Operation 'off' targeting node3 by ") + test.add_log_pattern("Operation 'off' targeting node3 by ") + + ### Test that multiple mergers occur with topologies used + test = self.new_test("cpg_custom_merge_with_topology", + "Verify multiple overlapping identical fencing operations are merged with fencing levels.", + True) + test.add_cmd("stonith_admin", "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"") + test.add_cmd("stonith_admin", "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\" ") + test.add_cmd("stonith_admin", "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false2") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true1") + test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10") + test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10") + test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10") + test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node3 -t 10") + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 10") + ### 4 mergers should occur + test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") + test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") + test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") + test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client") + ### the pattern below signifies that both the original and duplicate operation completed + test.add_log_pattern("Operation 'off' targeting node3 by ") + test.add_log_pattern("Operation 'off' targeting node3 by ") + test.add_log_pattern("Operation 'off' targeting node3 by ") + test.add_log_pattern("Operation 'off' targeting node3 by ") + test.add_log_pattern("Operation 'off' targeting node3 by ") + + def build_fence_no_merge_tests(self): + """ Register tests to verify when fence operations should not be merged """ + + test = self.new_test("cpg_custom_no_merge", + "Verify differing fencing operations are not merged", True) + test.add_cmd("stonith_admin", "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3 node2\"") + test.add_cmd("stonith_admin", "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3 node2\" ") + test.add_cmd("stonith_admin", "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3 node2\"") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false2") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true1") + test.add_cmd_no_wait("stonith_admin", "--output-as=xml -F node2 -t 10") + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 10") + test.add_log_pattern("Merging fencing action 'off' targeting node3 originating from client", + negative=True) + + def build_standalone_tests(self): + """ Register a grab bag of tests that can be executed in standalone or corosync mode """ + + test_types = [ + { + "prefix" : "standalone", + "use_cpg" : False, + }, + { + "prefix" : "cpg", + "use_cpg" : True, + }, + ] + + # test what happens when all devices timeout + for test_type in test_types: + test = self.new_test("%s_fence_multi_device_failure" % test_type["prefix"], + "Verify that all devices timeout, a fencing failure is returned.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R false3 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + if test_type["use_cpg"]: + test.add_cmd_expected_fail("stonith_admin", "--output-as=xml -F node3 -t 2", ExitStatus.TIMEOUT) + test.add_log_pattern("Total timeout set to 7") + else: + test.add_cmd_expected_fail("stonith_admin", "--output-as=xml -F node3 -t 2", ExitStatus.ERROR) + + test.add_log_pattern("targeting node3 using false1 returned ") + test.add_log_pattern("targeting node3 using false2 returned ") + test.add_log_pattern("targeting node3 using false3 returned ") + + # test what happens when multiple devices can fence a node, but the first device fails. + for test_type in test_types: + test = self.new_test("%s_fence_device_failure_rollover" % test_type["prefix"], + "Verify that when one fence device fails for a node, the others are tried.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5") + + if test_type["use_cpg"]: + test.add_log_pattern("Total timeout set to 18") + + # test what happens when we try to use a missing fence-agent. + for test_type in test_types: + test = self.new_test("%s_fence_missing_agent" % test_type["prefix"], + "Verify proper error-handling when using a non-existent fence-agent.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_missing -o \"mode=pass\" -o \"pcmk_host_list=node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node2\"") + + test.add_cmd_expected_fail("stonith_admin", + "--output-as=xml -F node3 -t 5", + ExitStatus.NOSUCH) + test.add_cmd("stonith_admin", "--output-as=xml -F node2 -t 5") + + # simple topology test for one device + for test_type in test_types: + if not test_type["use_cpg"]: + continue + + test = self.new_test("%s_topology_simple" % test_type["prefix"], + "Verify all fencing devices at a level are used.", test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R true -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v true") + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5") + + test.add_log_pattern("Total timeout set to 6") + test.add_log_pattern("targeting node3 using true returned 0") + + + # add topology, delete topology, verify fencing still works + for test_type in test_types: + if not test_type["use_cpg"]: + continue + + test = self.new_test("%s_topology_add_remove" % test_type["prefix"], + "Verify fencing occurrs after all topology levels are removed", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R true -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v true") + test.add_cmd("stonith_admin", "--output-as=xml -d node3 -i 1") + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5") + + test.add_log_pattern("Total timeout set to 6") + test.add_log_pattern("targeting node3 using true returned 0") + + # test what happens when the first fencing level has multiple devices. + for test_type in test_types: + if not test_type["use_cpg"]: + continue + + test = self.new_test("%s_topology_device_fails" % test_type["prefix"], + "Verify if one device in a level fails, the other is tried.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R false -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true") + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 20") + + test.add_log_pattern("Total timeout set to 48") + test.add_log_pattern("targeting node3 using false returned 1") + test.add_log_pattern("targeting node3 using true returned 0") + + # test what happens when the first fencing level fails. + for test_type in test_types: + if not test_type["use_cpg"]: + continue + + test = self.new_test("%s_topology_multi_level_fails" % test_type["prefix"], + "Verify if one level fails, the next leve is tried.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true3 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true4 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v true1") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true2") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v false2") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v true3") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v true4") + + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 3") + + test.add_log_pattern("Total timeout set to 21") + test.add_log_pattern("targeting node3 using false1 returned 1") + test.add_log_pattern("targeting node3 using false2 returned 1") + test.add_log_pattern("targeting node3 using true3 returned 0") + test.add_log_pattern("targeting node3 using true4 returned 0") + + + # test what happens when the first fencing level had devices that no one has registered + for test_type in test_types: + if not test_type["use_cpg"]: + continue + + test = self.new_test("%s_topology_missing_devices" % test_type["prefix"], + "Verify topology can continue with missing devices.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true3 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true4 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v true1") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true2") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v false2") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v true3") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v true4") + + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5") + + # Test what happens if multiple fencing levels are defined, and then the first one is removed. + for test_type in test_types: + if not test_type["use_cpg"]: + continue + + test = self.new_test("%s_topology_level_removal" % test_type["prefix"], + "Verify level removal works.", test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true3 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true4 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R false2 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\"") + + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v true1") + + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true2") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v false2") + + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v true3") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 3 -v true4") + + # Now remove level 2, verify none of the devices in level two are hit. + test.add_cmd("stonith_admin", "--output-as=xml -d node3 -i 2") + + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 20") + + test.add_log_pattern("Total timeout set to 96") + test.add_log_pattern("targeting node3 using false1 returned 1") + test.add_log_pattern("targeting node3 using false2 returned ", + negative=True) + test.add_log_pattern("targeting node3 using true3 returned 0") + test.add_log_pattern("targeting node3 using true4 returned 0") + + # Test targeting a topology level by node name pattern. + for test_type in test_types: + if not test_type["use_cpg"]: + continue + + test = self.new_test("%s_topology_level_pattern" % test_type["prefix"], + "Verify targeting topology by node name pattern works.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + """--output-as=xml -R true -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node1 node2 node3" """) + test.add_cmd("stonith_admin", """--output-as=xml -r '@node.*' -i 1 -v true""") + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5") + test.add_log_pattern("targeting node3 using true returned 0") + + # test allowing commas and semicolons as delimiters in pcmk_host_list + for test_type in test_types: + test = self.new_test("%s_host_list_delimiters" % test_type["prefix"], + "Verify commas and semicolons can be used as pcmk_host_list delimiters", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + """--output-as=xml -R true1 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node1,node2,node3" """) + test.add_cmd("stonith_admin", + """--output-as=xml -R true2 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=pcmk1;pcmk2;pcmk3" """) + test.add_cmd("stonith_admin", "stonith_admin --output-as=xml -F node2 -t 5") + test.add_cmd("stonith_admin", "stonith_admin --output-as=xml -F pcmk3 -t 5") + test.add_log_pattern("targeting node2 using true1 returned 0") + test.add_log_pattern("targeting pcmk3 using true2 returned 0") + + # test the stonith builds the correct list of devices that can fence a node. + for test_type in test_types: + test = self.new_test("%s_list_devices" % test_type["prefix"], + "Verify list of devices that can fence a node is correct", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true3 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd_check_stdout("stonith_admin", "--output-as=xml -l node1 -V", "true2", "true1") + test.add_cmd_check_stdout("stonith_admin", "--output-as=xml -l node1 -V", "true3", "true1") + + # simple test of device monitor + for test_type in test_types: + test = self.new_test("%s_monitor" % test_type["prefix"], + "Verify device is reachable", test_type["use_cpg"]) + test.add_cmd("stonith_admin", "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\"") + test.add_cmd("stonith_admin", "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node3\"") + + test.add_cmd("stonith_admin", "--output-as=xml -Q true1") + test.add_cmd("stonith_admin", "--output-as=xml -Q false1") + test.add_cmd_expected_fail("stonith_admin", + "--output-as=xml -Q true2", + ExitStatus.NOSUCH) + + # Verify monitor occurs for duration of timeout period on failure + for test_type in test_types: + test = self.new_test("%s_monitor_timeout" % test_type["prefix"], + "Verify monitor uses duration of timeout period given.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + '--output-as=xml -R true1 -a fence_dummy -o "mode=fail" -o "monitor_mode=fail" -o "pcmk_host_list=node3"') + test.add_cmd_expected_fail("stonith_admin", "--output-as=xml -Q true1 -t 5", ExitStatus.ERROR) + test.add_log_pattern("Attempt 2 to execute") + + # Verify monitor occurs for duration of timeout period on failure, but stops at max retries + for test_type in test_types: + test = self.new_test("%s_monitor_timeout_max_retries" % test_type["prefix"], + "Verify monitor retries until max retry value or timeout is hit.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + '--output-as=xml -R true1 -a fence_dummy -o "mode=fail" -o "monitor_mode=fail" -o "pcmk_host_list=node3"') + test.add_cmd_expected_fail("stonith_admin", "--output-as=xml -Q true1 -t 15", ExitStatus.ERROR) + test.add_log_pattern("Attempted to execute agent fence_dummy (list) the maximum number of times") + + # simple register test + for test_type in test_types: + test = self.new_test("%s_register" % test_type["prefix"], + "Verify devices can be registered and un-registered", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\"") + + test.add_cmd("stonith_admin", "--output-as=xml -Q true1") + + test.add_cmd("stonith_admin", "--output-as=xml -D true1") + + test.add_cmd_expected_fail("stonith_admin", + "--output-as=xml -Q true1", + ExitStatus.NOSUCH) + + # simple reboot test + for test_type in test_types: + test = self.new_test("%s_reboot" % test_type["prefix"], + "Verify devices can be rebooted", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\"") + + test.add_cmd("stonith_admin", "--output-as=xml -B node3 -t 5") + + test.add_cmd("stonith_admin", "--output-as=xml -D true1") + + test.add_cmd_expected_fail("stonith_admin", + "--output-as=xml -Q true1", + ExitStatus.NOSUCH) + + # test fencing history. + for test_type in test_types: + if not test_type["use_cpg"]: + continue + test = self.new_test("%s_fence_history" % test_type["prefix"], + "Verify last fencing operation is returned.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node3\"") + + test.add_cmd("stonith_admin", "--output-as=xml -F node3 -t 5 -V") + + test.add_cmd_check_stdout("stonith_admin", "--output-as=xml -H node3", 'action="off" target="node3" .* status="success"') + + # simple test of dynamic list query + for test_type in test_types: + test = self.new_test("%s_dynamic_list_query" % test_type["prefix"], + "Verify dynamic list of fencing devices can be retrieved.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", "--output-as=xml -R true1 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1") + test.add_cmd("stonith_admin", "--output-as=xml -R true2 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1") + test.add_cmd("stonith_admin", "--output-as=xml -R true3 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1") + + test.add_cmd_check_stdout("stonith_admin", "--output-as=xml -l fake_port_1", 'count="3"') + + + # fence using dynamic list query + for test_type in test_types: + test = self.new_test("%s_fence_dynamic_list_query" % test_type["prefix"], + "Verify dynamic list of fencing devices can be retrieved.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", "--output-as=xml -R true1 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1") + test.add_cmd("stonith_admin", "--output-as=xml -R true2 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1") + test.add_cmd("stonith_admin", "--output-as=xml -R true3 -a fence_dummy -o mode=pass -o mock_dynamic_hosts=fake_port_1") + + test.add_cmd("stonith_admin", "--output-as=xml -F fake_port_1 -t 5 -V") + + # simple test of query using status action + for test_type in test_types: + test = self.new_test("%s_status_query" % test_type["prefix"], + "Verify dynamic list of fencing devices can be retrieved.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_check=status\"") + test.add_cmd("stonith_admin", "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_check=status\"") + test.add_cmd("stonith_admin", "--output-as=xml -R true3 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_check=status\"") + + test.add_cmd_check_stdout("stonith_admin", "--output-as=xml -l fake_port_1", 'count="3"') + + # test what happens when no reboot action is advertised + for test_type in test_types: + test = self.new_test("%s_no_reboot_support" % test_type["prefix"], + "Verify reboot action defaults to off when no reboot action is advertised by agent.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy_no_reboot -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", "--output-as=xml -B node1 -t 5 -V") + test.add_log_pattern("does not support reboot") + test.add_log_pattern("using true1 returned 0") + + # make sure reboot is used when reboot action is advertised + for test_type in test_types: + test = self.new_test("%s_with_reboot_support" % test_type["prefix"], + "Verify reboot action can be used when metadata advertises it.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + test.add_cmd("stonith_admin", "--output-as=xml -B node1 -t 5 -V") + test.add_log_pattern("does not advertise support for 'reboot', performing 'off'", + negative=True) + test.add_log_pattern("using true1 returned 0") + + # make sure all fencing delays are applied correctly and taken into account by fencing timeouts with topology + for test_type in test_types: + if not test_type["use_cpg"]: + continue + + test = self.new_test("%s_topology_delays" % test_type["prefix"], + "Verify all fencing delays are applied correctly and taken into account by fencing timeouts with topology.", + test_type["use_cpg"]) + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\" -o \"pcmk_delay_base=1\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R false1 -a fence_dummy -o \"mode=fail\" -o \"pcmk_host_list=node1 node2 node3\" -o \"pcmk_delay_base=1\"") + # Resulting "random" delay will always be 1 since (rand() % (delay_max - delay_base)) is always 0 here. + test.add_cmd("stonith_admin", + "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\" -o \"pcmk_delay_base=1\" -o \"pcmk_delay_max=2\"") + test.add_cmd("stonith_admin", + "--output-as=xml -R true3 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=node1 node2 node3\"") + + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v true1") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 1 -v false1") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true2") + test.add_cmd("stonith_admin", "--output-as=xml -r node3 -i 2 -v true3") + + test.add_cmd("stonith_admin", "--output-as=xml -F node3 --delay 1") + + # Total fencing timeout takes all fencing delays into account. + test.add_log_pattern("Total timeout set to 582") + + # Fencing timeout for the first device takes the requested fencing delay into account. + # Fencing timeout also takes pcmk_delay_base into account. + test.add_log_pattern(r"Requesting that .* perform 'off' action targeting node3 using true1 .*146s.*", + regex=True) + # Requested fencing delay is applied only for the first device in the first level. + # Static delay from pcmk_delay_base is added. + test.add_log_pattern("Delaying 'off' action targeting node3 using true1 for 2s | timeout=120s requested_delay=1s base=1s max=1s") + + # Fencing timeout no longer takes the requested fencing delay into account for further devices. + test.add_log_pattern(r"Requesting that .* perform 'off' action targeting node3 using false1 .*145s.*", + regex=True) + # Requested fencing delay is no longer applied for further devices. + test.add_log_pattern("Delaying 'off' action targeting node3 using false1 for 1s | timeout=120s requested_delay=0s base=1s max=1s") + + # Fencing timeout takes pcmk_delay_max into account. + test.add_log_pattern(r"Requesting that .* perform 'off' action targeting node3 using true2 .*146s.*", + regex=True) + test.add_log_pattern("Delaying 'off' action targeting node3 using true2 for 1s | timeout=120s requested_delay=0s base=1s max=2s") + + test.add_log_pattern("Delaying 'off' action targeting node3 using true3", + negative=True) + + def build_nodeid_tests(self): + """ Register tests that use a corosync node id """ + + our_uname = localname() + + ### verify nodeid is supplied when nodeid is in the metadata parameters + test = self.new_test("cpg_supply_nodeid", + "Verify nodeid is given when fence agent has nodeid as parameter", True) + + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s\"" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -F %s -t 3" % (our_uname)) + test.add_log_pattern("as nodeid with fence action 'off' targeting %s" % (our_uname)) + + ### verify nodeid is _NOT_ supplied when nodeid is not in the metadata parameters + test = self.new_test("cpg_do_not_supply_nodeid", + "Verify nodeid is _NOT_ given when fence agent does not have nodeid as parameter", + True) + + # use a host name that won't be in corosync.conf + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=regr-test\"") + test.add_cmd("stonith_admin", "--output-as=xml -F regr-test -t 3") + test.add_log_pattern("as nodeid with fence action 'off' targeting regr-test", + negative=True) + + ### verify nodeid use doesn't explode standalone mode + test = self.new_test("standalone_do_not_supply_nodeid", + "Verify nodeid in metadata parameter list doesn't kill standalone mode", + False) + + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s\"" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -F %s -t 3" % (our_uname)) + test.add_log_pattern("as nodeid with fence action 'off' targeting %s" % our_uname, + negative=True) + + def build_unfence_tests(self): + """ Register tests that verify unfencing """ + + our_uname = localname() + + ### verify unfencing using automatic unfencing + test = self.new_test("cpg_unfence_required_1", + "Verify require unfencing on all devices when automatic=true in agent's metadata", + True) + test.add_cmd('stonith_admin', + '--output-as=xml -R true1 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s"' % (our_uname)) + test.add_cmd('stonith_admin', + '--output-as=xml -R true2 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s"' % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -U %s -t 3" % (our_uname)) + # both devices should be executed + test.add_log_pattern("using true1 returned 0") + test.add_log_pattern("using true2 returned 0") + + ### verify unfencing using automatic unfencing fails if any of the required agents fail + test = self.new_test("cpg_unfence_required_2", + "Verify require unfencing on all devices when automatic=true in agent's metadata", + True) + test.add_cmd('stonith_admin', + '--output-as=xml -R true1 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s"' % (our_uname)) + test.add_cmd('stonith_admin', + '--output-as=xml -R true2 -a fence_dummy_auto_unfence -o "mode=fail" -o "pcmk_host_list=%s"' % (our_uname)) + test.add_cmd_expected_fail("stonith_admin", "--output-as=xml -U %s -t 6" % (our_uname), ExitStatus.ERROR) + + ### verify unfencing using automatic devices with topology + test = self.new_test("cpg_unfence_required_3", + "Verify require unfencing on all devices even when at different topology levels", + True) + test.add_cmd('stonith_admin', + '--output-as=xml -R true1 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s node3"' % (our_uname)) + test.add_cmd('stonith_admin', + '--output-as=xml -R true2 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s node3"' % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 1 -v true1" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 2 -v true2" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -U %s -t 3" % (our_uname)) + test.add_log_pattern("using true1 returned 0") + test.add_log_pattern("using true2 returned 0") + + ### verify unfencing using automatic devices with topology + test = self.new_test("cpg_unfence_required_4", + "Verify all required devices are executed even with topology levels fail.", + True) + test.add_cmd('stonith_admin', + '--output-as=xml -R true1 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s node3"' % (our_uname)) + test.add_cmd('stonith_admin', + '--output-as=xml -R true2 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s node3"' % (our_uname)) + test.add_cmd('stonith_admin', + '--output-as=xml -R true3 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s node3"' % (our_uname)) + test.add_cmd('stonith_admin', + '--output-as=xml -R true4 -a fence_dummy_auto_unfence -o "mode=pass" -o "pcmk_host_list=%s node3"' % (our_uname)) + test.add_cmd('stonith_admin', + '--output-as=xml -R false1 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=%s node3"' % (our_uname)) + test.add_cmd('stonith_admin', + '--output-as=xml -R false2 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=%s node3"' % (our_uname)) + test.add_cmd('stonith_admin', + '--output-as=xml -R false3 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=%s node3"' % (our_uname)) + test.add_cmd('stonith_admin', + '--output-as=xml -R false4 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=%s node3"' % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 1 -v true1" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 1 -v false1" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 2 -v false2" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 2 -v true2" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 2 -v false3" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 2 -v true3" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 3 -v false4" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 4 -v true4" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -U %s -t 3" % (our_uname)) + test.add_log_pattern("using true1 returned 0") + test.add_log_pattern("using true2 returned 0") + test.add_log_pattern("using true3 returned 0") + test.add_log_pattern("using true4 returned 0") + + def build_unfence_on_target_tests(self): + """ Register tests that verify unfencing that runs on the target """ + + our_uname = localname() + + ### verify unfencing using on_target device + test = self.new_test("cpg_unfence_on_target_1", + "Verify unfencing with on_target = true", True) + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s\"" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -U %s -t 3" % (our_uname)) + test.add_log_pattern("(on) to be executed on target") + + ### verify failure of unfencing using on_target device + test = self.new_test("cpg_unfence_on_target_2", + "Verify failure unfencing with on_target = true", + True) + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s node_fake_1234\"" % (our_uname)) + test.add_cmd_expected_fail("stonith_admin", + "--output-as=xml -U node_fake_1234 -t 3", + ExitStatus.NOSUCH) + test.add_log_pattern("(on) to be executed on target") + + ### verify unfencing using on_target device with topology + test = self.new_test("cpg_unfence_on_target_3", + "Verify unfencing with on_target = true using topology", + True) + + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s node3\"" % (our_uname)) + test.add_cmd("stonith_admin", + "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s node3\"" % (our_uname)) + + test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 1 -v true1" % (our_uname)) + test.add_cmd("stonith_admin", "--output-as=xml -r %s -i 2 -v true2" % (our_uname)) + + test.add_cmd("stonith_admin", "--output-as=xml -U %s -t 3" % (our_uname)) + test.add_log_pattern("(on) to be executed on target") + + ### verify unfencing using on_target device with topology fails when target node doesn't exist + test = self.new_test("cpg_unfence_on_target_4", + "Verify unfencing failure with on_target = true using topology", + True) + + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s node_fake\"" % (our_uname)) + test.add_cmd("stonith_admin", + "--output-as=xml -R true2 -a fence_dummy -o \"mode=pass\" -o \"pcmk_host_list=%s node_fake\"" % (our_uname)) + + test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 1 -v true1") + test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 2 -v true2") + + test.add_cmd_expected_fail("stonith_admin", + "--output-as=xml -U node_fake -t 3", + ExitStatus.NOSUCH) + test.add_log_pattern("(on) to be executed on target") + + def build_remap_tests(self): + """ Register tests that verify remapping of reboots to off-on """ + + test = self.new_test("cpg_remap_simple", + "Verify sequential topology reboot is remapped to all-off-then-all-on", True) + test.add_cmd("stonith_admin", + """--output-as=xml -R true1 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """ + """-o "pcmk_off_timeout=1" -o "pcmk_reboot_timeout=10" """) + test.add_cmd("stonith_admin", + """--output-as=xml -R true2 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """ + """-o "pcmk_off_timeout=2" -o "pcmk_reboot_timeout=20" """) + test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 1 -v true1 -v true2") + test.add_cmd("stonith_admin", "--output-as=xml -B node_fake -t 5") + test.add_log_pattern("Remapping multiple-device reboot targeting node_fake") + # timeout should be sum of off timeouts (1+2=3), not reboot timeouts (10+20=30) + test.add_log_pattern("Total timeout set to 3 for peer's fencing targeting node_fake") + test.add_log_pattern("perform 'off' action targeting node_fake using true1") + test.add_log_pattern("perform 'off' action targeting node_fake using true2") + test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'") + # fence_dummy sets "on" as an on_target action + test.add_log_pattern("Ignoring true1 'on' failure (no capable peers) targeting node_fake") + test.add_log_pattern("Ignoring true2 'on' failure (no capable peers) targeting node_fake") + test.add_log_pattern("Undoing remap of reboot targeting node_fake") + + test = self.new_test("cpg_remap_simple_off", + "Verify sequential topology reboot skips 'on' if " + "pcmk_reboot_action=off or agent doesn't support " + "'on'", True) + test.add_cmd("stonith_admin", + "--output-as=xml -R true1 -a fence_dummy -o mode=pass " + "-o pcmk_host_list=node_fake -o pcmk_off_timeout=1 " + "-o pcmk_reboot_timeout=10 -o pcmk_reboot_action=off") + test.add_cmd("stonith_admin", + "--output-as=xml -R true2 -a fence_dummy_no_on " + "-o mode=pass -o pcmk_host_list=node_fake " + "-o pcmk_off_timeout=2 -o pcmk_reboot_timeout=20") + test.add_cmd("stonith_admin", + "--output-as=xml -r node_fake -i 1 -v true1 -v true2") + test.add_cmd("stonith_admin", "--output-as=xml -B node_fake -t 5") + test.add_log_pattern("Remapping multiple-device reboot targeting node_fake") + # timeout should be sum of off timeouts (1+2=3), not reboot timeouts (10+20=30) + test.add_log_pattern("Total timeout set to 3 for peer's fencing targeting node_fake") + test.add_log_pattern("perform 'off' action targeting node_fake using true1") + test.add_log_pattern("perform 'off' action targeting node_fake using true2") + test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'") + # "on" should be skipped + test.add_log_pattern("Not turning node_fake back on using " + "true1 because the device is configured " + "to stay off") + test.add_log_pattern("Not turning node_fake back on using true2" + " because the agent doesn't support 'on'") + test.add_log_pattern("Undoing remap of reboot targeting node_fake") + + test = self.new_test("cpg_remap_automatic", + "Verify remapped topology reboot skips automatic 'on'", True) + test.add_cmd("stonith_admin", + """--output-as=xml -R true1 -a fence_dummy_auto_unfence """ + """-o "mode=pass" -o "pcmk_host_list=node_fake" """) + test.add_cmd("stonith_admin", + """--output-as=xml -R true2 -a fence_dummy_auto_unfence """ + """-o "mode=pass" -o "pcmk_host_list=node_fake" """) + test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 1 -v true1 -v true2") + test.add_cmd("stonith_admin", "--output-as=xml -B node_fake -t 5") + test.add_log_pattern("Remapping multiple-device reboot targeting node_fake") + test.add_log_pattern("perform 'off' action targeting node_fake using true1") + test.add_log_pattern("perform 'off' action targeting node_fake using true2") + test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'") + test.add_log_pattern("Undoing remap of reboot targeting node_fake") + test.add_log_pattern("perform 'on' action targeting node_fake using", + negative=True) + test.add_log_pattern("'on' failure", + negative=True) + + test = self.new_test("cpg_remap_complex_1", + "Verify remapped topology reboot in second level works if non-remapped first level fails", + True) + test.add_cmd("stonith_admin", """--output-as=xml -R false1 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node_fake" """) + test.add_cmd("stonith_admin", """--output-as=xml -R true1 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """) + test.add_cmd("stonith_admin", """--output-as=xml -R true2 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """) + test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 1 -v false1") + test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 2 -v true1 -v true2") + test.add_cmd("stonith_admin", "--output-as=xml -B node_fake -t 5") + test.add_log_pattern("perform 'reboot' action targeting node_fake using false1") + test.add_log_pattern("Remapping multiple-device reboot targeting node_fake") + test.add_log_pattern("perform 'off' action targeting node_fake using true1") + test.add_log_pattern("perform 'off' action targeting node_fake using true2") + test.add_log_pattern("Remapped 'off' targeting node_fake complete, remapping to 'on'") + test.add_log_pattern("Ignoring true1 'on' failure (no capable peers) targeting node_fake") + test.add_log_pattern("Ignoring true2 'on' failure (no capable peers) targeting node_fake") + test.add_log_pattern("Undoing remap of reboot targeting node_fake") + + test = self.new_test("cpg_remap_complex_2", + "Verify remapped topology reboot failure in second level proceeds to third level", + True) + test.add_cmd("stonith_admin", """--output-as=xml -R false1 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node_fake" """) + test.add_cmd("stonith_admin", """--output-as=xml -R false2 -a fence_dummy -o "mode=fail" -o "pcmk_host_list=node_fake" """) + test.add_cmd("stonith_admin", """--output-as=xml -R true1 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """) + test.add_cmd("stonith_admin", """--output-as=xml -R true2 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """) + test.add_cmd("stonith_admin", """--output-as=xml -R true3 -a fence_dummy -o "mode=pass" -o "pcmk_host_list=node_fake" """) + test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 1 -v false1") + test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 2 -v true1 -v false2 -v true3") + test.add_cmd("stonith_admin", "--output-as=xml -r node_fake -i 3 -v true2") + test.add_cmd("stonith_admin", "--output-as=xml -B node_fake -t 5") + test.add_log_pattern("perform 'reboot' action targeting node_fake using false1") + test.add_log_pattern("Remapping multiple-device reboot targeting node_fake") + test.add_log_pattern("perform 'off' action targeting node_fake using true1") + test.add_log_pattern("perform 'off' action targeting node_fake using false2") + test.add_log_pattern("Attempted to execute agent fence_dummy (off) the maximum number of times") + test.add_log_pattern("Undoing remap of reboot targeting node_fake") + test.add_log_pattern("perform 'reboot' action targeting node_fake using true2") + test.add_log_pattern("node_fake with true3", + negative=True) + + def build_query_tests(self): + """ run stonith_admin --metadata for the fence_dummy agent and check command output """ + + test = self.new_test("get_metadata", + "Run stonith_admin --metadata for the fence_dummy agent", True) + test.add_cmd_check_stdout("stonith_admin", "--output-as=xml -a fence_dummy --metadata", '<shortdesc lang') + + def build_metadata_tests(self): + """ run fence-agents coming with pacemaker with -o metadata and check for valid xml """ + + test = self.new_test("check_metadata_dummy", + "Run fence_dummy -o metadata and check for valid xml", False) + test.add_cmd("fence_dummy", "-o metadata", check_rng=False, check_stderr=False) + # fence_dummy prints on stderr to check that tools just listen on stdout + + test = self.new_test("check_metadata_watchdog", + "Run fence_watchdog -o metadata and check for valid xml", False) + test.add_cmd("fence_watchdog", "-o metadata", check_rng=False) + + def build_validate_tests(self): + """ run stonith_admin --validate for the fence_dummy agent and check command output """ + + test = self.new_test("validate_dummy", + "Run stonith_admin --validate-all and check output", False) + test.add_cmd_expected_fail("stonith_admin", "-a fence_dummy --validate --output-as=xml") + test.add_cmd("stonith_admin", """-a fence_dummy --validate -o "delay=5" --output-as=xml""", check_rng=False) + test.add_cmd_expected_fail("stonith_admin", """-a fence_dummy --validate -o "delay=15" --output-as=xml""") + + def setup_environment(self, use_corosync): + """ Prepare the host before executing any tests """ + + if use_corosync: + self._corosync.start(kill_first=True) + + subprocess.call(["cts-support", "install"]) + + def cleanup_environment(self, use_corosync): + """ Clean up the host after executing desired tests """ + + if use_corosync: + self._corosync.stop() + + subprocess.call(["cts-support", "uninstall"]) + + +def build_options(): + parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, + description="Run pacemaker-fenced regression tests", + epilog="Example: Run only the test 'start_stop'\n" + "\t " + sys.argv[0] + " --run-only start_stop\n\n" + "Example: Run only the tests with the string 'systemd' present in them\n" + "\t " + sys.argv[0] + " --run-only-pattern systemd") + parser.add_argument("-c", "--cpg-only", action="store_true", + help="Only run tests that require corosync") + parser.add_argument("-l", "--list-tests", action="store_true", + help="Print out all registered tests") + parser.add_argument("-n", "--no-cpg", action="store_true", + help="Only run tests that do not require corosync") + parser.add_argument("-p", "--run-only-pattern", metavar='PATTERN', + help="Run only tests matching the given pattern") + parser.add_argument("-r", "--run-only", metavar='TEST', + help="Run a specific test") + parser.add_argument("-t", "--timeout", type=float, default=2, + help="Up to how many seconds each test case waits for the daemon to " + "be initialized. Defaults to 2. The value 0 means no limit.") + parser.add_argument("-w", "--force-wait", action="store_true", + help="Each test case waits the default/specified --timeout for the " + "daemon without tracking the log") + parser.add_argument("-V", "--verbose", action="store_true", + help="Verbose output") + + args = parser.parse_args() + return args + + +def main(): + """ Run fencing regression tests as specified by arguments """ + + update_path() + + # Ensure all command output is in portable locale for comparison + os.environ['LC_ALL'] = "C" + + opts = build_options() + + exit_if_proc_running("pacemaker-fenced") + + use_corosync = not opts.no_cpg + if use_corosync: + exit_if_proc_running("corosync") + + # Create a temporary directory for log files (the directory and its + # contents will automatically be erased when done) + with tempfile.TemporaryDirectory(prefix="cts-fencing-") as logdir: + tests = FenceTests(verbose=opts.verbose, timeout=opts.timeout, + force_wait=opts.force_wait, logdir=logdir) + + tests.build_standalone_tests() + tests.build_custom_timeout_tests() + tests.build_api_sanity_tests() + tests.build_fence_merge_tests() + tests.build_fence_no_merge_tests() + tests.build_unfence_tests() + tests.build_unfence_on_target_tests() + tests.build_nodeid_tests() + tests.build_remap_tests() + tests.build_query_tests() + tests.build_metadata_tests() + tests.build_validate_tests() + + if opts.list_tests: + tests.print_list() + sys.exit(ExitStatus.OK) + + print("Starting ...") + + try: + tests.setup_environment(use_corosync) + except TimeoutError: + print("corosync did not start in time, exiting") + sys.exit(ExitStatus.TIMEOUT) + + if opts.run_only_pattern: + tests.run_tests_matching(opts.run_only_pattern) + tests.print_results() + elif opts.run_only: + tests.run_single(opts.run_only) + tests.print_results() + elif opts.no_cpg: + tests.run_no_cpg() + tests.print_results() + elif opts.cpg_only: + tests.run_cpg_only() + tests.print_results() + else: + tests.run_tests() + tests.print_results() + + tests.cleanup_environment(use_corosync) + + tests.exit() + + +if __name__ == "__main__": + main() |