diff options
Diffstat (limited to 'test/unittests/test_bootstrap.py')
| -rw-r--r-- | test/unittests/test_bootstrap.py | 1905 |
1 files changed, 1905 insertions, 0 deletions
diff --git a/test/unittests/test_bootstrap.py b/test/unittests/test_bootstrap.py new file mode 100644 index 0000000..45bf03d --- /dev/null +++ b/test/unittests/test_bootstrap.py @@ -0,0 +1,1905 @@ +""" +Unitary tests for crmsh/bootstrap.py + +:author: xinliang +:organization: SUSE Linux GmbH +:contact: XLiang@suse.de + +:since: 2019-10-21 +""" + +# pylint:disable=C0103,C0111,W0212,W0611 + +import subprocess +import unittest +import yaml +import socket + +import crmsh.sh +import crmsh.ssh_key +import crmsh.user_of_host +import crmsh.utils +from crmsh.ui_node import NodeMgmt + +try: + from unittest import mock +except ImportError: + import mock + +from crmsh import bootstrap +from crmsh import constants +from crmsh import qdevice + + +class TestContext(unittest.TestCase): + """ + Unitary tests for crmsh.bootstrap.Context + """ + + @classmethod + def setUpClass(cls): + """ + Global setUp. + """ + + def setUp(self): + """ + Test setUp. + """ + self.ctx_inst = bootstrap.Context() + + def tearDown(self): + """ + Test tearDown. + """ + + @classmethod + def tearDownClass(cls): + """ + Global tearDown. + """ + + @mock.patch('crmsh.bootstrap.Context.initialize_user') + def test_set_context(self, mock_initialize_user: mock.MagicMock): + options = mock.Mock(yes_to_all=True, ipv6=False) + ctx = self.ctx_inst.set_context(options) + self.assertEqual(ctx.yes_to_all, True) + self.assertEqual(ctx.ipv6, False) + mock_initialize_user.assert_called_once() + + @mock.patch('crmsh.qdevice.QDevice') + def test_initialize_qdevice_return(self, mock_qdevice): + self.ctx_inst.initialize_qdevice() + mock_qdevice.assert_not_called() + + @mock.patch('crmsh.qdevice.QDevice') + def test_initialize_qdevice(self, mock_qdevice): + ctx = crmsh.bootstrap.Context() + ctx.qnetd_addr = "node3" + ctx.qdevice_port = 123 + ctx.stage = "" + ctx.initialize_qdevice() + mock_qdevice.assert_called_once_with('node3', port=123, ssh_user=None, algo=None, tie_breaker=None, tls=None, cmds=None, mode=None, is_stage=False) + + @mock.patch('crmsh.qdevice.QDevice') + def test_initialize_qdevice_with_user(self, mock_qdevice): + ctx = crmsh.bootstrap.Context() + ctx.qnetd_addr = "alice@node3" + ctx.qdevice_port = 123 + ctx.stage = "" + ctx.initialize_qdevice() + mock_qdevice.assert_called_once_with('node3', port=123, ssh_user='alice', algo=None, tie_breaker=None, tls=None, cmds=None, mode=None, is_stage=False) + + @mock.patch('crmsh.utils.fatal') + def test_validate_sbd_option_error_together(self, mock_error): + mock_error.side_effect = SystemExit + ctx = crmsh.bootstrap.Context() + ctx.sbd_devices = ["/dev/sda1"] + ctx.diskless_sbd = True + with self.assertRaises(SystemExit): + ctx._validate_sbd_option() + mock_error.assert_called_once_with("Can't use -s and -S options together") + + @mock.patch('crmsh.utils.fatal') + def test_validate_sbd_option_error_sbd_stage_no_option(self, mock_error): + mock_error.side_effect = SystemExit + ctx = crmsh.bootstrap.Context() + ctx.stage = "sbd" + ctx.yes_to_all = True + with self.assertRaises(SystemExit): + ctx._validate_sbd_option() + mock_error.assert_called_once_with("Stage sbd should specify sbd device by -s or diskless sbd by -S option") + + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + def test_validate_sbd_option_error_sbd_stage_service(self, mock_active, mock_error): + mock_error.side_effect = SystemExit + ctx = crmsh.bootstrap.Context() + ctx.stage = "sbd" + ctx.diskless_sbd = True + mock_active.return_value = True + with self.assertRaises(SystemExit): + ctx._validate_sbd_option() + mock_error.assert_called_once_with("Can't configure stage sbd: sbd.service already running! Please use crm option '-F' if need to redeploy") + mock_active.assert_called_once_with("sbd.service") + + @mock.patch('crmsh.utils.check_all_nodes_reachable') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + def test_validate_sbd_option_error_sbd_stage(self, mock_active, mock_check_all): + options = mock.Mock(stage="sbd", diskless_sbd=True, cluster_is_running=True) + ctx = crmsh.bootstrap.Context() + ctx.stage = "sbd" + ctx.diskless_sbd = True + ctx.cluster_is_running = True + mock_active.return_value = False + ctx._validate_sbd_option() + mock_active.assert_called_once_with("sbd.service") + mock_check_all.assert_called_once_with() + + @mock.patch('crmsh.utils.fatal') + def test_validate_option_error_nic_number(self, mock_error): + mock_error.side_effect = SystemExit + ctx = crmsh.bootstrap.Context() + ctx.nic_list = ["eth1", "eth2", "eth3"] + with self.assertRaises(SystemExit): + ctx.validate_option() + mock_error.assert_called_once_with("Maximum number of interface is 2") + + @mock.patch('crmsh.utils.fatal') + @mock.patch('socket.gethostbyname') + @mock.patch('crmsh.utils.InterfacesInfo.ip_in_local') + def test_validate_cluster_node_same_name(self, mock_ip_in_local, mock_gethost, mock_fatal): + options = mock.Mock(cluster_node="me", type="join") + ctx = crmsh.bootstrap.Context() + ctx.cluster_node = "me" + ctx.type = "join" + mock_fatal.side_effect = SystemExit + mock_gethost.return_value = ("10.10.10.41", None) + mock_ip_in_local.return_value = True + with self.assertRaises(SystemExit): + ctx._validate_cluster_node() + mock_fatal.assert_called_once_with("Please specify peer node's hostname or IP address") + + @mock.patch('crmsh.utils.fatal') + @mock.patch('socket.gethostbyname') + def test_validate_cluster_node_unknown_name(self, mock_gethost, mock_fatal): + ctx = crmsh.bootstrap.Context() + ctx.cluster_node = "xxxx" + ctx.type = "join" + mock_fatal.side_effect = SystemExit + mock_gethost.side_effect = socket.gaierror("gethostbyname error") + with self.assertRaises(SystemExit): + ctx._validate_cluster_node() + mock_fatal.assert_called_once_with('"xxxx": gethostbyname error') + + @mock.patch('logging.Logger.warning') + @mock.patch('crmsh.bootstrap.Validation.valid_admin_ip') + def test_validate_option(self, mock_admin_ip, mock_warn): + ctx = crmsh.bootstrap.Context() + ctx.admin_ip = "10.10.10.123" + ctx.qdevice_inst = mock.Mock() + ctx._validate_sbd_option = mock.Mock() + ctx._validate_nodes_option = mock.Mock() + ctx.validate_option() + mock_admin_ip.assert_called_once_with("10.10.10.123") + ctx.qdevice_inst.valid_qdevice_options.assert_called_once_with() + ctx._validate_sbd_option.assert_called_once_with() + + @mock.patch('logging.Logger.info') + def test_load_specific_profile_return(self, mock_status): + res = self.ctx_inst.load_specific_profile(None) + assert res == {} + mock_status.assert_not_called() + + @mock.patch('logging.Logger.info') + def test_load_specific_profile_not_exist(self, mock_status): + self.ctx_inst.profiles_data = {"name": "test"} + res = self.ctx_inst.load_specific_profile("newname") + assert res == {} + mock_status.assert_called_once_with("\"newname\" profile does not exist in {}".format(bootstrap.PROFILES_FILE)) + + @mock.patch('logging.Logger.info') + def test_load_specific_profile(self, mock_status): + self.ctx_inst.profiles_data = {"name": "test"} + res = self.ctx_inst.load_specific_profile("name") + assert res == "test" + mock_status.assert_called_once_with("Loading \"name\" profile from {}".format(bootstrap.PROFILES_FILE)) + + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.utils.detect_cloud') + @mock.patch('os.uname') + def test_detect_platform_s390(self, mock_uname, mock_cloud, mock_status): + mock_uname.return_value = mock.Mock(machine="s390") + res = self.ctx_inst.detect_platform() + self.assertEqual(res, bootstrap.Context.S390_PROFILE_NAME) + mock_uname.assert_called_once_with() + mock_cloud.assert_not_called() + mock_status.assert_called_once_with("Detected \"{}\" platform".format(res)) + + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.utils.detect_cloud') + @mock.patch('os.uname') + def test_detect_platform(self, mock_uname, mock_cloud, mock_status): + mock_uname.return_value = mock.Mock(machine="xxx") + mock_cloud.return_value = "azure" + res = self.ctx_inst.detect_platform() + self.assertEqual(res, "azure") + mock_uname.assert_called_once_with() + mock_cloud.assert_called_once_with() + mock_status.assert_called_once_with("Detected \"{}\" platform".format(res)) + + @mock.patch('os.path.exists') + @mock.patch('crmsh.bootstrap.Context.detect_platform') + def test_load_profiles_file_not_exist(self, mock_platform, mock_exists): + mock_platform.return_value = "s390" + mock_exists.return_value = False + self.ctx_inst.load_profiles() + mock_platform.assert_called_once_with() + mock_exists.assert_called_once_with(bootstrap.PROFILES_FILE) + + @mock.patch('yaml.load') + @mock.patch('builtins.open', new_callable=mock.mock_open, read_data="") + @mock.patch('os.path.exists') + @mock.patch('crmsh.bootstrap.Context.detect_platform') + def test_load_profiles_file_empty(self, mock_platform, mock_exists, mock_open_file, mock_load): + mock_platform.return_value = "s390" + mock_exists.return_value = True + mock_load.return_value = "" + self.ctx_inst.load_profiles() + mock_platform.assert_called_once_with() + mock_exists.assert_called_once_with(bootstrap.PROFILES_FILE) + mock_open_file.assert_called_once_with(bootstrap.PROFILES_FILE) + mock_load.assert_called_once_with(mock_open_file.return_value, Loader=yaml.SafeLoader) + + @mock.patch('crmsh.bootstrap.Context.load_specific_profile') + @mock.patch('yaml.load') + @mock.patch('builtins.open', new_callable=mock.mock_open, read_data="") + @mock.patch('os.path.exists') + @mock.patch('crmsh.bootstrap.Context.detect_platform') + def test_load_profiles_file(self, mock_platform, mock_exists, mock_open_file, mock_load, mock_load_specific): + mock_platform.return_value = "s390" + mock_exists.return_value = True + mock_load.return_value = "data" + mock_load_specific.side_effect = [ + {"name": "xin", "age": 18}, + {"name": "wang"} + ] + + self.ctx_inst.load_profiles() + assert self.ctx_inst.profiles_dict == {"name": "wang", "age": 18} + + mock_platform.assert_called_once_with() + mock_exists.assert_called_once_with(bootstrap.PROFILES_FILE) + mock_open_file.assert_called_once_with(bootstrap.PROFILES_FILE) + mock_load.assert_called_once_with(mock_open_file.return_value, Loader=yaml.SafeLoader) + mock_load_specific.assert_has_calls([ + mock.call(bootstrap.Context.DEFAULT_PROFILE_NAME), + mock.call("s390") + ]) + + @mock.patch('crmsh.userdir.get_sudoer') + @mock.patch('crmsh.userdir.getuser') + def test_initialize_user_without_args_without_sudoer(self, mock_getuser: mock.MagicMock, mock_get_sudoer: mock.MagicMock): + mock_getuser.return_value = 'root' + mock_get_sudoer.return_value = None + context = bootstrap.Context() + context.cluster_node = None + context.user_at_node_list = None + context.initialize_user() + self.assertEqual('root', context.current_user) + + @mock.patch('crmsh.userdir.get_sudoer') + @mock.patch('crmsh.userdir.getuser') + def test_initialize_user_without_args_with_sudoer(self, mock_getuser: mock.MagicMock, mock_get_sudoer: mock.MagicMock): + mock_getuser.return_value = 'root' + mock_get_sudoer.return_value = 'alice' + context = bootstrap.Context() + context.cluster_node = None + context.user_at_node_list = None + context.initialize_user() + self.assertEqual('root', context.current_user) + + @mock.patch('crmsh.userdir.get_sudoer') + @mock.patch('crmsh.userdir.getuser') + def test_initialize_user_cluster_node_without_user_without_sudoer(self, mock_getuser: mock.MagicMock, mock_get_sudoer: mock.MagicMock): + mock_getuser.return_value = 'root' + mock_get_sudoer.return_value = None + context = bootstrap.Context() + context.cluster_node = 'node1' + context.user_at_node_list = None + context.initialize_user() + self.assertEqual('root', context.current_user) + + @mock.patch('crmsh.userdir.get_sudoer') + @mock.patch('crmsh.userdir.getuser') + def test_initialize_user_cluster_node_with_user_without_sudoer(self, mock_getuser: mock.MagicMock, mock_get_sudoer: mock.MagicMock): + mock_getuser.return_value = 'root' + mock_get_sudoer.return_value = None + context = bootstrap.Context() + context.cluster_node = 'alice@node1' + context.user_at_node_list = None + with self.assertRaises(ValueError): + context.initialize_user() + + @mock.patch('crmsh.userdir.get_sudoer') + @mock.patch('crmsh.userdir.getuser') + def test_initialize_user_cluster_node_without_user_with_sudoer(self, mock_getuser: mock.MagicMock, mock_get_sudoer: mock.MagicMock): + mock_getuser.return_value = 'root' + mock_get_sudoer.return_value = 'bob' + context = bootstrap.Context() + context.cluster_node = 'node1' + context.user_at_node_list = None + context.initialize_user() + self.assertEqual('root', context.current_user) + + @mock.patch('crmsh.userdir.get_sudoer') + @mock.patch('crmsh.userdir.getuser') + def test_initialize_user_cluster_node_with_user_with_sudoer(self, mock_getuser: mock.MagicMock, mock_get_sudoer: mock.MagicMock): + mock_getuser.return_value = 'root' + mock_get_sudoer.return_value = 'bob' + context = bootstrap.Context() + context.cluster_node = 'alice@node1' + context.user_at_node_list = None + context.initialize_user() + self.assertEqual('bob', context.current_user) + + @mock.patch('crmsh.userdir.get_sudoer') + @mock.patch('crmsh.userdir.getuser') + def test_initialize_user_node_list_without_user_without_sudoer(self, mock_getuser: mock.MagicMock, mock_get_sudoer: mock.MagicMock): + mock_getuser.return_value = 'root' + mock_get_sudoer.return_value = None + context = bootstrap.Context() + context.user_at_node_list = ['node1', 'node2'] + context.cluster_node = None + context.initialize_user() + self.assertEqual('root', context.current_user) + + @mock.patch('crmsh.userdir.get_sudoer') + @mock.patch('crmsh.userdir.getuser') + def test_initialize_user_node_list_with_user_without_sudoer(self, mock_getuser: mock.MagicMock, mock_get_sudoer: mock.MagicMock): + mock_getuser.return_value = 'root' + mock_get_sudoer.return_value = None + context = bootstrap.Context() + context.user_at_node_list = ['alice@node1', 'alice@node2'] + context.cluster_node = None + with self.assertRaises(ValueError): + context.initialize_user() + + @mock.patch('crmsh.userdir.get_sudoer') + @mock.patch('crmsh.userdir.getuser') + def test_initialize_user_node_list_without_user_with_sudoer(self, mock_getuser: mock.MagicMock, mock_get_sudoer: mock.MagicMock): + mock_getuser.return_value = 'root' + mock_get_sudoer.return_value = 'bob' + context = bootstrap.Context() + context.user_at_node_list = ['node1', 'node2'] + context.cluster_node = None + context.initialize_user() + self.assertEqual('root', context.current_user) + + @mock.patch('crmsh.userdir.get_sudoer') + @mock.patch('crmsh.userdir.getuser') + def test_initialize_user_node_list_with_user_with_sudoer(self, mock_getuser: mock.MagicMock, mock_get_sudoer: mock.MagicMock): + mock_getuser.return_value = 'root' + mock_get_sudoer.return_value = 'bob' + context = bootstrap.Context() + context.user_at_node_list = ['alice@node1', 'alice@node2'] + context.cluster_node = None + context.initialize_user() + self.assertEqual('bob', context.current_user) + + +class TestBootstrap(unittest.TestCase): + """ + Unitary tests for crmsh/bootstrap.py + """ + + @classmethod + def setUpClass(cls): + """ + Global setUp. + """ + + def setUp(self): + """ + Test setUp. + """ + self.qdevice_with_ip = qdevice.QDevice("10.10.10.123") + + def tearDown(self): + """ + Test tearDown. + """ + + @classmethod + def tearDownClass(cls): + """ + Global tearDown. + """ + + @mock.patch('crmsh.parallax.parallax_call') + @mock.patch('crmsh.service_manager.ServiceManager.start_service') + @mock.patch('crmsh.sbd.SBDTimeout.is_sbd_delay_start') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_enabled') + @mock.patch('crmsh.utils.package_is_installed') + def test_start_pacemaker(self, mock_installed, mock_enabled, mock_delay_start, mock_start, mock_parallax_call): + bootstrap._context = None + mock_installed.return_value = True + mock_enabled.return_value = True + mock_delay_start.return_value = True + node_list = ["node1", "node2", "node3", "node4", "node5", "node6"] + bootstrap.start_pacemaker(node_list) + mock_start.assert_has_calls([ + mock.call("corosync.service", remote_addr="node1"), + mock.call("corosync.service", remote_addr="node2"), + mock.call("corosync.service", remote_addr="node3"), + mock.call("corosync.service", remote_addr="node4"), + mock.call("corosync.service", remote_addr="node5"), + mock.call("corosync.service", remote_addr="node6"), + mock.call("pacemaker.service", enable=False, node_list=node_list) + ]) + mock_parallax_call.assert_has_calls([ + mock.call(node_list, 'mkdir -p /run/systemd/system/sbd.service.d/'), + mock.call(node_list, "echo -e '[Service]\nUnsetEnvironment=SBD_DELAY_START' > /run/systemd/system/sbd.service.d/sbd_delay_start_disabled.conf"), + mock.call(node_list, "systemctl daemon-reload"), + ]) + + @mock.patch('crmsh.bootstrap.configure_ssh_key') + @mock.patch('crmsh.service_manager.ServiceManager.start_service') + def test_init_ssh(self, mock_start_service, mock_config_ssh): + bootstrap._context = mock.Mock(current_user="alice", user_at_node_list=[], use_ssh_agent=False) + bootstrap.init_ssh() + mock_start_service.assert_called_once_with("sshd.service", enable=True) + mock_config_ssh.assert_has_calls([ + mock.call("alice") + ]) + + @mock.patch('crmsh.userdir.gethomedir') + def test_key_files(self, mock_gethome): + mock_gethome.return_value = "/root" + expected_res = {"private": "/root/.ssh/id_rsa", "public": "/root/.ssh/id_rsa.pub", "authorized": "/root/.ssh/authorized_keys"} + self.assertEqual(bootstrap.key_files("root"), expected_res) + mock_gethome.assert_called_once_with("root") + + @mock.patch('builtins.open') + def test_is_nologin(self, mock_open_file): + data = "hacluster:x:90:90:heartbeat processes:/var/lib/heartbeat/cores/hacluster:/sbin/nologin" + mock_open_file.return_value = mock.mock_open(read_data=data).return_value + assert bootstrap.is_nologin("hacluster") is not None + mock_open_file.assert_called_once_with("/etc/passwd") + + @mock.patch('crmsh.bootstrap.confirm') + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.bootstrap.is_nologin') + def test_change_user_shell_return(self, mock_nologin, mock_status, mock_confirm): + bootstrap._context = mock.Mock(yes_to_all=False) + mock_nologin.return_value = True + mock_confirm.return_value = False + + bootstrap.change_user_shell("hacluster") + + mock_nologin.assert_called_once_with("hacluster", None) + mock_confirm.assert_called_once_with("Continue?") + + @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') + @mock.patch('crmsh.bootstrap.is_nologin') + def test_change_user_shell(self, mock_nologin, mock_invoke): + bootstrap._context = mock.Mock(yes_to_all=True) + mock_nologin.return_value = True + + bootstrap.change_user_shell("hacluster") + + mock_nologin.assert_called_once_with("hacluster", None) + mock_invoke.assert_called_once_with("usermod -s /bin/bash hacluster", None) + + @mock.patch('crmsh.sh.LocalShell.su_subprocess_run') + def test_generate_ssh_key_pair_on_remote(self, mock_su: mock.MagicMock): + mock_su.return_value = mock.Mock(returncode=0, stdout=b'') + bootstrap.generate_ssh_key_pair_on_remote('local_sudoer', 'remote_host', 'remote_sudoer', 'remote_user') + mock_su.assert_has_calls([ + mock.call( + 'local_sudoer', + 'ssh -o StrictHostKeyChecking=no remote_sudoer@remote_host sudo -H -u remote_user /bin/sh', + input=''' +[ -f ~/.ssh/id_rsa ] || ssh-keygen -q -t rsa -f ~/.ssh/id_rsa -C "Cluster internal on $(hostname)" -N '' +[ -f ~/.ssh/id_rsa.pub ] || ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub +'''.encode('utf-8'), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ), + mock.call( + 'local_sudoer', + 'ssh -o StrictHostKeyChecking=no remote_sudoer@remote_host sudo -H -u remote_user /bin/sh', + input='cat ~/.ssh/id_rsa.pub'.encode('utf-8'), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ), + ]) + + @mock.patch('crmsh.bootstrap.append_unique') + @mock.patch('crmsh.sh.LocalShell.get_stdout_or_raise_error') + @mock.patch('crmsh.utils.detect_file') + @mock.patch('crmsh.bootstrap.key_files') + @mock.patch('crmsh.bootstrap.change_user_shell') + def _test_configure_ssh_key(self, mock_change_shell, mock_key_files, mock_detect, mock_su, mock_append_unique): + mock_key_files.return_value = {"private": "/test/.ssh/id_rsa", "public": "/test/.ssh/id_rsa.pub", "authorized": "/test/.ssh/authorized_keys"} + mock_detect.side_effect = [True, True, False] + + bootstrap.configure_ssh_key("test") + + mock_change_shell.assert_called_once_with("test") + mock_key_files.assert_called_once_with("test") + mock_detect.assert_has_calls([ + mock.call("/test/.ssh/id_rsa"), + mock.call("/test/.ssh/id_rsa.pub"), + mock.call("/test/.ssh/authorized_keys") + ]) + mock_append_unique.assert_called_once_with("/test/.ssh/id_rsa.pub", "/test/.ssh/authorized_keys", "test") + mock_su.assert_called_once_with('test', 'touch /test/.ssh/authorized_keys') + + @mock.patch('crmsh.ssh_key.AuthorizedKeyManager.add') + @mock.patch('crmsh.ssh_key.KeyFileManager.ensure_key_pair_exists_for_user') + def test_configure_ssh_key(self, mock_ensure_key_pair, mock_add): + public_key = crmsh.ssh_key.InMemoryPublicKey('foo') + mock_ensure_key_pair.return_value = (True, [public_key]) + bootstrap.configure_ssh_key('alice') + mock_ensure_key_pair.assert_called_once_with(None, 'alice') + mock_add.assert_called_once_with(None, 'alice', public_key) + + @mock.patch('crmsh.bootstrap.append_to_remote_file') + @mock.patch('crmsh.utils.check_file_content_included') + def test_append_unique_remote(self, mock_check, mock_append): + mock_check.return_value = False + bootstrap.append_unique("fromfile", "tofile", user="root", remote="node1", from_local=True) + mock_check.assert_called_once_with("fromfile", "tofile", remote="node1", source_local=True) + mock_append.assert_called_once_with("fromfile", "root", "node1", "tofile") + + @mock.patch('crmsh.bootstrap.append') + @mock.patch('crmsh.utils.check_file_content_included') + def test_append_unique(self, mock_check, mock_append): + mock_check.return_value = False + bootstrap.append_unique("fromfile", "tofile") + mock_check.assert_called_once_with("fromfile", "tofile", remote=None, source_local=False) + mock_append.assert_called_once_with("fromfile", "tofile", remote=None) + + @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') + def test_append_to_remote_file(self, mock_run): + bootstrap.append_to_remote_file("fromfile", "root", "node1", "tofile") + cmd = "cat fromfile | ssh {} root@node1 'cat >> tofile'".format(constants.SSH_OPTION) + mock_run.assert_called_once_with(cmd) + + @mock.patch('crmsh.utils.fatal') + def test_join_ssh_no_seed_host(self, mock_error): + mock_error.side_effect = ValueError + with self.assertRaises(ValueError): + bootstrap.join_ssh(None, None) + mock_error.assert_called_once_with("No existing IP/hostname specified (use -c option)") + + @mock.patch('crmsh.bootstrap.swap_public_ssh_key_for_secondary_user') + @mock.patch('crmsh.bootstrap.change_user_shell') + @mock.patch('crmsh.sh.LocalShell.get_stdout_or_raise_error') + @mock.patch('crmsh.bootstrap.swap_public_ssh_key') + @mock.patch('crmsh.utils.ssh_copy_id_no_raise') + @mock.patch('crmsh.bootstrap.configure_ssh_key') + @mock.patch('crmsh.service_manager.ServiceManager.start_service') + def test_join_ssh( + self, + mock_start_service, mock_config_ssh, mock_ssh_copy_id, mock_swap, mock_invoke, mock_change, mock_swap_2, + ): + bootstrap._context = mock.Mock(current_user="bob", default_nic_list=["eth1"], use_ssh_agent=False) + mock_invoke.return_value = '' + mock_swap.return_value = None + mock_ssh_copy_id.return_value = 0 + + bootstrap.join_ssh("node1", "alice") + + mock_start_service.assert_called_once_with("sshd.service", enable=True) + mock_config_ssh.assert_has_calls([ + mock.call("bob"), + mock.call("hacluster"), + ]) + mock_ssh_copy_id.assert_called_once_with("bob", "alice", "node1") + mock_swap.assert_called_once_with("node1", "bob", "alice", "bob", "alice", add=True) + mock_invoke.assert_called_once_with( + "bob", + "ssh {} alice@node1 sudo crm cluster init -i eth1 ssh_remote".format(constants.SSH_OPTION), + ) + mock_swap_2.assert_called_once() + args, kwargs = mock_swap_2.call_args + self.assertEqual(3, len(args)) + self.assertEqual('node1', args[1]) + self.assertEqual('hacluster', args[2]) + + @mock.patch('crmsh.ssh_key.AuthorizedKeyManager.add') + @mock.patch('crmsh.ssh_key.KeyFile.public_key') + @mock.patch('crmsh.ssh_key.KeyFileManager.ensure_key_pair_exists_for_user') + @mock.patch('crmsh.ssh_key.KeyFileManager.list_public_key_for_user') + @mock.patch('logging.Logger.info') + def test_swap_public_ssh_key_for_secondary_user( + self, + mock_log_info, + mock_list_public_key_for_user, + mock_ensure_key_pair_exists_for_user, + mock_public_key, + mock_authorized_key_manager_add, + ): + mock_shell = mock.Mock( + crmsh.sh.ClusterShell, + local_shell=mock.Mock(crmsh.sh.LocalShell), + user_of_host=mock.Mock(crmsh.user_of_host.UserOfHost), + ) + mock_list_public_key_for_user.return_value = ['~/.ssh/id_rsa', '~/.ssh/id_ed25519'] + mock_ensure_key_pair_exists_for_user.return_value = (True, [ + crmsh.ssh_key.InMemoryPublicKey('foo'), + crmsh.ssh_key.InMemoryPublicKey('bar'), + ]) + mock_public_key.return_value = 'public_key' + crmsh.bootstrap.swap_public_ssh_key_for_secondary_user(mock_shell, 'node1', 'alice') + mock_list_public_key_for_user.assert_called_once_with(None, 'alice') + mock_ensure_key_pair_exists_for_user.assert_called_once_with('node1', 'alice') + mock_authorized_key_manager_add.assert_has_calls([ + mock.call(None, 'alice', crmsh.ssh_key.InMemoryPublicKey('foo')), + mock.call('node1', 'alice', crmsh.ssh_key.KeyFile('~/.ssh/id_rsa')), + ]) + mock_log_info.assert_called_with("A new ssh keypair is generated for user %s@%s.", 'alice', 'node1') + + @mock.patch('crmsh.bootstrap.change_user_shell') + @mock.patch('crmsh.sh.LocalShell.get_stdout_or_raise_error') + @mock.patch('crmsh.bootstrap.swap_public_ssh_key') + @mock.patch('crmsh.utils.ssh_copy_id_no_raise') + @mock.patch('crmsh.bootstrap.configure_ssh_key') + @mock.patch('crmsh.service_manager.ServiceManager.start_service') + def test_join_ssh_bad_credential(self, mock_start_service, mock_config_ssh, mock_ssh_copy_id, mock_swap, mock_invoke, mock_change): + bootstrap._context = mock.Mock(current_user="bob", default_nic_list=["eth1"], use_ssh_agent=False) + mock_invoke.return_value = '' + mock_swap.return_value = None + mock_ssh_copy_id.return_value = 255 + + with self.assertRaises(ValueError): + bootstrap.join_ssh("node1", "alice") + + mock_start_service.assert_called_once_with("sshd.service", enable=True) + mock_config_ssh.assert_has_calls([ + mock.call("bob"), + ]) + mock_ssh_copy_id.assert_called_once_with("bob", "alice", "node1") + mock_swap.assert_not_called() + mock_invoke.assert_not_called() + + + @mock.patch('crmsh.bootstrap.import_ssh_key') + @mock.patch('crmsh.bootstrap.export_ssh_key_non_interactive') + @mock.patch('logging.Logger.warning') + @mock.patch('crmsh.utils.check_ssh_passwd_need') + def test_swap_public_ssh_key_exception(self, mock_check_passwd, mock_warn, mock_export_ssh_key, mock_import_ssh): + mock_check_passwd.return_value = False + mock_import_ssh.side_effect = ValueError("Can't get the remote id_rsa.pub from {}: {}") + + bootstrap.swap_public_ssh_key("node1", "bob", "bob", "alice", "alice") + + mock_check_passwd.assert_called_once_with("bob", "bob", "node1") + mock_import_ssh.assert_called_once_with("bob", "bob", "alice", "node1", "alice") + mock_warn.assert_called_once_with(mock_import_ssh.side_effect) + + @mock.patch('crmsh.bootstrap.import_ssh_key') + @mock.patch('crmsh.bootstrap.export_ssh_key_non_interactive') + @mock.patch('crmsh.utils.check_ssh_passwd_need') + def test_swap_public_ssh_key(self, mock_check_passwd, mock_export_ssh, mock_import_ssh): + mock_check_passwd.return_value = True + + bootstrap.swap_public_ssh_key("node1", "bob", "bob", "alice", "alice") + + mock_check_passwd.assert_called_once_with("bob", "bob", "node1") + mock_export_ssh.assert_called_once_with("bob", "bob", "node1", "alice", "alice") + mock_import_ssh.assert_called_once_with("bob", "bob", "alice", "node1", "alice") + + @mock.patch('crmsh.utils.this_node') + def test_bootstrap_add_return(self, mock_this_node): + ctx = mock.Mock(user_at_node_list=[], use_ssh_agent=False) + bootstrap.bootstrap_add(ctx) + mock_this_node.assert_not_called() + + @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.utils.this_node') + def test_bootstrap_add(self, mock_this_node, mock_info, mock_run): + ctx = mock.Mock(current_user="alice", user_at_node_list=["bob@node2", "carol@node3"], nic_list=["eth1"], use_ssh_agent=False) + mock_this_node.return_value = "node1" + bootstrap.bootstrap_add(ctx) + mock_info.assert_has_calls([ + mock.call("Adding node node2 to cluster"), + mock.call("Running command on node2: crm cluster join -y -i eth1 -c alice@node1"), + mock.call("Adding node node3 to cluster"), + mock.call("Running command on node3: crm cluster join -y -i eth1 -c alice@node1") + ]) + + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.sh.ClusterShell.get_rc_stdout_stderr_without_input') + def test_setup_passwordless_with_other_nodes_failed_fetch_nodelist(self, mock_run, mock_error): + bootstrap._context = mock.Mock(current_user="carol", use_ssh_agent=False) + mock_run.return_value = (1, None, None) + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap.setup_passwordless_with_other_nodes("node1", "alice") + + mock_run.assert_called_once_with('node1', 'crm_node -l') + mock_error.assert_called_once_with("Can't fetch cluster nodes list from node1: None") + + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.utils.HostUserConfig') + @mock.patch('crmsh.bootstrap._fetch_core_hosts') + @mock.patch('crmsh.sh.ClusterShell.get_rc_stdout_stderr_without_input') + def test_setup_passwordless_with_other_nodes_failed_fetch_hostname( + self, + mock_run, + mock_fetch_core_hosts, + mock_host_user_config_class, + mock_error, + ): + bootstrap._context = mock.Mock(current_user="carol", use_ssh_agent=False) + out_node_list = """1 node1 member + 2 node2 member""" + mock_run.side_effect = [ + (0, out_node_list, None), + (1, None, None) + ] + mock_fetch_core_hosts.return_value = (["alice", "bob"], ["node1", "node2"]) + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap.setup_passwordless_with_other_nodes("node1", "alice") + + mock_run.assert_has_calls([ + mock.call('node1', 'crm_node -l'), + mock.call('node1', 'hostname'), + ]) + mock_error.assert_called_once_with("Can't fetch hostname of node1: None") + + @mock.patch('crmsh.bootstrap.swap_key_for_hacluster') + @mock.patch('crmsh.bootstrap.change_user_shell') + @mock.patch('crmsh.utils.HostUserConfig') + @mock.patch('crmsh.bootstrap._fetch_core_hosts') + @mock.patch('crmsh.utils.ssh_copy_id') + @mock.patch('crmsh.utils.user_of') + @mock.patch('crmsh.bootstrap.swap_public_ssh_key') + @mock.patch('crmsh.sh.ClusterShell.get_rc_stdout_stderr_without_input') + def test_setup_passwordless_with_other_nodes( + self, + mock_run, + mock_swap, + mock_userof, + mock_ssh_copy_id: mock.MagicMock, + mock_fetch_core_hosts, + mock_host_user_config_class, + mock_change_shell, + mock_swap_hacluster + ): + bootstrap._context = mock.Mock(current_user="carol", use_ssh_agent=False) + mock_fetch_core_hosts.return_value = (["alice", "bob"], ["node1", "node2"]) + mock_userof.return_value = "bob" + out_node_list = """1 node1 member + 2 node2 member""" + mock_run.side_effect = [ + (0, out_node_list, None), + (0, "node1", None) + ] + + bootstrap.setup_passwordless_with_other_nodes("node1", "alice") + + mock_run.assert_has_calls([ + mock.call('node1', 'crm_node -l'), + mock.call('node1', 'hostname'), + ]) + mock_userof.assert_called_once_with("node2") + mock_ssh_copy_id.assert_has_calls([ + mock.call('carol', 'bob', 'node2') + ]) + mock_swap.assert_has_calls([ + mock.call('node2', "carol", "bob", "carol", "bob"), + mock.call('node2', 'hacluster', 'hacluster', 'carol', 'bob', add=True) + ]) + + @mock.patch('crmsh.userdir.getuser') + @mock.patch('crmsh.bootstrap.key_files') + @mock.patch('builtins.open') + @mock.patch('crmsh.bootstrap.append') + @mock.patch('os.path.join') + @mock.patch('os.path.exists') + def test_init_ssh_remote_no_sshkey(self, mock_exists, mock_join, mock_append, mock_open_file, mock_key_files, mock_getuser): + mock_getuser.return_value = "alice" + mock_key_files.return_value = {"private": "/home/alice/.ssh/id_rsa", "public": "/home/alice/.ssh/id_rsa.pub", "authorized": "/home/alice/.ssh/authorized_keys"} + mock_exists.side_effect = [False, True, False, False, False] + mock_join.side_effect = ["/home/alice/.ssh/id_rsa", + "/home/alice/.ssh/id_dsa", + "/home/alice/.ssh/id_ecdsa", + "/home/alice/.ssh/id_ed25519"] + mock_open_file.side_effect = [ + mock.mock_open().return_value, + mock.mock_open(read_data="data1 data2").return_value, + mock.mock_open(read_data="data1111").return_value + ] + + bootstrap.init_ssh_remote() + + mock_getuser.assert_called_once_with() + mock_key_files.assert_called_once_with("alice") + + mock_open_file.assert_has_calls([ + mock.call("/home/alice/.ssh/authorized_keys", 'w'), + mock.call("/home/alice/.ssh/authorized_keys", "r+"), + mock.call("/home/alice/.ssh/id_rsa.pub") + ]) + mock_exists.assert_has_calls([ + mock.call("/home/alice/.ssh/authorized_keys"), + mock.call("/home/alice/.ssh/id_rsa"), + mock.call("/home/alice/.ssh/id_dsa"), + mock.call("/home/alice/.ssh/id_ecdsa"), + mock.call("/home/alice/.ssh/id_ed25519"), + ]) + mock_append.assert_called_once_with("/home/alice/.ssh/id_rsa.pub", "/home/alice/.ssh/authorized_keys") + + @mock.patch('crmsh.sh.ClusterShell.get_rc_stdout_stderr_without_input') + def test_get_node_canonical_hostname(self, mock_run): + mock_run.return_value = (0, "Node1", None) + + peer_node = bootstrap.get_node_canonical_hostname('node1') + self.assertEqual('Node1', peer_node) + mock_run.assert_called_once_with('node1', 'crm_node --name') + + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.sh.ClusterShell.get_rc_stdout_stderr_without_input') + def test_get_node_canonical_hostname_error(self, mock_run, mock_error): + mock_run.return_value = (1, None, "error") + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap.get_node_canonical_hostname('node1') + + mock_run.assert_called_once_with("node1", "crm_node --name") + mock_error.assert_called_once_with("error") + + @mock.patch('crmsh.utils.this_node') + @mock.patch('crmsh.bootstrap.get_node_canonical_hostname') + @mock.patch('crmsh.xmlutil.CrmMonXmlParser.is_node_online') + def test_is_online_local_offline(self, mock_is_online, mock_get_hostname, mock_this_node): + bootstrap._context = mock.Mock(cluster_node='node2') + mock_this_node.return_value = "node1" + mock_is_online.return_value = False + + assert bootstrap.is_online() is False + + mock_this_node.assert_called_once_with() + mock_get_hostname.assert_not_called() + mock_is_online.assert_called_once_with("node1") + + @mock.patch('crmsh.utils.this_node') + @mock.patch('crmsh.bootstrap.get_node_canonical_hostname') + @mock.patch('crmsh.xmlutil.CrmMonXmlParser.is_node_online') + def test_is_online_on_init_node(self, mock_is_online, mock_get_hostname, mock_this_node): + bootstrap._context = mock.Mock(cluster_node=None) + mock_this_node.return_value = "node1" + mock_is_online.return_value = True + + assert bootstrap.is_online() is True + + mock_this_node.assert_called_once_with() + mock_get_hostname.assert_not_called() + mock_is_online.assert_called_once_with("node1") + + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.service_manager.ServiceManager.stop_service') + @mock.patch('crmsh.bootstrap.sync_file') + @mock.patch('crmsh.corosync.conf') + @mock.patch('shutil.copy') + @mock.patch('crmsh.utils.this_node') + @mock.patch('crmsh.bootstrap.get_node_canonical_hostname') + @mock.patch('crmsh.xmlutil.CrmMonXmlParser.is_node_online') + def test_is_online_peer_offline(self, mock_is_online, mock_get_hostname, mock_this_node, + mock_copy, mock_corosync_conf, mock_csync2, mock_stop_service, mock_error): + bootstrap._context = mock.Mock(cluster_node='node1') + mock_is_online.side_effect = [True, False] + bootstrap.COROSYNC_CONF_ORIG = "/tmp/crmsh_tmpfile" + mock_this_node.return_value = "node2" + mock_get_hostname.return_value = "node1" + mock_corosync_conf.side_effect = [ "/etc/corosync/corosync.conf", + "/etc/corosync/corosync.conf"] + + bootstrap.is_online() + + mock_this_node.assert_called_once_with() + mock_get_hostname.assert_called_once_with('node1') + mock_corosync_conf.assert_has_calls([ + mock.call(), + mock.call() + ]) + mock_copy.assert_called_once_with(bootstrap.COROSYNC_CONF_ORIG, "/etc/corosync/corosync.conf") + mock_csync2.assert_called_once_with("/etc/corosync/corosync.conf") + mock_stop_service.assert_called_once_with("corosync") + mock_error.assert_called_once_with("Cannot see peer node \"node1\", please check the communication IP") + + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.service_manager.ServiceManager.stop_service') + @mock.patch('crmsh.bootstrap.sync_file') + @mock.patch('crmsh.corosync.conf') + @mock.patch('shutil.copy') + @mock.patch('crmsh.utils.this_node') + @mock.patch('crmsh.bootstrap.get_node_canonical_hostname') + @mock.patch('crmsh.xmlutil.CrmMonXmlParser.is_node_online') + def test_is_online_both_online(self, mock_is_online, mock_get_hostname, mock_this_node, + mock_copy, mock_corosync_conf, mock_csync2, mock_stop_service, mock_error): + bootstrap._context = mock.Mock(cluster_node='node2') + mock_is_online.side_effect = [True, True] + mock_this_node.return_value = "node2" + mock_get_hostname.return_value = "node2" + + assert bootstrap.is_online() is True + + mock_this_node.assert_called_once_with() + mock_get_hostname.assert_called_once_with('node2') + mock_corosync_conf.assert_not_called() + mock_copy.assert_not_called() + mock_csync2.assert_not_called() + mock_stop_service.assert_not_called() + mock_error.assert_not_called() + + @mock.patch('crmsh.bootstrap.invokerc') + @mock.patch('crmsh.bootstrap.invoke') + def test_csync2_update_no_conflicts(self, mock_invoke, mock_invokerc): + mock_invokerc.return_value = True + bootstrap.csync2_update("/etc/corosync.conf") + mock_invoke.assert_called_once_with("csync2 -rm /etc/corosync.conf") + mock_invokerc.assert_called_once_with("csync2 -rxv /etc/corosync.conf") + + @mock.patch('logging.Logger.warning') + @mock.patch('crmsh.bootstrap.invokerc') + @mock.patch('crmsh.bootstrap.invoke') + def test_csync2_update(self, mock_invoke, mock_invokerc, mock_warn): + mock_invokerc.side_effect = [False, False] + bootstrap.csync2_update("/etc/corosync.conf") + mock_invoke.assert_has_calls([ + mock.call("csync2 -rm /etc/corosync.conf"), + mock.call("csync2 -rf /etc/corosync.conf") + ]) + mock_invokerc.assert_has_calls([ + mock.call("csync2 -rxv /etc/corosync.conf"), + mock.call("csync2 -rxv /etc/corosync.conf") + ]) + mock_warn.assert_called_once_with("/etc/corosync.conf was not synced") + + @mock.patch('crmsh.utils.InterfacesInfo') + def test_init_network(self, mock_interfaces): + mock_interfaces_inst = mock.Mock() + mock_interfaces.return_value = mock_interfaces_inst + mock_interfaces_inst.get_default_nic_list_from_route.return_value = ["eth0", "eth1"] + bootstrap._context = mock.Mock(ipv6=False, second_heartbeat=False, nic_list=["eth0", "eth1"], default_nic_list=["eth0", "eth1"]) + + bootstrap.init_network() + + mock_interfaces.assert_called_once_with(False, False, bootstrap._context.nic_list) + mock_interfaces_inst.get_interfaces_info.assert_called_once_with() + mock_interfaces_inst.get_default_nic_list_from_route.assert_called_once_with() + mock_interfaces_inst.get_default_ip_list.assert_called_once_with() + + @mock.patch('crmsh.service_manager.ServiceManager.disable_service') + @mock.patch('logging.Logger.info') + def test_init_qdevice_no_config(self, mock_status, mock_disable): + bootstrap._context = mock.Mock(qdevice_inst=None) + bootstrap.init_qdevice() + mock_status.assert_not_called() + mock_disable.assert_called_once_with("corosync-qdevice.service") + + @mock.patch('crmsh.utils.HostUserConfig') + @mock.patch('crmsh.user_of_host.UserOfHost.instance') + @mock.patch('crmsh.utils.list_cluster_nodes') + @mock.patch('crmsh.utils.ssh_copy_id_no_raise') + @mock.patch('crmsh.bootstrap.configure_ssh_key') + @mock.patch('crmsh.utils.check_ssh_passwd_need') + @mock.patch('logging.Logger.info') + def test_init_qdevice_copy_ssh_key_failed( + self, + mock_status, mock_check_ssh_passwd_need, + mock_configure_ssh_key, mock_ssh_copy_id, mock_list_nodes, mock_user_of_host, + mock_host_user_config_class, + ): + mock_list_nodes.return_value = [] + bootstrap._context = mock.Mock(qdevice_inst=self.qdevice_with_ip, current_user="bob") + mock_check_ssh_passwd_need.return_value = True + mock_ssh_copy_id.return_value = 255 + mock_user_of_host.return_value = mock.MagicMock(crmsh.user_of_host.UserOfHost) + mock_user_of_host.return_value.user_pair_for_ssh.return_value = "bob", "bob" + mock_user_of_host.return_value.use_ssh_agent.return_value = False + + with self.assertRaises(ValueError): + bootstrap.init_qdevice() + + mock_status.assert_has_calls([ + mock.call("Configure Qdevice/Qnetd:"), + ]) + mock_check_ssh_passwd_need.assert_called_once_with("bob", "bob", "10.10.10.123") + mock_configure_ssh_key.assert_called_once_with('bob') + mock_ssh_copy_id.assert_called_once_with('bob', 'bob', '10.10.10.123') + + @mock.patch('crmsh.utils.HostUserConfig') + @mock.patch('crmsh.user_of_host.UserOfHost.instance') + @mock.patch('crmsh.utils.list_cluster_nodes') + @mock.patch('crmsh.bootstrap.confirm') + @mock.patch('crmsh.utils.is_qdevice_configured') + @mock.patch('crmsh.bootstrap.configure_ssh_key') + @mock.patch('crmsh.utils.check_ssh_passwd_need') + @mock.patch('logging.Logger.info') + def test_init_qdevice_already_configured( + self, + mock_status, mock_ssh, mock_configure_ssh_key, + mock_qdevice_configured, mock_confirm, mock_list_nodes, mock_user_of_host, + mock_host_user_config_class, + ): + mock_list_nodes.return_value = [] + bootstrap._context = mock.Mock(qdevice_inst=self.qdevice_with_ip, current_user="bob") + mock_ssh.return_value = False + mock_user_of_host.return_value = mock.MagicMock(crmsh.user_of_host.UserOfHost) + mock_user_of_host.return_value.user_pair_for_ssh.return_value = "bob", "bob" + mock_user_of_host.return_value.use_ssh_agent.return_value = False + mock_qdevice_configured.return_value = True + mock_confirm.return_value = False + self.qdevice_with_ip.start_qdevice_service = mock.Mock() + + bootstrap.init_qdevice() + + mock_status.assert_called_once_with("Configure Qdevice/Qnetd:") + mock_ssh.assert_called_once_with("bob", "bob", "10.10.10.123") + mock_configure_ssh_key.assert_not_called() + mock_host_user_config_class.return_value.save_remote.assert_called_once_with(mock_list_nodes.return_value) + mock_qdevice_configured.assert_called_once_with() + mock_confirm.assert_called_once_with("Qdevice is already configured - overwrite?") + self.qdevice_with_ip.start_qdevice_service.assert_called_once_with() + + @mock.patch('crmsh.utils.HostUserConfig') + @mock.patch('crmsh.user_of_host.UserOfHost.instance') + @mock.patch('crmsh.bootstrap.adjust_priority_fencing_delay') + @mock.patch('crmsh.bootstrap.adjust_priority_in_rsc_defaults') + @mock.patch('crmsh.utils.list_cluster_nodes') + @mock.patch('crmsh.utils.this_node') + @mock.patch('crmsh.utils.is_qdevice_configured') + @mock.patch('crmsh.bootstrap.configure_ssh_key') + @mock.patch('crmsh.utils.check_ssh_passwd_need') + @mock.patch('logging.Logger.info') + def test_init_qdevice(self, mock_info, mock_ssh, mock_configure_ssh_key, mock_qdevice_configured, + mock_this_node, mock_list_nodes, mock_adjust_priority, mock_adjust_fence_delay, + mock_user_of_host, mock_host_user_config_class): + bootstrap._context = mock.Mock(qdevice_inst=self.qdevice_with_ip, current_user="bob") + mock_this_node.return_value = "192.0.2.100" + mock_list_nodes.return_value = [] + mock_ssh.return_value = False + mock_user_of_host.return_value = mock.MagicMock(crmsh.user_of_host.UserOfHost) + mock_user_of_host.return_value.user_pair_for_ssh.return_value = "bob", "bob" + mock_user_of_host.return_value.use_ssh_agent.return_value = False + mock_qdevice_configured.return_value = False + self.qdevice_with_ip.set_cluster_name = mock.Mock() + self.qdevice_with_ip.valid_qnetd = mock.Mock() + self.qdevice_with_ip.config_and_start_qdevice = mock.Mock() + + bootstrap.init_qdevice() + + mock_info.assert_called_once_with("Configure Qdevice/Qnetd:") + mock_ssh.assert_called_once_with("bob", "bob", "10.10.10.123") + mock_host_user_config_class.return_value.add.assert_has_calls([ + mock.call('bob', '192.0.2.100'), + mock.call('bob', '10.10.10.123'), + ]) + mock_host_user_config_class.return_value.save_remote.assert_called_once_with(mock_list_nodes.return_value) + mock_qdevice_configured.assert_called_once_with() + self.qdevice_with_ip.set_cluster_name.assert_called_once_with() + self.qdevice_with_ip.valid_qnetd.assert_called_once_with() + self.qdevice_with_ip.config_and_start_qdevice.assert_called_once_with() + + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.utils.HostUserConfig') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_available') + @mock.patch('crmsh.utils.list_cluster_nodes') + @mock.patch('logging.Logger.info') + def test_init_qdevice_service_not_available( + self, + mock_info, mock_list_nodes, mock_available, + mock_host_user_config_class, + mock_fatal, + ): + bootstrap._context = mock.Mock(qdevice_inst=self.qdevice_with_ip) + mock_list_nodes.return_value = ["node1"] + mock_available.return_value = False + mock_fatal.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap.init_qdevice() + + mock_host_user_config_class.return_value.save_local.assert_not_called() + mock_host_user_config_class.return_value.save_remote.assert_not_called() + mock_fatal.assert_called_once_with("corosync-qdevice.service is not available on node1") + mock_available.assert_called_once_with("corosync-qdevice.service", "node1") + mock_info.assert_called_once_with("Configure Qdevice/Qnetd:") + + @mock.patch('crmsh.bootstrap.prompt_for_string') + def test_configure_qdevice_interactive_return(self, mock_prompt): + bootstrap._context = mock.Mock(yes_to_all=True) + bootstrap.configure_qdevice_interactive() + mock_prompt.assert_not_called() + + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.bootstrap.confirm') + def test_configure_qdevice_interactive_not_confirm(self, mock_confirm, mock_info): + bootstrap._context = mock.Mock(yes_to_all=False) + mock_confirm.return_value = False + bootstrap.configure_qdevice_interactive() + mock_confirm.assert_called_once_with("Do you want to configure QDevice?") + + @mock.patch('logging.Logger.error') + @mock.patch('crmsh.qdevice.QDevice.check_package_installed') + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.bootstrap.confirm') + def test_configure_qdevice_interactive_not_installed(self, mock_confirm, mock_info, mock_installed, mock_error): + bootstrap._context = mock.Mock(yes_to_all=False) + mock_confirm.side_effect = [True, False] + mock_installed.side_effect = ValueError("corosync-qdevice not installed") + bootstrap.configure_qdevice_interactive() + mock_confirm.assert_has_calls([ + mock.call("Do you want to configure QDevice?"), + mock.call("Please install the package manually and press 'y' to continue") + ]) + + @mock.patch('crmsh.qdevice.QDevice') + @mock.patch('crmsh.bootstrap.prompt_for_string') + @mock.patch('crmsh.qdevice.QDevice.check_package_installed') + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.bootstrap.confirm') + def test_configure_qdevice_interactive(self, mock_confirm, mock_info, mock_installed, mock_prompt, mock_qdevice): + bootstrap._context = mock.Mock(yes_to_all=False) + mock_confirm.return_value = True + mock_prompt.side_effect = ["alice@qnetd-node", 5403, "ffsplit", "lowest", "on", None] + mock_qdevice_inst = mock.Mock() + mock_qdevice.return_value = mock_qdevice_inst + + bootstrap.configure_qdevice_interactive() + mock_confirm.assert_called_once_with("Do you want to configure QDevice?") + mock_prompt.assert_has_calls([ + mock.call("HOST or IP of the QNetd server to be used", + valid_func=qdevice.QDevice.check_qnetd_addr), + mock.call("TCP PORT of QNetd server", default=5403, + valid_func=qdevice.QDevice.check_qdevice_port), + mock.call("QNetd decision ALGORITHM (ffsplit/lms)", default="ffsplit", + valid_func=qdevice.QDevice.check_qdevice_algo), + mock.call("QNetd TIE_BREAKER (lowest/highest/valid node id)", default="lowest", + valid_func=qdevice.QDevice.check_qdevice_tie_breaker), + mock.call("Whether using TLS on QDevice/QNetd (on/off/required)", default="on", + valid_func=qdevice.QDevice.check_qdevice_tls), + mock.call("Heuristics COMMAND to run with absolute path; For multiple commands, use \";\" to separate", + valid_func=qdevice.QDevice.check_qdevice_heuristics, + allow_empty=True) + ]) + mock_qdevice.assert_called_once_with('qnetd-node', port=5403, ssh_user='alice', algo='ffsplit', tie_breaker='lowest', tls='on', cmds=None, mode=None, is_stage=False) + + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.utils.is_qdevice_configured') + def test_remove_qdevice_no_configured(self, mock_qdevice_configured, mock_error): + mock_qdevice_configured.return_value = False + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap.remove_qdevice() + + mock_qdevice_configured.assert_called_once_with() + mock_error.assert_called_once_with("No QDevice configuration in this cluster") + + @mock.patch('crmsh.bootstrap.confirm') + @mock.patch('crmsh.utils.is_qdevice_configured') + def test_remove_qdevice_not_confirmed(self, mock_qdevice_configured, mock_confirm): + mock_qdevice_configured.return_value = True + mock_confirm.return_value = False + + bootstrap.remove_qdevice() + + mock_qdevice_configured.assert_called_once_with() + mock_confirm.assert_called_once_with("Removing QDevice service and configuration from cluster: Are you sure?") + + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + @mock.patch('crmsh.bootstrap.adjust_priority_fencing_delay') + @mock.patch('crmsh.bootstrap.adjust_priority_in_rsc_defaults') + @mock.patch('crmsh.qdevice.QDevice.remove_certification_files_on_qnetd') + @mock.patch('crmsh.qdevice.QDevice.remove_qdevice_db') + @mock.patch('crmsh.qdevice.QDevice.remove_qdevice_config') + @mock.patch('crmsh.bootstrap.update_expected_votes') + @mock.patch('crmsh.log.LoggerUtils.status_long') + @mock.patch('crmsh.bootstrap.invoke') + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.qdevice.evaluate_qdevice_quorum_effect') + @mock.patch('crmsh.utils.check_all_nodes_reachable') + @mock.patch('crmsh.bootstrap.confirm') + @mock.patch('crmsh.utils.is_qdevice_configured') + def test_remove_qdevice_reload(self, mock_qdevice_configured, mock_confirm, mock_reachable, mock_evaluate, + mock_status, mock_invoke, mock_status_long, mock_update_votes, mock_remove_config, mock_remove_db, + mock_remove_files, mock_adjust_priority, mock_adjust_fence_delay, mock_service_is_active): + mock_qdevice_configured.return_value = True + mock_confirm.return_value = True + mock_evaluate.return_value = qdevice.QdevicePolicy.QDEVICE_RELOAD + mock_service_is_active.return_value = False + + bootstrap.remove_qdevice() + + mock_qdevice_configured.assert_called_once_with() + mock_confirm.assert_called_once_with("Removing QDevice service and configuration from cluster: Are you sure?") + mock_reachable.assert_called_once_with() + mock_evaluate.assert_called_once_with(qdevice.QDEVICE_REMOVE) + mock_status.assert_has_calls([ + mock.call("Disable corosync-qdevice.service"), + mock.call("Stopping corosync-qdevice.service") + ]) + mock_invoke.assert_has_calls([ + mock.call("crm cluster run 'systemctl disable corosync-qdevice'"), + mock.call("crm cluster run 'systemctl stop corosync-qdevice'"), + mock.call("crm cluster run 'crm corosync reload'") + ] ) + mock_status_long.assert_called_once_with("Removing QDevice configuration from cluster") + mock_update_votes.assert_called_once_with() + mock_remove_config.assert_called_once_with() + mock_remove_db.assert_called_once_with() + + @mock.patch('crmsh.service_manager.ServiceManager.start_service') + @mock.patch('crmsh.qdevice.QDevice') + @mock.patch('crmsh.corosync.get_value') + @mock.patch('crmsh.utils.is_qdevice_tls_on') + @mock.patch('crmsh.bootstrap.invoke') + @mock.patch('crmsh.bootstrap.sync_file') + @mock.patch('crmsh.corosync.conf') + @mock.patch('crmsh.corosync.add_nodelist_from_cmaptool') + @mock.patch('crmsh.corosync.is_unicast') + @mock.patch('crmsh.log.LoggerUtils.status_long') + def test_start_qdevice_on_join_node(self, mock_status_long, mock_is_unicast, mock_add_nodelist, + mock_conf, mock_csync2_update, mock_invoke, mock_qdevice_tls, + mock_get_value, mock_qdevice, mock_start_service): + mock_is_unicast.return_value = False + mock_qdevice_tls.return_value = True + mock_conf.return_value = "corosync.conf" + mock_get_value.return_value = "10.10.10.123" + mock_qdevice_inst = mock.Mock() + mock_qdevice.return_value = mock_qdevice_inst + mock_qdevice_inst.certificate_process_on_join = mock.Mock() + + bootstrap.start_qdevice_on_join_node("node2") + + mock_status_long.assert_called_once_with("Starting corosync-qdevice.service") + mock_is_unicast.assert_called_once_with() + mock_add_nodelist.assert_called_once_with() + mock_conf.assert_called_once_with() + mock_csync2_update.assert_called_once_with("corosync.conf") + mock_invoke.assert_called_once_with("crm corosync reload") + mock_qdevice_tls.assert_called_once_with() + mock_get_value.assert_called_once_with("quorum.device.net.host") + mock_qdevice.assert_called_once_with("10.10.10.123", cluster_node="node2") + mock_qdevice_inst.certificate_process_on_join.assert_called_once_with() + mock_start_service.assert_called_once_with("corosync-qdevice.service", enable=True) + + @mock.patch('crmsh.sh.ShellUtils.get_stdout_stderr') + @mock.patch('crmsh.log.LoggerUtils.log_only_to_file') + def test_invoke(self, mock_log, mock_run): + mock_run.return_value = (0, "output", "error") + res = bootstrap.invoke("cmd --option") + self.assertEqual(res, (True, "output", "error")) + mock_log.assert_has_calls([ + mock.call('invoke: cmd --option'), + mock.call('stdout: output'), + mock.call('stderr: error') + ]) + + @mock.patch('crmsh.bootstrap.invoke') + def test_invokerc(self, mock_invoke): + mock_invoke.return_value = (True, None, None) + res = bootstrap.invokerc("cmd") + self.assertEqual(res, True) + mock_invoke.assert_called_once_with("cmd") + + @mock.patch('crmsh.utils.cluster_run_cmd') + @mock.patch('os.path.isfile') + def test_sync_files_to_disk(self, mock_isfile, mock_cluster_cmd): + bootstrap.FILES_TO_SYNC = ("file1", "file2") + mock_isfile.side_effect = [True, True] + bootstrap.sync_files_to_disk() + mock_isfile.assert_has_calls([mock.call("file1"), mock.call("file2")]) + mock_cluster_cmd.assert_called_once_with("sync file1 file2") + + @mock.patch('logging.Logger.debug') + @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') + @mock.patch('crmsh.bootstrap.cib_factory') + def test_adjust_pcmk_delay_2node(self, mock_cib_factory, mock_run, mock_debug): + mock_cib_factory.refresh = mock.Mock() + mock_cib_factory.fence_id_list_without_pcmk_delay = mock.Mock() + mock_cib_factory.fence_id_list_without_pcmk_delay.return_value = ["res_1"] + bootstrap.adjust_pcmk_delay_max(True) + mock_run.assert_called_once_with("crm resource param res_1 set pcmk_delay_max {}s".format(constants.PCMK_DELAY_MAX)) + + @mock.patch('logging.Logger.debug') + @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') + @mock.patch('crmsh.bootstrap.cib_factory') + def test_adjust_pcmk_delay(self, mock_cib_factory, mock_run, mock_debug): + mock_cib_factory.refresh = mock.Mock() + mock_cib_factory.fence_id_list_with_pcmk_delay = mock.Mock() + mock_cib_factory.fence_id_list_with_pcmk_delay.return_value = ["res_1"] + bootstrap.adjust_pcmk_delay_max(False) + mock_run.assert_called_once_with("crm resource param res_1 delete pcmk_delay_max") + + @mock.patch('crmsh.sbd.SBDTimeout') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + def test_adjust_stonith_timeout_sbd(self, mock_is_active, mock_sbd_timeout): + mock_is_active.return_value = True + mock_sbd_timeout.adjust_sbd_timeout_related_cluster_configuration = mock.Mock() + bootstrap.adjust_stonith_timeout() + mock_sbd_timeout.adjust_sbd_timeout_related_cluster_configuration.assert_called_once_with() + + @mock.patch('crmsh.utils.set_property') + @mock.patch('crmsh.bootstrap.get_stonith_timeout_generally_expected') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + def test_adjust_stonith_timeout(self, mock_is_active, mock_get_timeout, mock_set): + mock_is_active.return_value = False + mock_get_timeout.return_value = 30 + bootstrap.adjust_stonith_timeout() + mock_set.assert_called_once_with("stonith-timeout", 30, conditional=True) + + @mock.patch('crmsh.utils.set_property') + def test_adjust_priority_in_rsc_defaults_2node(self, mock_set): + bootstrap.adjust_priority_in_rsc_defaults(True) + mock_set.assert_called_once_with('priority', 1, property_type='rsc_defaults', conditional=True) + + @mock.patch('crmsh.utils.set_property') + def test_adjust_priority_in_rsc_defaults(self, mock_set): + bootstrap.adjust_priority_in_rsc_defaults(False) + mock_set.assert_called_once_with('priority', 0, property_type='rsc_defaults') + + @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') + def test_adjust_priority_fencing_delay_no_fence_agent(self, mock_run): + mock_run.return_value = None + bootstrap.adjust_priority_fencing_delay(False) + mock_run.assert_called_once_with("crm configure show related:stonith") + + @mock.patch('crmsh.utils.set_property') + @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') + def test_adjust_priority_fencing_delay_no_pcmk_delay(self, mock_run, mock_set): + mock_run.return_value = "data" + bootstrap.adjust_priority_fencing_delay(False) + mock_run.assert_called_once_with("crm configure show related:stonith") + mock_set.assert_called_once_with("priority-fencing-delay", 0) + + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + def test_adjust_properties_no_service(self, mock_is_active): + mock_is_active.return_value = False + bootstrap.adjust_properties() + mock_is_active.assert_called_once_with("pacemaker.service") + + @mock.patch('crmsh.bootstrap.adjust_priority_fencing_delay') + @mock.patch('crmsh.bootstrap.adjust_priority_in_rsc_defaults') + @mock.patch('crmsh.bootstrap.adjust_stonith_timeout') + @mock.patch('crmsh.bootstrap.adjust_pcmk_delay_max') + @mock.patch('crmsh.utils.is_2node_cluster_without_qdevice') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + def test_adjust_properties(self, mock_is_active, mock_2node_qdevice, mock_adj_pcmk, mock_adj_stonith, mock_adj_priority, mock_adj_fence): + mock_is_active.return_value = True + mock_2node_qdevice.return_value = True + bootstrap.adjust_properties() + mock_is_active.assert_called_once_with("pacemaker.service") + mock_adj_pcmk.assert_called_once_with(True) + mock_adj_stonith.assert_called_once_with() + mock_adj_priority.assert_called_once_with(True) + mock_adj_fence.assert_called_once_with(True) + + @mock.patch('crmsh.utils.cluster_copy_file') + def test_sync_file_skip_csync2(self, mock_copy): + bootstrap._context = mock.Mock(skip_csync2=True, node_list_in_cluster=["node1", "node2"]) + bootstrap.sync_file("/file1") + mock_copy.assert_called_once_with("/file1", nodes=["node1", "node2"], output=False) + + @mock.patch('crmsh.bootstrap.csync2_update') + def test_sync_file(self, mock_csync2_update): + bootstrap._context = mock.Mock(skip_csync2=False) + bootstrap.sync_file("/file1") + mock_csync2_update.assert_called_once_with("/file1") + + +class TestValidation(unittest.TestCase): + """ + Unitary tests for class bootstrap.Validation + """ + @classmethod + def setUpClass(cls): + """ + Global setUp. + """ + + def setUp(self): + """ + Test setUp. + """ + self.validate_inst = bootstrap.Validation("10.10.10.1") + self.validate_port_inst_in_use = bootstrap.Validation("4567", ["4568"]) + self.validate_port_inst_out_of_range = bootstrap.Validation("456766") + + def tearDown(self): + """ + Test tearDown. + """ + + @classmethod + def tearDownClass(cls): + """ + Global tearDown. + """ + + @mock.patch('crmsh.utils.IP.is_mcast') + def test_is_mcast_addr(self, mock_mcast): + mock_mcast.return_value = False + with self.assertRaises(ValueError) as err: + self.validate_inst._is_mcast_addr() + self.assertEqual("10.10.10.1 is not multicast address", str(err.exception)) + mock_mcast.assert_called_once_with("10.10.10.1") + + def test_is_local_addr(self): + with self.assertRaises(ValueError) as err: + self.validate_inst._is_local_addr(["20.20.20.1", "20.20.20.2"]) + self.assertEqual("Address must be a local address (one of ['20.20.20.1', '20.20.20.2'])", str(err.exception)) + + def test_is_valid_port_in_use(self): + with self.assertRaises(ValueError) as err: + self.validate_port_inst_in_use._is_valid_port() + self.assertEqual("Port 4567 is already in use by corosync. Leave a gap between multiple rings.", str(err.exception)) + + def test_is_valid_port_out_of_range(self): + with self.assertRaises(ValueError) as err: + self.validate_port_inst_out_of_range._is_valid_port() + self.assertEqual("Valid port range should be 1025-65535", str(err.exception)) + + @mock.patch('crmsh.bootstrap.Validation._is_mcast_addr') + def test_valid_mcast_address(self, mock_mcast): + bootstrap.Validation.valid_mcast_address("10.10.10.1") + mock_mcast.assert_called_once_with() + + @mock.patch('crmsh.bootstrap.Validation._is_local_addr') + def test_valid_ucast_ip(self, mock_local_addr): + bootstrap._context = mock.Mock(local_ip_list=["10.10.10.2", "10.10.10.3"]) + bootstrap.Validation.valid_ucast_ip("10.10.10.1") + mock_local_addr.assert_called_once_with(["10.10.10.2", "10.10.10.3"]) + + @mock.patch('crmsh.bootstrap.Validation._is_local_addr') + def test_valid_mcast_ip(self, mock_local_addr): + bootstrap._context = mock.Mock(local_ip_list=["10.10.10.2", "10.10.10.3"], + local_network_list=["10.10.10.0"]) + bootstrap.Validation.valid_mcast_ip("10.10.10.1") + mock_local_addr.assert_called_once_with(["10.10.10.2", "10.10.10.3", "10.10.10.0"]) + + @mock.patch('crmsh.bootstrap.Validation._is_valid_port') + def test_valid_port(self, mock_port): + bootstrap.Validation.valid_port("10.10.10.1") + mock_port.assert_called_once_with() + + @mock.patch('crmsh.bootstrap.invokerc') + @mock.patch('crmsh.utils.IP.is_ipv6') + def test_valid_admin_ip_in_use(self, mock_ipv6, mock_invoke): + mock_ipv6.return_value = False + mock_invoke.return_value = True + + with self.assertRaises(ValueError) as err: + self.validate_inst.valid_admin_ip("10.10.10.1") + self.assertEqual("Address already in use: 10.10.10.1", str(err.exception)) + + mock_ipv6.assert_called_once_with("10.10.10.1") + mock_invoke.assert_called_once_with("ping -c 1 10.10.10.1") + + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + @mock.patch('crmsh.bootstrap.init') + @mock.patch('crmsh.bootstrap.Context') + def test_bootstrap_remove_cluster_is_active(self, mock_context, mock_init, mock_active, + mock_error): + mock_context_inst = mock.Mock() + mock_context.return_value = mock_context_inst + mock_active.return_value = False + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap.bootstrap_remove(mock_context_inst) + + mock_init.assert_called_once_with() + mock_active.assert_called_once_with("corosync.service") + mock_error.assert_called_once_with("Cluster is not active - can't execute removing action") + + @mock.patch('crmsh.bootstrap.remove_qdevice') + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + @mock.patch('crmsh.bootstrap.init') + @mock.patch('crmsh.bootstrap.Context') + def test_bootstrap_remove_qdevice(self, mock_context, mock_init, mock_active, + mock_error, mock_qdevice): + mock_context_inst = mock.Mock(qdevice=True, cluster_node=None) + mock_context.return_value = mock_context_inst + mock_active.return_value = [True, True] + + bootstrap.bootstrap_remove(mock_context_inst) + + mock_init.assert_called_once_with() + mock_active.assert_has_calls([ + mock.call("corosync.service"), + mock.call("csync2.socket") + ]) + mock_error.assert_not_called() + mock_qdevice.assert_called_once_with() + + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + @mock.patch('crmsh.bootstrap.init') + @mock.patch('crmsh.bootstrap.Context') + def test_bootstrap_remove_qdevice_cluster_node(self, mock_context, mock_init, mock_active, mock_error): + mock_context_inst = mock.Mock(qdevice=True, cluster_node="node1") + mock_context.return_value = mock_context_inst + mock_active.return_value = True + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap.bootstrap_remove(mock_context_inst) + + mock_init.assert_called_once_with() + mock_active.assert_called_once_with("corosync.service") + mock_error.assert_called_once_with("Either remove node or qdevice") + + @mock.patch('crmsh.bootstrap.prompt_for_string') + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.bootstrap.remove_qdevice') + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + @mock.patch('crmsh.bootstrap.init') + @mock.patch('crmsh.bootstrap.Context') + def test_bootstrap_remove_no_cluster_node(self, mock_context, mock_init, mock_active, + mock_error, mock_qdevice, mock_status, mock_prompt): + mock_context_inst = mock.Mock(yes_to_all=False, cluster_node=None, qdevice_rm_flag=None) + mock_context.return_value = mock_context_inst + mock_active.return_value = [True, True] + mock_prompt.return_value = None + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap.bootstrap_remove(mock_context_inst) + + mock_init.assert_called_once_with() + mock_active.assert_has_calls([ + mock.call("corosync.service"), + mock.call("csync2.socket") + ]) + mock_qdevice.assert_not_called() + mock_status.assert_called_once_with('Remove This Node from Cluster:\n You will be asked for the IP address or name of an existing node,\n which will be removed from the cluster. This command must be\n executed from a different node in the cluster.\n') + mock_prompt.assert_called_once_with("IP address or hostname of cluster node (e.g.: 192.168.1.1)", ".+") + mock_error.assert_called_once_with("No existing IP/hostname specified (use -c option)") + + @mock.patch('crmsh.bootstrap.confirm') + @mock.patch('crmsh.bootstrap.get_node_canonical_hostname') + @mock.patch('crmsh.bootstrap.remove_qdevice') + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + @mock.patch('crmsh.bootstrap.init') + @mock.patch('crmsh.bootstrap.Context') + def test_bootstrap_remove_no_confirm(self, mock_context, mock_init, mock_active, + mock_error, mock_qdevice, mock_hostname, mock_confirm): + mock_context_inst = mock.Mock(cluster_node="node1", force=False, qdevice_rm_flag=None) + mock_context.return_value = mock_context_inst + mock_active.return_value = [True, True] + mock_hostname.return_value = "node1" + mock_confirm.return_value = False + + bootstrap.bootstrap_remove(mock_context_inst) + + mock_init.assert_called_once_with() + mock_active.assert_has_calls([ + mock.call("corosync.service"), + mock.call("csync2.socket") + ]) + mock_qdevice.assert_not_called() + mock_error.assert_not_called() + mock_hostname.assert_called_once_with('node1') + mock_confirm.assert_called_once_with('Removing node "node1" from the cluster: Are you sure?') + + @mock.patch('crmsh.utils.this_node') + @mock.patch('crmsh.bootstrap.confirm') + @mock.patch('crmsh.bootstrap.get_node_canonical_hostname') + @mock.patch('crmsh.bootstrap.remove_qdevice') + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + @mock.patch('crmsh.bootstrap.init') + @mock.patch('crmsh.bootstrap.Context') + def test_bootstrap_remove_self_need_force(self, mock_context, mock_init, mock_active, + mock_error, mock_qdevice, mock_hostname, mock_confirm, mock_this_node): + mock_context_inst = mock.Mock(cluster_node="node1", force=False, qdevice_rm_flag=None) + mock_context.return_value = mock_context_inst + mock_active.return_value = [True, True] + mock_hostname.return_value = "node1" + mock_confirm.return_value = True + mock_this_node.return_value = "node1" + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap.bootstrap_remove(mock_context_inst) + + mock_init.assert_called_once_with() + mock_active.assert_has_calls([ + mock.call("corosync.service"), + mock.call("csync2.socket") + ]) + mock_qdevice.assert_not_called() + mock_hostname.assert_called_once_with('node1') + mock_confirm.assert_called_once_with('Removing node "node1" from the cluster: Are you sure?') + mock_this_node.assert_called_once_with() + mock_error.assert_called_once_with("Removing self requires --force") + + @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') + @mock.patch('crmsh.bootstrap.remove_self') + @mock.patch('crmsh.utils.this_node') + @mock.patch('crmsh.bootstrap.confirm') + @mock.patch('crmsh.bootstrap.get_node_canonical_hostname') + @mock.patch('crmsh.bootstrap.remove_qdevice') + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + @mock.patch('crmsh.bootstrap.init') + @mock.patch('crmsh.bootstrap.Context') + def test_bootstrap_remove_self(self, mock_context, mock_init, mock_active, + mock_error, mock_qdevice, mock_hostname, mock_confirm, mock_this_node, mock_self, mock_run): + mock_context_inst = mock.Mock(cluster_node="node1", force=True, qdevice_rm_flag=None) + mock_context.return_value = mock_context_inst + mock_active.return_value = [True, True] + mock_hostname.return_value = "node1" + mock_this_node.return_value = "node1" + + bootstrap.bootstrap_remove(mock_context_inst) + + mock_init.assert_called_once_with() + mock_active.assert_has_calls([ + mock.call("corosync.service"), + mock.call("csync2.socket") + ]) + mock_qdevice.assert_not_called() + mock_hostname.assert_called_once_with('node1') + mock_confirm.assert_not_called() + mock_this_node.assert_called_once_with() + mock_error.assert_not_called() + mock_self.assert_called_once_with(True) + mock_run.assert_called_once_with('rm -rf /var/lib/crmsh', 'node1') + + @mock.patch('crmsh.xmlutil.listnodes') + @mock.patch('crmsh.utils.this_node') + @mock.patch('crmsh.bootstrap.confirm') + @mock.patch('crmsh.bootstrap.get_node_canonical_hostname') + @mock.patch('crmsh.bootstrap.remove_qdevice') + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + @mock.patch('crmsh.bootstrap.init') + @mock.patch('crmsh.bootstrap.Context') + def test_bootstrap_remove_not_in_cluster(self, mock_context, mock_init, mock_active, + mock_error, mock_qdevice, mock_hostname, mock_confirm, mock_this_node, mock_list): + mock_context_inst = mock.Mock(cluster_node="node2", force=True, qdevice_rm_flag=None) + mock_context.return_value = mock_context_inst + mock_active.return_value = [True, True] + mock_hostname.return_value = "node2" + mock_this_node.return_value = "node1" + mock_list.return_value = ["node1", "node3"] + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap.bootstrap_remove(mock_context_inst) + + mock_init.assert_called_once_with() + mock_active.assert_has_calls([ + mock.call("corosync.service"), + mock.call("csync2.socket") + ]) + mock_qdevice.assert_not_called() + mock_hostname.assert_called_once_with('node2') + mock_confirm.assert_not_called() + mock_this_node.assert_called_once_with() + mock_error.assert_called_once_with("Specified node node2 is not configured in cluster! Unable to remove.") + + @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') + @mock.patch('crmsh.utils.fetch_cluster_node_list_from_node') + @mock.patch('crmsh.bootstrap.remove_node_from_cluster') + @mock.patch('crmsh.xmlutil.listnodes') + @mock.patch('crmsh.utils.this_node') + @mock.patch('crmsh.bootstrap.confirm') + @mock.patch('crmsh.bootstrap.get_node_canonical_hostname') + @mock.patch('crmsh.bootstrap.remove_qdevice') + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + @mock.patch('crmsh.bootstrap.init') + @mock.patch('crmsh.bootstrap.Context') + def test_bootstrap_remove(self, mock_context, mock_init, mock_active, + mock_error, mock_qdevice, mock_hostname, mock_confirm, mock_this_node, + mock_list, mock_remove, mock_fetch, mock_run): + mock_context_inst = mock.Mock(cluster_node="node2", qdevice_rm_flag=None, force=True) + mock_context.return_value = mock_context_inst + mock_active.side_effect = [True, False] + mock_hostname.return_value = "node2" + mock_this_node.return_value = "node1" + mock_list.return_value = ["node1", "node2"] + mock_fetch.return_value = ["node1", "node2"] + + bootstrap.bootstrap_remove(mock_context_inst) + + mock_init.assert_called_once_with() + mock_active.assert_has_calls([ + mock.call("corosync.service"), + mock.call("csync2.socket") + ]) + mock_qdevice.assert_not_called() + mock_hostname.assert_called_once_with('node2') + mock_confirm.assert_not_called() + mock_error.assert_not_called() + mock_remove.assert_called_once_with('node2') + mock_run.assert_called_once_with('rm -rf /var/lib/crmsh', 'node2') + + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.sh.ClusterShell.get_rc_stdout_stderr_without_input') + @mock.patch('crmsh.xmlutil.listnodes') + @mock.patch('crmsh.utils.this_node') + def test_remove_self_other_nodes(self, mock_this_node, mock_list, mock_run, mock_error): + mock_this_node.return_value = 'node1' + mock_list.return_value = ["node1", "node2"] + mock_run.return_value = (1, '', 'err') + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap._context = mock.Mock(cluster_node="node1", yes_to_all=True) + bootstrap.remove_self() + + mock_list.assert_called_once_with(include_remote_nodes=False) + mock_run.assert_called_once_with("node2", "crm cluster remove -y -c node1") + mock_error.assert_called_once_with("Failed to remove this node from node2") + + @mock.patch('crmsh.utils.package_is_installed') + @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error') + def test_rm_configuration_files(self, mock_run, mock_installed): + bootstrap._context = mock.Mock(rm_list=["file1", "file2"]) + mock_installed.return_value = True + bootstrap.rm_configuration_files() + mock_run.assert_has_calls([ + mock.call('rm -f file1 file2', None), + mock.call('cp /usr/share/fillup-templates/sysconfig.sbd /etc/sysconfig/sbd', None) + ]) + + @mock.patch('crmsh.utils.get_iplist_from_name') + @mock.patch('crmsh.corosync.get_values') + def test_get_cluster_node_ip_host(self, mock_get_values, mock_get_iplist): + mock_get_values.return_value = ["node1", "node2"] + self.assertIsNone(bootstrap.get_cluster_node_ip('node1')) + mock_get_values.assert_called_once_with("nodelist.node.ring0_addr") + mock_get_iplist.assert_not_called() + + @mock.patch('crmsh.utils.get_iplist_from_name') + @mock.patch('crmsh.corosync.get_values') + def test_get_cluster_node_ip(self, mock_get_values, mock_get_iplist): + mock_get_values.return_value = ["10.10.10.1", "10.10.10.2"] + mock_get_iplist.return_value = ["10.10.10.1"] + self.assertEqual("10.10.10.1", bootstrap.get_cluster_node_ip('node1')) + mock_get_values.assert_called_once_with("nodelist.node.ring0_addr") + mock_get_iplist.assert_called_once_with('node1') + + @mock.patch('crmsh.service_manager.ServiceManager.stop_service') + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + def test_stop_services(self, mock_active, mock_status, mock_stop): + mock_active.side_effect = [True, True, True, True] + bootstrap.stop_services(bootstrap.SERVICES_STOP_LIST) + mock_active.assert_has_calls([ + mock.call("corosync-qdevice.service", remote_addr=None), + mock.call("corosync.service", remote_addr=None), + mock.call("hawk.service", remote_addr=None), + mock.call("csync2.socket", remote_addr=None) + ]) + mock_status.assert_has_calls([ + mock.call('Stopping the %s%s', 'corosync-qdevice.service', ''), + mock.call('Stopping the %s%s', 'corosync.service', ''), + mock.call('Stopping the %s%s', 'hawk.service', ''), + mock.call('Stopping the %s%s', 'csync2.socket', '') + ]) + mock_stop.assert_has_calls([ + mock.call("corosync-qdevice.service", disable=True, remote_addr=None), + mock.call("corosync.service", disable=True, remote_addr=None), + mock.call("hawk.service", disable=True, remote_addr=None), + mock.call("csync2.socket", disable=True, remote_addr=None) + ]) + + @mock.patch.object(NodeMgmt, 'call_delnode') + @mock.patch('crmsh.bootstrap.rm_configuration_files') + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.bootstrap.invoke') + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.bootstrap.stop_services') + @mock.patch('crmsh.bootstrap.get_cluster_node_ip') + def test_remove_node_from_cluster_rm_node_failed(self, mock_get_ip, mock_stop, mock_status, mock_invoke, mock_error, mock_rm_conf_files, mock_call_delnode): + mock_get_ip.return_value = '192.0.2.100' + mock_call_delnode.return_value = False + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap._context = mock.Mock(rm_list=["file1", "file2"]) + bootstrap.remove_node_from_cluster('node1') + + mock_get_ip.assert_called_once_with('node1') + mock_status.assert_called_once_with("Removing the node node1") + mock_stop.assert_called_once_with(bootstrap.SERVICES_STOP_LIST, remote_addr="node1") + mock_invoke.assert_not_called() + mock_call_delnode.assert_called_once_with("node1") + mock_error.assert_called_once_with("Failed to remove node1.") + + @mock.patch.object(NodeMgmt, 'call_delnode') + @mock.patch('crmsh.bootstrap.rm_configuration_files') + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.bootstrap.invokerc') + @mock.patch('crmsh.bootstrap.invoke') + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.bootstrap.stop_services') + @mock.patch('crmsh.bootstrap.get_cluster_node_ip') + def test_remove_node_from_cluster_rm_csync_failed(self, mock_get_ip, mock_stop, mock_status, mock_invoke, mock_invokerc, mock_error, mock_rm_conf_files, mock_call_delnode): + mock_get_ip.return_value = '192.0.2.100' + mock_call_delnode.return_value = True + mock_invokerc.return_value = False + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + bootstrap._context = mock.Mock(rm_list=["file1", "file2"]) + bootstrap.remove_node_from_cluster('node1') + + mock_get_ip.assert_called_once_with('node1') + mock_status.assert_called_once_with("Removing the node node1") + mock_stop.assert_called_once_with(bootstrap.SERVICES_STOP_LIST, remote_addr="node1") + mock_invoke.assert_not_called() + mock_call_delnode.assert_called_once_with("node1") + mock_invokerc.assert_has_calls([ + mock.call("sed -i /node1/d {}".format(bootstrap.CSYNC2_CFG)) + ]) + mock_error.assert_called_once_with("Removing the node node1 from {} failed".format(bootstrap.CSYNC2_CFG)) + + @mock.patch.object(NodeMgmt, 'call_delnode') + @mock.patch('crmsh.service_manager.ServiceManager.service_is_active') + @mock.patch('crmsh.bootstrap.rm_configuration_files') + @mock.patch('crmsh.bootstrap.adjust_priority_fencing_delay') + @mock.patch('crmsh.bootstrap.adjust_priority_in_rsc_defaults') + @mock.patch('crmsh.bootstrap.sync_file') + @mock.patch('crmsh.bootstrap.decrease_expected_votes') + @mock.patch('crmsh.corosync.del_node') + @mock.patch('crmsh.corosync.get_values') + @mock.patch('crmsh.utils.fatal') + @mock.patch('crmsh.bootstrap.invokerc') + @mock.patch('crmsh.bootstrap.invoke') + @mock.patch('logging.Logger.info') + @mock.patch('crmsh.bootstrap.stop_services') + @mock.patch('crmsh.bootstrap.get_cluster_node_ip') + def test_remove_node_from_cluster_hostname(self, mock_get_ip, mock_stop, mock_status, + mock_invoke, mock_invokerc, mock_error, mock_get_values, mock_del, mock_decrease, mock_csync2, + mock_adjust_priority, mock_adjust_fence_delay, mock_rm_conf_files, mock_is_active, mock_cal_delnode): + mock_get_ip.return_value = "10.10.10.1" + mock_cal_delnode.return_value = True + mock_invoke.side_effect = [(True, None, None)] + mock_invokerc.return_value = True + mock_get_values.return_value = ["10.10.10.1"] + mock_is_active.return_value = False + + bootstrap._context = mock.Mock(cluster_node="node1", rm_list=["file1", "file2"]) + bootstrap.remove_node_from_cluster('node1') + + mock_get_ip.assert_called_once_with('node1') + mock_status.assert_has_calls([ + mock.call("Removing the node node1"), + mock.call("Propagating configuration changes across the remaining nodes") + ]) + mock_stop.assert_called_once_with(bootstrap.SERVICES_STOP_LIST, remote_addr="node1") + mock_cal_delnode.assert_called_once_with("node1") + mock_invoke.assert_has_calls([ + mock.call("corosync-cfgtool -R") + ]) + mock_invokerc.assert_called_once_with("sed -i /node1/d {}".format(bootstrap.CSYNC2_CFG)) + mock_error.assert_not_called() + mock_get_values.assert_called_once_with("nodelist.node.ring0_addr") + mock_del.assert_called_once_with("10.10.10.1") + mock_decrease.assert_called_once_with() + mock_csync2.assert_has_calls([ + mock.call(bootstrap.CSYNC2_CFG), + mock.call("/etc/corosync/corosync.conf") + ]) |