Adding upstream version 4.6.0.upstream/4.6.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1031 insertions, 0 deletions

diff --git a/test/unittests/test_qdevice.py b/test/unittests/test_qdevice.py
new file mode 100644
index 0000000..f6b2f13
--- /dev/null
+++ b/test/unittests/test_qdevice.py
@@ -0,0 +1,1031 @@
+import os
+import unittest
+import socket
+
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+from crmsh import bootstrap
+from crmsh import sbd
+from crmsh import qdevice, lock
+
+
+F2 = open(os.path.join(os.path.dirname(__file__), 'corosync.conf.2')).read()
+F4 = open(os.path.join(os.path.dirname(__file__), 'corosync.conf.3')).read()
+
+
+@mock.patch('crmsh.utils.calculate_quorate_status')
+@mock.patch('crmsh.utils.get_quorum_votes_dict')
+def test_evaluate_qdevice_quorum_effect_restart(mock_get_dict, mock_quorate):
+    mock_get_dict.return_value = {'Expected': '1', 'Total': '1'}
+    mock_quorate.return_value = False
+    res = qdevice.evaluate_qdevice_quorum_effect(qdevice.QDEVICE_ADD, False, False)
+    assert res == qdevice.QdevicePolicy.QDEVICE_RESTART
+    mock_get_dict.assert_called_once_with()
+    mock_quorate.assert_called_once_with(2, 1)
+
+
+@mock.patch('crmsh.utils.calculate_quorate_status')
+@mock.patch('crmsh.utils.get_quorum_votes_dict')
+def test_evaluate_qdevice_quorum_effect_reload(mock_get_dict, mock_quorate):
+    mock_get_dict.return_value = {'Expected': '2', 'Total': '2'}
+    mock_quorate.return_value = True
+    res = qdevice.evaluate_qdevice_quorum_effect(qdevice.QDEVICE_ADD)
+    assert res == qdevice.QdevicePolicy.QDEVICE_RELOAD
+    mock_get_dict.assert_called_once_with()
+    mock_quorate.assert_called_once_with(3, 2)
+
+
+@mock.patch('crmsh.xmlutil.CrmMonXmlParser')
+@mock.patch('crmsh.utils.calculate_quorate_status')
+@mock.patch('crmsh.utils.get_quorum_votes_dict')
+def test_evaluate_qdevice_quorum_effect_later(mock_get_dict, mock_quorate, mock_parser):
+    mock_get_dict.return_value = {'Expected': '2', 'Total': '2'}
+    mock_quorate.return_value = False
+    mock_parser().is_any_resource_running.return_value = True
+    res = qdevice.evaluate_qdevice_quorum_effect(qdevice.QDEVICE_REMOVE)
+    assert res == qdevice.QdevicePolicy.QDEVICE_RESTART_LATER
+    mock_get_dict.assert_called_once_with()
+    mock_quorate.assert_called_once_with(2, 1)
+
+
+@mock.patch('crmsh.xmlutil.CrmMonXmlParser')
+@mock.patch('crmsh.utils.calculate_quorate_status')
+@mock.patch('crmsh.utils.get_quorum_votes_dict')
+def test_evaluate_qdevice_quorum_effect(mock_get_dict, mock_quorate, mock_parser):
+    mock_get_dict.return_value = {'Expected': '2', 'Total': '2'}
+    mock_quorate.return_value = False
+    mock_parser().is_any_resource_running.return_value = False
+    res = qdevice.evaluate_qdevice_quorum_effect(qdevice.QDEVICE_REMOVE)
+    assert res == qdevice.QdevicePolicy.QDEVICE_RESTART
+    mock_get_dict.assert_called_once_with()
+    mock_quorate.assert_called_once_with(2, 1)
+
+
+@mock.patch('crmsh.lock.RemoteLock')
+def test_qnetd_lock_for_same_cluster_name(mock_remote_lock):
+    _context = mock.Mock(qnetd_addr="qnetd-node", cluster_name="cluster1")
+    remote_lock_inst = mock.Mock()
+    mock_remote_lock.return_value = remote_lock_inst
+    remote_lock_inst.lock.return_value.__enter__ = mock.Mock()
+    remote_lock_inst.lock.return_value.__exit__ = mock.Mock()
+    @qdevice.qnetd_lock_for_same_cluster_name
+    def decorated(ctx):
+        return
+    decorated(_context)
+    mock_remote_lock.assert_called_once_with("qnetd-node", for_join=False,
+            lock_dir="/run/.crmsh_qdevice_lock_for_cluster1", wait=False)
+
+
+@mock.patch('crmsh.utils.fatal')
+@mock.patch('crmsh.lock.RemoteLock')
+def test_qnetd_lock_for_same_cluster_name_claim_error(mock_remote_lock, mock_fatal):
+    _context = mock.Mock(qnetd_addr="qnetd-node", cluster_name="cluster1")
+    remote_lock_inst = mock.Mock()
+    mock_remote_lock.return_value = remote_lock_inst
+    remote_lock_inst.lock.side_effect = lock.ClaimLockError
+    @qdevice.qnetd_lock_for_same_cluster_name
+    def decorated(ctx):
+        return
+    decorated(_context)
+    mock_fatal.assert_called_once_with("Duplicated cluster name \"cluster1\"!")
+    mock_remote_lock.assert_called_once_with("qnetd-node", for_join=False,
+            lock_dir="/run/.crmsh_qdevice_lock_for_cluster1", wait=False)
+
+
+@mock.patch('crmsh.utils.fatal')
+@mock.patch('crmsh.lock.RemoteLock')
+def test_qnetd_lock_for_same_cluster_name_ssh_error(mock_remote_lock, mock_fatal):
+    _context = mock.Mock(qnetd_addr="qnetd-node", cluster_name="cluster1")
+    remote_lock_inst = mock.Mock()
+    mock_remote_lock.return_value = remote_lock_inst
+    remote_lock_inst.lock.side_effect = lock.SSHError("ssh error!")
+    @qdevice.qnetd_lock_for_same_cluster_name
+    def decorated(ctx):
+        return
+    decorated(_context)
+    mock_remote_lock.assert_called_once_with("qnetd-node", for_join=False,
+            lock_dir="/run/.crmsh_qdevice_lock_for_cluster1", wait=False) 
+
+
+@mock.patch('crmsh.lock.RemoteLock')
+def test_qnetd_lock_for_multi_cluster(mock_remote_lock):
+    _context = mock.Mock(qnetd_addr="qnetd-node")
+    remote_lock_inst = mock.Mock()
+    mock_remote_lock.return_value = remote_lock_inst
+    remote_lock_inst.lock.return_value.__enter__ = mock.Mock()
+    remote_lock_inst.lock.return_value.__exit__ = mock.Mock()
+    @qdevice.qnetd_lock_for_multi_cluster
+    def decorated(ctx):
+        return
+    decorated(_context)
+    mock_remote_lock.assert_called_once_with("qnetd-node", for_join=False, no_warn=True)
+
+
+@mock.patch('crmsh.utils.fatal')
+@mock.patch('crmsh.lock.RemoteLock')
+def test_qnetd_lock_for_multi_cluster_error(mock_remote_lock, mock_fatal):
+    _context = mock.Mock(qnetd_addr="qnetd-node")
+    remote_lock_inst = mock.Mock()
+    mock_remote_lock.return_value = remote_lock_inst
+    remote_lock_inst.lock.side_effect = lock.SSHError("ssh error!")
+    @qdevice.qnetd_lock_for_multi_cluster
+    def decorated(ctx):
+        return
+    decorated(_context)
+    mock_remote_lock.assert_called_once_with("qnetd-node", for_join=False, no_warn=True)
+
+
+class TestQDevice(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        """
+        Global setUp.
+        """
+
+    def setUp(self):
+        """
+        Test setUp.
+        """
+        # Use the setup to create a fresh instance for each test
+        self.qdevice_with_ip = qdevice.QDevice("10.10.10.123")
+        self.qdevice_with_hostname = qdevice.QDevice("node.qnetd")
+        self.qdevice_with_invalid_port = qdevice.QDevice("10.10.10.123", port=100)
+        self.qdevice_with_invalid_tie_breaker = qdevice.QDevice("10.10.10.123", tie_breaker="wrong")
+        self.qdevice_with_ip_cluster_node = qdevice.QDevice("10.10.10.123", cluster_node="node1.com")
+        self.qdevice_with_invalid_cmds_relative_path = qdevice.QDevice("10.10.10.123", cmds="ls")
+        self.qdevice_with_invalid_cmds_not_exist = qdevice.QDevice("10.10.10.123", cmds="/not_exist")
+        self.qdevice_with_cluster_name = qdevice.QDevice("10.10.10.123", cluster_name="hacluster1")
+        self.qdevice_with_stage_cluster_name = qdevice.QDevice("10.10.10.123", is_stage=True, cluster_name="cluster1")
+
+    def tearDown(self):
+        """
+        Test tearDown.
+        """
+
+    @classmethod
+    def tearDownClass(cls):
+        """
+        Global tearDown.
+        """
+
+    def test_qnetd_cacert_on_local(self):
+        res = self.qdevice_with_ip.qnetd_cacert_on_local
+        self.assertEqual(res, "/etc/corosync/qdevice/net/10.10.10.123/qnetd-cacert.crt")
+
+    def test_qnetd_cacert_on_cluster(self):
+        res = self.qdevice_with_ip_cluster_node.qnetd_cacert_on_cluster
+        self.assertEqual(res, "/etc/corosync/qdevice/net/node1.com/qnetd-cacert.crt")
+
+    def test_qdevice_crq_on_qnetd(self):
+        res = self.qdevice_with_cluster_name.qdevice_crq_on_qnetd
+        self.assertEqual(res, "/etc/corosync/qnetd/nssdb/qdevice-net-node.crq.hacluster1")
+
+    def test_qdevice_crq_on_local(self):
+        res = self.qdevice_with_ip.qdevice_crq_on_local
+        self.assertEqual(res, "/etc/corosync/qdevice/net/nssdb/qdevice-net-node.crq")
+
+    def test_qnetd_cluster_crt_on_qnetd(self):
+        res = self.qdevice_with_ip.qnetd_cluster_crt_on_qnetd
+        self.assertEqual(res, "/etc/corosync/qnetd/nssdb/cluster-None.crt")
+
+    @mock.patch('os.path.basename')
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cluster_crt_on_qnetd", new_callable=mock.PropertyMock)
+    def test_qnetd_cluster_crt_on_local(self, mock_qnetd_crt, mock_basename):
+        mock_qnetd_crt.return_value = "crt_file"
+        mock_basename.return_value = "crt_file"
+        res = self.qdevice_with_ip.qnetd_cluster_crt_on_local
+        self.assertEqual(res, "/etc/corosync/qdevice/net/10.10.10.123/crt_file")
+
+    def test_qdevice_p12_on_local(self):
+        res = self.qdevice_with_ip.qdevice_p12_on_local
+        self.assertEqual(res, "/etc/corosync/qdevice/net/nssdb/qdevice-net-node.p12")
+
+    def test_qdevice_p12_on_cluster(self):
+        res = self.qdevice_with_ip_cluster_node.qdevice_p12_on_cluster
+        self.assertEqual(res, "/etc/corosync/qdevice/net/node1.com/qdevice-net-node.p12")
+
+    @mock.patch('crmsh.utils.check_port_open')
+    @mock.patch('crmsh.utils.InterfacesInfo.ip_in_local')
+    @mock.patch('crmsh.utils.ping_node')
+    @mock.patch('socket.getaddrinfo')
+    def test_check_qnetd_addr_port_error(self, mock_getaddrinfo, mock_ping, mock_in_local, mock_check):
+        mock_getaddrinfo.return_value = [(None, ("10.10.10.123",)),]
+        mock_in_local.return_value = False
+        mock_check.return_value = False
+        with self.assertRaises(ValueError) as err:
+            qdevice.QDevice.check_qnetd_addr("qnetd-node")
+        excepted_err_string = "ssh service on \"qnetd-node\" not available"
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+    @mock.patch('crmsh.utils.InterfacesInfo.ip_in_local')
+    @mock.patch('crmsh.utils.ping_node')
+    @mock.patch('socket.getaddrinfo')
+    def test_check_qnetd_addr_local(self, mock_getaddrinfo, mock_ping, mock_in_local):
+        mock_getaddrinfo.return_value = [(None, ("10.10.10.123",)),]
+        mock_in_local.return_value = True
+        with self.assertRaises(ValueError) as err:
+            qdevice.QDevice.check_qnetd_addr("qnetd-node")
+        excepted_err_string = "host for qnetd must be a remote one"
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+    @mock.patch('socket.getaddrinfo')
+    def test_check_qnetd_addr(self, mock_getaddrinfo):
+        mock_getaddrinfo.side_effect = socket.error
+        with self.assertRaises(ValueError) as err:
+            qdevice.QDevice.check_qnetd_addr("qnetd-node")
+        excepted_err_string = "host \"qnetd-node\" is unreachable"
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+    @mock.patch('crmsh.utils.valid_port')
+    def test_check_qdevice_port(self, mock_port):
+        mock_port.return_value = False
+        with self.assertRaises(ValueError) as err:
+            qdevice.QDevice.check_qdevice_port("1")
+        excepted_err_string = "invalid qdevice port range(1024 - 65535)"
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+    def test_check_qdevice_algo(self):
+        with self.assertRaises(ValueError) as err:
+            qdevice.QDevice.check_qdevice_algo("1")
+        excepted_err_string = "invalid ALGORITHM choice: '1' (choose from 'ffsplit', 'lms')"
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+    @mock.patch('crmsh.service_manager.ServiceManager.service_is_active')
+    def test_check_qdevice_tie_breaker(self, mock_is_active):
+        mock_is_active.return_value = False
+        with self.assertRaises(ValueError) as err:
+            qdevice.QDevice.check_qdevice_tie_breaker("1")
+        excepted_err_string = "invalid qdevice tie_breaker(lowest/highest/valid_node_id)"
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+    def test_check_qdevice_tls(self):
+        with self.assertRaises(ValueError) as err:
+            qdevice.QDevice.check_qdevice_tls("1")
+        excepted_err_string = "invalid TLS choice: '1' (choose from 'on', 'off', 'required')"
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+    def test_check_qdevice_hm(self):
+        with self.assertRaises(ValueError) as err:
+            qdevice.QDevice.check_qdevice_heuristics_mode("1")
+        excepted_err_string = "invalid MODE choice: '1' (choose from 'on', 'sync', 'off')"
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+    def test_check_qdevice_he_path_error(self):
+        with self.assertRaises(ValueError) as err:
+            qdevice.QDevice.check_qdevice_heuristics("command1")
+        excepted_err_string = "commands for heuristics should be absolute path"
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+    @mock.patch('os.path.exists')
+    def test_check_qdevice_he_not_exist_erro(self, mock_exists):
+        mock_exists.return_value = False
+        with self.assertRaises(ValueError) as err:
+            qdevice.QDevice.check_qdevice_heuristics("/usr/bin/testst")
+        excepted_err_string = "command /usr/bin/testst not exist"
+        self.assertEqual(excepted_err_string, str(err.exception))
+    
+    @mock.patch('crmsh.utils.package_is_installed')
+    def test_check_package_installed(self, mock_installed):
+        mock_installed.return_value = False
+        with self.assertRaises(ValueError) as err:
+            qdevice.QDevice.check_package_installed("corosync-qdevice")
+        excepted_err_string = "Package \"corosync-qdevice\" not installed on this node"
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+    @mock.patch('crmsh.qdevice.QDevice.check_qdevice_heuristics_mode')
+    @mock.patch('crmsh.qdevice.QDevice.check_qdevice_heuristics')
+    @mock.patch('crmsh.qdevice.QDevice.check_qdevice_tls')
+    @mock.patch('crmsh.qdevice.QDevice.check_qdevice_tie_breaker')
+    @mock.patch('crmsh.qdevice.QDevice.check_qdevice_algo')
+    @mock.patch('crmsh.qdevice.QDevice.check_qdevice_port')
+    @mock.patch('crmsh.qdevice.QDevice.check_qnetd_addr')
+    @mock.patch('crmsh.qdevice.QDevice.check_package_installed')
+    def test_valid_qdevice_options(self, mock_installed, mock_check_qnetd, mock_check_port,
+            mock_check_algo, mock_check_tie, mock_check_tls, mock_check_h, mock_check_hm):
+        self.qdevice_with_ip.valid_qdevice_options()
+        mock_installed.assert_called_once_with("corosync-qdevice")
+        mock_check_qnetd.assert_called_once_with("10.10.10.123")
+
+    @mock.patch("crmsh.utils.package_is_installed")
+    def test_valid_qnetd_not_installed(self, mock_installed):
+        self.qdevice_with_ip.qnetd_ip = "10.10.10.123"
+        mock_installed.return_value = False
+        excepted_err_string = 'Package "corosync-qnetd" not installed on 10.10.10.123!\nCluster service already successfully started on this node except qdevice service.\nIf you still want to use qdevice, install "corosync-qnetd" on 10.10.10.123.\nThen run command "crm cluster init" with "qdevice" stage, like:\n  crm cluster init qdevice qdevice_related_options\nThat command will setup qdevice separately.'
+        self.maxDiff = None
+
+        with self.assertRaises(ValueError) as err:
+            self.qdevice_with_ip.valid_qnetd()
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+        mock_installed.assert_called_once_with("corosync-qnetd", remote_addr="10.10.10.123")
+
+    @mock.patch("crmsh.sh.ClusterShell.get_stdout_or_raise_error")
+    @mock.patch("crmsh.service_manager.ServiceManager.service_is_active")
+    @mock.patch("crmsh.utils.package_is_installed")
+    def test_valid_qnetd_duplicated_with_qnetd_running(self, mock_installed, mock_is_active, mock_run):
+        mock_installed.return_value = True
+        mock_is_active.return_value = True
+        mock_run.return_value = "data"
+        excepted_err_string = "This cluster's name \"cluster1\" already exists on qnetd server!\nPlease consider to use the different cluster-name property."
+        self.maxDiff = None
+
+        with self.assertRaises(ValueError) as err:
+            self.qdevice_with_stage_cluster_name.valid_qnetd()
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+        mock_installed.assert_called_once_with("corosync-qnetd", remote_addr="10.10.10.123")
+        mock_is_active.assert_called_once_with("corosync-qnetd", remote_addr="10.10.10.123")
+        mock_run.assert_called_once_with("corosync-qnetd-tool -l -c cluster1", "10.10.10.123")
+
+    @mock.patch("crmsh.sh.ClusterShell.get_stdout_or_raise_error")
+    @mock.patch("crmsh.service_manager.ServiceManager.service_is_active")
+    @mock.patch("crmsh.utils.package_is_installed")
+    def test_valid_qnetd_duplicated_without_qnetd_running(self, mock_installed, mock_is_active, mock_run):
+        mock_installed.return_value = True
+        mock_is_active.return_value = False
+        excepted_err_string = "This cluster's name \"hacluster1\" already exists on qnetd server!\nCluster service already successfully started on this node except qdevice service.\nIf you still want to use qdevice, consider to use the different cluster-name property.\nThen run command \"crm cluster init\" with \"qdevice\" stage, like:\n  crm cluster init qdevice qdevice_related_options\nThat command will setup qdevice separately."
+        self.maxDiff = None
+
+        with self.assertRaises(ValueError) as err:
+            self.qdevice_with_cluster_name.valid_qnetd()
+        self.assertEqual(excepted_err_string, str(err.exception))
+
+        mock_installed.assert_called_once_with("corosync-qnetd", remote_addr="10.10.10.123")
+        mock_is_active.assert_called_once_with("corosync-qnetd", remote_addr="10.10.10.123")
+        mock_run.assert_called_once_with("test -f /etc/corosync/qnetd/nssdb/cluster-hacluster1.crt", "10.10.10.123")
+
+    @mock.patch("crmsh.service_manager.ServiceManager.enable_service")
+    def test_enable_qnetd(self, mock_enable):
+        self.qdevice_with_ip.enable_qnetd()
+        mock_enable.assert_called_once_with("corosync-qnetd.service", remote_addr="10.10.10.123")
+
+    @mock.patch("crmsh.service_manager.ServiceManager.disable_service")
+    def test_disable_qnetd(self, mock_disable):
+        self.qdevice_with_ip.disable_qnetd()
+        mock_disable.assert_called_once_with("corosync-qnetd.service", remote_addr="10.10.10.123")
+
+    @mock.patch("crmsh.service_manager.ServiceManager.start_service")
+    def test_start_qnetd(self, mock_start):
+        self.qdevice_with_ip.start_qnetd()
+        mock_start.assert_called_once_with("corosync-qnetd.service", remote_addr="10.10.10.123")
+
+    @mock.patch("crmsh.service_manager.ServiceManager.stop_service")
+    def test_stop_qnetd(self, mock_stop):
+        self.qdevice_with_ip.stop_qnetd()
+        mock_stop.assert_called_once_with("corosync-qnetd.service", remote_addr="10.10.10.123")
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.parallax.parallax_call")
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_qnetd", new_callable=mock.PropertyMock)
+    def test_init_db_on_qnetd_already_exists(self, mock_qnetd_cacert, mock_call, mock_log):
+        mock_call.return_value = [("10.10.10.123", (0, None, None))]
+        mock_qnetd_cacert.return_value = "/etc/corosync/qnetd/nssdb/qnetd-cacert.crt"
+        self.qdevice_with_ip.init_db_on_qnetd.__wrapped__(self.qdevice_with_ip)
+        mock_call.assert_called_once_with(["10.10.10.123"],
+                                          "test -f {}".format(mock_qnetd_cacert.return_value))
+        mock_qnetd_cacert.assert_called_once_with()
+        mock_log.assert_not_called()
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.parallax.parallax_call")
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_qnetd", new_callable=mock.PropertyMock)
+    def test_init_db_on_qnetd(self, mock_qnetd_cacert, mock_call, mock_log):
+        mock_call.side_effect = [ValueError(mock.Mock(), "Failed on 10.10.10.123: error happen"),
+                                 [("10.10.10.123", (0, None, None))]]
+        mock_qnetd_cacert.return_value = "/etc/corosync/qnetd/nssdb/qnetd-cacert.crt"
+
+        self.qdevice_with_ip.init_db_on_qnetd.__wrapped__(self.qdevice_with_ip)
+
+        mock_call.assert_has_calls([
+            mock.call(["10.10.10.123"], "test -f {}".format(mock_qnetd_cacert.return_value)),
+            mock.call(["10.10.10.123"], "corosync-qnetd-certutil -i")
+        ])
+        mock_qnetd_cacert.assert_called_once_with()
+        mock_log.assert_called_once_with("Step 1: Initialize database on 10.10.10.123",
+                'corosync-qnetd-certutil -i')
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("os.path.exists")
+    @mock.patch("crmsh.parallax.parallax_slurp")
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_local", new_callable=mock.PropertyMock)
+    def test_fetch_qnetd_crt_from_qnetd_exist(self, mock_qnetd_cacert_local,
+                                              mock_parallax_slurp, mock_exists, mock_log):
+        mock_qnetd_cacert_local.return_value = "/etc/corosync/qdevice/net/10.10.10.123/qnetd-cacert.crt"
+        mock_exists.return_value = True
+
+        self.qdevice_with_ip.fetch_qnetd_crt_from_qnetd()
+
+        mock_exists.assert_called_once_with(mock_qnetd_cacert_local.return_value)
+        mock_qnetd_cacert_local.assert_called_once_with()
+        mock_parallax_slurp.assert_not_called()
+        mock_log.assert_not_called()
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("os.path.exists")
+    @mock.patch("crmsh.parallax.parallax_slurp")
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_local", new_callable=mock.PropertyMock)
+    def test_fetch_qnetd_crt_from_qnetd(self, mock_qnetd_cacert_local,
+                                        mock_parallax_slurp, mock_exists, mock_log):
+        mock_qnetd_cacert_local.return_value = "/etc/corosync/qdevice/net/10.10.10.123/qnetd-cacert.crt"
+        mock_exists.return_value = False
+
+        self.qdevice_with_ip.fetch_qnetd_crt_from_qnetd()
+
+        mock_exists.assert_called_once_with(mock_qnetd_cacert_local.return_value)
+        mock_qnetd_cacert_local.assert_called_once_with()
+        mock_log.assert_called_once_with("Step 2: Fetch qnetd-cacert.crt from 10.10.10.123")
+        mock_parallax_slurp.assert_called_once_with(["10.10.10.123"], "/etc/corosync/qdevice/net", "/etc/corosync/qnetd/nssdb/qnetd-cacert.crt")
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.utils.list_cluster_nodes")
+    @mock.patch("crmsh.utils.this_node")
+    @mock.patch("crmsh.parallax.parallax_copy")
+    def test_copy_qnetd_crt_to_cluster_one_node(self, mock_copy, mock_this_node, mock_list_nodes, mock_log):
+        mock_this_node.return_value = "node1.com"
+        mock_list_nodes.return_value = ["node1.com"]
+
+        self.qdevice_with_ip.copy_qnetd_crt_to_cluster()
+
+        mock_this_node.assert_called_once_with()
+        mock_list_nodes.assert_called_once_with()
+        mock_copy.assert_not_called()
+        mock_log.assert_not_called()
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.utils.list_cluster_nodes")
+    @mock.patch("crmsh.utils.this_node")
+    @mock.patch("crmsh.parallax.parallax_copy")
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_local", new_callable=mock.PropertyMock)
+    @mock.patch("os.path.dirname")
+    def test_copy_qnetd_crt_to_cluster(self, mock_dirname, mock_qnetd_cacert_local,
+                                       mock_copy, mock_this_node, mock_list_nodes, mock_log):
+        mock_qnetd_cacert_local.return_value = "/etc/corosync/qdevice/net/10.10.10.123/qnetd-cacert.crt"
+        mock_dirname.return_value = "/etc/corosync/qdevice/net/10.10.10.123"
+        mock_this_node.return_value = "node1.com"
+        mock_list_nodes.return_value = ["node1.com", "node2.com"]
+
+        self.qdevice_with_ip.copy_qnetd_crt_to_cluster()
+
+        mock_this_node.assert_called_once_with()
+        mock_list_nodes.assert_called_once_with()
+        mock_log.assert_called_once_with("Step 3: Copy exported qnetd-cacert.crt to ['node2.com']")
+        mock_copy.assert_called_once_with(["node2.com"], mock_dirname.return_value,
+                                          "/etc/corosync/qdevice/net", True)
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.parallax.parallax_call")
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_local", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.utils.list_cluster_nodes")
+    def test_init_db_on_cluster(self, mock_list_nodes, mock_qnetd_cacert_local, mock_call, mock_log):
+        mock_list_nodes.return_value = ["node1", "node2"]
+        mock_qnetd_cacert_local.return_value = "/etc/corosync/qdevice/net/10.10.10.123/qnetd-cacert.crt"
+        mock_call.return_value = [("node1", (0, None, None)), ("node2", (0, None, None))]
+
+        self.qdevice_with_ip.init_db_on_cluster()
+
+        mock_list_nodes.assert_called_once_with()
+        mock_qnetd_cacert_local.assert_called_once_with()
+        mock_log.assert_called_once_with("Step 4: Initialize database on ['node1', 'node2']",
+                'corosync-qdevice-net-certutil -i -c /etc/corosync/qdevice/net/10.10.10.123/qnetd-cacert.crt')
+        mock_call.assert_called_once_with(mock_list_nodes.return_value,
+                                          "corosync-qdevice-net-certutil -i -c {}".format(mock_qnetd_cacert_local.return_value))
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.sh.ClusterShell.get_stdout_or_raise_error")
+    def test_create_ca_request(self, mock_stdout_stderr, mock_log):
+        mock_stdout_stderr.return_value = (0, None, None)
+
+        self.qdevice_with_cluster_name.create_ca_request()
+
+        mock_log.assert_called_once_with("Step 5: Generate certificate request qdevice-net-node.crq",
+                'corosync-qdevice-net-certutil -r -n hacluster1')
+        mock_stdout_stderr.assert_called_once_with("corosync-qdevice-net-certutil -r -n hacluster1")
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.qdevice.QDevice.qdevice_crq_on_qnetd", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.qdevice.QDevice.qdevice_crq_on_local", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.parallax.parallax_copy")
+    def test_copy_crq_to_qnetd(self, mock_copy, mock_qdevice_crq_local,
+                               mock_qdevice_crq_qnetd, mock_log):
+        mock_qdevice_crq_local.return_value = "/etc/corosync/qdevice/net/nssdb/qdevice-net-node.crq"
+        mock_qdevice_crq_qnetd.return_value = "/etc/corosync/qnetd/nssdb/qdevice-net-node.crq"
+
+        self.qdevice_with_ip.copy_crq_to_qnetd()
+
+        mock_log.assert_called_once_with("Step 6: Copy qdevice-net-node.crq to 10.10.10.123")
+        mock_copy.assert_called_once_with(["10.10.10.123"], mock_qdevice_crq_local.return_value,
+                                          mock_qdevice_crq_qnetd.return_value, False)
+        mock_qdevice_crq_local.assert_called_once_with()
+        mock_qdevice_crq_qnetd.assert_called_once_with()
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.parallax.parallax_call")
+    @mock.patch("crmsh.qdevice.QDevice.qdevice_crq_on_qnetd", new_callable=mock.PropertyMock)
+    def test_sign_crq_on_qnetd(self, mock_qdevice_crq_qnetd, mock_call, mock_log):
+        mock_qdevice_crq_qnetd.return_value = "/etc/corosync/qnetd/nssdb/qdevice-net-node.crq"
+        mock_call.return_value = ["10.10.10.123", (0, None, None)]
+
+        self.qdevice_with_ip.cluster_name = "hacluster"
+        self.qdevice_with_ip.sign_crq_on_qnetd()
+
+        mock_log.assert_called_once_with("Step 7: Sign and export cluster certificate on 10.10.10.123",
+                'corosync-qnetd-certutil -s -c /etc/corosync/qnetd/nssdb/qdevice-net-node.crq -n hacluster')
+        mock_qdevice_crq_qnetd.assert_called_once_with()
+        mock_call.assert_called_once_with(["10.10.10.123"],
+                                          "corosync-qnetd-certutil -s -c {} -n hacluster".format(mock_qdevice_crq_qnetd.return_value))
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cluster_crt_on_qnetd", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.parallax.parallax_slurp")
+    def test_fetch_cluster_crt_from_qnetd(self, mock_parallax_slurp, mock_crt_on_qnetd, mock_log):
+        mock_crt_on_qnetd.return_value = "/etc/corosync/qnetd/nssdb/cluster-hacluster.crt"
+
+        self.qdevice_with_ip.cluster_name = "hacluster"
+        self.qdevice_with_ip.fetch_cluster_crt_from_qnetd()
+
+        mock_log.assert_called_once_with("Step 8: Fetch cluster-hacluster.crt from 10.10.10.123")
+        mock_crt_on_qnetd.assert_has_calls([mock.call(), mock.call()])
+        mock_parallax_slurp.assert_called_once_with(["10.10.10.123"], "/etc/corosync/qdevice/net", mock_crt_on_qnetd.return_value)
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.sh.ClusterShell.get_stdout_or_raise_error")
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cluster_crt_on_local", new_callable=mock.PropertyMock)
+    def test_import_cluster_crt(self, mock_crt_on_local, mock_stdout_stderr, mock_log):
+        mock_crt_on_local.return_value = "/etc/corosync/qdevice/net/10.10.10.123/cluster-hacluster.crt"
+
+        self.qdevice_with_ip.import_cluster_crt()
+
+        mock_log.assert_called_once_with("Step 9: Import certificate file cluster-hacluster.crt on local",
+                'corosync-qdevice-net-certutil -M -c /etc/corosync/qdevice/net/10.10.10.123/cluster-hacluster.crt')
+        mock_crt_on_local.assert_has_calls([mock.call(), mock.call()])
+        mock_stdout_stderr.assert_called_once_with("corosync-qdevice-net-certutil -M -c {}".format(mock_crt_on_local.return_value))
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.utils.list_cluster_nodes")
+    @mock.patch("crmsh.utils.this_node")
+    @mock.patch("crmsh.parallax.parallax_copy")
+    def test_copy_p12_to_cluster_one_node(self, mock_copy, mock_this_node, mock_list_nodes, mock_log):
+        mock_this_node.return_value = "node1.com"
+        mock_list_nodes.return_value = ["node1.com"]
+
+        self.qdevice_with_ip.copy_p12_to_cluster()
+
+        mock_log.assert_not_called()
+        mock_this_node.assert_called_once_with()
+        mock_list_nodes.assert_called_once_with()
+        mock_copy.assert_not_called()
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.utils.list_cluster_nodes")
+    @mock.patch("crmsh.utils.this_node")
+    @mock.patch("crmsh.parallax.parallax_copy")
+    @mock.patch("crmsh.qdevice.QDevice.qdevice_p12_on_local", new_callable=mock.PropertyMock)
+    def test_copy_p12_to_cluster(self, mock_p12_on_local,
+                                       mock_copy, mock_this_node, mock_list_nodes, mock_log):
+        mock_this_node.return_value = "node1.com"
+        mock_list_nodes.return_value = ["node1.com", "node2.com"]
+        mock_p12_on_local.return_value = "/etc/corosync/qdevice/net/nssdb/qdevice-net-node.p12"
+
+        self.qdevice_with_ip.copy_p12_to_cluster()
+
+        mock_log.assert_called_once_with("Step 10: Copy qdevice-net-node.p12 to ['node2.com']")
+        mock_this_node.assert_called_once_with()
+        mock_list_nodes.assert_called_once_with()
+        mock_copy.assert_called_once_with(["node2.com"], mock_p12_on_local.return_value,
+                                          mock_p12_on_local.return_value, False)
+        mock_p12_on_local.assert_has_calls([mock.call(), mock.call()])
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.parallax.parallax_call")
+    @mock.patch("crmsh.utils.list_cluster_nodes_except_me")
+    def test_import_p12_on_cluster_one_node(self, mock_list_nodes, mock_call, mock_log):
+        mock_list_nodes.return_value = []
+
+        self.qdevice_with_ip.import_p12_on_cluster()
+
+        mock_log.assert_not_called()
+        mock_list_nodes.assert_called_once_with()
+        mock_call.assert_not_called()
+
+    @mock.patch("crmsh.parallax.parallax_call")
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.qdevice.QDevice.qdevice_p12_on_local", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.utils.list_cluster_nodes_except_me")
+    def test_import_p12_on_cluster(self, mock_list_nodes, mock_p12_on_local, mock_log, mock_call):
+        mock_list_nodes.return_value = ["node2", "node3"]
+        mock_p12_on_local.return_value = "/etc/corosync/qdevice/net/nssdb/qdevice-net-node.p12"
+        mock_call.return_value = [("node2", (0, None, None)), ("node3", (0, None, None))]
+
+        self.qdevice_with_ip.import_p12_on_cluster()
+
+        mock_log.assert_called_once_with("Step 11: Import qdevice-net-node.p12 on ['node2', 'node3']",
+                'corosync-qdevice-net-certutil -m -c /etc/corosync/qdevice/net/nssdb/qdevice-net-node.p12')
+        mock_list_nodes.assert_called_once_with()
+        mock_call.assert_called_once_with(
+                ["node2", "node3"],
+                "corosync-qdevice-net-certutil -m -c {}".format(mock_p12_on_local.return_value))
+        mock_p12_on_local.assert_called_once_with()
+
+    @mock.patch("crmsh.qdevice.QDevice.import_p12_on_cluster")
+    @mock.patch("crmsh.qdevice.QDevice.copy_p12_to_cluster")
+    @mock.patch("crmsh.qdevice.QDevice.import_cluster_crt")
+    @mock.patch("crmsh.qdevice.QDevice.fetch_cluster_crt_from_qnetd")
+    @mock.patch("crmsh.qdevice.QDevice.sign_crq_on_qnetd")
+    @mock.patch("crmsh.qdevice.QDevice.copy_crq_to_qnetd")
+    @mock.patch("crmsh.qdevice.QDevice.create_ca_request")
+    @mock.patch("crmsh.qdevice.QDevice.init_db_on_cluster")
+    @mock.patch("crmsh.qdevice.QDevice.copy_qnetd_crt_to_cluster")
+    @mock.patch("crmsh.qdevice.QDevice.fetch_qnetd_crt_from_qnetd")
+    @mock.patch("crmsh.qdevice.QDevice.init_db_on_qnetd")
+    def test_certificate_process_on_init(self, mock_init_db_on_qnetd, mock_fetch_qnetd_crt_from_qnetd,
+            mock_copy_qnetd_crt_to_cluster, mock_init_db_on_cluster, mock_create_ca_request,
+            mock_copy_crq_to_qnetd, mock_sign_crq_on_qnetd, mock_fetch_cluster_crt_from_qnetd,
+            mock_import_cluster_crt, mock_copy_p12_to_cluster, mock_import_p12_on_cluster):
+
+        self.qdevice_with_ip.certificate_process_on_init()
+        mock_init_db_on_qnetd.assert_called_once_with()
+        mock_fetch_qnetd_crt_from_qnetd.assert_called_once_with()
+        mock_copy_qnetd_crt_to_cluster.assert_called_once_with()
+        mock_init_db_on_cluster.assert_called_once_with()
+        mock_create_ca_request.assert_called_once_with()
+        mock_copy_crq_to_qnetd.assert_called_once_with()
+        mock_sign_crq_on_qnetd.assert_called_once_with()
+        mock_fetch_cluster_crt_from_qnetd.assert_called_once_with()
+        mock_import_cluster_crt.assert_called_once_with()
+        mock_copy_p12_to_cluster.assert_called_once_with()
+        mock_import_p12_on_cluster.assert_called_once_with()
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("os.path.exists")
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_cluster", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_local", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.parallax.parallax_slurp")
+    def test_fetch_qnetd_crt_from_cluster_exist(self, mock_parallax_slurp, mock_qnetd_cacert_local,
+                                                mock_qnetd_cacert_cluster, mock_exists, mock_log):
+        mock_exists.return_value = True
+        mock_qnetd_cacert_cluster.return_value = "/etc/corosync/qdevice/net/node1.com/qnetd-cacert.crt"
+
+        self.qdevice_with_ip_cluster_node.fetch_qnetd_crt_from_cluster()
+
+        mock_log.assert_not_called()
+        mock_exists.assert_called_once_with(mock_qnetd_cacert_cluster.return_value)
+        mock_qnetd_cacert_cluster.assert_called_once_with()
+        mock_qnetd_cacert_local.assert_not_called()
+        mock_parallax_slurp.assert_not_called()
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("os.path.exists")
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_cluster", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_local", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.parallax.parallax_slurp")
+    def test_fetch_qnetd_crt_from_cluster(self, mock_parallax_slurp, mock_qnetd_cacert_local,
+                                          mock_qnetd_cacert_cluster, mock_exists, mock_log):
+        mock_exists.return_value = False
+        mock_qnetd_cacert_cluster.return_value = "/etc/corosync/qdevice/net/node1.com/qnetd-cacert.crt"
+        mock_qnetd_cacert_local.return_value = "/etc/corosync/qdevice/net/10.10.10.123/qnetd-cacert.crt"
+
+        self.qdevice_with_ip_cluster_node.fetch_qnetd_crt_from_cluster()
+
+        mock_log.assert_called_once_with("Step 1: Fetch qnetd-cacert.crt from node1.com")
+        mock_exists.assert_called_once_with(mock_qnetd_cacert_cluster.return_value)
+        mock_qnetd_cacert_cluster.assert_called_once_with()
+        mock_qnetd_cacert_local.assert_called_once_with()
+        mock_parallax_slurp.assert_called_once_with(["node1.com"], "/etc/corosync/qdevice/net", mock_qnetd_cacert_local.return_value)
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.sh.ClusterShell.get_stdout_or_raise_error")
+    @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_cluster", new_callable=mock.PropertyMock)
+    def test_init_db_on_local(self, mock_qnetd_cacert_cluster, mock_stdout_stderr, mock_log):
+        mock_qnetd_cacert_cluster.return_value = "/etc/corosync/qdevice/net/node1.com/qnetd-cacert.crt"
+        mock_stdout_stderr.return_value = (0, None, None)
+
+        self.qdevice_with_ip_cluster_node.init_db_on_local()
+
+        mock_log.assert_called_once_with("Step 2: Initialize database on local",
+                'corosync-qdevice-net-certutil -i -c /etc/corosync/qdevice/net/node1.com/qnetd-cacert.crt')
+        mock_qnetd_cacert_cluster.assert_called_once_with()
+        mock_stdout_stderr.assert_called_once_with("corosync-qdevice-net-certutil -i -c {}".format(mock_qnetd_cacert_cluster.return_value))
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("os.path.exists")
+    @mock.patch("crmsh.qdevice.QDevice.qdevice_p12_on_cluster", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.qdevice.QDevice.qdevice_p12_on_local", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.parallax.parallax_slurp")
+    def test_fetch_p12_from_cluster_exist(self, mock_parallax_slurp, mock_p12_on_local,
+                                          mock_p12_on_cluster, mock_exists, mock_log):
+        mock_exists.return_value = True
+        mock_p12_on_cluster.return_value = "/etc/corosync/qdevice/net/node1.com/qdevice-net-node.p12"
+
+        self.qdevice_with_ip_cluster_node.fetch_p12_from_cluster()
+
+        mock_log.assert_not_called()
+        mock_exists.assert_called_once_with(mock_p12_on_cluster.return_value)
+        mock_p12_on_cluster.assert_called_once_with()
+        mock_p12_on_local.assert_not_called()
+        mock_parallax_slurp.assert_not_called()
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("os.path.exists")
+    @mock.patch("crmsh.qdevice.QDevice.qdevice_p12_on_cluster", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.qdevice.QDevice.qdevice_p12_on_local", new_callable=mock.PropertyMock)
+    @mock.patch("crmsh.parallax.parallax_slurp")
+    def test_fetch_p12_from_cluster(self, mock_parallax_slurp, mock_p12_on_local,
+                                    mock_p12_on_cluster, mock_exists, mock_log):
+        mock_exists.return_value = False
+        mock_p12_on_cluster.return_value = "/etc/corosync/qdevice/net/node1.com/qdevice-net-node.p12"
+        mock_p12_on_local.return_value = "/etc/corosync/qdevice/net/nssdb/qdevice-net-node.p12"
+
+        self.qdevice_with_ip_cluster_node.fetch_p12_from_cluster()
+
+        mock_log.assert_called_once_with("Step 3: Fetch qdevice-net-node.p12 from node1.com")
+        mock_exists.assert_called_once_with(mock_p12_on_cluster.return_value)
+        mock_p12_on_cluster.assert_called_once_with()
+        mock_p12_on_local.assert_called_once_with()
+        mock_parallax_slurp.assert_called_once_with(["node1.com"], '/etc/corosync/qdevice/net', mock_p12_on_local.return_value)
+
+    @mock.patch("crmsh.qdevice.QDevice.log_only_to_file")
+    @mock.patch("crmsh.sh.ClusterShell.get_stdout_or_raise_error")
+    @mock.patch("crmsh.qdevice.QDevice.qdevice_p12_on_cluster", new_callable=mock.PropertyMock)
+    def test_import_p12_on_local(self, mock_p12_on_cluster, mock_stdout_stderr, mock_log):
+        mock_p12_on_cluster.return_value = "/etc/corosync/qdevice/net/node1.com/qdevice-net-node.p12"
+
+        self.qdevice_with_ip_cluster_node.import_p12_on_local()
+
+        mock_log.assert_called_once_with("Step 4: Import cluster certificate and key",
+                'corosync-qdevice-net-certutil -m -c /etc/corosync/qdevice/net/node1.com/qdevice-net-node.p12')
+        mock_p12_on_cluster.assert_called_once_with()
+        mock_stdout_stderr.assert_called_once_with("corosync-qdevice-net-certutil -m -c {}".format(mock_p12_on_cluster.return_value))
+
+    @mock.patch("crmsh.qdevice.QDevice.import_p12_on_local")
+    @mock.patch("crmsh.qdevice.QDevice.fetch_p12_from_cluster")
+    @mock.patch("crmsh.qdevice.QDevice.init_db_on_local")
+    @mock.patch("crmsh.qdevice.QDevice.fetch_qnetd_crt_from_cluster")
+    def test_certificate_process_on_join(self, mock_fetch_qnetd_crt_from_cluster, mock_init_db_on_local,
+            mock_fetch_p12_from_cluster, mock_import_p12_on_local):
+        self.qdevice_with_ip.certificate_process_on_join()
+        mock_fetch_qnetd_crt_from_cluster.assert_called_once_with()
+        mock_init_db_on_local.assert_called_once_with()
+        mock_fetch_p12_from_cluster.assert_called_once_with()
+        mock_import_p12_on_local.assert_called_once_with()
+
+    @mock.patch("crmsh.utils.str2file")
+    @mock.patch("crmsh.corosync.make_section")
+    @mock.patch("crmsh.corosync.Parser")
+    @mock.patch("crmsh.corosync.conf")
+    @mock.patch("crmsh.utils.read_from_file")
+    def test_write_qdevice_config(self, mock_read_file, mock_conf, mock_parser, mock_mksection, mock_str2file):
+        mock_mksection.side_effect = [
+            ["device {", "}"],
+            ["net {", "}"]
+        ]
+        mock_read_file.return_value = "data"
+        mock_conf.side_effect = ["corosync.conf", "corosync.conf"]
+        mock_instance = mock.Mock()
+        mock_parser.return_value = mock_instance
+        mock_instance.to_string.return_value = "string data"
+
+        self.qdevice_with_ip.write_qdevice_config()
+
+        mock_conf.assert_has_calls([mock.call(), mock.call()])
+        mock_parser.assert_called_once_with("data")
+        mock_instance.remove.assert_called_once_with("quorum.device")
+        mock_instance.add.assert_has_calls([
+            mock.call('quorum', ["device {", "}"]),
+            mock.call('quorum.device', ["net {", "}"])
+        ])
+        mock_instance.set.assert_has_calls([
+            mock.call('quorum.device.votes', '1'),
+            mock.call('quorum.device.model', 'net'),
+            mock.call('quorum.device.net.tls', 'on'),
+            mock.call('quorum.device.net.host', '10.10.10.123'),
+            mock.call('quorum.device.net.port', 5403),
+            mock.call('quorum.device.net.algorithm', 'ffsplit'),
+            mock.call('quorum.device.net.tie_breaker', 'lowest')
+        ])
+        mock_instance.to_string.assert_called_once_with()
+        mock_mksection.assert_has_calls([
+            mock.call('quorum.device', []),
+            mock.call('quorum.device.net', [])
+        ])
+        mock_str2file.assert_called_once_with("string data", "corosync.conf")
+
+    @mock.patch("crmsh.utils.str2file")
+    @mock.patch("crmsh.corosync.Parser")
+    @mock.patch("crmsh.corosync.conf")
+    @mock.patch("crmsh.utils.read_from_file")
+    def test_remove_qdevice_config(self, mock_read_file, mock_conf, mock_parser, mock_str2file):
+        mock_conf.side_effect = ["corosync.conf", "corosync.conf"]
+        mock_read_file.return_value = "data"
+        mock_instance = mock.Mock()
+        mock_parser.return_value = mock_instance
+        mock_instance.to_string.return_value = "string data"
+
+        self.qdevice_with_ip.remove_qdevice_config()
+
+        mock_conf.assert_has_calls([mock.call(), mock.call()])
+        mock_parser.assert_called_once_with("data")
+        mock_instance.remove.assert_called_once_with("quorum.device")
+        mock_instance.to_string.assert_called_once_with()
+        mock_str2file.assert_called_once_with("string data", "corosync.conf")
+
+    @mock.patch("crmsh.parallax.parallax_call")
+    @mock.patch('crmsh.utils.list_cluster_nodes')
+    @mock.patch('os.path.exists')
+    def test_remove_qdevice_db_not_exist(self, mock_exists, mock_list_nodes, mock_call):
+        mock_exists.return_value = False
+
+        self.qdevice_with_ip.remove_qdevice_db()
+
+        mock_exists.assert_called_once_with('/etc/corosync/qdevice/net/nssdb')
+        mock_list_nodes.assert_not_called()
+        mock_call.assert_not_called()
+
+    @mock.patch("crmsh.parallax.parallax_call")
+    @mock.patch('crmsh.utils.list_cluster_nodes')
+    @mock.patch('os.path.exists')
+    def test_remove_qdevice_db(self, mock_exists, mock_list_nodes, mock_call):
+        mock_exists.return_value = True
+        mock_list_nodes.return_value = ["node1.com", "node2.com"]
+        mock_call.return_value = [("node1.com", (0, None, None)), ("node2.com", (0, None, None))]
+
+        self.qdevice_with_ip.remove_qdevice_db()
+
+        mock_exists.assert_called_once_with('/etc/corosync/qdevice/net/nssdb')
+        mock_list_nodes.assert_called_once_with()
+        mock_call.assert_called_once_with(mock_list_nodes.return_value,
+                                          'rm -rf /etc/corosync/qdevice/net/*'.format())
+
+    @mock.patch('logging.Logger.warning')
+    @mock.patch('crmsh.corosync.get_value')
+    @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error')
+    def test_check_qdevice_vote(self, mock_run, mock_get_value, mock_warning):
+        data = """
+Membership information
+----------------------
+    Nodeid      Votes    Qdevice Name
+         1          1    A,V,NMW 192.168.122.221 (local)
+         0          0            Qdevice
+        """
+        mock_run.return_value = data
+        mock_get_value.return_value = "qnetd-node"
+        qdevice.QDevice.check_qdevice_vote()
+        mock_run.assert_called_once_with("corosync-quorumtool -s", success_exit_status={0, 2})
+        mock_get_value.assert_called_once_with("quorum.device.net.host")
+        mock_warning.assert_called_once_with("Qdevice's vote is 0, which simply means Qdevice can't talk to Qnetd(qnetd-node) for various reasons.")
+
+    @mock.patch('crmsh.qdevice.evaluate_qdevice_quorum_effect')
+    @mock.patch('crmsh.log.LoggerUtils.status_long')
+    @mock.patch('crmsh.qdevice.QDevice.remove_qdevice_db')
+    def test_config_and_start_qdevice(self, mock_rm_db, mock_status_long, mock_evaluate):
+        mock_status_long.return_value.__enter__ = mock.Mock()
+        mock_status_long.return_value.__exit__ = mock.Mock()
+        self.qdevice_with_ip.certificate_process_on_init = mock.Mock()
+        self.qdevice_with_ip.adjust_sbd_watchdog_timeout_with_qdevice = mock.Mock()
+        self.qdevice_with_ip.config_qdevice = mock.Mock()
+        self.qdevice_with_ip.start_qdevice_service = mock.Mock()
+
+        self.qdevice_with_ip.config_and_start_qdevice.__wrapped__(self.qdevice_with_ip)
+
+        mock_rm_db.assert_called_once_with()
+        mock_status_long.assert_called_once_with("Qdevice certification process")
+        self.qdevice_with_ip.certificate_process_on_init.assert_called_once_with()
+        self.qdevice_with_ip.adjust_sbd_watchdog_timeout_with_qdevice.assert_called_once_with()
+        self.qdevice_with_ip.config_qdevice.assert_called_once_with()
+        self.qdevice_with_ip.start_qdevice_service.assert_called_once_with()
+
+    @mock.patch('crmsh.utils.set_property')
+    @mock.patch('crmsh.sbd.SBDTimeout.get_stonith_timeout')
+    @mock.patch('crmsh.sbd.SBDManager.update_configuration')
+    @mock.patch('crmsh.sbd.SBDManager.get_sbd_value_from_config')
+    @mock.patch('crmsh.sbd.SBDManager.is_using_diskless_sbd')
+    @mock.patch('crmsh.utils.check_all_nodes_reachable')
+    def test_adjust_sbd_watchdog_timeout_with_qdevice(self, mock_check_reachable, mock_using_diskless_sbd, mock_get_sbd_value, mock_update_config, mock_get_timeout, mock_set_property):
+        mock_using_diskless_sbd.return_value = True
+        mock_get_sbd_value.return_value = ""
+        mock_get_timeout.return_value = 100
+
+        self.qdevice_with_stage_cluster_name.adjust_sbd_watchdog_timeout_with_qdevice()
+
+        mock_check_reachable.assert_called_once_with()
+        mock_using_diskless_sbd.assert_called_once_with()
+        mock_get_sbd_value.assert_called_once_with("SBD_WATCHDOG_TIMEOUT")
+        mock_update_config.assert_called_once_with({"SBD_WATCHDOG_TIMEOUT": str(sbd.SBDTimeout.SBD_WATCHDOG_TIMEOUT_DEFAULT_WITH_QDEVICE)})
+        mock_set_property.assert_called_once_with("stonith-timeout", 100)
+
+    @mock.patch('crmsh.qdevice.QDevice.start_qnetd')
+    @mock.patch('crmsh.qdevice.QDevice.enable_qnetd')
+    @mock.patch('crmsh.utils.cluster_run_cmd')
+    @mock.patch('logging.Logger.info')
+    def test_start_qdevice_service_reload(self, mock_status, mock_cluster_run, mock_enable_qnetd, mock_start_qnetd):
+        self.qdevice_with_ip.qdevice_reload_policy = qdevice.QdevicePolicy.QDEVICE_RELOAD
+
+        self.qdevice_with_ip.start_qdevice_service()
+
+        mock_status.assert_has_calls([
+            mock.call("Enable corosync-qdevice.service in cluster"),
+            mock.call("Starting corosync-qdevice.service in cluster"),
+            mock.call("Enable corosync-qnetd.service on 10.10.10.123"),
+            mock.call("Starting corosync-qnetd.service on 10.10.10.123")
+            ])
+        mock_cluster_run.assert_has_calls([
+            mock.call("systemctl enable corosync-qdevice"),
+            mock.call("systemctl restart corosync-qdevice")
+            ])
+        mock_enable_qnetd.assert_called_once_with()
+        mock_start_qnetd.assert_called_once_with()
+
+    @mock.patch('crmsh.qdevice.QDevice.start_qnetd')
+    @mock.patch('crmsh.qdevice.QDevice.enable_qnetd')
+    @mock.patch('crmsh.bootstrap.wait_for_cluster')
+    @mock.patch('crmsh.utils.cluster_run_cmd')
+    @mock.patch('logging.Logger.info')
+    def test_start_qdevice_service_restart(self, mock_status, mock_cluster_run, mock_wait, mock_enable_qnetd, mock_start_qnetd):
+        self.qdevice_with_ip.qdevice_reload_policy = qdevice.QdevicePolicy.QDEVICE_RESTART
+
+        self.qdevice_with_ip.start_qdevice_service()
+
+        mock_status.assert_has_calls([
+            mock.call("Enable corosync-qdevice.service in cluster"),
+            mock.call("Restarting cluster service"),
+            mock.call("Enable corosync-qnetd.service on 10.10.10.123"),
+            mock.call("Starting corosync-qnetd.service on 10.10.10.123")
+            ])
+        mock_wait.assert_called_once_with()
+        mock_cluster_run.assert_has_calls([
+            mock.call("systemctl enable corosync-qdevice"),
+            mock.call("crm cluster restart")
+            ])
+        mock_enable_qnetd.assert_called_once_with()
+        mock_start_qnetd.assert_called_once_with()
+
+    @mock.patch('crmsh.qdevice.QDevice.start_qnetd')
+    @mock.patch('crmsh.qdevice.QDevice.enable_qnetd')
+    @mock.patch('logging.Logger.warning')
+    @mock.patch('crmsh.utils.cluster_run_cmd')
+    @mock.patch('logging.Logger.info')
+    def test_start_qdevice_service_warn(self, mock_status, mock_cluster_run, mock_warn, mock_enable_qnetd, mock_start_qnetd):
+        self.qdevice_with_ip.qdevice_reload_policy = qdevice.QdevicePolicy.QDEVICE_RESTART_LATER
+
+        self.qdevice_with_ip.start_qdevice_service()
+
+        mock_status.assert_has_calls([
+            mock.call("Enable corosync-qdevice.service in cluster"),
+            mock.call("Enable corosync-qnetd.service on 10.10.10.123"),
+            mock.call("Starting corosync-qnetd.service on 10.10.10.123")
+            ])
+        mock_cluster_run.assert_has_calls([
+            mock.call("systemctl enable corosync-qdevice"),
+            ])
+        mock_warn.assert_called_once_with("To use qdevice service, need to restart cluster service manually on each node")
+        mock_enable_qnetd.assert_called_once_with()
+        mock_start_qnetd.assert_called_once_with()
+
+    @mock.patch('crmsh.utils.cluster_run_cmd')
+    @mock.patch('crmsh.bootstrap.update_expected_votes')
+    @mock.patch('crmsh.log.LoggerUtils.status_long')
+    @mock.patch('crmsh.corosync.add_nodelist_from_cmaptool')
+    @mock.patch('crmsh.corosync.is_unicast')
+    @mock.patch('crmsh.qdevice.QDevice.write_qdevice_config')
+    def test_config_qdevice(self, mock_write, mock_is_unicast, mock_add_nodelist, mock_status_long,
+            mock_update_votes, mock_run):
+        mock_is_unicast.return_value = False
+        mock_status_long.return_value.__enter__ = mock.Mock()
+        mock_status_long.return_value.__exit__ = mock.Mock()
+        self.qdevice_with_ip.qdevice_reload_policy = qdevice.QdevicePolicy.QDEVICE_RELOAD
+
+        self.qdevice_with_ip.config_qdevice()
+
+        mock_write.assert_called_once_with()
+        mock_is_unicast.assert_called_once_with()
+        mock_add_nodelist.assert_called_once_with()
+        mock_status_long.assert_called_once_with("Update configuration")
+        mock_update_votes.assert_called_once_with()
+        mock_run.assert_called_once_with("crm corosync reload")
+
+    @mock.patch('crmsh.utils.is_qdevice_configured')
+    def test_remove_certification_files_on_qnetd_return(self, mock_configured):
+        mock_configured.return_value = False
+        qdevice.QDevice.remove_certification_files_on_qnetd()
+        mock_configured.assert_called_once_with()
+
+    @mock.patch('crmsh.sh.ClusterShell.get_stdout_or_raise_error')
+    @mock.patch('crmsh.corosync.get_value')
+    @mock.patch('crmsh.utils.is_qdevice_configured')
+    def test_remove_certification_files_on_qnetd(self, mock_configured, mock_get_value, mock_run):
+        mock_configured.return_value = True
+        mock_get_value.side_effect = ["qnetd-node", "cluster1"]
+        qdevice.QDevice.remove_certification_files_on_qnetd()
+        mock_configured.assert_called_once_with()
+        mock_get_value.assert_has_calls([
+            mock.call("quorum.device.net.host"),
+            mock.call("totem.cluster_name")])
+        crt_file = "/etc/corosync/qnetd/nssdb/cluster-cluster1.crt"
+        crt_cmd = "test -f {crt_file} && rm -f {crt_file}".format(crt_file=crt_file)
+        crq_file = "/etc/corosync/qnetd/nssdb/qdevice-net-node.crq.cluster1"
+        crq_cmd = "test -f {crq_file} && rm -f {crq_file}".format(crq_file=crq_file)
+        mock_run.assert_has_calls([
+            mock.call(crt_cmd, "qnetd-node"),
+            mock.call(crq_cmd, "qnetd-node")])