diff options
Diffstat (limited to 'test/features/steps')
| -rw-r--r-- | test/features/steps/__init__.py | 0 | ||||
| -rwxr-xr-x | test/features/steps/behave_agent.py | 134 | ||||
| -rw-r--r-- | test/features/steps/const.py | 353 | ||||
| -rw-r--r-- | test/features/steps/step_implementation.py | 575 | ||||
| -rw-r--r-- | test/features/steps/utils.py | 177 |
5 files changed, 1239 insertions, 0 deletions
diff --git a/test/features/steps/__init__.py b/test/features/steps/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/features/steps/__init__.py diff --git a/test/features/steps/behave_agent.py b/test/features/steps/behave_agent.py new file mode 100755 index 0000000..eafeedd --- /dev/null +++ b/test/features/steps/behave_agent.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# behave_agent.py - a simple agent to execute command +# NO AUTHENTICATIONS. It should only be used in behave test. +import io +import os +import pwd +import socket +import struct +import subprocess +import typing + + +MSG_EOF = 0 +MSG_USER = 1 +MSG_CMD = 2 +MSG_OUT = 4 +MSG_ERR = 5 +MSG_RC = 6 + + +class Message: + @staticmethod + def write(output, type: int, data: bytes): + output.write(struct.pack('!ii', type, len(data))) + output.write(data) + + @staticmethod + def read(input): + buf = input.read(8) + type, length = struct.unpack('!ii', buf) + if length > 0: + buf = input.read(length) + else: + buf = b'' + return type, buf + + +class SocketIO(io.RawIOBase): + def __init__(self, s: socket.socket): + self._socket = s + + def readable(self) -> bool: + return True + + def writable(self) -> bool: + return True + + def read(self, __size: int = -1) -> bytes: + return self._socket.recv(__size) + + def readinto(self, __buffer) -> int: + return self._socket.recv_into(__buffer) + + def readall(self) -> bytes: + raise NotImplementedError + + def write(self, __b) -> int: + return self._socket.send(__b) + + +def call(host: str, port: int, cmdline: str, user: typing.Optional[str] = None): + family, type, proto, _, sockaddr = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)[0] + with socket.socket(family, type, proto) as s: + s.connect(sockaddr) + sout = io.BufferedWriter(SocketIO(s), 4096) + Message.write(sout, MSG_USER, user.encode('utf-8') if user else _getuser().encode('utf-8')) + Message.write(sout, MSG_CMD, cmdline.encode('utf-8')) + Message.write(sout, MSG_EOF, b'') + sout.flush() + s.shutdown(socket.SHUT_WR) + rc = None + stdout = [] + stderr = [] + sin = io.BufferedReader(SocketIO(s), 4096) + while True: + type, buf = Message.read(sin) + if type == MSG_OUT: + stdout.append(buf) + elif type == MSG_ERR: + stderr.append(buf) + elif type == MSG_RC: + rc, = struct.unpack('!i', buf) + elif type == MSG_EOF: + assert rc is not None + return rc, b''.join(stdout), b''.join(stderr) + else: + raise ValueError(f"Unknown message type: {type}") + + +def serve(stdin, stdout, stderr): + # This is an xinetd-style service. + assert os.geteuid() == 0 + user = None + cmd = None + sin = io.BufferedReader(stdin) + while True: + type, buf = Message.read(sin) + if type == MSG_USER: + user = buf.decode('utf-8') + elif type == MSG_CMD: + cmd = buf.decode('utf-8') + elif type == MSG_EOF: + assert user is not None + assert cmd is not None + break + else: + raise ValueError(f"Unknown message type: {type}") + if user == 'root': + args = ['/bin/sh'] + else: + args = ['/bin/su', '-', user, '-c', '/bin/sh'] + result = subprocess.run( + args, + input=cmd.encode('utf-8'), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + sout = io.BufferedWriter(stdout) + Message.write(sout, MSG_RC, struct.pack('!i', result.returncode)) + Message.write(sout, MSG_OUT, result.stdout) + Message.write(sout, MSG_ERR, result.stderr) + Message.write(sout, MSG_EOF, b'') + stdout.flush() + + +def _getuser(): + return pwd.getpwuid(os.geteuid()).pw_name + + +if __name__ == '__main__': + with open(0, 'rb') as stdin, \ + open(1, 'wb') as stdout, \ + open(2, 'wb') as stderr: + serve(stdin, stdout, stderr) diff --git a/test/features/steps/const.py b/test/features/steps/const.py new file mode 100644 index 0000000..3ec8845 --- /dev/null +++ b/test/features/steps/const.py @@ -0,0 +1,353 @@ +CRM_H_OUTPUT = '''usage: crm [-h|--help] [OPTIONS] [SUBCOMMAND ARGS...] +or crm help SUBCOMMAND + +For a list of available subcommands, use crm help. + +Use crm without arguments for an interactive session. +Call a subcommand directly for a "single-shot" use. +Call crm with a level name as argument to start an interactive +session from that level. + +See the crm(8) man page or call crm help for more details. + +positional arguments: + SUBCOMMAND + +optional arguments: + -h, --help show this help message and exit + --version show program's version number and exit + -f FILE, --file FILE Load commands from the given file. If a dash (-) is + used in place of a file name, crm will read commands + from the shell standard input (stdin). + -c CIB, --cib CIB Start the session using the given shadow CIB file. + Equivalent to `cib use <CIB>`. + -D OUTPUT_TYPE, --display OUTPUT_TYPE + Choose one of the output options: plain, color-always, + color, or uppercase. The default is color if the + terminal emulation supports colors, else plain. + -F, --force Make crm proceed with applying changes where it would + normally ask the user to confirm before proceeding. + This option is mainly useful in scripts, and should be + used with care. + -n, --no Automatically answer no when prompted + -w, --wait Make crm wait for the cluster transition to finish + (for the changes to take effect) after each processed + line. + -H DIR|FILE|SESSION, --history DIR|FILE|SESSION + A directory or file containing a cluster report to + load into history, or the name of a previously saved + history session. + -d, --debug Print verbose debugging information. + -R, --regression-tests + Enables extra verbose trace logging used by the + regression tests. Logs all external calls made by + crmsh. + --scriptdir DIR Extra directory where crm looks for cluster scripts, + or a list of directories separated by semi-colons + (e.g. /dir1;/dir2;etc.). + -X PROFILE Collect profiling data and save in PROFILE. + -o OPTION=VALUE, --opt OPTION=VALUE + Set crmsh option temporarily. If the options are saved + using+options save+ then the value passed here will + also be saved.Multiple options can be set by using + +-o+ multiple times.''' + + +CRM_CLUSTER_INIT_H_OUTPUT = '''Initializes a new HA cluster + +usage: init [options] [STAGE] + +Initialize a cluster from scratch. This command configures +a complete cluster, and can also add additional cluster +nodes to the initial one-node cluster using the --nodes +option. + +optional arguments: + -h, --help Show this help message + -q, --quiet Be quiet (don't describe what's happening, just do it) + -y, --yes Answer "yes" to all prompts (use with caution, this is + destructive, especially those storage related + configurations and stages.) + -n NAME, --name NAME Set the name of the configured cluster. + -N [USER@]HOST, --node [USER@]HOST + The member node of the cluster. Note: the current node + is always get initialized during bootstrap in the + beginning. + -S, --enable-sbd Enable SBD even if no SBD device is configured + (diskless mode) + -w WATCHDOG, --watchdog WATCHDOG + Use the given watchdog device or driver name + -x, --skip-csync2-sync + Skip csync2 initialization (an experimental option) + --no-overwrite-sshkey + Avoid "/root/.ssh/id_rsa" overwrite if "-y" option is + used (False by default; Deprecated) + --use-ssh-agent Use an existing key from ssh-agent instead of creating + new key pairs + +Network configuration: + Options for configuring the network and messaging layer. + + -i IF, --interface IF + Bind to IP address on interface IF. Use -i second time + for second interface + -u, --unicast Configure corosync to communicate over unicast(udpu). + This is the default transport type + -U, --multicast Configure corosync to communicate over multicast. + Default is unicast + -A IP, --admin-ip IP Configure IP address as an administration virtual IP + -M, --multi-heartbeats + Configure corosync with second heartbeat line + -I, --ipv6 Configure corosync use IPv6 + +QDevice configuration: + QDevice participates in quorum decisions. With the assistance of + a third-party arbitrator Qnetd, it provides votes so that a cluster + is able to sustain more node failures than standard quorum rules + allow. It is recommended for clusters with an even number of nodes + and highly recommended for 2 node clusters. + + Options for configuring QDevice and QNetd. + + --qnetd-hostname [USER@]HOST + User and host of the QNetd server. The host can be + specified in either hostname or IP address. + --qdevice-port PORT TCP PORT of QNetd server (default:5403) + --qdevice-algo ALGORITHM + QNetd decision ALGORITHM (ffsplit/lms, + default:ffsplit) + --qdevice-tie-breaker TIE_BREAKER + QNetd TIE_BREAKER (lowest/highest/valid_node_id, + default:lowest) + --qdevice-tls TLS Whether using TLS on QDevice/QNetd (on/off/required, + default:on) + --qdevice-heuristics COMMAND + COMMAND to run with absolute path. For multiple + commands, use ";" to separate (details about + heuristics can see man 8 corosync-qdevice) + --qdevice-heuristics-mode MODE + MODE of operation of heuristics (on/sync/off, + default:sync) + +Storage configuration: + Options for configuring shared storage. + + -s DEVICE, --sbd-device DEVICE + Block device to use for SBD fencing, use ";" as + separator or -s multiple times for multi path (up to 3 + devices) + -o DEVICE, --ocfs2-device DEVICE + Block device to use for OCFS2; When using Cluster LVM2 + to manage the shared storage, user can specify one or + multiple raw disks, use ";" as separator or -o + multiple times for multi path (must specify -C option) + NOTE: this is a Technical Preview + -C, --cluster-lvm2 Use Cluster LVM2 (only valid together with -o option) + NOTE: this is a Technical Preview + -m MOUNT, --mount-point MOUNT + Mount point for OCFS2 device (default is + /srv/clusterfs, only valid together with -o option) + NOTE: this is a Technical Preview + +Stage can be one of: + ssh Create SSH keys for passwordless SSH between cluster nodes + csync2 Configure csync2 + corosync Configure corosync + sbd Configure SBD (requires -s <dev>) + cluster Bring the cluster online + ocfs2 Configure OCFS2 (requires -o <dev>) NOTE: this is a Technical Preview + vgfs Create volume group and filesystem (ocfs2 template only, + requires -o <dev>) NOTE: this stage is an alias of ocfs2 stage + admin Create administration virtual IP (optional) + qdevice Configure qdevice and qnetd + +Note: + - If stage is not specified, the script will run through each stage + in sequence, with prompts for required information. + +Examples: + # Setup the cluster on the current node + crm cluster init -y + + # Setup the cluster with multiple nodes + (NOTE: the current node will be part of the cluster even not listed in the -N option as below) + crm cluster init -N node1 -N node2 -N node3 -y + + # Setup the cluster on the current node, with two network interfaces + crm cluster init -i eth1 -i eth2 -y + + # Setup the cluster on the current node, with disk-based SBD + crm cluster init -s <share disk> -y + + # Setup the cluster on the current node, with diskless SBD + crm cluster init -S -y + + # Setup the cluster on the current node, with QDevice + crm cluster init --qnetd-hostname <qnetd addr> -y + + # Setup the cluster on the current node, with SBD+OCFS2 + crm cluster init -s <share disk1> -o <share disk2> -y + + # Setup the cluster on the current node, with SBD+OCFS2+Cluster LVM + crm cluster init -s <share disk1> -o <share disk2> -o <share disk3> -C -y + + # Add SBD on a running cluster + crm cluster init sbd -s <share disk> -y + + # Replace SBD device on a running cluster which already configured SBD + crm -F cluster init sbd -s <share disk> -y + + # Add diskless SBD on a running cluster + crm cluster init sbd -S -y + + # Add QDevice on a running cluster + crm cluster init qdevice --qnetd-hostname <qnetd addr> -y + + # Add OCFS2+Cluster LVM on a running cluster + crm cluster init ocfs2 -o <share disk1> -o <share disk2> -C -y''' + + +CRM_CLUSTER_JOIN_H_OUTPUT = '''Join existing cluster + +usage: join [options] [STAGE] + +Join the current node to an existing cluster. The +current node cannot be a member of a cluster already. +Pass any node in the existing cluster as the argument +to the -c option. + +optional arguments: + -h, --help Show this help message + -q, --quiet Be quiet (don't describe what's happening, just do it) + -y, --yes Answer "yes" to all prompts (use with caution) + -w WATCHDOG, --watchdog WATCHDOG + Use the given watchdog device + --use-ssh-agent Use an existing key from ssh-agent instead of creating + new key pairs + +Network configuration: + Options for configuring the network and messaging layer. + + -c [USER@]HOST, --cluster-node [USER@]HOST + User and host to login to an existing cluster node. + The host can be specified with either a hostname or an + IP. + -i IF, --interface IF + Bind to IP address on interface IF. Use -i second time + for second interface + +Stage can be one of: + ssh Obtain SSH keys from existing cluster node (requires -c <host>) + csync2 Configure csync2 (requires -c <host>) + ssh_merge Merge root's SSH known_hosts across all nodes (csync2 must + already be configured). + cluster Start the cluster on this node + +If stage is not specified, each stage will be invoked in sequence. + +Examples: + # Join with a cluster node + crm cluster join -c <node> -y + + # Join with a cluster node, with the same network interface used by that node + crm cluster join -c <node> -i eth1 -i eth2 -y''' + + +CRM_CLUSTER_REMOVE_H_OUTPUT = '''Remove node(s) from the cluster + +usage: remove [options] [<node> ...] + +Remove one or more nodes from the cluster. + +This command can remove the last node in the cluster, +thus effectively removing the whole cluster. To remove +the last node, pass --force argument to crm or set +the config.core.force option. + +optional arguments: + -h, --help Show this help message + -q, --quiet Be quiet (don't describe what's happening, just do it) + -y, --yes Answer "yes" to all prompts (use with caution) + -c HOST, --cluster-node HOST + IP address or hostname of cluster node which will be + deleted + -F, --force Remove current node + --qdevice Remove QDevice configuration and service from cluster''' + + +CRM_CLUSTER_GEO_INIT_H_OUTPUT = '''Configure cluster as geo cluster + +usage: geo-init [options] + +Create a new geo cluster with the current cluster as the +first member. Pass the complete geo cluster topology as +arguments to this command, and then use geo-join and +geo-init-arbitrator to add the remaining members to +the geo cluster. + +optional arguments: + -h, --help Show this help message + -q, --quiet Be quiet (don't describe what's happening, just do it) + -y, --yes Answer "yes" to all prompts (use with caution) + -a [USER@]HOST, --arbitrator [USER@]HOST + Geo cluster arbitrator + -s DESC, --clusters DESC + Geo cluster description (see details below) + -t LIST, --tickets LIST + Tickets to create (space-separated) + +Cluster Description + + This is a map of cluster names to IP addresses. + Each IP address will be configured as a virtual IP + representing that cluster in the geo cluster + configuration. + + Example with two clusters named paris and amsterdam: + + --clusters "paris=192.168.10.10 amsterdam=192.168.10.11" + + Name clusters using the --name parameter to + crm bootstrap init.''' + + +CRM_CLUSTER_GEO_JOIN_H_OUTPUT = '''Join cluster to existing geo cluster + +usage: geo-join [options] + +This command should be run from one of the nodes in a cluster +which is currently not a member of a geo cluster. The geo +cluster configuration will be fetched from the provided node, +and the cluster will be added to the geo cluster. + +Note that each cluster in a geo cluster needs to have a unique +name set. The cluster name can be set using the --name argument +to init, or by configuring corosync with the cluster name in +an existing cluster. + +optional arguments: + -h, --help Show this help message + -q, --quiet Be quiet (don't describe what's happening, just do it) + -y, --yes Answer "yes" to all prompts (use with caution) + -c [USER@]HOST, --cluster-node [USER@]HOST + An already-configured geo cluster or arbitrator + -s DESC, --clusters DESC + Geo cluster description (see geo-init for details)''' + + +CRM_CLUSTER_GEO_INIT_ARBIT_H_OUTPUT = '''Initialize node as geo cluster arbitrator + +usage: geo-init-arbitrator [options] + +Configure the current node as a geo arbitrator. The command +requires an existing geo cluster or geo arbitrator from which +to get the geo cluster configuration. + +optional arguments: + -h, --help Show this help message + -q, --quiet Be quiet (don't describe what's happening, just do it) + -y, --yes Answer "yes" to all prompts (use with caution) + -c [USER@]HOST, --cluster-node [USER@]HOST + An already-configured geo cluster + --use-ssh-agent Use an existing key from ssh-agent instead of creating + new key pairs''' diff --git a/test/features/steps/step_implementation.py b/test/features/steps/step_implementation.py new file mode 100644 index 0000000..74f0cc8 --- /dev/null +++ b/test/features/steps/step_implementation.py @@ -0,0 +1,575 @@ +import re +import time +import os +import datetime +import yaml + +import behave +from behave import given, when, then +import behave_agent +from crmsh import corosync, sbd, userdir, bootstrap +from crmsh import utils as crmutils +from crmsh.sh import ShellUtils +from utils import check_cluster_state, check_service_state, online, run_command, me, \ + run_command_local_or_remote, file_in_archive, \ + assert_eq, is_unclean, assert_in +import const + + +def _parse_str(text): + return text[1:-1].encode('utf-8').decode('unicode_escape') +_parse_str.pattern='".*"' + + +behave.use_step_matcher("cfparse") +behave.register_type(str=_parse_str) + + +@when('Write multi lines to file "{f}" on "{addr}"') +def step_impl(context, f, addr): + data_list = context.text.split('\n') + for line in data_list: + echo_option = " -n" if line == data_list[-1] else "" + cmd = "echo{} \"{}\"|sudo tee -a {}".format(echo_option, line, f) + if addr != me(): + sudoer = userdir.get_sudoer() + user = f"{sudoer}@" if sudoer else "" + cmd = f"ssh {user}{addr} '{cmd}'" + run_command(context, cmd) + + +@given('Cluster service is "{state}" on "{addr}"') +def step_impl(context, state, addr): + assert check_cluster_state(context, state, addr) is True + + +@given('Nodes [{nodes:str+}] are cleaned up') +def step_impl(context, nodes): + run_command(context, 'crm resource cleanup || true') + for node in nodes: + # wait for ssh service + for _ in range(10): + rc, _, _ = ShellUtils().get_stdout_stderr('ssh {} true'.format(node)) + if rc == 0: + break + time.sleep(1) + run_command_local_or_remote(context, "crm cluster stop {} || true".format(node), node) + assert check_cluster_state(context, 'stopped', node) is True + + +@given('Service "{name}" is "{state}" on "{addr}"') +def step_impl(context, name, state, addr): + assert check_service_state(context, name, state, addr) is True + + +@given('Has disk "{disk}" on "{addr}"') +def step_impl(context, disk, addr): + _, out, _ = run_command_local_or_remote(context, "fdisk -l", addr) + assert re.search(r'{} '.format(disk), out) is not None + + +@given('Online nodes are "{nodelist}"') +def step_impl(context, nodelist): + assert online(context, nodelist) is True + + +@given('Run "{cmd}" OK') +def step_impl(context, cmd): + rc, _, _ = run_command(context, cmd) + assert rc == 0 + + +@then('Run "{cmd}" OK') +def step_impl(context, cmd): + rc, _, _ = run_command(context, cmd) + assert rc == 0 + + +@when('Run "{cmd}" OK') +def step_impl(context, cmd): + rc, _, _ = run_command(context, cmd) + assert rc == 0 + + +@given('IP "{addr}" is belong to "{iface}"') +def step_impl(context, addr, iface): + cmd = 'ip address show dev {}'.format(iface) + res = re.search(r' {}/'.format(addr), run_command(context, cmd)[1]) + assert bool(res) is True + + +@given('Run "{cmd}" OK on "{addr}"') +def step_impl(context, cmd, addr): + _, out, _ = run_command_local_or_remote(context, cmd, addr, True) + +@when('Run "{cmd}" on "{addr}"') +def step_impl(context, cmd, addr): + _, out, _ = run_command_local_or_remote(context, cmd, addr) + + +@then('Run "{cmd}" OK on "{addr}"') +def step_impl(context, cmd, addr): + _, out, _ = run_command_local_or_remote(context, cmd, addr) + + +@then('Print stdout') +def step_impl(context): + context.logger.info("\n{}".format(context.stdout)) + + +@then('Print stderr') +def step_impl(context): + context.logger.info("\n{}".format(context.stderr)) + + +@then('No crmsh tracebacks') +def step_impl(context): + if "Traceback (most recent call last):" in context.stderr and \ + re.search('File "/usr/lib/python.*/crmsh/', context.stderr): + context.logger.info("\n{}".format(context.stderr)) + context.failed = True + + +@when('Try "{cmd}" on "{addr}"') +def step_impl(context, cmd, addr): + run_command_local_or_remote(context, cmd, addr, exit_on_fail=False) + + +@when('Try "{cmd}"') +def step_impl(context, cmd): + _, out, _ = run_command(context, cmd, exit_on_fail=False) + + +@when('Wait "{second}" seconds') +def step_impl(context, second): + time.sleep(int(second)) + + +@then('Got output "{msg}"') +def step_impl(context, msg): + assert context.stdout == msg + context.stdout = None + + +@then('Expected multiple lines') +def step_impl(context): + assert context.stdout == context.text + context.stdout = None + + +@then('Expected "{msg}" in stdout') +def step_impl(context, msg): + assert_in(msg, context.stdout) + context.stdout = None + + +@then('Expected "{msg}" in stderr') +def step_impl(context, msg): + assert_in(msg, context.stderr) + context.stderr = None + + +@then('Expected regrex "{reg_str}" in stdout') +def step_impl(context, reg_str): + res = re.search(reg_str, context.stdout) + assert res is not None + context.stdout = None + + +@then('Expected return code is "{num}"') +def step_impl(context, num): + assert context.return_code == int(num) + + +@then('Expected "{msg}" not in stdout') +def step_impl(context, msg): + assert msg not in context.stdout + context.stdout = None + + +@then('Expected "{msg}" not in stderr') +def step_impl(context, msg): + assert context.stderr is None or msg not in context.stderr + context.stderr = None + + +@then('Except "{msg}"') +def step_impl(context, msg): + assert_in(msg, context.stderr) + context.stderr = None + + +@then('Except multiple lines') +def step_impl(context): + assert_in(context.text, context.stderr) + context.stderr = None + + +@then('Expected multiple lines in output') +def step_impl(context): + assert_in(context.text, context.stdout) + context.stdout = None + + +@then('Except "{msg}" in stderr') +def step_impl(context, msg): + assert_in(msg, context.stderr) + context.stderr = None + + +@then('Cluster service is "{state}" on "{addr}"') +def step_impl(context, state, addr): + assert check_cluster_state(context, state, addr) is True + + +@then('Service "{name}" is "{state}" on "{addr}"') +def step_impl(context, name, state, addr): + assert check_service_state(context, name, state, addr) is True + + +@then('Online nodes are "{nodelist}"') +def step_impl(context, nodelist): + assert online(context, nodelist) is True + + +@then('Node "{node}" is standby') +def step_impl(context, node): + assert crmutils.is_standby(node) is True + + +@then('Node "{node}" is online') +def step_impl(context, node): + assert crmutils.is_standby(node) is False + + +@then('IP "{addr}" is used by corosync on "{node}"') +def step_impl(context, addr, node): + _, out, _ = run_command_local_or_remote(context, 'corosync-cfgtool -s', node) + res = re.search(r' {}\n'.format(addr), out) + assert bool(res) is True + + +@then('Cluster name is "{name}"') +def step_impl(context, name): + _, out, _ = run_command(context, 'corosync-cmapctl -b totem.cluster_name') + assert out.split()[-1] == name + + +@then('Cluster virtual IP is "{addr}"') +def step_impl(context, addr): + _, out, _ = run_command(context, 'crm configure show|grep -A1 IPaddr2') + res = re.search(r' ip={}'.format(addr), out) + assert bool(res) is True + + +@then('Cluster is using udpu transport mode') +def step_impl(context): + assert corosync.get_value('totem.transport') == 'udpu' + + +@then('Show cluster status on "{addr}"') +def step_impl(context, addr): + _, out, _ = run_command_local_or_remote(context, 'crm_mon -1', addr) + if out: + context.logger.info("\n{}".format(out)) + + +@then('Show corosync ring status') +def step_impl(context): + _, out, _ = run_command(context, 'crm corosync status ring') + if out: + context.logger.info("\n{}".format(out)) + + +@then('Show crm configure') +def step_impl(context): + _, out, _ = run_command(context, 'crm configure show') + if out: + context.logger.info("\n{}".format(out)) + + +@then('Show status from qnetd') +def step_impl(context): + _, out, _ = run_command(context, 'crm corosync status qnetd') + if out: + context.logger.info("\n{}".format(out)) + + +@then('Show qdevice status') +def step_impl(context): + _, out, _ = run_command(context, 'crm corosync status qdevice') + if out: + context.logger.info("\n{}".format(out)) + + +@then('Show corosync qdevice configuration') +def step_impl(context): + _, out, _ = run_command(context, "sed -n -e '/quorum/,/^}/ p' /etc/corosync/corosync.conf") + if out: + context.logger.info("\n{}".format(out)) + + +@then('Resource "{res}" type "{res_type}" is "{state}"') +def step_impl(context, res, res_type, state): + try_count = 0 + result = None + while try_count < 20: + time.sleep(1) + _, out, _ = run_command(context, "crm_mon -1rR") + if out: + result = re.search(r'\s{}\s+.*:+{}\):\s+{} '.format(res, res_type, state), out) + if not result: + try_count += 1 + else: + break + assert result is not None + + +@then('Resource "{res}" failcount on "{node}" is "{number}"') +def step_impl(context, res, node, number): + cmd = "crm resource failcount {} show {}".format(res, node) + _, out, _ = run_command(context, cmd) + if out: + result = re.search(r'name=fail-count-{} value={}'.format(res, number), out) + assert result is not None + + +@then('Resource "{res_type}" not configured') +def step_impl(context, res_type): + _, out, _ = run_command(context, "crm configure show") + result = re.search(r' {} '.format(res_type), out) + assert result is None + + +@then('Output is the same with expected "{cmd}" help output') +def step_impl(context, cmd): + cmd_help = {} + cmd_help["crm"] = const.CRM_H_OUTPUT + cmd_help["crm_cluster_init"] = const.CRM_CLUSTER_INIT_H_OUTPUT + cmd_help["crm_cluster_join"] = const.CRM_CLUSTER_JOIN_H_OUTPUT + cmd_help["crm_cluster_remove"] = const.CRM_CLUSTER_REMOVE_H_OUTPUT + cmd_help["crm_cluster_geo-init"] = const.CRM_CLUSTER_GEO_INIT_H_OUTPUT + cmd_help["crm_cluster_geo-join"] = const.CRM_CLUSTER_GEO_JOIN_H_OUTPUT + cmd_help["crm_cluster_geo-init-arbitrator"] = const.CRM_CLUSTER_GEO_INIT_ARBIT_H_OUTPUT + key = '_'.join(cmd.split()) + assert_eq(cmd_help[key], context.stdout) + + +@then('Corosync working on "{transport_type}" mode') +def step_impl(context, transport_type): + if transport_type == "multicast": + assert corosync.get_value("totem.transport") is None + if transport_type == "unicast": + assert_eq("udpu", corosync.get_value("totem.transport")) + + +@then('Expected votes will be "{votes}"') +def step_impl(context, votes): + assert_eq(int(votes), int(corosync.get_value("quorum.expected_votes"))) + + +@then('Directory "{directory}" created') +def step_impl(context, directory): + assert os.path.isdir(directory) is True + + +@then('Directory "{directory}" not created') +def step_impl(context, directory): + assert os.path.isdir(directory) is False + + +@then('Default crm_report tar file created') +def step_impl(context): + default_file_name = 'crm_report-{}.tar.bz2'.format(datetime.datetime.now().strftime("%a-%d-%b-%Y")) + assert os.path.exists(default_file_name) is True + + +@when('Remove default crm_report tar file') +def step_impl(context): + default_file_name = 'crm_report-{}.tar.bz2'.format(datetime.datetime.now().strftime("%a-%d-%b-%Y")) + os.remove(default_file_name) + + +@then('File "{f}" in "{archive}"') +def step_impl(context, f, archive): + assert file_in_archive(f, archive) is True + + +@then('Directory "{f}" in "{archive}"') +def step_impl(context, f, archive): + assert file_in_archive(f, archive) is True + + +@then('File "{f}" not in "{archive}"') +def step_impl(context, f, archive): + assert file_in_archive(f, archive) is False + + +@then('File "{f}" was synced in cluster') +def step_impl(context, f): + cmd = "crm cluster diff {}".format(f) + rc, out, _ = run_command(context, cmd) + assert_eq("", out) + + +@given('Resource "{res_id}" is started on "{node}"') +def step_impl(context, res_id, node): + rc, out, err = ShellUtils().get_stdout_stderr("crm_mon -1") + assert re.search(r'\*\s+{}\s+.*Started\s+{}'.format(res_id, node), out) is not None + + +@then('Resource "{res_id}" is started on "{node}"') +def step_impl(context, res_id, node): + rc, out, err = ShellUtils().get_stdout_stderr("crm_mon -1") + assert re.search(r'\*\s+{}\s+.*Started\s+{}'.format(res_id, node), out) is not None + + +@then('SBD option "{key}" value is "{value}"') +def step_impl(context, key, value): + res = sbd.SBDManager.get_sbd_value_from_config(key) + assert_eq(value, res) + + +@then('SBD option "{key}" value for "{dev}" is "{value}"') +def step_impl(context, key, dev, value): + res = sbd.SBDTimeout.get_sbd_msgwait(dev) + assert_eq(int(value), res) + + +@then('Cluster property "{key}" is "{value}"') +def step_impl(context, key, value): + res = crmutils.get_property(key) + assert res is not None + assert_eq(value, str(res)) + + +@then('Property "{key}" in "{type}" is "{value}"') +def step_impl(context, key, type, value): + res = crmutils.get_property(key, type) + assert res is not None + assert_eq(value, str(res)) + + +@then('Parameter "{param_name}" not configured in "{res_id}"') +def step_impl(context, param_name, res_id): + _, out, _ = run_command(context, "crm configure show {}".format(res_id)) + result = re.search("params {}=".format(param_name), out) + assert result is None + + +@then('Parameter "{param_name}" configured in "{res_id}"') +def step_impl(context, param_name, res_id): + _, out, _ = run_command(context, "crm configure show {}".format(res_id)) + result = re.search("params {}=".format(param_name), out) + assert result is not None + + +@given('Yaml "{path}" value is "{value}"') +def step_impl(context, path, value): + yaml_file = "/etc/crm/profiles.yml" + with open(yaml_file) as f: + data = yaml.load(f, Loader=yaml.SafeLoader) + sec_name, key = path.split(':') + assert_eq(str(value), str(data[sec_name][key])) + + +@when('Wait for DC') +def step_impl(context): + while True: + time.sleep(1) + if crmutils.get_dc(): + break + + +@then('File "{path}" exists on "{node}"') +def step_impl(context, path, node): + rc, _, stderr = behave_agent.call(node, 1122, 'test -f {}'.format(path), user='root') + assert rc == 0 + + +@then('File "{path}" not exist on "{node}"') +def step_impl(context, path, node): + cmd = '[ ! -f {} ]'.format(path) + rc, _, stderr = behave_agent.call(node, 1122, cmd, user='root') + assert rc == 0 + + +@then('Directory "{path}" is empty on "{node}"') +def step_impl(context, path, node): + cmd = '[ ! "$(ls -A {})" ]'.format(path) + rc, _, stderr = behave_agent.call(node, 1122, cmd, user='root') + assert rc == 0 + + +@then('Directory "{path}" not empty on "{node}"') +def step_impl(context, path, node): + cmd = '[ "$(ls -A {})" ]'.format(path) + rc, _, stderr = behave_agent.call(node, 1122, cmd, user='root') + assert rc == 0 + + +@then('Node "{node}" is UNCLEAN') +def step_impl(context, node): + assert is_unclean(node) is True + + +@then('Wait "{count}" seconds for "{node}" successfully fenced') +def step_impl(context, count, node): + index = 0 + while index <= int(count): + rc, out, _ = ShellUtils().get_stdout_stderr("stonith_admin -h {}".format(node)) + if "Node {} last fenced at:".format(node) in out: + return True + time.sleep(1) + index += 1 + return False + +@then('Check passwordless for hacluster between "{nodelist}"') +def step_impl(context, nodelist): + if userdir.getuser() != 'root' or userdir.get_sudoer(): + return True + failed = False + nodes = nodelist.split() + for i in range(0, len(nodes)): + for j in range(i + 1, len(nodes)): + rc, _, _ = behave_agent.call( + nodes[i], 1122, + f'ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 {nodes[j]} true', + user='hacluster', + ) + if rc != 0: + failed = True + context.logger.error(f"There is no passwordless configured from {nodes[i]} to {nodes[j]} under 'hacluster'") + assert not failed + + +@then('Check user shell for hacluster between "{nodelist}"') +def step_impl(context, nodelist): + if userdir.getuser() != 'root' or userdir.get_sudoer(): + return True + for node in nodelist.split(): + if node == me(): + assert bootstrap.is_nologin('hacluster') is False + else: + assert bootstrap.is_nologin('hacluster', node) is False + + +@given('ssh-agent is started at "{path}" on nodes [{nodes:str+}]') +def step_impl(context, path, nodes): + user = userdir.get_sudoer() + if not user: + user = userdir.getuser() + for node in nodes: + rc, _, _ = behave_agent.call(node, 1122, f"systemd-run --uid '{user}' -u ssh-agent /usr/bin/ssh-agent -D -a '{path}'", user='root') + assert 0 == rc + + +@then('This file "{target_file}" will trigger UnicodeDecodeError exception') +def step_impl(context, target_file): + try: + with open(target_file, "r", encoding="utf-8") as file: + content = file.read() + except UnicodeDecodeError as e: + return True + else: + return False diff --git a/test/features/steps/utils.py b/test/features/steps/utils.py new file mode 100644 index 0000000..675c2c4 --- /dev/null +++ b/test/features/steps/utils.py @@ -0,0 +1,177 @@ +import concurrent.futures +import difflib +import tarfile +import glob +import re +import socket +from crmsh import utils, userdir +from crmsh.sh import ShellUtils +import behave_agent + + +COLOR_MODE = r'\x1b\[[0-9]+m' + + +def get_file_type(file_path): + rc, out, _ = ShellUtils().get_stdout_stderr("file {}".format(file_path)) + if re.search(r'{}: bzip2'.format(file_path), out): + return "bzip2" + if re.search(r'{}: directory'.format(file_path), out): + return "directory" + + +def get_all_files(archive_path): + archive_type = get_file_type(archive_path) + if archive_type == "bzip2": + with tarfile.open(archive_path) as tar: + return tar.getnames() + if archive_type == "directory": + all_files = glob.glob("{}/*".format(archive_path)) + glob.glob("{}/*/*".format(archive_path)) + return all_files + + +def file_in_archive(f, archive_path): + for item in get_all_files(archive_path): + if re.search(r'/{}$'.format(f), item): + return True + return False + + +def me(): + return socket.gethostname() + + +def _wrap_cmd_non_root(cmd): + """ + When running command under sudoer, or the current user is not root, + wrap crm cluster join command with '<user>@', and for the -N option, too + """ + sudoer = userdir.get_sudoer() + current_user = userdir.getuser() + if sudoer: + user = sudoer + elif current_user != 'root': + user = current_user + else: + return cmd + if re.search('cluster (:?join|geo_join|geo_init_arbitrator)', cmd) and "@" not in cmd: + cmd = re.sub(r'''((?:-c|-N|--qnetd-hostname|--cluster-node|--arbitrator)(?:\s+|=)['"]?)(\S{2,}['"]?)''', f'\\1{user}@\\2', cmd) + elif "cluster init" in cmd and ("-N" in cmd or "--qnetd-hostname" in cmd) and "@" not in cmd: + cmd = re.sub(r'''((?:-c|-N|--qnetd-hostname|--cluster-node)(?:\s+|=)['"]?)(\S{2,}['"]?)''', f'\\1{user}@\\2', cmd) + elif "cluster init" in cmd and "--node" in cmd and "@" not in cmd: + search_patt = r"--node [\'\"](.*)[\'\"]" + res = re.search(search_patt, cmd) + if res: + node_str = ' '.join([f"{user}@{n}" for n in res.group(1).split()]) + cmd = re.sub(search_patt, f"--node '{node_str}'", cmd) + return cmd + + +def run_command(context, cmd, exit_on_fail=True): + cmd = _wrap_cmd_non_root(cmd) + rc, out, err = ShellUtils().get_stdout_stderr(cmd) + context.return_code = rc + if out: + out = re.sub(COLOR_MODE, '', out) + context.stdout = out + if err: + err = re.sub(COLOR_MODE, '', err) + context.stderr = err + if rc != 0 and exit_on_fail: + if out: + context.logger.info("\n{}\n".format(out)) + context.logger.error("\n{}\n".format(err)) + context.failed = True + return rc, out, err + + +def run_command_local_or_remote(context, cmd, addr, exit_on_fail=True): + if addr == me(): + return run_command(context, cmd, exit_on_fail) + cmd = _wrap_cmd_non_root(cmd) + sudoer = userdir.get_sudoer() + if sudoer is None: + user = None + else: + user = sudoer + cmd = f'sudo {cmd}' + hosts = addr.split(',') + with concurrent.futures.ThreadPoolExecutor(max_workers=len(hosts)) as executor: + results = list(executor.map(lambda x: (x, behave_agent.call(x, 1122, cmd, user=user)), hosts)) + out = utils.to_ascii(results[0][1][1]) + err = utils.to_ascii(results[0][1][2]) + context.stdout = out + context.stderr = err + context.return_code = 0 + for host, (rc, stdout, stderr) in results: + if rc != 0: + err = re.sub(COLOR_MODE, '', utils.to_ascii(stderr)) + context.stderr = err + if exit_on_fail: + import os + context.logger.error("Failed to run %s on %s@%s :%s", cmd, os.geteuid(), host, err) + raise ValueError("{}".format(err)) + else: + return + return 0, out, err + + +def check_service_state(context, service_name, state, addr): + if state not in ["started", "stopped", "enabled", "disabled"]: + context.logger.error("\nService state should be \"started/stopped/enabled/disabled\"\n") + context.failed = True + if state in {'enabled', 'disabled'}: + rc, _, _ = behave_agent.call(addr, 1122, f'systemctl is-enabled {service_name}', 'root') + return (state == 'enabled') == (rc == 0) + elif state in {'started', 'stopped'}: + rc, _, _ = behave_agent.call(addr, 1122, f'systemctl is-active {service_name}', 'root') + return (state == 'started') == (rc == 0) + else: + context.logger.error("\nService state should be \"started/stopped/enabled/disabled\"\n") + raise ValueError("Service state should be \"started/stopped/enabled/disabled\"") + + +def check_cluster_state(context, state, addr): + return check_service_state(context, 'pacemaker.service', state, addr) + + +def is_unclean(node): + rc, out, err = ShellUtils().get_stdout_stderr("crm_mon -1") + return "{}: UNCLEAN".format(node) in out + + +def online(context, nodelist): + rc = True + _, out = ShellUtils().get_stdout("sudo crm_node -l") + for node in nodelist.split(): + node_info = "{} member".format(node) + if not node_info in out: + rc = False + context.logger.error("\nNode \"{}\" not online\n".format(node)) + return rc + +def assert_eq(expected, actual): + if expected != actual: + msg = "\033[32m" "Expected" "\033[31m" " != Actual" "\033[0m" "\n" \ + "\033[32m" "Expected:" "\033[0m" " {}\n" \ + "\033[31m" "Actual:" "\033[0m" " {}".format(expected, actual) + if isinstance(expected, str) and '\n' in expected: + try: + diff = '\n'.join(difflib.unified_diff( + expected.splitlines(), + actual.splitlines(), + fromfile="expected", + tofile="actual", + lineterm="", + )) + msg = "{}\n" "\033[31m" "Diff:" "\033[0m" "\n{}".format(msg, diff) + except Exception: + pass + raise AssertionError(msg) + +def assert_in(expected, actual): + if expected not in actual: + msg = "\033[32m" "Expected" "\033[31m" " not in Actual" "\033[0m" "\n" \ + "\033[32m" "Expected:" "\033[0m" " {}\n" \ + "\033[31m" "Actual:" "\033[0m" " {}".format(expected, actual) + raise AssertionError(msg) |