diff options
Diffstat (limited to 'debian/tests/ansible-test-integration.py')
| -rwxr-xr-x | debian/tests/ansible-test-integration.py | 309 |
1 files changed, 309 insertions, 0 deletions
diff --git a/debian/tests/ansible-test-integration.py b/debian/tests/ansible-test-integration.py new file mode 100755 index 0000000..c9b657f --- /dev/null +++ b/debian/tests/ansible-test-integration.py @@ -0,0 +1,309 @@ +#!/usr/bin/python3 + +import argparse +import glob +import os +import subprocess +import sys + +# helper function to print log messages depending on verbosity +def log (level, *msg): + if args.verbose >= level: + print(' '.join([str(x) for x in msg]), flush=True) + +sys.dont_write_bytecode = True + +# helper function to run and log processes +def runprog (name, progargs, exit_on_failure=True): + log(1, 'Running', name) + proc = subprocess.run( + progargs, + timeout = 300, + capture_output = True, + text = True, + ) + if proc.returncode != 0: + log(0,"#"*72) + log(0,'Running', name, 'failed!') + log(0,"Returncode:", proc.returncode) + log(1,"#### STDOUT ####") + log(1, proc.stdout) + log(1, "#### STDERR ####") + log(1, proc.stderr) + log(0,"#"*72) + if exit_on_failure == True: + exit(proc.returncode) + return proc + +def locale_debug(name): + if args.verbose >= 3: + log(2, name) + log(2, 'output of /usr/bin/locale:') + subprocess.run(['/usr/bin/locale']) + prog = runprog('cat /etc/default/locale', ['cat', '/etc/default/locale']) + log(2, prog.stdout, prog.stderr) + # grep will exit 1 when it doesn't return anything, so don't fail on it. + prog = runprog('grep -v -P \'^#|^$\' /etc/locale.gen', ['grep', '-v', '-P', '^#|^$', '/etc/locale.gen'], exit_on_failure=False) + log(2, prog.stdout, prog.stderr) + +parser = argparse.ArgumentParser( + prog='ansible-test-integration.py', + description='python script to run ansible-test integration against the Debian source package', +) + +# Whether to run the default tests not in any other list +parser.add_argument( + '--default-tests', + action=argparse.BooleanOptionalAction, + default=True, + help='Run the default tests not listed anywhere else. (default: yes)', +) + +parser.add_argument( + '--requires-root', + action=argparse.BooleanOptionalAction, + default=False, + help='Run tests that require root. (default: no)', +) + +parser.add_argument( + '--requires-ssh', + action=argparse.BooleanOptionalAction, + default=False, + help='Run tests that require a specially configured SSH server. (default: no)' +) + +parser.add_argument( + '--requires-apt-mark-manual', + action=argparse.BooleanOptionalAction, + default=True, + help='Run tests that do "apt-mark manual" on certain packages. (default: yes)', +) + +parser.add_argument( + '--fails-on-pip', + action=argparse.BooleanOptionalAction, + default=False, + help='Run tests that run "pip3 install" on certain modules. (default: no)', +) + +parser.add_argument( + '--failing', + action=argparse.BooleanOptionalAction, + default=False, + help='Run tests that fail on other reasons. (default: no)', +) + +parser.add_argument( + '--setup', + action=argparse.BooleanOptionalAction, + default=True, + help='Setup testbed via sudo. (default: yes)', +) + +parser.add_argument( + '--dry-run', + action=argparse.BooleanOptionalAction, + default=False, + help='Print the list of targets without actually running them', +) + +parser.add_argument( + '--verbose', '-v', + action='count', + default=1, + help='verbosity between 0 and 5. 0 will only emit errors. 1=INFO. More for increasing levels of debug info. Defaults to 1.', +) + +args = parser.parse_args() + +locale_debug('locale before setting os.environ:') +os.environ['LANG'] = 'en_US.UTF-8' +locale_debug('locale after setting os.environ:') + +if args.setup == True: + proc = runprog('testbed-setup.sh', ['sudo', './debian/tests/testbed-setup.sh']) + log(2,"#### STDOUT ####") + log(2, proc.stdout) + log(2, "#### STDERR ####") + log(2, proc.stderr) + locale_debug('locale after running testbed-setup.sh:') + +# integration tests requiring root in some form +integration_requires_root = { + 'ansible-vault', # ignores --local and tries to use venv + 'apt_key', # add/removes apt keys + 'blockinfile', # setup_remote_tmp_dir handler fails (hidden output) + 'callback_default', # checks for an error that has root's homedir + 'debconf', # Writes to debconf database + 'gathering', # writes to /etc/ansible/facts.d/ + 'group', # wants to add/remove systemd groups + 'keyword_inheritance', # requires sudo + #'module_defaults', # requires sudo + 'noexec', # calls mount + #'omit', # requires sudo + 'systemd', # disables/enables services +} + +# integration tests requiring a running ssh server +integration_requires_ssh = { + 'become_unprivileged', + 'cli', + 'connection_paramiko_ssh', + 'connection_ssh', + 'delegate_to', + 'fetch', + 'module_tracebacks', +} + +# integration tests requiring root because the apt module is used to +# install missing packages, or to mark packages as manually installed +integration_requires_apt_mark_manual = { + 'ansible-galaxy-collection-scm', # apt-mark manual git + 'ansible-pull', # apt-mark manual git + #'debconf', # apt-mark manual debconf-utils + 'iptables', # apt-mark manual iptables + 'git', # apt-mark manual git +} + +integration_fails_on_pip = { + 'ansible-galaxy-collection-cli', # fails on pip + 'ansible-inventory', # pip error: externally-managed-environment ## upstream fix + 'builtin_vars_prompt', # passlib: pip error: externally-managed-environment + 'debugger', # pip installs pexpect + 'pause', # pip installs pexpect +} + +integration_failing = { + 'ansible-galaxy-role': 'dict object has no attribute lnk_source', ## needs upstream fix? + 'ansible-test-docker': "pwsh doesn't exist in Debian yet", + 'ansible-test': 'installs and runs python libs from remote', + 'ansible-test-sanity': 'checks are only valid for the source tree', + 'ansible-test-units-forked': '?????', + 'facts_d': 'seems to read an unreadable problem without error', ## needs upstream fix + 'infra': 'requires hacking/test-module.py not present', + 'interpreter_discovery_python': 'detects /usr/bin/python3.11, expect python3, detects os_version 12.6, expects it to compare > 10', +# 'preflight_encoding': 'fails due to missing en_US.UTF-8', # workaround in testbed-setup.sh + 'remote_tmp': 'Will often show false positive on: "Test tempdir is removed", needs upstream fixing', + 'service_facts': "Version comparison failed: '<' not supported between instances of 'str' and 'int'", # writes to /usr/sbin/ +# 'tags': 'fails due to missing en_US.UTF-8', # workaround in testbed-setup.sh + 'template_jinja2_non_native': 'no need to test against latest jinja2', +} + +# work around autopkgtest providing the source tree owned by a different user +runprog('git config hack', ['git', 'config', '--global', '--add', 'safe.directory', '*']) + +pyver = str(sys.version_info.major) + '.' + str(sys.version_info.minor) + +overall_test_rc = 0 +failed_tests = [] +succeeded_tests = [] + +# retrieve a list of all integration tests +all_targets_cmd = runprog( + 'ansible-test to retrieve list of targets', + ['./bin/ansible-test', 'integration', '--list-targets'], +) + +# Compile list of all targets +all_targets = set(all_targets_cmd.stdout.splitlines()) + +default_targets = list( + all_targets + - integration_requires_root + - integration_requires_ssh + - integration_requires_apt_mark_manual + - integration_fails_on_pip + - set(integration_failing.keys()) +) + +# compile a list of targets to run, depending on CLI parameters +targets = [] +skipped = [] + +if args.default_tests == True: + targets.extend(default_targets) +else: + skipped.extend(default_targets) + +if args.requires_root == True: + targets.extend(integration_requires_root) +else: + skipped.extend(integration_requires_root) + +if args.requires_ssh == True: + targets.extend(integration_requires_ssh) +else: + skipped.extend(integration_requires_ssh) + +if args.requires_apt_mark_manual == True: + targets.extend(integration_requires_apt_mark_manual) +else: + skipped.extend(integration_requires_apt_mark_manual) + +if args.fails_on_pip == True: + targets.extend(integration_fails_on_pip) +else: + skipped.extend(integration_fails_on_pip) + +if args.failing == True: + targets.extend(integration_failing) +else: + skipped.extend(integration_failing) + +targets.sort() +skipped.sort() + +for i in targets: + + if args.dry_run == True: + log(1, 'Would run ansible-test in', i) + skipped.append(i) + continue + + print ("\n" + "#"*72, flush=True) + print ("#### Running integration tests in", i, flush=True) + print ("#"*72, flush=True) + + proc = subprocess.run([ + './bin/ansible-test', + 'integration', + '--python-interpreter', + '/usr/bin/python3', + '--python', + pyver, + '--local', + '--color', 'yes', + i + ]) + + + if proc.returncode != 0: + failed_tests.append(i) + overall_test_rc = proc.returncode + else: + succeeded_tests.append(i) + +if overall_test_rc != 0: + print ("#"*72, flush=True) + print ("#### failed tests are:", flush=True) + for i in failed_tests: + print ("####", i, flush=True) + print ("#"*72, flush=True) + +log(1, '### TEST SUMMARY ###') +log(1, 'succeeded tests:', len(succeeded_tests)) +log(1, 'failed tests:', len(failed_tests)) +log(1, 'skipped tests:', len(skipped)) + +log(2, '### succeed test list:') +for i in succeeded_tests: + log(2, i) +log(2, '### failed test list:') +for i in failed_tests: + log(2, i) +log(2, '### skipped test list:') +for i in skipped: + log(2, i) + +exit(overall_test_rc) |