diff options
Diffstat (limited to '')
| -rw-r--r-- | debian/changelog | 7 | ||||
| -rwxr-xr-x | debian/tests/ansible-test-integration.py | 309 | ||||
| -rw-r--r-- | debian/tests/control | 87 | ||||
| -rwxr-xr-x | debian/tests/testbed-setup.sh | 31 |
4 files changed, 434 insertions, 0 deletions
diff --git a/debian/changelog b/debian/changelog index b355a36..2afdf64 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +ansible-core (2.17.2-1) unstable; urgency=medium + + * New upstream version 2.17.2 + * Add integration tests to autopkgtest + + -- Lee Garrett <debian@rocketjump.eu> Fri, 26 Jul 2024 02:07:36 +0200 + ansible-core (2.17.1-1~progress7.99u1) graograman-backports; urgency=medium * Uploading to graograman-backports, remaining changes: diff --git a/debian/tests/ansible-test-integration.py b/debian/tests/ansible-test-integration.py new file mode 100755 index 0000000..c9b657f --- /dev/null +++ b/debian/tests/ansible-test-integration.py @@ -0,0 +1,309 @@ +#!/usr/bin/python3 + +import argparse +import glob +import os +import subprocess +import sys + +# helper function to print log messages depending on verbosity +def log (level, *msg): + if args.verbose >= level: + print(' '.join([str(x) for x in msg]), flush=True) + +sys.dont_write_bytecode = True + +# helper function to run and log processes +def runprog (name, progargs, exit_on_failure=True): + log(1, 'Running', name) + proc = subprocess.run( + progargs, + timeout = 300, + capture_output = True, + text = True, + ) + if proc.returncode != 0: + log(0,"#"*72) + log(0,'Running', name, 'failed!') + log(0,"Returncode:", proc.returncode) + log(1,"#### STDOUT ####") + log(1, proc.stdout) + log(1, "#### STDERR ####") + log(1, proc.stderr) + log(0,"#"*72) + if exit_on_failure == True: + exit(proc.returncode) + return proc + +def locale_debug(name): + if args.verbose >= 3: + log(2, name) + log(2, 'output of /usr/bin/locale:') + subprocess.run(['/usr/bin/locale']) + prog = runprog('cat /etc/default/locale', ['cat', '/etc/default/locale']) + log(2, prog.stdout, prog.stderr) + # grep will exit 1 when it doesn't return anything, so don't fail on it. + prog = runprog('grep -v -P \'^#|^$\' /etc/locale.gen', ['grep', '-v', '-P', '^#|^$', '/etc/locale.gen'], exit_on_failure=False) + log(2, prog.stdout, prog.stderr) + +parser = argparse.ArgumentParser( + prog='ansible-test-integration.py', + description='python script to run ansible-test integration against the Debian source package', +) + +# Whether to run the default tests not in any other list +parser.add_argument( + '--default-tests', + action=argparse.BooleanOptionalAction, + default=True, + help='Run the default tests not listed anywhere else. (default: yes)', +) + +parser.add_argument( + '--requires-root', + action=argparse.BooleanOptionalAction, + default=False, + help='Run tests that require root. (default: no)', +) + +parser.add_argument( + '--requires-ssh', + action=argparse.BooleanOptionalAction, + default=False, + help='Run tests that require a specially configured SSH server. (default: no)' +) + +parser.add_argument( + '--requires-apt-mark-manual', + action=argparse.BooleanOptionalAction, + default=True, + help='Run tests that do "apt-mark manual" on certain packages. (default: yes)', +) + +parser.add_argument( + '--fails-on-pip', + action=argparse.BooleanOptionalAction, + default=False, + help='Run tests that run "pip3 install" on certain modules. (default: no)', +) + +parser.add_argument( + '--failing', + action=argparse.BooleanOptionalAction, + default=False, + help='Run tests that fail on other reasons. (default: no)', +) + +parser.add_argument( + '--setup', + action=argparse.BooleanOptionalAction, + default=True, + help='Setup testbed via sudo. (default: yes)', +) + +parser.add_argument( + '--dry-run', + action=argparse.BooleanOptionalAction, + default=False, + help='Print the list of targets without actually running them', +) + +parser.add_argument( + '--verbose', '-v', + action='count', + default=1, + help='verbosity between 0 and 5. 0 will only emit errors. 1=INFO. More for increasing levels of debug info. Defaults to 1.', +) + +args = parser.parse_args() + +locale_debug('locale before setting os.environ:') +os.environ['LANG'] = 'en_US.UTF-8' +locale_debug('locale after setting os.environ:') + +if args.setup == True: + proc = runprog('testbed-setup.sh', ['sudo', './debian/tests/testbed-setup.sh']) + log(2,"#### STDOUT ####") + log(2, proc.stdout) + log(2, "#### STDERR ####") + log(2, proc.stderr) + locale_debug('locale after running testbed-setup.sh:') + +# integration tests requiring root in some form +integration_requires_root = { + 'ansible-vault', # ignores --local and tries to use venv + 'apt_key', # add/removes apt keys + 'blockinfile', # setup_remote_tmp_dir handler fails (hidden output) + 'callback_default', # checks for an error that has root's homedir + 'debconf', # Writes to debconf database + 'gathering', # writes to /etc/ansible/facts.d/ + 'group', # wants to add/remove systemd groups + 'keyword_inheritance', # requires sudo + #'module_defaults', # requires sudo + 'noexec', # calls mount + #'omit', # requires sudo + 'systemd', # disables/enables services +} + +# integration tests requiring a running ssh server +integration_requires_ssh = { + 'become_unprivileged', + 'cli', + 'connection_paramiko_ssh', + 'connection_ssh', + 'delegate_to', + 'fetch', + 'module_tracebacks', +} + +# integration tests requiring root because the apt module is used to +# install missing packages, or to mark packages as manually installed +integration_requires_apt_mark_manual = { + 'ansible-galaxy-collection-scm', # apt-mark manual git + 'ansible-pull', # apt-mark manual git + #'debconf', # apt-mark manual debconf-utils + 'iptables', # apt-mark manual iptables + 'git', # apt-mark manual git +} + +integration_fails_on_pip = { + 'ansible-galaxy-collection-cli', # fails on pip + 'ansible-inventory', # pip error: externally-managed-environment ## upstream fix + 'builtin_vars_prompt', # passlib: pip error: externally-managed-environment + 'debugger', # pip installs pexpect + 'pause', # pip installs pexpect +} + +integration_failing = { + 'ansible-galaxy-role': 'dict object has no attribute lnk_source', ## needs upstream fix? + 'ansible-test-docker': "pwsh doesn't exist in Debian yet", + 'ansible-test': 'installs and runs python libs from remote', + 'ansible-test-sanity': 'checks are only valid for the source tree', + 'ansible-test-units-forked': '?????', + 'facts_d': 'seems to read an unreadable problem without error', ## needs upstream fix + 'infra': 'requires hacking/test-module.py not present', + 'interpreter_discovery_python': 'detects /usr/bin/python3.11, expect python3, detects os_version 12.6, expects it to compare > 10', +# 'preflight_encoding': 'fails due to missing en_US.UTF-8', # workaround in testbed-setup.sh + 'remote_tmp': 'Will often show false positive on: "Test tempdir is removed", needs upstream fixing', + 'service_facts': "Version comparison failed: '<' not supported between instances of 'str' and 'int'", # writes to /usr/sbin/ +# 'tags': 'fails due to missing en_US.UTF-8', # workaround in testbed-setup.sh + 'template_jinja2_non_native': 'no need to test against latest jinja2', +} + +# work around autopkgtest providing the source tree owned by a different user +runprog('git config hack', ['git', 'config', '--global', '--add', 'safe.directory', '*']) + +pyver = str(sys.version_info.major) + '.' + str(sys.version_info.minor) + +overall_test_rc = 0 +failed_tests = [] +succeeded_tests = [] + +# retrieve a list of all integration tests +all_targets_cmd = runprog( + 'ansible-test to retrieve list of targets', + ['./bin/ansible-test', 'integration', '--list-targets'], +) + +# Compile list of all targets +all_targets = set(all_targets_cmd.stdout.splitlines()) + +default_targets = list( + all_targets + - integration_requires_root + - integration_requires_ssh + - integration_requires_apt_mark_manual + - integration_fails_on_pip + - set(integration_failing.keys()) +) + +# compile a list of targets to run, depending on CLI parameters +targets = [] +skipped = [] + +if args.default_tests == True: + targets.extend(default_targets) +else: + skipped.extend(default_targets) + +if args.requires_root == True: + targets.extend(integration_requires_root) +else: + skipped.extend(integration_requires_root) + +if args.requires_ssh == True: + targets.extend(integration_requires_ssh) +else: + skipped.extend(integration_requires_ssh) + +if args.requires_apt_mark_manual == True: + targets.extend(integration_requires_apt_mark_manual) +else: + skipped.extend(integration_requires_apt_mark_manual) + +if args.fails_on_pip == True: + targets.extend(integration_fails_on_pip) +else: + skipped.extend(integration_fails_on_pip) + +if args.failing == True: + targets.extend(integration_failing) +else: + skipped.extend(integration_failing) + +targets.sort() +skipped.sort() + +for i in targets: + + if args.dry_run == True: + log(1, 'Would run ansible-test in', i) + skipped.append(i) + continue + + print ("\n" + "#"*72, flush=True) + print ("#### Running integration tests in", i, flush=True) + print ("#"*72, flush=True) + + proc = subprocess.run([ + './bin/ansible-test', + 'integration', + '--python-interpreter', + '/usr/bin/python3', + '--python', + pyver, + '--local', + '--color', 'yes', + i + ]) + + + if proc.returncode != 0: + failed_tests.append(i) + overall_test_rc = proc.returncode + else: + succeeded_tests.append(i) + +if overall_test_rc != 0: + print ("#"*72, flush=True) + print ("#### failed tests are:", flush=True) + for i in failed_tests: + print ("####", i, flush=True) + print ("#"*72, flush=True) + +log(1, '### TEST SUMMARY ###') +log(1, 'succeeded tests:', len(succeeded_tests)) +log(1, 'failed tests:', len(failed_tests)) +log(1, 'skipped tests:', len(skipped)) + +log(2, '### succeed test list:') +for i in succeeded_tests: + log(2, i) +log(2, '### failed test list:') +for i in failed_tests: + log(2, i) +log(2, '### skipped test list:') +for i in skipped: + log(2, i) + +exit(overall_test_rc) diff --git a/debian/tests/control b/debian/tests/control index a07ba12..d9f0582 100644 --- a/debian/tests/control +++ b/debian/tests/control @@ -16,3 +16,90 @@ Depends: @, python3-tz, python3-winrm, python3-yaml + +Tests: ansible-test-integration.py +Depends: @, + #bsdextrautils, + debconf-utils, # debconf + file, # stat + git, # used by most tests + #gnupg, + gzip, # git + iptables, # iptables + #iproute2, + locales, + #procps, + #python3-apt, + #python3-cryptography, + python3-distlib, # ansible-galaxy-collection-cli + #python3-jinja2, + #python3-junit.xml, + #python3-mock, + python3-passlib, # grep 'setup/always/setup_passlib' test/integration/targets/*/aliases + python3-pexpect, # grep 'setup/always/setup_pexpect' test/integration/targets/*/aliases + python3-pip, # ansible-test-config, builtin_vars_prompt, remove again and fix tests + #python3-pycryptodome, + python3-pytest, # ansible-test-units-assertions + python3-pytest-mock, # ansible-test-units-assertions + #python3-pytest-xdist, + #python3-systemd, + #python3-tz, + python3-venv, # ansible-inventory; ./test/lib/ansible_test/_util/target/injector/virtualenv.sh + python3-virtualenv, # ansible-inventory, ansible-test-installed; ./test/lib/ansible_test/_util/target/injector/virtualenv.sh + #python3-winrm, + python3-yaml, # ansible-test-units-forked + #pwsh, # ansible-test-sanity-validate-modules, see RFP bug https://bugs.debian.org/834756, + #rsync, + shellcheck, + sudo, # needed for testbed-setup.sh + tar, # integration test git + unzip, # integration test git + zip, # integration test git +Restrictions: + allow-stderr, # lots of STDERR output + needs-internet, # used for ansible-galaxy and pip tests + needs-sudo, # used to call testbed-setup.sh + +Test-Command: ./debian/tests/ansible-test-integration.py --no-default-tests --requires-root --no-requires-apt-mark-manual +Features: test-name=ansible-test-integration-flaky +Depends: @, + #bsdextrautils, + debconf-utils, # debconf + file, # stat + git, # used by most tests + gnupg, # integration test apt-key + gzip, # git + iptables, # iptables + #iproute2, + locales, + #procps, + #python3-apt, + #python3-cryptography, + python3-distlib, # ansible-galaxy-collection-cli + #python3-jinja2, + #python3-junit.xml, + #python3-mock, + python3-passlib, # grep 'setup/always/setup_passlib' test/integration/targets/*/aliases + python3-pexpect, # grep 'setup/always/setup_pexpect' test/integration/targets/*/aliases + python3-pip, # ansible-test-config, builtin_vars_prompt, remove again and fix tests + #python3-pycryptodome, + python3-pytest, # ansible-test-units-assertions + python3-pytest-mock, # ansible-test-units-assertions + #python3-pytest-xdist, + #python3-systemd, + #python3-tz, + python3-venv, # ansible-inventory; ./test/lib/ansible_test/_util/target/injector/virtualenv.sh + python3-virtualenv, # ansible-inventory, ansible-test-installed; ./test/lib/ansible_test/_util/target/injector/virtualenv.sh + #python3-winrm, + python3-yaml, # ansible-test-units-forked + #pwsh, # ansible-test-sanity-validate-modules, see RFP bug https://bugs.debian.org/834756, + #rsync, + shellcheck, + sudo, # needed for testbed-setup.sh + tar, # integration test git + unzip, # integration test git + zip, # integration test git +Restrictions: + allow-stderr, # lots of STDERR output + needs-internet, # used for ansible-galaxy and pip tests + needs-root, # we're running tests requiring root diff --git a/debian/tests/testbed-setup.sh b/debian/tests/testbed-setup.sh new file mode 100755 index 0000000..cbc119b --- /dev/null +++ b/debian/tests/testbed-setup.sh @@ -0,0 +1,31 @@ +#!/usr/bin/sh + +# helper script called from within ansible-test-integration.py to + +set -eu + +# workaround for various integration tests trying to do this and failing +# due to missing root permission +echo 'apt-mark changes' +/usr/bin/apt-mark manual debconf-utils +/usr/bin/apt-mark manual iptables +/usr/bin/apt-mark manual git + +# Set default locale to en_US.UTF-8 to remove lots of warnings when +# running the tests +echo 'debconf selection changes' +# These values get overwritten by what is in the config files when +# running dpkg-reconfigure, see https://bugs.debian.org/684134 +cat << EOF | debconf-set-selections +locales locales/default_environment_locale select en_US.UTF-8 +locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8 +EOF + +# So set these values manually in the config files. +echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen +echo 'LANG=en_US.UTF-8' > /etc/locale.conf +echo 'LANG=en_US.UTF-8' > /etc/default/locale + +echo 'dpkg-reconfigure changes' +dpkg-reconfigure --frontend=noninteractive locales + |