Merging debian version 2.17.2-1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

4 files changed, 434 insertions, 0 deletions

diff --git a/debian/changelog b/debian/changelog
index b355a36..2afdf64 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+ansible-core (2.17.2-1) unstable; urgency=medium
+
+  * New upstream version 2.17.2
+  * Add integration tests to autopkgtest
+
+ -- Lee Garrett <debian@rocketjump.eu>  Fri, 26 Jul 2024 02:07:36 +0200
+
 ansible-core (2.17.1-1~progress7.99u1) graograman-backports; urgency=medium
 
   * Uploading to graograman-backports, remaining changes:
diff --git a/debian/tests/ansible-test-integration.py b/debian/tests/ansible-test-integration.py
new file mode 100755
index 0000000..c9b657f
--- /dev/null
+++ b/debian/tests/ansible-test-integration.py
@@ -0,0 +1,309 @@
+#!/usr/bin/python3
+
+import argparse
+import glob
+import os
+import subprocess
+import sys
+
+# helper function to print log messages depending on verbosity
+def log (level, *msg):
+    if args.verbose >= level:
+        print(' '.join([str(x) for x in msg]), flush=True)
+
+sys.dont_write_bytecode = True
+
+# helper function to run and log processes
+def runprog (name, progargs, exit_on_failure=True):
+    log(1, 'Running', name)
+    proc = subprocess.run(
+        progargs,
+        timeout = 300,
+        capture_output = True,
+        text = True,
+    )
+    if proc.returncode != 0:
+        log(0,"#"*72)
+        log(0,'Running', name, 'failed!')
+        log(0,"Returncode:", proc.returncode)
+        log(1,"#### STDOUT ####")
+        log(1, proc.stdout)
+        log(1, "#### STDERR ####")
+        log(1, proc.stderr)
+        log(0,"#"*72)
+        if exit_on_failure == True:
+            exit(proc.returncode)
+    return proc
+
+def locale_debug(name):
+    if args.verbose >= 3:
+        log(2, name)
+        log(2, 'output of /usr/bin/locale:')
+        subprocess.run(['/usr/bin/locale'])
+        prog = runprog('cat /etc/default/locale', ['cat', '/etc/default/locale'])
+        log(2, prog.stdout, prog.stderr)
+        # grep will exit 1 when it doesn't return anything, so don't fail on it.
+        prog = runprog('grep -v -P \'^#|^$\' /etc/locale.gen', ['grep', '-v', '-P', '^#|^$', '/etc/locale.gen'], exit_on_failure=False)
+        log(2, prog.stdout, prog.stderr)
+
+parser = argparse.ArgumentParser(
+    prog='ansible-test-integration.py',
+    description='python script to run ansible-test integration against the Debian source package',
+)
+
+# Whether to run the default tests not in any other list
+parser.add_argument(
+    '--default-tests',
+    action=argparse.BooleanOptionalAction,
+    default=True,
+    help='Run the default tests not listed anywhere else. (default: yes)',
+)
+
+parser.add_argument(
+    '--requires-root',
+    action=argparse.BooleanOptionalAction,
+    default=False,
+    help='Run tests that require root. (default: no)',
+)
+
+parser.add_argument(
+    '--requires-ssh',
+    action=argparse.BooleanOptionalAction,
+    default=False,
+    help='Run tests that require a specially configured SSH server. (default: no)'
+)
+
+parser.add_argument(
+    '--requires-apt-mark-manual',
+    action=argparse.BooleanOptionalAction,
+    default=True,
+    help='Run tests that do "apt-mark manual" on certain packages. (default: yes)',
+)
+
+parser.add_argument(
+    '--fails-on-pip',
+    action=argparse.BooleanOptionalAction,
+    default=False,
+    help='Run tests that run "pip3 install" on certain modules. (default: no)',
+)
+
+parser.add_argument(
+    '--failing',
+    action=argparse.BooleanOptionalAction,
+    default=False,
+    help='Run tests that fail on other reasons. (default: no)',
+)
+
+parser.add_argument(
+    '--setup',
+    action=argparse.BooleanOptionalAction,
+    default=True,
+    help='Setup testbed via sudo. (default: yes)',
+)
+
+parser.add_argument(
+    '--dry-run',
+    action=argparse.BooleanOptionalAction,
+    default=False,
+    help='Print the list of targets without actually running them',
+)
+
+parser.add_argument(
+    '--verbose', '-v',
+    action='count',
+    default=1,
+    help='verbosity between 0 and 5. 0 will only emit errors. 1=INFO. More for increasing levels of debug info. Defaults to 1.',
+)
+
+args = parser.parse_args()
+
+locale_debug('locale before setting os.environ:')
+os.environ['LANG'] = 'en_US.UTF-8'
+locale_debug('locale after setting os.environ:')
+
+if args.setup == True:
+    proc = runprog('testbed-setup.sh', ['sudo', './debian/tests/testbed-setup.sh'])
+    log(2,"#### STDOUT ####")
+    log(2, proc.stdout)
+    log(2, "#### STDERR ####")
+    log(2, proc.stderr)
+    locale_debug('locale after running testbed-setup.sh:')
+
+# integration tests requiring root in some form
+integration_requires_root = {
+    'ansible-vault',    # ignores --local and tries to use venv
+    'apt_key',          # add/removes apt keys
+    'blockinfile',      # setup_remote_tmp_dir handler fails (hidden output)
+    'callback_default', # checks for an error that has root's homedir
+    'debconf',          # Writes to debconf database
+    'gathering',        # writes to /etc/ansible/facts.d/
+    'group',            # wants to add/remove systemd groups
+    'keyword_inheritance', # requires sudo
+    #'module_defaults',  # requires sudo
+    'noexec',           # calls mount
+    #'omit',             # requires sudo
+    'systemd',          # disables/enables services
+}
+
+# integration tests requiring a running ssh server
+integration_requires_ssh = {
+    'become_unprivileged',
+    'cli',
+    'connection_paramiko_ssh',
+    'connection_ssh',
+    'delegate_to',
+    'fetch',
+    'module_tracebacks',
+}
+
+# integration tests requiring root because the apt module is used to
+# install missing packages, or to mark packages as manually installed
+integration_requires_apt_mark_manual = {
+    'ansible-galaxy-collection-scm',   # apt-mark manual git
+    'ansible-pull',                    # apt-mark manual git
+    #'debconf',                         # apt-mark manual debconf-utils
+    'iptables',                        # apt-mark manual iptables
+    'git',                             # apt-mark manual git
+}
+
+integration_fails_on_pip = {
+    'ansible-galaxy-collection-cli',   # fails on pip
+    'ansible-inventory',               # pip error: externally-managed-environment ## upstream fix
+    'builtin_vars_prompt',             # passlib: pip error: externally-managed-environment
+    'debugger',                        # pip installs pexpect
+    'pause',                           # pip installs pexpect
+}
+
+integration_failing = {
+    'ansible-galaxy-role':            'dict object has no attribute lnk_source',  ## needs upstream fix?
+    'ansible-test-docker':            "pwsh doesn't exist in Debian yet",
+    'ansible-test':                   'installs and runs python libs from remote',
+    'ansible-test-sanity':            'checks are only valid for the source tree',
+    'ansible-test-units-forked':      '?????',
+    'facts_d':                        'seems to read an unreadable problem without error', ## needs upstream fix
+    'infra':                          'requires hacking/test-module.py not present',
+    'interpreter_discovery_python':   'detects /usr/bin/python3.11, expect python3, detects os_version 12.6, expects it to compare > 10',
+#    'preflight_encoding':             'fails due to missing en_US.UTF-8', # workaround in testbed-setup.sh
+    'remote_tmp':                     'Will often show false positive on: "Test tempdir is removed", needs upstream fixing',
+    'service_facts':                  "Version comparison failed: '<' not supported between instances of 'str' and 'int'",    # writes to /usr/sbin/
+#    'tags':                           'fails due to missing en_US.UTF-8', # workaround in testbed-setup.sh
+    'template_jinja2_non_native':     'no need to test against latest jinja2',
+}
+
+# work around autopkgtest providing the source tree owned by a different user
+runprog('git config hack', ['git', 'config', '--global', '--add', 'safe.directory', '*'])
+
+pyver = str(sys.version_info.major) + '.' + str(sys.version_info.minor)
+
+overall_test_rc = 0
+failed_tests = []
+succeeded_tests = []
+
+# retrieve a list of all integration tests
+all_targets_cmd = runprog(
+    'ansible-test to retrieve list of targets',
+    ['./bin/ansible-test', 'integration', '--list-targets'],
+)
+
+# Compile list of all targets
+all_targets = set(all_targets_cmd.stdout.splitlines())
+
+default_targets = list(
+    all_targets
+    - integration_requires_root
+    - integration_requires_ssh
+    - integration_requires_apt_mark_manual
+    - integration_fails_on_pip
+    - set(integration_failing.keys())
+)
+
+# compile a list of targets to run, depending on CLI parameters
+targets = []
+skipped = []
+
+if args.default_tests == True:
+    targets.extend(default_targets)
+else:
+    skipped.extend(default_targets)
+
+if args.requires_root == True:
+    targets.extend(integration_requires_root)
+else:
+    skipped.extend(integration_requires_root)
+
+if args.requires_ssh == True:
+    targets.extend(integration_requires_ssh)
+else:
+    skipped.extend(integration_requires_ssh)
+
+if args.requires_apt_mark_manual == True:
+    targets.extend(integration_requires_apt_mark_manual)
+else:
+    skipped.extend(integration_requires_apt_mark_manual)
+
+if args.fails_on_pip == True:
+    targets.extend(integration_fails_on_pip)
+else:
+    skipped.extend(integration_fails_on_pip)
+
+if args.failing == True:
+    targets.extend(integration_failing)
+else:
+    skipped.extend(integration_failing)
+
+targets.sort()
+skipped.sort()
+
+for i in targets:
+
+    if args.dry_run == True:
+        log(1, 'Would run ansible-test in', i)
+        skipped.append(i)
+        continue
+
+    print ("\n" + "#"*72, flush=True)
+    print ("#### Running integration tests in", i, flush=True)
+    print ("#"*72, flush=True)
+
+    proc = subprocess.run([
+        './bin/ansible-test',
+        'integration',
+        '--python-interpreter',
+        '/usr/bin/python3',
+        '--python',
+        pyver,
+        '--local',
+        '--color', 'yes',
+        i
+    ])
+
+
+    if proc.returncode != 0:
+        failed_tests.append(i)
+        overall_test_rc = proc.returncode
+    else:
+        succeeded_tests.append(i)
+
+if overall_test_rc != 0:
+    print ("#"*72, flush=True)
+    print ("#### failed tests are:", flush=True)
+    for i in failed_tests:
+        print ("####", i, flush=True)
+    print ("#"*72, flush=True)
+
+log(1, '### TEST SUMMARY ###')
+log(1, 'succeeded tests:', len(succeeded_tests))
+log(1, 'failed tests:', len(failed_tests))
+log(1, 'skipped tests:', len(skipped))
+
+log(2, '### succeed test list:')
+for i in succeeded_tests:
+    log(2, i)
+log(2, '### failed test list:')
+for i in failed_tests:
+    log(2, i)
+log(2, '### skipped test list:')
+for i in skipped:
+    log(2, i)
+
+exit(overall_test_rc)
diff --git a/debian/tests/control b/debian/tests/control
index a07ba12..d9f0582 100644
--- a/debian/tests/control
+++ b/debian/tests/control
@@ -16,3 +16,90 @@ Depends: @,
          python3-tz,
          python3-winrm,
          python3-yaml
+
+Tests: ansible-test-integration.py
+Depends: @,
+         #bsdextrautils,
+         debconf-utils,             # debconf
+         file,                      # stat
+         git,                       # used by most tests
+         #gnupg,
+         gzip,                      # git
+         iptables,                  # iptables
+         #iproute2,
+         locales,
+         #procps,
+         #python3-apt,
+         #python3-cryptography,
+         python3-distlib,           # ansible-galaxy-collection-cli
+         #python3-jinja2,
+         #python3-junit.xml,
+         #python3-mock,
+         python3-passlib,            # grep 'setup/always/setup_passlib' test/integration/targets/*/aliases
+         python3-pexpect,            # grep 'setup/always/setup_pexpect' test/integration/targets/*/aliases
+         python3-pip,               # ansible-test-config, builtin_vars_prompt, remove again and fix tests
+         #python3-pycryptodome,
+         python3-pytest,            # ansible-test-units-assertions
+         python3-pytest-mock,       # ansible-test-units-assertions
+         #python3-pytest-xdist,
+         #python3-systemd,
+         #python3-tz,
+         python3-venv,              # ansible-inventory; ./test/lib/ansible_test/_util/target/injector/virtualenv.sh
+         python3-virtualenv,        # ansible-inventory, ansible-test-installed; ./test/lib/ansible_test/_util/target/injector/virtualenv.sh
+         #python3-winrm,
+         python3-yaml,              # ansible-test-units-forked
+         #pwsh,                      # ansible-test-sanity-validate-modules, see RFP bug https://bugs.debian.org/834756,
+         #rsync,
+         shellcheck,
+         sudo,                      # needed for testbed-setup.sh
+         tar,                       # integration test git
+         unzip,                     # integration test git
+         zip,                       # integration test git
+Restrictions:
+  allow-stderr,     # lots of STDERR output
+  needs-internet,   # used for ansible-galaxy and pip tests
+  needs-sudo,       # used to call testbed-setup.sh
+
+Test-Command: ./debian/tests/ansible-test-integration.py --no-default-tests --requires-root --no-requires-apt-mark-manual
+Features: test-name=ansible-test-integration-flaky
+Depends: @,
+         #bsdextrautils,
+         debconf-utils,             # debconf
+         file,                      # stat
+         git,                       # used by most tests
+         gnupg,                     # integration test apt-key
+         gzip,                      # git
+         iptables,                  # iptables
+         #iproute2,
+         locales,
+         #procps,
+         #python3-apt,
+         #python3-cryptography,
+         python3-distlib,           # ansible-galaxy-collection-cli
+         #python3-jinja2,
+         #python3-junit.xml,
+         #python3-mock,
+         python3-passlib,            # grep 'setup/always/setup_passlib' test/integration/targets/*/aliases
+         python3-pexpect,            # grep 'setup/always/setup_pexpect' test/integration/targets/*/aliases
+         python3-pip,               # ansible-test-config, builtin_vars_prompt, remove again and fix tests
+         #python3-pycryptodome,
+         python3-pytest,            # ansible-test-units-assertions
+         python3-pytest-mock,       # ansible-test-units-assertions
+         #python3-pytest-xdist,
+         #python3-systemd,
+         #python3-tz,
+         python3-venv,              # ansible-inventory; ./test/lib/ansible_test/_util/target/injector/virtualenv.sh
+         python3-virtualenv,        # ansible-inventory, ansible-test-installed; ./test/lib/ansible_test/_util/target/injector/virtualenv.sh
+         #python3-winrm,
+         python3-yaml,              # ansible-test-units-forked
+         #pwsh,                      # ansible-test-sanity-validate-modules, see RFP bug https://bugs.debian.org/834756,
+         #rsync,
+         shellcheck,
+         sudo,                      # needed for testbed-setup.sh
+         tar,                       # integration test git
+         unzip,                     # integration test git
+         zip,                       # integration test git
+Restrictions:
+  allow-stderr,     # lots of STDERR output
+  needs-internet,   # used for ansible-galaxy and pip tests
+  needs-root,       # we're running tests requiring root
diff --git a/debian/tests/testbed-setup.sh b/debian/tests/testbed-setup.sh
new file mode 100755
index 0000000..cbc119b
--- /dev/null
+++ b/debian/tests/testbed-setup.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/sh
+
+# helper script called from within ansible-test-integration.py to
+
+set -eu
+
+# workaround for various integration tests trying to do this and failing
+# due to missing root permission
+echo 'apt-mark changes'
+/usr/bin/apt-mark manual debconf-utils
+/usr/bin/apt-mark manual iptables
+/usr/bin/apt-mark manual git
+
+# Set default locale to en_US.UTF-8 to remove lots of warnings when
+# running the tests
+echo 'debconf selection changes'
+# These values get overwritten by what is in the config files when
+# running dpkg-reconfigure, see  https://bugs.debian.org/684134
+cat << EOF | debconf-set-selections
+locales	locales/default_environment_locale	select	en_US.UTF-8
+locales	locales/locales_to_be_generated	multiselect	en_US.UTF-8 UTF-8
+EOF
+
+# So set these values manually in the config files.
+echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen
+echo 'LANG=en_US.UTF-8' > /etc/locale.conf
+echo 'LANG=en_US.UTF-8' > /etc/default/locale
+
+echo 'dpkg-reconfigure changes'
+dpkg-reconfigure --frontend=noninteractive locales
+