diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-05 16:16:49 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-05 16:16:49 +0000 |
| commit | 48e387c5c12026a567eb7b293a3a590241c0cecb (patch) | |
| tree | 80f2573be2d7d534b8ac4d2a852fe43f7ac35324 /test/lib | |
| parent | Releasing progress-linux version 2.16.6-1~progress7.99u1. (diff) | |
| download | ansible-core-48e387c5c12026a567eb7b293a3a590241c0cecb.tar.xz ansible-core-48e387c5c12026a567eb7b293a3a590241c0cecb.zip | |
Merging upstream version 2.17.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'test/lib')
80 files changed, 220 insertions, 956 deletions
diff --git a/test/lib/ansible_test/__init__.py b/test/lib/ansible_test/__init__.py index 527d413..223f9b4 100644 --- a/test/lib/ansible_test/__init__.py +++ b/test/lib/ansible_test/__init__.py @@ -1,2 +1 @@ -# Empty __init__.py to allow importing of `ansible_test._util.target.common` under Python 2.x. -# This allows the ansible-test entry point to report supported Python versions before exiting. +# Empty __init__.py to allow relative imports to work under mypy. diff --git a/test/lib/ansible_test/_data/completion/docker.txt b/test/lib/ansible_test/_data/completion/docker.txt index a863ecb..b8603ec 100644 --- a/test/lib/ansible_test/_data/completion/docker.txt +++ b/test/lib/ansible_test/_data/completion/docker.txt @@ -1,9 +1,7 @@ -base image=quay.io/ansible/base-test-container:5.10.0 python=3.12,2.7,3.6,3.7,3.8,3.9,3.10,3.11 -default image=quay.io/ansible/default-test-container:8.12.0 python=3.12,2.7,3.6,3.7,3.8,3.9,3.10,3.11 context=collection -default image=quay.io/ansible/ansible-core-test-container:8.12.0 python=3.12,2.7,3.6,3.7,3.8,3.9,3.10,3.11 context=ansible-core -alpine3 image=quay.io/ansible/alpine3-test-container:6.3.0 python=3.11 cgroup=none audit=none -centos7 image=quay.io/ansible/centos7-test-container:6.3.0 python=2.7 cgroup=v1-only -fedora38 image=quay.io/ansible/fedora38-test-container:6.3.0 python=3.11 -opensuse15 image=quay.io/ansible/opensuse15-test-container:6.3.0 python=3.6 -ubuntu2004 image=quay.io/ansible/ubuntu2004-test-container:6.3.0 python=3.8 -ubuntu2204 image=quay.io/ansible/ubuntu2204-test-container:6.3.0 python=3.10 +base image=quay.io/ansible/base-test-container:6.2.0 python=3.12,3.7,3.8,3.9,3.10,3.11 +default image=quay.io/ansible/default-test-container:9.6.0 python=3.12,3.7,3.8,3.9,3.10,3.11 context=collection +default image=quay.io/ansible/ansible-core-test-container:9.6.0 python=3.12,3.7,3.8,3.9,3.10,3.11 context=ansible-core +alpine319 image=quay.io/ansible/alpine319-test-container:7.1.0 python=3.11 cgroup=none audit=none +fedora39 image=quay.io/ansible/fedora39-test-container:7.1.0 python=3.12 +ubuntu2004 image=quay.io/ansible/ubuntu2004-test-container:7.1.0 python=3.8 +ubuntu2204 image=quay.io/ansible/ubuntu2204-test-container:7.1.0 python=3.10 diff --git a/test/lib/ansible_test/_data/completion/remote.txt b/test/lib/ansible_test/_data/completion/remote.txt index 06d4b5e..cad7fa4 100644 --- a/test/lib/ansible_test/_data/completion/remote.txt +++ b/test/lib/ansible_test/_data/completion/remote.txt @@ -1,14 +1,13 @@ -alpine/3.18 python=3.11 become=doas_sudo provider=aws arch=x86_64 +alpine/3.19 python=3.11 become=doas_sudo provider=aws arch=x86_64 alpine become=doas_sudo provider=aws arch=x86_64 -fedora/38 python=3.11 become=sudo provider=aws arch=x86_64 +fedora/39 python=3.12 become=sudo provider=aws arch=x86_64 fedora become=sudo provider=aws arch=x86_64 -freebsd/13.2 python=3.9,3.11 python_dir=/usr/local/bin become=su_sudo provider=aws arch=x86_64 +freebsd/13.3 python=3.9,3.11 python_dir=/usr/local/bin become=su_sudo provider=aws arch=x86_64 +freebsd/14.0 python=3.9,3.11 python_dir=/usr/local/bin become=su_sudo provider=aws arch=x86_64 freebsd python_dir=/usr/local/bin become=su_sudo provider=aws arch=x86_64 -macos/13.2 python=3.11 python_dir=/usr/local/bin become=sudo provider=parallels arch=x86_64 +macos/14.3 python=3.11 python_dir=/usr/local/bin become=sudo provider=parallels arch=x86_64 macos python_dir=/usr/local/bin become=sudo provider=parallels arch=x86_64 -rhel/7.9 python=2.7 become=sudo provider=aws arch=x86_64 -rhel/8.8 python=3.6,3.11 become=sudo provider=aws arch=x86_64 -rhel/9.2 python=3.9,3.11 become=sudo provider=aws arch=x86_64 +rhel/9.3 python=3.9,3.11 become=sudo provider=aws arch=x86_64 rhel become=sudo provider=aws arch=x86_64 ubuntu/22.04 python=3.10 become=sudo provider=aws arch=x86_64 ubuntu become=sudo provider=aws arch=x86_64 diff --git a/test/lib/ansible_test/_data/requirements/ansible-test.txt b/test/lib/ansible_test/_data/requirements/ansible-test.txt index 17662f0..e0bb945 100644 --- a/test/lib/ansible_test/_data/requirements/ansible-test.txt +++ b/test/lib/ansible_test/_data/requirements/ansible-test.txt @@ -1,5 +1,3 @@ # The test-constraints sanity test verifies this file, but changes must be made manually to keep it in up-to-date. -virtualenv == 16.7.12 ; python_version < '3' coverage == 7.3.2 ; python_version >= '3.8' and python_version <= '3.12' coverage == 6.5.0 ; python_version >= '3.7' and python_version <= '3.7' -coverage == 4.5.4 ; python_version >= '2.6' and python_version <= '3.6' diff --git a/test/lib/ansible_test/_data/requirements/constraints.txt b/test/lib/ansible_test/_data/requirements/constraints.txt index dd837e3..17f3638 100644 --- a/test/lib/ansible_test/_data/requirements/constraints.txt +++ b/test/lib/ansible_test/_data/requirements/constraints.txt @@ -1,15 +1,11 @@ # do not add a cryptography or pyopenssl constraint to this file, they require special handling, see get_cryptography_requirements in python_requirements.py # do not add a coverage constraint to this file, it is handled internally by ansible-test -packaging < 21.0 ; python_version < '3.6' # packaging 21.0 requires Python 3.6 or newer +pypsrp < 1.0.0 # in case the next major version is too big of a change pywinrm >= 0.3.0 ; python_version < '3.11' # message encryption support pywinrm >= 0.4.3 ; python_version >= '3.11' # support for Python 3.11 -pytest < 5.0.0, >= 4.5.0 ; python_version == '2.7' # pytest 5.0.0 and later will no longer support python 2.7 -pytest >= 4.5.0 ; python_version > '2.7' # pytest 4.5.0 added support for --strict-markers +pytest >= 4.5.0 # pytest 4.5.0 added support for --strict-markers ntlm-auth >= 1.3.0 # message encryption support using cryptography requests-ntlm >= 1.1.0 # message encryption support requests-credssp >= 0.1.0 # message encryption support -pyparsing < 3.0.0 ; python_version < '3.5' # pyparsing 3 and later require python 3.5 or later mock >= 2.0.0 # needed for features backported from Python 3.6 unittest.mock (assert_called, assert_called_once...) pytest-mock >= 1.4.0 # needed for mock_use_standalone_module pytest option -setuptools < 45 ; python_version == '2.7' # setuptools 45 and later require python 3.5 or later -wheel < 0.38.0 ; python_version < '3.7' # wheel 0.38.0 and later require python 3.7 or later diff --git a/test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt b/test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt index 6680145..60ef86f 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt @@ -1,5 +1,5 @@ # edit "sanity.ansible-doc.in" and generate with: hacking/update-sanity-requirements.py --test ansible-doc -Jinja2==3.1.2 -MarkupSafe==2.1.3 -packaging==23.2 +Jinja2==3.1.3 +MarkupSafe==2.1.5 +packaging==24.0 PyYAML==6.0.1 diff --git a/test/lib/ansible_test/_data/requirements/sanity.changelog.txt b/test/lib/ansible_test/_data/requirements/sanity.changelog.txt index d763bad..cc5b635 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.changelog.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.changelog.txt @@ -1,9 +1,9 @@ # edit "sanity.changelog.in" and generate with: hacking/update-sanity-requirements.py --test changelog -antsibull-changelog==0.23.0 +antsibull-changelog==0.26.0 docutils==0.18.1 -packaging==23.2 +packaging==24.0 PyYAML==6.0.1 rstcheck==5.0.0 semantic-version==2.10.0 types-docutils==0.18.3 -typing_extensions==4.8.0 +typing_extensions==4.10.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt b/test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt index 56366b7..9116ed9 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt @@ -1,4 +1,4 @@ # edit "sanity.import.plugin.in" and generate with: hacking/update-sanity-requirements.py --test import.plugin -Jinja2==3.1.2 -MarkupSafe==2.1.3 +Jinja2==3.1.3 +MarkupSafe==2.1.5 PyYAML==6.0.1 diff --git a/test/lib/ansible_test/_data/requirements/sanity.mypy.txt b/test/lib/ansible_test/_data/requirements/sanity.mypy.txt index f6a47fb..651aea8 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.mypy.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.mypy.txt @@ -1,18 +1,18 @@ # edit "sanity.mypy.in" and generate with: hacking/update-sanity-requirements.py --test mypy cffi==1.16.0 -cryptography==41.0.4 -Jinja2==3.1.2 -MarkupSafe==2.1.3 -mypy==1.5.1 +cryptography==42.0.5 +Jinja2==3.1.3 +MarkupSafe==2.1.5 +mypy==1.9.0 mypy-extensions==1.0.0 -packaging==23.2 +packaging==24.0 pycparser==2.21 tomli==2.0.1 types-backports==0.1.3 -types-paramiko==3.3.0.0 -types-PyYAML==6.0.12.12 -types-requests==2.31.0.7 -types-setuptools==68.2.0.0 -types-toml==0.10.8.7 -typing_extensions==4.8.0 -urllib3==2.0.6 +types-paramiko==3.4.0.20240311 +types-PyYAML==6.0.12.20240311 +types-requests==2.31.0.20240311 +types-setuptools==69.2.0.20240317 +types-toml==0.10.8.20240310 +typing_extensions==4.10.0 +urllib3==2.2.1 diff --git a/test/lib/ansible_test/_data/requirements/sanity.pep8.txt b/test/lib/ansible_test/_data/requirements/sanity.pep8.txt index 1a36d4d..51d2b64 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.pep8.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.pep8.txt @@ -1,2 +1,2 @@ # edit "sanity.pep8.in" and generate with: hacking/update-sanity-requirements.py --test pep8 -pycodestyle==2.11.0 +pycodestyle==2.11.1 diff --git a/test/lib/ansible_test/_data/requirements/sanity.pylint.txt b/test/lib/ansible_test/_data/requirements/sanity.pylint.txt index c3144fe..b6bdec5 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.pylint.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.pylint.txt @@ -1,11 +1,11 @@ # edit "sanity.pylint.in" and generate with: hacking/update-sanity-requirements.py --test pylint -astroid==3.0.0 -dill==0.3.7 -isort==5.12.0 +astroid==3.1.0 +dill==0.3.8 +isort==5.13.2 mccabe==0.7.0 -platformdirs==3.11.0 -pylint==3.0.1 +platformdirs==4.2.0 +pylint==3.1.0 PyYAML==6.0.1 tomli==2.0.1 -tomlkit==0.12.1 -typing_extensions==4.8.0 +tomlkit==0.12.4 +typing_extensions==4.10.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt b/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt index 4af9b95..8e6e2ce 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt @@ -1,3 +1,3 @@ # edit "sanity.runtime-metadata.in" and generate with: hacking/update-sanity-requirements.py --test runtime-metadata PyYAML==6.0.1 -voluptuous==0.13.1 +voluptuous==0.14.2 diff --git a/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt b/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt index 4e24d64..fba0da1 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt @@ -1,6 +1,6 @@ # edit "sanity.validate-modules.in" and generate with: hacking/update-sanity-requirements.py --test validate-modules antsibull-docs-parser==1.0.0 -Jinja2==3.1.2 -MarkupSafe==2.1.3 +Jinja2==3.1.3 +MarkupSafe==2.1.5 PyYAML==6.0.1 -voluptuous==0.13.1 +voluptuous==0.14.2 diff --git a/test/lib/ansible_test/_data/requirements/sanity.yamllint.txt b/test/lib/ansible_test/_data/requirements/sanity.yamllint.txt index bafd30b..2c91d9e 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.yamllint.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.yamllint.txt @@ -1,4 +1,4 @@ # edit "sanity.yamllint.in" and generate with: hacking/update-sanity-requirements.py --test yamllint -pathspec==0.11.2 +pathspec==0.12.1 PyYAML==6.0.1 -yamllint==1.32.0 +yamllint==1.35.1 diff --git a/test/lib/ansible_test/_internal/bootstrap.py b/test/lib/ansible_test/_internal/bootstrap.py index b0cfb60..a9dd637 100644 --- a/test/lib/ansible_test/_internal/bootstrap.py +++ b/test/lib/ansible_test/_internal/bootstrap.py @@ -28,7 +28,7 @@ class Bootstrap: """Base class for bootstrapping systems.""" controller: bool - python_versions: list[str] + python_interpreters: dict[str, str] ssh_key: SshKey @property @@ -41,7 +41,7 @@ class Bootstrap: return dict( bootstrap_type=self.bootstrap_type, controller='yes' if self.controller else '', - python_versions=self.python_versions, + python_interpreters=[f'{key}:{value}' for key, value in self.python_interpreters.items()], ssh_key_type=self.ssh_key.KEY_TYPE, ssh_private_key=self.ssh_key.key_contents, ssh_public_key=self.ssh_key.pub_contents, diff --git a/test/lib/ansible_test/_internal/classification/__init__.py b/test/lib/ansible_test/_internal/classification/__init__.py index deda27e..b512284 100644 --- a/test/lib/ansible_test/_internal/classification/__init__.py +++ b/test/lib/ansible_test/_internal/classification/__init__.py @@ -674,11 +674,6 @@ class PathMapper: # Early classification that needs to occur before common classification belongs here. - if path.startswith('test/units/compat/'): - return { - 'units': 'test/units/', - } - if dirname == '.azure-pipelines/commands': test_map = { 'cloud.sh': 'integration:cloud/', diff --git a/test/lib/ansible_test/_internal/commands/integration/cloud/cs.py b/test/lib/ansible_test/_internal/commands/integration/cloud/cs.py index 8060804..ebb2734 100644 --- a/test/lib/ansible_test/_internal/commands/integration/cloud/cs.py +++ b/test/lib/ansible_test/_internal/commands/integration/cloud/cs.py @@ -38,7 +38,7 @@ class CsCloudProvider(CloudProvider): def __init__(self, args: IntegrationConfig) -> None: super().__init__(args) - self.image = os.environ.get('ANSIBLE_CLOUDSTACK_CONTAINER', 'quay.io/ansible/cloudstack-test-container:1.6.1') + self.image = os.environ.get('ANSIBLE_CLOUDSTACK_CONTAINER', 'quay.io/ansible/cloudstack-test-container:1.7.0') self.host = '' self.port = 0 diff --git a/test/lib/ansible_test/_internal/commands/integration/cloud/nios.py b/test/lib/ansible_test/_internal/commands/integration/cloud/nios.py index 62dd155..876968f 100644 --- a/test/lib/ansible_test/_internal/commands/integration/cloud/nios.py +++ b/test/lib/ansible_test/_internal/commands/integration/cloud/nios.py @@ -28,7 +28,7 @@ class NiosProvider(CloudProvider): # # It's source source itself resides at: # https://github.com/ansible/nios-test-container - DOCKER_IMAGE = 'quay.io/ansible/nios-test-container:2.0.0' + DOCKER_IMAGE = 'quay.io/ansible/nios-test-container:3.0.0' def __init__(self, args: IntegrationConfig) -> None: super().__init__(args) diff --git a/test/lib/ansible_test/_internal/commands/sanity/__init__.py b/test/lib/ansible_test/_internal/commands/sanity/__init__.py index 9b675e4..143fe33 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/__init__.py +++ b/test/lib/ansible_test/_internal/commands/sanity/__init__.py @@ -765,11 +765,6 @@ class SanityTest(metaclass=abc.ABCMeta): return False @property - def py2_compat(self) -> bool: - """True if the test only applies to code that runs on Python 2.x.""" - return False - - @property def supported_python_versions(self) -> t.Optional[tuple[str, ...]]: """A tuple of supported Python versions or None if the test does not depend on specific Python versions.""" return CONTROLLER_PYTHON_VERSIONS @@ -786,23 +781,11 @@ class SanityTest(metaclass=abc.ABCMeta): def filter_targets_by_version(self, args: SanityConfig, targets: list[TestTarget], python_version: str) -> list[TestTarget]: """Return the given list of test targets, filtered to include only those relevant for the test, taking into account the Python version.""" + del args # args is not used here, but derived classes may make use of it del python_version # python_version is not used here, but derived classes may make use of it targets = self.filter_targets(targets) - if self.py2_compat: - # This sanity test is a Python 2.x compatibility test. - content_config = get_content_config(args) - - if content_config.py2_support: - # This collection supports Python 2.x. - # Filter targets to include only those that require support for remote-only Python versions. - targets = self.filter_remote_targets(targets) - else: - # This collection does not support Python 2.x. - # There are no targets to test. - targets = [] - return targets @staticmethod @@ -878,7 +861,6 @@ class SanityCodeSmellTest(SanitySingleVersion): self.__no_targets: bool = self.config.get('no_targets') self.__include_directories: bool = self.config.get('include_directories') self.__include_symlinks: bool = self.config.get('include_symlinks') - self.__py2_compat: bool = self.config.get('py2_compat', False) self.__error_code: str | None = self.config.get('error_code', None) else: self.output = None @@ -894,7 +876,6 @@ class SanityCodeSmellTest(SanitySingleVersion): self.__no_targets = True self.__include_directories = False self.__include_symlinks = False - self.__py2_compat = False self.__error_code = None if self.no_targets: @@ -940,11 +921,6 @@ class SanityCodeSmellTest(SanitySingleVersion): return self.__include_symlinks @property - def py2_compat(self) -> bool: - """True if the test only applies to code that runs on Python 2.x.""" - return self.__py2_compat - - @property def supported_python_versions(self) -> t.Optional[tuple[str, ...]]: """A tuple of supported Python versions or None if the test does not depend on specific Python versions.""" versions = super().supported_python_versions @@ -1141,10 +1117,8 @@ def create_sanity_virtualenv( commands = collect_requirements( # create_sanity_virtualenv() python=python, controller=True, - virtualenv=False, command=None, ansible=False, - cryptography=False, coverage=coverage, minimize=minimize, sanity=name, diff --git a/test/lib/ansible_test/_internal/commands/sanity/import.py b/test/lib/ansible_test/_internal/commands/sanity/import.py index 36f5241..55b90f7 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/import.py +++ b/test/lib/ansible_test/_internal/commands/sanity/import.py @@ -47,11 +47,6 @@ from ...ansible_util import ( ansible_environment, ) -from ...python_requirements import ( - PipUnavailableError, - install_requirements, -) - from ...config import ( SanityConfig, ) @@ -68,10 +63,6 @@ from ...host_configs import ( PythonConfig, ) -from ...venv import ( - get_virtualenv_version, -) - def _get_module_test(module_restrictions: bool) -> c.Callable[[str], bool]: """Create a predicate which tests whether a path can be used by modules or not.""" @@ -109,15 +100,6 @@ class ImportTest(SanityMultipleVersion): paths = [target.path for target in targets.include] - if python.version.startswith('2.') and (get_virtualenv_version(args, python.path) or (0,)) < (13,): - # hack to make sure that virtualenv is available under Python 2.x - # on Python 3.x we can use the built-in venv - # version 13+ is required to use the `--no-wheel` option - try: - install_requirements(args, python, virtualenv=True, controller=False) # sanity (import) - except PipUnavailableError as ex: - display.warning(str(ex)) - temp_root = os.path.join(ResultType.TMP.path, 'sanity', 'import') messages = [] diff --git a/test/lib/ansible_test/_internal/commands/sanity/mypy.py b/test/lib/ansible_test/_internal/commands/sanity/mypy.py index c93474e..0465951 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/mypy.py +++ b/test/lib/ansible_test/_internal/commands/sanity/mypy.py @@ -66,7 +66,6 @@ class MypyTest(SanityMultipleVersion): vendored_paths = ( 'lib/ansible/module_utils/six/__init__.py', 'lib/ansible/module_utils/distro/_distro.py', - 'lib/ansible/module_utils/compat/_selectors2.py', ) def filter_targets(self, targets: list[TestTarget]) -> list[TestTarget]: @@ -77,15 +76,6 @@ class MypyTest(SanityMultipleVersion): or target.path.startswith('test/lib/ansible_test/_util/target/sanity/import/'))] @property - def supported_python_versions(self) -> t.Optional[tuple[str, ...]]: - """A tuple of supported Python versions or None if the test does not depend on specific Python versions.""" - # mypy 0.981 dropped support for Python 2 - # see: https://mypy-lang.blogspot.com/2022/09/mypy-0981-released.html - # cryptography dropped support for Python 3.5 in version 3.3 - # see: https://cryptography.io/en/latest/changelog/#v3-3 - return tuple(version for version in SUPPORTED_PYTHON_VERSIONS if str_to_version(version) >= (3, 6)) - - @property def error_code(self) -> t.Optional[str]: """Error code for ansible-test matching the format used by the underlying test program, or None if the program does not use error codes.""" return 'ansible-test' @@ -95,6 +85,13 @@ class MypyTest(SanityMultipleVersion): """True if the test requires PyPI, otherwise False.""" return True + @property + def supported_python_versions(self) -> t.Optional[tuple[str, ...]]: + """A tuple of supported Python versions or None if the test does not depend on specific Python versions.""" + # Because the version of typeshed mypy use in 1.9 doesn't support 3.7, neither does mypy 1.9. + # see: https://mypy-lang.blogspot.com/2024/03/mypy-19-released.html + return tuple(version for version in SUPPORTED_PYTHON_VERSIONS if str_to_version(version) >= (3, 8)) + def test(self, args: SanityConfig, targets: SanityTargets, python: PythonConfig) -> TestResult: settings = self.load_processor(args, python.version) diff --git a/test/lib/ansible_test/_internal/commands/units/__init__.py b/test/lib/ansible_test/_internal/commands/units/__init__.py index 71ce5c4..9427262 100644 --- a/test/lib/ansible_test/_internal/commands/units/__init__.py +++ b/test/lib/ansible_test/_internal/commands/units/__init__.py @@ -245,7 +245,7 @@ def command_units(args: UnitsConfig) -> None: # # NOTE: This only affects use of pytest-mock. # Collection unit tests may directly import mock, which will be provided by ansible-test when it installs requirements using pip. - # Although mock is available for ansible-core unit tests, they should import units.compat.mock instead. + # Although mock is available for ansible-core unit tests, they should import unittest.mock instead. if str_to_version(python.version) < (3, 8): config_name = 'legacy.ini' else: diff --git a/test/lib/ansible_test/_internal/config.py b/test/lib/ansible_test/_internal/config.py index dbc137b..18d815c 100644 --- a/test/lib/ansible_test/_internal/config.py +++ b/test/lib/ansible_test/_internal/config.py @@ -65,7 +65,6 @@ class ContentConfig: modules: ModulesConfig python_versions: tuple[str, ...] - py2_support: bool class EnvironmentConfig(CommonConfig): diff --git a/test/lib/ansible_test/_internal/content_config.py b/test/lib/ansible_test/_internal/content_config.py index 7ac1876..c9c37df 100644 --- a/test/lib/ansible_test/_internal/content_config.py +++ b/test/lib/ansible_test/_internal/content_config.py @@ -29,7 +29,6 @@ from .io import ( from .util import ( ApplicationError, display, - str_to_version, ) from .data import ( @@ -75,13 +74,9 @@ def parse_content_config(data: t.Any) -> ContentConfig: python_versions = tuple(version for version in SUPPORTED_PYTHON_VERSIONS if version in CONTROLLER_PYTHON_VERSIONS or version in modules.python_versions) - # True if Python 2.x is supported. - py2_support = any(version for version in python_versions if str_to_version(version)[0] == 2) - return ContentConfig( modules=modules, python_versions=python_versions, - py2_support=py2_support, ) diff --git a/test/lib/ansible_test/_internal/coverage_util.py b/test/lib/ansible_test/_internal/coverage_util.py index 3017623..28e66b5 100644 --- a/test/lib/ansible_test/_internal/coverage_util.py +++ b/test/lib/ansible_test/_internal/coverage_util.py @@ -71,7 +71,6 @@ COVERAGE_VERSIONS = ( # IMPORTANT: Keep this in sync with the ansible-test.txt requirements file. CoverageVersion('7.3.2', 7, (3, 8), (3, 12)), CoverageVersion('6.5.0', 7, (3, 7), (3, 7)), - CoverageVersion('4.5.4', 0, (2, 6), (3, 6)), ) """ This tuple specifies the coverage version to use for Python version ranges. diff --git a/test/lib/ansible_test/_internal/docker_util.py b/test/lib/ansible_test/_internal/docker_util.py index 52b9691..97c022c 100644 --- a/test/lib/ansible_test/_internal/docker_util.py +++ b/test/lib/ansible_test/_internal/docker_util.py @@ -496,7 +496,7 @@ def get_docker_hostname() -> str: """Return the hostname of the Docker service.""" docker_host = os.environ.get('DOCKER_HOST') - if docker_host and docker_host.startswith('tcp://'): + if docker_host and docker_host.startswith(('tcp://', 'ssh://')): try: hostname = urllib.parse.urlparse(docker_host)[1].split(':')[0] display.info('Detected Docker host: %s' % hostname, verbosity=1) diff --git a/test/lib/ansible_test/_internal/host_profiles.py b/test/lib/ansible_test/_internal/host_profiles.py index 0981245..39fe7d2 100644 --- a/test/lib/ansible_test/_internal/host_profiles.py +++ b/test/lib/ansible_test/_internal/host_profiles.py @@ -958,7 +958,7 @@ class DockerProfile(ControllerHostProfile[DockerConfig], SshTargetHostProfile[Do """Perform out-of-band setup before delegation.""" bootstrapper = BootstrapDocker( controller=self.controller, - python_versions=[self.python.version], + python_interpreters={self.python.version: self.python.path}, ssh_key=SshKey(self.args), ) @@ -1214,8 +1214,9 @@ class PosixRemoteProfile(ControllerHostProfile[PosixRemoteConfig], RemoteProfile def configure(self) -> None: """Perform in-band configuration. Executed before delegation for the controller and after delegation for targets.""" # a target uses a single python version, but a controller may include additional versions for targets running on the controller - python_versions = [self.python.version] + [target.python.version for target in self.targets if isinstance(target, ControllerConfig)] - python_versions = sorted_versions(list(set(python_versions))) + python_interpreters = {self.python.version: self.python.path} + python_interpreters.update({target.python.version: target.python.path for target in self.targets if isinstance(target, ControllerConfig)}) + python_interpreters = {version: python_interpreters[version] for version in sorted_versions(list(python_interpreters.keys()))} core_ci = self.wait_for_instance() pwd = self.wait_until_ready() @@ -1226,7 +1227,7 @@ class PosixRemoteProfile(ControllerHostProfile[PosixRemoteConfig], RemoteProfile controller=self.controller, platform=self.config.platform, platform_version=self.config.version, - python_versions=python_versions, + python_interpreters=python_interpreters, ssh_key=core_ci.ssh_key, ) diff --git a/test/lib/ansible_test/_internal/python_requirements.py b/test/lib/ansible_test/_internal/python_requirements.py index 81006e4..404f2cd 100644 --- a/test/lib/ansible_test/_internal/python_requirements.py +++ b/test/lib/ansible_test/_internal/python_requirements.py @@ -5,7 +5,6 @@ import base64 import dataclasses import json import os -import re import typing as t from .encoding import ( @@ -20,14 +19,9 @@ from .io import ( from .util import ( ANSIBLE_TEST_DATA_ROOT, ANSIBLE_TEST_TARGET_ROOT, - ANSIBLE_TEST_TOOLS_ROOT, ApplicationError, SubprocessError, display, - find_executable, - raw_command, - str_to_version, - version_to_str, ) from .util_common import ( @@ -63,9 +57,6 @@ from .coverage_util import ( QUIET_PIP_SCRIPT_PATH = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'quiet_pip.py') REQUIREMENTS_SCRIPT_PATH = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'requirements.py') -# IMPORTANT: Keep this in sync with the ansible-test.txt requirements file. -VIRTUALENV_VERSION = '16.7.12' - # Pip Abstraction @@ -132,7 +123,6 @@ def install_requirements( ansible: bool = False, command: bool = False, coverage: bool = False, - virtualenv: bool = False, controller: bool = True, connection: t.Optional[Connection] = None, ) -> None: @@ -145,8 +135,6 @@ def install_requirements( if command and isinstance(args, (UnitsConfig, IntegrationConfig)) and args.coverage: coverage = True - cryptography = False - if ansible: try: ansible_cache = install_requirements.ansible_cache # type: ignore[attr-defined] @@ -160,19 +148,12 @@ def install_requirements( else: ansible_cache[python.path] = True - # Install the latest cryptography version that the current requirements can support if it is not already available. - # This avoids downgrading cryptography when OS packages provide a newer version than we are able to install using pip. - # If not installed here, later install commands may try to install a version of cryptography which cannot be installed. - cryptography = not is_cryptography_available(python.path) - commands = collect_requirements( python=python, controller=controller, ansible=ansible, - cryptography=cryptography, command=args.command if command else None, coverage=coverage, - virtualenv=virtualenv, minimize=False, sanity=None, ) @@ -205,9 +186,7 @@ def collect_requirements( python: PythonConfig, controller: bool, ansible: bool, - cryptography: bool, coverage: bool, - virtualenv: bool, minimize: bool, command: t.Optional[str], sanity: t.Optional[str], @@ -215,17 +194,9 @@ def collect_requirements( """Collect requirements for the given Python using the specified arguments.""" commands: list[PipCommand] = [] - if virtualenv: - # sanity tests on Python 2.x install virtualenv when it is too old or is not already installed and the `--requirements` option is given - # the last version of virtualenv with no dependencies is used to minimize the changes made outside a virtual environment - commands.extend(collect_package_install(packages=[f'virtualenv=={VIRTUALENV_VERSION}'], constraints=False)) - if coverage: commands.extend(collect_package_install(packages=[f'coverage=={get_coverage_version(python.version).coverage_version}'], constraints=False)) - if cryptography: - commands.extend(collect_package_install(packages=get_cryptography_requirements(python))) - if ansible or command: commands.extend(collect_general_install(command, ansible)) @@ -446,17 +417,7 @@ def get_venv_packages(python: PythonConfig) -> dict[str, str]: wheel='0.37.1', ) - override_packages = { - '2.7': dict( - pip='20.3.4', # 21.0 requires Python 3.6+ - setuptools='44.1.1', # 45.0.0 requires Python 3.5+ - wheel=None, - ), - '3.6': dict( - pip='21.3.1', # 22.0 requires Python 3.7+ - setuptools='59.6.0', # 59.7.0 requires Python 3.7+ - wheel=None, - ), + override_packages: dict[str, dict[str, str]] = { } packages = {name: version or default_packages[name] for name, version in override_packages.get(python.version, default_packages).items()} @@ -510,82 +471,3 @@ def prepare_pip_script(commands: list[PipCommand]) -> str: def usable_pip_file(path: t.Optional[str]) -> bool: """Return True if the specified pip file is usable, otherwise False.""" return bool(path) and os.path.exists(path) and bool(os.path.getsize(path)) - - -# Cryptography - - -def is_cryptography_available(python: str) -> bool: - """Return True if cryptography is available for the given python.""" - try: - raw_command([python, '-c', 'import cryptography'], capture=True) - except SubprocessError: - return False - - return True - - -def get_cryptography_requirements(python: PythonConfig) -> list[str]: - """ - Return the correct cryptography and pyopenssl requirements for the given python version. - The version of cryptography installed depends on the python version and openssl version. - """ - openssl_version = get_openssl_version(python) - - if openssl_version and openssl_version < (1, 1, 0): - # cryptography 3.2 requires openssl 1.1.x or later - # see https://cryptography.io/en/latest/changelog.html#v3-2 - cryptography = 'cryptography < 3.2' - # pyopenssl 20.0.0 requires cryptography 3.2 or later - pyopenssl = 'pyopenssl < 20.0.0' - else: - # cryptography 3.4+ builds require a working rust toolchain - # systems bootstrapped using ansible-core-ci can access additional wheels through the spare-tire package index - cryptography = 'cryptography' - # any future installation of pyopenssl is free to use any compatible version of cryptography - pyopenssl = '' - - requirements = [ - cryptography, - pyopenssl, - ] - - requirements = [requirement for requirement in requirements if requirement] - - return requirements - - -def get_openssl_version(python: PythonConfig) -> t.Optional[tuple[int, ...]]: - """Return the openssl version.""" - if not python.version.startswith('2.'): - # OpenSSL version checking only works on Python 3.x. - # This should be the most accurate, since it is the Python we will be using. - version = json.loads(raw_command([python.path, os.path.join(ANSIBLE_TEST_TOOLS_ROOT, 'sslcheck.py')], capture=True)[0])['version'] - - if version: - display.info(f'Detected OpenSSL version {version_to_str(version)} under Python {python.version}.', verbosity=1) - - return tuple(version) - - # Fall back to detecting the OpenSSL version from the CLI. - # This should provide an adequate solution on Python 2.x. - openssl_path = find_executable('openssl', required=False) - - if openssl_path: - try: - result = raw_command([openssl_path, 'version'], capture=True)[0] - except SubprocessError: - result = '' - - match = re.search(r'^OpenSSL (?P<version>[0-9]+\.[0-9]+\.[0-9]+)', result) - - if match: - version = str_to_version(match.group('version')) - - display.info(f'Detected OpenSSL version {version_to_str(version)} using the openssl CLI.', verbosity=1) - - return version - - display.info('Unable to detect OpenSSL version.', verbosity=1) - - return None diff --git a/test/lib/ansible_test/_internal/ssh.py b/test/lib/ansible_test/_internal/ssh.py index b2a2678..8125768 100644 --- a/test/lib/ansible_test/_internal/ssh.py +++ b/test/lib/ansible_test/_internal/ssh.py @@ -245,6 +245,7 @@ def create_ssh_port_forwards( """ options: dict[str, t.Union[str, int]] = dict( LogLevel='INFO', # info level required to get messages on stderr indicating the ports assigned to each forward + ControlPath='none', # if the user has ControlPath set up for every host, it will prevent creation of forwards ) cli_args = [] diff --git a/test/lib/ansible_test/_internal/util.py b/test/lib/ansible_test/_internal/util.py index 394c263..903cbcc 100644 --- a/test/lib/ansible_test/_internal/util.py +++ b/test/lib/ansible_test/_internal/util.py @@ -998,7 +998,7 @@ def retry(func: t.Callable[..., TValue], ex_type: t.Type[BaseException] = Subpro def parse_to_list_of_dict(pattern: str, value: str) -> list[dict[str, str]]: """Parse lines from the given value using the specified pattern and return the extracted list of key/value pair dictionaries.""" matched = [] - unmatched = [] + unmatched: list[str] = [] for line in value.splitlines(): match = re.search(pattern, line) diff --git a/test/lib/ansible_test/_internal/util_common.py b/test/lib/ansible_test/_internal/util_common.py index 77a6165..a690497 100644 --- a/test/lib/ansible_test/_internal/util_common.py +++ b/test/lib/ansible_test/_internal/util_common.py @@ -409,7 +409,7 @@ def create_interpreter_wrapper(interpreter: str, injected_interpreter: str) -> N code = textwrap.dedent(''' #!%s - from __future__ import absolute_import + from __future__ import annotations from os import execv from sys import argv diff --git a/test/lib/ansible_test/_internal/venv.py b/test/lib/ansible_test/_internal/venv.py index a83fc8b..cdd73b0 100644 --- a/test/lib/ansible_test/_internal/venv.py +++ b/test/lib/ansible_test/_internal/venv.py @@ -15,7 +15,6 @@ from .config import ( from .util import ( find_python, SubprocessError, - get_available_python_versions, ANSIBLE_TEST_TARGET_TOOLS_ROOT, display, remove_tree, @@ -85,49 +84,21 @@ def create_virtual_environment( system_site_packages: bool = False, pip: bool = False, ) -> bool: - """Create a virtual environment using venv or virtualenv for the requested Python version.""" + """Create a virtual environment using venv for the requested Python version.""" if not os.path.exists(python.path): # the requested python version could not be found return False - if str_to_version(python.version) >= (3, 0): - # use the built-in 'venv' module on Python 3.x - # creating a virtual environment using 'venv' when running in a virtual environment created by 'virtualenv' results - # in a copy of the original virtual environment instead of creation of a new one - # avoid this issue by only using "real" python interpreters to invoke 'venv' - for real_python in iterate_real_pythons(python.version): - if run_venv(args, real_python, system_site_packages, pip, path): - display.info('Created Python %s virtual environment using "venv": %s' % (python.version, path), verbosity=1) - return True - - # something went wrong, most likely the package maintainer for the Python installation removed ensurepip - # which will prevent creation of a virtual environment without installation of other OS packages - - # use the installed 'virtualenv' module on the Python requested version - if run_virtualenv(args, python.path, python.path, system_site_packages, pip, path): - display.info('Created Python %s virtual environment using "virtualenv": %s' % (python.version, path), verbosity=1) - return True - - available_pythons = get_available_python_versions() - - for available_python_version, available_python_interpreter in sorted(available_pythons.items()): - if available_python_interpreter == python.path: - # already attempted to use this interpreter - continue - - virtualenv_version = get_virtualenv_version(args, available_python_interpreter) - - if not virtualenv_version: - # virtualenv not available for this Python or we were unable to detect the version - continue - - # try using 'virtualenv' from another Python to setup the desired version - if run_virtualenv(args, available_python_interpreter, python.path, system_site_packages, pip, path): - display.info('Created Python %s virtual environment using "virtualenv" on Python %s: %s' % (python.version, available_python_version, path), - verbosity=1) + # creating a virtual environment using 'venv' when running in a virtual environment created by 'virtualenv' results + # in a copy of the original virtual environment instead of creation of a new one + # avoid this issue by only using "real" python interpreters to invoke 'venv' + for real_python in iterate_real_pythons(python.version): + if run_venv(args, real_python, system_site_packages, pip, path): + display.info('Created Python %s virtual environment using "venv": %s' % (python.version, path), verbosity=1) return True - # no suitable 'virtualenv' available + # something went wrong, most likely the package maintainer for the Python installation removed ensurepip + # which will prevent creation of a virtual environment without installation of other OS packages return False @@ -188,7 +159,7 @@ def run_venv( pip: bool, path: str, ) -> bool: - """Create a virtual environment using the 'venv' module. Not available on Python 2.x.""" + """Create a virtual environment using the 'venv' module.""" cmd = [run_python, '-m', 'venv'] if system_site_packages: @@ -210,72 +181,3 @@ def run_venv( return False return True - - -def run_virtualenv( - args: EnvironmentConfig, - run_python: str, - env_python: str, - system_site_packages: bool, - pip: bool, - path: str, -) -> bool: - """Create a virtual environment using the 'virtualenv' module.""" - # always specify which interpreter to use to guarantee the desired interpreter is provided - # otherwise virtualenv may select a different interpreter than the one running virtualenv - cmd = [run_python, '-m', 'virtualenv', '--python', env_python] - - if system_site_packages: - cmd.append('--system-site-packages') - - if not pip: - cmd.append('--no-pip') - # these options provide consistency with venv, which does not install them without pip - cmd.append('--no-setuptools') - cmd.append('--no-wheel') - - cmd.append(path) - - try: - run_command(args, cmd, capture=True) - except SubprocessError as ex: - remove_tree(path) - - if args.verbosity > 1: - display.error(ex.message) - - return False - - return True - - -def get_virtualenv_version(args: EnvironmentConfig, python: str) -> t.Optional[tuple[int, ...]]: - """Get the virtualenv version for the given python interpreter, if available, otherwise return None.""" - try: - cache = get_virtualenv_version.cache # type: ignore[attr-defined] - except AttributeError: - cache = get_virtualenv_version.cache = {} # type: ignore[attr-defined] - - if python not in cache: - try: - stdout = run_command(args, [python, '-m', 'virtualenv', '--version'], capture=True)[0] - except SubprocessError as ex: - stdout = '' - - if args.verbosity > 1: - display.error(ex.message) - - version = None - - if stdout: - # noinspection PyBroadException - try: - version = str_to_version(stdout.strip()) - except Exception: # pylint: disable=broad-except - pass - - cache[python] = version - - version = cache[python] - - return version diff --git a/test/lib/ansible_test/_util/__init__.py b/test/lib/ansible_test/_util/__init__.py index 527d413..8f58623 100644 --- a/test/lib/ansible_test/_util/__init__.py +++ b/test/lib/ansible_test/_util/__init__.py @@ -1,2 +1 @@ -# Empty __init__.py to allow importing of `ansible_test._util.target.common` under Python 2.x. -# This allows the ansible-test entry point to report supported Python versions before exiting. +# Empty __init__.py to keep pylint happy. diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.json deleted file mode 100644 index 4ebce32..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extensions": [ - ".py" - ], - "py2_compat": true, - "output": "path-message" -} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.py deleted file mode 100644 index 7b39c37..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Enforce proper usage of __future__ imports.""" -from __future__ import annotations - -import ast -import sys - - -def main(): - """Main entry point.""" - for path in sys.argv[1:] or sys.stdin.read().splitlines(): - with open(path, 'rb') as path_fd: - lines = path_fd.read().splitlines() - - missing = True - if not lines: - # Files are allowed to be empty of everything including boilerplate - missing = False - - for text in lines: - if text in (b'from __future__ import (absolute_import, division, print_function)', - b'from __future__ import absolute_import, division, print_function'): - missing = False - break - - if missing: - with open(path, encoding='utf-8') as file: - contents = file.read() - - # noinspection PyBroadException - try: - node = ast.parse(contents) - - # files consisting of only assignments have no need for future import boilerplate - # the only exception would be division during assignment, but we'll overlook that for simplicity - # the most likely case is that of a documentation only python file - if all(isinstance(statement, ast.Assign) for statement in node.body): - missing = False - except Exception: # pylint: disable=broad-except - pass # the compile sanity test will report this error - - if missing: - print('%s: missing: from __future__ import (absolute_import, division, print_function)' % path) - - -if __name__ == '__main__': - main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.json deleted file mode 100644 index 4ebce32..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extensions": [ - ".py" - ], - "py2_compat": true, - "output": "path-message" -} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.py deleted file mode 100644 index 8bdcfc9..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Require __metaclass__ boilerplate for code that supports Python 2.x.""" -from __future__ import annotations - -import ast -import sys - - -def main(): - """Main entry point.""" - for path in sys.argv[1:] or sys.stdin.read().splitlines(): - with open(path, 'rb') as path_fd: - lines = path_fd.read().splitlines() - - missing = True - if not lines: - # Files are allowed to be empty of everything including boilerplate - missing = False - - for text in lines: - if text == b'__metaclass__ = type': - missing = False - break - - if missing: - with open(path, encoding='utf-8') as file: - contents = file.read() - - # noinspection PyBroadException - try: - node = ast.parse(contents) - - # files consisting of only assignments have no need for metaclass boilerplate - # the most likely case is that of a documentation only python file - if all(isinstance(statement, ast.Assign) for statement in node.body): - missing = False - except Exception: # pylint: disable=broad-except - pass # the compile sanity test will report this error - - if missing: - print('%s: missing: __metaclass__ = type' % path) - - -if __name__ == '__main__': - main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.json deleted file mode 100644 index 88858ae..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extensions": [ - ".py" - ], - "ignore_self": true, - "output": "path-line-column-message" -} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.py deleted file mode 100644 index 74e38d7..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Disallow use of basestring isinstance checks.""" -from __future__ import annotations - -import re -import sys - - -def main(): - """Main entry point.""" - for path in sys.argv[1:] or sys.stdin.read().splitlines(): - with open(path, 'r', encoding='utf-8') as path_fd: - for line, text in enumerate(path_fd.readlines()): - match = re.search(r'(isinstance.*basestring)', text) - - if match: - print('%s:%d:%d: do not use `isinstance(s, basestring)`' % ( - path, line + 1, match.start(1) + 1)) - - -if __name__ == '__main__': - main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.json deleted file mode 100644 index 88858ae..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extensions": [ - ".py" - ], - "ignore_self": true, - "output": "path-line-column-message" -} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.py deleted file mode 100644 index b4e4002..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Disallow use of the dict.iteritems function.""" -from __future__ import annotations - -import re -import sys - - -def main(): - """Main entry point.""" - for path in sys.argv[1:] or sys.stdin.read().splitlines(): - with open(path, 'r', encoding='utf-8') as path_fd: - for line, text in enumerate(path_fd.readlines()): - match = re.search(r'(?<! six)\.(iteritems)', text) - - if match: - print('%s:%d:%d: use `dict.items` or `ansible.module_utils.six.iteritems` instead of `dict.iteritems`' % ( - path, line + 1, match.start(1) + 1)) - - -if __name__ == '__main__': - main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.json deleted file mode 100644 index 88858ae..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extensions": [ - ".py" - ], - "ignore_self": true, - "output": "path-line-column-message" -} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.py deleted file mode 100644 index 00c8703..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Disallow use of the dict.iterkeys function.""" -from __future__ import annotations - -import re -import sys - - -def main(): - """Main entry point.""" - for path in sys.argv[1:] or sys.stdin.read().splitlines(): - with open(path, 'r', encoding='utf-8') as path_fd: - for line, text in enumerate(path_fd.readlines()): - match = re.search(r'\.(iterkeys)', text) - - if match: - print('%s:%d:%d: use `dict.keys` or `for key in dict:` instead of `dict.iterkeys`' % ( - path, line + 1, match.start(1) + 1)) - - -if __name__ == '__main__': - main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.json deleted file mode 100644 index 88858ae..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extensions": [ - ".py" - ], - "ignore_self": true, - "output": "path-line-column-message" -} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.py deleted file mode 100644 index 2e8036a..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Disallow use of the dict.itervalues function.""" -from __future__ import annotations - -import re -import sys - - -def main(): - """Main entry point.""" - for path in sys.argv[1:] or sys.stdin.read().splitlines(): - with open(path, 'r', encoding='utf-8') as path_fd: - for line, text in enumerate(path_fd.readlines()): - match = re.search(r'(?<! six)\.(itervalues)', text) - - if match: - print('%s:%d:%d: use `dict.values` or `ansible.module_utils.six.itervalues` instead of `dict.itervalues`' % ( - path, line + 1, match.start(1) + 1)) - - -if __name__ == '__main__': - main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.json deleted file mode 100644 index ccee80a..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extensions": [ - ".py" - ], - "prefixes": [ - "lib/ansible/", - "plugins/" - ], - "output": "path-line-column-message" -} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.py deleted file mode 100644 index eb5987d..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Disallow importing display from __main__.""" -from __future__ import annotations - -import sys - -MAIN_DISPLAY_IMPORT = 'from __main__ import display' - - -def main(): - """Main entry point.""" - for path in sys.argv[1:] or sys.stdin.read().splitlines(): - with open(path, 'r', encoding='utf-8') as file: - for i, line in enumerate(file.readlines()): - if MAIN_DISPLAY_IMPORT in line: - lineno = i + 1 - colno = line.index(MAIN_DISPLAY_IMPORT) + 1 - print('%s:%d:%d: Display is a singleton, just import and instantiate' % (path, lineno, colno)) - - -if __name__ == '__main__': - main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.json deleted file mode 100644 index 88858ae..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extensions": [ - ".py" - ], - "ignore_self": true, - "output": "path-line-column-message" -} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.py deleted file mode 100644 index 75c34f2..0000000 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Disallow use of the unicode_literals future.""" -from __future__ import annotations - -import re -import sys - - -def main(): - """Main entry point.""" - for path in sys.argv[1:] or sys.stdin.read().splitlines(): - with open(path, 'r', encoding='utf-8') as path_fd: - for line, text in enumerate(path_fd.readlines()): - match = re.search(r'(unicode_literals)', text) - - if match: - print('%s:%d:%d: do not use `unicode_literals`' % ( - path, line + 1, match.start(1) + 1)) - - -if __name__ == '__main__': - main() diff --git a/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-core.ini b/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-core.ini index 41d824b..0251f67 100644 --- a/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-core.ini +++ b/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-core.ini @@ -46,9 +46,6 @@ ignore_missing_imports = True [mypy-lxml.*] ignore_missing_imports = True -[mypy-yum.*] -ignore_missing_imports = True - [mypy-rpmUtils.*] ignore_missing_imports = True @@ -85,9 +82,6 @@ ignore_missing_imports = True [mypy-distro.*] ignore_missing_imports = True -[mypy-selectors2.*] -ignore_missing_imports = True - [mypy-resolvelib.*] ignore_missing_imports = True diff --git a/test/lib/ansible_test/_util/controller/sanity/mypy/modules.ini b/test/lib/ansible_test/_util/controller/sanity/mypy/modules.ini index d6a608f..b4e7b05 100644 --- a/test/lib/ansible_test/_util/controller/sanity/mypy/modules.ini +++ b/test/lib/ansible_test/_util/controller/sanity/mypy/modules.ini @@ -13,9 +13,6 @@ ignore_missing_imports = True [mypy-md5.*] ignore_missing_imports = True -[mypy-yum.*] -ignore_missing_imports = True - [mypy-rpmUtils.*] ignore_missing_imports = True @@ -52,9 +49,6 @@ ignore_missing_imports = True [mypy-distro.*] ignore_missing_imports = True -[mypy-selectors2.*] -ignore_missing_imports = True - [mypy-selinux.*] ignore_missing_imports = True diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg index f8a0a8a..51d0bb0 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg @@ -35,7 +35,6 @@ bad-names= tutu, good-names= - __metaclass__, C, ex, i, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg index 5bec36f..801adbe 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg @@ -34,7 +34,6 @@ bad-names= tutu, good-names= - __metaclass__, C, ex, i, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg index c30eb37..adc0b95 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg @@ -33,7 +33,6 @@ bad-names= tutu, good-names= - __metaclass__, C, ex, i, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/collection.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/collection.cfg index 762d488..bb6c3e4 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/collection.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/collection.cfg @@ -102,7 +102,6 @@ disable= undefined-loop-variable, unexpected-keyword-arg, ungrouped-imports, - unidiomatic-typecheck, unnecessary-pass, unnecessary-dunder-call, unsubscriptable-object, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/default.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/default.cfg index 825e5df..6264948 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/default.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/default.cfg @@ -95,7 +95,6 @@ disable= undefined-loop-variable, unexpected-keyword-arg, ungrouped-imports, - unidiomatic-typecheck, unnecessary-pass, unsubscriptable-object, unsupported-assignment-operation, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py index f6c8337..be4dba5 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py @@ -32,7 +32,6 @@ except ImportError: from pylint.checkers import BaseChecker, BaseTokenChecker from ansible.module_utils.compat.version import LooseVersion -from ansible.module_utils.six import string_types from ansible.release import __version__ as ansible_version_raw from ansible.utils.version import SemanticVersion @@ -137,7 +136,7 @@ def _get_func_name(node): def parse_isodate(value): """Parse an ISO 8601 date string.""" msg = 'Expected ISO 8601 date string (YYYY-MM-DD)' - if not isinstance(value, string_types): + if not isinstance(value, str): raise ValueError(msg) # From Python 3.7 in, there is datetime.date.fromisoformat(). For older versions, # we have to do things manually. diff --git a/test/lib/ansible_test/_util/controller/sanity/shellcheck/exclude.txt b/test/lib/ansible_test/_util/controller/sanity/shellcheck/exclude.txt index 29588dd..6563f8f 100644 --- a/test/lib/ansible_test/_util/controller/sanity/shellcheck/exclude.txt +++ b/test/lib/ansible_test/_util/controller/sanity/shellcheck/exclude.txt @@ -1,3 +1,2 @@ SC1090 SC1091 -SC2164 diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py index 2b92a56..5e3a07e 100644 --- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py @@ -70,13 +70,12 @@ from ansible.module_utils.common.collections import is_iterable from ansible.module_utils.common.parameters import DEFAULT_TYPE_VALIDATORS from ansible.module_utils.compat.version import StrictVersion, LooseVersion from ansible.module_utils.basic import to_bytes -from ansible.module_utils.six import PY3, with_metaclass, string_types from ansible.plugins.loader import fragment_loader from ansible.plugins.list import IGNORE as REJECTLIST from ansible.utils.plugin_docs import add_collection_to_versions_and_dates, add_fragments, get_docstring from ansible.utils.version import SemanticVersion -from .module_args import AnsibleModuleImportError, AnsibleModuleNotInitialized, get_argument_spec +from .module_args import AnsibleModuleImportError, AnsibleModuleNotInitialized, get_py_argument_spec, get_ps_argument_spec from .schema import ( ansible_module_kwargs_schema, @@ -87,18 +86,14 @@ from .schema import ( from .utils import CaptureStd, NoArgsAnsibleModule, compare_unordered_lists, parse_yaml, parse_isodate -if PY3: - # Because there is no ast.TryExcept in Python 3 ast module - TRY_EXCEPT = ast.Try - # REPLACER_WINDOWS from ansible.executor.module_common is byte - # string but we need unicode for Python 3 - REPLACER_WINDOWS = REPLACER_WINDOWS.decode('utf-8') -else: - TRY_EXCEPT = ast.TryExcept +# Because there is no ast.TryExcept in Python 3 ast module +TRY_EXCEPT = ast.Try +# REPLACER_WINDOWS from ansible.executor.module_common is byte +# string but we need unicode for Python 3 +REPLACER_WINDOWS = REPLACER_WINDOWS.decode('utf-8') REJECTLIST_DIRS = frozenset(('.git', 'test', '.github', '.idea')) INDENT_REGEX = re.compile(r'([\t]*)') -TYPE_REGEX = re.compile(r'.*(if|or)(\s+[^"\']*|\s+)(?<!_)(?<!str\()type\([^)].*') SYS_EXIT_REGEX = re.compile(r'[^#]*sys.exit\s*\(.*') NO_LOG_REGEX = re.compile(r'(?:pass(?!ive)|secret|token|key)', re.I) @@ -269,7 +264,7 @@ class Reporter: return 3 if sum(ret) else 0 -class Validator(with_metaclass(abc.ABCMeta, object)): +class Validator(metaclass=abc.ABCMeta): """Validator instances are intended to be run on a single object. if you are scanning multiple objects for problems, you'll want to have a separate Validator for each one.""" @@ -335,8 +330,6 @@ class ModuleValidator(Validator): self.git_cache = git_cache self.base_module = self.git_cache.get_original_path(self.path) - self._python_module_override = False - with open(path) as f: self.text = f.read() self.length = len(self.text.splitlines()) @@ -383,7 +376,7 @@ class ModuleValidator(Validator): pass def _python_module(self): - if self.path.endswith('.py') or self._python_module_override: + if self.path.endswith('.py'): return True return False @@ -421,7 +414,7 @@ class ModuleValidator(Validator): return self.git_cache.is_new(self.path) def _check_interpreter(self, powershell=False): - if powershell: + if self._powershell_module(): if not self.text.startswith('#!powershell\n'): self.reporter.error( path=self.object_path, @@ -430,34 +423,20 @@ class ModuleValidator(Validator): ) return - missing_python_interpreter = False - - if not self.text.startswith('#!/usr/bin/python'): - if NEW_STYLE_PYTHON_MODULE_RE.search(to_bytes(self.text)): - missing_python_interpreter = self.text.startswith('#!') # shebang optional, but if present must match - else: - missing_python_interpreter = True # shebang required + if self._python_module(): + missing_python_interpreter = False - if missing_python_interpreter: - self.reporter.error( - path=self.object_path, - code='missing-python-interpreter', - msg='Interpreter line is not "#!/usr/bin/python"', - ) + if not self.text.startswith('#!/usr/bin/python'): + if NEW_STYLE_PYTHON_MODULE_RE.search(to_bytes(self.text)): + missing_python_interpreter = self.text.startswith('#!') # shebang optional, but if present must match + else: + missing_python_interpreter = True # shebang required - def _check_type_instead_of_isinstance(self, powershell=False): - if powershell: - return - for line_no, line in enumerate(self.text.splitlines()): - typekeyword = TYPE_REGEX.match(line) - if typekeyword: - # TODO: add column + if missing_python_interpreter: self.reporter.error( path=self.object_path, - code='unidiomatic-typecheck', - msg=('Type comparison using type() found. ' - 'Use isinstance() instead'), - line=line_no + 1 + code='missing-python-interpreter', + msg='Interpreter line is not "#!/usr/bin/python"', ) def _check_for_sys_exit(self): @@ -1120,14 +1099,6 @@ class ModuleValidator(Validator): ' documentation for removed' ) else: - # We are testing a collection - if self.object_name.startswith('_'): - self.reporter.error( - path=self.object_path, - code='collections-no-underscore-on-deprecation', - msg='Deprecated content in collections MUST NOT start with "_", update meta/runtime.yml instead', - ) - if not (doc_deprecated == routing_says_deprecated): # DOCUMENTATION.deprecated and meta/runtime.yml disagree self.reporter.error( @@ -1193,7 +1164,7 @@ class ModuleValidator(Validator): for entry in object: self._validate_semantic_markup(entry) return - if not isinstance(object, string_types): + if not isinstance(object, str): return if self.collection: @@ -1312,7 +1283,12 @@ class ModuleValidator(Validator): def _validate_ansible_module_call(self, docs): try: - spec, kwargs = get_argument_spec(self.path, self.collection) + if self._python_module(): + spec, kwargs = get_py_argument_spec(self.path, self.collection) + elif self._powershell_module(): + spec, kwargs = get_ps_argument_spec(self.path, self.collection) + else: + raise NotImplementedError() except AnsibleModuleNotInitialized: self.reporter.error( path=self.object_path, @@ -1374,7 +1350,7 @@ class ModuleValidator(Validator): continue bad_term = False for term in check: - if not isinstance(term, string_types): + if not isinstance(term, str): msg = name if context: msg += " found in %s" % " -> ".join(context) @@ -1442,7 +1418,7 @@ class ModuleValidator(Validator): continue bad_term = False for term in requirements: - if not isinstance(term, string_types): + if not isinstance(term, str): msg = "required_if" if context: msg += " found in %s" % " -> ".join(context) @@ -1525,13 +1501,13 @@ class ModuleValidator(Validator): # This is already reported by schema checking return for key, value in terms.items(): - if isinstance(value, string_types): + if isinstance(value, str): value = [value] if not isinstance(value, (list, tuple)): # This is already reported by schema checking continue for term in value: - if not isinstance(term, string_types): + if not isinstance(term, str): # This is already reported by schema checking continue if len(set(value)) != len(value) or key in value: @@ -2268,7 +2244,6 @@ class ModuleValidator(Validator): 'extension for python modules or a .ps1 ' 'for powershell modules') ) - self._python_module_override = True if self._python_module() and self.ast is None: self.reporter.error( @@ -2380,10 +2355,7 @@ class ModuleValidator(Validator): self._check_gpl3_header() if not self._just_docs() and not self._sidecar_doc() and not end_of_deprecation_should_be_removed_only: if self.plugin_type == 'module': - self._check_interpreter(powershell=self._powershell_module()) - self._check_type_instead_of_isinstance( - powershell=self._powershell_module() - ) + self._check_interpreter() class PythonPackageValidator(Validator): diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py index 1b71217..bff9306 100644 --- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py @@ -167,10 +167,3 @@ def get_py_argument_spec(filename, collection): return argument_spec, fake.kwargs except (TypeError, IndexError): return {}, {} - - -def get_argument_spec(filename, collection): - if filename.endswith('.py'): - return get_py_argument_spec(filename, collection) - else: - return get_ps_argument_spec(filename, collection) diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py index a6068c6..ba4e188 100644 --- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py @@ -297,6 +297,7 @@ def argument_spec_schema(for_collection): [is_callable, list_string_types], ), 'choices': Any([object], (object,)), + 'context': dict, 'required': bool, 'no_log': bool, 'aliases': Any(list_string_types, tuple(list_string_types)), @@ -487,10 +488,17 @@ def check_option_choices(v): type_checker, type_name = get_type_checker({'type': v.get('elements')}) else: type_checker, type_name = get_type_checker(v) + if type_checker is None: return v - for value in v_choices: + if isinstance(v_choices, dict): + # choices are still a list (the keys) but dict form serves to document each choice. + iterate = v_choices.keys() + else: + iterate = v_choices + + for value in iterate: try: type_checker(value) except Exception as exc: @@ -542,7 +550,7 @@ def list_dict_option_schema(for_collection, plugin_type): basic_option_schema = { Required('description'): doc_string_or_strings, 'required': bool, - 'choices': list, + 'choices': Any(list, {object: doc_string_or_strings}), 'aliases': Any(list_string_types), 'version_added': version(for_collection), 'version_added_collection': collection_name, diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/utils.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/utils.py index 15cb703..84bab28 100644 --- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/utils.py +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/utils.py @@ -192,7 +192,7 @@ def compare_unordered_lists(a, b): - unordered lists - unhashable elements """ - return len(a) == len(b) and all(x in b for x in a) + return len(a) == len(b) and all(x in b for x in a) and all(x in a for x in b) class NoArgsAnsibleModule(AnsibleModule): diff --git a/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py b/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py index ed1afcf..0c39fc7 100644 --- a/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py +++ b/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py @@ -126,19 +126,31 @@ class YamlChecker: yaml_data = yaml_data[1:] lineno += 1 - self.check_parsable(path, yaml_data, lineno) + multiple_docs_allowed = [ + "EXAMPLES", + ] + self.check_parsable(path, yaml_data, lineno, (key in multiple_docs_allowed), key) messages = list(linter.run(yaml_data, conf, path)) self.messages += [self.result_to_message(r, path, lineno - 1, key) for r in messages] - def check_parsable(self, path, contents, lineno=1): # type: (str, str, int) -> None + def check_parsable(self, path, contents, lineno=1, allow_multiple=False, prefix=""): # type: (str, str, int, bool) -> None """Check the given contents to verify they can be parsed as YAML.""" + prefix = f"{prefix}: " if prefix else "" try: - yaml.load(contents, Loader=TestLoader) + documents = len(list(yaml.load_all(contents, Loader=TestLoader))) + if documents > 1 and not allow_multiple: + self.messages += [{'code': 'multiple-yaml-documents', + 'message': f'{prefix}expected a single document in the stream', + 'path': path, + 'line': lineno, + 'column': 1, + 'level': 'error', + }] except MarkedYAMLError as ex: self.messages += [{'code': 'unparsable-with-libyaml', - 'message': '%s - %s' % (ex.args[0], ex.args[2]), + 'message': f'{prefix}{ex.args[0]} - {ex.args[2]}', 'path': path, 'line': ex.problem_mark.line + lineno, 'column': ex.problem_mark.column + 1, diff --git a/test/lib/ansible_test/_util/controller/tools/sslcheck.py b/test/lib/ansible_test/_util/controller/tools/sslcheck.py deleted file mode 100644 index c25fed6..0000000 --- a/test/lib/ansible_test/_util/controller/tools/sslcheck.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Show openssl version.""" -from __future__ import annotations - -import json - -# noinspection PyBroadException -try: - from ssl import OPENSSL_VERSION_INFO - VERSION = list(OPENSSL_VERSION_INFO[:3]) -except Exception: # pylint: disable=broad-except - VERSION = None - - -def main(): - """Main program entry point.""" - print(json.dumps(dict( - version=VERSION, - ))) - - -if __name__ == '__main__': - main() diff --git a/test/lib/ansible_test/_util/target/__init__.py b/test/lib/ansible_test/_util/target/__init__.py index 527d413..8f58623 100644 --- a/test/lib/ansible_test/_util/target/__init__.py +++ b/test/lib/ansible_test/_util/target/__init__.py @@ -1,2 +1 @@ -# Empty __init__.py to allow importing of `ansible_test._util.target.common` under Python 2.x. -# This allows the ansible-test entry point to report supported Python versions before exiting. +# Empty __init__.py to keep pylint happy. diff --git a/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py b/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py index 930654f..9cb5d04 100755 --- a/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py +++ b/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py @@ -4,8 +4,7 @@ # NOTE: This file resides in the _util/target directory to ensure compatibility with all supported Python versions. -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations import os import sys diff --git a/test/lib/ansible_test/_util/target/common/__init__.py b/test/lib/ansible_test/_util/target/common/__init__.py deleted file mode 100644 index 527d413..0000000 --- a/test/lib/ansible_test/_util/target/common/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Empty __init__.py to allow importing of `ansible_test._util.target.common` under Python 2.x. -# This allows the ansible-test entry point to report supported Python versions before exiting. diff --git a/test/lib/ansible_test/_util/target/common/constants.py b/test/lib/ansible_test/_util/target/common/constants.py index 36a5a2c..fdad9f2 100644 --- a/test/lib/ansible_test/_util/target/common/constants.py +++ b/test/lib/ansible_test/_util/target/common/constants.py @@ -2,12 +2,9 @@ # NOTE: This file resides in the _util/target directory to ensure compatibility with all supported Python versions. -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations REMOTE_ONLY_PYTHON_VERSIONS = ( - '2.7', - '3.6', '3.7', '3.8', '3.9', diff --git a/test/lib/ansible_test/_util/target/injector/python.py b/test/lib/ansible_test/_util/target/injector/python.py index c1e88a9..82f59b1 100644 --- a/test/lib/ansible_test/_util/target/injector/python.py +++ b/test/lib/ansible_test/_util/target/injector/python.py @@ -1,8 +1,8 @@ # auto-shebang """Provides an entry point for python scripts and python modules on the controller with the current python interpreter and optional code coverage collection.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations +import importlib.util import os import sys @@ -19,22 +19,7 @@ def main(): if coverage_output: args += ['-m', 'coverage.__main__', 'run', '--rcfile', coverage_config] else: - if sys.version_info >= (3, 4): - # noinspection PyUnresolvedReferences - import importlib.util - - # noinspection PyUnresolvedReferences - found = bool(importlib.util.find_spec('coverage')) - else: - # noinspection PyDeprecation - import imp - - try: - # noinspection PyDeprecation - imp.find_module('coverage') - found = True - except ImportError: - found = False + found = bool(importlib.util.find_spec('coverage')) if not found: sys.exit('ERROR: Could not find `coverage` module. ' @@ -62,7 +47,7 @@ def find_program(name, executable): # type: (str, bool) -> str Raises an exception if the program is not found. """ path = os.environ.get('PATH', os.path.defpath) - seen = set([os.path.abspath(__file__)]) + seen = {os.path.abspath(__file__)} mode = os.F_OK | os.X_OK if executable else os.F_OK for base in path.split(os.path.pathsep): diff --git a/test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py index d00d9e9..9e98359 100644 --- a/test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py +++ b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py @@ -4,21 +4,14 @@ # https://github.com/pytest-dev/pytest-forked # https://github.com/pytest-dev/py # TIP: Disable pytest-xdist when debugging internal errors in this plugin. -from __future__ import absolute_import, division, print_function - -__metaclass__ = type +from __future__ import annotations import os import pickle import tempfile import warnings -from pytest import Item, hookimpl - -try: - from pytest import TestReport -except ImportError: - from _pytest.runner import TestReport # Backwards compatibility with pytest < 7. Remove once Python 2.7 is not supported. +from pytest import Item, hookimpl, TestReport from _pytest.runner import runtestprotocol diff --git a/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_collections.py b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_collections.py index 2f77c03..3aa2e12 100644 --- a/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_collections.py +++ b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_collections.py @@ -1,6 +1,5 @@ """Enable unit testing of Ansible collections. PYTEST_DONT_REWRITE""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations import os @@ -39,9 +38,6 @@ def enable_assertion_rewriting_hook(): # type: () -> None """ import sys - if sys.version_info[0] == 2: - return # Python 2.x is not supported - hook_name = '_pytest.assertion.rewrite.AssertionRewritingHook' hooks = [hook for hook in sys.meta_path if hook.__class__.__module__ + '.' + hook.__class__.__qualname__ == hook_name] diff --git a/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_coverage.py b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_coverage.py index b05298a..577a498 100644 --- a/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_coverage.py +++ b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_coverage.py @@ -1,6 +1,5 @@ """Monkey patch os._exit when running under coverage so we don't lose coverage data in forks, such as with `pytest --boxed`. PYTEST_DONT_REWRITE""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations def pytest_configure(): diff --git a/test/lib/ansible_test/_util/target/sanity/compile/compile.py b/test/lib/ansible_test/_util/target/sanity/compile/compile.py index bd2446f..3dfec39 100644 --- a/test/lib/ansible_test/_util/target/sanity/compile/compile.py +++ b/test/lib/ansible_test/_util/target/sanity/compile/compile.py @@ -1,12 +1,10 @@ """Python syntax checker with lint friendly output.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations import sys ENCODING = 'utf-8' ERRORS = 'replace' -Text = type(u'') def main(): @@ -29,21 +27,14 @@ def compile_source(path): else: return - # In some situations offset can be None. This can happen for syntax errors on Python 2.6 - # (__future__ import following after a regular import). - offset = offset or 0 - result = "%s:%d:%d: %s: %s" % (path, lineno, offset, extype.__name__, safe_message(message)) - if sys.version_info <= (3,): - result = result.encode(ENCODING, ERRORS) - print(result) def safe_message(value): - """Given an input value as text or bytes, return the first non-empty line as text, ensuring it can be round-tripped as UTF-8.""" - if isinstance(value, Text): + """Given an input value as str or bytes, return the first non-empty line as str, ensuring it can be round-tripped as UTF-8.""" + if isinstance(value, str): value = value.encode(ENCODING, ERRORS) value = value.decode(ENCODING, ERRORS) diff --git a/test/lib/ansible_test/_util/target/sanity/import/importer.py b/test/lib/ansible_test/_util/target/sanity/import/importer.py index 38a7364..28f3f85 100644 --- a/test/lib/ansible_test/_util/target/sanity/import/importer.py +++ b/test/lib/ansible_test/_util/target/sanity/import/importer.py @@ -1,6 +1,5 @@ """Import the given python module(s) and report error(s) encountered.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations def main(): @@ -542,16 +541,6 @@ def main(): "ignore", "AnsibleCollectionFinder has already been configured") - if sys.version_info[0] == 2: - warnings.filterwarnings( - "ignore", - "Python 2 is no longer supported by the Python core team. Support for it is now deprecated in cryptography," - " and will be removed in a future release.") - warnings.filterwarnings( - "ignore", - "Python 2 is no longer supported by the Python core team. Support for it is now deprecated in cryptography," - " and will be removed in the next release.") - # ansible.utils.unsafe_proxy attempts patching sys.intern generating a warning if it was already patched warnings.filterwarnings( "ignore", diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh index 65673da..16e6d38 100644 --- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -2,6 +2,16 @@ set -eu +remove_externally_managed_marker() +{ + "${python_interpreter}" -c ' +import pathlib +import sysconfig +path = pathlib.Path(sysconfig.get_path("stdlib")) / "EXTERNALLY-MANAGED" +path.unlink(missing_ok=True) +' +} + install_ssh_keys() { if [ ! -f "${ssh_private_key_path}" ]; then @@ -49,9 +59,6 @@ customize_bashrc() install_pip() { if ! "${python_interpreter}" -m pip.__main__ --version --disable-pip-version-check 2>/dev/null; then case "${python_version}" in - "2.7") - pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-20.3.4.py" - ;; *) pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-23.1.2.py" ;; @@ -68,18 +75,6 @@ install_pip() { fi } -pip_install() { - pip_packages="$1" - - while true; do - # shellcheck disable=SC2086 - "${python_interpreter}" -m pip install --disable-pip-version-check ${pip_packages} \ - && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 - done -} - bootstrap_remote_alpine() { py_pkg_prefix="py3" @@ -168,19 +163,39 @@ bootstrap_remote_freebsd() jinja2_pkg="py${python_package_version}-jinja2" cryptography_pkg="py${python_package_version}-cryptography" pyyaml_pkg="py${python_package_version}-yaml" + packaging_pkg="py${python_package_version}-packaging" # Declare platform/python version combinations which do not have supporting OS packages available. # For these combinations ansible-test will use pip to install the requirements instead. case "${platform_version}/${python_version}" in + 13.3/3.9) + # defaults above 'just work'TM + ;; + 13.3/3.11) + jinja2_pkg="" # not available + cryptography_pkg="" # not available + pyyaml_pkg="" # not available + ;; + 14.0/3.9) + # defaults above 'just work'TM + ;; + 14.0/3.11) + cryptography_pkg="" # not available + jinja2_pkg="" # not available + pyyaml_pkg="" # not available + ;; *) + # just assume nothing is available jinja2_pkg="" # not available cryptography_pkg="" # not available pyyaml_pkg="" # not available + packaging_pkg="" # not available ;; esac packages=" ${packages} + ${packaging_pkg} libyaml ${pyyaml_pkg} ${jinja2_pkg} @@ -242,79 +257,6 @@ bootstrap_remote_macos() echo 'PATH="/usr/local/bin:$PATH"' > /etc/zshenv } -bootstrap_remote_rhel_7() -{ - packages=" - gcc - python-devel - python-virtualenv - " - - while true; do - # shellcheck disable=SC2086 - yum install -q -y ${packages} \ - && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 - done - - install_pip - - bootstrap_remote_rhel_pinned_pip_packages -} - -bootstrap_remote_rhel_8() -{ - if [ "${python_version}" = "3.6" ]; then - py_pkg_prefix="python3" - else - py_pkg_prefix="python${python_version}" - fi - - packages=" - gcc - ${py_pkg_prefix}-devel - " - - # pip isn't included in the Python devel package under Python 3.11 - if [ "${python_version}" != "3.6" ]; then - packages=" - ${packages} - ${py_pkg_prefix}-pip - " - fi - - # Jinja2 is not installed with an OS package since the provided version is too old. - # Instead, ansible-test will install it using pip. - if [ "${controller}" ]; then - packages=" - ${packages} - ${py_pkg_prefix}-cryptography - " - fi - - # Python 3.11 isn't a module like the earlier versions - if [ "${python_version}" = "3.6" ]; then - while true; do - # shellcheck disable=SC2086 - yum module install -q -y "python${python_package_version}" \ - && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 - done - fi - - while true; do - # shellcheck disable=SC2086 - yum install -q -y ${packages} \ - && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 - done - - bootstrap_remote_rhel_pinned_pip_packages -} - bootstrap_remote_rhel_9() { if [ "${python_version}" = "3.9" ]; then @@ -360,23 +302,10 @@ bootstrap_remote_rhel_9() bootstrap_remote_rhel() { case "${platform_version}" in - 7.*) bootstrap_remote_rhel_7 ;; - 8.*) bootstrap_remote_rhel_8 ;; 9.*) bootstrap_remote_rhel_9 ;; esac } -bootstrap_remote_rhel_pinned_pip_packages() -{ - # pin packaging and pyparsing to match the downstream vendored versions - pip_packages=" - packaging==20.4 - pyparsing==2.4.7 - " - - pip_install "${pip_packages}" -} - bootstrap_remote_ubuntu() { py_pkg_prefix="python3" @@ -430,14 +359,27 @@ bootstrap_docker() { # Required for newer mysql-server packages to install/upgrade on Ubuntu 16.04. rm -f /usr/sbin/policy-rc.d + + for key_value in ${python_interpreters}; do + IFS=':' read -r python_version python_interpreter << EOF +${key_value} +EOF + + echo "Bootstrapping Python ${python_version} at: ${python_interpreter}" + + remove_externally_managed_marker + done } bootstrap_remote() { - for python_version in ${python_versions}; do - echo "Bootstrapping Python ${python_version}" + for key_value in ${python_interpreters}; do + IFS=':' read -r python_version python_interpreter << EOF +${key_value} +EOF + + echo "Bootstrapping Python ${python_version} at: ${python_interpreter}" - python_interpreter="python${python_version}" python_package_version="$(echo "${python_version}" | tr -d '.')" case "${platform}" in @@ -448,6 +390,8 @@ bootstrap_remote() "rhel") bootstrap_remote_rhel ;; "ubuntu") bootstrap_remote_ubuntu ;; esac + + remove_externally_managed_marker done } @@ -474,7 +418,7 @@ bootstrap_type=#{bootstrap_type} controller=#{controller} platform=#{platform} platform_version=#{platform_version} -python_versions=#{python_versions} +python_interpreters=#{python_interpreters} ssh_key_type=#{ssh_key_type} ssh_private_key=#{ssh_private_key} ssh_public_key=#{ssh_public_key} diff --git a/test/lib/ansible_test/_util/target/setup/probe_cgroups.py b/test/lib/ansible_test/_util/target/setup/probe_cgroups.py index 2ac7ecb..a09c024 100644 --- a/test/lib/ansible_test/_util/target/setup/probe_cgroups.py +++ b/test/lib/ansible_test/_util/target/setup/probe_cgroups.py @@ -1,6 +1,5 @@ """A tool for probing cgroups to determine write access.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations import json import os diff --git a/test/lib/ansible_test/_util/target/setup/quiet_pip.py b/test/lib/ansible_test/_util/target/setup/quiet_pip.py index 171ff8f..c2e9ba2 100644 --- a/test/lib/ansible_test/_util/target/setup/quiet_pip.py +++ b/test/lib/ansible_test/_util/target/setup/quiet_pip.py @@ -1,20 +1,17 @@ """Custom entry-point for pip that filters out unwanted logging and warnings.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations import logging import os import re import runpy import sys -import warnings BUILTIN_FILTERER_FILTER = logging.Filterer.filter LOGGING_MESSAGE_FILTER = re.compile("^(" ".*Running pip install with root privileges is generally not a good idea.*|" # custom Fedora patch [1] ".*Running pip as the 'root' user can result in broken permissions .*|" # pip 21.1 - "DEPRECATION: Python 2.7 will reach the end of its life .*|" # pip 19.2.3 "Ignoring .*: markers .* don't match your environment|" "Looking in indexes: .*|" # pypi-test-container "Requirement already satisfied.*" @@ -22,13 +19,6 @@ LOGGING_MESSAGE_FILTER = re.compile("^(" # [1] https://src.fedoraproject.org/rpms/python-pip/blob/f34/f/emit-a-warning-when-running-with-root-privileges.patch -WARNING_MESSAGE_FILTERS = ( - # DEPRECATION: Python 2.7 reached the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 is no longer maintained. - # pip 21.0 will drop support for Python 2.7 in January 2021. - # More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support - 'DEPRECATION: Python 2.7 reached the end of its life ', -) - def custom_filterer_filter(self, record): """Globally omit logging of unwanted messages.""" @@ -44,11 +34,6 @@ def main(): # It also avoids problems with loss of color output and mixing up the order of stdout/stderr messages. logging.Filterer.filter = custom_filterer_filter - for message_filter in WARNING_MESSAGE_FILTERS: - # Setting filterwarnings in code is necessary because of the following: - # Python 2.7 cannot use the -W option to match warning text after a colon. This makes it impossible to match specific warning messages. - warnings.filterwarnings('ignore', message_filter) - get_pip = os.environ.get('GET_PIP') try: diff --git a/test/lib/ansible_test/_util/target/setup/requirements.py b/test/lib/ansible_test/_util/target/setup/requirements.py index b145fde..a320dfc 100644 --- a/test/lib/ansible_test/_util/target/setup/requirements.py +++ b/test/lib/ansible_test/_util/target/setup/requirements.py @@ -1,6 +1,5 @@ """A tool for installing test requirements on the controller and target host.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations # pylint: disable=wrong-import-position @@ -250,6 +249,14 @@ def common_pip_environment(): # type: () -> t.Dict[str, str] """Return common environment variables used to run pip.""" env = os.environ.copy() + # When ansible-test installs requirements outside a virtual environment, it does so under one of two conditions: + # 1) The environment is an ephemeral one provisioned by ansible-test. + # 2) The user has provided the `--requirements` option to force installation of requirements. + # It seems reasonable to bypass PEP 668 checks in both of these cases. + # Doing so with an environment variable allows it to work under any version of pip which supports it, without breaking older versions. + # NOTE: pip version 23.0 enforces PEP 668 but does not support the override, in which case upgrading pip is required. + env.update(PIP_BREAK_SYSTEM_PACKAGES='1') + return env diff --git a/test/lib/ansible_test/_util/target/tools/virtualenvcheck.py b/test/lib/ansible_test/_util/target/tools/virtualenvcheck.py index a38ad07..49b308b 100644 --- a/test/lib/ansible_test/_util/target/tools/virtualenvcheck.py +++ b/test/lib/ansible_test/_util/target/tools/virtualenvcheck.py @@ -1,6 +1,5 @@ """Detect the real python interpreter when running in a virtual environment created by the 'virtualenv' module.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations import json diff --git a/test/lib/ansible_test/_util/target/tools/yamlcheck.py b/test/lib/ansible_test/_util/target/tools/yamlcheck.py index dfd08e5..07dccca 100644 --- a/test/lib/ansible_test/_util/target/tools/yamlcheck.py +++ b/test/lib/ansible_test/_util/target/tools/yamlcheck.py @@ -1,6 +1,5 @@ """Show availability of PyYAML and libyaml support.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type +from __future__ import annotations import json |