diff options
Diffstat (limited to 'test/lib/ansible_test/_util')
95 files changed, 9365 insertions, 0 deletions
diff --git a/test/lib/ansible_test/_util/__init__.py b/test/lib/ansible_test/_util/__init__.py new file mode 100644 index 0000000..527d413 --- /dev/null +++ b/test/lib/ansible_test/_util/__init__.py @@ -0,0 +1,2 @@ +# Empty __init__.py to allow importing of `ansible_test._util.target.common` under Python 2.x. +# This allows the ansible-test entry point to report supported Python versions before exiting. diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/action-plugin-docs.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/action-plugin-docs.json new file mode 100644 index 0000000..12bbe0d --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/action-plugin-docs.json @@ -0,0 +1,13 @@ +{ + "all_targets": true, + "prefixes": [ + "lib/ansible/modules/", + "lib/ansible/plugins/action/", + "plugins/modules/", + "plugins/action/" + ], + "extensions": [ + ".py" + ], + "output": "path-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/action-plugin-docs.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/action-plugin-docs.py new file mode 100644 index 0000000..a319d1a --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/action-plugin-docs.py @@ -0,0 +1,66 @@ +"""Test to verify action plugins have an associated module to provide documentation.""" +from __future__ import annotations + +import os +import sys + + +def main(): + """Main entry point.""" + paths = sys.argv[1:] or sys.stdin.read().splitlines() + + module_names = set() + + module_prefixes = { + 'lib/ansible/modules/': True, + 'plugins/modules/': False, + } + + action_prefixes = { + 'lib/ansible/plugins/action/': True, + 'plugins/action/': False, + } + + for path in paths: + full_name = get_full_name(path, module_prefixes) + + if full_name: + module_names.add(full_name) + + for path in paths: + full_name = get_full_name(path, action_prefixes) + + if full_name and full_name not in module_names: + print('%s: action plugin has no matching module to provide documentation' % path) + + +def get_full_name(path, prefixes): + """Return the full name of the plugin at the given path by matching against the given path prefixes, or None if no match is found.""" + for prefix, flat in prefixes.items(): + if path.startswith(prefix): + relative_path = os.path.relpath(path, prefix) + + if flat: + full_name = os.path.basename(relative_path) + else: + full_name = relative_path + + full_name = os.path.splitext(full_name)[0] + + name = os.path.basename(full_name) + + if name == '__init__': + return None + + if name.startswith('_'): + name = name[1:] + + full_name = os.path.join(os.path.dirname(full_name), name).replace(os.path.sep, '.') + + return full_name + + return None + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.json new file mode 100644 index 0000000..7d19f10 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.json @@ -0,0 +1,8 @@ +{ + "intercept": true, + "prefixes": [ + "changelogs/config.yaml", + "changelogs/fragments/" + ], + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.py new file mode 100644 index 0000000..924e5af --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.py @@ -0,0 +1,60 @@ +"""Check changelog fragment naming, syntax, etc.""" +from __future__ import annotations + +import os +import sys +import subprocess + + +def main(): + """Main entry point.""" + paths = sys.argv[1:] or sys.stdin.read().splitlines() + + allowed_extensions = ('.yml', '.yaml') + config_path = 'changelogs/config.yaml' + + # config must be detected independent of the file list since the file list only contains files under test (changed) + has_config = os.path.exists(config_path) + paths_to_check = [] + for path in paths: + if path == config_path: + continue + + if path.startswith('changelogs/fragments/.'): + if path in ('changelogs/fragments/.keep', 'changelogs/fragments/.gitkeep'): + continue + + print('%s:%d:%d: file must not be a dotfile' % (path, 0, 0)) + continue + + ext = os.path.splitext(path)[1] + + if ext not in allowed_extensions: + print('%s:%d:%d: extension must be one of: %s' % (path, 0, 0, ', '.join(allowed_extensions))) + + paths_to_check.append(path) + + if not has_config: + print('changelogs/config.yaml:0:0: config file does not exist') + return + + if not paths_to_check: + return + + cmd = [sys.executable, '-m', 'antsibull_changelog', 'lint'] + paths_to_check + + # The sphinx module is a soft dependency for rstcheck, which is used by the changelog linter. + # If sphinx is found it will be loaded by rstcheck, which can affect the results of the test. + # To maintain consistency across environments, loading of sphinx is blocked, since any version (or no version) of sphinx may be present. + env = os.environ.copy() + env.update(PYTHONPATH='%s:%s' % (os.path.join(os.path.dirname(__file__), 'changelog'), env['PYTHONPATH'])) + + # ignore the return code, rely on the output instead + process = subprocess.run(cmd, stdin=subprocess.DEVNULL, capture_output=True, text=True, env=env, check=False) + + sys.stdout.write(process.stdout) + sys.stderr.write(process.stderr) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog/sphinx.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog/sphinx.py new file mode 100644 index 0000000..7eab0f5 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog/sphinx.py @@ -0,0 +1,4 @@ +"""Block the sphinx module from being loaded.""" +from __future__ import annotations + +raise ImportError('The sphinx module has been prevented from loading to maintain consistent test results.') diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/empty-init.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/empty-init.json new file mode 100644 index 0000000..9835f9b --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/empty-init.json @@ -0,0 +1,14 @@ +{ + "prefixes": [ + "lib/ansible/modules/", + "lib/ansible/module_utils/", + "plugins/modules/", + "plugins/module_utils/", + "test/units/", + "tests/unit/" + ], + "files": [ + "__init__.py" + ], + "output": "path-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/empty-init.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/empty-init.py new file mode 100644 index 0000000..01aef69 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/empty-init.py @@ -0,0 +1,16 @@ +"""Require empty __init__.py files.""" +from __future__ import annotations + +import os +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + if os.path.getsize(path) > 0: + print('%s: empty __init__.py required' % path) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.json new file mode 100644 index 0000000..4ebce32 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.json @@ -0,0 +1,7 @@ +{ + "extensions": [ + ".py" + ], + "py2_compat": true, + "output": "path-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.py new file mode 100644 index 0000000..7b39c37 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.py @@ -0,0 +1,46 @@ +"""Enforce proper usage of __future__ imports.""" +from __future__ import annotations + +import ast +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'rb') as path_fd: + lines = path_fd.read().splitlines() + + missing = True + if not lines: + # Files are allowed to be empty of everything including boilerplate + missing = False + + for text in lines: + if text in (b'from __future__ import (absolute_import, division, print_function)', + b'from __future__ import absolute_import, division, print_function'): + missing = False + break + + if missing: + with open(path, encoding='utf-8') as file: + contents = file.read() + + # noinspection PyBroadException + try: + node = ast.parse(contents) + + # files consisting of only assignments have no need for future import boilerplate + # the only exception would be division during assignment, but we'll overlook that for simplicity + # the most likely case is that of a documentation only python file + if all(isinstance(statement, ast.Assign) for statement in node.body): + missing = False + except Exception: # pylint: disable=broad-except + pass # the compile sanity test will report this error + + if missing: + print('%s: missing: from __future__ import (absolute_import, division, print_function)' % path) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/line-endings.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/line-endings.json new file mode 100644 index 0000000..db5c3c9 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/line-endings.json @@ -0,0 +1,4 @@ +{ + "text": true, + "output": "path-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/line-endings.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/line-endings.py new file mode 100644 index 0000000..31f97ad --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/line-endings.py @@ -0,0 +1,18 @@ +"""Require Unix line endings.""" +from __future__ import annotations + +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'rb') as path_fd: + contents = path_fd.read() + + if b'\r' in contents: + print('%s: use "\\n" for line endings instead of "\\r\\n"' % path) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.json new file mode 100644 index 0000000..4ebce32 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.json @@ -0,0 +1,7 @@ +{ + "extensions": [ + ".py" + ], + "py2_compat": true, + "output": "path-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.py new file mode 100644 index 0000000..8bdcfc9 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.py @@ -0,0 +1,44 @@ +"""Require __metaclass__ boilerplate for code that supports Python 2.x.""" +from __future__ import annotations + +import ast +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'rb') as path_fd: + lines = path_fd.read().splitlines() + + missing = True + if not lines: + # Files are allowed to be empty of everything including boilerplate + missing = False + + for text in lines: + if text == b'__metaclass__ = type': + missing = False + break + + if missing: + with open(path, encoding='utf-8') as file: + contents = file.read() + + # noinspection PyBroadException + try: + node = ast.parse(contents) + + # files consisting of only assignments have no need for metaclass boilerplate + # the most likely case is that of a documentation only python file + if all(isinstance(statement, ast.Assign) for statement in node.body): + missing = False + except Exception: # pylint: disable=broad-except + pass # the compile sanity test will report this error + + if missing: + print('%s: missing: __metaclass__ = type' % path) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-assert.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-assert.json new file mode 100644 index 0000000..ccee80a --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-assert.json @@ -0,0 +1,10 @@ +{ + "extensions": [ + ".py" + ], + "prefixes": [ + "lib/ansible/", + "plugins/" + ], + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-assert.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-assert.py new file mode 100644 index 0000000..8c1c027 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-assert.py @@ -0,0 +1,24 @@ +"""Disallow use of assert.""" +from __future__ import annotations + +import re +import sys + +ASSERT_RE = re.compile(r'^\s*assert[^a-z0-9_:]') + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'r', encoding='utf-8') as file: + for i, line in enumerate(file.readlines()): + matches = ASSERT_RE.findall(line) + + if matches: + lineno = i + 1 + colno = line.index('assert') + 1 + print('%s:%d:%d: raise AssertionError instead of: %s' % (path, lineno, colno, matches[0][colno - 1:])) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.json new file mode 100644 index 0000000..88858ae --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.json @@ -0,0 +1,7 @@ +{ + "extensions": [ + ".py" + ], + "ignore_self": true, + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.py new file mode 100644 index 0000000..74e38d7 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.py @@ -0,0 +1,21 @@ +"""Disallow use of basestring isinstance checks.""" +from __future__ import annotations + +import re +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'r', encoding='utf-8') as path_fd: + for line, text in enumerate(path_fd.readlines()): + match = re.search(r'(isinstance.*basestring)', text) + + if match: + print('%s:%d:%d: do not use `isinstance(s, basestring)`' % ( + path, line + 1, match.start(1) + 1)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.json new file mode 100644 index 0000000..88858ae --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.json @@ -0,0 +1,7 @@ +{ + "extensions": [ + ".py" + ], + "ignore_self": true, + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.py new file mode 100644 index 0000000..b4e4002 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.py @@ -0,0 +1,21 @@ +"""Disallow use of the dict.iteritems function.""" +from __future__ import annotations + +import re +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'r', encoding='utf-8') as path_fd: + for line, text in enumerate(path_fd.readlines()): + match = re.search(r'(?<! six)\.(iteritems)', text) + + if match: + print('%s:%d:%d: use `dict.items` or `ansible.module_utils.six.iteritems` instead of `dict.iteritems`' % ( + path, line + 1, match.start(1) + 1)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.json new file mode 100644 index 0000000..88858ae --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.json @@ -0,0 +1,7 @@ +{ + "extensions": [ + ".py" + ], + "ignore_self": true, + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.py new file mode 100644 index 0000000..00c8703 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.py @@ -0,0 +1,21 @@ +"""Disallow use of the dict.iterkeys function.""" +from __future__ import annotations + +import re +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'r', encoding='utf-8') as path_fd: + for line, text in enumerate(path_fd.readlines()): + match = re.search(r'\.(iterkeys)', text) + + if match: + print('%s:%d:%d: use `dict.keys` or `for key in dict:` instead of `dict.iterkeys`' % ( + path, line + 1, match.start(1) + 1)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.json new file mode 100644 index 0000000..88858ae --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.json @@ -0,0 +1,7 @@ +{ + "extensions": [ + ".py" + ], + "ignore_self": true, + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.py new file mode 100644 index 0000000..2e8036a --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.py @@ -0,0 +1,21 @@ +"""Disallow use of the dict.itervalues function.""" +from __future__ import annotations + +import re +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'r', encoding='utf-8') as path_fd: + for line, text in enumerate(path_fd.readlines()): + match = re.search(r'(?<! six)\.(itervalues)', text) + + if match: + print('%s:%d:%d: use `dict.values` or `ansible.module_utils.six.itervalues` instead of `dict.itervalues`' % ( + path, line + 1, match.start(1) + 1)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-get-exception.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-get-exception.json new file mode 100644 index 0000000..88858ae --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-get-exception.json @@ -0,0 +1,7 @@ +{ + "extensions": [ + ".py" + ], + "ignore_self": true, + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-get-exception.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-get-exception.py new file mode 100644 index 0000000..0abb23d --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-get-exception.py @@ -0,0 +1,28 @@ +"""Disallow use of the get_exception function.""" +from __future__ import annotations + +import re +import sys + + +def main(): + """Main entry point.""" + basic_allow_once = True + + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'r', encoding='utf-8') as path_fd: + for line, text in enumerate(path_fd.readlines()): + match = re.search(r'([^a-zA-Z0-9_]get_exception[^a-zA-Z0-9_])', text) + + if match: + if path == 'lib/ansible/module_utils/basic.py' and basic_allow_once: + # basic.py is allowed to import get_exception for backwards compatibility but should not call it anywhere + basic_allow_once = False + continue + + print('%s:%d:%d: do not use `get_exception`' % ( + path, line + 1, match.start(1) + 1)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-illegal-filenames.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-illegal-filenames.json new file mode 100644 index 0000000..6f13c86 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-illegal-filenames.json @@ -0,0 +1,5 @@ +{ + "include_directories": true, + "include_symlinks": true, + "output": "path-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-illegal-filenames.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-illegal-filenames.py new file mode 100644 index 0000000..10bf4aa --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-illegal-filenames.py @@ -0,0 +1,83 @@ +""" +Check for illegal filenames on various operating systems. +The main rules are derived from restrictions on Windows: +https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions +""" +from __future__ import annotations + +import os +import struct +import sys + +from ansible.module_utils.basic import to_bytes + +ILLEGAL_CHARS = [ + b'<', + b'>', + b':', + b'"', + b'/', + b'\\', + b'|', + b'?', + b'*' +] + [struct.pack("b", i) for i in range(32)] + +ILLEGAL_NAMES = [ + "CON", + "PRN", + "AUX", + "NUL", + "COM1", + "COM2", + "COM3", + "COM4", + "COM5", + "COM6", + "COM7", + "COM8", + "COM9", + "LPT1", + "LPT2", + "LPT3", + "LPT4", + "LPT5", + "LPT6", + "LPT7", + "LPT8", + "LPT9", +] + +ILLEGAL_END_CHARS = [ + '.', + ' ', +] + + +def check_path(path, is_dir=False): + """Check the specified path for unwanted characters and names.""" + type_name = 'directory' if is_dir else 'file' + file_name = os.path.basename(path.rstrip(os.path.sep)) + name = os.path.splitext(file_name)[0] + + if name.upper() in ILLEGAL_NAMES: + print("%s: illegal %s name %s" % (path, type_name, name.upper())) + + if file_name[-1] in ILLEGAL_END_CHARS: + print("%s: illegal %s name end-char '%s'" % (path, type_name, file_name[-1])) + + bfile = to_bytes(file_name, encoding='utf-8') + for char in ILLEGAL_CHARS: + if char in bfile: + bpath = to_bytes(path, encoding='utf-8') + print("%s: illegal char '%s' in %s name" % (bpath, char, type_name)) + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + check_path(path, is_dir=path.endswith(os.path.sep)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.json new file mode 100644 index 0000000..ccee80a --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.json @@ -0,0 +1,10 @@ +{ + "extensions": [ + ".py" + ], + "prefixes": [ + "lib/ansible/", + "plugins/" + ], + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.py new file mode 100644 index 0000000..eb5987d --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.py @@ -0,0 +1,21 @@ +"""Disallow importing display from __main__.""" +from __future__ import annotations + +import sys + +MAIN_DISPLAY_IMPORT = 'from __main__ import display' + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'r', encoding='utf-8') as file: + for i, line in enumerate(file.readlines()): + if MAIN_DISPLAY_IMPORT in line: + lineno = i + 1 + colno = line.index(MAIN_DISPLAY_IMPORT) + 1 + print('%s:%d:%d: Display is a singleton, just import and instantiate' % (path, lineno, colno)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-smart-quotes.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-smart-quotes.json new file mode 100644 index 0000000..54d9fff --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-smart-quotes.json @@ -0,0 +1,5 @@ +{ + "text": true, + "ignore_self": true, + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-smart-quotes.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-smart-quotes.py new file mode 100644 index 0000000..461033d --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-smart-quotes.py @@ -0,0 +1,28 @@ +"""Disallow use of Unicode quotes.""" +# -*- coding: utf-8 -*- +from __future__ import annotations + +import re +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'rb') as path_fd: + for line, text in enumerate(path_fd.readlines()): + try: + text = text.decode('utf-8') + except UnicodeDecodeError as ex: + print('%s:%d:%d: UnicodeDecodeError: %s' % (path, line + 1, ex.start + 1, ex)) + continue + + match = re.search('([‘’“”])', text) + + if match: + print('%s:%d:%d: use ASCII quotes `\'` and `"` instead of Unicode quotes' % ( + path, line + 1, match.start(1) + 1)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.json new file mode 100644 index 0000000..88858ae --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.json @@ -0,0 +1,7 @@ +{ + "extensions": [ + ".py" + ], + "ignore_self": true, + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.py new file mode 100644 index 0000000..75c34f2 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.py @@ -0,0 +1,21 @@ +"""Disallow use of the unicode_literals future.""" +from __future__ import annotations + +import re +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'r', encoding='utf-8') as path_fd: + for line, text in enumerate(path_fd.readlines()): + match = re.search(r'(unicode_literals)', text) + + if match: + print('%s:%d:%d: do not use `unicode_literals`' % ( + path, line + 1, match.start(1) + 1)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/replace-urlopen.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/replace-urlopen.json new file mode 100644 index 0000000..88858ae --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/replace-urlopen.json @@ -0,0 +1,7 @@ +{ + "extensions": [ + ".py" + ], + "ignore_self": true, + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/replace-urlopen.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/replace-urlopen.py new file mode 100644 index 0000000..a6dd5aa --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/replace-urlopen.py @@ -0,0 +1,21 @@ +"""Disallow use of the urlopen function.""" +from __future__ import annotations + +import re +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'r', encoding='utf-8') as path_fd: + for line, text in enumerate(path_fd.readlines()): + match = re.search(r'^(?:[^#]*?)(urlopen)', text) + + if match: + print('%s:%d:%d: use `ansible.module_utils.urls.open_url` instead of `urlopen`' % ( + path, line + 1, match.start(1) + 1)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/runtime-metadata.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/runtime-metadata.json new file mode 100644 index 0000000..44003ec --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/runtime-metadata.json @@ -0,0 +1,11 @@ +{ + "prefixes": [ + "lib/ansible/config/ansible_builtin_runtime.yml", + "meta/routing.yml", + "meta/runtime.yml" + ], + "extensions": [ + ".yml" + ], + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/runtime-metadata.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/runtime-metadata.py new file mode 100644 index 0000000..6cf2777 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/runtime-metadata.py @@ -0,0 +1,277 @@ +"""Schema validation of ansible-core's ansible_builtin_runtime.yml and collection's meta/runtime.yml""" +from __future__ import annotations + +import datetime +import os +import re +import sys + +from functools import partial + +import yaml + +from voluptuous import All, Any, MultipleInvalid, PREVENT_EXTRA +from voluptuous import Required, Schema, Invalid +from voluptuous.humanize import humanize_error + +from ansible.module_utils.compat.version import StrictVersion, LooseVersion +from ansible.module_utils.six import string_types +from ansible.utils.version import SemanticVersion + + +def isodate(value, check_deprecation_date=False, is_tombstone=False): + """Validate a datetime.date or ISO 8601 date string.""" + # datetime.date objects come from YAML dates, these are ok + if isinstance(value, datetime.date): + removal_date = value + else: + # make sure we have a string + msg = 'Expected ISO 8601 date string (YYYY-MM-DD), or YAML date' + if not isinstance(value, string_types): + raise Invalid(msg) + # From Python 3.7 in, there is datetime.date.fromisoformat(). For older versions, + # we have to do things manually. + if not re.match('^[0-9]{4}-[0-9]{2}-[0-9]{2}$', value): + raise Invalid(msg) + try: + removal_date = datetime.datetime.strptime(value, '%Y-%m-%d').date() + except ValueError: + raise Invalid(msg) + # Make sure date is correct + today = datetime.date.today() + if is_tombstone: + # For a tombstone, the removal date must be in the past + if today < removal_date: + raise Invalid( + 'The tombstone removal_date (%s) must not be after today (%s)' % (removal_date, today)) + else: + # For a deprecation, the removal date must be in the future. Only test this if + # check_deprecation_date is truish, to avoid checks to suddenly start to fail. + if check_deprecation_date and today > removal_date: + raise Invalid( + 'The deprecation removal_date (%s) must be after today (%s)' % (removal_date, today)) + return value + + +def removal_version(value, is_ansible, current_version=None, is_tombstone=False): + """Validate a removal version string.""" + msg = ( + 'Removal version must be a string' if is_ansible else + 'Removal version must be a semantic version (https://semver.org/)' + ) + if not isinstance(value, string_types): + raise Invalid(msg) + try: + if is_ansible: + version = StrictVersion() + version.parse(value) + version = LooseVersion(value) # We're storing Ansible's version as a LooseVersion + else: + version = SemanticVersion() + version.parse(value) + if version.major != 0 and (version.minor != 0 or version.patch != 0): + raise Invalid('removal_version (%r) must be a major release, not a minor or patch release ' + '(see specification at https://semver.org/)' % (value, )) + if current_version is not None: + if is_tombstone: + # For a tombstone, the removal version must not be in the future + if version > current_version: + raise Invalid('The tombstone removal_version (%r) must not be after the ' + 'current version (%s)' % (value, current_version)) + else: + # For a deprecation, the removal version must be in the future + if version <= current_version: + raise Invalid('The deprecation removal_version (%r) must be after the ' + 'current version (%s)' % (value, current_version)) + except ValueError: + raise Invalid(msg) + return value + + +def any_value(value): + """Accepts anything.""" + return value + + +def get_ansible_version(): + """Return current ansible-core version""" + from ansible.release import __version__ + + return LooseVersion('.'.join(__version__.split('.')[:3])) + + +def get_collection_version(): + """Return current collection version, or None if it is not available""" + import importlib.util + + collection_detail_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'tools', 'collection_detail.py') + collection_detail_spec = importlib.util.spec_from_file_location('collection_detail', collection_detail_path) + collection_detail = importlib.util.module_from_spec(collection_detail_spec) + sys.modules['collection_detail'] = collection_detail + collection_detail_spec.loader.exec_module(collection_detail) + + # noinspection PyBroadException + try: + result = collection_detail.read_manifest_json('.') or collection_detail.read_galaxy_yml('.') + return SemanticVersion(result['version']) + except Exception: # pylint: disable=broad-except + # We do not care why it fails, in case we cannot get the version + # just return None to indicate "we don't know". + return None + + +def validate_metadata_file(path, is_ansible, check_deprecation_dates=False): + """Validate explicit runtime metadata file""" + try: + with open(path, 'r', encoding='utf-8') as f_path: + routing = yaml.safe_load(f_path) + except yaml.error.MarkedYAMLError as ex: + print('%s:%d:%d: YAML load failed: %s' % (path, ex.context_mark.line + + 1, ex.context_mark.column + 1, re.sub(r'\s+', ' ', str(ex)))) + return + except Exception as ex: # pylint: disable=broad-except + print('%s:%d:%d: YAML load failed: %s' % + (path, 0, 0, re.sub(r'\s+', ' ', str(ex)))) + return + + if is_ansible: + current_version = get_ansible_version() + else: + current_version = get_collection_version() + + # Updates to schema MUST also be reflected in the documentation + # ~https://docs.ansible.com/ansible-core/devel/dev_guide/developing_collections.html + + # plugin_routing schema + + avoid_additional_data = Schema( + Any( + { + Required('removal_version'): any_value, + 'warning_text': any_value, + }, + { + Required('removal_date'): any_value, + 'warning_text': any_value, + } + ), + extra=PREVENT_EXTRA + ) + + deprecation_schema = All( + # The first schema validates the input, and the second makes sure no extra keys are specified + Schema( + { + 'removal_version': partial(removal_version, is_ansible=is_ansible, + current_version=current_version), + 'removal_date': partial(isodate, check_deprecation_date=check_deprecation_dates), + 'warning_text': Any(*string_types), + } + ), + avoid_additional_data + ) + + tombstoning_schema = All( + # The first schema validates the input, and the second makes sure no extra keys are specified + Schema( + { + 'removal_version': partial(removal_version, is_ansible=is_ansible, + current_version=current_version, is_tombstone=True), + 'removal_date': partial(isodate, is_tombstone=True), + 'warning_text': Any(*string_types), + } + ), + avoid_additional_data + ) + + plugin_routing_schema = Any( + Schema({ + ('deprecation'): Any(deprecation_schema), + ('tombstone'): Any(tombstoning_schema), + ('redirect'): Any(*string_types), + }, extra=PREVENT_EXTRA), + ) + + list_dict_plugin_routing_schema = [{str_type: plugin_routing_schema} + for str_type in string_types] + + plugin_schema = Schema({ + ('action'): Any(None, *list_dict_plugin_routing_schema), + ('become'): Any(None, *list_dict_plugin_routing_schema), + ('cache'): Any(None, *list_dict_plugin_routing_schema), + ('callback'): Any(None, *list_dict_plugin_routing_schema), + ('cliconf'): Any(None, *list_dict_plugin_routing_schema), + ('connection'): Any(None, *list_dict_plugin_routing_schema), + ('doc_fragments'): Any(None, *list_dict_plugin_routing_schema), + ('filter'): Any(None, *list_dict_plugin_routing_schema), + ('httpapi'): Any(None, *list_dict_plugin_routing_schema), + ('inventory'): Any(None, *list_dict_plugin_routing_schema), + ('lookup'): Any(None, *list_dict_plugin_routing_schema), + ('module_utils'): Any(None, *list_dict_plugin_routing_schema), + ('modules'): Any(None, *list_dict_plugin_routing_schema), + ('netconf'): Any(None, *list_dict_plugin_routing_schema), + ('shell'): Any(None, *list_dict_plugin_routing_schema), + ('strategy'): Any(None, *list_dict_plugin_routing_schema), + ('terminal'): Any(None, *list_dict_plugin_routing_schema), + ('test'): Any(None, *list_dict_plugin_routing_schema), + ('vars'): Any(None, *list_dict_plugin_routing_schema), + }, extra=PREVENT_EXTRA) + + # import_redirection schema + + import_redirection_schema = Any( + Schema({ + ('redirect'): Any(*string_types), + # import_redirect doesn't currently support deprecation + }, extra=PREVENT_EXTRA) + ) + + list_dict_import_redirection_schema = [{str_type: import_redirection_schema} + for str_type in string_types] + + # top level schema + + schema = Schema({ + # All of these are optional + ('plugin_routing'): Any(plugin_schema), + ('import_redirection'): Any(None, *list_dict_import_redirection_schema), + # requires_ansible: In the future we should validate this with SpecifierSet + ('requires_ansible'): Any(*string_types), + ('action_groups'): dict, + }, extra=PREVENT_EXTRA) + + # Ensure schema is valid + + try: + schema(routing) + except MultipleInvalid as ex: + for error in ex.errors: + # No way to get line/column numbers + print('%s:%d:%d: %s' % (path, 0, 0, humanize_error(routing, error))) + + +def main(): + """Main entry point.""" + paths = sys.argv[1:] or sys.stdin.read().splitlines() + + collection_legacy_file = 'meta/routing.yml' + collection_runtime_file = 'meta/runtime.yml' + + # This is currently disabled, because if it is enabled this test can start failing + # at a random date. For this to be properly activated, we (a) need to be able to return + # codes for this test, and (b) make this error optional. + check_deprecation_dates = False + + for path in paths: + if path == collection_legacy_file: + print('%s:%d:%d: %s' % (path, 0, 0, ("Should be called '%s'" % collection_runtime_file))) + continue + + validate_metadata_file( + path, + is_ansible=path not in (collection_legacy_file, collection_runtime_file), + check_deprecation_dates=check_deprecation_dates) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.json new file mode 100644 index 0000000..5648429 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.json @@ -0,0 +1,4 @@ +{ + "text": true, + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py new file mode 100644 index 0000000..b0b1319 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py @@ -0,0 +1,124 @@ +"""Check shebangs, execute bits and byte order marks.""" +from __future__ import annotations + +import os +import re +import stat +import sys + + +def main(): + """Main entry point.""" + standard_shebangs = set([ + b'#!/bin/bash -eu', + b'#!/bin/bash -eux', + b'#!/bin/sh', + b'#!/usr/bin/env bash', + b'#!/usr/bin/env fish', + b'#!/usr/bin/env pwsh', + b'#!/usr/bin/env python', + b'#!/usr/bin/make -f', + ]) + + integration_shebangs = set([ + b'#!/bin/sh', + b'#!/usr/bin/env bash', + b'#!/usr/bin/env python', + ]) + + module_shebangs = { + '': b'#!/usr/bin/python', + '.py': b'#!/usr/bin/python', + '.ps1': b'#!powershell', + } + + # see https://unicode.org/faq/utf_bom.html#bom1 + byte_order_marks = ( + (b'\x00\x00\xFE\xFF', 'UTF-32 (BE)'), + (b'\xFF\xFE\x00\x00', 'UTF-32 (LE)'), + (b'\xFE\xFF', 'UTF-16 (BE)'), + (b'\xFF\xFE', 'UTF-16 (LE)'), + (b'\xEF\xBB\xBF', 'UTF-8'), + ) + + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'rb') as path_fd: + shebang = path_fd.readline().strip() + mode = os.stat(path).st_mode + executable = (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) & mode + + if not shebang or not shebang.startswith(b'#!'): + if executable: + print('%s:%d:%d: file without shebang should not be executable' % (path, 0, 0)) + + for mark, name in byte_order_marks: + if shebang.startswith(mark): + print('%s:%d:%d: file starts with a %s byte order mark' % (path, 0, 0, name)) + break + + continue + + is_module = False + is_integration = False + + dirname = os.path.dirname(path) + + if path.startswith('lib/ansible/modules/'): + is_module = True + elif re.search('^test/support/[^/]+/plugins/modules/', path): + is_module = True + elif re.search('^test/support/[^/]+/collections/ansible_collections/[^/]+/[^/]+/plugins/modules/', path): + is_module = True + elif path == 'test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py': + pass # ansible-test entry point must be executable and have a shebang + elif re.search(r'^lib/ansible/cli/[^/]+\.py', path): + pass # cli entry points must be executable and have a shebang + elif path.startswith('examples/'): + continue # examples trigger some false positives due to location + elif path.startswith('lib/') or path.startswith('test/lib/'): + if executable: + print('%s:%d:%d: should not be executable' % (path, 0, 0)) + + if shebang: + print('%s:%d:%d: should not have a shebang' % (path, 0, 0)) + + continue + elif path.startswith('test/integration/targets/') or path.startswith('tests/integration/targets/'): + is_integration = True + + if dirname.endswith('/library') or '/plugins/modules' in dirname or dirname in ( + # non-standard module library directories + 'test/integration/targets/module_precedence/lib_no_extension', + 'test/integration/targets/module_precedence/lib_with_extension', + ): + is_module = True + elif path.startswith('plugins/modules/'): + is_module = True + + if is_module: + if executable: + print('%s:%d:%d: module should not be executable' % (path, 0, 0)) + + ext = os.path.splitext(path)[1] + expected_shebang = module_shebangs.get(ext) + expected_ext = ' or '.join(['"%s"' % k for k in module_shebangs]) + + if expected_shebang: + if shebang == expected_shebang: + continue + + print('%s:%d:%d: expected module shebang "%s" but found: %s' % (path, 1, 1, expected_shebang, shebang)) + else: + print('%s:%d:%d: expected module extension %s but found: %s' % (path, 0, 0, expected_ext, ext)) + else: + if is_integration: + allowed = integration_shebangs + else: + allowed = standard_shebangs + + if shebang not in allowed: + print('%s:%d:%d: unexpected non-module shebang: %s' % (path, 1, 1, shebang)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/symlinks.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/symlinks.json new file mode 100644 index 0000000..6f13c86 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/symlinks.json @@ -0,0 +1,5 @@ +{ + "include_directories": true, + "include_symlinks": true, + "output": "path-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/symlinks.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/symlinks.py new file mode 100644 index 0000000..5cffc69 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/symlinks.py @@ -0,0 +1,32 @@ +"""Check for unwanted symbolic links.""" +from __future__ import annotations + +import os +import sys + + +def main(): + """Main entry point.""" + root_dir = os.getcwd() + os.path.sep + + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + if not os.path.islink(path.rstrip(os.path.sep)): + continue + + if not os.path.exists(path): + print('%s: broken symlinks are not allowed' % path) + continue + + if path.endswith(os.path.sep): + print('%s: symlinks to directories are not allowed' % path) + continue + + real_path = os.path.realpath(path) + + if not real_path.startswith(root_dir): + print('%s: symlinks outside content tree are not allowed: %s' % (path, os.path.relpath(real_path, os.path.dirname(path)))) + continue + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/use-argspec-type-path.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/use-argspec-type-path.json new file mode 100644 index 0000000..3610305 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/use-argspec-type-path.json @@ -0,0 +1,10 @@ +{ + "prefixes": [ + "lib/ansible/modules/", + "plugins/modules/" + ], + "extensions": [ + ".py" + ], + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/use-argspec-type-path.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/use-argspec-type-path.py new file mode 100644 index 0000000..0faeff3 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/use-argspec-type-path.py @@ -0,0 +1,21 @@ +"""Disallow use of the expanduser function.""" +from __future__ import annotations + +import re +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'r', encoding='utf-8') as path_fd: + for line, text in enumerate(path_fd.readlines()): + match = re.search(r'(expanduser)', text) + + if match: + print('%s:%d:%d: use argspec type="path" instead of type="str" to avoid use of `expanduser`' % ( + path, line + 1, match.start(1) + 1)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/use-compat-six.json b/test/lib/ansible_test/_util/controller/sanity/code-smell/use-compat-six.json new file mode 100644 index 0000000..776590b --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/use-compat-six.json @@ -0,0 +1,6 @@ +{ + "extensions": [ + ".py" + ], + "output": "path-line-column-message" +} diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/use-compat-six.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/use-compat-six.py new file mode 100644 index 0000000..db42fec --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/use-compat-six.py @@ -0,0 +1,21 @@ +"""Disallow importing of the six module.""" +from __future__ import annotations + +import re +import sys + + +def main(): + """Main entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + with open(path, 'r', encoding='utf-8') as path_fd: + for line, text in enumerate(path_fd.readlines()): + match = re.search(r'((^\s*import\s+six\b)|(^\s*from\s+six\b))', text) + + if match: + print('%s:%d:%d: use `ansible.module_utils.six` instead of `six`' % ( + path, line + 1, match.start(1) + 1)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/integration-aliases/yaml_to_json.py b/test/lib/ansible_test/_util/controller/sanity/integration-aliases/yaml_to_json.py new file mode 100644 index 0000000..af11dd8 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/integration-aliases/yaml_to_json.py @@ -0,0 +1,14 @@ +"""Read YAML from stdin and write JSON to stdout.""" +from __future__ import annotations + +import json +import sys + +from yaml import load + +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + +json.dump(load(sys.stdin, Loader=SafeLoader), sys.stdout) diff --git a/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-core.ini b/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-core.ini new file mode 100644 index 0000000..4d93f35 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-core.ini @@ -0,0 +1,119 @@ +# IMPORTANT +# Set "ignore_missing_imports" per package below, rather than globally. +# That will help identify missing type stubs that should be added to the sanity test environment. + +[mypy] +# There are ~20 errors reported in ansible-core when strict optional checking is enabled. +# Until the number of occurrences are reduced, it's better to disable strict checking. +strict_optional = False +# There are ~70 errors reported in ansible-core when checking attributes. +# Until the number of occurrences are reduced, it's better to disable the check. +disable_error_code = attr-defined + +[mypy-ansible.module_utils.six.moves.*] +ignore_missing_imports = True + +[mypy-passlib.*] +ignore_missing_imports = True + +[mypy-pexpect.*] +ignore_missing_imports = True + +[mypy-pypsrp.*] +ignore_missing_imports = True + +[mypy-winrm.*] +ignore_missing_imports = True + +[mypy-kerberos.*] +ignore_missing_imports = True + +[mypy-xmltodict.*] +ignore_missing_imports = True + +[mypy-md5.*] +ignore_missing_imports = True + +[mypy-scp.*] +ignore_missing_imports = True + +[mypy-ncclient.*] +ignore_missing_imports = True + +[mypy-lxml.*] +ignore_missing_imports = True + +[mypy-yum.*] +ignore_missing_imports = True + +[mypy-rpmUtils.*] +ignore_missing_imports = True + +[mypy-rpm.*] +ignore_missing_imports = True + +[mypy-psutil.*] +ignore_missing_imports = True + +[mypy-dnf.*] +ignore_missing_imports = True + +[mypy-apt.*] +ignore_missing_imports = True + +[mypy-apt_pkg.*] +ignore_missing_imports = True + +[mypy-gssapi.*] +ignore_missing_imports = True + +[mypy-_ssl.*] +ignore_missing_imports = True + +[mypy-urllib_gssapi.*] +ignore_missing_imports = True + +[mypy-systemd.*] +ignore_missing_imports = True + +[mypy-sha.*] +ignore_missing_imports = True + +[mypy-distro.*] +ignore_missing_imports = True + +[mypy-selectors2.*] +ignore_missing_imports = True + +[mypy-resolvelib.*] +ignore_missing_imports = True + +[mypy-urlparse.*] +ignore_missing_imports = True + +[mypy-argcomplete.*] +ignore_missing_imports = True + +[mypy-selinux.*] +ignore_missing_imports = True + +[mypy-urllib2.*] +ignore_missing_imports = True + +[mypy-httplib.*] +ignore_missing_imports = True + +[mypy-compiler.*] +ignore_missing_imports = True + +[mypy-aptsources.*] +ignore_missing_imports = True + +[mypy-urllib3.*] +ignore_missing_imports = True + +[mypy-requests.*] +ignore_missing_imports = True + +[mypy-jinja2.nativetypes] +ignore_missing_imports = True diff --git a/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini b/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini new file mode 100644 index 0000000..190e952 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini @@ -0,0 +1,24 @@ +# IMPORTANT +# Set "ignore_missing_imports" per package below, rather than globally. +# That will help identify missing type stubs that should be added to the sanity test environment. + +[mypy] +# There are ~350 errors reported in ansible-test when strict optional checking is enabled. +# Until the number of occurrences are greatly reduced, it's better to disable strict checking. +strict_optional = False +# There are ~25 errors reported in ansible-test under the 'misc' code. +# The majority of those errors are "Only concrete class can be given", which is due to a limitation of mypy. +# See: https://github.com/python/mypy/issues/5374 +disable_error_code = misc + +[mypy-argcomplete] +ignore_missing_imports = True + +[mypy-coverage] +ignore_missing_imports = True + +[mypy-ansible_release] +ignore_missing_imports = True + +[mypy-StringIO] +ignore_missing_imports = True diff --git a/test/lib/ansible_test/_util/controller/sanity/mypy/modules.ini b/test/lib/ansible_test/_util/controller/sanity/mypy/modules.ini new file mode 100644 index 0000000..d6a608f --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/mypy/modules.ini @@ -0,0 +1,98 @@ +# IMPORTANT +# Set "ignore_missing_imports" per package below, rather than globally. +# That will help identify missing type stubs that should be added to the sanity test environment. + +[mypy] + +[mypy-ansible.module_utils.six.moves.*] +ignore_missing_imports = True + +[mypy-pexpect.*] +ignore_missing_imports = True + +[mypy-md5.*] +ignore_missing_imports = True + +[mypy-yum.*] +ignore_missing_imports = True + +[mypy-rpmUtils.*] +ignore_missing_imports = True + +[mypy-rpm.*] +ignore_missing_imports = True + +[mypy-psutil.*] +ignore_missing_imports = True + +[mypy-dnf.*] +ignore_missing_imports = True + +[mypy-apt.*] +ignore_missing_imports = True + +[mypy-apt_pkg.*] +ignore_missing_imports = True + +[mypy-gssapi.*] +ignore_missing_imports = True + +[mypy-_ssl.*] +ignore_missing_imports = True + +[mypy-urllib_gssapi.*] +ignore_missing_imports = True + +[mypy-systemd.*] +ignore_missing_imports = True + +[mypy-sha.*] +ignore_missing_imports = True + +[mypy-distro.*] +ignore_missing_imports = True + +[mypy-selectors2.*] +ignore_missing_imports = True + +[mypy-selinux.*] +ignore_missing_imports = True + +[mypy-urllib2.*] +ignore_missing_imports = True + +[mypy-httplib.*] +ignore_missing_imports = True + +[mypy-compiler.*] +ignore_missing_imports = True + +[mypy-aptsources.*] +ignore_missing_imports = True + +[mypy-urllib3.*] +ignore_missing_imports = True + +[mypy-requests.*] +ignore_missing_imports = True + +[mypy-pkg_resources.*] +ignore_missing_imports = True + +[mypy-urllib.*] +ignore_missing_imports = True + +[mypy-email.*] +ignore_missing_imports = True + +[mypy-selectors.*] +ignore_missing_imports = True + +[mypy-importlib.*] +ignore_missing_imports = True + +[mypy-collections.*] +ignore_missing_imports = True + +[mypy-http.*] +ignore_missing_imports = True diff --git a/test/lib/ansible_test/_util/controller/sanity/pep8/current-ignore.txt b/test/lib/ansible_test/_util/controller/sanity/pep8/current-ignore.txt new file mode 100644 index 0000000..659c7f5 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pep8/current-ignore.txt @@ -0,0 +1,4 @@ +E402 +W503 +W504 +E741 diff --git a/test/lib/ansible_test/_util/controller/sanity/pslint/pslint.ps1 b/test/lib/ansible_test/_util/controller/sanity/pslint/pslint.ps1 new file mode 100644 index 0000000..0cf3c7f --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pslint/pslint.ps1 @@ -0,0 +1,37 @@ +#Requires -Version 6 +#Requires -Modules PSScriptAnalyzer, PSSA-PSCustomUseLiteralPath + +$ErrorActionPreference = "Stop" +$WarningPreference = "Stop" + +$LiteralPathRule = Import-Module -Name PSSA-PSCustomUseLiteralPath -PassThru +$LiteralPathRulePath = Join-Path -Path $LiteralPathRule.ModuleBase -ChildPath $LiteralPathRule.RootModule + +$PSSAParams = @{ + CustomRulePath = @($LiteralPathRulePath) + IncludeDefaultRules = $true + Setting = (Join-Path -Path $PSScriptRoot -ChildPath "settings.psd1") +} + +$Results = @( + ForEach ($Path in $Args) { + $Retries = 3 + + Do { + Try { + Invoke-ScriptAnalyzer -Path $Path @PSSAParams 3> $null + $Retries = 0 + } + Catch { + If (--$Retries -le 0) { + Throw + } + } + } + Until ($Retries -le 0) + } +) + +# Since pwsh 7.1 results that exceed depth will produce a warning which fails the process. +# Ignore warnings only for this step. +ConvertTo-Json -InputObject $Results -Depth 1 -WarningAction SilentlyContinue diff --git a/test/lib/ansible_test/_util/controller/sanity/pslint/settings.psd1 b/test/lib/ansible_test/_util/controller/sanity/pslint/settings.psd1 new file mode 100644 index 0000000..2ae13b4 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pslint/settings.psd1 @@ -0,0 +1,52 @@ +@{ + Rules = @{ + PSAvoidLongLines = @{ + Enable = $true + MaximumLineLength = 160 + } + PSPlaceOpenBrace = @{ + Enable = $true + OnSameLine = $true + IgnoreOneLineBlock = $true + NewLineAfter = $true + } + PSPlaceCloseBrace = @{ + Enable = $true + IgnoreOneLineBlock = $true + NewLineAfter = $true + NoEmptyLineBefore = $false + } + PSUseConsistentIndentation = @{ + Enable = $true + IndentationSize = 4 + PipelineIndentation = 'IncreaseIndentationForFirstPipeline' + Kind = 'space' + } + PSUseConsistentWhitespace = @{ + Enable = $true + CheckInnerBrace = $true + CheckOpenBrace = $true + CheckOpenParen = $true + CheckOperator = $true + CheckPipe = $true + CheckPipeForRedundantWhitespace = $false + CheckSeparator = $true + CheckParameter = $false + IgnoreAssignmentOperatorInsideHashTable = $false + } + } + ExcludeRules = @( + 'PSUseOutputTypeCorrectly', + 'PSUseShouldProcessForStateChangingFunctions', + # We send strings as plaintext so will always come across the 3 issues + 'PSAvoidUsingPlainTextForPassword', + 'PSAvoidUsingConvertToSecureStringWithPlainText', + 'PSAvoidUsingUserNameAndPassWordParams', + # We send the module as a base64 encoded string and a BOM will cause + # issues here + 'PSUseBOMForUnicodeEncodedFile', + # Too many false positives, there are many cases where shared utils + # invoke user defined code but not all parameters are used. + 'PSReviewUnusedParameter' + ) +} diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg new file mode 100644 index 0000000..aa34772 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg @@ -0,0 +1,57 @@ +[MESSAGES CONTROL] + +disable= + consider-using-f-string, # Python 2.x support still required + cyclic-import, # consistent results require running with --jobs 1 and testing all files + deprecated-method, # results vary by Python version + deprecated-module, # results vary by Python version + duplicate-code, # consistent results require running with --jobs 1 and testing all files + import-outside-toplevel, # common pattern in ansible related code + raise-missing-from, # Python 2.x does not support raise from + super-with-arguments, # Python 2.x does not support super without arguments + redundant-u-string-prefix, # Python 2.x support still required + too-few-public-methods, + too-many-arguments, + too-many-branches, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-nested-blocks, + too-many-return-statements, + too-many-statements, + useless-return, # complains about returning None when the return type is optional + +[BASIC] + +bad-names= + _, + bar, + baz, + foo, + tata, + toto, + tutu, + +good-names= + __metaclass__, + C, + ex, + i, + j, + k, + Run, + +class-attribute-rgx=[A-Za-z_][A-Za-z0-9_]{1,40}$ +attr-rgx=[a-z_][a-z0-9_]{1,40}$ +method-rgx=[a-z_][a-z0-9_]{1,40}$ +function-rgx=[a-z_][a-z0-9_]{1,40}$ + +[IMPORTS] + +preferred-modules = + distutils.version:ansible.module_utils.compat.version, + +# These modules are used by ansible-test, but will not be present in the virtual environment running pylint. +# Listing them here makes it possible to enable the import-error check. +ignored-modules = + py, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg new file mode 100644 index 0000000..1c03472 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg @@ -0,0 +1,63 @@ +[MESSAGES CONTROL] + +disable= + consider-using-f-string, # many occurrences + cyclic-import, # consistent results require running with --jobs 1 and testing all files + deprecated-method, # results vary by Python version + deprecated-module, # results vary by Python version + duplicate-code, # consistent results require running with --jobs 1 and testing all files + import-outside-toplevel, # common pattern in ansible related code + raise-missing-from, # Python 2.x does not support raise from + too-few-public-methods, + too-many-public-methods, + too-many-arguments, + too-many-branches, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-nested-blocks, + too-many-return-statements, + too-many-statements, + unspecified-encoding, # always run with UTF-8 encoding enforced + useless-return, # complains about returning None when the return type is optional + +[BASIC] + +bad-names= + _, + bar, + baz, + foo, + tata, + toto, + tutu, + +good-names= + __metaclass__, + C, + ex, + i, + j, + k, + Run, + +class-attribute-rgx=[A-Za-z_][A-Za-z0-9_]{1,40}$ +attr-rgx=[a-z_][a-z0-9_]{1,40}$ +method-rgx=[a-z_][a-z0-9_]{1,40}$ +function-rgx=[a-z_][a-z0-9_]{1,40}$ + +# Use the regex from earlier versions of pylint. +# See: https://github.com/PyCQA/pylint/pull/7322 +typevar-rgx=^_{0,2}(?:[^\W\da-z_]+|(?:[^\W\da-z_]+[^\WA-Z_]+)+T?(?<!Type))(?:_co(?:ntra)?)?$ + +[IMPORTS] + +preferred-modules = + distutils.version:ansible.module_utils.compat.version, + +# These modules are used by ansible-test, but will not be present in the virtual environment running pylint. +# Listing them here makes it possible to enable the import-error check. +ignored-modules = + cryptography, + coverage, + yamllint, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg new file mode 100644 index 0000000..e3aa8ee --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg @@ -0,0 +1,57 @@ +[MESSAGES CONTROL] + +disable= + consider-using-f-string, # many occurrences + cyclic-import, # consistent results require running with --jobs 1 and testing all files + deprecated-method, # results vary by Python version + deprecated-module, # results vary by Python version + duplicate-code, # consistent results require running with --jobs 1 and testing all files + import-outside-toplevel, # common pattern in ansible related code + raise-missing-from, # Python 2.x does not support raise from + too-few-public-methods, + too-many-arguments, + too-many-branches, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-nested-blocks, + too-many-return-statements, + too-many-statements, + unspecified-encoding, # always run with UTF-8 encoding enforced + useless-return, # complains about returning None when the return type is optional + +[BASIC] + +bad-names= + _, + bar, + baz, + foo, + tata, + toto, + tutu, + +good-names= + __metaclass__, + C, + ex, + i, + j, + k, + Run, + +class-attribute-rgx=[A-Za-z_][A-Za-z0-9_]{1,40}$ +attr-rgx=[a-z_][a-z0-9_]{1,40}$ +method-rgx=[a-z_][a-z0-9_]{1,40}$ +function-rgx=[a-z_][a-z0-9_]{1,40}$ +module-rgx=[a-z_][a-z0-9_-]{2,40}$ + +[IMPORTS] + +preferred-modules = + distutils.version:ansible.module_utils.compat.version, + +# These modules are used by ansible-test, but will not be present in the virtual environment running pylint. +# Listing them here makes it possible to enable the import-error check. +ignored-modules = + voluptuous, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/collection.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/collection.cfg new file mode 100644 index 0000000..38b8d2d --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/collection.cfg @@ -0,0 +1,147 @@ +[MESSAGES CONTROL] + +disable= + abstract-method, + access-member-before-definition, + arguments-differ, + assignment-from-no-return, + assignment-from-none, + attribute-defined-outside-init, + bad-indentation, + bad-mcs-classmethod-argument, + broad-except, + c-extension-no-member, + cell-var-from-loop, + chained-comparison, + comparison-with-callable, + consider-iterating-dictionary, + consider-merging-isinstance, + consider-using-dict-comprehension, # requires Python 2.7+, but we still require Python 2.6 support + consider-using-dict-items, + consider-using-enumerate, + consider-using-f-string, # Python 2.x support still required + consider-using-generator, + consider-using-get, + consider-using-in, + consider-using-set-comprehension, # requires Python 2.7+, but we still require Python 2.6 support + consider-using-ternary, + consider-using-with, + consider-using-max-builtin, + consider-using-min-builtin, + cyclic-import, # consistent results require running with --jobs 1 and testing all files + deprecated-method, # results vary by Python version + deprecated-module, # results vary by Python version + duplicate-code, # consistent results require running with --jobs 1 and testing all files + eval-used, + exec-used, + expression-not-assigned, + fixme, + function-redefined, + global-statement, + global-variable-undefined, + import-error, # inconsistent results which depend on the availability of imports + import-outside-toplevel, # common pattern in ansible related code + import-self, + inconsistent-return-statements, + invalid-envvar-default, + invalid-name, + invalid-sequence-index, + keyword-arg-before-vararg, + len-as-condition, + line-too-long, + literal-comparison, + locally-disabled, + method-hidden, + missing-docstring, + no-else-break, + no-else-continue, + no-else-raise, + no-else-return, + no-member, + no-name-in-module, # inconsistent results which depend on the availability of imports + no-value-for-parameter, + non-iterator-returned, + not-a-mapping, + not-an-iterable, + not-callable, + pointless-statement, + pointless-string-statement, + possibly-unused-variable, + protected-access, + raise-missing-from, # Python 2.x does not support raise from + redefined-argument-from-local, + redefined-builtin, + redefined-outer-name, + redundant-u-string-prefix, # Python 2.x support still required + reimported, + relative-beyond-top-level, # https://github.com/PyCQA/pylint/issues/2967 + signature-differs, + simplifiable-if-expression, + simplifiable-if-statement, + subprocess-popen-preexec-fn, + super-init-not-called, + super-with-arguments, # Python 2.x does not support super without arguments + superfluous-parens, + too-few-public-methods, + too-many-ancestors, # inconsistent results between python 3.6 and 3.7+ + too-many-arguments, + too-many-boolean-expressions, + too-many-branches, + too-many-function-args, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-nested-blocks, + too-many-public-methods, + too-many-return-statements, + too-many-statements, + trailing-comma-tuple, + trailing-comma-tuple, + try-except-raise, + unbalanced-tuple-unpacking, + undefined-loop-variable, + unexpected-keyword-arg, + ungrouped-imports, + unidiomatic-typecheck, + unnecessary-pass, + unnecessary-dunder-call, + unsubscriptable-object, + unsupported-assignment-operation, + unsupported-delete-operation, + unsupported-membership-test, + unused-argument, + unused-import, + unused-variable, + unspecified-encoding, # always run with UTF-8 encoding enforced + use-dict-literal, # many occurrences + use-list-literal, # many occurrences + use-implicit-booleaness-not-comparison, # many occurrences + useless-object-inheritance, + useless-return, + useless-super-delegation, + useless-option-value, # provide backwards compatibility with previous versions of ansible-test + wrong-import-order, + wrong-import-position, + +[BASIC] + +bad-names= + _, + bar, + baz, + foo, + tata, + toto, + tutu, + +good-names= + ex, + i, + j, + k, + Run, + +[TYPECHECK] + +ignored-modules= + _MovedItems, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/default.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/default.cfg new file mode 100644 index 0000000..6a242b8 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/default.cfg @@ -0,0 +1,146 @@ +[MESSAGES CONTROL] + +disable= + import-outside-toplevel, # common pattern in ansible related code + abstract-method, + access-member-before-definition, + arguments-differ, + assignment-from-no-return, + assignment-from-none, + attribute-defined-outside-init, + bad-indentation, + bad-mcs-classmethod-argument, + broad-except, + c-extension-no-member, + cell-var-from-loop, + chained-comparison, + comparison-with-callable, + consider-iterating-dictionary, + consider-merging-isinstance, + consider-using-dict-items, + consider-using-enumerate, + consider-using-f-string, # Python 2.x support still required + consider-using-get, + consider-using-in, + consider-using-ternary, + consider-using-with, + consider-using-max-builtin, + consider-using-min-builtin, + cyclic-import, # consistent results require running with --jobs 1 and testing all files + deprecated-method, # results vary by Python version + deprecated-module, # results vary by Python version + duplicate-code, # consistent results require running with --jobs 1 and testing all files + eval-used, + exec-used, + expression-not-assigned, + fixme, + function-redefined, + global-statement, + global-variable-undefined, + import-error, # inconsistent results which depend on the availability of imports + import-self, + inconsistent-return-statements, + invalid-envvar-default, + invalid-name, + invalid-sequence-index, + keyword-arg-before-vararg, + len-as-condition, + line-too-long, + literal-comparison, + locally-disabled, + method-hidden, + missing-docstring, + no-else-break, + no-else-continue, + no-else-raise, + no-else-return, + no-member, + no-name-in-module, # inconsistent results which depend on the availability of imports + no-value-for-parameter, + non-iterator-returned, + not-a-mapping, + not-an-iterable, + not-callable, + pointless-statement, + pointless-string-statement, + possibly-unused-variable, + protected-access, + raise-missing-from, # Python 2.x does not support raise from + redefined-argument-from-local, + redefined-builtin, + redefined-outer-name, + redundant-u-string-prefix, # Python 2.x support still required + reimported, + signature-differs, + simplifiable-if-expression, + simplifiable-if-statement, + subprocess-popen-preexec-fn, + super-init-not-called, + super-with-arguments, # Python 2.x does not support super without arguments + superfluous-parens, + too-few-public-methods, + too-many-ancestors, # inconsistent results between python 3.6 and 3.7+ + too-many-arguments, + too-many-boolean-expressions, + too-many-branches, + too-many-function-args, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-nested-blocks, + too-many-public-methods, + too-many-return-statements, + too-many-statements, + trailing-comma-tuple, + trailing-comma-tuple, + try-except-raise, + unbalanced-tuple-unpacking, + undefined-loop-variable, + unexpected-keyword-arg, + ungrouped-imports, + unidiomatic-typecheck, + unnecessary-pass, + unsubscriptable-object, + unsupported-assignment-operation, + unsupported-delete-operation, + unsupported-membership-test, + unused-argument, + unused-import, + unused-variable, + unspecified-encoding, # always run with UTF-8 encoding enforced + use-dict-literal, # many occurrences + use-list-literal, # many occurrences + use-implicit-booleaness-not-comparison, # many occurrences + useless-object-inheritance, + useless-return, + useless-super-delegation, + wrong-import-order, + wrong-import-position, + +[BASIC] + +bad-names= + _, + bar, + baz, + foo, + tata, + toto, + tutu, + +good-names= + ex, + i, + j, + k, + Run, + +[TYPECHECK] + +ignored-modules= + _MovedItems, + +[IMPORTS] + +preferred-modules = + distutils.version:ansible.module_utils.compat.version, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py new file mode 100644 index 0000000..79b8bf1 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py @@ -0,0 +1,263 @@ +"""Ansible specific plyint plugin for checking deprecations.""" +# (c) 2018, Matt Martz <matt@sivel.net> +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# -*- coding: utf-8 -*- +from __future__ import annotations + +import datetime +import re +import typing as t + +import astroid + +from pylint.interfaces import IAstroidChecker +from pylint.checkers import BaseChecker +from pylint.checkers.utils import check_messages + +from ansible.module_utils.compat.version import LooseVersion +from ansible.module_utils.six import string_types +from ansible.release import __version__ as ansible_version_raw +from ansible.utils.version import SemanticVersion + +MSGS = { + 'E9501': ("Deprecated version (%r) found in call to Display.deprecated " + "or AnsibleModule.deprecate", + "ansible-deprecated-version", + "Used when a call to Display.deprecated specifies a version " + "less than or equal to the current version of Ansible", + {'minversion': (2, 6)}), + 'E9502': ("Display.deprecated call without a version or date", + "ansible-deprecated-no-version", + "Used when a call to Display.deprecated does not specify a " + "version or date", + {'minversion': (2, 6)}), + 'E9503': ("Invalid deprecated version (%r) found in call to " + "Display.deprecated or AnsibleModule.deprecate", + "ansible-invalid-deprecated-version", + "Used when a call to Display.deprecated specifies an invalid " + "Ansible version number", + {'minversion': (2, 6)}), + 'E9504': ("Deprecated version (%r) found in call to Display.deprecated " + "or AnsibleModule.deprecate", + "collection-deprecated-version", + "Used when a call to Display.deprecated specifies a collection " + "version less than or equal to the current version of this " + "collection", + {'minversion': (2, 6)}), + 'E9505': ("Invalid deprecated version (%r) found in call to " + "Display.deprecated or AnsibleModule.deprecate", + "collection-invalid-deprecated-version", + "Used when a call to Display.deprecated specifies an invalid " + "collection version number", + {'minversion': (2, 6)}), + 'E9506': ("No collection name found in call to Display.deprecated or " + "AnsibleModule.deprecate", + "ansible-deprecated-no-collection-name", + "The current collection name in format `namespace.name` must " + "be provided as collection_name when calling Display.deprecated " + "or AnsibleModule.deprecate (`ansible.builtin` for ansible-core)", + {'minversion': (2, 6)}), + 'E9507': ("Wrong collection name (%r) found in call to " + "Display.deprecated or AnsibleModule.deprecate", + "wrong-collection-deprecated", + "The name of the current collection must be passed to the " + "Display.deprecated resp. AnsibleModule.deprecate calls " + "(`ansible.builtin` for ansible-core)", + {'minversion': (2, 6)}), + 'E9508': ("Expired date (%r) found in call to Display.deprecated " + "or AnsibleModule.deprecate", + "ansible-deprecated-date", + "Used when a call to Display.deprecated specifies a date " + "before today", + {'minversion': (2, 6)}), + 'E9509': ("Invalid deprecated date (%r) found in call to " + "Display.deprecated or AnsibleModule.deprecate", + "ansible-invalid-deprecated-date", + "Used when a call to Display.deprecated specifies an invalid " + "date. It must be a string in format `YYYY-MM-DD` (ISO 8601)", + {'minversion': (2, 6)}), + 'E9510': ("Both version and date found in call to " + "Display.deprecated or AnsibleModule.deprecate", + "ansible-deprecated-both-version-and-date", + "Only one of version and date must be specified", + {'minversion': (2, 6)}), + 'E9511': ("Removal version (%r) must be a major release, not a minor or " + "patch release (see the specification at https://semver.org/)", + "removal-version-must-be-major", + "Used when a call to Display.deprecated or " + "AnsibleModule.deprecate for a collection specifies a version " + "which is not of the form x.0.0", + {'minversion': (2, 6)}), +} + + +ANSIBLE_VERSION = LooseVersion('.'.join(ansible_version_raw.split('.')[:3])) + + +def _get_expr_name(node): + """Funciton to get either ``attrname`` or ``name`` from ``node.func.expr`` + + Created specifically for the case of ``display.deprecated`` or ``self._display.deprecated`` + """ + try: + return node.func.expr.attrname + except AttributeError: + # If this fails too, we'll let it raise, the caller should catch it + return node.func.expr.name + + +def parse_isodate(value): + """Parse an ISO 8601 date string.""" + msg = 'Expected ISO 8601 date string (YYYY-MM-DD)' + if not isinstance(value, string_types): + raise ValueError(msg) + # From Python 3.7 in, there is datetime.date.fromisoformat(). For older versions, + # we have to do things manually. + if not re.match('^[0-9]{4}-[0-9]{2}-[0-9]{2}$', value): + raise ValueError(msg) + try: + return datetime.datetime.strptime(value, '%Y-%m-%d').date() + except ValueError: + raise ValueError(msg) + + +class AnsibleDeprecatedChecker(BaseChecker): + """Checks for Display.deprecated calls to ensure that the ``version`` + has not passed or met the time for removal + """ + + __implements__ = (IAstroidChecker,) + name = 'deprecated' + msgs = MSGS + + options = ( + ('collection-name', { + 'default': None, + 'type': 'string', + 'metavar': '<name>', + 'help': 'The collection\'s name used to check collection names in deprecations.', + }), + ('collection-version', { + 'default': None, + 'type': 'string', + 'metavar': '<version>', + 'help': 'The collection\'s version number used to check deprecations.', + }), + ) + + def _check_date(self, node, date): + if not isinstance(date, str): + self.add_message('ansible-invalid-deprecated-date', node=node, args=(date,)) + return + + try: + date_parsed = parse_isodate(date) + except ValueError: + self.add_message('ansible-invalid-deprecated-date', node=node, args=(date,)) + return + + if date_parsed < datetime.date.today(): + self.add_message('ansible-deprecated-date', node=node, args=(date,)) + + def _check_version(self, node, version, collection_name): + if not isinstance(version, (str, float)): + if collection_name == 'ansible.builtin': + symbol = 'ansible-invalid-deprecated-version' + else: + symbol = 'collection-invalid-deprecated-version' + self.add_message(symbol, node=node, args=(version,)) + return + + version_no = str(version) + + if collection_name == 'ansible.builtin': + # Ansible-base + try: + if not version_no: + raise ValueError('Version string should not be empty') + loose_version = LooseVersion(str(version_no)) + if ANSIBLE_VERSION >= loose_version: + self.add_message('ansible-deprecated-version', node=node, args=(version,)) + except ValueError: + self.add_message('ansible-invalid-deprecated-version', node=node, args=(version,)) + elif collection_name: + # Collections + try: + if not version_no: + raise ValueError('Version string should not be empty') + semantic_version = SemanticVersion(version_no) + if collection_name == self.collection_name and self.collection_version is not None: + if self.collection_version >= semantic_version: + self.add_message('collection-deprecated-version', node=node, args=(version,)) + if semantic_version.major != 0 and (semantic_version.minor != 0 or semantic_version.patch != 0): + self.add_message('removal-version-must-be-major', node=node, args=(version,)) + except ValueError: + self.add_message('collection-invalid-deprecated-version', node=node, args=(version,)) + + @property + def collection_name(self) -> t.Optional[str]: + """Return the collection name, or None if ansible-core is being tested.""" + return self.config.collection_name + + @property + def collection_version(self) -> t.Optional[SemanticVersion]: + """Return the collection version, or None if ansible-core is being tested.""" + return SemanticVersion(self.config.collection_version) if self.config.collection_version is not None else None + + @check_messages(*(MSGS.keys())) + def visit_call(self, node): + """Visit a call node.""" + version = None + date = None + collection_name = None + try: + if (node.func.attrname == 'deprecated' and 'display' in _get_expr_name(node) or + node.func.attrname == 'deprecate' and _get_expr_name(node)): + if node.keywords: + for keyword in node.keywords: + if len(node.keywords) == 1 and keyword.arg is None: + # This is likely a **kwargs splat + return + if keyword.arg == 'version': + if isinstance(keyword.value.value, astroid.Name): + # This is likely a variable + return + version = keyword.value.value + if keyword.arg == 'date': + if isinstance(keyword.value.value, astroid.Name): + # This is likely a variable + return + date = keyword.value.value + if keyword.arg == 'collection_name': + if isinstance(keyword.value.value, astroid.Name): + # This is likely a variable + return + collection_name = keyword.value.value + if not version and not date: + try: + version = node.args[1].value + except IndexError: + self.add_message('ansible-deprecated-no-version', node=node) + return + if version and date: + self.add_message('ansible-deprecated-both-version-and-date', node=node) + + if collection_name: + this_collection = collection_name == (self.collection_name or 'ansible.builtin') + if not this_collection: + self.add_message('wrong-collection-deprecated', node=node, args=(collection_name,)) + elif self.collection_name is not None: + self.add_message('ansible-deprecated-no-collection-name', node=node) + + if date: + self._check_date(node, date) + elif version: + self._check_version(node, version, collection_name) + except AttributeError: + # Not the type of node we are interested in + pass + + +def register(linter): + """required method to auto register this checker """ + linter.register_checker(AnsibleDeprecatedChecker(linter)) diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/string_format.py b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/string_format.py new file mode 100644 index 0000000..934a9ae --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/string_format.py @@ -0,0 +1,85 @@ +"""Ansible specific pylint plugin for checking format string usage.""" +# (c) 2018, Matt Martz <matt@sivel.net> +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# -*- coding: utf-8 -*- +from __future__ import annotations + +import astroid +from pylint.interfaces import IAstroidChecker +from pylint.checkers import BaseChecker +from pylint.checkers import utils +from pylint.checkers.utils import check_messages +try: + from pylint.checkers.utils import parse_format_method_string +except ImportError: + # noinspection PyUnresolvedReferences + from pylint.checkers.strings import parse_format_method_string + +MSGS = { + 'E9305': ("Format string contains automatic field numbering " + "specification", + "ansible-format-automatic-specification", + "Used when a PEP 3101 format string contains automatic " + "field numbering (e.g. '{}').", + {'minversion': (2, 6)}), + 'E9390': ("bytes object has no .format attribute", + "ansible-no-format-on-bytestring", + "Used when a bytestring was used as a PEP 3101 format string " + "as Python3 bytestrings do not have a .format attribute", + {'minversion': (3, 0)}), +} + + +class AnsibleStringFormatChecker(BaseChecker): + """Checks string formatting operations to ensure that the format string + is valid and the arguments match the format string. + """ + + __implements__ = (IAstroidChecker,) + name = 'string' + msgs = MSGS + + @check_messages(*(MSGS.keys())) + def visit_call(self, node): + """Visit a call node.""" + func = utils.safe_infer(node.func) + if (isinstance(func, astroid.BoundMethod) + and isinstance(func.bound, astroid.Instance) + and func.bound.name in ('str', 'unicode', 'bytes')): + if func.name == 'format': + self._check_new_format(node, func) + + def _check_new_format(self, node, func): + """ Check the new string formatting """ + if (isinstance(node.func, astroid.Attribute) + and not isinstance(node.func.expr, astroid.Const)): + return + try: + strnode = next(func.bound.infer()) + except astroid.InferenceError: + return + if not isinstance(strnode, astroid.Const): + return + + if isinstance(strnode.value, bytes): + self.add_message('ansible-no-format-on-bytestring', node=node) + return + if not isinstance(strnode.value, str): + return + + if node.starargs or node.kwargs: + return + try: + num_args = parse_format_method_string(strnode.value)[1] + except utils.IncompleteFormatString: + return + + if num_args: + self.add_message('ansible-format-automatic-specification', + node=node) + return + + +def register(linter): + """required method to auto register this checker """ + linter.register_checker(AnsibleStringFormatChecker(linter)) diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/unwanted.py b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/unwanted.py new file mode 100644 index 0000000..1be42f5 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/unwanted.py @@ -0,0 +1,223 @@ +"""A plugin for pylint to identify imports and functions which should not be used.""" +from __future__ import annotations + +import os +import typing as t + +import astroid + +from pylint.checkers import BaseChecker +from pylint.interfaces import IAstroidChecker + +ANSIBLE_TEST_MODULES_PATH = os.environ['ANSIBLE_TEST_MODULES_PATH'] +ANSIBLE_TEST_MODULE_UTILS_PATH = os.environ['ANSIBLE_TEST_MODULE_UTILS_PATH'] + + +class UnwantedEntry: + """Defines an unwanted import.""" + def __init__( + self, + alternative, # type: str + modules_only=False, # type: bool + names=None, # type: t.Optional[t.Tuple[str, ...]] + ignore_paths=None, # type: t.Optional[t.Tuple[str, ...]] + ansible_test_only=False, # type: bool + ): # type: (...) -> None + self.alternative = alternative + self.modules_only = modules_only + self.names = set(names) if names else set() + self.ignore_paths = ignore_paths + self.ansible_test_only = ansible_test_only + + def applies_to(self, path, name=None): # type: (str, t.Optional[str]) -> bool + """Return True if this entry applies to the given path, otherwise return False.""" + if self.names: + if not name: + return False + + if name not in self.names: + return False + + if self.ignore_paths and any(path.endswith(ignore_path) for ignore_path in self.ignore_paths): + return False + + if self.ansible_test_only and '/test/lib/ansible_test/_internal/' not in path: + return False + + if self.modules_only: + return is_module_path(path) + + return True + + +def is_module_path(path): # type: (str) -> bool + """Return True if the given path is a module or module_utils path, otherwise return False.""" + return path.startswith(ANSIBLE_TEST_MODULES_PATH) or path.startswith(ANSIBLE_TEST_MODULE_UTILS_PATH) + + +class AnsibleUnwantedChecker(BaseChecker): + """Checker for unwanted imports and functions.""" + __implements__ = (IAstroidChecker,) + + name = 'unwanted' + + BAD_IMPORT = 'ansible-bad-import' + BAD_IMPORT_FROM = 'ansible-bad-import-from' + BAD_FUNCTION = 'ansible-bad-function' + BAD_MODULE_IMPORT = 'ansible-bad-module-import' + + msgs = dict( + E5101=('Import %s instead of %s', + BAD_IMPORT, + 'Identifies imports which should not be used.'), + E5102=('Import %s from %s instead of %s', + BAD_IMPORT_FROM, + 'Identifies imports which should not be used.'), + E5103=('Call %s instead of %s', + BAD_FUNCTION, + 'Identifies functions which should not be used.'), + E5104=('Import external package or ansible.module_utils not %s', + BAD_MODULE_IMPORT, + 'Identifies imports which should not be used.'), + ) + + unwanted_imports = dict( + # Additional imports that we may want to start checking: + # boto=UnwantedEntry('boto3', modules_only=True), + # requests=UnwantedEntry('ansible.module_utils.urls', modules_only=True), + # urllib=UnwantedEntry('ansible.module_utils.urls', modules_only=True), + + # see https://docs.python.org/2/library/urllib2.html + urllib2=UnwantedEntry('ansible.module_utils.urls', + ignore_paths=( + '/lib/ansible/module_utils/urls.py', + )), + + # see https://docs.python.org/3/library/collections.abc.html + collections=UnwantedEntry('ansible.module_utils.common._collections_compat', + ignore_paths=( + '/lib/ansible/module_utils/common/_collections_compat.py', + ), + names=( + 'MappingView', + 'ItemsView', + 'KeysView', + 'ValuesView', + 'Mapping', 'MutableMapping', + 'Sequence', 'MutableSequence', + 'Set', 'MutableSet', + 'Container', + 'Hashable', + 'Sized', + 'Callable', + 'Iterable', + 'Iterator', + )), + ) + + unwanted_functions = { + # see https://docs.python.org/3/library/tempfile.html#tempfile.mktemp + 'tempfile.mktemp': UnwantedEntry('tempfile.mkstemp'), + + # os.chmod resolves as posix.chmod + 'posix.chmod': UnwantedEntry('verified_chmod', + ansible_test_only=True), + + 'sys.exit': UnwantedEntry('exit_json or fail_json', + ignore_paths=( + '/lib/ansible/module_utils/basic.py', + '/lib/ansible/modules/async_wrapper.py', + ), + modules_only=True), + + 'builtins.print': UnwantedEntry('module.log or module.debug', + ignore_paths=( + '/lib/ansible/module_utils/basic.py', + ), + modules_only=True), + } + + def visit_import(self, node): # type: (astroid.node_classes.Import) -> None + """Visit an import node.""" + for name in node.names: + self._check_import(node, name[0]) + + def visit_importfrom(self, node): # type: (astroid.node_classes.ImportFrom) -> None + """Visit an import from node.""" + self._check_importfrom(node, node.modname, node.names) + + def visit_attribute(self, node): # type: (astroid.node_classes.Attribute) -> None + """Visit an attribute node.""" + last_child = node.last_child() + + # this is faster than using type inference and will catch the most common cases + if not isinstance(last_child, astroid.node_classes.Name): + return + + module = last_child.name + + entry = self.unwanted_imports.get(module) + + if entry and entry.names: + if entry.applies_to(self.linter.current_file, node.attrname): + self.add_message(self.BAD_IMPORT_FROM, args=(node.attrname, entry.alternative, module), node=node) + + def visit_call(self, node): # type: (astroid.node_classes.Call) -> None + """Visit a call node.""" + try: + for i in node.func.inferred(): + func = None + + if isinstance(i, astroid.scoped_nodes.FunctionDef) and isinstance(i.parent, astroid.scoped_nodes.Module): + func = '%s.%s' % (i.parent.name, i.name) + + if not func: + continue + + entry = self.unwanted_functions.get(func) + + if entry and entry.applies_to(self.linter.current_file): + self.add_message(self.BAD_FUNCTION, args=(entry.alternative, func), node=node) + except astroid.exceptions.InferenceError: + pass + + def _check_import(self, node, modname): # type: (astroid.node_classes.Import, str) -> None + """Check the imports on the specified import node.""" + self._check_module_import(node, modname) + + entry = self.unwanted_imports.get(modname) + + if not entry: + return + + if entry.applies_to(self.linter.current_file): + self.add_message(self.BAD_IMPORT, args=(entry.alternative, modname), node=node) + + def _check_importfrom(self, node, modname, names): # type: (astroid.node_classes.ImportFrom, str, t.List[str]) -> None + """Check the imports on the specified import from node.""" + self._check_module_import(node, modname) + + entry = self.unwanted_imports.get(modname) + + if not entry: + return + + for name in names: + if entry.applies_to(self.linter.current_file, name[0]): + self.add_message(self.BAD_IMPORT_FROM, args=(name[0], entry.alternative, modname), node=node) + + def _check_module_import(self, node, modname): # type: (t.Union[astroid.node_classes.Import, astroid.node_classes.ImportFrom], str) -> None + """Check the module import on the given import or import from node.""" + if not is_module_path(self.linter.current_file): + return + + if modname == 'ansible.module_utils' or modname.startswith('ansible.module_utils.'): + return + + if modname == 'ansible' or modname.startswith('ansible.'): + self.add_message(self.BAD_MODULE_IMPORT, args=(modname,), node=node) + + +def register(linter): + """required method to auto register this checker """ + linter.register_checker(AnsibleUnwantedChecker(linter)) diff --git a/test/lib/ansible_test/_util/controller/sanity/shellcheck/exclude.txt b/test/lib/ansible_test/_util/controller/sanity/shellcheck/exclude.txt new file mode 100644 index 0000000..29588dd --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/shellcheck/exclude.txt @@ -0,0 +1,3 @@ +SC1090 +SC1091 +SC2164 diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate.py new file mode 100644 index 0000000..ee7e832 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +from validate_modules.main import main + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/__init__.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/__init__.py new file mode 100644 index 0000000..1cfd6ac --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2015 Matt Martz <matt@sivel.net> +# Copyright (C) 2015 Rackspace US, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +from __future__ import annotations diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py new file mode 100644 index 0000000..270c9f4 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py @@ -0,0 +1,2520 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2015 Matt Martz <matt@sivel.net> +# Copyright (C) 2015 Rackspace US, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +from __future__ import annotations + +import abc +import argparse +import ast +import datetime +import json +import os +import re +import subprocess +import sys +import tempfile +import traceback +import warnings + +from collections import OrderedDict +from collections.abc import Mapping +from contextlib import contextmanager +from fnmatch import fnmatch + +import yaml + +from voluptuous.humanize import humanize_error + + +def setup_collection_loader(): + """ + Configure the collection loader if a collection is being tested. + This must be done before the plugin loader is imported. + """ + if '--collection' not in sys.argv: + return + + # noinspection PyProtectedMember + from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder + + collections_paths = os.environ.get('ANSIBLE_COLLECTIONS_PATH', '').split(os.pathsep) + collection_loader = _AnsibleCollectionFinder(collections_paths) + # noinspection PyProtectedMember + collection_loader._install() # pylint: disable=protected-access + + warnings.filterwarnings( + "ignore", + "AnsibleCollectionFinder has already been configured") + + +setup_collection_loader() + +from ansible import __version__ as ansible_version +from ansible.executor.module_common import REPLACER_WINDOWS, NEW_STYLE_PYTHON_MODULE_RE +from ansible.module_utils.common.parameters import DEFAULT_TYPE_VALIDATORS +from ansible.module_utils.compat.version import StrictVersion, LooseVersion +from ansible.module_utils.basic import to_bytes +from ansible.module_utils.six import PY3, with_metaclass, string_types +from ansible.plugins.loader import fragment_loader +from ansible.plugins.list import IGNORE as REJECTLIST +from ansible.utils.plugin_docs import add_collection_to_versions_and_dates, add_fragments, get_docstring +from ansible.utils.version import SemanticVersion + +from .module_args import AnsibleModuleImportError, AnsibleModuleNotInitialized, get_argument_spec + +from .schema import ansible_module_kwargs_schema, doc_schema, return_schema + +from .utils import CaptureStd, NoArgsAnsibleModule, compare_unordered_lists, is_empty, parse_yaml, parse_isodate + + +if PY3: + # Because there is no ast.TryExcept in Python 3 ast module + TRY_EXCEPT = ast.Try + # REPLACER_WINDOWS from ansible.executor.module_common is byte + # string but we need unicode for Python 3 + REPLACER_WINDOWS = REPLACER_WINDOWS.decode('utf-8') +else: + TRY_EXCEPT = ast.TryExcept + +REJECTLIST_DIRS = frozenset(('.git', 'test', '.github', '.idea')) +INDENT_REGEX = re.compile(r'([\t]*)') +TYPE_REGEX = re.compile(r'.*(if|or)(\s+[^"\']*|\s+)(?<!_)(?<!str\()type\([^)].*') +SYS_EXIT_REGEX = re.compile(r'[^#]*sys.exit\s*\(.*') +NO_LOG_REGEX = re.compile(r'(?:pass(?!ive)|secret|token|key)', re.I) + + +REJECTLIST_IMPORTS = { + 'requests': { + 'new_only': True, + 'error': { + 'code': 'use-module-utils-urls', + 'msg': ('requests import found, should use ' + 'ansible.module_utils.urls instead') + } + }, + r'boto(?:\.|$)': { + 'new_only': True, + 'error': { + 'code': 'use-boto3', + 'msg': 'boto import found, new modules should use boto3' + } + }, +} +SUBPROCESS_REGEX = re.compile(r'subprocess\.Po.*') +OS_CALL_REGEX = re.compile(r'os\.call.*') + + +LOOSE_ANSIBLE_VERSION = LooseVersion('.'.join(ansible_version.split('.')[:3])) + + +PLUGINS_WITH_RETURN_VALUES = ('module', ) +PLUGINS_WITH_EXAMPLES = ('module', ) +PLUGINS_WITH_YAML_EXAMPLES = ('module', ) + + +def is_potential_secret_option(option_name): + if not NO_LOG_REGEX.search(option_name): + return False + # If this is a count, type, algorithm, timeout, filename, or name, it is probably not a secret + if option_name.endswith(( + '_count', '_type', '_alg', '_algorithm', '_timeout', '_name', '_comment', + '_bits', '_id', '_identifier', '_period', '_file', '_filename', + )): + return False + # 'key' also matches 'publickey', which is generally not secret + if any(part in option_name for part in ( + 'publickey', 'public_key', 'keyusage', 'key_usage', 'keyserver', 'key_server', + 'keysize', 'key_size', 'keyservice', 'key_service', 'pub_key', 'pubkey', + 'keyboard', 'secretary', + )): + return False + return True + + +def compare_dates(d1, d2): + try: + date1 = parse_isodate(d1, allow_date=True) + date2 = parse_isodate(d2, allow_date=True) + return date1 == date2 + except ValueError: + # At least one of d1 and d2 cannot be parsed. Simply compare values. + return d1 == d2 + + +class ReporterEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, Exception): + return str(o) + + return json.JSONEncoder.default(self, o) + + +class Reporter: + def __init__(self): + self.files = OrderedDict() + + def _ensure_default_entry(self, path): + try: + self.files[path] + except KeyError: + self.files[path] = { + 'errors': [], + 'warnings': [], + 'traces': [], + 'warning_traces': [] + } + + def _log(self, path, code, msg, level='error', line=0, column=0): + self._ensure_default_entry(path) + lvl_dct = self.files[path]['%ss' % level] + lvl_dct.append({ + 'code': code, + 'msg': msg, + 'line': line, + 'column': column + }) + + def error(self, *args, **kwargs): + self._log(*args, level='error', **kwargs) + + def warning(self, *args, **kwargs): + self._log(*args, level='warning', **kwargs) + + def trace(self, path, tracebk): + self._ensure_default_entry(path) + self.files[path]['traces'].append(tracebk) + + def warning_trace(self, path, tracebk): + self._ensure_default_entry(path) + self.files[path]['warning_traces'].append(tracebk) + + @staticmethod + @contextmanager + def _output_handle(output): + if output != '-': + handle = open(output, 'w+') + else: + handle = sys.stdout + + yield handle + + handle.flush() + handle.close() + + @staticmethod + def _filter_out_ok(reports): + temp_reports = OrderedDict() + for path, report in reports.items(): + if report['errors'] or report['warnings']: + temp_reports[path] = report + + return temp_reports + + def plain(self, warnings=False, output='-'): + """Print out the test results in plain format + + output is ignored here for now + """ + ret = [] + + for path, report in Reporter._filter_out_ok(self.files).items(): + traces = report['traces'][:] + if warnings and report['warnings']: + traces.extend(report['warning_traces']) + + for trace in traces: + print('TRACE:') + print('\n '.join((' %s' % trace).splitlines())) + for error in report['errors']: + error['path'] = path + print('%(path)s:%(line)d:%(column)d: E%(code)s %(msg)s' % error) + ret.append(1) + if warnings: + for warning in report['warnings']: + warning['path'] = path + print('%(path)s:%(line)d:%(column)d: W%(code)s %(msg)s' % warning) + + return 3 if ret else 0 + + def json(self, warnings=False, output='-'): + """Print out the test results in json format + + warnings is not respected in this output + """ + ret = [len(r['errors']) for r in self.files.values()] + + with Reporter._output_handle(output) as handle: + print(json.dumps(Reporter._filter_out_ok(self.files), indent=4, cls=ReporterEncoder), file=handle) + + return 3 if sum(ret) else 0 + + +class Validator(with_metaclass(abc.ABCMeta, object)): + """Validator instances are intended to be run on a single object. if you + are scanning multiple objects for problems, you'll want to have a separate + Validator for each one.""" + + def __init__(self, reporter=None): + self.reporter = reporter + + @property + @abc.abstractmethod + def object_name(self): + """Name of the object we validated""" + pass + + @property + @abc.abstractmethod + def object_path(self): + """Path of the object we validated""" + pass + + @abc.abstractmethod + def validate(self): + """Run this method to generate the test results""" + pass + + +class ModuleValidator(Validator): + REJECTLIST_PATTERNS = ('.git*', '*.pyc', '*.pyo', '.*', '*.md', '*.rst', '*.txt') + REJECTLIST_FILES = frozenset(('.git', '.gitignore', '.travis.yml', + '.gitattributes', '.gitmodules', 'COPYING', + '__init__.py', 'VERSION', 'test-docs.sh')) + REJECTLIST = REJECTLIST_FILES.union(REJECTLIST['module']) + + # win_dsc is a dynamic arg spec, the docs won't ever match + PS_ARG_VALIDATE_REJECTLIST = frozenset(('win_dsc.ps1', )) + + ACCEPTLIST_FUTURE_IMPORTS = frozenset(('absolute_import', 'division', 'print_function')) + + def __init__(self, path, analyze_arg_spec=False, collection=None, collection_version=None, + base_branch=None, git_cache=None, reporter=None, routing=None, plugin_type='module'): + super(ModuleValidator, self).__init__(reporter=reporter or Reporter()) + + self.path = path + self.basename = os.path.basename(self.path) + self.name = os.path.splitext(self.basename)[0] + self.plugin_type = plugin_type + + self.analyze_arg_spec = analyze_arg_spec and plugin_type == 'module' + + self._Version = LooseVersion + self._StrictVersion = StrictVersion + + self.collection = collection + self.collection_name = 'ansible.builtin' + if self.collection: + self._Version = SemanticVersion + self._StrictVersion = SemanticVersion + collection_namespace_path, collection_name = os.path.split(self.collection) + self.collection_name = '%s.%s' % (os.path.basename(collection_namespace_path), collection_name) + self.routing = routing + self.collection_version = None + if collection_version is not None: + self.collection_version_str = collection_version + self.collection_version = SemanticVersion(collection_version) + + self.base_branch = base_branch + self.git_cache = git_cache or GitCache() + + self._python_module_override = False + + with open(path) as f: + self.text = f.read() + self.length = len(self.text.splitlines()) + try: + self.ast = ast.parse(self.text) + except Exception: + self.ast = None + + if base_branch: + self.base_module = self._get_base_file() + else: + self.base_module = None + + def _create_version(self, v, collection_name=None): + if not v: + raise ValueError('Empty string is not a valid version') + if collection_name == 'ansible.builtin': + return LooseVersion(v) + if collection_name is not None: + return SemanticVersion(v) + return self._Version(v) + + def _create_strict_version(self, v, collection_name=None): + if not v: + raise ValueError('Empty string is not a valid version') + if collection_name == 'ansible.builtin': + return StrictVersion(v) + if collection_name is not None: + return SemanticVersion(v) + return self._StrictVersion(v) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if not self.base_module: + return + + try: + os.remove(self.base_module) + except Exception: + pass + + @property + def object_name(self): + return self.basename + + @property + def object_path(self): + return self.path + + def _get_collection_meta(self): + """Implement if we need this for version_added comparisons + """ + pass + + def _python_module(self): + if self.path.endswith('.py') or self._python_module_override: + return True + return False + + def _powershell_module(self): + if self.path.endswith('.ps1'): + return True + return False + + def _sidecar_doc(self): + if self.path.endswith('.yml') or self.path.endswith('.yaml'): + return True + return False + + def _just_docs(self): + """Module can contain just docs and from __future__ boilerplate + """ + try: + for child in self.ast.body: + if not isinstance(child, ast.Assign): + # allow string constant expressions (these are docstrings) + if isinstance(child, ast.Expr) and isinstance(child.value, ast.Constant) and isinstance(child.value.value, str): + continue + + # allowed from __future__ imports + if isinstance(child, ast.ImportFrom) and child.module == '__future__': + for future_import in child.names: + if future_import.name not in self.ACCEPTLIST_FUTURE_IMPORTS: + break + else: + continue + return False + return True + except AttributeError: + return False + + def _get_base_branch_module_path(self): + """List all paths within lib/ansible/modules to try and match a moved module""" + return self.git_cache.base_module_paths.get(self.object_name) + + def _has_alias(self): + """Return true if the module has any aliases.""" + return self.object_name in self.git_cache.head_aliased_modules + + def _get_base_file(self): + # In case of module moves, look for the original location + base_path = self._get_base_branch_module_path() + ext = os.path.splitext(base_path or self.path)[1] + + command = ['git', 'show', '%s:%s' % (self.base_branch, base_path or self.path)] + p = subprocess.run(command, stdin=subprocess.DEVNULL, capture_output=True, check=False) + + if int(p.returncode) != 0: + return None + + t = tempfile.NamedTemporaryFile(delete=False, suffix=ext) + t.write(p.stdout) + t.close() + + return t.name + + def _is_new_module(self): + if self._has_alias(): + return False + + return not self.object_name.startswith('_') and bool(self.base_branch) and not bool(self.base_module) + + def _check_interpreter(self, powershell=False): + if powershell: + if not self.text.startswith('#!powershell\n'): + self.reporter.error( + path=self.object_path, + code='missing-powershell-interpreter', + msg='Interpreter line is not "#!powershell"' + ) + return + + missing_python_interpreter = False + + if not self.text.startswith('#!/usr/bin/python'): + if NEW_STYLE_PYTHON_MODULE_RE.search(to_bytes(self.text)): + missing_python_interpreter = self.text.startswith('#!') # shebang optional, but if present must match + else: + missing_python_interpreter = True # shebang required + + if missing_python_interpreter: + self.reporter.error( + path=self.object_path, + code='missing-python-interpreter', + msg='Interpreter line is not "#!/usr/bin/python"', + ) + + def _check_type_instead_of_isinstance(self, powershell=False): + if powershell: + return + for line_no, line in enumerate(self.text.splitlines()): + typekeyword = TYPE_REGEX.match(line) + if typekeyword: + # TODO: add column + self.reporter.error( + path=self.object_path, + code='unidiomatic-typecheck', + msg=('Type comparison using type() found. ' + 'Use isinstance() instead'), + line=line_no + 1 + ) + + def _check_for_sys_exit(self): + # Optimize out the happy path + if 'sys.exit' not in self.text: + return + + for line_no, line in enumerate(self.text.splitlines()): + sys_exit_usage = SYS_EXIT_REGEX.match(line) + if sys_exit_usage: + # TODO: add column + self.reporter.error( + path=self.object_path, + code='use-fail-json-not-sys-exit', + msg='sys.exit() call found. Should be exit_json/fail_json', + line=line_no + 1 + ) + + def _check_gpl3_header(self): + header = '\n'.join(self.text.split('\n')[:20]) + if ('GNU General Public License' not in header or + ('version 3' not in header and 'v3.0' not in header)): + self.reporter.error( + path=self.object_path, + code='missing-gplv3-license', + msg='GPLv3 license header not found in the first 20 lines of the module' + ) + elif self._is_new_module(): + if len([line for line in header + if 'GNU General Public License' in line]) > 1: + self.reporter.error( + path=self.object_path, + code='use-short-gplv3-license', + msg='Found old style GPLv3 license header: ' + 'https://docs.ansible.com/ansible-core/devel/dev_guide/developing_modules_documenting.html#copyright' + ) + + def _check_for_subprocess(self): + for child in self.ast.body: + if isinstance(child, ast.Import): + if child.names[0].name == 'subprocess': + for line_no, line in enumerate(self.text.splitlines()): + sp_match = SUBPROCESS_REGEX.search(line) + if sp_match: + self.reporter.error( + path=self.object_path, + code='use-run-command-not-popen', + msg=('subprocess.Popen call found. Should be module.run_command'), + line=(line_no + 1), + column=(sp_match.span()[0] + 1) + ) + + def _check_for_os_call(self): + if 'os.call' in self.text: + for line_no, line in enumerate(self.text.splitlines()): + os_call_match = OS_CALL_REGEX.search(line) + if os_call_match: + self.reporter.error( + path=self.object_path, + code='use-run-command-not-os-call', + msg=('os.call() call found. Should be module.run_command'), + line=(line_no + 1), + column=(os_call_match.span()[0] + 1) + ) + + def _find_rejectlist_imports(self): + for child in self.ast.body: + names = [] + if isinstance(child, ast.Import): + names.extend(child.names) + elif isinstance(child, TRY_EXCEPT): + bodies = child.body + for handler in child.handlers: + bodies.extend(handler.body) + for grandchild in bodies: + if isinstance(grandchild, ast.Import): + names.extend(grandchild.names) + for name in names: + # TODO: Add line/col + for rejectlist_import, options in REJECTLIST_IMPORTS.items(): + if re.search(rejectlist_import, name.name): + new_only = options['new_only'] + if self._is_new_module() and new_only: + self.reporter.error( + path=self.object_path, + **options['error'] + ) + elif not new_only: + self.reporter.error( + path=self.object_path, + **options['error'] + ) + + def _find_module_utils(self): + linenos = [] + found_basic = False + for child in self.ast.body: + if isinstance(child, (ast.Import, ast.ImportFrom)): + names = [] + try: + names.append(child.module) + if child.module.endswith('.basic'): + found_basic = True + except AttributeError: + pass + names.extend([n.name for n in child.names]) + + if [n for n in names if n.startswith('ansible.module_utils')]: + linenos.append(child.lineno) + + for name in child.names: + if ('module_utils' in getattr(child, 'module', '') and + isinstance(name, ast.alias) and + name.name == '*'): + msg = ( + 'module-utils-specific-import', + ('module_utils imports should import specific ' + 'components, not "*"') + ) + if self._is_new_module(): + self.reporter.error( + path=self.object_path, + code=msg[0], + msg=msg[1], + line=child.lineno + ) + else: + self.reporter.warning( + path=self.object_path, + code=msg[0], + msg=msg[1], + line=child.lineno + ) + + if (isinstance(name, ast.alias) and + name.name == 'basic'): + found_basic = True + + if not found_basic: + self.reporter.warning( + path=self.object_path, + code='missing-module-utils-basic-import', + msg='Did not find "ansible.module_utils.basic" import' + ) + + return linenos + + def _get_first_callable(self): + linenos = [] + for child in self.ast.body: + if isinstance(child, (ast.FunctionDef, ast.ClassDef)): + linenos.append(child.lineno) + + return min(linenos) if linenos else None + + def _find_has_import(self): + for child in self.ast.body: + found_try_except_import = False + found_has = False + if isinstance(child, TRY_EXCEPT): + bodies = child.body + for handler in child.handlers: + bodies.extend(handler.body) + for grandchild in bodies: + if isinstance(grandchild, ast.Import): + found_try_except_import = True + if isinstance(grandchild, ast.Assign): + for target in grandchild.targets: + if not isinstance(target, ast.Name): + continue + if target.id.lower().startswith('has_'): + found_has = True + if found_try_except_import and not found_has: + # TODO: Add line/col + self.reporter.warning( + path=self.object_path, + code='try-except-missing-has', + msg='Found Try/Except block without HAS_ assignment' + ) + + def _ensure_imports_below_docs(self, doc_info, first_callable): + min_doc_line = min(doc_info[key]['lineno'] for key in doc_info) + max_doc_line = max(doc_info[key]['end_lineno'] for key in doc_info) + + import_lines = [] + + for child in self.ast.body: + if isinstance(child, (ast.Import, ast.ImportFrom)): + if isinstance(child, ast.ImportFrom) and child.module == '__future__': + # allowed from __future__ imports + for future_import in child.names: + if future_import.name not in self.ACCEPTLIST_FUTURE_IMPORTS: + self.reporter.error( + path=self.object_path, + code='illegal-future-imports', + msg=('Only the following from __future__ imports are allowed: %s' + % ', '.join(self.ACCEPTLIST_FUTURE_IMPORTS)), + line=child.lineno + ) + break + else: # for-else. If we didn't find a problem nad break out of the loop, then this is a legal import + continue + import_lines.append(child.lineno) + if child.lineno < min_doc_line: + self.reporter.error( + path=self.object_path, + code='import-before-documentation', + msg=('Import found before documentation variables. ' + 'All imports must appear below ' + 'DOCUMENTATION/EXAMPLES/RETURN.'), + line=child.lineno + ) + break + elif isinstance(child, TRY_EXCEPT): + bodies = child.body + for handler in child.handlers: + bodies.extend(handler.body) + for grandchild in bodies: + if isinstance(grandchild, (ast.Import, ast.ImportFrom)): + import_lines.append(grandchild.lineno) + if grandchild.lineno < min_doc_line: + self.reporter.error( + path=self.object_path, + code='import-before-documentation', + msg=('Import found before documentation ' + 'variables. All imports must appear below ' + 'DOCUMENTATION/EXAMPLES/RETURN.'), + line=child.lineno + ) + break + + for import_line in import_lines: + if not (max_doc_line < import_line < first_callable): + msg = ( + 'import-placement', + ('Imports should be directly below DOCUMENTATION/EXAMPLES/' + 'RETURN.') + ) + if self._is_new_module(): + self.reporter.error( + path=self.object_path, + code=msg[0], + msg=msg[1], + line=import_line + ) + else: + self.reporter.warning( + path=self.object_path, + code=msg[0], + msg=msg[1], + line=import_line + ) + + def _validate_ps_replacers(self): + # loop all (for/else + error) + # get module list for each + # check "shape" of each module name + + module_requires = r'(?im)^#\s*requires\s+\-module(?:s?)\s*(Ansible\.ModuleUtils\..+)' + csharp_requires = r'(?im)^#\s*ansiblerequires\s+\-csharputil\s*(Ansible\..+)' + found_requires = False + + for req_stmt in re.finditer(module_requires, self.text): + found_requires = True + # this will bomb on dictionary format - "don't do that" + module_list = [x.strip() for x in req_stmt.group(1).split(',')] + if len(module_list) > 1: + self.reporter.error( + path=self.object_path, + code='multiple-utils-per-requires', + msg='Ansible.ModuleUtils requirements do not support multiple modules per statement: "%s"' % req_stmt.group(0) + ) + continue + + module_name = module_list[0] + + if module_name.lower().endswith('.psm1'): + self.reporter.error( + path=self.object_path, + code='invalid-requires-extension', + msg='Module #Requires should not end in .psm1: "%s"' % module_name + ) + + for req_stmt in re.finditer(csharp_requires, self.text): + found_requires = True + # this will bomb on dictionary format - "don't do that" + module_list = [x.strip() for x in req_stmt.group(1).split(',')] + if len(module_list) > 1: + self.reporter.error( + path=self.object_path, + code='multiple-csharp-utils-per-requires', + msg='Ansible C# util requirements do not support multiple utils per statement: "%s"' % req_stmt.group(0) + ) + continue + + module_name = module_list[0] + + if module_name.lower().endswith('.cs'): + self.reporter.error( + path=self.object_path, + code='illegal-extension-cs', + msg='Module #AnsibleRequires -CSharpUtil should not end in .cs: "%s"' % module_name + ) + + # also accept the legacy #POWERSHELL_COMMON replacer signal + if not found_requires and REPLACER_WINDOWS not in self.text: + self.reporter.error( + path=self.object_path, + code='missing-module-utils-import-csharp-requirements', + msg='No Ansible.ModuleUtils or C# Ansible util requirements/imports found' + ) + + def _find_ps_docs_file(self): + sidecar = self._find_sidecar_docs() + if sidecar: + return sidecar + + py_path = self.path.replace('.ps1', '.py') + if not os.path.isfile(py_path): + self.reporter.error( + path=self.object_path, + code='missing-documentation', + msg='No DOCUMENTATION provided' + ) + return py_path + + def _find_sidecar_docs(self): + base_path = os.path.splitext(self.path)[0] + for ext in ('.yml', '.yaml'): + doc_path = f"{base_path}{ext}" + if os.path.isfile(doc_path): + return doc_path + + def _get_py_docs(self): + docs = { + 'DOCUMENTATION': { + 'value': None, + 'lineno': 0, + 'end_lineno': 0, + }, + 'EXAMPLES': { + 'value': None, + 'lineno': 0, + 'end_lineno': 0, + }, + 'RETURN': { + 'value': None, + 'lineno': 0, + 'end_lineno': 0, + }, + } + for child in self.ast.body: + if isinstance(child, ast.Assign): + for grandchild in child.targets: + if not isinstance(grandchild, ast.Name): + continue + + if grandchild.id == 'DOCUMENTATION': + docs['DOCUMENTATION']['value'] = child.value.s + docs['DOCUMENTATION']['lineno'] = child.lineno + docs['DOCUMENTATION']['end_lineno'] = ( + child.lineno + len(child.value.s.splitlines()) + ) + elif grandchild.id == 'EXAMPLES': + docs['EXAMPLES']['value'] = child.value.s + docs['EXAMPLES']['lineno'] = child.lineno + docs['EXAMPLES']['end_lineno'] = ( + child.lineno + len(child.value.s.splitlines()) + ) + elif grandchild.id == 'RETURN': + docs['RETURN']['value'] = child.value.s + docs['RETURN']['lineno'] = child.lineno + docs['RETURN']['end_lineno'] = ( + child.lineno + len(child.value.s.splitlines()) + ) + + return docs + + def _validate_docs_schema(self, doc, schema, name, error_code): + # TODO: Add line/col + errors = [] + try: + schema(doc) + except Exception as e: + for error in e.errors: + error.data = doc + errors.extend(e.errors) + + for error in errors: + path = [str(p) for p in error.path] + + local_error_code = getattr(error, 'ansible_error_code', error_code) + + if isinstance(error.data, dict): + error_message = humanize_error(error.data, error) + else: + error_message = error + + if path: + combined_path = '%s.%s' % (name, '.'.join(path)) + else: + combined_path = name + + self.reporter.error( + path=self.object_path, + code=local_error_code, + msg='%s: %s' % (combined_path, error_message) + ) + + def _validate_docs(self): + doc = None + # We have three ways of marking deprecated/removed files. Have to check each one + # individually and then make sure they all agree + filename_deprecated_or_removed = False + deprecated = False + doc_deprecated = None # doc legally might not exist + routing_says_deprecated = False + + if self.object_name.startswith('_') and not os.path.islink(self.object_path): + filename_deprecated_or_removed = True + + # We are testing a collection + if self.routing: + routing_deprecation = self.routing.get('plugin_routing', {}) + routing_deprecation = routing_deprecation.get('modules' if self.plugin_type == 'module' else self.plugin_type, {}) + routing_deprecation = routing_deprecation.get(self.name, {}).get('deprecation', {}) + if routing_deprecation: + # meta/runtime.yml says this is deprecated + routing_says_deprecated = True + deprecated = True + + if self._python_module(): + doc_info = self._get_py_docs() + else: + doc_info = None + + sidecar_text = None + if self._sidecar_doc(): + sidecar_text = self.text + elif sidecar_path := self._find_sidecar_docs(): + with open(sidecar_path, mode='r', encoding='utf-8') as fd: + sidecar_text = fd.read() + + if sidecar_text: + sidecar_doc, errors, traces = parse_yaml(sidecar_text, 0, self.name, 'DOCUMENTATION') + for error in errors: + self.reporter.error( + path=self.object_path, + code='documentation-syntax-error', + **error + ) + for trace in traces: + self.reporter.trace( + path=self.object_path, + tracebk=trace + ) + + doc = sidecar_doc.get('DOCUMENTATION', None) + examples_raw = sidecar_doc.get('EXAMPLES', None) + examples_lineno = 1 + returns = sidecar_doc.get('RETURN', None) + + elif doc_info: + if bool(doc_info['DOCUMENTATION']['value']): + doc, errors, traces = parse_yaml( + doc_info['DOCUMENTATION']['value'], + doc_info['DOCUMENTATION']['lineno'], + self.name, 'DOCUMENTATION' + ) + + for error in errors: + self.reporter.error( + path=self.object_path, + code='documentation-syntax-error', + **error + ) + for trace in traces: + self.reporter.trace( + path=self.object_path, + tracebk=trace + ) + + examples_raw = doc_info['EXAMPLES']['value'] + examples_lineno = doc_info['EXAMPLES']['lineno'] + + returns = None + if bool(doc_info['RETURN']['value']): + returns, errors, traces = parse_yaml(doc_info['RETURN']['value'], + doc_info['RETURN']['lineno'], + self.name, 'RETURN') + + for error in errors: + self.reporter.error( + path=self.object_path, + code='return-syntax-error', + **error + ) + for trace in traces: + self.reporter.trace( + path=self.object_path, + tracebk=trace + ) + + if doc: + add_collection_to_versions_and_dates(doc, self.collection_name, + is_module=self.plugin_type == 'module') + + missing_fragment = False + with CaptureStd(): + try: + get_docstring(self.path, fragment_loader=fragment_loader, + verbose=True, + collection_name=self.collection_name, + plugin_type=self.plugin_type) + except AssertionError: + fragment = doc['extends_documentation_fragment'] + self.reporter.error( + path=self.object_path, + code='missing-doc-fragment', + msg='DOCUMENTATION fragment missing: %s' % fragment + ) + missing_fragment = True + except Exception as e: + self.reporter.trace( + path=self.object_path, + tracebk=traceback.format_exc() + ) + self.reporter.error( + path=self.object_path, + code='documentation-error', + msg='Unknown DOCUMENTATION error, see TRACE: %s' % e + ) + + if not missing_fragment: + add_fragments(doc, self.object_path, fragment_loader=fragment_loader, + is_module=self.plugin_type == 'module') + + if 'options' in doc and doc['options'] is None: + self.reporter.error( + path=self.object_path, + code='invalid-documentation-options', + msg='DOCUMENTATION.options must be a dictionary/hash when used', + ) + + if 'deprecated' in doc and doc.get('deprecated'): + doc_deprecated = True + doc_deprecation = doc['deprecated'] + documentation_collection = doc_deprecation.get('removed_from_collection') + if documentation_collection != self.collection_name: + self.reporter.error( + path=self.object_path, + code='deprecation-wrong-collection', + msg='"DOCUMENTATION.deprecation.removed_from_collection must be the current collection name: %r vs. %r' % ( + documentation_collection, self.collection_name) + ) + else: + doc_deprecated = False + + if os.path.islink(self.object_path): + # This module has an alias, which we can tell as it's a symlink + # Rather than checking for `module: $filename` we need to check against the true filename + self._validate_docs_schema( + doc, + doc_schema( + os.readlink(self.object_path).split('.')[0], + for_collection=bool(self.collection), + deprecated_module=deprecated, + plugin_type=self.plugin_type, + ), + 'DOCUMENTATION', + 'invalid-documentation', + ) + else: + # This is the normal case + self._validate_docs_schema( + doc, + doc_schema( + self.object_name.split('.')[0], + for_collection=bool(self.collection), + deprecated_module=deprecated, + plugin_type=self.plugin_type, + ), + 'DOCUMENTATION', + 'invalid-documentation', + ) + + if not self.collection: + existing_doc = self._check_for_new_args(doc) + self._check_version_added(doc, existing_doc) + else: + self.reporter.error( + path=self.object_path, + code='missing-documentation', + msg='No DOCUMENTATION provided', + ) + + if not examples_raw and self.plugin_type in PLUGINS_WITH_EXAMPLES: + if self.plugin_type in PLUGINS_WITH_EXAMPLES: + self.reporter.error( + path=self.object_path, + code='missing-examples', + msg='No EXAMPLES provided' + ) + + elif self.plugin_type in PLUGINS_WITH_YAML_EXAMPLES: + dummy, errors, traces = parse_yaml(examples_raw, + examples_lineno, + self.name, 'EXAMPLES', + load_all=True, + ansible_loader=True) + for error in errors: + self.reporter.error( + path=self.object_path, + code='invalid-examples', + **error + ) + for trace in traces: + self.reporter.trace( + path=self.object_path, + tracebk=trace + ) + + if returns: + if returns: + add_collection_to_versions_and_dates( + returns, + self.collection_name, + is_module=self.plugin_type == 'module', + return_docs=True) + self._validate_docs_schema( + returns, + return_schema(for_collection=bool(self.collection), plugin_type=self.plugin_type), + 'RETURN', 'return-syntax-error') + + elif self.plugin_type in PLUGINS_WITH_RETURN_VALUES: + if self._is_new_module(): + self.reporter.error( + path=self.object_path, + code='missing-return', + msg='No RETURN provided' + ) + else: + self.reporter.warning( + path=self.object_path, + code='missing-return-legacy', + msg='No RETURN provided' + ) + + # Check for mismatched deprecation + if not self.collection: + mismatched_deprecation = True + if not (filename_deprecated_or_removed or deprecated or doc_deprecated): + mismatched_deprecation = False + else: + if (filename_deprecated_or_removed and doc_deprecated): + mismatched_deprecation = False + if (filename_deprecated_or_removed and not doc): + mismatched_deprecation = False + + if mismatched_deprecation: + self.reporter.error( + path=self.object_path, + code='deprecation-mismatch', + msg='Module deprecation/removed must agree in documentation, by prepending filename with' + ' "_", and setting DOCUMENTATION.deprecated for deprecation or by removing all' + ' documentation for removed' + ) + else: + # We are testing a collection + if self.object_name.startswith('_'): + self.reporter.error( + path=self.object_path, + code='collections-no-underscore-on-deprecation', + msg='Deprecated content in collections MUST NOT start with "_", update meta/runtime.yml instead', + ) + + if not (doc_deprecated == routing_says_deprecated): + # DOCUMENTATION.deprecated and meta/runtime.yml disagree + self.reporter.error( + path=self.object_path, + code='deprecation-mismatch', + msg='"meta/runtime.yml" and DOCUMENTATION.deprecation do not agree.' + ) + elif routing_says_deprecated: + # Both DOCUMENTATION.deprecated and meta/runtime.yml agree that the module is deprecated. + # Make sure they give the same version or date. + routing_date = routing_deprecation.get('removal_date') + routing_version = routing_deprecation.get('removal_version') + # The versions and dates in the module documentation are auto-tagged, so remove the tag + # to make comparison possible and to avoid confusing the user. + documentation_date = doc_deprecation.get('removed_at_date') + documentation_version = doc_deprecation.get('removed_in') + if not compare_dates(routing_date, documentation_date): + self.reporter.error( + path=self.object_path, + code='deprecation-mismatch', + msg='"meta/runtime.yml" and DOCUMENTATION.deprecation do not agree on removal date: %r vs. %r' % ( + routing_date, documentation_date) + ) + if routing_version != documentation_version: + self.reporter.error( + path=self.object_path, + code='deprecation-mismatch', + msg='"meta/runtime.yml" and DOCUMENTATION.deprecation do not agree on removal version: %r vs. %r' % ( + routing_version, documentation_version) + ) + + # In the future we should error if ANSIBLE_METADATA exists in a collection + + return doc_info, doc + + def _check_version_added(self, doc, existing_doc): + version_added_raw = doc.get('version_added') + try: + collection_name = doc.get('version_added_collection') + version_added = self._create_strict_version( + str(version_added_raw or '0.0'), + collection_name=collection_name) + except ValueError as e: + version_added = version_added_raw or '0.0' + if self._is_new_module() or version_added != 'historical': + # already reported during schema validation, except: + if version_added == 'historical': + self.reporter.error( + path=self.object_path, + code='module-invalid-version-added', + msg='version_added is not a valid version number: %r. Error: %s' % (version_added, e) + ) + return + + if existing_doc and str(version_added_raw) != str(existing_doc.get('version_added')): + self.reporter.error( + path=self.object_path, + code='module-incorrect-version-added', + msg='version_added should be %r. Currently %r' % (existing_doc.get('version_added'), version_added_raw) + ) + + if not self._is_new_module(): + return + + should_be = '.'.join(ansible_version.split('.')[:2]) + strict_ansible_version = self._create_strict_version(should_be, collection_name='ansible.builtin') + + if (version_added < strict_ansible_version or + strict_ansible_version < version_added): + self.reporter.error( + path=self.object_path, + code='module-incorrect-version-added', + msg='version_added should be %r. Currently %r' % (should_be, version_added_raw) + ) + + def _validate_ansible_module_call(self, docs): + try: + spec, kwargs = get_argument_spec(self.path, self.collection) + except AnsibleModuleNotInitialized: + self.reporter.error( + path=self.object_path, + code='ansible-module-not-initialized', + msg="Execution of the module did not result in initialization of AnsibleModule", + ) + return + except AnsibleModuleImportError as e: + self.reporter.error( + path=self.object_path, + code='import-error', + msg="Exception attempting to import module for argument_spec introspection, '%s'" % e + ) + self.reporter.trace( + path=self.object_path, + tracebk=traceback.format_exc() + ) + return + + schema = ansible_module_kwargs_schema(self.object_name.split('.')[0], for_collection=bool(self.collection)) + self._validate_docs_schema(kwargs, schema, 'AnsibleModule', 'invalid-ansiblemodule-schema') + + self._validate_argument_spec(docs, spec, kwargs) + + def _validate_list_of_module_args(self, name, terms, spec, context): + if terms is None: + return + if not isinstance(terms, (list, tuple)): + # This is already reported by schema checking + return + for check in terms: + if not isinstance(check, (list, tuple)): + # This is already reported by schema checking + continue + bad_term = False + for term in check: + if not isinstance(term, string_types): + msg = name + if context: + msg += " found in %s" % " -> ".join(context) + msg += " must contain strings in the lists or tuples; found value %r" % (term, ) + self.reporter.error( + path=self.object_path, + code=name + '-type', + msg=msg, + ) + bad_term = True + if bad_term: + continue + if len(set(check)) != len(check): + msg = name + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has repeated terms" + self.reporter.error( + path=self.object_path, + code=name + '-collision', + msg=msg, + ) + if not set(check) <= set(spec): + msg = name + if context: + msg += " found in %s" % " -> ".join(context) + msg += " contains terms which are not part of argument_spec: %s" % ", ".join(sorted(set(check).difference(set(spec)))) + self.reporter.error( + path=self.object_path, + code=name + '-unknown', + msg=msg, + ) + + def _validate_required_if(self, terms, spec, context, module): + if terms is None: + return + if not isinstance(terms, (list, tuple)): + # This is already reported by schema checking + return + for check in terms: + if not isinstance(check, (list, tuple)) or len(check) not in [3, 4]: + # This is already reported by schema checking + continue + if len(check) == 4 and not isinstance(check[3], bool): + msg = "required_if" + if context: + msg += " found in %s" % " -> ".join(context) + msg += " must have forth value omitted or of type bool; got %r" % (check[3], ) + self.reporter.error( + path=self.object_path, + code='required_if-is_one_of-type', + msg=msg, + ) + requirements = check[2] + if not isinstance(requirements, (list, tuple)): + msg = "required_if" + if context: + msg += " found in %s" % " -> ".join(context) + msg += " must have third value (requirements) being a list or tuple; got type %r" % (requirements, ) + self.reporter.error( + path=self.object_path, + code='required_if-requirements-type', + msg=msg, + ) + continue + bad_term = False + for term in requirements: + if not isinstance(term, string_types): + msg = "required_if" + if context: + msg += " found in %s" % " -> ".join(context) + msg += " must have only strings in third value (requirements); got %r" % (term, ) + self.reporter.error( + path=self.object_path, + code='required_if-requirements-type', + msg=msg, + ) + bad_term = True + if bad_term: + continue + if len(set(requirements)) != len(requirements): + msg = "required_if" + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has repeated terms in requirements" + self.reporter.error( + path=self.object_path, + code='required_if-requirements-collision', + msg=msg, + ) + if not set(requirements) <= set(spec): + msg = "required_if" + if context: + msg += " found in %s" % " -> ".join(context) + msg += " contains terms in requirements which are not part of argument_spec: %s" % ", ".join(sorted(set(requirements).difference(set(spec)))) + self.reporter.error( + path=self.object_path, + code='required_if-requirements-unknown', + msg=msg, + ) + key = check[0] + if key not in spec: + msg = "required_if" + if context: + msg += " found in %s" % " -> ".join(context) + msg += " must have its key %s in argument_spec" % key + self.reporter.error( + path=self.object_path, + code='required_if-unknown-key', + msg=msg, + ) + continue + if key in requirements: + msg = "required_if" + if context: + msg += " found in %s" % " -> ".join(context) + msg += " contains its key %s in requirements" % key + self.reporter.error( + path=self.object_path, + code='required_if-key-in-requirements', + msg=msg, + ) + value = check[1] + if value is not None: + _type = spec[key].get('type', 'str') + if callable(_type): + _type_checker = _type + else: + _type_checker = DEFAULT_TYPE_VALIDATORS.get(_type) + try: + with CaptureStd(): + dummy = _type_checker(value) + except (Exception, SystemExit): + msg = "required_if" + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has value %r which does not fit to %s's parameter type %r" % (value, key, _type) + self.reporter.error( + path=self.object_path, + code='required_if-value-type', + msg=msg, + ) + + def _validate_required_by(self, terms, spec, context): + if terms is None: + return + if not isinstance(terms, Mapping): + # This is already reported by schema checking + return + for key, value in terms.items(): + if isinstance(value, string_types): + value = [value] + if not isinstance(value, (list, tuple)): + # This is already reported by schema checking + continue + for term in value: + if not isinstance(term, string_types): + # This is already reported by schema checking + continue + if len(set(value)) != len(value) or key in value: + msg = "required_by" + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has repeated terms" + self.reporter.error( + path=self.object_path, + code='required_by-collision', + msg=msg, + ) + if not set(value) <= set(spec) or key not in spec: + msg = "required_by" + if context: + msg += " found in %s" % " -> ".join(context) + msg += " contains terms which are not part of argument_spec: %s" % ", ".join(sorted(set(value).difference(set(spec)))) + self.reporter.error( + path=self.object_path, + code='required_by-unknown', + msg=msg, + ) + + def _validate_argument_spec(self, docs, spec, kwargs, context=None, last_context_spec=None): + if not self.analyze_arg_spec: + return + + if docs is None: + docs = {} + + if context is None: + context = [] + + if last_context_spec is None: + last_context_spec = kwargs + + try: + if not context: + add_fragments(docs, self.object_path, fragment_loader=fragment_loader, + is_module=self.plugin_type == 'module') + except Exception: + # Cannot merge fragments + return + + # Use this to access type checkers later + module = NoArgsAnsibleModule({}) + + self._validate_list_of_module_args('mutually_exclusive', last_context_spec.get('mutually_exclusive'), spec, context) + self._validate_list_of_module_args('required_together', last_context_spec.get('required_together'), spec, context) + self._validate_list_of_module_args('required_one_of', last_context_spec.get('required_one_of'), spec, context) + self._validate_required_if(last_context_spec.get('required_if'), spec, context, module) + self._validate_required_by(last_context_spec.get('required_by'), spec, context) + + provider_args = set() + args_from_argspec = set() + deprecated_args_from_argspec = set() + doc_options = docs.get('options', {}) + if doc_options is None: + doc_options = {} + for arg, data in spec.items(): + restricted_argument_names = ('message', 'syslog_facility') + if arg.lower() in restricted_argument_names: + msg = "Argument '%s' in argument_spec " % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += "must not be one of %s as it is used " \ + "internally by Ansible Core Engine" % (",".join(restricted_argument_names)) + self.reporter.error( + path=self.object_path, + code='invalid-argument-name', + msg=msg, + ) + continue + if 'aliases' in data: + for al in data['aliases']: + if al.lower() in restricted_argument_names: + msg = "Argument alias '%s' in argument_spec " % al + if context: + msg += " found in %s" % " -> ".join(context) + msg += "must not be one of %s as it is used " \ + "internally by Ansible Core Engine" % (",".join(restricted_argument_names)) + self.reporter.error( + path=self.object_path, + code='invalid-argument-name', + msg=msg, + ) + continue + + # Could this a place where secrets are leaked? + # If it is type: path we know it's not a secret key as it's a file path. + # If it is type: bool it is more likely a flag indicating that something is secret, than an actual secret. + if all(( + data.get('no_log') is None, is_potential_secret_option(arg), + data.get('type') not in ("path", "bool"), data.get('choices') is None, + )): + msg = "Argument '%s' in argument_spec could be a secret, though doesn't have `no_log` set" % arg + if context: + msg += " found in %s" % " -> ".join(context) + self.reporter.error( + path=self.object_path, + code='no-log-needed', + msg=msg, + ) + + if not isinstance(data, dict): + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " must be a dictionary/hash when used" + self.reporter.error( + path=self.object_path, + code='invalid-argument-spec', + msg=msg, + ) + continue + + removed_at_date = data.get('removed_at_date', None) + if removed_at_date is not None: + try: + if parse_isodate(removed_at_date, allow_date=False) < datetime.date.today(): + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has a removed_at_date '%s' before today" % removed_at_date + self.reporter.error( + path=self.object_path, + code='deprecated-date', + msg=msg, + ) + except ValueError: + # This should only happen when removed_at_date is not in ISO format. Since schema + # validation already reported this as an error, don't report it a second time. + pass + + deprecated_aliases = data.get('deprecated_aliases', None) + if deprecated_aliases is not None: + for deprecated_alias in deprecated_aliases: + if 'name' in deprecated_alias and 'date' in deprecated_alias: + try: + date = deprecated_alias['date'] + if parse_isodate(date, allow_date=False) < datetime.date.today(): + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has deprecated aliases '%s' with removal date '%s' before today" % ( + deprecated_alias['name'], deprecated_alias['date']) + self.reporter.error( + path=self.object_path, + code='deprecated-date', + msg=msg, + ) + except ValueError: + # This should only happen when deprecated_alias['date'] is not in ISO format. Since + # schema validation already reported this as an error, don't report it a second + # time. + pass + + has_version = False + if self.collection and self.collection_version is not None: + compare_version = self.collection_version + version_of_what = "this collection (%s)" % self.collection_version_str + code_prefix = 'collection' + has_version = True + elif not self.collection: + compare_version = LOOSE_ANSIBLE_VERSION + version_of_what = "Ansible (%s)" % ansible_version + code_prefix = 'ansible' + has_version = True + + removed_in_version = data.get('removed_in_version', None) + if removed_in_version is not None: + try: + collection_name = data.get('removed_from_collection') + removed_in = self._create_version(str(removed_in_version), collection_name=collection_name) + if has_version and collection_name == self.collection_name and compare_version >= removed_in: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has a deprecated removed_in_version %r," % removed_in_version + msg += " i.e. the version is less than or equal to the current version of %s" % version_of_what + self.reporter.error( + path=self.object_path, + code=code_prefix + '-deprecated-version', + msg=msg, + ) + except ValueError as e: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has an invalid removed_in_version number %r: %s" % (removed_in_version, e) + self.reporter.error( + path=self.object_path, + code='invalid-deprecated-version', + msg=msg, + ) + except TypeError: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has an invalid removed_in_version number %r: " % (removed_in_version, ) + msg += " error while comparing to version of %s" % version_of_what + self.reporter.error( + path=self.object_path, + code='invalid-deprecated-version', + msg=msg, + ) + + if deprecated_aliases is not None: + for deprecated_alias in deprecated_aliases: + if 'name' in deprecated_alias and 'version' in deprecated_alias: + try: + collection_name = deprecated_alias.get('collection_name') + version = self._create_version(str(deprecated_alias['version']), collection_name=collection_name) + if has_version and collection_name == self.collection_name and compare_version >= version: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has deprecated aliases '%s' with removal in version %r," % ( + deprecated_alias['name'], deprecated_alias['version']) + msg += " i.e. the version is less than or equal to the current version of %s" % version_of_what + self.reporter.error( + path=self.object_path, + code=code_prefix + '-deprecated-version', + msg=msg, + ) + except ValueError as e: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has deprecated aliases '%s' with invalid removal version %r: %s" % ( + deprecated_alias['name'], deprecated_alias['version'], e) + self.reporter.error( + path=self.object_path, + code='invalid-deprecated-version', + msg=msg, + ) + except TypeError: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has deprecated aliases '%s' with invalid removal version %r:" % ( + deprecated_alias['name'], deprecated_alias['version']) + msg += " error while comparing to version of %s" % version_of_what + self.reporter.error( + path=self.object_path, + code='invalid-deprecated-version', + msg=msg, + ) + + aliases = data.get('aliases', []) + if arg in aliases: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " is specified as its own alias" + self.reporter.error( + path=self.object_path, + code='parameter-alias-self', + msg=msg + ) + if len(aliases) > len(set(aliases)): + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has at least one alias specified multiple times in aliases" + self.reporter.error( + path=self.object_path, + code='parameter-alias-repeated', + msg=msg + ) + if not context and arg == 'state': + bad_states = set(['list', 'info', 'get']) & set(data.get('choices', set())) + for bad_state in bad_states: + self.reporter.error( + path=self.object_path, + code='parameter-state-invalid-choice', + msg="Argument 'state' includes the value '%s' as a choice" % bad_state) + if not data.get('removed_in_version', None) and not data.get('removed_at_date', None): + args_from_argspec.add(arg) + args_from_argspec.update(aliases) + else: + deprecated_args_from_argspec.add(arg) + deprecated_args_from_argspec.update(aliases) + if arg == 'provider' and self.object_path.startswith('lib/ansible/modules/network/'): + if data.get('options') is not None and not isinstance(data.get('options'), Mapping): + self.reporter.error( + path=self.object_path, + code='invalid-argument-spec-options', + msg="Argument 'options' in argument_spec['provider'] must be a dictionary/hash when used", + ) + elif data.get('options'): + # Record provider options from network modules, for later comparison + for provider_arg, provider_data in data.get('options', {}).items(): + provider_args.add(provider_arg) + provider_args.update(provider_data.get('aliases', [])) + + if data.get('required') and data.get('default', object) != object: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " is marked as required but specifies a default. Arguments with a" \ + " default should not be marked as required" + self.reporter.error( + path=self.object_path, + code='no-default-for-required-parameter', + msg=msg + ) + + if arg in provider_args: + # Provider args are being removed from network module top level + # don't validate docs<->arg_spec checks below + continue + + _type = data.get('type', 'str') + if callable(_type): + _type_checker = _type + else: + _type_checker = DEFAULT_TYPE_VALIDATORS.get(_type) + + _elements = data.get('elements') + if (_type == 'list') and not _elements: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " defines type as list but elements is not defined" + self.reporter.error( + path=self.object_path, + code='parameter-list-no-elements', + msg=msg + ) + if _elements: + if not callable(_elements): + DEFAULT_TYPE_VALIDATORS.get(_elements) + if _type != 'list': + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " defines elements as %s but it is valid only when value of parameter type is list" % _elements + self.reporter.error( + path=self.object_path, + code='parameter-invalid-elements', + msg=msg + ) + + arg_default = None + if 'default' in data and not is_empty(data['default']): + try: + with CaptureStd(): + arg_default = _type_checker(data['default']) + except (Exception, SystemExit): + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " defines default as (%r) but this is incompatible with parameter type %r" % (data['default'], _type) + self.reporter.error( + path=self.object_path, + code='incompatible-default-type', + msg=msg + ) + continue + + doc_options_args = [] + for alias in sorted(set([arg] + list(aliases))): + if alias in doc_options: + doc_options_args.append(alias) + if len(doc_options_args) == 0: + # Undocumented arguments will be handled later (search for undocumented-parameter) + doc_options_arg = {} + else: + doc_options_arg = doc_options[doc_options_args[0]] + if len(doc_options_args) > 1: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " with aliases %s is documented multiple times, namely as %s" % ( + ", ".join([("'%s'" % alias) for alias in aliases]), + ", ".join([("'%s'" % alias) for alias in doc_options_args]) + ) + self.reporter.error( + path=self.object_path, + code='parameter-documented-multiple-times', + msg=msg + ) + + try: + doc_default = None + if 'default' in doc_options_arg and not is_empty(doc_options_arg['default']): + with CaptureStd(): + doc_default = _type_checker(doc_options_arg['default']) + except (Exception, SystemExit): + msg = "Argument '%s' in documentation" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " defines default as (%r) but this is incompatible with parameter type %r" % (doc_options_arg.get('default'), _type) + self.reporter.error( + path=self.object_path, + code='doc-default-incompatible-type', + msg=msg + ) + continue + + if arg_default != doc_default: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " defines default as (%r) but documentation defines default as (%r)" % (arg_default, doc_default) + self.reporter.error( + path=self.object_path, + code='doc-default-does-not-match-spec', + msg=msg + ) + + doc_type = doc_options_arg.get('type') + if 'type' in data and data['type'] is not None: + if doc_type is None: + if not arg.startswith('_'): # hidden parameter, for example _raw_params + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " defines type as %r but documentation doesn't define type" % (data['type']) + self.reporter.error( + path=self.object_path, + code='parameter-type-not-in-doc', + msg=msg + ) + elif data['type'] != doc_type: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " defines type as %r but documentation defines type as %r" % (data['type'], doc_type) + self.reporter.error( + path=self.object_path, + code='doc-type-does-not-match-spec', + msg=msg + ) + else: + if doc_type is None: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " uses default type ('str') but documentation doesn't define type" + self.reporter.error( + path=self.object_path, + code='doc-missing-type', + msg=msg + ) + elif doc_type != 'str': + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " implies type as 'str' but documentation defines as %r" % doc_type + self.reporter.error( + path=self.object_path, + code='implied-parameter-type-mismatch', + msg=msg + ) + + doc_choices = [] + try: + for choice in doc_options_arg.get('choices', []): + try: + with CaptureStd(): + doc_choices.append(_type_checker(choice)) + except (Exception, SystemExit): + msg = "Argument '%s' in documentation" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " defines choices as (%r) but this is incompatible with argument type %r" % (choice, _type) + self.reporter.error( + path=self.object_path, + code='doc-choices-incompatible-type', + msg=msg + ) + raise StopIteration() + except StopIteration: + continue + + arg_choices = [] + try: + for choice in data.get('choices', []): + try: + with CaptureStd(): + arg_choices.append(_type_checker(choice)) + except (Exception, SystemExit): + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " defines choices as (%r) but this is incompatible with argument type %r" % (choice, _type) + self.reporter.error( + path=self.object_path, + code='incompatible-choices', + msg=msg + ) + raise StopIteration() + except StopIteration: + continue + + if not compare_unordered_lists(arg_choices, doc_choices): + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " defines choices as (%r) but documentation defines choices as (%r)" % (arg_choices, doc_choices) + self.reporter.error( + path=self.object_path, + code='doc-choices-do-not-match-spec', + msg=msg + ) + + doc_required = doc_options_arg.get('required', False) + data_required = data.get('required', False) + if (doc_required or data_required) and not (doc_required and data_required): + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + if doc_required: + msg += " is not required, but is documented as being required" + else: + msg += " is required, but is not documented as being required" + self.reporter.error( + path=self.object_path, + code='doc-required-mismatch', + msg=msg + ) + + doc_elements = doc_options_arg.get('elements', None) + doc_type = doc_options_arg.get('type', 'str') + data_elements = data.get('elements', None) + if (doc_elements or data_elements) and not (doc_elements == data_elements): + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + if data_elements: + msg += " specifies elements as %s," % data_elements + else: + msg += " does not specify elements," + if doc_elements: + msg += "but elements is documented as being %s" % doc_elements + else: + msg += "but elements is not documented" + self.reporter.error( + path=self.object_path, + code='doc-elements-mismatch', + msg=msg + ) + + spec_suboptions = data.get('options') + doc_suboptions = doc_options_arg.get('suboptions', {}) + if spec_suboptions: + if not doc_suboptions: + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " has sub-options but documentation does not define it" + self.reporter.error( + path=self.object_path, + code='missing-suboption-docs', + msg=msg + ) + self._validate_argument_spec({'options': doc_suboptions}, spec_suboptions, kwargs, + context=context + [arg], last_context_spec=data) + + for arg in args_from_argspec: + if not str(arg).isidentifier(): + msg = "Argument '%s' in argument_spec" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " is not a valid python identifier" + self.reporter.error( + path=self.object_path, + code='parameter-invalid', + msg=msg + ) + + if docs: + args_from_docs = set() + for arg, data in doc_options.items(): + args_from_docs.add(arg) + args_from_docs.update(data.get('aliases', [])) + + args_missing_from_docs = args_from_argspec.difference(args_from_docs) + docs_missing_from_args = args_from_docs.difference(args_from_argspec | deprecated_args_from_argspec) + for arg in args_missing_from_docs: + if arg in provider_args: + # Provider args are being removed from network module top level + # So they are likely not documented on purpose + continue + msg = "Argument '%s'" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " is listed in the argument_spec, but not documented in the module documentation" + self.reporter.error( + path=self.object_path, + code='undocumented-parameter', + msg=msg + ) + for arg in docs_missing_from_args: + msg = "Argument '%s'" % arg + if context: + msg += " found in %s" % " -> ".join(context) + msg += " is listed in DOCUMENTATION.options, but not accepted by the module argument_spec" + self.reporter.error( + path=self.object_path, + code='nonexistent-parameter-documented', + msg=msg + ) + + def _check_for_new_args(self, doc): + if not self.base_branch or self._is_new_module(): + return + + with CaptureStd(): + try: + existing_doc, dummy_examples, dummy_return, existing_metadata = get_docstring( + self.base_module, fragment_loader, verbose=True, collection_name=self.collection_name, + is_module=self.plugin_type == 'module') + existing_options = existing_doc.get('options', {}) or {} + except AssertionError: + fragment = doc['extends_documentation_fragment'] + self.reporter.warning( + path=self.object_path, + code='missing-existing-doc-fragment', + msg='Pre-existing DOCUMENTATION fragment missing: %s' % fragment + ) + return + except Exception as e: + self.reporter.warning_trace( + path=self.object_path, + tracebk=e + ) + self.reporter.warning( + path=self.object_path, + code='unknown-doc-fragment', + msg=('Unknown pre-existing DOCUMENTATION error, see TRACE. Submodule refs may need updated') + ) + return + + try: + mod_collection_name = existing_doc.get('version_added_collection') + mod_version_added = self._create_strict_version( + str(existing_doc.get('version_added', '0.0')), + collection_name=mod_collection_name) + except ValueError: + mod_collection_name = self.collection_name + mod_version_added = self._create_strict_version('0.0') + + options = doc.get('options', {}) or {} + + should_be = '.'.join(ansible_version.split('.')[:2]) + strict_ansible_version = self._create_strict_version(should_be, collection_name='ansible.builtin') + + for option, details in options.items(): + try: + names = [option] + details.get('aliases', []) + except (TypeError, AttributeError): + # Reporting of this syntax error will be handled by schema validation. + continue + + if any(name in existing_options for name in names): + # The option already existed. Make sure version_added didn't change. + for name in names: + existing_collection_name = existing_options.get(name, {}).get('version_added_collection') + existing_version = existing_options.get(name, {}).get('version_added') + if existing_version: + break + current_collection_name = details.get('version_added_collection') + current_version = details.get('version_added') + if current_collection_name != existing_collection_name: + self.reporter.error( + path=self.object_path, + code='option-incorrect-version-added-collection', + msg=('version_added for existing option (%s) should ' + 'belong to collection %r. Currently belongs to %r' % + (option, current_collection_name, existing_collection_name)) + ) + elif str(current_version) != str(existing_version): + self.reporter.error( + path=self.object_path, + code='option-incorrect-version-added', + msg=('version_added for existing option (%s) should ' + 'be %r. Currently %r' % + (option, existing_version, current_version)) + ) + continue + + try: + collection_name = details.get('version_added_collection') + version_added = self._create_strict_version( + str(details.get('version_added', '0.0')), + collection_name=collection_name) + except ValueError as e: + # already reported during schema validation + continue + + builtin = self.collection_name == 'ansible.builtin' and collection_name in ('ansible.builtin', None) + if not builtin and collection_name != self.collection_name: + continue + if (strict_ansible_version != mod_version_added and + (version_added < strict_ansible_version or + strict_ansible_version < version_added)): + self.reporter.error( + path=self.object_path, + code='option-incorrect-version-added', + msg=('version_added for new option (%s) should ' + 'be %r. Currently %r' % + (option, should_be, version_added)) + ) + + return existing_doc + + @staticmethod + def is_on_rejectlist(path): + base_name = os.path.basename(path) + file_name = os.path.splitext(base_name)[0] + + if file_name.startswith('_') and os.path.islink(path): + return True + + if not frozenset((base_name, file_name)).isdisjoint(ModuleValidator.REJECTLIST): + return True + + for pat in ModuleValidator.REJECTLIST_PATTERNS: + if fnmatch(base_name, pat): + return True + + return False + + def validate(self): + super(ModuleValidator, self).validate() + if not self._python_module() and not self._powershell_module() and not self._sidecar_doc(): + self.reporter.error( + path=self.object_path, + code='invalid-extension', + msg=('Official Ansible modules must have a .py ' + 'extension for python modules or a .ps1 ' + 'for powershell modules') + ) + self._python_module_override = True + + if self._python_module() and self.ast is None: + self.reporter.error( + path=self.object_path, + code='python-syntax-error', + msg='Python SyntaxError while parsing module' + ) + try: + compile(self.text, self.path, 'exec') + except Exception: + self.reporter.trace( + path=self.object_path, + tracebk=traceback.format_exc() + ) + return + + end_of_deprecation_should_be_removed_only = False + doc_info = None + if self._python_module() or self._sidecar_doc(): + doc_info, docs = self._validate_docs() + + # See if current version => deprecated.removed_in, ie, should be docs only + if docs and docs.get('deprecated', False): + + if 'removed_in' in docs['deprecated']: + removed_in = None + collection_name = docs['deprecated'].get('removed_from_collection') + version = docs['deprecated']['removed_in'] + if collection_name != self.collection_name: + self.reporter.error( + path=self.object_path, + code='invalid-module-deprecation-source', + msg=('The deprecation version for a module must be added in this collection') + ) + else: + try: + removed_in = self._create_strict_version(str(version), collection_name=collection_name) + except ValueError as e: + self.reporter.error( + path=self.object_path, + code='invalid-module-deprecation-version', + msg=('The deprecation version %r cannot be parsed: %s' % (version, e)) + ) + + if removed_in: + if not self.collection: + strict_ansible_version = self._create_strict_version( + '.'.join(ansible_version.split('.')[:2]), self.collection_name) + end_of_deprecation_should_be_removed_only = strict_ansible_version >= removed_in + + if end_of_deprecation_should_be_removed_only: + self.reporter.error( + path=self.object_path, + code='ansible-deprecated-module', + msg='Module is marked for removal in version %s of Ansible when the current version is %s' % ( + version, ansible_version), + ) + elif self.collection_version: + strict_ansible_version = self.collection_version + end_of_deprecation_should_be_removed_only = strict_ansible_version >= removed_in + + if end_of_deprecation_should_be_removed_only: + self.reporter.error( + path=self.object_path, + code='collection-deprecated-module', + msg='Module is marked for removal in version %s of this collection when the current version is %s' % ( + version, self.collection_version_str), + ) + + # handle deprecation by date + if 'removed_at_date' in docs['deprecated']: + try: + removed_at_date = docs['deprecated']['removed_at_date'] + if parse_isodate(removed_at_date, allow_date=True) < datetime.date.today(): + msg = "Module's deprecated.removed_at_date date '%s' is before today" % removed_at_date + self.reporter.error(path=self.object_path, code='deprecated-date', msg=msg) + except ValueError: + # This happens if the date cannot be parsed. This is already checked by the schema. + pass + + if self._python_module() and not self._just_docs() and not end_of_deprecation_should_be_removed_only: + if self.plugin_type == 'module': + self._validate_ansible_module_call(docs) + self._check_for_sys_exit() + self._find_rejectlist_imports() + if self.plugin_type == 'module': + self._find_module_utils() + self._find_has_import() + + if doc_info: + first_callable = self._get_first_callable() or 1000000 # use a bogus "high" line number if no callable exists + self._ensure_imports_below_docs(doc_info, first_callable) + + if self.plugin_type == 'module': + self._check_for_subprocess() + self._check_for_os_call() + + if self._powershell_module(): + self._validate_ps_replacers() + docs_path = self._find_ps_docs_file() + + # We can only validate PowerShell arg spec if it is using the new Ansible.Basic.AnsibleModule util + pattern = r'(?im)^#\s*ansiblerequires\s+\-csharputil\s*Ansible\.Basic' + if re.search(pattern, self.text) and self.object_name not in self.PS_ARG_VALIDATE_REJECTLIST: + with ModuleValidator(docs_path, base_branch=self.base_branch, git_cache=self.git_cache) as docs_mv: + docs = docs_mv._validate_docs()[1] + self._validate_ansible_module_call(docs) + + self._check_gpl3_header() + if not self._just_docs() and not self._sidecar_doc() and not end_of_deprecation_should_be_removed_only: + if self.plugin_type == 'module': + self._check_interpreter(powershell=self._powershell_module()) + self._check_type_instead_of_isinstance( + powershell=self._powershell_module() + ) + + +class PythonPackageValidator(Validator): + REJECTLIST_FILES = frozenset(('__pycache__',)) + + def __init__(self, path, reporter=None): + super(PythonPackageValidator, self).__init__(reporter=reporter or Reporter()) + + self.path = path + self.basename = os.path.basename(path) + + @property + def object_name(self): + return self.basename + + @property + def object_path(self): + return self.path + + def validate(self): + super(PythonPackageValidator, self).validate() + + if self.basename in self.REJECTLIST_FILES: + return + + init_file = os.path.join(self.path, '__init__.py') + if not os.path.exists(init_file): + self.reporter.error( + path=self.object_path, + code='subdirectory-missing-init', + msg='Ansible module subdirectories must contain an __init__.py' + ) + + +def re_compile(value): + """ + Argparse expects things to raise TypeError, re.compile raises an re.error + exception + + This function is a shorthand to convert the re.error exception to a + TypeError + """ + + try: + return re.compile(value) + except re.error as e: + raise TypeError(e) + + +def run(): + parser = argparse.ArgumentParser(prog="validate-modules") + parser.add_argument('plugins', nargs='+', + help='Path to module/plugin or module/plugin directory') + parser.add_argument('-w', '--warnings', help='Show warnings', + action='store_true') + parser.add_argument('--exclude', help='RegEx exclusion pattern', + type=re_compile) + parser.add_argument('--arg-spec', help='Analyze module argument spec', + action='store_true', default=False) + parser.add_argument('--base-branch', default=None, + help='Used in determining if new options were added') + parser.add_argument('--format', choices=['json', 'plain'], default='plain', + help='Output format. Default: "%(default)s"') + parser.add_argument('--output', default='-', + help='Output location, use "-" for stdout. ' + 'Default "%(default)s"') + parser.add_argument('--collection', + help='Specifies the path to the collection, when ' + 'validating files within a collection. Ensure ' + 'that ANSIBLE_COLLECTIONS_PATH is set so the ' + 'contents of the collection can be located') + parser.add_argument('--collection-version', + help='The collection\'s version number used to check ' + 'deprecations') + parser.add_argument('--plugin-type', + default='module', + help='The plugin type to validate. Defaults to %(default)s') + + args = parser.parse_args() + + args.plugins = [m.rstrip('/') for m in args.plugins] + + reporter = Reporter() + git_cache = GitCache(args.base_branch, args.plugin_type) + + check_dirs = set() + + routing = None + if args.collection: + routing_file = 'meta/runtime.yml' + # Load meta/runtime.yml if it exists, as it may contain deprecation information + if os.path.isfile(routing_file): + try: + with open(routing_file) as f: + routing = yaml.safe_load(f) + except yaml.error.MarkedYAMLError as ex: + print('%s:%d:%d: YAML load failed: %s' % (routing_file, ex.context_mark.line + 1, ex.context_mark.column + 1, re.sub(r'\s+', ' ', str(ex)))) + except Exception as ex: # pylint: disable=broad-except + print('%s:%d:%d: YAML load failed: %s' % (routing_file, 0, 0, re.sub(r'\s+', ' ', str(ex)))) + + for plugin in args.plugins: + if os.path.isfile(plugin): + path = plugin + if args.exclude and args.exclude.search(path): + continue + if ModuleValidator.is_on_rejectlist(path): + continue + with ModuleValidator(path, collection=args.collection, collection_version=args.collection_version, + analyze_arg_spec=args.arg_spec, base_branch=args.base_branch, + git_cache=git_cache, reporter=reporter, routing=routing, + plugin_type=args.plugin_type) as mv1: + mv1.validate() + check_dirs.add(os.path.dirname(path)) + + for root, dirs, files in os.walk(plugin): + basedir = root[len(plugin) + 1:].split('/', 1)[0] + if basedir in REJECTLIST_DIRS: + continue + for dirname in dirs: + if root == plugin and dirname in REJECTLIST_DIRS: + continue + path = os.path.join(root, dirname) + if args.exclude and args.exclude.search(path): + continue + check_dirs.add(path) + + for filename in files: + path = os.path.join(root, filename) + if args.exclude and args.exclude.search(path): + continue + if ModuleValidator.is_on_rejectlist(path): + continue + with ModuleValidator(path, collection=args.collection, collection_version=args.collection_version, + analyze_arg_spec=args.arg_spec, base_branch=args.base_branch, + git_cache=git_cache, reporter=reporter, routing=routing, + plugin_type=args.plugin_type) as mv2: + mv2.validate() + + if not args.collection and args.plugin_type == 'module': + for path in sorted(check_dirs): + pv = PythonPackageValidator(path, reporter=reporter) + pv.validate() + + if args.format == 'plain': + sys.exit(reporter.plain(warnings=args.warnings, output=args.output)) + else: + sys.exit(reporter.json(warnings=args.warnings, output=args.output)) + + +class GitCache: + def __init__(self, base_branch, plugin_type): + self.base_branch = base_branch + self.plugin_type = plugin_type + + self.rel_path = 'lib/ansible/modules/' + if plugin_type != 'module': + self.rel_path = 'lib/ansible/plugins/%s/' % plugin_type + + if self.base_branch: + self.base_tree = self._git(['ls-tree', '-r', '--name-only', self.base_branch, self.rel_path]) + else: + self.base_tree = [] + + try: + self.head_tree = self._git(['ls-tree', '-r', '--name-only', 'HEAD', self.rel_path]) + except GitError as ex: + if ex.status == 128: + # fallback when there is no .git directory + self.head_tree = self._get_module_files() + else: + raise + except FileNotFoundError: + # fallback when git is not installed + self.head_tree = self._get_module_files() + + allowed_exts = ('.py', '.ps1') + if plugin_type != 'module': + allowed_exts = ('.py', ) + self.base_module_paths = dict((os.path.basename(p), p) for p in self.base_tree if os.path.splitext(p)[1] in allowed_exts) + + self.base_module_paths.pop('__init__.py', None) + + self.head_aliased_modules = set() + + for path in self.head_tree: + filename = os.path.basename(path) + + if filename.startswith('_') and filename != '__init__.py': + if os.path.islink(path): + self.head_aliased_modules.add(os.path.basename(os.path.realpath(path))) + + def _get_module_files(self): + module_files = [] + + for (dir_path, dir_names, file_names) in os.walk(self.rel_path): + for file_name in file_names: + module_files.append(os.path.join(dir_path, file_name)) + + return module_files + + @staticmethod + def _git(args): + cmd = ['git'] + args + p = subprocess.run(cmd, stdin=subprocess.DEVNULL, capture_output=True, text=True, check=False) + + if p.returncode != 0: + raise GitError(p.stderr, p.returncode) + + return p.stdout.splitlines() + + +class GitError(Exception): + def __init__(self, message, status): + super(GitError, self).__init__(message) + + self.status = status + + +def main(): + try: + run() + except KeyboardInterrupt: + pass diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py new file mode 100644 index 0000000..03a1401 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2016 Matt Martz <matt@sivel.net> +# Copyright (C) 2016 Rackspace US, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +from __future__ import annotations + +import runpy +import inspect +import json +import os +import subprocess +import sys + +from contextlib import contextmanager + +from ansible.executor.powershell.module_manifest import PSModuleDepFinder +from ansible.module_utils.basic import FILE_COMMON_ARGUMENTS, AnsibleModule +from ansible.module_utils.six import reraise +from ansible.module_utils._text import to_bytes, to_text + +from .utils import CaptureStd, find_executable, get_module_name_from_filename + + +ANSIBLE_MODULE_CONSTRUCTOR_ARGS = tuple(list(inspect.signature(AnsibleModule.__init__).parameters)[1:]) + + +class AnsibleModuleCallError(RuntimeError): + pass + + +class AnsibleModuleImportError(ImportError): + pass + + +class AnsibleModuleNotInitialized(Exception): + pass + + +class _FakeAnsibleModuleInit: + def __init__(self): + self.args = tuple() + self.kwargs = {} + self.called = False + + def __call__(self, *args, **kwargs): + if args and isinstance(args[0], AnsibleModule): + # Make sure, due to creative calling, that we didn't end up with + # ``self`` in ``args`` + self.args = args[1:] + else: + self.args = args + self.kwargs = kwargs + self.called = True + raise AnsibleModuleCallError('AnsibleModuleCallError') + + +def _fake_load_params(): + pass + + +@contextmanager +def setup_env(filename): + # Used to clean up imports later + pre_sys_modules = list(sys.modules.keys()) + + fake = _FakeAnsibleModuleInit() + module = __import__('ansible.module_utils.basic').module_utils.basic + _original_init = module.AnsibleModule.__init__ + _original_load_params = module._load_params + setattr(module.AnsibleModule, '__init__', fake) + setattr(module, '_load_params', _fake_load_params) + + try: + yield fake + finally: + setattr(module.AnsibleModule, '__init__', _original_init) + setattr(module, '_load_params', _original_load_params) + + # Clean up imports to prevent issues with mutable data being used in modules + for k in list(sys.modules.keys()): + # It's faster if we limit to items in ansible.module_utils + # But if this causes problems later, we should remove it + if k not in pre_sys_modules and k.startswith('ansible.module_utils.'): + del sys.modules[k] + + +def get_ps_argument_spec(filename, collection): + fqc_name = get_module_name_from_filename(filename, collection) + + pwsh = find_executable('pwsh') + if not pwsh: + raise FileNotFoundError('Required program for PowerShell arg spec inspection "pwsh" not found.') + + module_path = os.path.join(os.getcwd(), filename) + b_module_path = to_bytes(module_path, errors='surrogate_or_strict') + with open(b_module_path, mode='rb') as module_fd: + b_module_data = module_fd.read() + + ps_dep_finder = PSModuleDepFinder() + ps_dep_finder.scan_module(b_module_data, fqn=fqc_name) + + # For ps_argspec.ps1 to compile Ansible.Basic it also needs the AddType module_util. + ps_dep_finder._add_module(name=b"Ansible.ModuleUtils.AddType", ext=".psm1", fqn=None, optional=False, wrapper=False) + + util_manifest = json.dumps({ + 'module_path': to_text(module_path, errors='surrogate_or_strict'), + 'ansible_basic': ps_dep_finder.cs_utils_module["Ansible.Basic"]['path'], + 'ps_utils': {name: info['path'] for name, info in ps_dep_finder.ps_modules.items()} + }) + + script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ps_argspec.ps1') + proc = subprocess.run(['pwsh', script_path, util_manifest], stdin=subprocess.DEVNULL, capture_output=True, text=True, check=False) + + if proc.returncode != 0: + raise AnsibleModuleImportError("STDOUT:\n%s\nSTDERR:\n%s" % (proc.stdout, proc.stderr)) + + kwargs = json.loads(proc.stdout) + + # the validate-modules code expects the options spec to be under the argument_spec key not options as set in PS + kwargs['argument_spec'] = kwargs.pop('options', {}) + + return kwargs['argument_spec'], kwargs + + +def get_py_argument_spec(filename, collection): + name = get_module_name_from_filename(filename, collection) + + with setup_env(filename) as fake: + try: + with CaptureStd(): + runpy.run_module(name, run_name='__main__', alter_sys=True) + except AnsibleModuleCallError: + pass + except BaseException as e: + # we want to catch all exceptions here, including sys.exit + reraise(AnsibleModuleImportError, AnsibleModuleImportError('%s' % e), sys.exc_info()[2]) + + if not fake.called: + raise AnsibleModuleNotInitialized() + + try: + # Convert positional arguments to kwargs to make sure that all parameters are actually checked + for arg, arg_name in zip(fake.args, ANSIBLE_MODULE_CONSTRUCTOR_ARGS): + fake.kwargs[arg_name] = arg + # for ping kwargs == {'argument_spec':{'data':{'type':'str','default':'pong'}}, 'supports_check_mode':True} + argument_spec = fake.kwargs.get('argument_spec') or {} + # If add_file_common_args is truish, add options from FILE_COMMON_ARGUMENTS when not present. + # This is the only modification to argument_spec done by AnsibleModule itself, and which is + # not caught by setup_env's AnsibleModule replacement + if fake.kwargs.get('add_file_common_args'): + for k, v in FILE_COMMON_ARGUMENTS.items(): + if k not in argument_spec: + argument_spec[k] = v + return argument_spec, fake.kwargs + except (TypeError, IndexError): + return {}, {} + + +def get_argument_spec(filename, collection): + if filename.endswith('.py'): + return get_py_argument_spec(filename, collection) + else: + return get_ps_argument_spec(filename, collection) diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/ps_argspec.ps1 b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/ps_argspec.ps1 new file mode 100644 index 0000000..4183b2b --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/ps_argspec.ps1 @@ -0,0 +1,121 @@ +#Requires -Version 6 + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" +$WarningPreference = "Stop" + +Function Resolve-CircularReference { + <# + .SYNOPSIS + Removes known types that cause a circular reference in their json serialization. + + .PARAMETER Hash + The hash to scan for circular references + #> + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true)] + [System.Collections.IDictionary] + $Hash + ) + + foreach ($key in [String[]]$Hash.Keys) { + $value = $Hash[$key] + if ($value -is [System.Collections.IDictionary]) { + Resolve-CircularReference -Hash $value + } + elseif ($value -is [Array] -or $value -is [System.Collections.IList]) { + $values = @(foreach ($v in $value) { + if ($v -is [System.Collections.IDictionary]) { + Resolve-CircularReference -Hash $v + } + , $v + }) + $Hash[$key] = $values + } + elseif ($value -is [DateTime]) { + $Hash[$key] = $value.ToString("yyyy-MM-dd") + } + elseif ($value -is [delegate]) { + # Type can be set to a delegate function which defines it's own type. For the documentation we just + # reflection that as raw + if ($key -eq 'type') { + $Hash[$key] = 'raw' + } + else { + $Hash[$key] = $value.ToString() # Shouldn't ever happen but just in case. + } + } + } +} + +$manifest = ConvertFrom-Json -InputObject $args[0] -AsHashtable +if (-not $manifest.Contains('module_path') -or -not $manifest.module_path) { + Write-Error -Message "No module specified." + exit 1 +} +$module_path = $manifest.module_path + +# Check if the path is relative and get the full path to the module +if (-not ([System.IO.Path]::IsPathRooted($module_path))) { + $module_path = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($module_path) +} + +if (-not (Test-Path -LiteralPath $module_path -PathType Leaf)) { + Write-Error -Message "The module at '$module_path' does not exist." + exit 1 +} + +$module_code = Get-Content -LiteralPath $module_path -Raw + +$powershell = [PowerShell]::Create() +$powershell.Runspace.SessionStateProxy.SetVariable("ErrorActionPreference", "Stop") + +# Load the PowerShell module utils as the module may be using them to refer to shared module options. Currently we +# can only load the PowerShell utils due to cross platform compatibility issues. +if ($manifest.Contains('ps_utils')) { + foreach ($util_info in $manifest.ps_utils.GetEnumerator()) { + $util_name = $util_info.Key + $util_path = $util_info.Value + + if (-not (Test-Path -LiteralPath $util_path -PathType Leaf)) { + # Failed to find the util path, just silently ignore for now and hope for the best. + continue + } + + $util_sb = [ScriptBlock]::Create((Get-Content -LiteralPath $util_path -Raw)) + $powershell.AddCommand('New-Module').AddParameters(@{ + Name = $util_name + ScriptBlock = $util_sb + }) > $null + $powershell.AddCommand('Import-Module').AddParameter('WarningAction', 'SilentlyContinue') > $null + $powershell.AddCommand('Out-Null').AddStatement() > $null + + # Also import it into the current runspace in case ps_argspec.ps1 needs to use it. + $null = New-Module -Name $util_name -ScriptBlock $util_sb | Import-Module -WarningAction SilentlyContinue + } +} + +Add-CSharpType -References @(Get-Content -LiteralPath $manifest.ansible_basic -Raw) +[Ansible.Basic.AnsibleModule]::_DebugArgSpec = $true + +$powershell.AddScript($module_code) > $null +$powershell.Invoke() > $null +$arg_spec = $powershell.Runspace.SessionStateProxy.GetVariable('ansibleTestArgSpec') + +if (-not $arg_spec) { + $err = $powershell.Streams.Error + if ($err) { + $err + } + else { + "Unknown error trying to get PowerShell arg spec" + } + + exit 1 +} + + +Resolve-CircularReference -Hash $arg_spec + +ConvertTo-Json -InputObject $arg_spec -Compress -Depth 99 diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py new file mode 100644 index 0000000..b2623ff --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py @@ -0,0 +1,899 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2015, Matt Martz <matt@sivel.net> +# Copyright: (c) 2015, Rackspace US, Inc. +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import annotations + +import re + +from ansible.module_utils.compat.version import StrictVersion +from functools import partial +from urllib.parse import urlparse + +from voluptuous import ALLOW_EXTRA, PREVENT_EXTRA, All, Any, Invalid, Length, Required, Schema, Self, ValueInvalid, Exclusive +from ansible.module_utils.six import string_types +from ansible.module_utils.common.collections import is_iterable +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.parsing.quoting import unquote +from ansible.utils.version import SemanticVersion +from ansible.release import __version__ + +from .utils import parse_isodate + +list_string_types = list(string_types) +tuple_string_types = tuple(string_types) +any_string_types = Any(*string_types) + +# Valid DOCUMENTATION.author lines +# Based on Ansibulbot's extract_github_id() +# author: First Last (@name) [optional anything] +# "Ansible Core Team" - Used by the Bot +# "Michael DeHaan" - nop +# "OpenStack Ansible SIG" - OpenStack does not use GitHub +# "Name (!UNKNOWN)" - For the few untraceable authors +author_line = re.compile(r'^\w.*(\(@([\w-]+)\)|!UNKNOWN)(?![\w.])|^Ansible Core Team$|^Michael DeHaan$|^OpenStack Ansible SIG$') + + +def _add_ansible_error_code(exception, error_code): + setattr(exception, 'ansible_error_code', error_code) + return exception + + +def isodate(v, error_code=None): + try: + parse_isodate(v, allow_date=True) + except ValueError as e: + raise _add_ansible_error_code(Invalid(str(e)), error_code or 'ansible-invalid-date') + return v + + +COLLECTION_NAME_RE = re.compile(r'^\w+(?:\.\w+)+$') +FULLY_QUALIFIED_COLLECTION_RESOURCE_RE = re.compile(r'^\w+(?:\.\w+){2,}$') + + +def collection_name(v, error_code=None): + if not isinstance(v, string_types): + raise _add_ansible_error_code( + Invalid('Collection name must be a string'), error_code or 'collection-invalid-name') + m = COLLECTION_NAME_RE.match(v) + if not m: + raise _add_ansible_error_code( + Invalid('Collection name must be of format `<namespace>.<name>`'), error_code or 'collection-invalid-name') + return v + + +def deprecation_versions(): + """Create a list of valid version for deprecation entries, current+4""" + major, minor = [int(version) for version in __version__.split('.')[0:2]] + return Any(*['{0}.{1}'.format(major, minor + increment) for increment in range(0, 5)]) + + +def version(for_collection=False): + if for_collection: + # We do not accept floats for versions in collections + return Any(*string_types) + return Any(float, *string_types) + + +def date(error_code=None): + return Any(isodate, error_code=error_code) + + +_MODULE = re.compile(r"\bM\(([^)]+)\)") +_LINK = re.compile(r"\bL\(([^)]+)\)") +_URL = re.compile(r"\bU\(([^)]+)\)") +_REF = re.compile(r"\bR\(([^)]+)\)") + + +def _check_module_link(directive, content): + if not FULLY_QUALIFIED_COLLECTION_RESOURCE_RE.match(content): + raise _add_ansible_error_code( + Invalid('Directive "%s" must contain a FQCN' % directive), 'invalid-documentation-markup') + + +def _check_link(directive, content): + if ',' not in content: + raise _add_ansible_error_code( + Invalid('Directive "%s" must contain a comma' % directive), 'invalid-documentation-markup') + idx = content.rindex(',') + title = content[:idx] + url = content[idx + 1:].lstrip(' ') + _check_url(directive, url) + + +def _check_url(directive, content): + try: + parsed_url = urlparse(content) + if parsed_url.scheme not in ('', 'http', 'https'): + raise ValueError('Schema must be HTTP, HTTPS, or not specified') + except ValueError as exc: + raise _add_ansible_error_code( + Invalid('Directive "%s" must contain an URL' % directive), 'invalid-documentation-markup') + + +def _check_ref(directive, content): + if ',' not in content: + raise _add_ansible_error_code( + Invalid('Directive "%s" must contain a comma' % directive), 'invalid-documentation-markup') + + +def doc_string(v): + """Match a documentation string.""" + if not isinstance(v, string_types): + raise _add_ansible_error_code( + Invalid('Must be a string'), 'invalid-documentation') + for m in _MODULE.finditer(v): + _check_module_link(m.group(0), m.group(1)) + for m in _LINK.finditer(v): + _check_link(m.group(0), m.group(1)) + for m in _URL.finditer(v): + _check_url(m.group(0), m.group(1)) + for m in _REF.finditer(v): + _check_ref(m.group(0), m.group(1)) + return v + + +def doc_string_or_strings(v): + """Match a documentation string, or list of strings.""" + if isinstance(v, string_types): + return doc_string(v) + if isinstance(v, (list, tuple)): + return [doc_string(vv) for vv in v] + raise _add_ansible_error_code( + Invalid('Must be a string or list of strings'), 'invalid-documentation') + + +def is_callable(v): + if not callable(v): + raise ValueInvalid('not a valid value') + return v + + +def sequence_of_sequences(min=None, max=None): + return All( + Any( + None, + [Any(list, tuple)], + tuple([Any(list, tuple)]), + ), + Any( + None, + [Length(min=min, max=max)], + tuple([Length(min=min, max=max)]), + ), + ) + + +seealso_schema = Schema( + [ + Any( + { + Required('module'): Any(*string_types), + 'description': doc_string, + }, + { + Required('ref'): Any(*string_types), + Required('description'): doc_string, + }, + { + Required('name'): Any(*string_types), + Required('link'): Any(*string_types), + Required('description'): doc_string, + }, + ), + ] +) + + +argument_spec_types = ['bits', 'bool', 'bytes', 'dict', 'float', 'int', 'json', 'jsonarg', 'list', 'path', 'raw', + 'sid', 'str'] + + +argument_spec_modifiers = { + 'mutually_exclusive': sequence_of_sequences(min=2), + 'required_together': sequence_of_sequences(min=2), + 'required_one_of': sequence_of_sequences(min=2), + 'required_if': sequence_of_sequences(min=3, max=4), + 'required_by': Schema({str: Any(list_string_types, tuple_string_types, *string_types)}), +} + + +def no_required_with_default(v): + if v.get('default') and v.get('required'): + raise Invalid('required=True cannot be supplied with a default') + return v + + +def elements_with_list(v): + if v.get('elements') and v.get('type') != 'list': + raise Invalid('type must be list to use elements') + return v + + +def options_with_apply_defaults(v): + if v.get('apply_defaults') and not v.get('options'): + raise Invalid('apply_defaults=True requires options to be set') + return v + + +def check_removal_version(v, version_field, collection_name_field, error_code='invalid-removal-version'): + version = v.get(version_field) + collection_name = v.get(collection_name_field) + if not isinstance(version, string_types) or not isinstance(collection_name, string_types): + # If they are not strings, schema validation will have already complained. + return v + if collection_name == 'ansible.builtin': + try: + parsed_version = StrictVersion() + parsed_version.parse(version) + except ValueError as exc: + raise _add_ansible_error_code( + Invalid('%s (%r) is not a valid ansible-core version: %s' % (version_field, version, exc)), + error_code=error_code) + return v + try: + parsed_version = SemanticVersion() + parsed_version.parse(version) + if parsed_version.major != 0 and (parsed_version.minor != 0 or parsed_version.patch != 0): + raise _add_ansible_error_code( + Invalid('%s (%r) must be a major release, not a minor or patch release (see specification at ' + 'https://semver.org/)' % (version_field, version)), + error_code='removal-version-must-be-major') + except ValueError as exc: + raise _add_ansible_error_code( + Invalid('%s (%r) is not a valid collection version (see specification at https://semver.org/): ' + '%s' % (version_field, version, exc)), + error_code=error_code) + return v + + +def option_deprecation(v): + if v.get('removed_in_version') or v.get('removed_at_date'): + if v.get('removed_in_version') and v.get('removed_at_date'): + raise _add_ansible_error_code( + Invalid('Only one of removed_in_version and removed_at_date must be specified'), + error_code='deprecation-either-date-or-version') + if not v.get('removed_from_collection'): + raise _add_ansible_error_code( + Invalid('If removed_in_version or removed_at_date is specified, ' + 'removed_from_collection must be specified as well'), + error_code='deprecation-collection-missing') + check_removal_version(v, + version_field='removed_in_version', + collection_name_field='removed_from_collection', + error_code='invalid-removal-version') + return + if v.get('removed_from_collection'): + raise Invalid('removed_from_collection cannot be specified without either ' + 'removed_in_version or removed_at_date') + + +def argument_spec_schema(for_collection): + any_string_types = Any(*string_types) + schema = { + any_string_types: { + 'type': Any(is_callable, *argument_spec_types), + 'elements': Any(*argument_spec_types), + 'default': object, + 'fallback': Any( + (is_callable, list_string_types), + [is_callable, list_string_types], + ), + 'choices': Any([object], (object,)), + 'required': bool, + 'no_log': bool, + 'aliases': Any(list_string_types, tuple(list_string_types)), + 'apply_defaults': bool, + 'removed_in_version': version(for_collection), + 'removed_at_date': date(), + 'removed_from_collection': collection_name, + 'options': Self, + 'deprecated_aliases': Any([All( + Any( + { + Required('name'): Any(*string_types), + Required('date'): date(), + Required('collection_name'): collection_name, + }, + { + Required('name'): Any(*string_types), + Required('version'): version(for_collection), + Required('collection_name'): collection_name, + }, + ), + partial(check_removal_version, + version_field='version', + collection_name_field='collection_name', + error_code='invalid-removal-version') + )]), + } + } + schema[any_string_types].update(argument_spec_modifiers) + schemas = All( + schema, + Schema({any_string_types: no_required_with_default}), + Schema({any_string_types: elements_with_list}), + Schema({any_string_types: options_with_apply_defaults}), + Schema({any_string_types: option_deprecation}), + ) + return Schema(schemas) + + +def ansible_module_kwargs_schema(module_name, for_collection): + schema = { + 'argument_spec': argument_spec_schema(for_collection), + 'bypass_checks': bool, + 'no_log': bool, + 'check_invalid_arguments': Any(None, bool), + 'add_file_common_args': bool, + 'supports_check_mode': bool, + } + if module_name.endswith(('_info', '_facts')): + del schema['supports_check_mode'] + schema[Required('supports_check_mode')] = True + schema.update(argument_spec_modifiers) + return Schema(schema) + + +json_value = Schema(Any( + None, + int, + float, + [Self], + *(list({str_type: Self} for str_type in string_types) + list(string_types)) +)) + + +def version_added(v, error_code='version-added-invalid', accept_historical=False): + if 'version_added' in v: + version_added = v.get('version_added') + if isinstance(version_added, string_types): + # If it is not a string, schema validation will have already complained + # - or we have a float and we are in ansible/ansible, in which case we're + # also happy. + if v.get('version_added_collection') == 'ansible.builtin': + if version_added == 'historical' and accept_historical: + return v + try: + version = StrictVersion() + version.parse(version_added) + except ValueError as exc: + raise _add_ansible_error_code( + Invalid('version_added (%r) is not a valid ansible-core version: ' + '%s' % (version_added, exc)), + error_code=error_code) + else: + try: + version = SemanticVersion() + version.parse(version_added) + if version.major != 0 and version.patch != 0: + raise _add_ansible_error_code( + Invalid('version_added (%r) must be a major or minor release, ' + 'not a patch release (see specification at ' + 'https://semver.org/)' % (version_added, )), + error_code='version-added-must-be-major-or-minor') + except ValueError as exc: + raise _add_ansible_error_code( + Invalid('version_added (%r) is not a valid collection version ' + '(see specification at https://semver.org/): ' + '%s' % (version_added, exc)), + error_code=error_code) + elif 'version_added_collection' in v: + # Must have been manual intervention, since version_added_collection is only + # added automatically when version_added is present + raise Invalid('version_added_collection cannot be specified without version_added') + return v + + +def check_option_elements(v): + # Check whether elements is there iff type == 'list' + v_type = v.get('type') + v_elements = v.get('elements') + if v_type == 'list' and v_elements is None: + raise _add_ansible_error_code( + Invalid('Argument defines type as list but elements is not defined'), + error_code='parameter-list-no-elements') # FIXME: adjust error code? + if v_type != 'list' and v_elements is not None: + raise _add_ansible_error_code( + Invalid('Argument defines parameter elements as %s but it is valid only when value of parameter type is list' % (v_elements, )), + error_code='doc-elements-invalid') + return v + + +def get_type_checker(v): + v_type = v.get('type') + if v_type == 'list': + elt_checker, elt_name = get_type_checker({'type': v.get('elements')}) + + def list_checker(value): + if isinstance(value, string_types): + value = [unquote(x.strip()) for x in value.split(',')] + if not isinstance(value, list): + raise ValueError('Value must be a list') + if elt_checker: + for elt in value: + try: + elt_checker(elt) + except Exception as exc: + raise ValueError('Entry %r is not of type %s: %s' % (elt, elt_name, exc)) + + return list_checker, ('list of %s' % elt_name) if elt_checker else 'list' + + if v_type in ('boolean', 'bool'): + return partial(boolean, strict=False), v_type + + if v_type in ('integer', 'int'): + return int, v_type + + if v_type == 'float': + return float, v_type + + if v_type == 'none': + def none_checker(value): + if value not in ('None', None): + raise ValueError('Value must be "None" or none') + + return none_checker, v_type + + if v_type in ('str', 'string', 'path', 'tmp', 'temppath', 'tmppath'): + def str_checker(value): + if not isinstance(value, string_types): + raise ValueError('Value must be string') + + return str_checker, v_type + + if v_type in ('pathspec', 'pathlist'): + def path_list_checker(value): + if not isinstance(value, string_types) and not is_iterable(value): + raise ValueError('Value must be string or list of strings') + + return path_list_checker, v_type + + if v_type in ('dict', 'dictionary'): + def dict_checker(value): + if not isinstance(value, dict): + raise ValueError('Value must be dictionary') + + return dict_checker, v_type + + return None, 'unknown' + + +def check_option_choices(v): + # Check whether choices have the correct type + v_choices = v.get('choices') + if not is_iterable(v_choices): + return v + + if v.get('type') == 'list': + # choices for a list type means that every list element must be one of these choices + type_checker, type_name = get_type_checker({'type': v.get('elements')}) + else: + type_checker, type_name = get_type_checker(v) + if type_checker is None: + return v + + for value in v_choices: + try: + type_checker(value) + except Exception as exc: + raise _add_ansible_error_code( + Invalid( + 'Argument defines choices as (%r) but this is incompatible with argument type %s: %s' % (value, type_name, exc)), + error_code='doc-choices-incompatible-type') + + return v + + +def check_option_default(v): + # Check whether default is only present if required=False, and whether default has correct type + v_default = v.get('default') + if v.get('required') and v_default is not None: + raise _add_ansible_error_code( + Invalid( + 'Argument is marked as required but specifies a default.' + ' Arguments with a default should not be marked as required'), + error_code='no-default-for-required-parameter') # FIXME: adjust error code? + + if v_default is None: + return v + + type_checker, type_name = get_type_checker(v) + if type_checker is None: + return v + + try: + type_checker(v_default) + except Exception as exc: + raise _add_ansible_error_code( + Invalid( + 'Argument defines default as (%r) but this is incompatible with parameter type %s: %s' % (v_default, type_name, exc)), + error_code='incompatible-default-type') + + return v + + +def list_dict_option_schema(for_collection, plugin_type): + if plugin_type == 'module': + option_types = Any(None, 'bits', 'bool', 'bytes', 'dict', 'float', 'int', 'json', 'jsonarg', 'list', 'path', 'raw', 'sid', 'str') + element_types = option_types + else: + option_types = Any(None, 'boolean', 'bool', 'integer', 'int', 'float', 'list', 'dict', 'dictionary', 'none', + 'path', 'tmp', 'temppath', 'tmppath', 'pathspec', 'pathlist', 'str', 'string', 'raw') + element_types = Any(None, 'boolean', 'bool', 'integer', 'int', 'float', 'list', 'dict', 'dictionary', 'path', 'str', 'string', 'raw') + + basic_option_schema = { + Required('description'): doc_string_or_strings, + 'required': bool, + 'choices': list, + 'aliases': Any(list_string_types), + 'version_added': version(for_collection), + 'version_added_collection': collection_name, + 'default': json_value, + # Note: Types are strings, not literal bools, such as True or False + 'type': option_types, + # in case of type='list' elements define type of individual item in list + 'elements': element_types, + } + if plugin_type != 'module': + basic_option_schema['name'] = Any(*string_types) + deprecated_schema = All( + Schema( + All( + { + # This definition makes sure everything has the correct types/values + 'why': doc_string, + 'alternatives': doc_string, + # vod stands for 'version or date'; this is the name of the exclusive group + Exclusive('removed_at_date', 'vod'): date(), + Exclusive('version', 'vod'): version(for_collection), + 'collection_name': collection_name, + }, + { + # This definition makes sure that everything we require is there + Required('why'): Any(*string_types), + 'alternatives': Any(*string_types), + Required(Any('removed_at_date', 'version')): Any(*string_types), + Required('collection_name'): Any(*string_types), + }, + ), + extra=PREVENT_EXTRA + ), + partial(check_removal_version, + version_field='version', + collection_name_field='collection_name', + error_code='invalid-removal-version'), + ) + env_schema = All( + Schema({ + Required('name'): Any(*string_types), + 'deprecated': deprecated_schema, + 'version_added': version(for_collection), + 'version_added_collection': collection_name, + }, extra=PREVENT_EXTRA), + partial(version_added, error_code='option-invalid-version-added') + ) + ini_schema = All( + Schema({ + Required('key'): Any(*string_types), + Required('section'): Any(*string_types), + 'deprecated': deprecated_schema, + 'version_added': version(for_collection), + 'version_added_collection': collection_name, + }, extra=PREVENT_EXTRA), + partial(version_added, error_code='option-invalid-version-added') + ) + vars_schema = All( + Schema({ + Required('name'): Any(*string_types), + 'deprecated': deprecated_schema, + 'version_added': version(for_collection), + 'version_added_collection': collection_name, + }, extra=PREVENT_EXTRA), + partial(version_added, error_code='option-invalid-version-added') + ) + cli_schema = All( + Schema({ + Required('name'): Any(*string_types), + 'option': Any(*string_types), + 'deprecated': deprecated_schema, + 'version_added': version(for_collection), + 'version_added_collection': collection_name, + }, extra=PREVENT_EXTRA), + partial(version_added, error_code='option-invalid-version-added') + ) + keyword_schema = All( + Schema({ + Required('name'): Any(*string_types), + 'deprecated': deprecated_schema, + 'version_added': version(for_collection), + 'version_added_collection': collection_name, + }, extra=PREVENT_EXTRA), + partial(version_added, error_code='option-invalid-version-added') + ) + basic_option_schema.update({ + 'env': [env_schema], + 'ini': [ini_schema], + 'vars': [vars_schema], + 'cli': [cli_schema], + 'keyword': [keyword_schema], + 'deprecated': deprecated_schema, + }) + + suboption_schema = dict(basic_option_schema) + suboption_schema.update({ + # Recursive suboptions + 'suboptions': Any(None, *list({str_type: Self} for str_type in string_types)), + }) + suboption_schema = Schema(All( + suboption_schema, + check_option_elements, + check_option_choices, + check_option_default, + ), extra=PREVENT_EXTRA) + + # This generates list of dicts with keys from string_types and suboption_schema value + # for example in Python 3: {str: suboption_schema} + list_dict_suboption_schema = [{str_type: suboption_schema} for str_type in string_types] + + option_schema = dict(basic_option_schema) + option_schema.update({ + 'suboptions': Any(None, *list_dict_suboption_schema), + }) + option_schema = Schema(All( + option_schema, + check_option_elements, + check_option_choices, + check_option_default, + ), extra=PREVENT_EXTRA) + + option_version_added = Schema( + All({ + 'suboptions': Any(None, *[{str_type: Self} for str_type in string_types]), + }, partial(version_added, error_code='option-invalid-version-added')), + extra=ALLOW_EXTRA + ) + + # This generates list of dicts with keys from string_types and option_schema value + # for example in Python 3: {str: option_schema} + return [{str_type: All(option_schema, option_version_added)} for str_type in string_types] + + +def return_contains(v): + schema = Schema( + { + Required('contains'): Any(dict, list, *string_types) + }, + extra=ALLOW_EXTRA + ) + if v.get('type') == 'complex': + return schema(v) + return v + + +def return_schema(for_collection, plugin_type='module'): + if plugin_type == 'module': + return_types = Any('bool', 'complex', 'dict', 'float', 'int', 'list', 'raw', 'str') + element_types = Any(None, 'bits', 'bool', 'bytes', 'dict', 'float', 'int', 'json', 'jsonarg', 'list', 'path', 'raw', 'sid', 'str') + else: + return_types = Any(None, 'boolean', 'bool', 'integer', 'int', 'float', 'list', 'dict', 'dictionary', 'path', 'str', 'string', 'raw') + element_types = return_types + + basic_return_option_schema = { + Required('description'): doc_string_or_strings, + 'returned': doc_string, + 'version_added': version(for_collection), + 'version_added_collection': collection_name, + 'sample': json_value, + 'example': json_value, + # in case of type='list' elements define type of individual item in list + 'elements': element_types, + 'choices': Any([object], (object,)), + } + if plugin_type == 'module': + # type is only required for modules right now + basic_return_option_schema[Required('type')] = return_types + else: + basic_return_option_schema['type'] = return_types + + inner_return_option_schema = dict(basic_return_option_schema) + inner_return_option_schema.update({ + 'contains': Any(None, *list({str_type: Self} for str_type in string_types)), + }) + return_contains_schema = Any( + All( + Schema(inner_return_option_schema), + Schema(return_contains), + Schema(partial(version_added, error_code='option-invalid-version-added')), + ), + Schema(type(None)), + ) + + # This generates list of dicts with keys from string_types and return_contains_schema value + # for example in Python 3: {str: return_contains_schema} + list_dict_return_contains_schema = [{str_type: return_contains_schema} for str_type in string_types] + + return_option_schema = dict(basic_return_option_schema) + return_option_schema.update({ + 'contains': Any(None, *list_dict_return_contains_schema), + }) + if plugin_type == 'module': + # 'returned' is required on top-level + del return_option_schema['returned'] + return_option_schema[Required('returned')] = Any(*string_types) + return Any( + All( + Schema( + { + any_string_types: return_option_schema + } + ), + Schema({any_string_types: return_contains}), + Schema({any_string_types: partial(version_added, error_code='option-invalid-version-added')}), + ), + Schema(type(None)), + ) + + +def deprecation_schema(for_collection): + main_fields = { + Required('why'): doc_string, + Required('alternative'): doc_string, + Required('removed_from_collection'): collection_name, + 'removed': Any(True), + } + + date_schema = { + Required('removed_at_date'): date(), + } + date_schema.update(main_fields) + + if for_collection: + version_schema = { + Required('removed_in'): version(for_collection), + } + else: + version_schema = { + Required('removed_in'): deprecation_versions(), + } + version_schema.update(main_fields) + + result = Any( + Schema(version_schema, extra=PREVENT_EXTRA), + Schema(date_schema, extra=PREVENT_EXTRA), + ) + + if for_collection: + result = All( + result, + partial(check_removal_version, + version_field='removed_in', + collection_name_field='removed_from_collection', + error_code='invalid-removal-version')) + + return result + + +def author(value): + if value is None: + return value # let schema checks handle + + if not is_iterable(value): + value = [value] + + for line in value: + if not isinstance(line, string_types): + continue # let schema checks handle + m = author_line.search(line) + if not m: + raise Invalid("Invalid author") + + return value + + +def doc_schema(module_name, for_collection=False, deprecated_module=False, plugin_type='module'): + + if module_name.startswith('_'): + module_name = module_name[1:] + deprecated_module = True + if for_collection is False and plugin_type == 'connection' and module_name == 'paramiko_ssh': + # The plugin loader has a hard-coded exception: when the builtin connection 'paramiko' is + # referenced, it loads 'paramiko_ssh' instead. That's why in this plugin, the name must be + # 'paramiko' and not 'paramiko_ssh'. + module_name = 'paramiko' + doc_schema_dict = { + Required('module' if plugin_type == 'module' else 'name'): module_name, + Required('short_description'): doc_string, + Required('description'): doc_string_or_strings, + 'notes': Any(None, [doc_string]), + 'seealso': Any(None, seealso_schema), + 'requirements': [doc_string], + 'todo': Any(None, doc_string_or_strings), + 'options': Any(None, *list_dict_option_schema(for_collection, plugin_type)), + 'extends_documentation_fragment': Any(list_string_types, *string_types), + 'version_added_collection': collection_name, + } + if plugin_type == 'module': + doc_schema_dict[Required('author')] = All(Any(None, list_string_types, *string_types), author) + else: + # author is optional for plugins (for now) + doc_schema_dict['author'] = All(Any(None, list_string_types, *string_types), author) + if plugin_type == 'callback': + doc_schema_dict[Required('type')] = Any('aggregate', 'notification', 'stdout') + + if for_collection: + # Optional + doc_schema_dict['version_added'] = version(for_collection=True) + else: + doc_schema_dict[Required('version_added')] = version(for_collection=False) + + if deprecated_module: + deprecation_required_scheme = { + Required('deprecated'): Any(deprecation_schema(for_collection=for_collection)), + } + + doc_schema_dict.update(deprecation_required_scheme) + + def add_default_attributes(more=None): + schema = { + 'description': doc_string_or_strings, + 'details': doc_string_or_strings, + 'support': any_string_types, + 'version_added_collection': any_string_types, + 'version_added': any_string_types, + } + if more: + schema.update(more) + return schema + + doc_schema_dict['attributes'] = Schema( + All( + Schema({ + any_string_types: { + Required('description'): doc_string_or_strings, + Required('support'): Any('full', 'partial', 'none', 'N/A'), + 'details': doc_string_or_strings, + 'version_added_collection': collection_name, + 'version_added': version(for_collection=for_collection), + }, + }, extra=ALLOW_EXTRA), + partial(version_added, error_code='attribute-invalid-version-added', accept_historical=False), + Schema({ + any_string_types: add_default_attributes(), + 'action_group': add_default_attributes({ + Required('membership'): list_string_types, + }), + 'forced_action_plugin': add_default_attributes({ + Required('action_plugin'): any_string_types, + }), + 'platform': add_default_attributes({ + Required('platforms'): Any(list_string_types, *string_types) + }), + }, extra=PREVENT_EXTRA), + ) + ) + return Schema( + All( + Schema( + doc_schema_dict, + extra=PREVENT_EXTRA + ), + partial(version_added, error_code='module-invalid-version-added', accept_historical=not for_collection), + ) + ) + + +# Things to add soon +#################### +# 1) Recursively validate `type: complex` fields +# This will improve documentation, though require fair amount of module tidyup + +# Possible Future Enhancements +############################## + +# 1) Don't allow empty options for choices, aliases, etc +# 2) If type: bool ensure choices isn't set - perhaps use Exclusive +# 3) both version_added should be quoted floats + +# Tool that takes JSON and generates RETURN skeleton (needs to support complex structures) diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/utils.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/utils.py new file mode 100644 index 0000000..88d5b01 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/utils.py @@ -0,0 +1,222 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2015 Matt Martz <matt@sivel.net> +# Copyright (C) 2015 Rackspace US, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +from __future__ import annotations + +import ast +import datetime +import os +import re +import sys + +from io import BytesIO, TextIOWrapper + +import yaml +import yaml.reader + +from ansible.module_utils._text import to_text +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.common.yaml import SafeLoader +from ansible.module_utils.six import string_types +from ansible.parsing.yaml.loader import AnsibleLoader + + +class AnsibleTextIOWrapper(TextIOWrapper): + def write(self, s): + super(AnsibleTextIOWrapper, self).write(to_text(s, self.encoding, errors='replace')) + + +def find_executable(executable, cwd=None, path=None): + """Finds the full path to the executable specified""" + match = None + real_cwd = os.getcwd() + + if not cwd: + cwd = real_cwd + + if os.path.dirname(executable): + target = os.path.join(cwd, executable) + if os.path.exists(target) and os.access(target, os.F_OK | os.X_OK): + match = executable + else: + path = os.environ.get('PATH', os.path.defpath) + + path_dirs = path.split(os.path.pathsep) + seen_dirs = set() + + for path_dir in path_dirs: + if path_dir in seen_dirs: + continue + + seen_dirs.add(path_dir) + + if os.path.abspath(path_dir) == real_cwd: + path_dir = cwd + + candidate = os.path.join(path_dir, executable) + + if os.path.exists(candidate) and os.access(candidate, os.F_OK | os.X_OK): + match = candidate + break + + return match + + +def find_globals(g, tree): + """Uses AST to find globals in an ast tree""" + for child in tree: + if hasattr(child, 'body') and isinstance(child.body, list): + find_globals(g, child.body) + elif isinstance(child, (ast.FunctionDef, ast.ClassDef)): + g.add(child.name) + continue + elif isinstance(child, ast.Assign): + try: + g.add(child.targets[0].id) + except (IndexError, AttributeError): + pass + elif isinstance(child, ast.Import): + g.add(child.names[0].name) + elif isinstance(child, ast.ImportFrom): + for name in child.names: + g_name = name.asname or name.name + if g_name == '*': + continue + g.add(g_name) + + +class CaptureStd(): + """Context manager to handle capturing stderr and stdout""" + + def __enter__(self): + self.sys_stdout = sys.stdout + self.sys_stderr = sys.stderr + sys.stdout = self.stdout = AnsibleTextIOWrapper(BytesIO(), encoding=self.sys_stdout.encoding) + sys.stderr = self.stderr = AnsibleTextIOWrapper(BytesIO(), encoding=self.sys_stderr.encoding) + return self + + def __exit__(self, exc_type, exc_value, traceback): + sys.stdout = self.sys_stdout + sys.stderr = self.sys_stderr + + def get(self): + """Return ``(stdout, stderr)``""" + + return self.stdout.buffer.getvalue(), self.stderr.buffer.getvalue() + + +def get_module_name_from_filename(filename, collection): + # Calculate the module's name so that relative imports work correctly + if collection: + # collection is a relative path, example: ansible_collections/my_namespace/my_collection + # filename is a relative path, example: plugins/modules/my_module.py + path = os.path.join(collection, filename) + else: + # filename is a relative path, example: lib/ansible/modules/system/ping.py + path = os.path.relpath(filename, 'lib') + + name = os.path.splitext(path)[0].replace(os.path.sep, '.') + + return name + + +def parse_yaml(value, lineno, module, name, load_all=False, ansible_loader=False): + traces = [] + errors = [] + data = None + + if load_all: + yaml_load = yaml.load_all + else: + yaml_load = yaml.load + + if ansible_loader: + loader = AnsibleLoader + else: + loader = SafeLoader + + try: + data = yaml_load(value, Loader=loader) + if load_all: + data = list(data) + except yaml.MarkedYAMLError as e: + errors.append({ + 'msg': '%s is not valid YAML' % name, + 'line': e.problem_mark.line + lineno, + 'column': e.problem_mark.column + 1 + }) + traces.append(e) + except yaml.reader.ReaderError as e: + traces.append(e) + # TODO: Better line/column detection + errors.append({ + 'msg': ('%s is not valid YAML. Character ' + '0x%x at position %d.' % (name, e.character, e.position)), + 'line': lineno + }) + except yaml.YAMLError as e: + traces.append(e) + errors.append({ + 'msg': '%s is not valid YAML: %s: %s' % (name, type(e), e), + 'line': lineno + }) + + return data, errors, traces + + +def is_empty(value): + """Evaluate null like values excluding False""" + if value is False: + return False + return not bool(value) + + +def compare_unordered_lists(a, b): + """Safe list comparisons + + Supports: + - unordered lists + - unhashable elements + """ + return len(a) == len(b) and all(x in b for x in a) + + +class NoArgsAnsibleModule(AnsibleModule): + """AnsibleModule that does not actually load params. This is used to get access to the + methods within AnsibleModule without having to fake a bunch of data + """ + def _load_params(self): + self.params = {'_ansible_selinux_special_fs': [], '_ansible_remote_tmp': '/tmp', '_ansible_keep_remote_files': False, '_ansible_check_mode': False} + + +def parse_isodate(v, allow_date): + if allow_date: + if isinstance(v, datetime.date): + return v + msg = 'Expected ISO 8601 date string (YYYY-MM-DD) or YAML date' + else: + msg = 'Expected ISO 8601 date string (YYYY-MM-DD)' + if not isinstance(v, string_types): + raise ValueError(msg) + # From Python 3.7 in, there is datetime.date.fromisoformat(). For older versions, + # we have to do things manually. + if not re.match('^[0-9]{4}-[0-9]{2}-[0-9]{2}$', v): + raise ValueError(msg) + try: + return datetime.datetime.strptime(v, '%Y-%m-%d').date() + except ValueError: + raise ValueError(msg) diff --git a/test/lib/ansible_test/_util/controller/sanity/yamllint/config/default.yml b/test/lib/ansible_test/_util/controller/sanity/yamllint/config/default.yml new file mode 100644 index 0000000..45d8b7a --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/yamllint/config/default.yml @@ -0,0 +1,19 @@ +extends: default + +rules: + braces: {max-spaces-inside: 1, level: error} + brackets: {max-spaces-inside: 1, level: error} + colons: {max-spaces-after: -1, level: error} + commas: {max-spaces-after: -1, level: error} + comments: disable + comments-indentation: disable + document-start: disable + empty-lines: {max: 3, level: error} + hyphens: {level: error} + indentation: disable + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: {type: unix} + trailing-spaces: disable + truthy: disable diff --git a/test/lib/ansible_test/_util/controller/sanity/yamllint/config/modules.yml b/test/lib/ansible_test/_util/controller/sanity/yamllint/config/modules.yml new file mode 100644 index 0000000..da7e604 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/yamllint/config/modules.yml @@ -0,0 +1,19 @@ +extends: default + +rules: + braces: disable + brackets: disable + colons: disable + commas: disable + comments: disable + comments-indentation: disable + document-start: disable + empty-lines: disable + hyphens: disable + indentation: disable + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: {type: unix} + trailing-spaces: disable + truthy: disable diff --git a/test/lib/ansible_test/_util/controller/sanity/yamllint/config/plugins.yml b/test/lib/ansible_test/_util/controller/sanity/yamllint/config/plugins.yml new file mode 100644 index 0000000..6d41813 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/yamllint/config/plugins.yml @@ -0,0 +1,19 @@ +extends: default + +rules: + braces: disable + brackets: disable + colons: disable + commas: disable + comments: disable + comments-indentation: disable + document-start: disable + empty-lines: disable + hyphens: disable + indentation: disable + key-duplicates: disable + line-length: disable + new-line-at-end-of-file: disable + new-lines: {type: unix} + trailing-spaces: disable + truthy: disable diff --git a/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py b/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py new file mode 100644 index 0000000..d6de611 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py @@ -0,0 +1,246 @@ +"""Wrapper around yamllint that supports YAML embedded in Ansible modules.""" +from __future__ import annotations + +import ast +import json +import os +import re +import sys +import typing as t + +import yaml +from yaml.resolver import Resolver +from yaml.constructor import SafeConstructor +from yaml.error import MarkedYAMLError +from yaml.cyaml import CParser + +from yamllint import linter +from yamllint.config import YamlLintConfig + + +def main(): + """Main program body.""" + paths = sys.argv[1:] or sys.stdin.read().splitlines() + + checker = YamlChecker() + checker.check(paths) + checker.report() + + +class TestConstructor(SafeConstructor): + """Yaml Safe Constructor that knows about Ansible tags.""" + def construct_yaml_unsafe(self, node): + """Construct an unsafe tag.""" + try: + constructor = getattr(node, 'id', 'object') + if constructor is not None: + constructor = getattr(self, 'construct_%s' % constructor) + except AttributeError: + constructor = self.construct_object + + value = constructor(node) + + return value + + +TestConstructor.add_constructor( + '!unsafe', + TestConstructor.construct_yaml_unsafe) + + +TestConstructor.add_constructor( + '!vault', + TestConstructor.construct_yaml_str) + + +TestConstructor.add_constructor( + '!vault-encrypted', + TestConstructor.construct_yaml_str) + + +class TestLoader(CParser, TestConstructor, Resolver): + """Custom YAML loader that recognizes custom Ansible tags.""" + def __init__(self, stream): + CParser.__init__(self, stream) + TestConstructor.__init__(self) + Resolver.__init__(self) + + +class YamlChecker: + """Wrapper around yamllint that supports YAML embedded in Ansible modules.""" + def __init__(self): + self.messages = [] + + def report(self): + """Print yamllint report to stdout.""" + report = dict( + messages=self.messages, + ) + + print(json.dumps(report, indent=4, sort_keys=True)) + + def check(self, paths): # type: (t.List[str]) -> None + """Check the specified paths.""" + config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config') + + yaml_conf = YamlLintConfig(file=os.path.join(config_path, 'default.yml')) + module_conf = YamlLintConfig(file=os.path.join(config_path, 'modules.yml')) + plugin_conf = YamlLintConfig(file=os.path.join(config_path, 'plugins.yml')) + + for path in paths: + extension = os.path.splitext(path)[1] + + with open(path, encoding='utf-8') as file: + contents = file.read() + + if extension in ('.yml', '.yaml'): + self.check_yaml(yaml_conf, path, contents) + elif extension == '.py': + if path.startswith('lib/ansible/modules/') or path.startswith('plugins/modules/'): + conf = module_conf + else: + conf = plugin_conf + + self.check_module(conf, path, contents) + else: + raise Exception('unsupported extension: %s' % extension) + + def check_yaml(self, conf, path, contents): # type: (YamlLintConfig, str, str) -> None + """Check the given YAML.""" + self.check_parsable(path, contents) + self.messages += [self.result_to_message(r, path) for r in linter.run(contents, conf, path)] + + def check_module(self, conf, path, contents): # type: (YamlLintConfig, str, str) -> None + """Check the given module.""" + docs = self.get_module_docs(path, contents) + + for key, value in docs.items(): + yaml_data = value['yaml'] + lineno = value['lineno'] + fmt = value['fmt'] + + if fmt != 'yaml': + continue + + if yaml_data.startswith('\n'): + yaml_data = yaml_data[1:] + lineno += 1 + + self.check_parsable(path, yaml_data, lineno) + + messages = list(linter.run(yaml_data, conf, path)) + + self.messages += [self.result_to_message(r, path, lineno - 1, key) for r in messages] + + def check_parsable(self, path, contents, lineno=1): # type: (str, str, int) -> None + """Check the given contents to verify they can be parsed as YAML.""" + try: + yaml.load(contents, Loader=TestLoader) + except MarkedYAMLError as ex: + self.messages += [{'code': 'unparsable-with-libyaml', + 'message': '%s - %s' % (ex.args[0], ex.args[2]), + 'path': path, + 'line': ex.problem_mark.line + lineno, + 'column': ex.problem_mark.column + 1, + 'level': 'error', + }] + + @staticmethod + def result_to_message(result, path, line_offset=0, prefix=''): # type: (t.Any, str, int, str) -> t.Dict[str, t.Any] + """Convert the given result to a dictionary and return it.""" + if prefix: + prefix = '%s: ' % prefix + + return dict( + code=result.rule or result.level, + message=prefix + result.desc, + path=path, + line=result.line + line_offset, + column=result.column, + level=result.level, + ) + + def get_module_docs(self, path, contents): # type: (str, str) -> t.Dict[str, t.Any] + """Return the module documentation for the given module contents.""" + module_doc_types = [ + 'DOCUMENTATION', + 'EXAMPLES', + 'RETURN', + ] + + docs = {} + + fmt_re = re.compile(r'^# fmt:\s+(\S+)') + + def check_assignment(statement, doc_types=None): + """Check the given statement for a documentation assignment.""" + for target in statement.targets: + if not isinstance(target, ast.Name): + continue + + if doc_types and target.id not in doc_types: + continue + + fmt_match = fmt_re.match(statement.value.s.lstrip()) + fmt = 'yaml' + if fmt_match: + fmt = fmt_match.group(1) + + docs[target.id] = dict( + yaml=statement.value.s, + lineno=statement.lineno, + end_lineno=statement.lineno + len(statement.value.s.splitlines()), + fmt=fmt.lower(), + ) + + module_ast = self.parse_module(path, contents) + + if not module_ast: + return {} + + is_plugin = path.startswith('lib/ansible/modules/') or path.startswith('lib/ansible/plugins/') or path.startswith('plugins/') + is_doc_fragment = path.startswith('lib/ansible/plugins/doc_fragments/') or path.startswith('plugins/doc_fragments/') + + if is_plugin and not is_doc_fragment: + for body_statement in module_ast.body: + if isinstance(body_statement, ast.Assign): + check_assignment(body_statement, module_doc_types) + elif is_doc_fragment: + for body_statement in module_ast.body: + if isinstance(body_statement, ast.ClassDef): + for class_statement in body_statement.body: + if isinstance(class_statement, ast.Assign): + check_assignment(class_statement) + else: + raise Exception('unsupported path: %s' % path) + + return docs + + def parse_module(self, path, contents): # type: (str, str) -> t.Optional[ast.Module] + """Parse the given contents and return a module if successful, otherwise return None.""" + try: + return ast.parse(contents) + except SyntaxError as ex: + self.messages.append(dict( + code='python-syntax-error', + message=str(ex), + path=path, + line=ex.lineno, + column=ex.offset, + level='error', + )) + except Exception as ex: # pylint: disable=broad-except + self.messages.append(dict( + code='python-parse-error', + message=str(ex), + path=path, + line=0, + column=0, + level='error', + )) + + return None + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/tools/collection_detail.py b/test/lib/ansible_test/_util/controller/tools/collection_detail.py new file mode 100644 index 0000000..870ea59 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/tools/collection_detail.py @@ -0,0 +1,94 @@ +"""Retrieve collection detail.""" +from __future__ import annotations + +import json +import os +import re +import sys + +import yaml + + +# See semantic versioning specification (https://semver.org/) +NUMERIC_IDENTIFIER = r'(?:0|[1-9][0-9]*)' +ALPHANUMERIC_IDENTIFIER = r'(?:[0-9]*[a-zA-Z-][a-zA-Z0-9-]*)' + +PRE_RELEASE_IDENTIFIER = r'(?:' + NUMERIC_IDENTIFIER + r'|' + ALPHANUMERIC_IDENTIFIER + r')' +BUILD_IDENTIFIER = r'[a-zA-Z0-9-]+' # equivalent to r'(?:[0-9]+|' + ALPHANUMERIC_IDENTIFIER + r')' + +VERSION_CORE = NUMERIC_IDENTIFIER + r'\.' + NUMERIC_IDENTIFIER + r'\.' + NUMERIC_IDENTIFIER +PRE_RELEASE = r'(?:-' + PRE_RELEASE_IDENTIFIER + r'(?:\.' + PRE_RELEASE_IDENTIFIER + r')*)?' +BUILD = r'(?:\+' + BUILD_IDENTIFIER + r'(?:\.' + BUILD_IDENTIFIER + r')*)?' + +SEMVER_REGULAR_EXPRESSION = r'^' + VERSION_CORE + PRE_RELEASE + BUILD + r'$' + + +def validate_version(version): + """Raise exception if the provided version is not None or a valid semantic version.""" + if version is None: + return + if not re.match(SEMVER_REGULAR_EXPRESSION, version): + raise Exception('Invalid version number "{0}". Collection version numbers must ' + 'follow semantic versioning (https://semver.org/).'.format(version)) + + +def read_manifest_json(collection_path): + """Return collection information from the MANIFEST.json file.""" + manifest_path = os.path.join(collection_path, 'MANIFEST.json') + + if not os.path.exists(manifest_path): + return None + + try: + with open(manifest_path, encoding='utf-8') as manifest_file: + manifest = json.load(manifest_file) + + collection_info = manifest.get('collection_info') or {} + + result = dict( + version=collection_info.get('version'), + ) + validate_version(result['version']) + except Exception as ex: # pylint: disable=broad-except + raise Exception('{0}: {1}'.format(os.path.basename(manifest_path), ex)) + + return result + + +def read_galaxy_yml(collection_path): + """Return collection information from the galaxy.yml file.""" + galaxy_path = os.path.join(collection_path, 'galaxy.yml') + + if not os.path.exists(galaxy_path): + return None + + try: + with open(galaxy_path, encoding='utf-8') as galaxy_file: + galaxy = yaml.safe_load(galaxy_file) + + result = dict( + version=galaxy.get('version'), + ) + validate_version(result['version']) + except Exception as ex: # pylint: disable=broad-except + raise Exception('{0}: {1}'.format(os.path.basename(galaxy_path), ex)) + + return result + + +def main(): + """Retrieve collection detail.""" + collection_path = sys.argv[1] + + try: + result = read_manifest_json(collection_path) or read_galaxy_yml(collection_path) or {} + except Exception as ex: # pylint: disable=broad-except + result = dict( + error='{0}'.format(ex), + ) + + print(json.dumps(result)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/tools/coverage_stub.ps1 b/test/lib/ansible_test/_util/controller/tools/coverage_stub.ps1 new file mode 100644 index 0000000..fcc4570 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/tools/coverage_stub.ps1 @@ -0,0 +1,40 @@ +<# +.SYNOPSIS +Gets the lines to hit from a sourcefile for coverage stubs. +#> +[CmdletBinding()] +param ( + [Parameter(Mandatory, ValueFromRemainingArguments)] + [String[]] + $Path +) + +$stubInfo = @( + foreach ($sourcePath in $Path) { + # Default is to just no lines for missing files + [Collections.Generic.HashSet[int]]$lines = @() + + if (Test-Path -LiteralPath $sourcePath) { + $code = [ScriptBlock]::Create([IO.File]::ReadAllText($sourcePath)) + + # We set our breakpoints with this predicate so our stubs should match + # that logic. + $predicate = { + $args[0] -is [System.Management.Automation.Language.CommandBaseAst] + } + $cmds = $code.Ast.FindAll($predicate, $true) + + # We only care about unique lines not multiple commands on 1 line. + $lines = @(foreach ($cmd in $cmds) { + $cmd.Extent.StartLineNumber + }) + } + + [PSCustomObject]@{ + Path = $sourcePath + Lines = $lines + } + } +) + +ConvertTo-Json -InputObject $stubInfo -Depth 2 -Compress diff --git a/test/lib/ansible_test/_util/controller/tools/sslcheck.py b/test/lib/ansible_test/_util/controller/tools/sslcheck.py new file mode 100644 index 0000000..c25fed6 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/tools/sslcheck.py @@ -0,0 +1,22 @@ +"""Show openssl version.""" +from __future__ import annotations + +import json + +# noinspection PyBroadException +try: + from ssl import OPENSSL_VERSION_INFO + VERSION = list(OPENSSL_VERSION_INFO[:3]) +except Exception: # pylint: disable=broad-except + VERSION = None + + +def main(): + """Main program entry point.""" + print(json.dumps(dict( + version=VERSION, + ))) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/controller/tools/yaml_to_json.py b/test/lib/ansible_test/_util/controller/tools/yaml_to_json.py new file mode 100644 index 0000000..e2a15bf --- /dev/null +++ b/test/lib/ansible_test/_util/controller/tools/yaml_to_json.py @@ -0,0 +1,27 @@ +"""Read YAML from stdin and write JSON to stdout.""" +from __future__ import annotations + +import datetime +import json +import sys + +from yaml import load + +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + +# unique ISO date marker matching the one present in importer.py +ISO_DATE_MARKER = 'isodate:f23983df-f3df-453c-9904-bcd08af468cc:' + + +def default(value): + """Custom default serializer which supports datetime.date types.""" + if isinstance(value, datetime.date): + return '%s%s' % (ISO_DATE_MARKER, value.isoformat()) + + raise TypeError('cannot serialize type: %s' % type(value)) + + +json.dump(load(sys.stdin, Loader=SafeLoader), sys.stdout, default=default) diff --git a/test/lib/ansible_test/_util/target/__init__.py b/test/lib/ansible_test/_util/target/__init__.py new file mode 100644 index 0000000..527d413 --- /dev/null +++ b/test/lib/ansible_test/_util/target/__init__.py @@ -0,0 +1,2 @@ +# Empty __init__.py to allow importing of `ansible_test._util.target.common` under Python 2.x. +# This allows the ansible-test entry point to report supported Python versions before exiting. diff --git a/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py b/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py new file mode 100755 index 0000000..930654f --- /dev/null +++ b/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# PYTHON_ARGCOMPLETE_OK +"""Command line entry point for ansible-test.""" + +# NOTE: This file resides in the _util/target directory to ensure compatibility with all supported Python versions. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import sys + + +def main(args=None): + """Main program entry point.""" + ansible_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + source_root = os.path.join(ansible_root, 'test', 'lib') + + if os.path.exists(os.path.join(source_root, 'ansible_test', '_internal', '__init__.py')): + # running from source, use that version of ansible-test instead of any version that may already be installed + sys.path.insert(0, source_root) + + # noinspection PyProtectedMember + from ansible_test._util.target.common.constants import CONTROLLER_PYTHON_VERSIONS + + if version_to_str(sys.version_info[:2]) not in CONTROLLER_PYTHON_VERSIONS: + raise SystemExit('This version of ansible-test cannot be executed with Python version %s. Supported Python versions are: %s' % ( + version_to_str(sys.version_info[:3]), ', '.join(CONTROLLER_PYTHON_VERSIONS))) + + if any(not os.get_blocking(handle.fileno()) for handle in (sys.stdin, sys.stdout, sys.stderr)): + raise SystemExit('Standard input, output and error file handles must be blocking to run ansible-test.') + + # noinspection PyProtectedMember + from ansible_test._internal import main as cli_main + + cli_main(args) + + +def version_to_str(version): + """Return a version string from a version tuple.""" + return '.'.join(str(n) for n in version) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/target/common/__init__.py b/test/lib/ansible_test/_util/target/common/__init__.py new file mode 100644 index 0000000..527d413 --- /dev/null +++ b/test/lib/ansible_test/_util/target/common/__init__.py @@ -0,0 +1,2 @@ +# Empty __init__.py to allow importing of `ansible_test._util.target.common` under Python 2.x. +# This allows the ansible-test entry point to report supported Python versions before exiting. diff --git a/test/lib/ansible_test/_util/target/common/constants.py b/test/lib/ansible_test/_util/target/common/constants.py new file mode 100644 index 0000000..9bddfaf --- /dev/null +++ b/test/lib/ansible_test/_util/target/common/constants.py @@ -0,0 +1,20 @@ +"""Constants used by ansible-test's CLI entry point (as well as the rest of ansible-test). Imports should not be used in this file.""" + +# NOTE: This file resides in the _util/target directory to ensure compatibility with all supported Python versions. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +REMOTE_ONLY_PYTHON_VERSIONS = ( + '2.7', + '3.5', + '3.6', + '3.7', + '3.8', +) + +CONTROLLER_PYTHON_VERSIONS = ( + '3.9', + '3.10', + '3.11', +) diff --git a/test/lib/ansible_test/_util/target/injector/python.py b/test/lib/ansible_test/_util/target/injector/python.py new file mode 100644 index 0000000..c1e88a9 --- /dev/null +++ b/test/lib/ansible_test/_util/target/injector/python.py @@ -0,0 +1,83 @@ +# auto-shebang +"""Provides an entry point for python scripts and python modules on the controller with the current python interpreter and optional code coverage collection.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import sys + + +def main(): + """Main entry point.""" + name = os.path.basename(__file__) + args = [sys.executable] + + coverage_config = os.environ.get('COVERAGE_CONF') + coverage_output = os.environ.get('COVERAGE_FILE') + + if coverage_config: + if coverage_output: + args += ['-m', 'coverage.__main__', 'run', '--rcfile', coverage_config] + else: + if sys.version_info >= (3, 4): + # noinspection PyUnresolvedReferences + import importlib.util + + # noinspection PyUnresolvedReferences + found = bool(importlib.util.find_spec('coverage')) + else: + # noinspection PyDeprecation + import imp + + try: + # noinspection PyDeprecation + imp.find_module('coverage') + found = True + except ImportError: + found = False + + if not found: + sys.exit('ERROR: Could not find `coverage` module. ' + 'Did you use a virtualenv created without --system-site-packages or with the wrong interpreter?') + + if name == 'python.py': + if sys.argv[1] == '-c': + # prevent simple misuse of python.py with -c which does not work with coverage + sys.exit('ERROR: Use `python -c` instead of `python.py -c` to avoid errors when code coverage is collected.') + elif name == 'pytest': + args += ['-m', 'pytest'] + elif name == 'importer.py': + args += [find_program(name, False)] + else: + args += [find_program(name, True)] + + args += sys.argv[1:] + + os.execv(args[0], args) + + +def find_program(name, executable): # type: (str, bool) -> str + """ + Find and return the full path to the named program, optionally requiring it to be executable. + Raises an exception if the program is not found. + """ + path = os.environ.get('PATH', os.path.defpath) + seen = set([os.path.abspath(__file__)]) + mode = os.F_OK | os.X_OK if executable else os.F_OK + + for base in path.split(os.path.pathsep): + candidate = os.path.abspath(os.path.join(base, name)) + + if candidate in seen: + continue + + seen.add(candidate) + + if os.path.exists(candidate) and os.access(candidate, mode): + return candidate + + raise Exception('Executable "%s" not found in path: %s' % (name, path)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/target/injector/virtualenv.sh b/test/lib/ansible_test/_util/target/injector/virtualenv.sh new file mode 100644 index 0000000..5dcbe0e --- /dev/null +++ b/test/lib/ansible_test/_util/target/injector/virtualenv.sh @@ -0,0 +1,14 @@ +# shellcheck shell=bash +# Create and activate a fresh virtual environment with `source virtualenv.sh`. + +rm -rf "${OUTPUT_DIR}/venv" + +# Try to use 'venv' if it is available, then fallback to 'virtualenv' since some systems provide 'venv' although it is non-functional. +if [[ "${ANSIBLE_TEST_PYTHON_VERSION}" =~ ^2\. ]] || ! "${ANSIBLE_TEST_PYTHON_INTERPRETER}" -m venv --system-site-packages "${OUTPUT_DIR}/venv" > /dev/null 2>&1; then + rm -rf "${OUTPUT_DIR}/venv" + "${ANSIBLE_TEST_PYTHON_INTERPRETER}" -m virtualenv --system-site-packages --python "${ANSIBLE_TEST_PYTHON_INTERPRETER}" "${OUTPUT_DIR}/venv" +fi + +set +ux +source "${OUTPUT_DIR}/venv/bin/activate" +set -ux diff --git a/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_collections.py b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_collections.py new file mode 100644 index 0000000..fefd6b0 --- /dev/null +++ b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_collections.py @@ -0,0 +1,70 @@ +"""Enable unit testing of Ansible collections. PYTEST_DONT_REWRITE""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os + +# set by ansible-test to a single directory, rather than a list of directories as supported by Ansible itself +ANSIBLE_COLLECTIONS_PATH = os.path.join(os.environ['ANSIBLE_COLLECTIONS_PATH'], 'ansible_collections') + +# set by ansible-test to the minimum python version supported on the controller +ANSIBLE_CONTROLLER_MIN_PYTHON_VERSION = tuple(int(x) for x in os.environ['ANSIBLE_CONTROLLER_MIN_PYTHON_VERSION'].split('.')) + + +# this monkeypatch to _pytest.pathlib.resolve_package_path fixes PEP420 resolution for collections in pytest >= 6.0.0 +# NB: this code should never run under py2 +def collection_resolve_package_path(path): + """Configure the Python package path so that pytest can find our collections.""" + for parent in path.parents: + if str(parent) == ANSIBLE_COLLECTIONS_PATH: + return parent + + raise Exception('File "%s" not found in collection path "%s".' % (path, ANSIBLE_COLLECTIONS_PATH)) + + +# this monkeypatch to py.path.local.LocalPath.pypkgpath fixes PEP420 resolution for collections in pytest < 6.0.0 +def collection_pypkgpath(self): + """Configure the Python package path so that pytest can find our collections.""" + for parent in self.parts(reverse=True): + if str(parent) == ANSIBLE_COLLECTIONS_PATH: + return parent + + raise Exception('File "%s" not found in collection path "%s".' % (self.strpath, ANSIBLE_COLLECTIONS_PATH)) + + +def pytest_configure(): + """Configure this pytest plugin.""" + try: + if pytest_configure.executed: + return + except AttributeError: + pytest_configure.executed = True + + # noinspection PyProtectedMember + from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder + + # allow unit tests to import code from collections + + # noinspection PyProtectedMember + _AnsibleCollectionFinder(paths=[os.path.dirname(ANSIBLE_COLLECTIONS_PATH)])._install() # pylint: disable=protected-access + + try: + # noinspection PyProtectedMember + from _pytest import pathlib as _pytest_pathlib + except ImportError: + _pytest_pathlib = None + + if hasattr(_pytest_pathlib, 'resolve_package_path'): + _pytest_pathlib.resolve_package_path = collection_resolve_package_path + else: + # looks like pytest <= 6.0.0, use the old hack against py.path + # noinspection PyProtectedMember + import py._path.local + + # force collections unit tests to be loaded with the ansible_collections namespace + # original idea from https://stackoverflow.com/questions/50174130/how-do-i-pytest-a-project-using-pep-420-namespace-packages/50175552#50175552 + # noinspection PyProtectedMember + py._path.local.LocalPath.pypkgpath = collection_pypkgpath # pylint: disable=protected-access + + +pytest_configure() diff --git a/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_coverage.py b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_coverage.py new file mode 100644 index 0000000..b05298a --- /dev/null +++ b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_coverage.py @@ -0,0 +1,68 @@ +"""Monkey patch os._exit when running under coverage so we don't lose coverage data in forks, such as with `pytest --boxed`. PYTEST_DONT_REWRITE""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +def pytest_configure(): + """Configure this pytest plugin.""" + try: + if pytest_configure.executed: + return + except AttributeError: + pytest_configure.executed = True + + try: + import coverage + except ImportError: + coverage = None + + try: + coverage.Coverage + except AttributeError: + coverage = None + + if not coverage: + return + + import gc + import os + + coverage_instances = [] + + for obj in gc.get_objects(): + if isinstance(obj, coverage.Coverage): + coverage_instances.append(obj) + + if not coverage_instances: + coverage_config = os.environ.get('COVERAGE_CONF') + + if not coverage_config: + return + + coverage_output = os.environ.get('COVERAGE_FILE') + + if not coverage_output: + return + + cov = coverage.Coverage(config_file=coverage_config) + coverage_instances.append(cov) + else: + cov = None + + # noinspection PyProtectedMember + os_exit = os._exit # pylint: disable=protected-access + + def coverage_exit(*args, **kwargs): + for instance in coverage_instances: + instance.stop() + instance.save() + + os_exit(*args, **kwargs) + + os._exit = coverage_exit # pylint: disable=protected-access + + if cov: + cov.start() + + +pytest_configure() diff --git a/test/lib/ansible_test/_util/target/sanity/compile/compile.py b/test/lib/ansible_test/_util/target/sanity/compile/compile.py new file mode 100644 index 0000000..bd2446f --- /dev/null +++ b/test/lib/ansible_test/_util/target/sanity/compile/compile.py @@ -0,0 +1,56 @@ +"""Python syntax checker with lint friendly output.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import sys + +ENCODING = 'utf-8' +ERRORS = 'replace' +Text = type(u'') + + +def main(): + """Main program entry point.""" + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + compile_source(path) + + +def compile_source(path): + """Compile the specified source file, printing an error if one occurs.""" + with open(path, 'rb') as source_fd: + source = source_fd.read() + + try: + compile(source, path, 'exec', dont_inherit=True) + except SyntaxError as ex: + extype, message, lineno, offset = type(ex), ex.text, ex.lineno, ex.offset + except BaseException as ex: # pylint: disable=broad-except + extype, message, lineno, offset = type(ex), str(ex), 0, 0 + else: + return + + # In some situations offset can be None. This can happen for syntax errors on Python 2.6 + # (__future__ import following after a regular import). + offset = offset or 0 + + result = "%s:%d:%d: %s: %s" % (path, lineno, offset, extype.__name__, safe_message(message)) + + if sys.version_info <= (3,): + result = result.encode(ENCODING, ERRORS) + + print(result) + + +def safe_message(value): + """Given an input value as text or bytes, return the first non-empty line as text, ensuring it can be round-tripped as UTF-8.""" + if isinstance(value, Text): + value = value.encode(ENCODING, ERRORS) + + value = value.decode(ENCODING, ERRORS) + value = value.strip().splitlines()[0].strip() + + return value + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/target/sanity/import/importer.py b/test/lib/ansible_test/_util/target/sanity/import/importer.py new file mode 100644 index 0000000..44a5ddc --- /dev/null +++ b/test/lib/ansible_test/_util/target/sanity/import/importer.py @@ -0,0 +1,573 @@ +"""Import the given python module(s) and report error(s) encountered.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +def main(): + """ + Main program function used to isolate globals from imported code. + Changes to globals in imported modules on Python 2.x will overwrite our own globals. + """ + import os + import sys + import types + + # preload an empty ansible._vendor module to prevent use of any embedded modules during the import test + vendor_module_name = 'ansible._vendor' + + vendor_module = types.ModuleType(vendor_module_name) + vendor_module.__file__ = os.path.join(os.path.sep.join(os.path.abspath(__file__).split(os.path.sep)[:-8]), 'lib/ansible/_vendor/__init__.py') + vendor_module.__path__ = [] + vendor_module.__package__ = vendor_module_name + + sys.modules[vendor_module_name] = vendor_module + + import ansible + import contextlib + import datetime + import json + import re + import runpy + import subprocess + import traceback + import warnings + + ansible_path = os.path.dirname(os.path.dirname(ansible.__file__)) + temp_path = os.environ['SANITY_TEMP_PATH'] + os.path.sep + external_python = os.environ.get('SANITY_EXTERNAL_PYTHON') + yaml_to_json_path = os.environ.get('SANITY_YAML_TO_JSON') + collection_full_name = os.environ.get('SANITY_COLLECTION_FULL_NAME') + collection_root = os.environ.get('ANSIBLE_COLLECTIONS_PATH') + import_type = os.environ.get('SANITY_IMPORTER_TYPE') + + try: + # noinspection PyCompatibility + from importlib import import_module + except ImportError: + def import_module(name, package=None): # type: (str, str | None) -> types.ModuleType + assert package is None + __import__(name) + return sys.modules[name] + + from io import BytesIO, TextIOWrapper + + try: + from importlib.util import spec_from_loader, module_from_spec + from importlib.machinery import SourceFileLoader, ModuleSpec # pylint: disable=unused-import + except ImportError: + has_py3_loader = False + else: + has_py3_loader = True + + if collection_full_name: + # allow importing code from collections when testing a collection + from ansible.module_utils.common.text.converters import to_bytes, to_text, to_native, text_type + + # noinspection PyProtectedMember + from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder + from ansible.utils.collection_loader import _collection_finder + + yaml_to_dict_cache = {} + + # unique ISO date marker matching the one present in yaml_to_json.py + iso_date_marker = 'isodate:f23983df-f3df-453c-9904-bcd08af468cc:' + iso_date_re = re.compile('^%s([0-9]{4})-([0-9]{2})-([0-9]{2})$' % iso_date_marker) + + def parse_value(value): + """Custom value parser for JSON deserialization that recognizes our internal ISO date format.""" + if isinstance(value, text_type): + match = iso_date_re.search(value) + + if match: + value = datetime.date(int(match.group(1)), int(match.group(2)), int(match.group(3))) + + return value + + def object_hook(data): + """Object hook for custom ISO date deserialization from JSON.""" + return dict((key, parse_value(value)) for key, value in data.items()) + + def yaml_to_dict(yaml, content_id): + """ + Return a Python dict version of the provided YAML. + Conversion is done in a subprocess since the current Python interpreter does not have access to PyYAML. + """ + if content_id in yaml_to_dict_cache: + return yaml_to_dict_cache[content_id] + + try: + cmd = [external_python, yaml_to_json_path] + proc = subprocess.Popen([to_bytes(c) for c in cmd], # pylint: disable=consider-using-with + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout_bytes, stderr_bytes = proc.communicate(to_bytes(yaml)) + + if proc.returncode != 0: + raise Exception('command %s failed with return code %d: %s' % ([to_native(c) for c in cmd], proc.returncode, to_native(stderr_bytes))) + + data = yaml_to_dict_cache[content_id] = json.loads(to_text(stdout_bytes), object_hook=object_hook) + + return data + except Exception as ex: + raise Exception('internal importer error - failed to parse yaml: %s' % to_native(ex)) + + _collection_finder._meta_yml_to_dict = yaml_to_dict # pylint: disable=protected-access + + collection_loader = _AnsibleCollectionFinder(paths=[collection_root]) + # noinspection PyProtectedMember + collection_loader._install() # pylint: disable=protected-access + else: + # do not support collection loading when not testing a collection + collection_loader = None + + if collection_loader and import_type == 'plugin': + # do not unload ansible code for collection plugin (not module) tests + # doing so could result in the collection loader being initialized multiple times + pass + else: + # remove all modules under the ansible package, except the preloaded vendor module + list(map(sys.modules.pop, [m for m in sys.modules if m.partition('.')[0] == ansible.__name__ and m != vendor_module_name])) + + if import_type == 'module': + # pre-load an empty ansible package to prevent unwanted code in __init__.py from loading + # this more accurately reflects the environment that AnsiballZ runs modules under + # it also avoids issues with imports in the ansible package that are not allowed + ansible_module = types.ModuleType(ansible.__name__) + ansible_module.__file__ = ansible.__file__ + ansible_module.__path__ = ansible.__path__ + ansible_module.__package__ = ansible.__package__ + + sys.modules[ansible.__name__] = ansible_module + + class ImporterAnsibleModuleException(Exception): + """Exception thrown during initialization of ImporterAnsibleModule.""" + + class ImporterAnsibleModule: + """Replacement for AnsibleModule to support import testing.""" + def __init__(self, *args, **kwargs): + raise ImporterAnsibleModuleException() + + class RestrictedModuleLoader: + """Python module loader that restricts inappropriate imports.""" + def __init__(self, path, name, restrict_to_module_paths): + self.path = path + self.name = name + self.loaded_modules = set() + self.restrict_to_module_paths = restrict_to_module_paths + + def find_spec(self, fullname, path=None, target=None): # pylint: disable=unused-argument + # type: (RestrictedModuleLoader, str, list[str], types.ModuleType | None ) -> ModuleSpec | None | ImportError + """Return the spec from the loader or None""" + loader = self._get_loader(fullname, path=path) + if loader is not None: + if has_py3_loader: + # loader is expected to be Optional[importlib.abc.Loader], but RestrictedModuleLoader does not inherit from importlib.abc.Loder + return spec_from_loader(fullname, loader) # type: ignore[arg-type] + raise ImportError("Failed to import '%s' due to a bug in ansible-test. Check importlib imports for typos." % fullname) + return None + + def find_module(self, fullname, path=None): + # type: (RestrictedModuleLoader, str, list[str]) -> RestrictedModuleLoader | None + """Return self if the given fullname is restricted, otherwise return None.""" + return self._get_loader(fullname, path=path) + + def _get_loader(self, fullname, path=None): + # type: (RestrictedModuleLoader, str, list[str]) -> RestrictedModuleLoader | None + """Return self if the given fullname is restricted, otherwise return None.""" + if fullname in self.loaded_modules: + return None # ignore modules that are already being loaded + + if is_name_in_namepace(fullname, ['ansible']): + if not self.restrict_to_module_paths: + return None # for non-modules, everything in the ansible namespace is allowed + + if fullname in ('ansible.module_utils.basic',): + return self # intercept loading so we can modify the result + + if is_name_in_namepace(fullname, ['ansible.module_utils', self.name]): + return None # module_utils and module under test are always allowed + + if any(os.path.exists(candidate_path) for candidate_path in convert_ansible_name_to_absolute_paths(fullname)): + return self # restrict access to ansible files that exist + + return None # ansible file does not exist, do not restrict access + + if is_name_in_namepace(fullname, ['ansible_collections']): + if not collection_loader: + return self # restrict access to collections when we are not testing a collection + + if not self.restrict_to_module_paths: + return None # for non-modules, everything in the ansible namespace is allowed + + if is_name_in_namepace(fullname, ['ansible_collections...plugins.module_utils', self.name]): + return None # module_utils and module under test are always allowed + + if collection_loader.find_module(fullname, path): + return self # restrict access to collection files that exist + + return None # collection file does not exist, do not restrict access + + # not a namespace we care about + return None + + def create_module(self, spec): # pylint: disable=unused-argument + # type: (RestrictedModuleLoader, ModuleSpec) -> None + """Return None to use default module creation.""" + return None + + def exec_module(self, module): + # type: (RestrictedModuleLoader, types.ModuleType) -> None | ImportError + """Execute the module if the name is ansible.module_utils.basic and otherwise raise an ImportError""" + fullname = module.__spec__.name + if fullname == 'ansible.module_utils.basic': + self.loaded_modules.add(fullname) + for path in convert_ansible_name_to_absolute_paths(fullname): + if not os.path.exists(path): + continue + loader = SourceFileLoader(fullname, path) + spec = spec_from_loader(fullname, loader) + real_module = module_from_spec(spec) + loader.exec_module(real_module) + real_module.AnsibleModule = ImporterAnsibleModule # type: ignore[attr-defined] + real_module._load_params = lambda *args, **kwargs: {} # type: ignore[attr-defined] # pylint: disable=protected-access + sys.modules[fullname] = real_module + return None + raise ImportError('could not find "%s"' % fullname) + raise ImportError('import of "%s" is not allowed in this context' % fullname) + + def load_module(self, fullname): + # type: (RestrictedModuleLoader, str) -> types.ModuleType | ImportError + """Return the module if the name is ansible.module_utils.basic and otherwise raise an ImportError.""" + if fullname == 'ansible.module_utils.basic': + module = self.__load_module(fullname) + + # stop Ansible module execution during AnsibleModule instantiation + module.AnsibleModule = ImporterAnsibleModule # type: ignore[attr-defined] + # no-op for _load_params since it may be called before instantiating AnsibleModule + module._load_params = lambda *args, **kwargs: {} # type: ignore[attr-defined] # pylint: disable=protected-access + + return module + + raise ImportError('import of "%s" is not allowed in this context' % fullname) + + def __load_module(self, fullname): + # type: (RestrictedModuleLoader, str) -> types.ModuleType + """Load the requested module while avoiding infinite recursion.""" + self.loaded_modules.add(fullname) + return import_module(fullname) + + def run(restrict_to_module_paths): + """Main program function.""" + base_dir = os.getcwd() + messages = set() + + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + name = convert_relative_path_to_name(path) + test_python_module(path, name, base_dir, messages, restrict_to_module_paths) + + if messages: + sys.exit(10) + + def test_python_module(path, name, base_dir, messages, restrict_to_module_paths): + """Test the given python module by importing it. + :type path: str + :type name: str + :type base_dir: str + :type messages: set[str] + :type restrict_to_module_paths: bool + """ + if name in sys.modules: + return # cannot be tested because it has already been loaded + + is_ansible_module = (path.startswith('lib/ansible/modules/') or path.startswith('plugins/modules/')) and os.path.basename(path) != '__init__.py' + run_main = is_ansible_module + + if path == 'lib/ansible/modules/async_wrapper.py': + # async_wrapper is a non-standard Ansible module (does not use AnsibleModule) so we cannot test the main function + run_main = False + + capture_normal = Capture() + capture_main = Capture() + + run_module_ok = False + + try: + with monitor_sys_modules(path, messages): + with restrict_imports(path, name, messages, restrict_to_module_paths): + with capture_output(capture_normal): + import_module(name) + + if run_main: + run_module_ok = is_ansible_module + + with monitor_sys_modules(path, messages): + with restrict_imports(path, name, messages, restrict_to_module_paths): + with capture_output(capture_main): + runpy.run_module(name, run_name='__main__', alter_sys=True) + except ImporterAnsibleModuleException: + # module instantiated AnsibleModule without raising an exception + if not run_module_ok: + if is_ansible_module: + report_message(path, 0, 0, 'module-guard', "AnsibleModule instantiation not guarded by `if __name__ == '__main__'`", messages) + else: + report_message(path, 0, 0, 'non-module', "AnsibleModule instantiated by import of non-module", messages) + except BaseException as ex: # pylint: disable=locally-disabled, broad-except + # intentionally catch all exceptions, including calls to sys.exit + exc_type, _exc, exc_tb = sys.exc_info() + message = str(ex) + results = list(reversed(traceback.extract_tb(exc_tb))) + line = 0 + offset = 0 + full_path = os.path.join(base_dir, path) + base_path = base_dir + os.path.sep + source = None + + # avoid line wraps in messages + message = re.sub(r'\n *', ': ', message) + + for result in results: + if result[0] == full_path: + # save the line number for the file under test + line = result[1] or 0 + + if not source and result[0].startswith(base_path) and not result[0].startswith(temp_path): + # save the first path and line number in the traceback which is in our source tree + source = (os.path.relpath(result[0], base_path), result[1] or 0, 0) + + if isinstance(ex, SyntaxError): + # SyntaxError has better information than the traceback + if ex.filename == full_path: # pylint: disable=locally-disabled, no-member + # syntax error was reported in the file under test + line = ex.lineno or 0 # pylint: disable=locally-disabled, no-member + offset = ex.offset or 0 # pylint: disable=locally-disabled, no-member + elif ex.filename.startswith(base_path) and not ex.filename.startswith(temp_path): # pylint: disable=locally-disabled, no-member + # syntax error was reported in our source tree + source = (os.path.relpath(ex.filename, base_path), ex.lineno or 0, ex.offset or 0) # pylint: disable=locally-disabled, no-member + + # remove the filename and line number from the message + # either it was extracted above, or it's not really useful information + message = re.sub(r' \(.*?, line [0-9]+\)$', '', message) + + if source and source[0] != path: + message += ' (at %s:%d:%d)' % (source[0], source[1], source[2]) + + report_message(path, line, offset, 'traceback', '%s: %s' % (exc_type.__name__, message), messages) + finally: + capture_report(path, capture_normal, messages) + capture_report(path, capture_main, messages) + + def is_name_in_namepace(name, namespaces): + """Returns True if the given name is one of the given namespaces, otherwise returns False.""" + name_parts = name.split('.') + + for namespace in namespaces: + namespace_parts = namespace.split('.') + length = min(len(name_parts), len(namespace_parts)) + + truncated_name = name_parts[0:length] + truncated_namespace = namespace_parts[0:length] + + # empty parts in the namespace are treated as wildcards + # to simplify the comparison, use those empty parts to indicate the positions in the name to be empty as well + for idx, part in enumerate(truncated_namespace): + if not part: + truncated_name[idx] = part + + # example: name=ansible, allowed_name=ansible.module_utils + # example: name=ansible.module_utils.system.ping, allowed_name=ansible.module_utils + if truncated_name == truncated_namespace: + return True + + return False + + def check_sys_modules(path, before, messages): + """Check for unwanted changes to sys.modules. + :type path: str + :type before: dict[str, module] + :type messages: set[str] + """ + after = sys.modules + removed = set(before.keys()) - set(after.keys()) + changed = set(key for key, value in before.items() if key in after and value != after[key]) + + # additions are checked by our custom PEP 302 loader, so we don't need to check them again here + + for module in sorted(removed): + report_message(path, 0, 0, 'unload', 'unloading of "%s" in sys.modules is not supported' % module, messages) + + for module in sorted(changed): + report_message(path, 0, 0, 'reload', 'reloading of "%s" in sys.modules is not supported' % module, messages) + + def convert_ansible_name_to_absolute_paths(name): + """Calculate the module path from the given name. + :type name: str + :rtype: list[str] + """ + return [ + os.path.join(ansible_path, name.replace('.', os.path.sep)), + os.path.join(ansible_path, name.replace('.', os.path.sep)) + '.py', + ] + + def convert_relative_path_to_name(path): + """Calculate the module name from the given path. + :type path: str + :rtype: str + """ + if path.endswith('/__init__.py'): + clean_path = os.path.dirname(path) + else: + clean_path = path + + clean_path = os.path.splitext(clean_path)[0] + + name = clean_path.replace(os.path.sep, '.') + + if collection_loader: + # when testing collections the relative paths (and names) being tested are within the collection under test + name = 'ansible_collections.%s.%s' % (collection_full_name, name) + else: + # when testing ansible all files being imported reside under the lib directory + name = name[len('lib/'):] + + return name + + class Capture: + """Captured output and/or exception.""" + def __init__(self): + # use buffered IO to simulate StringIO; allows Ansible's stream patching to behave without warnings + self.stdout = TextIOWrapper(BytesIO()) + self.stderr = TextIOWrapper(BytesIO()) + + def capture_report(path, capture, messages): + """Report on captured output. + :type path: str + :type capture: Capture + :type messages: set[str] + """ + # since we're using buffered IO, flush before checking for data + capture.stdout.flush() + capture.stderr.flush() + stdout_value = capture.stdout.buffer.getvalue() + if stdout_value: + first = stdout_value.decode().strip().splitlines()[0].strip() + report_message(path, 0, 0, 'stdout', first, messages) + + stderr_value = capture.stderr.buffer.getvalue() + if stderr_value: + first = stderr_value.decode().strip().splitlines()[0].strip() + report_message(path, 0, 0, 'stderr', first, messages) + + def report_message(path, line, column, code, message, messages): + """Report message if not already reported. + :type path: str + :type line: int + :type column: int + :type code: str + :type message: str + :type messages: set[str] + """ + message = '%s:%d:%d: %s: %s' % (path, line, column, code, message) + + if message not in messages: + messages.add(message) + print(message) + + @contextlib.contextmanager + def restrict_imports(path, name, messages, restrict_to_module_paths): + """Restrict available imports. + :type path: str + :type name: str + :type messages: set[str] + :type restrict_to_module_paths: bool + """ + restricted_loader = RestrictedModuleLoader(path, name, restrict_to_module_paths) + + # noinspection PyTypeChecker + sys.meta_path.insert(0, restricted_loader) + sys.path_importer_cache.clear() + + try: + yield + finally: + if import_type == 'plugin' and not collection_loader: + from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder + _AnsibleCollectionFinder._remove() # pylint: disable=protected-access + + if sys.meta_path[0] != restricted_loader: + report_message(path, 0, 0, 'metapath', 'changes to sys.meta_path[0] are not permitted', messages) + + while restricted_loader in sys.meta_path: + # noinspection PyTypeChecker + sys.meta_path.remove(restricted_loader) + + sys.path_importer_cache.clear() + + @contextlib.contextmanager + def monitor_sys_modules(path, messages): + """Monitor sys.modules for unwanted changes, reverting any additions made to our own namespaces.""" + snapshot = sys.modules.copy() + + try: + yield + finally: + check_sys_modules(path, snapshot, messages) + + for key in set(sys.modules.keys()) - set(snapshot.keys()): + if is_name_in_namepace(key, ('ansible', 'ansible_collections')): + del sys.modules[key] # only unload our own code since we know it's native Python + + @contextlib.contextmanager + def capture_output(capture): + """Capture sys.stdout and sys.stderr. + :type capture: Capture + """ + old_stdout = sys.stdout + old_stderr = sys.stderr + + sys.stdout = capture.stdout + sys.stderr = capture.stderr + + # clear all warnings registries to make all warnings available + for module in sys.modules.values(): + try: + # noinspection PyUnresolvedReferences + module.__warningregistry__.clear() + except AttributeError: + pass + + with warnings.catch_warnings(): + warnings.simplefilter('error') + + if collection_loader and import_type == 'plugin': + warnings.filterwarnings( + "ignore", + "AnsibleCollectionFinder has already been configured") + + if sys.version_info[0] == 2: + warnings.filterwarnings( + "ignore", + "Python 2 is no longer supported by the Python core team. Support for it is now deprecated in cryptography," + " and will be removed in a future release.") + warnings.filterwarnings( + "ignore", + "Python 2 is no longer supported by the Python core team. Support for it is now deprecated in cryptography," + " and will be removed in the next release.") + + if sys.version_info[:2] == (3, 5): + warnings.filterwarnings( + "ignore", + "Python 3.5 support will be dropped in the next release ofcryptography. Please upgrade your Python.") + warnings.filterwarnings( + "ignore", + "Python 3.5 support will be dropped in the next release of cryptography. Please upgrade your Python.") + + try: + yield + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + + run(import_type == 'module') + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/target/setup/ConfigureRemotingForAnsible.ps1 b/test/lib/ansible_test/_util/target/setup/ConfigureRemotingForAnsible.ps1 new file mode 100644 index 0000000..7cc86ab --- /dev/null +++ b/test/lib/ansible_test/_util/target/setup/ConfigureRemotingForAnsible.ps1 @@ -0,0 +1,435 @@ +#Requires -Version 3.0 + +# Configure a Windows host for remote management with Ansible +# ----------------------------------------------------------- +# +# This script checks the current WinRM (PS Remoting) configuration and makes +# the necessary changes to allow Ansible to connect, authenticate and +# execute PowerShell commands. +# +# IMPORTANT: This script uses self-signed certificates and authentication mechanisms +# that are intended for development environments and evaluation purposes only. +# Production environments and deployments that are exposed on the network should +# use CA-signed certificates and secure authentication mechanisms such as Kerberos. +# +# To run this script in Powershell: +# +# [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +# $url = "https://raw.githubusercontent.com/ansible/ansible/devel/examples/scripts/ConfigureRemotingForAnsible.ps1" +# $file = "$env:temp\ConfigureRemotingForAnsible.ps1" +# +# (New-Object -TypeName System.Net.WebClient).DownloadFile($url, $file) +# +# powershell.exe -ExecutionPolicy ByPass -File $file +# +# All events are logged to the Windows EventLog, useful for unattended runs. +# +# Use option -Verbose in order to see the verbose output messages. +# +# Use option -CertValidityDays to specify how long this certificate is valid +# starting from today. So you would specify -CertValidityDays 3650 to get +# a 10-year valid certificate. +# +# Use option -ForceNewSSLCert if the system has been SysPreped and a new +# SSL Certificate must be forced on the WinRM Listener when re-running this +# script. This is necessary when a new SID and CN name is created. +# +# Use option -EnableCredSSP to enable CredSSP as an authentication option. +# +# Use option -DisableBasicAuth to disable basic authentication. +# +# Use option -SkipNetworkProfileCheck to skip the network profile check. +# Without specifying this the script will only run if the device's interfaces +# are in DOMAIN or PRIVATE zones. Provide this switch if you want to enable +# WinRM on a device with an interface in PUBLIC zone. +# +# Use option -SubjectName to specify the CN name of the certificate. This +# defaults to the system's hostname and generally should not be specified. + +# Written by Trond Hindenes <trond@hindenes.com> +# Updated by Chris Church <cchurch@ansible.com> +# Updated by Michael Crilly <mike@autologic.cm> +# Updated by Anton Ouzounov <Anton.Ouzounov@careerbuilder.com> +# Updated by Nicolas Simond <contact@nicolas-simond.com> +# Updated by Dag Wieërs <dag@wieers.com> +# Updated by Jordan Borean <jborean93@gmail.com> +# Updated by Erwan Quélin <erwan.quelin@gmail.com> +# Updated by David Norman <david@dkn.email> +# +# Version 1.0 - 2014-07-06 +# Version 1.1 - 2014-11-11 +# Version 1.2 - 2015-05-15 +# Version 1.3 - 2016-04-04 +# Version 1.4 - 2017-01-05 +# Version 1.5 - 2017-02-09 +# Version 1.6 - 2017-04-18 +# Version 1.7 - 2017-11-23 +# Version 1.8 - 2018-02-23 +# Version 1.9 - 2018-09-21 + +# Support -Verbose option +[CmdletBinding()] + +Param ( + [string]$SubjectName = $env:COMPUTERNAME, + [int]$CertValidityDays = 1095, + [switch]$SkipNetworkProfileCheck, + $CreateSelfSignedCert = $true, + [switch]$ForceNewSSLCert, + [switch]$GlobalHttpFirewallAccess, + [switch]$DisableBasicAuth = $false, + [switch]$EnableCredSSP +) + +Function Write-ProgressLog { + $Message = $args[0] + Write-EventLog -LogName Application -Source $EventSource -EntryType Information -EventId 1 -Message $Message +} + +Function Write-VerboseLog { + $Message = $args[0] + Write-Verbose $Message + Write-ProgressLog $Message +} + +Function Write-HostLog { + $Message = $args[0] + Write-Output $Message + Write-ProgressLog $Message +} + +Function New-LegacySelfSignedCert { + Param ( + [string]$SubjectName, + [int]$ValidDays = 1095 + ) + + $hostnonFQDN = $env:computerName + $hostFQDN = [System.Net.Dns]::GetHostByName(($env:computerName)).Hostname + $SignatureAlgorithm = "SHA256" + + $name = New-Object -COM "X509Enrollment.CX500DistinguishedName.1" + $name.Encode("CN=$SubjectName", 0) + + $key = New-Object -COM "X509Enrollment.CX509PrivateKey.1" + $key.ProviderName = "Microsoft Enhanced RSA and AES Cryptographic Provider" + $key.KeySpec = 1 + $key.Length = 4096 + $key.SecurityDescriptor = "D:PAI(A;;0xd01f01ff;;;SY)(A;;0xd01f01ff;;;BA)(A;;0x80120089;;;NS)" + $key.MachineContext = 1 + $key.Create() + + $serverauthoid = New-Object -COM "X509Enrollment.CObjectId.1" + $serverauthoid.InitializeFromValue("1.3.6.1.5.5.7.3.1") + $ekuoids = New-Object -COM "X509Enrollment.CObjectIds.1" + $ekuoids.Add($serverauthoid) + $ekuext = New-Object -COM "X509Enrollment.CX509ExtensionEnhancedKeyUsage.1" + $ekuext.InitializeEncode($ekuoids) + + $cert = New-Object -COM "X509Enrollment.CX509CertificateRequestCertificate.1" + $cert.InitializeFromPrivateKey(2, $key, "") + $cert.Subject = $name + $cert.Issuer = $cert.Subject + $cert.NotBefore = (Get-Date).AddDays(-1) + $cert.NotAfter = $cert.NotBefore.AddDays($ValidDays) + + $SigOID = New-Object -ComObject X509Enrollment.CObjectId + $SigOID.InitializeFromValue(([Security.Cryptography.Oid]$SignatureAlgorithm).Value) + + [string[]] $AlternativeName += $hostnonFQDN + $AlternativeName += $hostFQDN + $IAlternativeNames = New-Object -ComObject X509Enrollment.CAlternativeNames + + foreach ($AN in $AlternativeName) { + $AltName = New-Object -ComObject X509Enrollment.CAlternativeName + $AltName.InitializeFromString(0x3, $AN) + $IAlternativeNames.Add($AltName) + } + + $SubjectAlternativeName = New-Object -ComObject X509Enrollment.CX509ExtensionAlternativeNames + $SubjectAlternativeName.InitializeEncode($IAlternativeNames) + + [String[]]$KeyUsage = ("DigitalSignature", "KeyEncipherment") + $KeyUsageObj = New-Object -ComObject X509Enrollment.CX509ExtensionKeyUsage + $KeyUsageObj.InitializeEncode([int][Security.Cryptography.X509Certificates.X509KeyUsageFlags]($KeyUsage)) + $KeyUsageObj.Critical = $true + + $cert.X509Extensions.Add($KeyUsageObj) + $cert.X509Extensions.Add($ekuext) + $cert.SignatureInformation.HashAlgorithm = $SigOID + $CERT.X509Extensions.Add($SubjectAlternativeName) + $cert.Encode() + + $enrollment = New-Object -COM "X509Enrollment.CX509Enrollment.1" + $enrollment.InitializeFromRequest($cert) + $certdata = $enrollment.CreateRequest(0) + $enrollment.InstallResponse(2, $certdata, 0, "") + + # extract/return the thumbprint from the generated cert + $parsed_cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2 + $parsed_cert.Import([System.Text.Encoding]::UTF8.GetBytes($certdata)) + + return $parsed_cert.Thumbprint +} + +Function Enable-GlobalHttpFirewallAccess { + Write-Verbose "Forcing global HTTP firewall access" + # this is a fairly naive implementation; could be more sophisticated about rule matching/collapsing + $fw = New-Object -ComObject HNetCfg.FWPolicy2 + + # try to find/enable the default rule first + $add_rule = $false + $matching_rules = $fw.Rules | Where-Object { $_.Name -eq "Windows Remote Management (HTTP-In)" } + $rule = $null + If ($matching_rules) { + If ($matching_rules -isnot [Array]) { + Write-Verbose "Editing existing single HTTP firewall rule" + $rule = $matching_rules + } + Else { + # try to find one with the All or Public profile first + Write-Verbose "Found multiple existing HTTP firewall rules..." + $rule = $matching_rules | ForEach-Object { $_.Profiles -band 4 }[0] + + If (-not $rule -or $rule -is [Array]) { + Write-Verbose "Editing an arbitrary single HTTP firewall rule (multiple existed)" + # oh well, just pick the first one + $rule = $matching_rules[0] + } + } + } + + If (-not $rule) { + Write-Verbose "Creating a new HTTP firewall rule" + $rule = New-Object -ComObject HNetCfg.FWRule + $rule.Name = "Windows Remote Management (HTTP-In)" + $rule.Description = "Inbound rule for Windows Remote Management via WS-Management. [TCP 5985]" + $add_rule = $true + } + + $rule.Profiles = 0x7FFFFFFF + $rule.Protocol = 6 + $rule.LocalPorts = 5985 + $rule.RemotePorts = "*" + $rule.LocalAddresses = "*" + $rule.RemoteAddresses = "*" + $rule.Enabled = $true + $rule.Direction = 1 + $rule.Action = 1 + $rule.Grouping = "Windows Remote Management" + + If ($add_rule) { + $fw.Rules.Add($rule) + } + + Write-Verbose "HTTP firewall rule $($rule.Name) updated" +} + +# Setup error handling. +Trap { + $_ + Exit 1 +} +$ErrorActionPreference = "Stop" + +# Get the ID and security principal of the current user account +$myWindowsID = [System.Security.Principal.WindowsIdentity]::GetCurrent() +$myWindowsPrincipal = new-object System.Security.Principal.WindowsPrincipal($myWindowsID) + +# Get the security principal for the Administrator role +$adminRole = [System.Security.Principal.WindowsBuiltInRole]::Administrator + +# Check to see if we are currently running "as Administrator" +if (-Not $myWindowsPrincipal.IsInRole($adminRole)) { + Write-Output "ERROR: You need elevated Administrator privileges in order to run this script." + Write-Output " Start Windows PowerShell by using the Run as Administrator option." + Exit 2 +} + +$EventSource = $MyInvocation.MyCommand.Name +If (-Not $EventSource) { + $EventSource = "Powershell CLI" +} + +If ([System.Diagnostics.EventLog]::Exists('Application') -eq $False -or [System.Diagnostics.EventLog]::SourceExists($EventSource) -eq $False) { + New-EventLog -LogName Application -Source $EventSource +} + +# Detect PowerShell version. +If ($PSVersionTable.PSVersion.Major -lt 3) { + Write-ProgressLog "PowerShell version 3 or higher is required." + Throw "PowerShell version 3 or higher is required." +} + +# Find and start the WinRM service. +Write-Verbose "Verifying WinRM service." +If (!(Get-Service "WinRM")) { + Write-ProgressLog "Unable to find the WinRM service." + Throw "Unable to find the WinRM service." +} +ElseIf ((Get-Service "WinRM").Status -ne "Running") { + Write-Verbose "Setting WinRM service to start automatically on boot." + Set-Service -Name "WinRM" -StartupType Automatic + Write-ProgressLog "Set WinRM service to start automatically on boot." + Write-Verbose "Starting WinRM service." + Start-Service -Name "WinRM" -ErrorAction Stop + Write-ProgressLog "Started WinRM service." + +} + +# WinRM should be running; check that we have a PS session config. +If (!(Get-PSSessionConfiguration -Verbose:$false) -or (!(Get-ChildItem WSMan:\localhost\Listener))) { + If ($SkipNetworkProfileCheck) { + Write-Verbose "Enabling PS Remoting without checking Network profile." + Enable-PSRemoting -SkipNetworkProfileCheck -Force -ErrorAction Stop + Write-ProgressLog "Enabled PS Remoting without checking Network profile." + } + Else { + Write-Verbose "Enabling PS Remoting." + Enable-PSRemoting -Force -ErrorAction Stop + Write-ProgressLog "Enabled PS Remoting." + } +} +Else { + Write-Verbose "PS Remoting is already enabled." +} + +# Ensure LocalAccountTokenFilterPolicy is set to 1 +# https://github.com/ansible/ansible/issues/42978 +$token_path = "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System" +$token_prop_name = "LocalAccountTokenFilterPolicy" +$token_key = Get-Item -Path $token_path +$token_value = $token_key.GetValue($token_prop_name, $null) +if ($token_value -ne 1) { + Write-Verbose "Setting LocalAccountTOkenFilterPolicy to 1" + if ($null -ne $token_value) { + Remove-ItemProperty -Path $token_path -Name $token_prop_name + } + New-ItemProperty -Path $token_path -Name $token_prop_name -Value 1 -PropertyType DWORD > $null +} + +# Make sure there is a SSL listener. +$listeners = Get-ChildItem WSMan:\localhost\Listener +If (!($listeners | Where-Object { $_.Keys -like "TRANSPORT=HTTPS" })) { + # We cannot use New-SelfSignedCertificate on 2012R2 and earlier + $thumbprint = New-LegacySelfSignedCert -SubjectName $SubjectName -ValidDays $CertValidityDays + Write-HostLog "Self-signed SSL certificate generated; thumbprint: $thumbprint" + + # Create the hashtables of settings to be used. + $valueset = @{ + Hostname = $SubjectName + CertificateThumbprint = $thumbprint + } + + $selectorset = @{ + Transport = "HTTPS" + Address = "*" + } + + Write-Verbose "Enabling SSL listener." + New-WSManInstance -ResourceURI 'winrm/config/Listener' -SelectorSet $selectorset -ValueSet $valueset + Write-ProgressLog "Enabled SSL listener." +} +Else { + Write-Verbose "SSL listener is already active." + + # Force a new SSL cert on Listener if the $ForceNewSSLCert + If ($ForceNewSSLCert) { + + # We cannot use New-SelfSignedCertificate on 2012R2 and earlier + $thumbprint = New-LegacySelfSignedCert -SubjectName $SubjectName -ValidDays $CertValidityDays + Write-HostLog "Self-signed SSL certificate generated; thumbprint: $thumbprint" + + $valueset = @{ + CertificateThumbprint = $thumbprint + Hostname = $SubjectName + } + + # Delete the listener for SSL + $selectorset = @{ + Address = "*" + Transport = "HTTPS" + } + Remove-WSManInstance -ResourceURI 'winrm/config/Listener' -SelectorSet $selectorset + + # Add new Listener with new SSL cert + New-WSManInstance -ResourceURI 'winrm/config/Listener' -SelectorSet $selectorset -ValueSet $valueset + } +} + +# Check for basic authentication. +$basicAuthSetting = Get-ChildItem WSMan:\localhost\Service\Auth | Where-Object { $_.Name -eq "Basic" } + +If ($DisableBasicAuth) { + If (($basicAuthSetting.Value) -eq $true) { + Write-Verbose "Disabling basic auth support." + Set-Item -Path "WSMan:\localhost\Service\Auth\Basic" -Value $false + Write-ProgressLog "Disabled basic auth support." + } + Else { + Write-Verbose "Basic auth is already disabled." + } +} +Else { + If (($basicAuthSetting.Value) -eq $false) { + Write-Verbose "Enabling basic auth support." + Set-Item -Path "WSMan:\localhost\Service\Auth\Basic" -Value $true + Write-ProgressLog "Enabled basic auth support." + } + Else { + Write-Verbose "Basic auth is already enabled." + } +} + +# If EnableCredSSP if set to true +If ($EnableCredSSP) { + # Check for CredSSP authentication + $credsspAuthSetting = Get-ChildItem WSMan:\localhost\Service\Auth | Where-Object { $_.Name -eq "CredSSP" } + If (($credsspAuthSetting.Value) -eq $false) { + Write-Verbose "Enabling CredSSP auth support." + Enable-WSManCredSSP -role server -Force + Write-ProgressLog "Enabled CredSSP auth support." + } +} + +If ($GlobalHttpFirewallAccess) { + Enable-GlobalHttpFirewallAccess +} + +# Configure firewall to allow WinRM HTTPS connections. +$fwtest1 = netsh advfirewall firewall show rule name="Allow WinRM HTTPS" +$fwtest2 = netsh advfirewall firewall show rule name="Allow WinRM HTTPS" profile=any +If ($fwtest1.count -lt 5) { + Write-Verbose "Adding firewall rule to allow WinRM HTTPS." + netsh advfirewall firewall add rule profile=any name="Allow WinRM HTTPS" dir=in localport=5986 protocol=TCP action=allow + Write-ProgressLog "Added firewall rule to allow WinRM HTTPS." +} +ElseIf (($fwtest1.count -ge 5) -and ($fwtest2.count -lt 5)) { + Write-Verbose "Updating firewall rule to allow WinRM HTTPS for any profile." + netsh advfirewall firewall set rule name="Allow WinRM HTTPS" new profile=any + Write-ProgressLog "Updated firewall rule to allow WinRM HTTPS for any profile." +} +Else { + Write-Verbose "Firewall rule already exists to allow WinRM HTTPS." +} + +# Test a remoting connection to localhost, which should work. +$httpResult = Invoke-Command -ComputerName "localhost" -ScriptBlock { $using:env:COMPUTERNAME } -ErrorVariable httpError -ErrorAction SilentlyContinue +$httpsOptions = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck + +$httpsResult = New-PSSession -UseSSL -ComputerName "localhost" -SessionOption $httpsOptions -ErrorVariable httpsError -ErrorAction SilentlyContinue + +If ($httpResult -and $httpsResult) { + Write-Verbose "HTTP: Enabled | HTTPS: Enabled" +} +ElseIf ($httpsResult -and !$httpResult) { + Write-Verbose "HTTP: Disabled | HTTPS: Enabled" +} +ElseIf ($httpResult -and !$httpsResult) { + Write-Verbose "HTTP: Enabled | HTTPS: Disabled" +} +Else { + Write-ProgressLog "Unable to establish an HTTP or HTTPS remoting session." + Throw "Unable to establish an HTTP or HTTPS remoting session." +} +Write-VerboseLog "PS Remoting has been successfully configured for Ansible." diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh new file mode 100644 index 0000000..732c122 --- /dev/null +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -0,0 +1,450 @@ +# shellcheck shell=sh + +set -eu + +install_ssh_keys() +{ + if [ ! -f "${ssh_private_key_path}" ]; then + # write public/private ssh key pair + public_key_path="${ssh_private_key_path}.pub" + + # shellcheck disable=SC2174 + mkdir -m 0700 -p "${ssh_path}" + touch "${public_key_path}" "${ssh_private_key_path}" + chmod 0600 "${public_key_path}" "${ssh_private_key_path}" + echo "${ssh_public_key}" > "${public_key_path}" + echo "${ssh_private_key}" > "${ssh_private_key_path}" + + # add public key to authorized_keys + authoried_keys_path="${HOME}/.ssh/authorized_keys" + + # the existing file is overwritten to avoid conflicts (ex: RHEL on EC2 blocks root login) + cat "${public_key_path}" > "${authoried_keys_path}" + chmod 0600 "${authoried_keys_path}" + + # add localhost's server keys to known_hosts + known_hosts_path="${HOME}/.ssh/known_hosts" + + for key in /etc/ssh/ssh_host_*_key.pub; do + echo "localhost $(cat "${key}")" >> "${known_hosts_path}" + done + fi +} + +customize_bashrc() +{ + true > ~/.bashrc + + # Show color `ls` results when available. + if ls --color > /dev/null 2>&1; then + echo "alias ls='ls --color'" >> ~/.bashrc + elif ls -G > /dev/null 2>&1; then + echo "alias ls='ls -G'" >> ~/.bashrc + fi + + # Improve shell prompts for interactive use. + echo "export PS1='\[\e]0;\u@\h: \w\a\]\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '" >> ~/.bashrc +} + +install_pip() { + if ! "${python_interpreter}" -m pip.__main__ --version --disable-pip-version-check 2>/dev/null; then + case "${python_version}" in + "2.7") + pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-20.3.4.py" + ;; + *) + pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-21.3.1.py" + ;; + esac + + while true; do + curl --silent --show-error "${pip_bootstrap_url}" -o /tmp/get-pip.py && \ + "${python_interpreter}" /tmp/get-pip.py --disable-pip-version-check --quiet && \ + rm /tmp/get-pip.py \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done + fi +} + +pip_install() { + pip_packages="$1" + + while true; do + # shellcheck disable=SC2086 + "${python_interpreter}" -m pip install --disable-pip-version-check ${pip_packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done +} + +bootstrap_remote_alpine() +{ + py_pkg_prefix="py3" + + packages=" + acl + bash + gcc + python3-dev + ${py_pkg_prefix}-pip + sudo + " + + if [ "${controller}" ]; then + packages=" + ${packages} + ${py_pkg_prefix}-cryptography + ${py_pkg_prefix}-packaging + ${py_pkg_prefix}-yaml + ${py_pkg_prefix}-jinja2 + ${py_pkg_prefix}-resolvelib + " + fi + + while true; do + # shellcheck disable=SC2086 + apk add -q ${packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done +} + +bootstrap_remote_fedora() +{ + py_pkg_prefix="python3" + + packages=" + acl + gcc + ${py_pkg_prefix}-devel + " + + if [ "${controller}" ]; then + packages=" + ${packages} + ${py_pkg_prefix}-cryptography + ${py_pkg_prefix}-jinja2 + ${py_pkg_prefix}-packaging + ${py_pkg_prefix}-pyyaml + ${py_pkg_prefix}-resolvelib + " + fi + + while true; do + # shellcheck disable=SC2086 + dnf install -q -y ${packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done +} + +bootstrap_remote_freebsd() +{ + packages=" + python${python_package_version} + py${python_package_version}-sqlite3 + bash + curl + gtar + sudo + " + + if [ "${controller}" ]; then + jinja2_pkg="py${python_package_version}-jinja2" + cryptography_pkg="py${python_package_version}-cryptography" + pyyaml_pkg="py${python_package_version}-yaml" + + # Declare platform/python version combinations which do not have supporting OS packages available. + # For these combinations ansible-test will use pip to install the requirements instead. + case "${platform_version}/${python_version}" in + *) + jinja2_pkg="" # not available + cryptography_pkg="" # not available + pyyaml_pkg="" # not available + ;; + esac + + packages=" + ${packages} + libyaml + ${pyyaml_pkg} + ${jinja2_pkg} + ${cryptography_pkg} + " + fi + + while true; do + # shellcheck disable=SC2086 + env ASSUME_ALWAYS_YES=YES pkg bootstrap && \ + pkg install -q -y ${packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done + + install_pip + + if ! grep '^PermitRootLogin yes$' /etc/ssh/sshd_config > /dev/null; then + sed -i '' 's/^# *PermitRootLogin.*$/PermitRootLogin yes/;' /etc/ssh/sshd_config + service sshd restart + fi + + # make additional wheels available for packages which lack them for this platform + echo "# generated by ansible-test +[global] +extra-index-url = https://spare-tire.testing.ansible.com/simple/ +prefer-binary = yes +" > /etc/pip.conf + + # enable ACL support on the root filesystem (required for become between unprivileged users) + fs_path="/" + fs_device="$(mount -v "${fs_path}" | cut -w -f 1)" + # shellcheck disable=SC2001 + fs_device_escaped=$(echo "${fs_device}" | sed 's|/|\\/|g') + + mount -o acls "${fs_device}" "${fs_path}" + awk 'BEGIN{FS=" "}; /'"${fs_device_escaped}"'/ {gsub(/^rw$/,"rw,acls", $4); print; next} // {print}' /etc/fstab > /etc/fstab.new + mv /etc/fstab.new /etc/fstab + + # enable sudo without a password for the wheel group, allowing ansible to use the sudo become plugin + echo '%wheel ALL=(ALL:ALL) NOPASSWD: ALL' > /usr/local/etc/sudoers.d/ansible-test +} + +bootstrap_remote_macos() +{ + # Silence macOS deprecation warning for bash. + echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bashrc + + # Make sure ~/ansible/ is the starting directory for interactive shells on the control node. + # The root home directory is under a symlink. Without this the real path will be displayed instead. + if [ "${controller}" ]; then + echo "cd ~/ansible/" >> ~/.bashrc + fi + + # Make sure commands like 'brew' can be found. + # This affects users with the 'zsh' shell, as well as 'root' accessed using 'sudo' from a user with 'zsh' for a shell. + # shellcheck disable=SC2016 + echo 'PATH="/usr/local/bin:$PATH"' > /etc/zshenv +} + +bootstrap_remote_rhel_7() +{ + packages=" + gcc + python-devel + python-virtualenv + " + + while true; do + # shellcheck disable=SC2086 + yum install -q -y ${packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done + + install_pip + + bootstrap_remote_rhel_pinned_pip_packages +} + +bootstrap_remote_rhel_8() +{ + if [ "${python_version}" = "3.6" ]; then + py_pkg_prefix="python3" + else + py_pkg_prefix="python${python_package_version}" + fi + + packages=" + gcc + ${py_pkg_prefix}-devel + " + + # Jinja2 is not installed with an OS package since the provided version is too old. + # Instead, ansible-test will install it using pip. + if [ "${controller}" ]; then + packages=" + ${packages} + ${py_pkg_prefix}-cryptography + " + fi + + while true; do + # shellcheck disable=SC2086 + yum module install -q -y "python${python_package_version}" && \ + yum install -q -y ${packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done + + bootstrap_remote_rhel_pinned_pip_packages +} + +bootstrap_remote_rhel_9() +{ + py_pkg_prefix="python3" + + packages=" + gcc + ${py_pkg_prefix}-devel + " + + # Jinja2 is not installed with an OS package since the provided version is too old. + # Instead, ansible-test will install it using pip. + if [ "${controller}" ]; then + packages=" + ${packages} + ${py_pkg_prefix}-cryptography + ${py_pkg_prefix}-packaging + ${py_pkg_prefix}-pyyaml + ${py_pkg_prefix}-resolvelib + " + fi + + while true; do + # shellcheck disable=SC2086 + dnf install -q -y ${packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done +} + +bootstrap_remote_rhel() +{ + case "${platform_version}" in + 7.*) bootstrap_remote_rhel_7 ;; + 8.*) bootstrap_remote_rhel_8 ;; + 9.*) bootstrap_remote_rhel_9 ;; + esac +} + +bootstrap_remote_rhel_pinned_pip_packages() +{ + # pin packaging and pyparsing to match the downstream vendored versions + pip_packages=" + packaging==20.4 + pyparsing==2.4.7 + " + + pip_install "${pip_packages}" +} + +bootstrap_remote_ubuntu() +{ + py_pkg_prefix="python3" + + packages=" + acl + gcc + python${python_version}-dev + python3-pip + python${python_version}-venv + " + + if [ "${controller}" ]; then + cryptography_pkg="${py_pkg_prefix}-cryptography" + jinja2_pkg="${py_pkg_prefix}-jinja2" + packaging_pkg="${py_pkg_prefix}-packaging" + pyyaml_pkg="${py_pkg_prefix}-yaml" + resolvelib_pkg="${py_pkg_prefix}-resolvelib" + + # Declare platforms which do not have supporting OS packages available. + # For these ansible-test will use pip to install the requirements instead. + # Only the platform is checked since Ubuntu shares Python packages across Python versions. + case "${platform_version}" in + "20.04") + jinja2_pkg="" # too old + resolvelib_pkg="" # not available + ;; + esac + + packages=" + ${packages} + ${cryptography_pkg} + ${jinja2_pkg} + ${packaging_pkg} + ${pyyaml_pkg} + ${resolvelib_pkg} + " + fi + + while true; do + # shellcheck disable=SC2086 + apt-get update -qq -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends ${packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done + + if [ "${controller}" ]; then + if [ "${platform_version}/${python_version}" = "20.04/3.9" ]; then + # Install pyyaml using pip so libyaml support is available on Python 3.9. + # The OS package install (which is installed by default) only has a .so file for Python 3.8. + pip_install "--upgrade pyyaml" + fi + fi +} + +bootstrap_docker() +{ + # Required for newer mysql-server packages to install/upgrade on Ubuntu 16.04. + rm -f /usr/sbin/policy-rc.d +} + +bootstrap_remote() +{ + for python_version in ${python_versions}; do + echo "Bootstrapping Python ${python_version}" + + python_interpreter="python${python_version}" + python_package_version="$(echo "${python_version}" | tr -d '.')" + + case "${platform}" in + "alpine") bootstrap_remote_alpine ;; + "fedora") bootstrap_remote_fedora ;; + "freebsd") bootstrap_remote_freebsd ;; + "macos") bootstrap_remote_macos ;; + "rhel") bootstrap_remote_rhel ;; + "ubuntu") bootstrap_remote_ubuntu ;; + esac + done +} + +bootstrap() +{ + ssh_path="${HOME}/.ssh" + ssh_private_key_path="${ssh_path}/id_${ssh_key_type}" + + install_ssh_keys + customize_bashrc + + # allow tests to detect ansible-test bootstrapped instances, as well as the bootstrap type + echo "${bootstrap_type}" > /etc/ansible-test.bootstrap + + case "${bootstrap_type}" in + "docker") bootstrap_docker ;; + "remote") bootstrap_remote ;; + esac +} + +# These variables will be templated before sending the script to the host. +# They are at the end of the script to maintain line numbers for debugging purposes. +bootstrap_type=#{bootstrap_type} +controller=#{controller} +platform=#{platform} +platform_version=#{platform_version} +python_versions=#{python_versions} +ssh_key_type=#{ssh_key_type} +ssh_private_key=#{ssh_private_key} +ssh_public_key=#{ssh_public_key} + +bootstrap diff --git a/test/lib/ansible_test/_util/target/setup/check_systemd_cgroup_v1.sh b/test/lib/ansible_test/_util/target/setup/check_systemd_cgroup_v1.sh new file mode 100644 index 0000000..3b05a3f --- /dev/null +++ b/test/lib/ansible_test/_util/target/setup/check_systemd_cgroup_v1.sh @@ -0,0 +1,17 @@ +# shellcheck shell=sh + +set -eu + +>&2 echo "@MARKER@" + +cgroup_path="$(awk -F: '$2 ~ /^name=systemd$/ { print "/sys/fs/cgroup/systemd"$3 }' /proc/1/cgroup)" + +if [ "${cgroup_path}" ] && [ -d "${cgroup_path}" ]; then + probe_path="${cgroup_path%/}/ansible-test-probe-@LABEL@" + mkdir "${probe_path}" + rmdir "${probe_path}" + exit 0 +fi + +>&2 echo "No systemd cgroup v1 hierarchy found" +exit 1 diff --git a/test/lib/ansible_test/_util/target/setup/probe_cgroups.py b/test/lib/ansible_test/_util/target/setup/probe_cgroups.py new file mode 100644 index 0000000..2ac7ecb --- /dev/null +++ b/test/lib/ansible_test/_util/target/setup/probe_cgroups.py @@ -0,0 +1,31 @@ +"""A tool for probing cgroups to determine write access.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json +import os +import sys + + +def main(): # type: () -> None + """Main program entry point.""" + probe_dir = sys.argv[1] + paths = sys.argv[2:] + results = {} + + for path in paths: + probe_path = os.path.join(path, probe_dir) + + try: + os.mkdir(probe_path) + os.rmdir(probe_path) + except Exception as ex: # pylint: disable=broad-except + results[path] = str(ex) + else: + results[path] = None + + print(json.dumps(results, sort_keys=True)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/target/setup/quiet_pip.py b/test/lib/ansible_test/_util/target/setup/quiet_pip.py new file mode 100644 index 0000000..54f0f86 --- /dev/null +++ b/test/lib/ansible_test/_util/target/setup/quiet_pip.py @@ -0,0 +1,72 @@ +"""Custom entry-point for pip that filters out unwanted logging and warnings.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import logging +import os +import re +import runpy +import sys +import warnings + +BUILTIN_FILTERER_FILTER = logging.Filterer.filter + +LOGGING_MESSAGE_FILTER = re.compile("^(" + ".*Running pip install with root privileges is generally not a good idea.*|" # custom Fedora patch [1] + ".*Running pip as the 'root' user can result in broken permissions .*|" # pip 21.1 + "DEPRECATION: Python 2.7 will reach the end of its life .*|" # pip 19.2.3 + "Ignoring .*: markers .* don't match your environment|" + "Looking in indexes: .*|" # pypi-test-container + "Requirement already satisfied.*" + ")$") + +# [1] https://src.fedoraproject.org/rpms/python-pip/blob/f34/f/emit-a-warning-when-running-with-root-privileges.patch + +WARNING_MESSAGE_FILTERS = ( + # DEPRECATION: Python 2.7 reached the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 is no longer maintained. + # pip 21.0 will drop support for Python 2.7 in January 2021. + # More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support + 'DEPRECATION: Python 2.7 reached the end of its life ', + + # DEPRECATION: Python 3.5 reached the end of its life on September 13th, 2020. Please upgrade your Python as Python 3.5 is no longer maintained. + # pip 21.0 will drop support for Python 3.5 in January 2021. pip 21.0 will remove support for this functionality. + 'DEPRECATION: Python 3.5 reached the end of its life ', +) + + +def custom_filterer_filter(self, record): + """Globally omit logging of unwanted messages.""" + if LOGGING_MESSAGE_FILTER.search(record.getMessage()): + return 0 + + return BUILTIN_FILTERER_FILTER(self, record) + + +def main(): + """Main program entry point.""" + # Filtering logging output globally avoids having to intercept stdout/stderr. + # It also avoids problems with loss of color output and mixing up the order of stdout/stderr messages. + logging.Filterer.filter = custom_filterer_filter + + for message_filter in WARNING_MESSAGE_FILTERS: + # Setting filterwarnings in code is necessary because of the following: + # Python 2.7 cannot use the -W option to match warning text after a colon. This makes it impossible to match specific warning messages. + warnings.filterwarnings('ignore', message_filter) + + get_pip = os.environ.get('GET_PIP') + + try: + if get_pip: + directory, filename = os.path.split(get_pip) + module = os.path.splitext(filename)[0] + sys.path.insert(0, directory) + runpy.run_module(module, run_name='__main__', alter_sys=True) + else: + runpy.run_module('pip.__main__', run_name='__main__', alter_sys=True) + except ImportError as ex: + print('pip is unavailable: %s' % ex) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/target/setup/requirements.py b/test/lib/ansible_test/_util/target/setup/requirements.py new file mode 100644 index 0000000..4fe9a6c --- /dev/null +++ b/test/lib/ansible_test/_util/target/setup/requirements.py @@ -0,0 +1,337 @@ +"""A tool for installing test requirements on the controller and target host.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +# pylint: disable=wrong-import-position + +import resource + +# Setting a low soft RLIMIT_NOFILE value will improve the performance of subprocess.Popen on Python 2.x when close_fds=True. +# This will affect all Python subprocesses. It will also affect the current Python process if set before subprocess is imported for the first time. +SOFT_RLIMIT_NOFILE = 1024 + +CURRENT_RLIMIT_NOFILE = resource.getrlimit(resource.RLIMIT_NOFILE) +DESIRED_RLIMIT_NOFILE = (SOFT_RLIMIT_NOFILE, CURRENT_RLIMIT_NOFILE[1]) + +if DESIRED_RLIMIT_NOFILE < CURRENT_RLIMIT_NOFILE: + resource.setrlimit(resource.RLIMIT_NOFILE, DESIRED_RLIMIT_NOFILE) + CURRENT_RLIMIT_NOFILE = DESIRED_RLIMIT_NOFILE + +import base64 +import contextlib +import errno +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile + +try: + import typing as t +except ImportError: + t = None + +try: + from shlex import quote as cmd_quote +except ImportError: + # noinspection PyProtectedMember + from pipes import quote as cmd_quote + +try: + from urllib.request import urlopen +except ImportError: + # noinspection PyCompatibility,PyUnresolvedReferences + from urllib2 import urlopen # pylint: disable=ansible-bad-import-from + +ENCODING = 'utf-8' + +Text = type(u'') + +VERBOSITY = 0 +CONSOLE = sys.stderr + + +def main(): # type: () -> None + """Main program entry point.""" + global VERBOSITY # pylint: disable=global-statement + + payload = json.loads(to_text(base64.b64decode(PAYLOAD))) + + VERBOSITY = payload['verbosity'] + + script = payload['script'] + commands = payload['commands'] + + with tempfile.NamedTemporaryFile(prefix='ansible-test-', suffix='-pip.py') as pip: + pip.write(to_bytes(script)) + pip.flush() + + for name, options in commands: + try: + globals()[name](pip.name, options) + except ApplicationError as ex: + print(ex) + sys.exit(1) + + +# noinspection PyUnusedLocal +def bootstrap(pip, options): # type: (str, t.Dict[str, t.Any]) -> None + """Bootstrap pip and related packages in an empty virtual environment.""" + pip_version = options['pip_version'] + packages = options['packages'] + + url = 'https://ci-files.testing.ansible.com/ansible-test/get-pip-%s.py' % pip_version + cache_path = os.path.expanduser('~/.ansible/test/cache/get_pip_%s.py' % pip_version.replace(".", "_")) + temp_path = cache_path + '.download' + + if os.path.exists(cache_path): + log('Using cached pip %s bootstrap script: %s' % (pip_version, cache_path)) + else: + log('Downloading pip %s bootstrap script: %s' % (pip_version, url)) + + make_dirs(os.path.dirname(cache_path)) + + try: + download_file(url, temp_path) + except Exception as ex: + raise ApplicationError((''' +Download failed: %s + +The bootstrap script can be manually downloaded and saved to: %s + +If you're behind a proxy, consider commenting on the following GitHub issue: + +https://github.com/ansible/ansible/issues/77304 +''' % (ex, cache_path)).strip()) + + shutil.move(temp_path, cache_path) + + log('Cached pip %s bootstrap script: %s' % (pip_version, cache_path)) + + env = common_pip_environment() + env.update(GET_PIP=cache_path) + + options = common_pip_options() + options.extend(packages) + + command = [sys.executable, pip] + options + + execute_command(command, env=env) + + +def install(pip, options): # type: (str, t.Dict[str, t.Any]) -> None + """Perform a pip install.""" + requirements = options['requirements'] + constraints = options['constraints'] + packages = options['packages'] + + tempdir = tempfile.mkdtemp(prefix='ansible-test-', suffix='-requirements') + + try: + options = common_pip_options() + options.extend(packages) + + for path, content in requirements: + write_text_file(os.path.join(tempdir, path), content, True) + options.extend(['-r', path]) + + for path, content in constraints: + write_text_file(os.path.join(tempdir, path), content, True) + options.extend(['-c', path]) + + command = [sys.executable, pip, 'install'] + options + + env = common_pip_environment() + + execute_command(command, env=env, cwd=tempdir) + finally: + remove_tree(tempdir) + + +def uninstall(pip, options): # type: (str, t.Dict[str, t.Any]) -> None + """Perform a pip uninstall.""" + packages = options['packages'] + ignore_errors = options['ignore_errors'] + + options = common_pip_options() + options.extend(packages) + + command = [sys.executable, pip, 'uninstall', '-y'] + options + + env = common_pip_environment() + + try: + execute_command(command, env=env, capture=True) + except SubprocessError: + if not ignore_errors: + raise + + +# noinspection PyUnusedLocal +def version(pip, options): # type: (str, t.Dict[str, t.Any]) -> None + """Report the pip version.""" + del options + + options = common_pip_options() + + command = [sys.executable, pip, '-V'] + options + + env = common_pip_environment() + + execute_command(command, env=env, capture=True) + + +def common_pip_environment(): # type: () -> t.Dict[str, str] + """Return common environment variables used to run pip.""" + env = os.environ.copy() + + return env + + +def common_pip_options(): # type: () -> t.List[str] + """Return a list of common pip options.""" + return [ + '--disable-pip-version-check', + ] + + +def devnull(): # type: () -> t.IO[bytes] + """Return a file object that references devnull.""" + try: + return devnull.file + except AttributeError: + devnull.file = open(os.devnull, 'w+b') # pylint: disable=consider-using-with + + return devnull.file + + +def download_file(url, path): # type: (str, str) -> None + """Download the given URL to the specified file path.""" + with open(to_bytes(path), 'wb') as saved_file: + with contextlib.closing(urlopen(url)) as download: + shutil.copyfileobj(download, saved_file) + + +class ApplicationError(Exception): + """Base class for application exceptions.""" + + +class SubprocessError(ApplicationError): + """A command returned a non-zero status.""" + def __init__(self, cmd, status, stdout, stderr): # type: (t.List[str], int, str, str) -> None + message = 'A command failed with status %d: %s' % (status, ' '.join(cmd_quote(c) for c in cmd)) + + if stderr: + message += '\n>>> Standard Error\n%s' % stderr.strip() + + if stdout: + message += '\n>>> Standard Output\n%s' % stdout.strip() + + super(SubprocessError, self).__init__(message) + + +def log(message, verbosity=0): # type: (str, int) -> None + """Log a message to the console if the verbosity is high enough.""" + if verbosity > VERBOSITY: + return + + print(message, file=CONSOLE) + CONSOLE.flush() + + +def execute_command(cmd, cwd=None, capture=False, env=None): # type: (t.List[str], t.Optional[str], bool, t.Optional[t.Dict[str, str]]) -> None + """Execute the specified command.""" + log('Execute command: %s' % ' '.join(cmd_quote(c) for c in cmd), verbosity=1) + + cmd_bytes = [to_bytes(c) for c in cmd] + + if capture: + stdout = subprocess.PIPE + stderr = subprocess.PIPE + else: + stdout = None + stderr = None + + cwd_bytes = to_optional_bytes(cwd) + process = subprocess.Popen(cmd_bytes, cwd=cwd_bytes, stdin=devnull(), stdout=stdout, stderr=stderr, env=env) # pylint: disable=consider-using-with + stdout_bytes, stderr_bytes = process.communicate() + stdout_text = to_optional_text(stdout_bytes) or u'' + stderr_text = to_optional_text(stderr_bytes) or u'' + + if process.returncode != 0: + raise SubprocessError(cmd, process.returncode, stdout_text, stderr_text) + + +def write_text_file(path, content, create_directories=False): # type: (str, str, bool) -> None + """Write the given text content to the specified path, optionally creating missing directories.""" + if create_directories: + make_dirs(os.path.dirname(path)) + + with open_binary_file(path, 'wb') as file_obj: + file_obj.write(to_bytes(content)) + + +def remove_tree(path): # type: (str) -> None + """Remove the specified directory tree.""" + try: + shutil.rmtree(to_bytes(path)) + except OSError as ex: + if ex.errno != errno.ENOENT: + raise + + +def make_dirs(path): # type: (str) -> None + """Create a directory at path, including any necessary parent directories.""" + try: + os.makedirs(to_bytes(path)) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + + +def open_binary_file(path, mode='rb'): # type: (str, str) -> t.IO[bytes] + """Open the given path for binary access.""" + if 'b' not in mode: + raise Exception('mode must include "b" for binary files: %s' % mode) + + return io.open(to_bytes(path), mode) # pylint: disable=consider-using-with,unspecified-encoding + + +def to_optional_bytes(value, errors='strict'): # type: (t.Optional[t.AnyStr], str) -> t.Optional[bytes] + """Return the given value as bytes encoded using UTF-8 if not already bytes, or None if the value is None.""" + return None if value is None else to_bytes(value, errors) + + +def to_optional_text(value, errors='strict'): # type: (t.Optional[t.AnyStr], str) -> t.Optional[t.Text] + """Return the given value as text decoded using UTF-8 if not already text, or None if the value is None.""" + return None if value is None else to_text(value, errors) + + +def to_bytes(value, errors='strict'): # type: (t.AnyStr, str) -> bytes + """Return the given value as bytes encoded using UTF-8 if not already bytes.""" + if isinstance(value, bytes): + return value + + if isinstance(value, Text): + return value.encode(ENCODING, errors) + + raise Exception('value is not bytes or text: %s' % type(value)) + + +def to_text(value, errors='strict'): # type: (t.AnyStr, str) -> t.Text + """Return the given value as text decoded using UTF-8 if not already text.""" + if isinstance(value, bytes): + return value.decode(ENCODING, errors) + + if isinstance(value, Text): + return value + + raise Exception('value is not bytes or text: %s' % type(value)) + + +PAYLOAD = b'{payload}' # base-64 encoded JSON payload which will be populated before this script is executed + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/target/tools/virtualenvcheck.py b/test/lib/ansible_test/_util/target/tools/virtualenvcheck.py new file mode 100644 index 0000000..a38ad07 --- /dev/null +++ b/test/lib/ansible_test/_util/target/tools/virtualenvcheck.py @@ -0,0 +1,21 @@ +"""Detect the real python interpreter when running in a virtual environment created by the 'virtualenv' module.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + +try: + # virtualenv <20 + from sys import real_prefix +except ImportError: + real_prefix = None + +try: + # venv and virtualenv >= 20 + from sys import base_exec_prefix +except ImportError: + base_exec_prefix = None + +print(json.dumps(dict( + real_prefix=real_prefix or base_exec_prefix, +))) diff --git a/test/lib/ansible_test/_util/target/tools/yamlcheck.py b/test/lib/ansible_test/_util/target/tools/yamlcheck.py new file mode 100644 index 0000000..dfd08e5 --- /dev/null +++ b/test/lib/ansible_test/_util/target/tools/yamlcheck.py @@ -0,0 +1,20 @@ +"""Show availability of PyYAML and libyaml support.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + +try: + import yaml +except ImportError: + yaml = None + +try: + from yaml import CLoader +except ImportError: + CLoader = None + +print(json.dumps(dict( + yaml=bool(yaml), + cloader=bool(CLoader), +))) |