diff options
Diffstat (limited to '')
| -rw-r--r-- | testing/mozbase/moztest/moztest/resolve.py | 1032 |
1 files changed, 1032 insertions, 0 deletions
diff --git a/testing/mozbase/moztest/moztest/resolve.py b/testing/mozbase/moztest/moztest/resolve.py new file mode 100644 index 0000000000..1076d52865 --- /dev/null +++ b/testing/mozbase/moztest/moztest/resolve.py @@ -0,0 +1,1032 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import fnmatch +import os +import pickle +import sys +from abc import ABCMeta, abstractmethod +from collections import defaultdict + +import mozpack.path as mozpath +import six +from manifestparser import TestManifest, combine_fields +from mozbuild.base import MozbuildObject +from mozbuild.testing import REFTEST_FLAVORS, TEST_MANIFESTS +from mozbuild.util import OrderedDefaultDict +from mozpack.files import FileFinder + +here = os.path.abspath(os.path.dirname(__file__)) + +MOCHITEST_CHUNK_BY_DIR = 4 +MOCHITEST_TOTAL_CHUNKS = 5 + + +def WebglSuite(name): + return { + "aliases": (name,), + "build_flavor": "mochitest", + "mach_command": "mochitest", + "kwargs": {"flavor": "plain", "subsuite": name, "test_paths": None}, + "task_regex": [ + "mochitest-" + name + "($|.*(-1|[^0-9])$)", + "test-verify($|.*(-1|[^0-9])$)", + ], + } + + +TEST_SUITES = { + "cppunittest": { + "aliases": ("cpp",), + "mach_command": "cppunittest", + "kwargs": {"test_files": None}, + }, + "crashtest": { + "aliases": ("c", "rc"), + "build_flavor": "crashtest", + "mach_command": "crashtest", + "kwargs": {"test_file": None}, + "task_regex": ["crashtest($|.*(-1|[^0-9])$)", "test-verify($|.*(-1|[^0-9])$)"], + }, + "crashtest-qr": { + "aliases": ("c", "rc"), + "build_flavor": "crashtest", + "mach_command": "crashtest", + "kwargs": {"test_file": None}, + "task_regex": [ + "crashtest-qr($|.*(-1|[^0-9])$)", + "test-verify($|.*(-1|[^0-9])$)", + ], + }, + "firefox-ui-functional": { + "aliases": ("fxfn",), + "mach_command": "firefox-ui-functional", + "kwargs": {}, + }, + "firefox-ui-update": { + "aliases": ("fxup",), + "mach_command": "firefox-ui-update", + "kwargs": {}, + }, + "marionette": { + "aliases": ("mn",), + "mach_command": "marionette-test", + "kwargs": {"tests": None}, + "task_regex": ["marionette($|.*(-1|[^0-9])$)"], + }, + "mochitest-a11y": { + "aliases": ("a11y", "ally"), + "build_flavor": "a11y", + "mach_command": "mochitest", + "kwargs": { + "flavor": "a11y", + "test_paths": None, + "e10s": False, + "enable_fission": False, + }, + "task_regex": [ + "mochitest-a11y($|.*(-1|[^0-9])$)", + "test-verify($|.*(-1|[^0-9])$)", + ], + }, + "mochitest-browser-chrome": { + "aliases": ("bc", "browser"), + "build_flavor": "browser-chrome", + "mach_command": "mochitest", + "kwargs": {"flavor": "browser-chrome", "test_paths": None}, + "task_regex": [ + "mochitest-browser-chrome($|.*(-1|[^0-9])$)", + "test-verify($|.*(-1|[^0-9])$)", + ], + }, + "mochitest-browser-chrome-screenshots": { + "aliases": ("ss", "screenshots-chrome"), + "build_flavor": "browser-chrome", + "mach_command": "mochitest", + "kwargs": { + "flavor": "browser-chrome", + "subsuite": "screenshots", + "test_paths": None, + }, + "task_regex": ["browser-screenshots($|.*(-1|[^0-9])$)"], + }, + "mochitest-chrome": { + "aliases": ("mc",), + "build_flavor": "chrome", + "mach_command": "mochitest", + "kwargs": { + "flavor": "chrome", + "test_paths": None, + "e10s": False, + "enable_fission": False, + }, + "task_regex": [ + "mochitest-chrome($|.*(-1|[^0-9])$)", + "test-verify($|.*(-1|[^0-9])$)", + ], + }, + "mochitest-chrome-gpu": { + "aliases": ("gpu",), + "build_flavor": "chrome", + "mach_command": "mochitest", + "kwargs": { + "flavor": "chrome", + "subsuite": "gpu", + "test_paths": None, + "e10s": False, + "enable_fission": False, + }, + "task_regex": [ + "mochitest-gpu($|.*(-1|[^0-9])$)", + "test-verify($|.*(-1|[^0-9])$)", + ], + }, + "mochitest-devtools-chrome": { + "aliases": ("dt", "devtools"), + "build_flavor": "browser-chrome", + "mach_command": "mochitest", + "kwargs": { + "flavor": "browser-chrome", + "subsuite": "devtools", + "test_paths": None, + }, + "task_regex": [ + "mochitest-devtools-chrome($|.*(-1|[^0-9])$)", + "test-verify($|.*(-1|[^0-9])$)", + ], + }, + "mochitest-browser-a11y": { + "aliases": ("ba", "browser-a11y"), + "build_flavor": "browser-chrome", + "mach_command": "mochitest", + "kwargs": { + "flavor": "browser-chrome", + "subsuite": "a11y", + "test_paths": None, + }, + "task_regex": [ + "mochitest-browser-a11y($|.*(-1|[^0-9])$)", + "test-verify($|.*(-1|[^0-9])$)", + ], + }, + "mochitest-media": { + "aliases": ("mpm", "plain-media"), + "build_flavor": "mochitest", + "mach_command": "mochitest", + "kwargs": {"flavor": "plain", "subsuite": "media", "test_paths": None}, + "task_regex": [ + "mochitest-media($|.*(-1|[^0-9])$)", + "test-verify($|.*(-1|[^0-9])$)", + ], + }, + "mochitest-browser-media": { + "aliases": ("bmda", "browser-mda"), + "build_flavor": "browser-chrome", + "mach_command": "mochitest", + "kwargs": { + "flavor": "browser-chrome", + "subsuite": "media-bc", + "test_paths": None, + }, + "task_regex": [ + "mochitest-browser-media($|.*(-1|[^0-9])$)", + "test-verify($|.*(-1|[^0-9])$)", + ], + }, + "mochitest-plain": { + "aliases": ( + "mp", + "plain", + ), + "build_flavor": "mochitest", + "mach_command": "mochitest", + "kwargs": {"flavor": "plain", "test_paths": None}, + "task_regex": [ + "mochitest-plain($|.*(-1|[^0-9])$)", # noqa + "test-verify($|.*(-1|[^0-9])$)", + ], + }, + "mochitest-plain-gpu": { + "aliases": ("gpu",), + "build_flavor": "mochitest", + "mach_command": "mochitest", + "kwargs": {"flavor": "plain", "subsuite": "gpu", "test_paths": None}, + "task_regex": [ + "mochitest-gpu($|.*(-1|[^0-9])$)", + "test-verify($|.*(-1|[^0-9])$)", + ], + }, + "mochitest-remote": { + "aliases": ("remote",), + "build_flavor": "browser-chrome", + "mach_command": "mochitest", + "kwargs": { + "flavor": "browser-chrome", + "subsuite": "remote", + "test_paths": None, + }, + "task_regex": [ + "mochitest-remote($|.*(-1|[^0-9])$)", + "test-verify($|.*(-1|[^0-9])$)", + ], + }, + "mochitest-webgl1-core": WebglSuite("webgl1-core"), + "mochitest-webgl1-ext": WebglSuite("webgl1-ext"), + "mochitest-webgl2-core": WebglSuite("webgl2-core"), + "mochitest-webgl2-ext": WebglSuite("webgl2-ext"), + "mochitest-webgl2-deqp": WebglSuite("webgl2-deqp"), + "mochitest-webgpu": WebglSuite("webgpu"), + "puppeteer": { + "aliases": ("remote/test/puppeteer",), + "mach_command": "puppeteer-test", + "kwargs": {"headless": False}, + }, + "python": { + "build_flavor": "python", + "mach_command": "python-test", + "kwargs": {"tests": None}, + }, + "telemetry-tests-client": { + "aliases": ("ttc",), + "build_flavor": "telemetry-tests-client", + "mach_command": "telemetry-tests-client", + "kwargs": {}, + "task_regex": ["telemetry-tests-client($|.*(-1|[^0-9])$)"], + }, + "reftest": { + "aliases": ("rr",), + "build_flavor": "reftest", + "mach_command": "reftest", + "kwargs": {"tests": None}, + "task_regex": [ + "(opt|debug)(-geckoview)?-reftest($|.*(-1|[^0-9])$)", + "test-verify-gpu($|.*(-1|[^0-9])$)", + ], + }, + "reftest-qr": { + "aliases": ("rr",), + "build_flavor": "reftest", + "mach_command": "reftest", + "kwargs": {"tests": None}, + "task_regex": [ + "(opt|debug)(-geckoview)?-reftest-qr($|.*(-1|[^0-9])$)", + "test-verify-gpu($|.*(-1|[^0-9])$)", + ], + }, + "robocop": { + "mach_command": "robocop", + "kwargs": {"test_paths": None}, + "task_regex": ["robocop($|.*(-1|[^0-9])$)"], + }, + "web-platform-tests": { + "aliases": ("wpt",), + "mach_command": "web-platform-tests", + "build_flavor": "web-platform-tests", + "kwargs": {"subsuite": "testharness"}, + "task_regex": [ + "web-platform-tests(?!-crashtest|-reftest|-wdspec|-print)" + "($|.*(-1|[^0-9])$)", + "test-verify-wpt", + ], + }, + "web-platform-tests-crashtest": { + "aliases": ("wpt",), + "mach_command": "web-platform-tests", + "build_flavor": "web-platform-tests", + "kwargs": {"subsuite": "crashtest"}, + "task_regex": [ + "web-platform-tests-crashtest($|.*(-1|[^0-9])$)", + "test-verify-wpt", + ], + }, + "web-platform-tests-print-reftest": { + "aliases": ("wpt",), + "mach_command": "web-platform-tests", + "kwargs": {"subsuite": "print-reftest"}, + "task_regex": [ + "web-platform-tests-print-reftest($|.*(-1|[^0-9])$)", + "test-verify-wpt", + ], + }, + "web-platform-tests-reftest": { + "aliases": ("wpt",), + "mach_command": "web-platform-tests", + "build_flavor": "web-platform-tests", + "kwargs": {"subsuite": "reftest"}, + "task_regex": [ + "web-platform-tests-reftest($|.*(-1|[^0-9])$)", + "test-verify-wpt", + ], + }, + "web-platform-tests-wdspec": { + "aliases": ("wpt",), + "mach_command": "web-platform-tests", + "build_flavor": "web-platform-tests", + "kwargs": {"subsuite": "wdspec"}, + "task_regex": [ + "web-platform-tests-wdspec($|.*(-1|[^0-9])$)", + "test-verify-wpt", + ], + }, + "valgrind": { + "aliases": ("v",), + "mach_command": "valgrind-test", + "kwargs": {}, + }, + "xpcshell": { + "aliases": ("x",), + "build_flavor": "xpcshell", + "mach_command": "xpcshell-test", + "kwargs": {"test_file": "all"}, + "task_regex": ["xpcshell($|.*(-1|[^0-9])$)", "test-verify($|.*(-1|[^0-9])$)"], + }, + "xpcshell-msix": { + "aliases": ("x",), + "build_flavor": "xpcshell", + "mach_command": "xpcshell-test", + "kwargs": {"test_file": "all"}, + "task_regex": ["xpcshell($|.*(-1|[^0-9])$)", "test-verify($|.*(-1|[^0-9])$)"], + }, +} +"""Definitions of all test suites and the metadata needed to run and process +them. Each test suite definition can contain the following keys. + +Arguments: + aliases (tuple): A tuple containing shorthands used to refer to this suite. + build_flavor (str): The flavor assigned to this suite by the build system + in `mozbuild.testing.TEST_MANIFESTS` (or similar). + mach_command (str): Name of the mach command used to run this suite. + kwargs (dict): Arguments needed to pass into the mach command. + task_regex (list): A list of regexes used to filter task labels that run + this suite. +""" + +for i in range(1, MOCHITEST_TOTAL_CHUNKS + 1): + TEST_SUITES["mochitest-%d" % i] = { + "aliases": ("m%d" % i,), + "mach_command": "mochitest", + "kwargs": { + "flavor": "mochitest", + "subsuite": "default", + "chunk_by_dir": MOCHITEST_CHUNK_BY_DIR, + "total_chunks": MOCHITEST_TOTAL_CHUNKS, + "this_chunk": i, + "test_paths": None, + }, + } + + +WPT_TYPES = set() +for suite, data in TEST_SUITES.items(): + if suite.startswith("web-platform-tests"): + WPT_TYPES.add(data["kwargs"]["subsuite"]) + + +_test_flavors = { + "a11y": "mochitest-a11y", + "browser-chrome": "mochitest-browser-chrome", + "chrome": "mochitest-chrome", + "crashtest": "crashtest", + "firefox-ui-functional": "firefox-ui-functional", + "firefox-ui-update": "firefox-ui-update", + "marionette": "marionette", + "mochitest": "mochitest-plain", + "puppeteer": "puppeteer", + "python": "python", + "reftest": "reftest", + "telemetry-tests-client": "telemetry-tests-client", + "web-platform-tests": "web-platform-tests", + "xpcshell": "xpcshell", +} + +_test_subsuites = { + ("browser-chrome", "a11y"): "mochitest-browser-a11y", + ("browser-chrome", "devtools"): "mochitest-devtools-chrome", + ("browser-chrome", "media"): "mochitest-browser-media", + ("browser-chrome", "remote"): "mochitest-remote", + ("browser-chrome", "screenshots"): "mochitest-browser-chrome-screenshots", + ("chrome", "gpu"): "mochitest-chrome-gpu", + ("mochitest", "gpu"): "mochitest-plain-gpu", + ("mochitest", "media"): "mochitest-media", + ("mochitest", "robocop"): "robocop", + ("mochitest", "webgl1-core"): "mochitest-webgl1-core", + ("mochitest", "webgl1-ext"): "mochitest-webgl1-ext", + ("mochitest", "webgl2-core"): "mochitest-webgl2-core", + ("mochitest", "webgl2-ext"): "mochitest-webgl2-ext", + ("mochitest", "webgl2-deqp"): "mochitest-webgl2-deqp", + ("mochitest", "webgpu"): "mochitest-webgpu", + ("web-platform-tests", "testharness"): "web-platform-tests", + ("web-platform-tests", "crashtest"): "web-platform-tests-crashtest", + ("web-platform-tests", "print-reftest"): "web-platform-tests-print-reftest", + ("web-platform-tests", "reftest"): "web-platform-tests-reftest", + ("web-platform-tests", "wdspec"): "web-platform-tests-wdspec", +} + + +def get_suite_definition(flavor, subsuite=None, strict=False): + """Return a suite definition given a flavor and optional subsuite. + + If strict is True, a subsuite must have its own entry in TEST_SUITES. + Otherwise, the entry for 'flavor' will be returned with the 'subsuite' + keyword arg set. + + With or without strict mode, an empty dict will be returned if no + matching suite definition was found. + """ + if not subsuite: + suite_name = _test_flavors.get(flavor) + return suite_name, TEST_SUITES.get(suite_name, {}).copy() + + suite_name = _test_subsuites.get((flavor, subsuite)) + if suite_name or strict: + return suite_name, TEST_SUITES.get(suite_name, {}).copy() + + suite_name = _test_flavors.get(flavor) + if suite_name not in TEST_SUITES: + return suite_name, {} + + suite = TEST_SUITES[suite_name].copy() + suite.setdefault("kwargs", {}) + suite["kwargs"]["subsuite"] = subsuite + return suite_name, suite + + +def rewrite_test_base(test, new_base): + """Rewrite paths in a test to be under a new base path. + + This is useful for running tests from a separate location from where they + were defined. + """ + test["here"] = mozpath.join(new_base, test["dir_relpath"]) + test["path"] = mozpath.join(new_base, test["file_relpath"]) + return test + + +@six.add_metaclass(ABCMeta) +class TestLoader(MozbuildObject): + @abstractmethod + def __call__(self): + """Generate test metadata.""" + + +class BuildBackendLoader(TestLoader): + def __call__(self): + """Loads the test metadata generated by the TestManifest build backend. + + The data is stored in two files: + + - <objdir>/all-tests.pkl + - <objdir>/test-defaults.pkl + + The 'all-tests.pkl' file is a mapping of source path to test objects. The + 'test-defaults.pkl' file maps manifests to their DEFAULT configuration. + These manifest defaults will be merged into the test configuration of the + contained tests. + """ + # If installing tests is going to result in re-generating the build + # backend, we need to do this here, so that the updated contents of + # all-tests.pkl make it to the set of tests to run. + if self.backend_out_of_date( + mozpath.join(self.topobjdir, "backend.TestManifestBackend") + ): + print("Test configuration changed. Regenerating backend.") + from mozbuild.gen_test_backend import gen_test_backend + + gen_test_backend() + + all_tests = os.path.join(self.topobjdir, "all-tests.pkl") + test_defaults = os.path.join(self.topobjdir, "test-defaults.pkl") + + with open(all_tests, "rb") as fh: + test_data = pickle.load(fh) + + with open(test_defaults, "rb") as fh: + defaults = pickle.load(fh) + + # The keys in defaults use platform-specific path separators. + # self.topsrcdir was normalized to use /, revert back to \ if needed. + topsrcdir = os.path.normpath(self.topsrcdir) + + for path, tests in six.iteritems(test_data): + for metadata in tests: + defaults_manifests = [metadata["manifest"]] + + ancestor_manifest = metadata.get("ancestor_manifest") + if ancestor_manifest: + # The (ancestor manifest, included manifest) tuple + # contains the defaults of the included manifest, so + # use it instead of [metadata['manifest']]. + ancestor_manifest = os.path.join(topsrcdir, ancestor_manifest) + defaults_manifests[0] = (ancestor_manifest, metadata["manifest"]) + defaults_manifests.append(ancestor_manifest) + + for manifest in defaults_manifests: + manifest_defaults = defaults.get(manifest) + if manifest_defaults: + metadata = combine_fields(manifest_defaults, metadata) + + yield metadata + + +class TestManifestLoader(TestLoader): + def __init__(self, *args, **kwargs): + super(TestManifestLoader, self).__init__(*args, **kwargs) + self.finder = FileFinder(self.topsrcdir) + self.reader = self.mozbuild_reader(config_mode="empty") + self.variables = { + "{}_MANIFESTS".format(k): v[0] for k, v in six.iteritems(TEST_MANIFESTS) + } + self.variables.update( + {"{}_MANIFESTS".format(f.upper()): f for f in REFTEST_FLAVORS} + ) + + def _load_manifestparser_manifest(self, mpath): + mp = TestManifest( + manifests=[mpath], + strict=True, + rootdir=self.topsrcdir, + finder=self.finder, + handle_defaults=True, + ) + return (test for test in mp.tests) + + def _load_reftest_manifest(self, mpath): + import reftest + + manifest = reftest.ReftestManifest(finder=self.finder) + manifest.load(mpath) + + for test in sorted(manifest.tests, key=lambda x: x.get("path")): + test["manifest_relpath"] = test["manifest"][len(self.topsrcdir) + 1 :] + yield test + + def __call__(self): + for path, name, key, value in self.reader.find_variables_from_ast( + self.variables + ): + mpath = os.path.join(self.topsrcdir, os.path.dirname(path), value) + flavor = self.variables[name] + + if name.rsplit("_", 1)[0].lower() in REFTEST_FLAVORS: + tests = self._load_reftest_manifest(mpath) + else: + tests = self._load_manifestparser_manifest(mpath) + + for test in tests: + path = mozpath.normpath(test["path"]) + assert mozpath.basedir(path, [self.topsrcdir]) + relpath = path[len(self.topsrcdir) + 1 :] + + # Add these keys for compatibility with the build backend loader. + test["flavor"] = flavor + test["file_relpath"] = relpath + test["srcdir_relpath"] = relpath + test["dir_relpath"] = mozpath.dirname(relpath) + + yield test + + +class TestResolver(MozbuildObject): + """Helper to resolve tests from the current environment to test files.""" + + test_rewrites = { + "a11y": "_tests/testing/mochitest/a11y", + "browser-chrome": "_tests/testing/mochitest/browser", + "chrome": "_tests/testing/mochitest/chrome", + "mochitest": "_tests/testing/mochitest/tests", + "xpcshell": "_tests/xpcshell", + } + + def __init__(self, *args, **kwargs): + loader_cls = kwargs.pop("loader_cls", BuildBackendLoader) + super(TestResolver, self).__init__(*args, **kwargs) + + self.load_tests = self._spawn(loader_cls) + self._tests = [] + self._reset_state() + + # These suites aren't registered in moz.build so require special handling. + self._puppeteer_loaded = False + self._tests_loaded = False + self._wpt_loaded = False + + def _reset_state(self): + self._tests_by_path = OrderedDefaultDict(list) + self._tests_by_flavor = defaultdict(set) + self._tests_by_manifest = defaultdict(list) + self._test_dirs = set() + + @property + def tests(self): + if not self._tests_loaded: + self._reset_state() + for test in self.load_tests(): + self._tests.append(test) + self._tests_loaded = True + return self._tests + + @property + def tests_by_path(self): + if not self._tests_by_path: + for test in self.tests: + self._tests_by_path[test["file_relpath"]].append(test) + return self._tests_by_path + + @property + def tests_by_flavor(self): + if not self._tests_by_flavor: + for test in self.tests: + self._tests_by_flavor[test["flavor"]].add(test["file_relpath"]) + return self._tests_by_flavor + + @property + def tests_by_manifest(self): + if not self._tests_by_manifest: + for test in self.tests: + if test["flavor"] == "web-platform-tests": + # Use test ids instead of paths for WPT. + self._tests_by_manifest[test["manifest"]].append(test["name"]) + else: + relpath = mozpath.relpath( + test["path"], mozpath.dirname(test["manifest"]) + ) + self._tests_by_manifest[test["manifest_relpath"]].append(relpath) + return self._tests_by_manifest + + @property + def test_dirs(self): + if not self._test_dirs: + for test in self.tests: + self._test_dirs.add(test["dir_relpath"]) + return self._test_dirs + + def _resolve( + self, paths=None, flavor="", subsuite=None, under_path=None, tags=None + ): + """Given parameters, resolve them to produce an appropriate list of tests. + + Args: + paths (list): + By default, set to None. If provided as a list of paths, then + this method will attempt to load the appropriate set of tests + that live in this path. + + flavor (string): + By default, an empty string. If provided as a string, then this + method will attempt to load tests that belong to this flavor. + Additional filtering also takes the flavor into consideration. + + subsuite (string): + By default, set to None. If provided as a string, then this value + is used to perform filtering of a candidate set of tests. + """ + if tags: + tags = set(tags) + + def fltr(tests): + """Filters tests based on several criteria. + + Args: + tests (list): + List of tests that belong to the same candidate path. + + Returns: + test (dict): + If the test survived the filtering process, it is returned + as a valid test. + """ + for test in tests: + if flavor: + if flavor == "devtools" and test.get("flavor") != "browser-chrome": + continue + if flavor != "devtools" and test.get("flavor") != flavor: + continue + + if subsuite and test.get("subsuite", "undefined") != subsuite: + continue + + if tags and not (tags & set(test.get("tags", "").split())): + continue + + if under_path and not test["file_relpath"].startswith(under_path): + continue + + # Make a copy so modifications don't change the source. + yield dict(test) + + paths = paths or [] + paths = [mozpath.normpath(p) for p in paths] + if not paths: + paths = [None] + + if flavor in ("", "puppeteer", None) and ( + any(self.is_puppeteer_path(p) for p in paths) or paths == [None] + ): + self.add_puppeteer_manifest_data() + + if flavor in ("", "web-platform-tests", None) and ( + any(self.is_wpt_path(p) for p in paths) or paths == [None] + ): + self.add_wpt_manifest_data() + + candidate_paths = set() + + for path in sorted(paths): + if path is None: + candidate_paths |= set(self.tests_by_path.keys()) + continue + + if "*" in path: + candidate_paths |= { + p for p in self.tests_by_path if mozpath.match(p, path) + } + continue + + # If the path is a directory, or the path is a prefix of a directory + # containing tests, pull in all tests in that directory. + if path in self.test_dirs or any( + p.startswith(path) for p in self.tests_by_path + ): + candidate_paths |= {p for p in self.tests_by_path if p.startswith(path)} + continue + + # If the path is a manifest, add all tests defined in that manifest. + if any(path.endswith(e) for e in (".ini", ".list")): + key = "manifest" if os.path.isabs(path) else "manifest_relpath" + candidate_paths |= { + t["file_relpath"] + for t in self.tests + if mozpath.normpath(t[key]) == path + } + continue + + # If it's a test file, add just that file. + candidate_paths |= {p for p in self.tests_by_path if path in p} + + for p in sorted(candidate_paths): + tests = self.tests_by_path[p] + for test in fltr(tests): + yield test + + def is_puppeteer_path(self, path): + if path is None: + return True + return mozpath.match(path, "remote/test/puppeteer/test/**") + + def add_puppeteer_manifest_data(self): + if self._puppeteer_loaded: + return + + self._reset_state() + + test_path = os.path.join(self.topsrcdir, "remote", "test", "puppeteer", "test") + for root, dirs, paths in os.walk(test_path): + for filename in fnmatch.filter(paths, "*.spec.js"): + path = os.path.join(root, filename) + self._tests.append( + { + "path": os.path.abspath(path), + "flavor": "puppeteer", + "here": os.path.dirname(path), + "manifest": None, + "name": path, + "file_relpath": path, + "head": "", + "support-files": "", + "subsuite": "puppeteer", + "dir_relpath": os.path.dirname(path), + "srcdir_relpath": path, + } + ) + + self._puppeteer_loaded = True + + def is_wpt_path(self, path): + """Checks if path forms part of the known web-platform-test paths. + + Args: + path (str or None): + Path to check against the list of known web-platform-test paths. + + Returns: + Boolean value. True if path is part of web-platform-tests path, or + path is None. False otherwise. + """ + if path is None: + return True + if mozpath.match(path, "testing/web-platform/tests/**"): + return True + if mozpath.match(path, "testing/web-platform/mozilla/tests/**"): + return True + return False + + def get_wpt_group(self, test, depth=3): + """Given a test object set the group (aka manifest) that it belongs to. + + If a custom value for `depth` is provided, it will override the default + value of 3 path components. + + Args: + test (dict): Test object for the particular suite and subsuite. + depth (int, optional): Custom number of path elements. + + Returns: + str: The group the given test belongs to. + """ + # This takes into account that for mozilla-specific WPT tests, the path + # contains an extra '/_mozilla' prefix that must be accounted for. + if test["name"].startswith("/_mozilla"): + depth = depth + 1 + + # Webdriver tests are nested in "classic" and "bidi" folders. Increase + # the depth to avoid grouping all classic or bidi tests in one chunk. + if test["name"].startswith(("/webdriver", "/_mozilla/webdriver")): + depth = depth + 1 + + group = os.path.dirname(test["name"]) + while group.count("/") > depth: + group = os.path.dirname(group) + return group + + def add_wpt_manifest_data(self): + """Adds manifest data for web-platform-tests into the list of available tests. + + Upon invocation, this method will download from firefox-ci the most recent + version of the web-platform-tests manifests. + + Once manifest is downloaded, this method will add details about each test + into the list of available tests. + """ + if self._wpt_loaded: + return + + self._reset_state() + + wpt_path = os.path.join(self.topsrcdir, "testing", "web-platform") + sys.path = [wpt_path] + sys.path + + import logging + + import manifestupdate + + logger = logging.getLogger("manifestupdate") + logger.disabled = True + + manifests = manifestupdate.run( + self.topsrcdir, + self.topobjdir, + rebuild=False, + download=True, + config_path=None, + rewrite_config=True, + update=True, + logger=logger, + ) + if not manifests: + print("Loading wpt manifest failed") + return + + for manifest, data in six.iteritems(manifests): + tests_root = data[ + "tests_path" + ] # full path on disk until web-platform tests directory + + for test_type, path, tests in manifest: + full_path = mozpath.join(tests_root, path) + src_path = mozpath.relpath(full_path, self.topsrcdir) + if test_type not in WPT_TYPES: + continue + + full_path = mozpath.join(tests_root, path) # absolute path on disk + src_path = mozpath.relpath(full_path, self.topsrcdir) + + for test in tests: + testobj = { + "head": "", + "support-files": "", + "path": full_path, + "flavor": "web-platform-tests", + "subsuite": test_type, + "here": mozpath.dirname(path), + "name": test.id, + "file_relpath": src_path, + "srcdir_relpath": src_path, + "dir_relpath": mozpath.dirname(src_path), + } + group = self.get_wpt_group(testobj) + testobj["manifest"] = group + + test_root = "tests" + if group.startswith("/_mozilla"): + test_root = os.path.join("mozilla", "tests") + group = group[len("/_mozilla") :] + + group = group.lstrip("/") + testobj["manifest_relpath"] = os.path.join( + wpt_path, test_root, group + ) + self._tests.append(testobj) + + self._wpt_loaded = True + + def resolve_tests(self, cwd=None, **kwargs): + """Resolve tests from an identifier. + + This is a generator of dicts describing each test. All arguments are + optional. + + Paths in returned tests are automatically translated to the paths in + the _tests directory under the object directory. + + Args: + cwd (str): + If specified, we will limit our results to tests under this + directory. The directory should be defined as an absolute path + under topsrcdir or topobjdir. + + paths (list): + An iterable of values to use to identify tests to run. If an + entry is a known test file, tests associated with that file are + returned (there may be multiple configurations for a single + file). If an entry is a directory, or a prefix of a directory + containing tests, all tests in that directory are returned. If + the string appears in a known test file, that test file is + considered. If the path contains a wildcard pattern, tests + matching that pattern are returned. + + under_path (str): + If specified, will be used to filter out tests that aren't in + the specified path prefix relative to topsrcdir or the test's + installed dir. + + flavor (str): + If specified, will be used to filter returned tests to only be + the flavor specified. A flavor is something like ``xpcshell``. + + subsuite (str): + If specified will be used to filter returned tests to only be + in the subsuite specified. To filter only tests that *don't* + have any subsuite, pass the string 'undefined'. + + tags (list): + If specified, will be used to filter out tests that don't contain + a matching tag. + """ + if cwd: + norm_cwd = mozpath.normpath(cwd) + norm_srcdir = mozpath.normpath(self.topsrcdir) + norm_objdir = mozpath.normpath(self.topobjdir) + + reldir = None + + if norm_cwd.startswith(norm_objdir): + reldir = norm_cwd[len(norm_objdir) + 1 :] + elif norm_cwd.startswith(norm_srcdir): + reldir = norm_cwd[len(norm_srcdir) + 1 :] + + kwargs["under_path"] = reldir + + rewrite_base = None + for test in self._resolve(**kwargs): + rewrite_base = self.test_rewrites.get(test["flavor"], None) + + if rewrite_base: + rewrite_base = os.path.join( + self.topobjdir, os.path.normpath(rewrite_base) + ) + yield rewrite_test_base(test, rewrite_base) + else: + yield test + + def resolve_metadata(self, what): + """Resolve tests based on the given metadata. If not specified, metadata + from outgoing files will be used instead. + """ + # Parse arguments and assemble a test "plan." + run_suites = set() + run_tests = [] + + for entry in what: + # If the path matches the name or alias of an entire suite, run + # the entire suite. + if entry in TEST_SUITES: + run_suites.add(entry) + continue + suitefound = False + for suite, v in six.iteritems(TEST_SUITES): + if entry.lower() in v.get("aliases", []): + run_suites.add(suite) + suitefound = True + if suitefound: + continue + + # Now look for file/directory matches in the TestResolver. + relpath = self._wrap_path_argument(entry).relpath() + tests = list(self.resolve_tests(paths=[relpath])) + run_tests.extend(tests) + + if not tests: + print("UNKNOWN TEST: %s" % entry, file=sys.stderr) + + return run_suites, run_tests |