path: root/testing/mozbase/moztest/moztest/resolve.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import fnmatch
import os
import pickle
import sys
from abc import ABCMeta, abstractmethod
from collections import defaultdict

import mozpack.path as mozpath
import six
from manifestparser import TestManifest, combine_fields
from mozbuild.base import MozbuildObject
from mozbuild.testing import REFTEST_FLAVORS, TEST_MANIFESTS
from mozbuild.util import OrderedDefaultDict
from mozpack.files import FileFinder

here = os.path.abspath(os.path.dirname(__file__))

MOCHITEST_CHUNK_BY_DIR = 4
MOCHITEST_TOTAL_CHUNKS = 5


def WebglSuite(name):
    return {
        "aliases": (name,),
        "build_flavor": "mochitest",
        "mach_command": "mochitest",
        "kwargs": {"flavor": "plain", "subsuite": name, "test_paths": None},
        "task_regex": [
            "mochitest-" + name + "($|.*(-1|[^0-9])$)",
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    }


TEST_SUITES = {
    "cppunittest": {
        "aliases": ("cpp",),
        "mach_command": "cppunittest",
        "kwargs": {"test_files": None},
    },
    "crashtest": {
        "aliases": ("c", "rc"),
        "build_flavor": "crashtest",
        "mach_command": "crashtest",
        "kwargs": {"test_file": None},
        "task_regex": ["crashtest($|.*(-1|[^0-9])$)", "test-verify($|.*(-1|[^0-9])$)"],
    },
    "crashtest-qr": {
        "aliases": ("c", "rc"),
        "build_flavor": "crashtest",
        "mach_command": "crashtest",
        "kwargs": {"test_file": None},
        "task_regex": [
            "crashtest-qr($|.*(-1|[^0-9])$)",
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    },
    "firefox-ui-functional": {
        "aliases": ("fxfn",),
        "mach_command": "firefox-ui-functional",
        "kwargs": {},
    },
    "firefox-ui-update": {
        "aliases": ("fxup",),
        "mach_command": "firefox-ui-update",
        "kwargs": {},
    },
    "marionette": {
        "aliases": ("mn",),
        "mach_command": "marionette-test",
        "kwargs": {"tests": None},
        "task_regex": ["marionette($|.*(-1|[^0-9])$)"],
    },
    "mochitest-a11y": {
        "aliases": ("a11y", "ally"),
        "build_flavor": "a11y",
        "mach_command": "mochitest",
        "kwargs": {
            "flavor": "a11y",
            "test_paths": None,
            "e10s": False,
            "enable_fission": False,
        },
        "task_regex": [
            "mochitest-a11y($|.*(-1|[^0-9])$)",
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    },
    "mochitest-browser-chrome": {
        "aliases": ("bc", "browser"),
        "build_flavor": "browser-chrome",
        "mach_command": "mochitest",
        "kwargs": {"flavor": "browser-chrome", "test_paths": None},
        "task_regex": [
            "mochitest-browser-chrome($|.*(-1|[^0-9])$)",
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    },
    "mochitest-browser-screenshots": {
        "aliases": ("ss", "screenshots-chrome"),
        "build_flavor": "browser-chrome",
        "mach_command": "mochitest",
        "kwargs": {
            "flavor": "browser-chrome",
            "subsuite": "screenshots",
            "test_paths": None,
        },
        "task_regex": ["mochitest-browser-screenshots($|.*(-1|[^0-9])$)"],
    },
    "mochitest-chrome": {
        "aliases": ("mc",),
        "build_flavor": "chrome",
        "mach_command": "mochitest",
        "kwargs": {
            "flavor": "chrome",
            "test_paths": None,
            "e10s": False,
            "enable_fission": False,
        },
        "task_regex": [
            "mochitest-chrome($|.*(-1|[^0-9])$)",
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    },
    "mochitest-chrome-gpu": {
        "aliases": ("gpu",),
        "build_flavor": "chrome",
        "mach_command": "mochitest",
        "kwargs": {
            "flavor": "chrome",
            "subsuite": "gpu",
            "test_paths": None,
            "e10s": False,
            "enable_fission": False,
        },
        "task_regex": [
            "mochitest-gpu($|.*(-1|[^0-9])$)",
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    },
    "mochitest-devtools-chrome": {
        "aliases": ("dt", "devtools"),
        "build_flavor": "browser-chrome",
        "mach_command": "mochitest",
        "kwargs": {
            "flavor": "browser-chrome",
            "subsuite": "devtools",
            "test_paths": None,
        },
        "task_regex": [
            "mochitest-devtools-chrome($|.*(-1|[^0-9])$)",
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    },
    "mochitest-browser-a11y": {
        "aliases": ("ba", "browser-a11y"),
        "build_flavor": "browser-chrome",
        "mach_command": "mochitest",
        "kwargs": {
            "flavor": "browser-chrome",
            "subsuite": "a11y",
            "test_paths": None,
        },
        "task_regex": [
            "mochitest-browser-a11y($|.*(-1|[^0-9])$)",
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    },
    "mochitest-media": {
        "aliases": ("mpm", "plain-media"),
        "build_flavor": "mochitest",
        "mach_command": "mochitest",
        "kwargs": {"flavor": "plain", "subsuite": "media", "test_paths": None},
        "task_regex": [
            "mochitest-media($|.*(-1|[^0-9])$)",
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    },
    "mochitest-browser-media": {
        "aliases": ("bmda", "browser-mda"),
        "build_flavor": "browser-chrome",
        "mach_command": "mochitest",
        "kwargs": {
            "flavor": "browser-chrome",
            "subsuite": "media-bc",
            "test_paths": None,
        },
        "task_regex": [
            "mochitest-browser-media($|.*(-1|[^0-9])$)",
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    },
    "mochitest-plain": {
        "aliases": (
            "mp",
            "plain",
        ),
        "build_flavor": "mochitest",
        "mach_command": "mochitest",
        "kwargs": {"flavor": "plain", "test_paths": None},
        "task_regex": [
            "mochitest-plain($|.*(-1|[^0-9])$)",  # noqa
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    },
    "mochitest-plain-gpu": {
        "aliases": ("gpu",),
        "build_flavor": "mochitest",
        "mach_command": "mochitest",
        "kwargs": {"flavor": "plain", "subsuite": "gpu", "test_paths": None},
        "task_regex": [
            "mochitest-gpu($|.*(-1|[^0-9])$)",
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    },
    "mochitest-remote": {
        "aliases": ("remote",),
        "build_flavor": "browser-chrome",
        "mach_command": "mochitest",
        "kwargs": {
            "flavor": "browser-chrome",
            "subsuite": "remote",
            "test_paths": None,
        },
        "task_regex": [
            "mochitest-remote($|.*(-1|[^0-9])$)",
            "test-verify($|.*(-1|[^0-9])$)",
        ],
    },
    "mochitest-webgl1-core": WebglSuite("webgl1-core"),
    "mochitest-webgl1-ext": WebglSuite("webgl1-ext"),
    "mochitest-webgl2-core": WebglSuite("webgl2-core"),
    "mochitest-webgl2-ext": WebglSuite("webgl2-ext"),
    "mochitest-webgl2-deqp": WebglSuite("webgl2-deqp"),
    "mochitest-webgpu": WebglSuite("webgpu"),
    "puppeteer": {
        "aliases": ("remote/test/puppeteer",),
        "mach_command": "puppeteer-test",
        "kwargs": {"headless": False},
    },
    "python": {
        "build_flavor": "python",
        "mach_command": "python-test",
        "kwargs": {"tests": None},
    },
    "telemetry-tests-client": {
        "aliases": ("ttc",),
        "build_flavor": "telemetry-tests-client",
        "mach_command": "telemetry-tests-client",
        "kwargs": {},
        "task_regex": ["telemetry-tests-client($|.*(-1|[^0-9])$)"],
    },
    "reftest": {
        "aliases": ("rr",),
        "build_flavor": "reftest",
        "mach_command": "reftest",
        "kwargs": {"tests": None},
        "task_regex": [
            "(opt|debug)(-geckoview)?-reftest($|.*(-1|[^0-9])$)",
            "test-verify-gpu($|.*(-1|[^0-9])$)",
        ],
    },
    "reftest-qr": {
        "aliases": ("rr",),
        "build_flavor": "reftest",
        "mach_command": "reftest",
        "kwargs": {"tests": None},
        "task_regex": [
            "(opt|debug)(-geckoview)?-reftest-qr($|.*(-1|[^0-9])$)",
            "test-verify-gpu($|.*(-1|[^0-9])$)",
        ],
    },
    "robocop": {
        "mach_command": "robocop",
        "kwargs": {"test_paths": None},
        "task_regex": ["robocop($|.*(-1|[^0-9])$)"],
    },
    "web-platform-tests": {
        "aliases": ("wpt",),
        "mach_command": "web-platform-tests",
        "build_flavor": "web-platform-tests",
        "kwargs": {"subsuite": "testharness"},
        "task_regex": [
            "web-platform-tests(?!-crashtest|-reftest|-wdspec|-print)"
            "($|.*(-1|[^0-9])$)",
            "test-verify-wpt",
        ],
    },
    "web-platform-tests-crashtest": {
        "aliases": ("wpt",),
        "mach_command": "web-platform-tests",
        "build_flavor": "web-platform-tests",
        "kwargs": {"subsuite": "crashtest"},
        "task_regex": [
            "web-platform-tests-crashtest($|.*(-1|[^0-9])$)",
            "test-verify-wpt",
        ],
    },
    "web-platform-tests-print-reftest": {
        "aliases": ("wpt",),
        "mach_command": "web-platform-tests",
        "kwargs": {"subsuite": "print-reftest"},
        "task_regex": [
            "web-platform-tests-print-reftest($|.*(-1|[^0-9])$)",
            "test-verify-wpt",
        ],
    },
    "web-platform-tests-reftest": {
        "aliases": ("wpt",),
        "mach_command": "web-platform-tests",
        "build_flavor": "web-platform-tests",
        "kwargs": {"subsuite": "reftest"},
        "task_regex": [
            "web-platform-tests-reftest($|.*(-1|[^0-9])$)",
            "test-verify-wpt",
        ],
    },
    "web-platform-tests-wdspec": {
        "aliases": ("wpt",),
        "mach_command": "web-platform-tests",
        "build_flavor": "web-platform-tests",
        "kwargs": {"subsuite": "wdspec"},
        "task_regex": [
            "web-platform-tests-wdspec($|.*(-1|[^0-9])$)",
            "test-verify-wpt",
        ],
    },
    "valgrind": {
        "aliases": ("v",),
        "mach_command": "valgrind-test",
        "kwargs": {},
    },
    "xpcshell": {
        "aliases": ("x",),
        "build_flavor": "xpcshell",
        "mach_command": "xpcshell-test",
        "kwargs": {"test_file": "all"},
        "task_regex": ["xpcshell($|.*(-1|[^0-9])$)", "test-verify($|.*(-1|[^0-9])$)"],
    },
    "xpcshell-msix": {
        "aliases": ("x",),
        "build_flavor": "xpcshell",
        "mach_command": "xpcshell-test",
        "kwargs": {"test_file": "all"},
        "task_regex": ["xpcshell($|.*(-1|[^0-9])$)", "test-verify($|.*(-1|[^0-9])$)"],
    },
}
"""Definitions of all test suites and the metadata needed to run and process
them. Each test suite definition can contain the following keys.

Arguments:
    aliases (tuple): A tuple containing shorthands used to refer to this suite.
    build_flavor (str): The flavor assigned to this suite by the build system
                        in `mozbuild.testing.TEST_MANIFESTS` (or similar).
    mach_command (str): Name of the mach command used to run this suite.
    kwargs (dict): Arguments needed to pass into the mach command.
    task_regex (list): A list of regexes used to filter task labels that run
                       this suite.
"""

for i in range(1, MOCHITEST_TOTAL_CHUNKS + 1):
    TEST_SUITES["mochitest-%d" % i] = {
        "aliases": ("m%d" % i,),
        "mach_command": "mochitest",
        "kwargs": {
            "flavor": "mochitest",
            "subsuite": "default",
            "chunk_by_dir": MOCHITEST_CHUNK_BY_DIR,
            "total_chunks": MOCHITEST_TOTAL_CHUNKS,
            "this_chunk": i,
            "test_paths": None,
        },
    }


WPT_TYPES = set()
for suite, data in TEST_SUITES.items():
    if suite.startswith("web-platform-tests"):
        WPT_TYPES.add(data["kwargs"]["subsuite"])


_test_flavors = {
    "a11y": "mochitest-a11y",
    "browser-chrome": "mochitest-browser-chrome",
    "chrome": "mochitest-chrome",
    "crashtest": "crashtest",
    "firefox-ui-functional": "firefox-ui-functional",
    "firefox-ui-update": "firefox-ui-update",
    "marionette": "marionette",
    "mochitest": "mochitest-plain",
    "puppeteer": "puppeteer",
    "python": "python",
    "reftest": "reftest",
    "telemetry-tests-client": "telemetry-tests-client",
    "web-platform-tests": "web-platform-tests",
    "xpcshell": "xpcshell",
}

_test_subsuites = {
    ("browser-chrome", "a11y"): "mochitest-browser-a11y",
    ("browser-chrome", "devtools"): "mochitest-devtools-chrome",
    ("browser-chrome", "media"): "mochitest-browser-media",
    ("browser-chrome", "remote"): "mochitest-remote",
    ("browser-chrome", "screenshots"): "mochitest-browser-screenshots",
    ("chrome", "gpu"): "mochitest-chrome-gpu",
    ("mochitest", "gpu"): "mochitest-plain-gpu",
    ("mochitest", "media"): "mochitest-media",
    ("mochitest", "robocop"): "robocop",
    ("mochitest", "webgl1-core"): "mochitest-webgl1-core",
    ("mochitest", "webgl1-ext"): "mochitest-webgl1-ext",
    ("mochitest", "webgl2-core"): "mochitest-webgl2-core",
    ("mochitest", "webgl2-ext"): "mochitest-webgl2-ext",
    ("mochitest", "webgl2-deqp"): "mochitest-webgl2-deqp",
    ("mochitest", "webgpu"): "mochitest-webgpu",
    ("web-platform-tests", "testharness"): "web-platform-tests",
    ("web-platform-tests", "crashtest"): "web-platform-tests-crashtest",
    ("web-platform-tests", "print-reftest"): "web-platform-tests-print-reftest",
    ("web-platform-tests", "reftest"): "web-platform-tests-reftest",
    ("web-platform-tests", "wdspec"): "web-platform-tests-wdspec",
}


def get_suite_definition(flavor, subsuite=None, strict=False):
    """Return a suite definition given a flavor and optional subsuite.

    If strict is True, a subsuite must have its own entry in TEST_SUITES.
    Otherwise, the entry for 'flavor' will be returned with the 'subsuite'
    keyword arg set.

    With or without strict mode, an empty dict will be returned if no
    matching suite definition was found.
    """
    if not subsuite:
        suite_name = _test_flavors.get(flavor)
        return suite_name, TEST_SUITES.get(suite_name, {}).copy()

    suite_name = _test_subsuites.get((flavor, subsuite))
    if suite_name or strict:
        return suite_name, TEST_SUITES.get(suite_name, {}).copy()

    suite_name = _test_flavors.get(flavor)
    if suite_name not in TEST_SUITES:
        return suite_name, {}

    suite = TEST_SUITES[suite_name].copy()
    suite.setdefault("kwargs", {})
    suite["kwargs"]["subsuite"] = subsuite
    return suite_name, suite


def rewrite_test_base(test, new_base):
    """Rewrite paths in a test to be under a new base path.

    This is useful for running tests from a separate location from where they
    were defined.
    """
    test["here"] = mozpath.join(new_base, test["dir_relpath"])
    test["path"] = mozpath.join(new_base, test["file_relpath"])
    return test


@six.add_metaclass(ABCMeta)
class TestLoader(MozbuildObject):
    @abstractmethod
    def __call__(self):
        """Generate test metadata."""


class BuildBackendLoader(TestLoader):
    def __call__(self):
        """Loads the test metadata generated by the TestManifest build backend.

        The data is stored in two files:

            - <objdir>/all-tests.pkl
            - <objdir>/test-defaults.pkl

        The 'all-tests.pkl' file is a mapping of source path to test objects. The
        'test-defaults.pkl' file maps manifests to their DEFAULT configuration.
        These manifest defaults will be merged into the test configuration of the
        contained tests.
        """
        # If installing tests is going to result in re-generating the build
        # backend, we need to do this here, so that the updated contents of
        # all-tests.pkl make it to the set of tests to run.
        if self.backend_out_of_date(
            mozpath.join(self.topobjdir, "backend.TestManifestBackend")
        ):
            print("Test configuration changed. Regenerating backend.")
            from mozbuild.gen_test_backend import gen_test_backend

            gen_test_backend()

        all_tests = os.path.join(self.topobjdir, "all-tests.pkl")
        test_defaults = os.path.join(self.topobjdir, "test-defaults.pkl")

        with open(all_tests, "rb") as fh:
            test_data = pickle.load(fh)

        with open(test_defaults, "rb") as fh:
            defaults = pickle.load(fh)

        # The keys in defaults use platform-specific path separators.
        # self.topsrcdir was normalized to use /, revert back to \ if needed.
        topsrcdir = os.path.normpath(self.topsrcdir)

        for path, tests in six.iteritems(test_data):
            for metadata in tests:
                defaults_manifests = [metadata["manifest"]]

                ancestor_manifest = metadata.get("ancestor_manifest")
                if ancestor_manifest:
                    # The (ancestor manifest, included manifest) tuple
                    # contains the defaults of the included manifest, so
                    # use it instead of [metadata['manifest']].
                    ancestor_manifest = os.path.join(topsrcdir, ancestor_manifest)
                    defaults_manifests[0] = (ancestor_manifest, metadata["manifest"])
                    defaults_manifests.append(ancestor_manifest)

                for manifest in defaults_manifests:
                    manifest_defaults = defaults.get(manifest)
                    if manifest_defaults:
                        metadata = combine_fields(manifest_defaults, metadata)

                yield metadata


class TestManifestLoader(TestLoader):
    def __init__(self, *args, **kwargs):
        super(TestManifestLoader, self).__init__(*args, **kwargs)
        self.finder = FileFinder(self.topsrcdir)
        self.reader = self.mozbuild_reader(config_mode="empty")
        self.variables = {
            "{}_MANIFESTS".format(k): v[0] for k, v in six.iteritems(TEST_MANIFESTS)
        }
        self.variables.update(
            {"{}_MANIFESTS".format(f.upper()): f for f in REFTEST_FLAVORS}
        )

    def _load_manifestparser_manifest(self, mpath):
        mp = TestManifest(
            manifests=[mpath],
            strict=True,
            rootdir=self.topsrcdir,
            finder=self.finder,
            handle_defaults=True,
        )
        return (test for test in mp.tests)

    def _load_reftest_manifest(self, mpath):
        import reftest

        manifest = reftest.ReftestManifest(finder=self.finder)
        manifest.load(mpath)

        for test in sorted(manifest.tests, key=lambda x: x.get("path")):
            test["manifest_relpath"] = test["manifest"][len(self.topsrcdir) + 1 :]
            yield test

    def __call__(self):
        for path, name, key, value in self.reader.find_variables_from_ast(
            self.variables
        ):
            mpath = os.path.join(self.topsrcdir, os.path.dirname(path), value)
            flavor = self.variables[name]

            if name.rsplit("_", 1)[0].lower() in REFTEST_FLAVORS:
                tests = self._load_reftest_manifest(mpath)
            else:
                tests = self._load_manifestparser_manifest(mpath)

            for test in tests:
                path = mozpath.normpath(test["path"])
                assert mozpath.basedir(path, [self.topsrcdir])
                relpath = path[len(self.topsrcdir) + 1 :]

                # Add these keys for compatibility with the build backend loader.
                test["flavor"] = flavor
                test["file_relpath"] = relpath
                test["srcdir_relpath"] = relpath
                test["dir_relpath"] = mozpath.dirname(relpath)

                yield test


class TestResolver(MozbuildObject):
    """Helper to resolve tests from the current environment to test files."""

    test_rewrites = {
        "a11y": "_tests/testing/mochitest/a11y",
        "browser-chrome": "_tests/testing/mochitest/browser",
        "chrome": "_tests/testing/mochitest/chrome",
        "mochitest": "_tests/testing/mochitest/tests",
        "xpcshell": "_tests/xpcshell",
    }

    def __init__(self, *args, **kwargs):
        loader_cls = kwargs.pop("loader_cls", BuildBackendLoader)
        super(TestResolver, self).__init__(*args, **kwargs)

        self.load_tests = self._spawn(loader_cls)
        self._tests = []
        self._reset_state()

        # These suites aren't registered in moz.build so require special handling.
        self._puppeteer_loaded = False
        self._tests_loaded = False
        self._wpt_loaded = False

    def _reset_state(self):
        self._tests_by_path = OrderedDefaultDict(list)
        self._tests_by_flavor = defaultdict(set)
        self._tests_by_manifest = defaultdict(list)
        self._test_dirs = set()

    @property
    def tests(self):
        if not self._tests_loaded:
            self._reset_state()
            for test in self.load_tests():
                self._tests.append(test)
            self._tests_loaded = True
        return self._tests

    @property
    def tests_by_path(self):
        if not self._tests_by_path:
            for test in self.tests:
                self._tests_by_path[test["file_relpath"]].append(test)
        return self._tests_by_path

    @property
    def tests_by_flavor(self):
        if not self._tests_by_flavor:
            for test in self.tests:
                self._tests_by_flavor[test["flavor"]].add(test["file_relpath"])
        return self._tests_by_flavor

    @property
    def tests_by_manifest(self):
        if not self._tests_by_manifest:
            for test in self.tests:
                if test["flavor"] == "web-platform-tests":
                    # Use test ids instead of paths for WPT.
                    self._tests_by_manifest[test["manifest"]].append(test["name"])
                else:
                    relpath = mozpath.relpath(
                        test["path"], mozpath.dirname(test["manifest"])
                    )
                    self._tests_by_manifest[test["manifest_relpath"]].append(relpath)
        return self._tests_by_manifest

    @property
    def test_dirs(self):
        if not self._test_dirs:
            for test in self.tests:
                self._test_dirs.add(test["dir_relpath"])
        return self._test_dirs

    def _resolve(
        self, paths=None, flavor="", subsuite=None, under_path=None, tags=None
    ):
        """Given parameters, resolve them to produce an appropriate list of tests.

        Args:
            paths (list):
                By default, set to None. If provided as a list of paths, then
                this method will attempt to load the appropriate set of tests
                that live in this path.

            flavor (string):
                By default, an empty string. If provided as a string, then this
                method will attempt to load tests that belong to this flavor.
                Additional filtering also takes the flavor into consideration.

            subsuite (string):
                By default, set to None. If provided as a string, then this value
                is used to perform filtering of a candidate set of tests.
        """
        if tags:
            tags = set(tags)

        def fltr(tests):
            """Filters tests based on several criteria.

            Args:
                tests (list):
                    List of tests that belong to the same candidate path.

            Returns:
                test (dict):
                    If the test survived the filtering process, it is returned
                    as a valid test.
            """
            for test in tests:
                if flavor:
                    if flavor == "devtools" and test.get("flavor") != "browser-chrome":
                        continue
                    if flavor != "devtools" and test.get("flavor") != flavor:
                        continue

                if subsuite and test.get("subsuite", "undefined") != subsuite:
                    continue

                if tags and not (tags & set(test.get("tags", "").split())):
                    continue

                if under_path and not test["file_relpath"].startswith(under_path):
                    continue

                # Make a copy so modifications don't change the source.
                yield dict(test)

        paths = paths or []
        paths = [mozpath.normpath(p) for p in paths]
        if not paths:
            paths = [None]

        if flavor in ("", "puppeteer", None) and (
            any(self.is_puppeteer_path(p) for p in paths) or paths == [None]
        ):
            self.add_puppeteer_manifest_data()

        if flavor in ("", "web-platform-tests", None) and (
            any(self.is_wpt_path(p) for p in paths) or paths == [None]
        ):
            self.add_wpt_manifest_data()

        candidate_paths = set()

        for path in sorted(paths):
            if path is None:
                candidate_paths |= set(self.tests_by_path.keys())
                continue

            if "*" in path:
                candidate_paths |= {
                    p for p in self.tests_by_path if mozpath.match(p, path)
                }
                continue

            # If the path is a directory, or the path is a prefix of a directory
            # containing tests, pull in all tests in that directory.
            if path in self.test_dirs or any(
                p.startswith(path) for p in self.tests_by_path
            ):
                candidate_paths |= {p for p in self.tests_by_path if p.startswith(path)}
                continue

            # If the path is a manifest, add all tests defined in that manifest.
            if any(path.endswith(e) for e in (".toml", ".ini", ".list")):
                key = "manifest" if os.path.isabs(path) else "manifest_relpath"
                candidate_paths |= {
                    t["file_relpath"]
                    for t in self.tests
                    if mozpath.normpath(t[key]) == path
                }
                continue

            # If it's a test file, add just that file.
            candidate_paths |= {p for p in self.tests_by_path if path in p}

        for p in sorted(candidate_paths):
            tests = self.tests_by_path[p]
            for test in fltr(tests):
                yield test

    def is_puppeteer_path(self, path):
        if path is None:
            return True
        return mozpath.match(path, "remote/test/puppeteer/test/**")

    def add_puppeteer_manifest_data(self):
        if self._puppeteer_loaded:
            return

        self._reset_state()

        test_path = os.path.join(self.topsrcdir, "remote", "test", "puppeteer", "test")
        for root, dirs, paths in os.walk(test_path):
            for filename in fnmatch.filter(paths, "*.spec.js"):
                path = os.path.join(root, filename)
                self._tests.append(
                    {
                        "path": os.path.abspath(path),
                        "flavor": "puppeteer",
                        "here": os.path.dirname(path),
                        "manifest": None,
                        "name": path,
                        "file_relpath": path,
                        "head": "",
                        "support-files": "",
                        "subsuite": "puppeteer",
                        "dir_relpath": os.path.dirname(path),
                        "srcdir_relpath": path,
                    }
                )

        self._puppeteer_loaded = True

    def is_wpt_path(self, path):
        """Checks if path forms part of the known web-platform-test paths.

        Args:
            path (str or None):
                Path to check against the list of known web-platform-test paths.

        Returns:
            Boolean value. True if path is part of web-platform-tests path, or
            path is None. False otherwise.
        """
        if path is None:
            return True
        if mozpath.match(path, "testing/web-platform/tests/**"):
            return True
        if mozpath.match(path, "testing/web-platform/mozilla/tests/**"):
            return True
        return False

    def get_wpt_group(self, test, depth=3):
        """Given a test object set the group (aka manifest) that it belongs to.

        If a custom value for `depth` is provided, it will override the default
        value of 3 path components.

        Args:
            test (dict): Test object for the particular suite and subsuite.
            depth (int, optional): Custom number of path elements.

        Returns:
            str: The group the given test belongs to.
        """
        # This takes into account that for mozilla-specific WPT tests, the path
        # contains an extra '/_mozilla' prefix that must be accounted for.
        if test["name"].startswith("/_mozilla"):
            depth = depth + 1

        # Webdriver tests are nested in "classic" and "bidi" folders. Increase
        # the depth to avoid grouping all classic or bidi tests in one chunk.
        if test["name"].startswith(("/webdriver", "/_mozilla/webdriver")):
            depth = depth + 1

        # Webdriver BiDi tests are nested even further as tests are grouped by
        # module but also by command / event name.
        if test["name"].startswith(
            ("/webdriver/tests/bidi", "/_mozilla/webdriver/bidi")
        ):
            depth = depth + 1

        if test["name"].startswith("/_mozilla/webgpu"):
            depth = 9001

        group = os.path.dirname(test["name"])
        while group.count("/") > depth:
            group = os.path.dirname(group)
        return group

    def add_wpt_manifest_data(self):
        """Adds manifest data for web-platform-tests into the list of available tests.

        Upon invocation, this method will download from firefox-ci the most recent
        version of the web-platform-tests manifests.

        Once manifest is downloaded, this method will add details about each test
        into the list of available tests.
        """
        if self._wpt_loaded:
            return

        self._reset_state()

        wpt_path = os.path.join(self.topsrcdir, "testing", "web-platform")
        sys.path = [wpt_path] + sys.path

        import logging

        import manifestupdate

        logger = logging.getLogger("manifestupdate")
        logger.disabled = True

        manifests = manifestupdate.run(
            self.topsrcdir,
            self.topobjdir,
            rebuild=False,
            download=True,
            config_path=None,
            rewrite_config=True,
            update=True,
            logger=logger,
        )
        if not manifests:
            print("Loading wpt manifest failed")
            return

        for manifest, data in six.iteritems(manifests):
            tests_root = data[
                "tests_path"
            ]  # full path on disk until web-platform tests directory

            for test_type, path, tests in manifest:
                full_path = mozpath.join(tests_root, path)
                src_path = mozpath.relpath(full_path, self.topsrcdir)
                if test_type not in WPT_TYPES:
                    continue

                full_path = mozpath.join(tests_root, path)  # absolute path on disk
                src_path = mozpath.relpath(full_path, self.topsrcdir)

                for test in tests:
                    testobj = {
                        "head": "",
                        "support-files": "",
                        "path": full_path,
                        "flavor": "web-platform-tests",
                        "subsuite": test_type,
                        "here": mozpath.dirname(path),
                        "name": test.id,
                        "file_relpath": src_path,
                        "srcdir_relpath": src_path,
                        "dir_relpath": mozpath.dirname(src_path),
                    }
                    group = self.get_wpt_group(testobj)
                    testobj["manifest"] = group

                    test_root = "tests"
                    if group.startswith("/_mozilla"):
                        test_root = os.path.join("mozilla", "tests")
                        group = group[len("/_mozilla") :]

                    group = group.lstrip("/")
                    testobj["manifest_relpath"] = os.path.join(
                        wpt_path, test_root, group
                    )
                    self._tests.append(testobj)

        self._wpt_loaded = True

    def resolve_tests(self, cwd=None, **kwargs):
        """Resolve tests from an identifier.

        This is a generator of dicts describing each test. All arguments are
        optional.

        Paths in returned tests are automatically translated to the paths in
        the _tests directory under the object directory.

        Args:
            cwd (str):
                If specified, we will limit our results to tests under this
                directory. The directory should be defined as an absolute path
                under topsrcdir or topobjdir.

            paths (list):
                An iterable of values to use to identify tests to run. If an
                entry is a known test file, tests associated with that file are
                returned (there may be multiple configurations for a single
                file). If an entry is a directory, or a prefix of a directory
                containing tests, all tests in that directory are returned. If
                the string appears in a known test file, that test file is
                considered. If the path contains a wildcard pattern, tests
                matching that pattern are returned.

            under_path (str):
                If specified, will be used to filter out tests that aren't in
                the specified path prefix relative to topsrcdir or the test's
                installed dir.

            flavor (str):
                If specified, will be used to filter returned tests to only be
                the flavor specified. A flavor is something like ``xpcshell``.

            subsuite (str):
                If specified will be used to filter returned tests to only be
                in the subsuite specified. To filter only tests that *don't*
                have any subsuite, pass the string 'undefined'.

            tags (list):
                If specified, will be used to filter out tests that don't contain
                a matching tag.
        """
        if cwd:
            norm_cwd = mozpath.normpath(cwd)
            norm_srcdir = mozpath.normpath(self.topsrcdir)
            norm_objdir = mozpath.normpath(self.topobjdir)

            reldir = None

            if norm_cwd.startswith(norm_objdir):
                reldir = norm_cwd[len(norm_objdir) + 1 :]
            elif norm_cwd.startswith(norm_srcdir):
                reldir = norm_cwd[len(norm_srcdir) + 1 :]

            kwargs["under_path"] = reldir

        rewrite_base = None
        for test in self._resolve(**kwargs):
            rewrite_base = self.test_rewrites.get(test["flavor"], None)

            if rewrite_base:
                rewrite_base = os.path.join(
                    self.topobjdir, os.path.normpath(rewrite_base)
                )
                yield rewrite_test_base(test, rewrite_base)
            else:
                yield test

    def resolve_metadata(self, what):
        """Resolve tests based on the given metadata. If not specified, metadata
        from outgoing files will be used instead.
        """
        # Parse arguments and assemble a test "plan."
        run_suites = set()
        run_tests = []

        for entry in what:
            # If the path matches the name or alias of an entire suite, run
            # the entire suite.
            if entry in TEST_SUITES:
                run_suites.add(entry)
                continue
            suitefound = False
            for suite, v in six.iteritems(TEST_SUITES):
                if entry.lower() in v.get("aliases", []):
                    run_suites.add(suite)
                    suitefound = True
            if suitefound:
                continue

            # Now look for file/directory matches in the TestResolver.
            relpath = self._wrap_path_argument(entry).relpath()
            tests = list(self.resolve_tests(paths=[relpath]))
            run_tests.extend(tests)

            if not tests:
                print("UNKNOWN TEST: %s" % entry, file=sys.stderr)

        return run_suites, run_tests