summaryrefslogtreecommitdiffstats
path: root/testing/mozbase/moztest/tests/test_resolve.py
diff options
context:
space:
mode:
Diffstat (limited to 'testing/mozbase/moztest/tests/test_resolve.py')
-rw-r--r--testing/mozbase/moztest/tests/test_resolve.py577
1 files changed, 577 insertions, 0 deletions
diff --git a/testing/mozbase/moztest/tests/test_resolve.py b/testing/mozbase/moztest/tests/test_resolve.py
new file mode 100644
index 0000000000..acf7cbc6fa
--- /dev/null
+++ b/testing/mozbase/moztest/tests/test_resolve.py
@@ -0,0 +1,577 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# flake8: noqa: E501
+
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
+
+import json
+import os
+import re
+import shutil
+import tempfile
+from collections import defaultdict
+
+import manifestupdate
+import mozpack.path as mozpath
+import mozunit
+import pytest
+from mozbuild.base import MozbuildObject
+from mozbuild.frontend.reader import BuildReader
+from mozbuild.test.common import MockConfig
+from mozfile import NamedTemporaryFile
+from moztest.resolve import (
+ TEST_SUITES,
+ BuildBackendLoader,
+ TestManifestLoader,
+ TestResolver,
+)
+
+here = os.path.abspath(os.path.dirname(__file__))
+data_path = os.path.join(here, "data")
+
+
+@pytest.fixture(scope="module")
+def topsrcdir():
+ return mozpath.join(data_path, "srcdir")
+
+
+@pytest.fixture(scope="module")
+def create_tests(topsrcdir):
+ def inner(*paths, **defaults):
+ tests = defaultdict(list)
+ for path in paths:
+ if isinstance(path, tuple):
+ path, kwargs = path
+ else:
+ kwargs = {}
+
+ path = mozpath.normpath(path)
+ manifest_name = kwargs.get("flavor", defaults.get("flavor", "manifest"))
+ manifest = kwargs.pop(
+ "manifest",
+ defaults.pop(
+ "manifest",
+ mozpath.join(mozpath.dirname(path), manifest_name + ".ini"),
+ ),
+ )
+
+ manifest_abspath = mozpath.join(topsrcdir, manifest)
+ relpath = mozpath.relpath(path, mozpath.dirname(manifest))
+ test = {
+ "name": relpath,
+ "path": mozpath.join(topsrcdir, path),
+ "relpath": relpath,
+ "file_relpath": path,
+ "flavor": "faketest",
+ "dir_relpath": mozpath.dirname(path),
+ "here": mozpath.dirname(manifest_abspath),
+ "manifest": manifest_abspath,
+ "manifest_relpath": manifest,
+ }
+ test.update(**defaults)
+ test.update(**kwargs)
+
+ # Normalize paths to ensure that the fixture matches reality.
+ for k in [
+ "ancestor_manifest",
+ "manifest",
+ "manifest_relpath",
+ "path",
+ "relpath",
+ ]:
+ p = test.get(k)
+ if p:
+ test[k] = p.replace("/", os.path.sep)
+
+ tests[path].append(test)
+
+ # dump tests to stdout for easier debugging on failure
+ print("The 'create_tests' fixture returned:")
+ print(json.dumps(dict(tests), indent=2, sort_keys=True))
+ return tests
+
+ return inner
+
+
+@pytest.fixture(scope="module")
+def all_tests(create_tests):
+ return create_tests(
+ *[
+ (
+ "apple/test_a11y.html",
+ {
+ "expected": "pass",
+ "flavor": "a11y",
+ },
+ ),
+ (
+ "banana/currant/test_xpcshell_A.js",
+ {
+ "firefox-appdir": "browser",
+ "flavor": "xpcshell",
+ "head": "head_global.js head_helpers.js head_http.js",
+ },
+ ),
+ (
+ "banana/currant/test_xpcshell_B.js",
+ {
+ "firefox-appdir": "browser",
+ "flavor": "xpcshell",
+ "head": "head_global.js head_helpers.js head_http.js",
+ },
+ ),
+ (
+ "carrot/test_included.js",
+ {
+ "ancestor_manifest": "carrot/xpcshell-one.ini",
+ "manifest": "carrot/xpcshell-shared.ini",
+ "flavor": "xpcshell",
+ "stick": "one",
+ },
+ ),
+ (
+ "carrot/test_included.js",
+ {
+ "ancestor_manifest": "carrot/xpcshell-two.ini",
+ "manifest": "carrot/xpcshell-shared.ini",
+ "flavor": "xpcshell",
+ "stick": "two",
+ },
+ ),
+ (
+ "dragonfruit/elderberry/test_xpcshell_C.js",
+ {
+ "flavor": "xpcshell",
+ "generated-files": "head_update.js",
+ "head": "head_update.js",
+ "manifest": "dragonfruit/xpcshell.ini",
+ "reason": "busted",
+ "run-sequentially": "Launches application.",
+ "skip-if": "os == 'android'",
+ },
+ ),
+ (
+ "dragonfruit/elderberry/test_xpcshell_C.js",
+ {
+ "flavor": "xpcshell",
+ "generated-files": "head_update.js",
+ "head": "head_update.js head2.js",
+ "manifest": "dragonfruit/elderberry/xpcshell_updater.ini",
+ "reason": "don't work",
+ "run-sequentially": "Launches application.",
+ "skip-if": "os == 'android'",
+ },
+ ),
+ (
+ "fig/grape/src/TestInstrumentationA.java",
+ {
+ "flavor": "instrumentation",
+ "manifest": "fig/grape/instrumentation.ini",
+ "subsuite": "background",
+ },
+ ),
+ (
+ "fig/huckleberry/src/TestInstrumentationB.java",
+ {
+ "flavor": "instrumentation",
+ "manifest": "fig/huckleberry/instrumentation.ini",
+ "subsuite": "browser",
+ },
+ ),
+ (
+ "juniper/browser_chrome.js",
+ {
+ "flavor": "browser-chrome",
+ "manifest": "juniper/browser.ini",
+ "skip-if": "e10s # broken",
+ },
+ ),
+ (
+ "kiwi/browser_devtools.js",
+ {
+ "flavor": "browser-chrome",
+ "manifest": "kiwi/browser.ini",
+ "subsuite": "devtools",
+ "tags": "devtools",
+ },
+ ),
+ ]
+ )
+
+
+@pytest.fixture(scope="module")
+def defaults(topsrcdir):
+ def to_abspath(relpath):
+ # test-defaults.pkl uses absolute paths with platform-specific path separators.
+ # Use platform-specific separators if needed to avoid regressing on bug 1644223.
+ return os.path.normpath(os.path.join(topsrcdir, relpath))
+
+ return {
+ (to_abspath("dragonfruit/elderberry/xpcshell_updater.ini")): {
+ "support-files": "\ndata/**\nxpcshell_updater.ini"
+ },
+ (
+ to_abspath("carrot/xpcshell-one.ini"),
+ to_abspath("carrot/xpcshell-shared.ini"),
+ ): {
+ "head": "head_one.js",
+ },
+ (
+ to_abspath("carrot/xpcshell-two.ini"),
+ to_abspath("carrot/xpcshell-shared.ini"),
+ ): {
+ "head": "head_two.js",
+ },
+ }
+
+
+class WPTManifestNamespace(object):
+ """Stand-in object for various WPT classes."""
+
+ def __init__(self, *args):
+ self.args = args
+
+ def __hash__(self):
+ return hash(str(self))
+
+ def __eq__(self, other):
+ return self.args == other.args
+
+ def __iter__(self):
+ yield tuple(self.args)
+
+
+def fake_wpt_manifestupdate(topsrcdir, *args, **kwargs):
+ with open(os.path.join(topsrcdir, "wpt_manifest_data.json")) as fh:
+ data = json.load(fh)
+
+ items = {}
+ for tests_root, test_data in data.items():
+ kwargs = {"tests_path": os.path.join(topsrcdir, tests_root)}
+
+ for test_type, tests in test_data.items():
+ for test in tests:
+ obj = WPTManifestNamespace()
+ if "mozilla" in tests_root:
+ obj.id = "/_mozilla/" + test
+ else:
+ obj.id = "/" + test
+
+ items[WPTManifestNamespace(test_type, test, {obj})] = kwargs
+ return items
+
+
+@pytest.fixture(params=[BuildBackendLoader, TestManifestLoader])
+def resolver(request, tmpdir, monkeypatch, topsrcdir, all_tests, defaults):
+ topobjdir = tmpdir.mkdir("objdir").strpath
+ loader_cls = request.param
+
+ if loader_cls == BuildBackendLoader:
+ with open(os.path.join(topobjdir, "all-tests.pkl"), "wb") as fh:
+ pickle.dump(all_tests, fh)
+ with open(os.path.join(topobjdir, "test-defaults.pkl"), "wb") as fh:
+ pickle.dump(defaults, fh)
+
+ # The mock data already exists, so prevent BuildBackendLoader from regenerating
+ # the build information from the whole gecko tree...
+ class BuildBackendLoaderNeverOutOfDate(BuildBackendLoader):
+ def backend_out_of_date(self, backend_file):
+ return False
+
+ loader_cls = BuildBackendLoaderNeverOutOfDate
+
+ # Patch WPT's manifestupdate.run to return tests based on the contents of
+ # 'data/srcdir/wpt_manifest_data.json'.
+ monkeypatch.setattr(manifestupdate, "run", fake_wpt_manifestupdate)
+
+ resolver = TestResolver(
+ topsrcdir, None, None, topobjdir=topobjdir, loader_cls=loader_cls
+ )
+ resolver._puppeteer_loaded = True
+
+ if loader_cls == TestManifestLoader:
+ config = MockConfig(topsrcdir)
+ resolver.load_tests.reader = BuildReader(config)
+ return resolver
+
+
+def test_load(resolver):
+ assert len(resolver.tests_by_path) == 9
+
+ assert len(resolver.tests_by_flavor["mochitest-plain"]) == 0
+ assert len(resolver.tests_by_flavor["xpcshell"]) == 4
+ assert len(resolver.tests_by_flavor["web-platform-tests"]) == 0
+
+ assert len(resolver.tests_by_manifest) == 9
+
+ resolver.add_wpt_manifest_data()
+ assert len(resolver.tests_by_path) == 11
+ assert len(resolver.tests_by_flavor["web-platform-tests"]) == 2
+ assert len(resolver.tests_by_manifest) == 11
+ assert "/html" in resolver.tests_by_manifest
+ assert "/_mozilla/html" in resolver.tests_by_manifest
+
+
+def test_resolve_all(resolver):
+ assert len(list(resolver._resolve())) == 13
+
+
+def test_resolve_filter_flavor(resolver):
+ assert len(list(resolver._resolve(flavor="xpcshell"))) == 6
+
+
+def test_resolve_by_dir(resolver):
+ assert len(list(resolver._resolve(paths=["banana/currant"]))) == 2
+
+
+def test_resolve_under_path(resolver):
+ assert len(list(resolver._resolve(under_path="banana"))) == 2
+ assert len(list(resolver._resolve(flavor="xpcshell", under_path="banana"))) == 2
+
+
+def test_resolve_multiple_paths(resolver):
+ result = list(resolver.resolve_tests(paths=["banana", "dragonfruit"]))
+ assert len(result) == 4
+
+
+def test_resolve_support_files(resolver):
+ expected_support_files = "\ndata/**\nxpcshell_updater.ini"
+ tests = list(resolver.resolve_tests(paths=["dragonfruit"]))
+ assert len(tests) == 2
+
+ for test in tests:
+ if test["manifest"].endswith("xpcshell_updater.ini"):
+ assert test["support-files"] == expected_support_files
+ else:
+ assert "support-files" not in test
+
+
+def test_resolve_path_prefix(resolver):
+ tests = list(resolver._resolve(paths=["juniper"]))
+ assert len(tests) == 1
+
+ # relative manifest
+ tests = list(resolver._resolve(paths=["apple/a11y.ini"]))
+ assert len(tests) == 1
+ assert tests[0]["name"] == "test_a11y.html"
+
+ # absolute manifest
+ tests = list(
+ resolver._resolve(paths=[os.path.join(resolver.topsrcdir, "apple/a11y.ini")])
+ )
+ assert len(tests) == 1
+ assert tests[0]["name"] == "test_a11y.html"
+
+
+def test_cwd_children_only(resolver):
+ """If cwd is defined, only resolve tests under the specified cwd."""
+ # Pretend we're under '/services' and ask for 'common'. This should
+ # pick up all tests from '/services/common'
+ tests = list(
+ resolver.resolve_tests(
+ paths=["currant"], cwd=os.path.join(resolver.topsrcdir, "banana")
+ )
+ )
+
+ assert len(tests) == 2
+
+ # Tests should be rewritten to objdir.
+ for t in tests:
+ assert t["here"] == mozpath.join(
+ resolver.topobjdir, "_tests/xpcshell/banana/currant"
+ )
+
+
+def test_various_cwd(resolver):
+ """Test various cwd conditions are all equal."""
+ expected = list(resolver.resolve_tests(paths=["banana"]))
+ actual = list(resolver.resolve_tests(paths=["banana"], cwd="/"))
+ assert actual == expected
+
+ actual = list(resolver.resolve_tests(paths=["banana"], cwd=resolver.topsrcdir))
+ assert actual == expected
+
+ actual = list(resolver.resolve_tests(paths=["banana"], cwd=resolver.topobjdir))
+ assert actual == expected
+
+
+def test_subsuites(resolver):
+ """Test filtering by subsuite."""
+ tests = list(resolver.resolve_tests(paths=["fig"]))
+ assert len(tests) == 2
+
+ tests = list(resolver.resolve_tests(paths=["fig"], subsuite="browser"))
+ assert len(tests) == 1
+ assert tests[0]["name"] == "src/TestInstrumentationB.java"
+
+ tests = list(resolver.resolve_tests(paths=["fig"], subsuite="background"))
+ assert len(tests) == 1
+ assert tests[0]["name"] == "src/TestInstrumentationA.java"
+
+ # Resolve tests *without* a subsuite.
+ tests = list(resolver.resolve_tests(flavor="browser-chrome", subsuite="undefined"))
+ assert len(tests) == 1
+ assert tests[0]["name"] == "browser_chrome.js"
+
+
+def test_wildcard_patterns(resolver):
+ """Test matching paths by wildcard."""
+ tests = list(resolver.resolve_tests(paths=["fig/**"]))
+ assert len(tests) == 2
+ for t in tests:
+ assert t["file_relpath"].startswith("fig")
+
+ tests = list(resolver.resolve_tests(paths=["**/**.js", "apple/**"]))
+ assert len(tests) == 9
+ for t in tests:
+ path = t["file_relpath"]
+ assert path.startswith("apple") or path.endswith(".js")
+
+
+def test_resolve_metadata(resolver):
+ """Test finding metadata from outgoing files."""
+ suites, tests = resolver.resolve_metadata(["bc"])
+ assert suites == {"mochitest-browser-chrome"}
+ assert tests == []
+
+ suites, tests = resolver.resolve_metadata(
+ ["mochitest-a11y", "/browser", "xpcshell"]
+ )
+ assert suites == {"mochitest-a11y", "xpcshell"}
+ assert sorted(t["file_relpath"] for t in tests) == [
+ "juniper/browser_chrome.js",
+ "kiwi/browser_devtools.js",
+ ]
+
+
+def test_ancestor_manifest_defaults(resolver, topsrcdir, defaults):
+ """Test that defaults from ancestor manifests are found."""
+ tests = list(resolver._resolve(paths=["carrot/test_included.js"]))
+ assert len(tests) == 2
+
+ if tests[0]["ancestor_manifest"] == os.path.join("carrot", "xpcshell-one.ini"):
+ [testOne, testTwo] = tests
+ else:
+ [testTwo, testOne] = tests
+
+ assert testOne["ancestor_manifest"] == os.path.join("carrot", "xpcshell-one.ini")
+ assert testOne["manifest_relpath"] == os.path.join("carrot", "xpcshell-shared.ini")
+ assert testOne["head"] == "head_one.js"
+ assert testOne["stick"] == "one"
+
+ assert testTwo["ancestor_manifest"] == os.path.join("carrot", "xpcshell-two.ini")
+ assert testTwo["manifest_relpath"] == os.path.join("carrot", "xpcshell-shared.ini")
+ assert testTwo["head"] == "head_two.js"
+ assert testTwo["stick"] == "two"
+
+
+def test_task_regexes():
+ """Test the task_regexes defined in TEST_SUITES."""
+ task_labels = [
+ "test-linux64/opt-browser-screenshots-1",
+ "test-linux64/opt-browser-screenshots-e10s-1",
+ "test-linux64/opt-marionette",
+ "test-linux64/opt-mochitest-plain",
+ "test-linux64/debug-mochitest-plain-e10s",
+ "test-linux64/opt-mochitest-a11y",
+ "test-linux64/opt-mochitest-browser",
+ "test-linux64/opt-mochitest-browser-chrome",
+ "test-linux64/opt-mochitest-browser-chrome-e10s",
+ "test-linux64/opt-mochitest-browser-chrome-e10s-11",
+ "test-linux64/opt-mochitest-chrome",
+ "test-linux64/opt-mochitest-devtools",
+ "test-linux64/opt-mochitest-devtools-chrome",
+ "test-linux64/opt-mochitest-gpu",
+ "test-linux64/opt-mochitest-gpu-e10s",
+ "test-linux64/opt-mochitest-media-e10s-1",
+ "test-linux64/opt-mochitest-media-e10s-11",
+ "test-linux64/opt-mochitest-screenshots-1",
+ "test-linux64/opt-reftest",
+ "test-linux64/opt-geckoview-reftest",
+ "test-linux64/debug-reftest-e10s-1",
+ "test-linux64/debug-reftest-e10s-11",
+ "test-linux64/opt-robocop",
+ "test-linux64/opt-robocop-1",
+ "test-linux64/opt-robocop-e10s",
+ "test-linux64/opt-robocop-e10s-1",
+ "test-linux64/opt-robocop-e10s-11",
+ "test-linux64/opt-web-platform-tests-e10s-1",
+ "test-linux64/opt-web-platform-tests-reftest-e10s-1",
+ "test-linux64/opt-web-platform-tests-wdspec-e10s-1",
+ "test-linux64/opt-web-platform-tests-1",
+ "test-linux64/opt-web-platform-test-e10s-1",
+ "test-linux64/opt-xpcshell",
+ "test-linux64/opt-xpcshell-1",
+ "test-linux64/opt-xpcshell-2",
+ ]
+
+ test_cases = {
+ "mochitest-browser-chrome": [
+ "test-linux64/opt-mochitest-browser-chrome",
+ "test-linux64/opt-mochitest-browser-chrome-e10s",
+ ],
+ "mochitest-chrome": [
+ "test-linux64/opt-mochitest-chrome",
+ ],
+ "mochitest-devtools-chrome": [
+ "test-linux64/opt-mochitest-devtools-chrome",
+ ],
+ "mochitest-media": [
+ "test-linux64/opt-mochitest-media-e10s-1",
+ ],
+ "mochitest-plain": [
+ "test-linux64/opt-mochitest-plain",
+ "test-linux64/debug-mochitest-plain-e10s",
+ ],
+ "mochitest-plain-gpu": [
+ "test-linux64/opt-mochitest-gpu",
+ "test-linux64/opt-mochitest-gpu-e10s",
+ ],
+ "mochitest-browser-chrome-screenshots": [
+ "test-linux64/opt-browser-screenshots-1",
+ "test-linux64/opt-browser-screenshots-e10s-1",
+ ],
+ "reftest": [
+ "test-linux64/opt-reftest",
+ "test-linux64/opt-geckoview-reftest",
+ "test-linux64/debug-reftest-e10s-1",
+ ],
+ "robocop": [
+ "test-linux64/opt-robocop",
+ "test-linux64/opt-robocop-1",
+ "test-linux64/opt-robocop-e10s",
+ "test-linux64/opt-robocop-e10s-1",
+ ],
+ "web-platform-tests": [
+ "test-linux64/opt-web-platform-tests-e10s-1",
+ "test-linux64/opt-web-platform-tests-1",
+ ],
+ "web-platform-tests-reftest": [
+ "test-linux64/opt-web-platform-tests-reftest-e10s-1",
+ ],
+ "web-platform-tests-wdspec": [
+ "test-linux64/opt-web-platform-tests-wdspec-e10s-1",
+ ],
+ "xpcshell": [
+ "test-linux64/opt-xpcshell",
+ "test-linux64/opt-xpcshell-1",
+ ],
+ }
+
+ regexes = []
+
+ def match_task(task):
+ return any(re.search(pattern, task) for pattern in regexes)
+
+ for suite, expected in sorted(test_cases.items()):
+ print(suite)
+ regexes = TEST_SUITES[suite]["task_regex"]
+ assert set(filter(match_task, task_labels)) == set(expected)
+
+
+if __name__ == "__main__":
+ mozunit.main()