Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 315 insertions, 0 deletions

diff --git a/taskcluster/gecko_taskgraph/util/chunking.py b/taskcluster/gecko_taskgraph/util/chunking.py
new file mode 100644
index 0000000000..2c6c429a2c
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/util/chunking.py
@@ -0,0 +1,315 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+"""Utility functions to handle test chunking."""
+
+import json
+import logging
+import os
+from abc import ABCMeta, abstractmethod
+
+from manifestparser import TestManifest
+from manifestparser.filters import chunk_by_runtime, tags
+from mozbuild.util import memoize
+from moztest.resolve import TEST_SUITES, TestManifestLoader, TestResolver
+
+from gecko_taskgraph import GECKO
+from gecko_taskgraph.util.bugbug import CT_LOW, BugbugTimeoutException, push_schedules
+
+logger = logging.getLogger(__name__)
+here = os.path.abspath(os.path.dirname(__file__))
+resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader)
+
+
+def guess_mozinfo_from_task(task, repo=""):
+    """Attempt to build a mozinfo dict from a task definition.
+
+    This won't be perfect and many values used in the manifests will be missing. But
+    it should cover most of the major ones and be "good enough" for chunking in the
+    taskgraph.
+
+    Args:
+        task (dict): A task definition.
+
+    Returns:
+        A dict that can be used as a mozinfo replacement.
+    """
+    setting = task["test-setting"]
+    arch = setting["platform"]["arch"]
+    p_os = setting["platform"]["os"]
+
+    info = {
+        "asan": setting["build"].get("asan", False),
+        "bits": 32 if "32" in arch else 64,
+        "ccov": setting["build"].get("ccov", False),
+        "debug": setting["build"]["type"] in ("debug", "debug-isolated-process"),
+        "e10s": not setting["runtime"].get("1proc", False),
+        "no-fission": "no-fission" in setting["runtime"].keys(),
+        "fission": any(
+            "1proc" not in key or "no-fission" not in key
+            for key in setting["runtime"].keys()
+        ),
+        "headless": "-headless" in task["test-name"],
+        "condprof": "conditioned_profile" in setting["runtime"].keys(),
+        "tsan": setting["build"].get("tsan", False),
+        "xorigin": any("xorigin" in key for key in setting["runtime"].keys()),
+        "socketprocess_networking": "socketprocess_networking"
+        in setting["runtime"].keys(),
+        "nightly_build": repo in ["mozilla-central", "autoland", "try", ""],  # trunk
+        "http3": "http3" in setting["runtime"].keys(),
+    }
+    for platform in ("android", "linux", "mac", "win"):
+        if p_os["name"].startswith(platform):
+            info["os"] = platform
+            break
+    else:
+        raise ValueError("{} is not a known platform!".format(p_os["name"]))
+
+    # crashreporter is disabled for asan / tsan builds
+    if info["asan"] or info["tsan"]:
+        info["crashreporter"] = False
+    else:
+        info["crashreporter"] = True
+
+    info["appname"] = "fennec" if info["os"] == "android" else "firefox"
+
+    # guess processor
+    if arch == "aarch64":
+        info["processor"] = "aarch64"
+    elif info["os"] == "android" and "arm" in arch:
+        info["processor"] = "arm"
+    elif info["bits"] == 32:
+        info["processor"] = "x86"
+    else:
+        info["processor"] = "x86_64"
+
+    # guess toolkit
+    if info["os"] == "android":
+        info["toolkit"] = "android"
+    elif info["os"] == "win":
+        info["toolkit"] = "windows"
+    elif info["os"] == "mac":
+        info["toolkit"] = "cocoa"
+    else:
+        info["toolkit"] = "gtk"
+
+    # guess os_version
+    os_versions = {
+        ("linux", "1804"): "18.04",
+        ("macosx", "1015"): "10.15",
+        ("macosx", "1100"): "11.00",
+        ("windows", "7"): "6.1",
+        ("windows", "10"): "10.0",
+    }
+    for (name, old_ver), new_ver in os_versions.items():
+        if p_os["name"] == name and p_os["version"] == old_ver:
+            info["os_version"] = new_ver
+            break
+
+    return info
+
+
+@memoize
+def get_runtimes(platform, suite_name):
+    if not suite_name or not platform:
+        raise TypeError("suite_name and platform cannot be empty.")
+
+    base = os.path.join(GECKO, "testing", "runtimes", "manifest-runtimes-{}.json")
+    for key in ("android", "windows"):
+        if key in platform:
+            path = base.format(key)
+            break
+    else:
+        path = base.format("unix")
+
+    if not os.path.exists(path):
+        raise OSError(f"manifest runtime file at {path} not found.")
+
+    with open(path) as fh:
+        return json.load(fh)[suite_name]
+
+
+def chunk_manifests(suite, platform, chunks, manifests):
+    """Run the chunking algorithm.
+
+    Args:
+        platform (str): Platform used to find runtime info.
+        chunks (int): Number of chunks to split manifests into.
+        manifests(list): Manifests to chunk.
+
+    Returns:
+        A list of length `chunks` where each item contains a list of manifests
+        that run in that chunk.
+    """
+    manifests = set(manifests)
+
+    if "web-platform-tests" not in suite:
+        runtimes = {
+            k: v for k, v in get_runtimes(platform, suite).items() if k in manifests
+        }
+        return [
+            c[1]
+            for c in chunk_by_runtime(None, chunks, runtimes).get_chunked_manifests(
+                manifests
+            )
+        ]
+
+    # Keep track of test paths for each chunk, and the runtime information.
+    chunked_manifests = [[] for _ in range(chunks)]
+
+    # Spread out the test manifests evenly across all chunks.
+    for index, key in enumerate(sorted(manifests)):
+        chunked_manifests[index % chunks].append(key)
+
+    # One last sort by the number of manifests. Chunk size should be more or less
+    # equal in size.
+    chunked_manifests.sort(key=lambda x: len(x))
+
+    # Return just the chunked test paths.
+    return chunked_manifests
+
+
+class BaseManifestLoader(metaclass=ABCMeta):
+    def __init__(self, params):
+        self.params = params
+
+    @abstractmethod
+    def get_manifests(self, flavor, subsuite, mozinfo):
+        """Compute which manifests should run for the given flavor, subsuite and mozinfo.
+
+        This function returns skipped manifests separately so that more balanced
+        chunks can be achieved by only considering "active" manifests in the
+        chunking algorithm.
+
+        Args:
+            flavor (str): The suite to run. Values are defined by the 'build_flavor' key
+                in `moztest.resolve.TEST_SUITES`.
+            subsuite (str): The subsuite to run or 'undefined' to denote no subsuite.
+            mozinfo (frozenset): Set of data in the form of (<key>, <value>) used
+                                 for filtering.
+
+        Returns:
+            A tuple of two manifest lists. The first is the set of active manifests (will
+            run at least one test. The second is a list of skipped manifests (all tests are
+            skipped).
+        """
+
+
+class DefaultLoader(BaseManifestLoader):
+    """Load manifests using metadata from the TestResolver."""
+
+    @memoize
+    def get_tests(self, suite):
+        suite_definition = TEST_SUITES[suite]
+        return list(
+            resolver.resolve_tests(
+                flavor=suite_definition["build_flavor"],
+                subsuite=suite_definition.get("kwargs", {}).get(
+                    "subsuite", "undefined"
+                ),
+            )
+        )
+
+    @memoize
+    def get_manifests(self, suite, mozinfo):
+        mozinfo = dict(mozinfo)
+        # Compute all tests for the given suite/subsuite.
+        tests = self.get_tests(suite)
+
+        # TODO: the only exception here is we schedule webgpu as that is a --tag
+        if "web-platform-tests" in suite:
+            manifests = set()
+            for t in tests:
+                manifests.add(t["manifest"])
+            return {
+                "active": list(manifests),
+                "skipped": [],
+                "other_dirs": dict.fromkeys(manifests, ""),
+            }
+
+        manifests = {chunk_by_runtime.get_manifest(t) for t in tests}
+
+        filters = None
+        if mozinfo["condprof"]:
+            filters = [tags(["condprof"])]
+
+        # Compute  the active tests.
+        m = TestManifest()
+        m.tests = tests
+        tests = m.active_tests(disabled=False, exists=False, filters=filters, **mozinfo)
+        active = {}
+        # map manifests and 'other' directories included
+        for t in tests:
+            mp = chunk_by_runtime.get_manifest(t)
+            active.setdefault(mp, [])
+
+            if not mp.startswith(t["dir_relpath"]):
+                active[mp].append(t["dir_relpath"])
+
+        skipped = manifests - set(active.keys())
+        other = {}
+        for m in active:
+            if len(active[m]) > 0:
+                other[m] = list(set(active[m]))
+        return {
+            "active": list(active.keys()),
+            "skipped": list(skipped),
+            "other_dirs": other,
+        }
+
+
+class BugbugLoader(DefaultLoader):
+    """Load manifests using metadata from the TestResolver, and then
+    filter them based on a query to bugbug."""
+
+    CONFIDENCE_THRESHOLD = CT_LOW
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.timedout = False
+
+    @memoize
+    def get_manifests(self, suite, mozinfo):
+        manifests = super().get_manifests(suite, mozinfo)
+
+        # Don't prune any manifests if we're on a backstop push or there was a timeout.
+        if self.params["backstop"] or self.timedout:
+            return manifests
+
+        try:
+            data = push_schedules(self.params["project"], self.params["head_rev"])
+        except BugbugTimeoutException:
+            logger.warning("Timed out waiting for bugbug, loading all test manifests.")
+            self.timedout = True
+            return self.get_manifests(suite, mozinfo)
+
+        bugbug_manifests = {
+            m
+            for m, c in data.get("groups", {}).items()
+            if c >= self.CONFIDENCE_THRESHOLD
+        }
+
+        manifests["active"] = list(set(manifests["active"]) & bugbug_manifests)
+        manifests["skipped"] = list(set(manifests["skipped"]) & bugbug_manifests)
+        return manifests
+
+
+manifest_loaders = {
+    "bugbug": BugbugLoader,
+    "default": DefaultLoader,
+}
+
+_loader_cache = {}
+
+
+def get_manifest_loader(name, params):
+    # Ensure we never create more than one instance of the same loader type for
+    # performance reasons.
+    if name in _loader_cache:
+        return _loader_cache[name]
+
+    loader = manifest_loaders[name](dict(params))
+    _loader_cache[name] = loader
+    return loader