Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

4 files changed, 679 insertions, 0 deletions

diff --git a/tools/tryselect/selectors/perfselector/__init__.py b/tools/tryselect/selectors/perfselector/__init__.py
new file mode 100644
index 0000000000..c580d191c1
--- /dev/null
+++ b/tools/tryselect/selectors/perfselector/__init__.py
@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
diff --git a/tools/tryselect/selectors/perfselector/classification.py b/tools/tryselect/selectors/perfselector/classification.py
new file mode 100644
index 0000000000..4ed18ce3e5
--- /dev/null
+++ b/tools/tryselect/selectors/perfselector/classification.py
@@ -0,0 +1,374 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import enum
+
+
+class ClassificationEnum(enum.Enum):
+    """This class provides the ability to use Enums as array indices."""
+
+    @property
+    def value(self):
+        return self._value_["value"]
+
+    def __index__(self):
+        return self._value_["index"]
+
+    def __int__(self):
+        return self._value_["index"]
+
+
+class Platforms(ClassificationEnum):
+    ANDROID_A51 = {"value": "android-a51", "index": 0}
+    ANDROID = {"value": "android", "index": 1}
+    WINDOWS = {"value": "windows", "index": 2}
+    LINUX = {"value": "linux", "index": 3}
+    MACOSX = {"value": "macosx", "index": 4}
+    DESKTOP = {"value": "desktop", "index": 5}
+
+
+class Apps(ClassificationEnum):
+    FIREFOX = {"value": "firefox", "index": 0}
+    CHROME = {"value": "chrome", "index": 1}
+    CHROMIUM = {"value": "chromium", "index": 2}
+    GECKOVIEW = {"value": "geckoview", "index": 3}
+    FENIX = {"value": "fenix", "index": 4}
+    CHROME_M = {"value": "chrome-m", "index": 5}
+    SAFARI = {"value": "safari", "index": 6}
+    CHROMIUM_RELEASE = {"value": "custom-car", "index": 7}
+
+
+class Suites(ClassificationEnum):
+    RAPTOR = {"value": "raptor", "index": 0}
+    TALOS = {"value": "talos", "index": 1}
+    AWSY = {"value": "awsy", "index": 2}
+
+
+class Variants(ClassificationEnum):
+    NO_FISSION = {"value": "no-fission", "index": 0}
+    BYTECODE_CACHED = {"value": "bytecode-cached", "index": 1}
+    LIVE_SITES = {"value": "live-sites", "index": 2}
+    PROFILING = {"value": "profiling", "index": 3}
+    SWR = {"value": "swr", "index": 4}
+
+
+"""
+The following methods and constants are used for restricting
+certain platforms and applications such as chrome, safari, and
+android tests. These all require a flag such as --android to
+enable (see build_category_matrix for more info).
+"""
+
+
+def check_for_android(android=False, **kwargs):
+    return android
+
+
+def check_for_chrome(chrome=False, **kwargs):
+    return chrome
+
+
+def check_for_custom_car(custom_car=False, **kwargs):
+    return custom_car
+
+
+def check_for_safari(safari=False, **kwargs):
+    return safari
+
+
+def check_for_live_sites(live_sites=False, **kwargs):
+    return live_sites
+
+
+def check_for_profile(profile=False, **kwargs):
+    return profile
+
+
+class ClassificationProvider:
+    @property
+    def platforms(self):
+        return {
+            Platforms.ANDROID_A51.value: {
+                "query": "'android 'a51 'shippable 'aarch64",
+                "restriction": check_for_android,
+                "platform": Platforms.ANDROID.value,
+            },
+            Platforms.ANDROID.value: {
+                # The android, and android-a51 queries are expected to be the same,
+                # we don't want to run the tests on other mobile platforms.
+                "query": "'android 'a51 'shippable 'aarch64",
+                "restriction": check_for_android,
+                "platform": Platforms.ANDROID.value,
+            },
+            Platforms.WINDOWS.value: {
+                "query": "!-32 'windows 'shippable",
+                "platform": Platforms.DESKTOP.value,
+            },
+            Platforms.LINUX.value: {
+                "query": "!clang 'linux 'shippable",
+                "platform": Platforms.DESKTOP.value,
+            },
+            Platforms.MACOSX.value: {
+                "query": "'osx 'shippable",
+                "platform": Platforms.DESKTOP.value,
+            },
+            Platforms.DESKTOP.value: {
+                "query": "!android 'shippable !-32 !clang",
+                "platform": Platforms.DESKTOP.value,
+            },
+        }
+
+    @property
+    def apps(self):
+        return {
+            Apps.FIREFOX.value: {
+                "query": "!chrom !geckoview !fenix !safari !custom-car",
+                "platforms": [Platforms.DESKTOP.value],
+            },
+            Apps.CHROME.value: {
+                "query": "'chrome",
+                "negation": "!chrom",
+                "restriction": check_for_chrome,
+                "platforms": [Platforms.DESKTOP.value],
+            },
+            Apps.CHROMIUM.value: {
+                "query": "'chromium",
+                "negation": "!chrom",
+                "restriction": check_for_chrome,
+                "platforms": [Platforms.DESKTOP.value],
+            },
+            Apps.GECKOVIEW.value: {
+                "query": "'geckoview",
+                "platforms": [Platforms.ANDROID.value],
+            },
+            Apps.FENIX.value: {
+                "query": "'fenix",
+                "platforms": [Platforms.ANDROID.value],
+            },
+            Apps.CHROME_M.value: {
+                "query": "'chrome-m",
+                "negation": "!chrom",
+                "restriction": check_for_chrome,
+                "platforms": [Platforms.ANDROID.value],
+            },
+            Apps.SAFARI.value: {
+                "query": "'safari",
+                "negation": "!safari",
+                "restriction": check_for_safari,
+                "platforms": [Platforms.MACOSX.value],
+            },
+            Apps.CHROMIUM_RELEASE.value: {
+                "query": "'custom-car",
+                "negation": "!custom-car",
+                "restriction": check_for_custom_car,
+                "platforms": [Platforms.LINUX.value, Platforms.WINDOWS.value],
+            },
+        }
+
+    @property
+    def variants(self):
+        return {
+            Variants.NO_FISSION.value: {
+                "query": "'nofis",
+                "negation": "!nofis",
+                "platforms": [Platforms.ANDROID.value],
+                "apps": [Apps.FENIX.value, Apps.GECKOVIEW.value],
+            },
+            Variants.BYTECODE_CACHED.value: {
+                "query": "'bytecode",
+                "negation": "!bytecode",
+                "platforms": [Platforms.DESKTOP.value],
+                "apps": [Apps.FIREFOX.value],
+            },
+            Variants.LIVE_SITES.value: {
+                "query": "'live",
+                "negation": "!live",
+                "restriction": check_for_live_sites,
+                "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value],
+                "apps": [  # XXX No live CaR tests
+                    Apps.FIREFOX.value,
+                    Apps.CHROME.value,
+                    Apps.CHROMIUM.value,
+                    Apps.FENIX.value,
+                    Apps.GECKOVIEW.value,
+                    Apps.SAFARI.value,
+                ],
+            },
+            Variants.PROFILING.value: {
+                "query": "'profil",
+                "negation": "!profil",
+                "restriction": check_for_profile,
+                "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value],
+                "apps": [Apps.FIREFOX.value, Apps.GECKOVIEW.value, Apps.FENIX.value],
+            },
+            Variants.SWR.value: {
+                "query": "'swr",
+                "negation": "!swr",
+                "platforms": [Platforms.DESKTOP.value],
+                "apps": [Apps.FIREFOX.value],
+            },
+        }
+
+    @property
+    def suites(self):
+        return {
+            Suites.RAPTOR.value: {
+                "apps": list(self.apps.keys()),
+                "platforms": list(self.platforms.keys()),
+                "variants": [
+                    Variants.NO_FISSION.value,
+                    Variants.LIVE_SITES.value,
+                    Variants.PROFILING.value,
+                    Variants.BYTECODE_CACHED.value,
+                ],
+            },
+            Suites.TALOS.value: {
+                "apps": [Apps.FIREFOX.value],
+                "platforms": [Platforms.DESKTOP.value],
+                "variants": [
+                    Variants.PROFILING.value,
+                    Variants.SWR.value,
+                ],
+            },
+            Suites.AWSY.value: {
+                "apps": [Apps.FIREFOX.value],
+                "platforms": [Platforms.DESKTOP.value],
+                "variants": [],
+            },
+        }
+
+    """
+    Here you can find the base categories that are defined for the perf
+    selector. The following fields are available:
+        * query: Set the queries to use for each suite you need.
+        * suites: The suites that are needed for this category.
+        * tasks: A hard-coded list of tasks to select.
+        * platforms: The platforms that it can run on.
+        * app-restrictions: A list of apps that the category can run.
+        * variant-restrictions: A list of variants available for each suite.
+
+    Note that setting the App/Variant-Restriction fields should be used to
+    restrict the available apps and variants, not expand them.
+    """
+
+    @property
+    def categories(self):
+        return {
+            "Pageload": {
+                "query": {
+                    Suites.RAPTOR.value: ["'browsertime 'tp6"],
+                },
+                "suites": [Suites.RAPTOR.value],
+                "tasks": [],
+            },
+            "Pageload (essential)": {
+                "query": {
+                    Suites.RAPTOR.value: ["'browsertime 'tp6 'essential"],
+                },
+                "variant-restrictions": {
+                    Suites.RAPTOR.value: [Variants.NO_FISSION.value]
+                },
+                "suites": [Suites.RAPTOR.value],
+                "app-restrictions": {
+                    Suites.RAPTOR.value: [
+                        Apps.FIREFOX.value,
+                        Apps.CHROME.value,
+                        Apps.CHROMIUM.value,
+                        Apps.FENIX.value,
+                        Apps.GECKOVIEW.value,
+                        Apps.CHROMIUM_RELEASE.value,
+                    ],
+                },
+                "tasks": [],
+            },
+            "Speedometer 3": {
+                "query": {
+                    Suites.RAPTOR.value: ["'browsertime 'speedometer3"],
+                },
+                "variant-restrictions": {
+                    Suites.RAPTOR.value: [Variants.NO_FISSION.value]
+                },
+                "suites": [Suites.RAPTOR.value],
+                "app-restrictions": {},
+                "tasks": [],
+            },
+            "Responsiveness": {
+                "query": {
+                    Suites.RAPTOR.value: ["'browsertime 'responsive"],
+                },
+                "suites": [Suites.RAPTOR.value],
+                "variant-restrictions": {Suites.RAPTOR.value: []},
+                "app-restrictions": {
+                    Suites.RAPTOR.value: [
+                        Apps.FIREFOX.value,
+                        Apps.CHROME.value,
+                        Apps.CHROMIUM.value,
+                        Apps.FENIX.value,
+                        Apps.GECKOVIEW.value,
+                    ],
+                },
+                "tasks": [],
+            },
+            "Benchmarks": {
+                "query": {
+                    Suites.RAPTOR.value: ["'browsertime 'benchmark"],
+                },
+                "suites": [Suites.RAPTOR.value],
+                "variant-restrictions": {Suites.RAPTOR.value: []},
+                "tasks": [],
+            },
+            "DAMP (Devtools)": {
+                "query": {
+                    Suites.TALOS.value: ["'talos 'damp"],
+                },
+                "suites": [Suites.TALOS.value],
+                "tasks": [],
+            },
+            "Talos PerfTests": {
+                "query": {
+                    Suites.TALOS.value: ["'talos"],
+                },
+                "suites": [Suites.TALOS.value],
+                "tasks": [],
+            },
+            "Resource Usage": {
+                "query": {
+                    Suites.TALOS.value: ["'talos 'xperf | 'tp5"],
+                    Suites.RAPTOR.value: ["'power 'osx"],
+                    Suites.AWSY.value: ["'awsy"],
+                },
+                "suites": [Suites.TALOS.value, Suites.RAPTOR.value, Suites.AWSY.value],
+                "platform-restrictions": [Platforms.DESKTOP.value],
+                "variant-restrictions": {
+                    Suites.RAPTOR.value: [],
+                    Suites.TALOS.value: [],
+                },
+                "app-restrictions": {
+                    Suites.RAPTOR.value: [Apps.FIREFOX.value],
+                    Suites.TALOS.value: [Apps.FIREFOX.value],
+                },
+                "tasks": [],
+            },
+            "Graphics, & Media Playback": {
+                "query": {
+                    # XXX This might not be an exhaustive list for talos atm
+                    Suites.TALOS.value: ["'talos 'svgr | 'bcv | 'webgl"],
+                    Suites.RAPTOR.value: ["'browsertime 'youtube-playback"],
+                },
+                "suites": [Suites.TALOS.value, Suites.RAPTOR.value],
+                "variant-restrictions": {
+                    Suites.RAPTOR.value: [Variants.NO_FISSION.value]
+                },
+                "app-restrictions": {
+                    Suites.RAPTOR.value: [
+                        Apps.FIREFOX.value,
+                        Apps.CHROME.value,
+                        Apps.CHROMIUM.value,
+                        Apps.FENIX.value,
+                        Apps.GECKOVIEW.value,
+                    ],
+                },
+                "tasks": [],
+            },
+        }
diff --git a/tools/tryselect/selectors/perfselector/perfcomparators.py b/tools/tryselect/selectors/perfselector/perfcomparators.py
new file mode 100644
index 0000000000..fce35fe562
--- /dev/null
+++ b/tools/tryselect/selectors/perfselector/perfcomparators.py
@@ -0,0 +1,258 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import importlib
+import inspect
+import pathlib
+
+BUILTIN_COMPARATORS = {}
+
+
+class ComparatorNotFound(Exception):
+    """Raised when we can't find the specified comparator.
+
+    Triggered when either the comparator name is incorrect for a builtin one,
+    or when a path to a specified comparator cannot be found.
+    """
+
+    pass
+
+
+class GithubRequestFailure(Exception):
+    """Raised when we hit a failure during PR link parsing."""
+
+    pass
+
+
+class BadComparatorArgs(Exception):
+    """Raised when the args given to the comparator are incorrect."""
+
+    pass
+
+
+def comparator(comparator_klass):
+    BUILTIN_COMPARATORS[comparator_klass.__name__] = comparator_klass
+    return comparator_klass
+
+
+@comparator
+class BasePerfComparator:
+    def __init__(self, vcs, compare_commit, current_revision_ref, comparator_args):
+        """Initialize the standard/default settings for Comparators.
+
+        :param vcs object: Used for updating the local repo.
+        :param compare_commit str: The base revision found for the local repo.
+        :param current_revision_ref str: The current revision of the local repo.
+        :param comparator_args list: List of comparator args in the format NAME=VALUE.
+        """
+        self.vcs = vcs
+        self.compare_commit = compare_commit
+        self.current_revision_ref = current_revision_ref
+        self.comparator_args = comparator_args
+
+        # Used to ensure that the local repo gets cleaned up appropriately on failures
+        self._updated = False
+
+    def setup_base_revision(self, extra_args):
+        """Setup the base try run/revision.
+
+        In this case, we update to the repo to the base revision and
+        push that to try. The extra_args can be used to set additional
+        arguments for Raptor (not available for other harnesses).
+
+        :param extra_args list: A list of extra arguments to pass to the try tasks.
+        """
+        self.vcs.update(self.compare_commit)
+        self._updated = True
+
+    def teardown_base_revision(self):
+        """Teardown the setup for the base revision."""
+        if self._updated:
+            self.vcs.update(self.current_revision_ref)
+            self._updated = False
+
+    def setup_new_revision(self, extra_args):
+        """Setup the new try run/revision.
+
+        Note that the extra_args are reset between the base, and new revision runs.
+
+        :param extra_args list: A list of extra arguments to pass to the try tasks.
+        """
+        pass
+
+    def teardown_new_revision(self):
+        """Teardown the new run/revision setup."""
+        pass
+
+    def teardown(self):
+        """Teardown for failures.
+
+        This method can be used for ensuring that the repo is cleaned up
+        when a failure is hit at any point in the process of doing the
+        new/base revision setups, or the pushes to try.
+        """
+        self.teardown_base_revision()
+
+
+def get_github_pull_request_info(link):
+    """Returns information about a PR link.
+
+    This method accepts a Github link in either of these formats:
+        https://github.com/mozilla-mobile/firefox-android/pull/1627,
+        https://github.com/mozilla-mobile/firefox-android/pull/1876/commits/17c7350cc37a4a85cea140a7ce54e9fd037b5365 #noqa
+
+    and returns the Github link, branch, and revision of the commit.
+    """
+    from urllib.parse import urlparse
+
+    import requests
+
+    # Parse the url, and get all the necessary info
+    parsed_url = urlparse(link)
+    path_parts = parsed_url.path.strip("/").split("/")
+    owner, repo = path_parts[0], path_parts[1]
+    pr_number = path_parts[-1]
+
+    if "/pull/" not in parsed_url.path:
+        raise GithubRequestFailure(
+            f"Link for Github PR is invalid (missing /pull/): {link}"
+        )
+
+    # Get the commit being targeted in the PR
+    pr_commit = None
+    if "/commits/" in parsed_url.path:
+        pr_commit = path_parts[-1]
+        pr_number = path_parts[-3]
+
+    # Make the request, and get the PR info, otherwise,
+    # raise an exception if the response code is not 200
+    api_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}"
+    response = requests.get(api_url)
+    if response.status_code == 200:
+        link_info = response.json()
+        return (
+            link_info["head"]["repo"]["html_url"],
+            pr_commit if pr_commit else link_info["head"]["sha"],
+            link_info["head"]["ref"],
+        )
+
+    raise GithubRequestFailure(
+        f"The following url returned a non-200 status code: {api_url}"
+    )
+
+
+@comparator
+class BenchmarkComparator(BasePerfComparator):
+    def _get_benchmark_info(self, arg_prefix):
+        # Get the flag from the comparator args
+        benchmark_info = {"repo": None, "branch": None, "revision": None, "link": None}
+        for arg in self.comparator_args:
+            if arg.startswith(arg_prefix):
+                _, settings = arg.split(arg_prefix)
+                setting, val = settings.split("=")
+                if setting not in benchmark_info:
+                    raise BadComparatorArgs(
+                        f"Unknown argument provided `{setting}`. Only the following "
+                        f"are available (prefixed with `{arg_prefix}`): "
+                        f"{list(benchmark_info.keys())}"
+                    )
+                benchmark_info[setting] = val
+
+        # Parse the link for any required information
+        if benchmark_info.get("link", None) is not None:
+            (
+                benchmark_info["repo"],
+                benchmark_info["revision"],
+                benchmark_info["branch"],
+            ) = get_github_pull_request_info(benchmark_info["link"])
+
+        return benchmark_info
+
+    def _setup_benchmark_args(self, extra_args, benchmark_info):
+        # Setup the arguments for Raptor
+        extra_args.append(f"benchmark-repository={benchmark_info['repo']}")
+        extra_args.append(f"benchmark-revision={benchmark_info['revision']}")
+
+        if benchmark_info.get("branch", None):
+            extra_args.append(f"benchmark-branch={benchmark_info['branch']}")
+
+    def setup_base_revision(self, extra_args):
+        """Sets up the options for a base benchmark revision run.
+
+        Checks for a `base-link` in the
+        command and adds the appropriate commands to the extra_args
+        which will be added to the PERF_FLAGS environment variable.
+
+        If that isn't provided, then you must provide the repo, branch,
+        and revision directly through these (branch is optional):
+
+            base-repo=https://github.com/mozilla-mobile/firefox-android
+            base-branch=main
+            base-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365
+
+        Otherwise, we'll use the default mach try perf
+        base behaviour.
+
+        TODO: Get the information automatically from a commit link. Github
+        API doesn't provide the branch name from a link like that.
+        """
+        base_info = self._get_benchmark_info("base-")
+
+        # If no options were provided, use the default BasePerfComparator behaviour
+        if not any(v is not None for v in base_info.values()):
+            raise BadComparatorArgs(
+                f"Could not find the correct base-revision arguments in: {self.comparator_args}"
+            )
+
+        self._setup_benchmark_args(extra_args, base_info)
+
+    def setup_new_revision(self, extra_args):
+        """Sets up the options for a new benchmark revision run.
+
+        Same as `setup_base_revision`, except it uses
+         `new-` as the prefix instead of `base-`.
+        """
+        new_info = self._get_benchmark_info("new-")
+
+        # If no options were provided, use the default BasePerfComparator behaviour
+        if not any(v is not None for v in new_info.values()):
+            raise BadComparatorArgs(
+                f"Could not find the correct new-revision arguments in: {self.comparator_args}"
+            )
+
+        self._setup_benchmark_args(extra_args, new_info)
+
+
+def get_comparator(comparator):
+    if comparator in BUILTIN_COMPARATORS:
+        return BUILTIN_COMPARATORS[comparator]
+
+    file = pathlib.Path(comparator)
+    if not file.exists():
+        raise ComparatorNotFound(
+            f"Expected either a path to a file containing a comparator, or a "
+            f"builtin comparator from this list: {BUILTIN_COMPARATORS.keys()}"
+        )
+
+    # Importing a source file directly
+    spec = importlib.util.spec_from_file_location(name=file.name, location=comparator)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    members = inspect.getmembers(
+        module,
+        lambda c: inspect.isclass(c)
+        and issubclass(c, BasePerfComparator)
+        and c != BasePerfComparator,
+    )
+
+    if not members:
+        raise ComparatorNotFound(
+            f"The path {comparator} was found but it was not a valid comparator. "
+            f"Ensure it is a subclass of BasePerfComparator and optionally contains the  "
+            f"following methods: "
+            f"{', '.join(inspect.getmembers(BasePerfComparator, predicate=inspect.ismethod))}"
+        )
+
+    return members[0][-1]
diff --git a/tools/tryselect/selectors/perfselector/utils.py b/tools/tryselect/selectors/perfselector/utils.py
new file mode 100644
index 0000000000..105d003091
--- /dev/null
+++ b/tools/tryselect/selectors/perfselector/utils.py
@@ -0,0 +1,44 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+import sys
+
+REVISION_MATCHER = re.compile(r"remote:.*/try/rev/([\w]*)[ \t]*$")
+
+
+class LogProcessor:
+    def __init__(self):
+        self.buf = ""
+        self.stdout = sys.__stdout__
+        self._revision = None
+
+    @property
+    def revision(self):
+        return self._revision
+
+    def write(self, buf):
+        while buf:
+            try:
+                newline_index = buf.index("\n")
+            except ValueError:
+                # No newline, wait for next call
+                self.buf += buf
+                break
+
+            # Get data up to next newline and combine with previously buffered data
+            data = self.buf + buf[: newline_index + 1]
+            buf = buf[newline_index + 1 :]
+
+            # Reset buffer then output line
+            self.buf = ""
+            if data.strip() == "":
+                continue
+            self.stdout.write(data.strip("\n") + "\n")
+
+            # Check if a temporary commit wa created
+            match = REVISION_MATCHER.match(data)
+            if match:
+                # Last line found is the revision we want
+                self._revision = match.group(1)