diff options
Diffstat (limited to 'tools/tryselect/selectors/perfselector')
4 files changed, 692 insertions, 0 deletions
diff --git a/tools/tryselect/selectors/perfselector/__init__.py b/tools/tryselect/selectors/perfselector/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/selectors/perfselector/classification.py b/tools/tryselect/selectors/perfselector/classification.py new file mode 100644 index 0000000000..cabf2a323e --- /dev/null +++ b/tools/tryselect/selectors/perfselector/classification.py @@ -0,0 +1,387 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import enum + + +class ClassificationEnum(enum.Enum): + """This class provides the ability to use Enums as array indices.""" + + @property + def value(self): + return self._value_["value"] + + def __index__(self): + return self._value_["index"] + + def __int__(self): + return self._value_["index"] + + +class Platforms(ClassificationEnum): + ANDROID_A51 = {"value": "android-a51", "index": 0} + ANDROID = {"value": "android", "index": 1} + WINDOWS = {"value": "windows", "index": 2} + LINUX = {"value": "linux", "index": 3} + MACOSX = {"value": "macosx", "index": 4} + DESKTOP = {"value": "desktop", "index": 5} + + +class Apps(ClassificationEnum): + FIREFOX = {"value": "firefox", "index": 0} + CHROME = {"value": "chrome", "index": 1} + CHROMIUM = {"value": "chromium", "index": 2} + GECKOVIEW = {"value": "geckoview", "index": 3} + FENIX = {"value": "fenix", "index": 4} + CHROME_M = {"value": "chrome-m", "index": 5} + SAFARI = {"value": "safari", "index": 6} + CHROMIUM_RELEASE = {"value": "custom-car", "index": 7} + CHROMIUM_RELEASE_M = {"value": "cstm-car-m", "index": 8} + + +class Suites(ClassificationEnum): + RAPTOR = {"value": "raptor", "index": 0} + TALOS = {"value": "talos", "index": 1} + AWSY = {"value": "awsy", "index": 2} + + +class Variants(ClassificationEnum): + FISSION = {"value": "fission", "index": 0} + BYTECODE_CACHED = {"value": "bytecode-cached", "index": 1} + LIVE_SITES = {"value": "live-sites", "index": 2} + PROFILING = {"value": "profiling", "index": 3} + SWR = {"value": "swr", "index": 4} + + +""" +The following methods and constants are used for restricting +certain platforms and applications such as chrome, safari, and +android tests. These all require a flag such as --android to +enable (see build_category_matrix for more info). +""" + + +def check_for_android(android=False, **kwargs): + return android + + +def check_for_fenix(fenix=False, **kwargs): + return fenix or ("fenix" in kwargs.get("requested_apps", [])) + + +def check_for_chrome(chrome=False, **kwargs): + return chrome + + +def check_for_custom_car(custom_car=False, **kwargs): + return custom_car + + +def check_for_safari(safari=False, **kwargs): + return safari + + +def check_for_live_sites(live_sites=False, **kwargs): + return live_sites + + +def check_for_profile(profile=False, **kwargs): + return profile + + +class ClassificationProvider: + @property + def platforms(self): + return { + Platforms.ANDROID_A51.value: { + "query": "'android 'a51 'shippable 'aarch64", + "restriction": check_for_android, + "platform": Platforms.ANDROID.value, + }, + Platforms.ANDROID.value: { + # The android, and android-a51 queries are expected to be the same, + # we don't want to run the tests on other mobile platforms. + "query": "'android 'a51 'shippable 'aarch64", + "restriction": check_for_android, + "platform": Platforms.ANDROID.value, + }, + Platforms.WINDOWS.value: { + "query": "!-32 'windows 'shippable", + "platform": Platforms.DESKTOP.value, + }, + Platforms.LINUX.value: { + "query": "!clang 'linux 'shippable", + "platform": Platforms.DESKTOP.value, + }, + Platforms.MACOSX.value: { + "query": "'osx 'shippable", + "platform": Platforms.DESKTOP.value, + }, + Platforms.DESKTOP.value: { + "query": "!android 'shippable !-32 !clang", + "platform": Platforms.DESKTOP.value, + }, + } + + @property + def apps(self): + return { + Apps.FIREFOX.value: { + "query": "!chrom !geckoview !fenix !safari !m-car", + "platforms": [Platforms.DESKTOP.value], + }, + Apps.CHROME.value: { + "query": "'chrome", + "negation": "!chrom", + "restriction": check_for_chrome, + "platforms": [Platforms.DESKTOP.value], + }, + Apps.CHROMIUM.value: { + "query": "'chromium", + "negation": "!chrom", + "restriction": check_for_chrome, + "platforms": [Platforms.DESKTOP.value], + }, + Apps.GECKOVIEW.value: { + "query": "'geckoview", + "negation": "!geckoview", + "platforms": [Platforms.ANDROID.value], + }, + Apps.FENIX.value: { + "query": "'fenix", + "negation": "!fenix", + "restriction": check_for_fenix, + "platforms": [Platforms.ANDROID.value], + }, + Apps.CHROME_M.value: { + "query": "'chrome-m", + "negation": "!chrom", + "restriction": check_for_chrome, + "platforms": [Platforms.ANDROID.value], + }, + Apps.SAFARI.value: { + "query": "'safari", + "negation": "!safari", + "restriction": check_for_safari, + "platforms": [Platforms.MACOSX.value], + }, + Apps.CHROMIUM_RELEASE.value: { + "query": "'m-car", + "negation": "!m-car", + "restriction": check_for_custom_car, + "platforms": [ + Platforms.LINUX.value, + Platforms.WINDOWS.value, + Platforms.MACOSX.value, + ], + }, + Apps.CHROMIUM_RELEASE_M.value: { + "query": "'m-car", + "negation": "!m-car", + "restriction": check_for_custom_car, + "platforms": [Platforms.ANDROID.value], + }, + } + + @property + def variants(self): + return { + Variants.FISSION.value: { + "query": "!nofis", + "negation": "'nofis", + "platforms": [Platforms.ANDROID.value], + "apps": [Apps.FENIX.value, Apps.GECKOVIEW.value], + }, + Variants.BYTECODE_CACHED.value: { + "query": "'bytecode", + "negation": "!bytecode", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, + Variants.LIVE_SITES.value: { + "query": "'live", + "negation": "!live", + "restriction": check_for_live_sites, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": [ # XXX No live CaR tests + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + Apps.SAFARI.value, + ], + }, + Variants.PROFILING.value: { + "query": "'profil", + "negation": "!profil", + "restriction": check_for_profile, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": [Apps.FIREFOX.value, Apps.GECKOVIEW.value, Apps.FENIX.value], + }, + Variants.SWR.value: { + "query": "'swr", + "negation": "!swr", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, + } + + @property + def suites(self): + return { + Suites.RAPTOR.value: { + "apps": list(self.apps.keys()), + "platforms": list(self.platforms.keys()), + "variants": [ + Variants.FISSION.value, + Variants.LIVE_SITES.value, + Variants.PROFILING.value, + Variants.BYTECODE_CACHED.value, + ], + }, + Suites.TALOS.value: { + "apps": [Apps.FIREFOX.value], + "platforms": [Platforms.DESKTOP.value], + "variants": [ + Variants.PROFILING.value, + Variants.SWR.value, + ], + }, + Suites.AWSY.value: { + "apps": [Apps.FIREFOX.value], + "platforms": [Platforms.DESKTOP.value], + "variants": [], + }, + } + + """ + Here you can find the base categories that are defined for the perf + selector. The following fields are available: + * query: Set the queries to use for each suite you need. + * suites: The suites that are needed for this category. + * tasks: A hard-coded list of tasks to select. + * platforms: The platforms that it can run on. + * app-restrictions: A list of apps that the category can run. + * variant-restrictions: A list of variants available for each suite. + + Note that setting the App/Variant-Restriction fields should be used to + restrict the available apps and variants, not expand them. + """ + + @property + def categories(self): + return { + "Pageload": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'tp6 !tp6-bench"], + }, + "suites": [Suites.RAPTOR.value], + "tasks": [], + "description": "A group of tests that measures various important pageload metrics. More information " + "can about what is exactly measured can found here:" + " https://firefox-source-docs.mozilla.org/testing/perfdocs/raptor.html#desktop", + }, + "Speedometer 3": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'speedometer3"], + }, + "variant-restrictions": {Suites.RAPTOR.value: [Variants.FISSION.value]}, + "suites": [Suites.RAPTOR.value], + "app-restrictions": {}, + "tasks": [], + "description": "A group of Speedometer3 tests on various platforms and architectures, speedometer3 is" + "currently the best benchmark we have for a baseline on real-world web performance", + }, + "Responsiveness": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'responsive"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "app-restrictions": { + Suites.RAPTOR.value: [ + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + ], + }, + "tasks": [], + "description": "A group of tests that ensure that the interactive part of the browser stays fast and" + "responsive", + }, + "Benchmarks": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'benchmark !tp6-bench"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "tasks": [], + "description": "A group of tests that benchmark how the browser performs in various categories. " + "More information about what exact benchmarks we run can be found here: " + "https://firefox-source-docs.mozilla.org/testing/perfdocs/raptor.html#benchmarks", + }, + "DAMP (Devtools)": { + "query": { + Suites.TALOS.value: ["'talos 'damp"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + "description": "The DAMP tests are a group of tests that measure the performance of the browsers " + "devtools under certain conditiones. More information on the DAMP tests can be found" + " here: https://firefox-source-docs.mozilla.org/devtools/tests/performance-tests" + "-damp.html#what-does-it-do", + }, + "Talos PerfTests": { + "query": { + Suites.TALOS.value: ["'talos"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + "description": "This selects all of the talos performance tests. More information can be found here: " + "https://firefox-source-docs.mozilla.org/testing/perfdocs/talos.html#test-types", + }, + "Resource Usage": { + "query": { + Suites.TALOS.value: ["'talos 'xperf | 'tp5"], + Suites.RAPTOR.value: ["'power 'osx"], + Suites.AWSY.value: ["'awsy"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value, Suites.AWSY.value], + "platform-restrictions": [Platforms.DESKTOP.value], + "variant-restrictions": { + Suites.RAPTOR.value: [], + Suites.TALOS.value: [], + }, + "app-restrictions": { + Suites.RAPTOR.value: [Apps.FIREFOX.value], + Suites.TALOS.value: [Apps.FIREFOX.value], + }, + "tasks": [], + "description": "A group of tests that monitor resource usage of various metrics like power, CPU, and" + "memory", + }, + "Graphics, & Media Playback": { + "query": { + # XXX This might not be an exhaustive list for talos atm + Suites.TALOS.value: ["'talos 'svgr | 'bcv | 'webgl"], + Suites.RAPTOR.value: ["'browsertime 'youtube-playback"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: [Variants.FISSION.value]}, + "app-restrictions": { + Suites.RAPTOR.value: [ + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + ], + }, + "tasks": [], + "description": "A group of tests that monitor key graphics and media metrics to keep the browser fast", + }, + } diff --git a/tools/tryselect/selectors/perfselector/perfcomparators.py b/tools/tryselect/selectors/perfselector/perfcomparators.py new file mode 100644 index 0000000000..fce35fe562 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/perfcomparators.py @@ -0,0 +1,258 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import importlib +import inspect +import pathlib + +BUILTIN_COMPARATORS = {} + + +class ComparatorNotFound(Exception): + """Raised when we can't find the specified comparator. + + Triggered when either the comparator name is incorrect for a builtin one, + or when a path to a specified comparator cannot be found. + """ + + pass + + +class GithubRequestFailure(Exception): + """Raised when we hit a failure during PR link parsing.""" + + pass + + +class BadComparatorArgs(Exception): + """Raised when the args given to the comparator are incorrect.""" + + pass + + +def comparator(comparator_klass): + BUILTIN_COMPARATORS[comparator_klass.__name__] = comparator_klass + return comparator_klass + + +@comparator +class BasePerfComparator: + def __init__(self, vcs, compare_commit, current_revision_ref, comparator_args): + """Initialize the standard/default settings for Comparators. + + :param vcs object: Used for updating the local repo. + :param compare_commit str: The base revision found for the local repo. + :param current_revision_ref str: The current revision of the local repo. + :param comparator_args list: List of comparator args in the format NAME=VALUE. + """ + self.vcs = vcs + self.compare_commit = compare_commit + self.current_revision_ref = current_revision_ref + self.comparator_args = comparator_args + + # Used to ensure that the local repo gets cleaned up appropriately on failures + self._updated = False + + def setup_base_revision(self, extra_args): + """Setup the base try run/revision. + + In this case, we update to the repo to the base revision and + push that to try. The extra_args can be used to set additional + arguments for Raptor (not available for other harnesses). + + :param extra_args list: A list of extra arguments to pass to the try tasks. + """ + self.vcs.update(self.compare_commit) + self._updated = True + + def teardown_base_revision(self): + """Teardown the setup for the base revision.""" + if self._updated: + self.vcs.update(self.current_revision_ref) + self._updated = False + + def setup_new_revision(self, extra_args): + """Setup the new try run/revision. + + Note that the extra_args are reset between the base, and new revision runs. + + :param extra_args list: A list of extra arguments to pass to the try tasks. + """ + pass + + def teardown_new_revision(self): + """Teardown the new run/revision setup.""" + pass + + def teardown(self): + """Teardown for failures. + + This method can be used for ensuring that the repo is cleaned up + when a failure is hit at any point in the process of doing the + new/base revision setups, or the pushes to try. + """ + self.teardown_base_revision() + + +def get_github_pull_request_info(link): + """Returns information about a PR link. + + This method accepts a Github link in either of these formats: + https://github.com/mozilla-mobile/firefox-android/pull/1627, + https://github.com/mozilla-mobile/firefox-android/pull/1876/commits/17c7350cc37a4a85cea140a7ce54e9fd037b5365 #noqa + + and returns the Github link, branch, and revision of the commit. + """ + from urllib.parse import urlparse + + import requests + + # Parse the url, and get all the necessary info + parsed_url = urlparse(link) + path_parts = parsed_url.path.strip("/").split("/") + owner, repo = path_parts[0], path_parts[1] + pr_number = path_parts[-1] + + if "/pull/" not in parsed_url.path: + raise GithubRequestFailure( + f"Link for Github PR is invalid (missing /pull/): {link}" + ) + + # Get the commit being targeted in the PR + pr_commit = None + if "/commits/" in parsed_url.path: + pr_commit = path_parts[-1] + pr_number = path_parts[-3] + + # Make the request, and get the PR info, otherwise, + # raise an exception if the response code is not 200 + api_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}" + response = requests.get(api_url) + if response.status_code == 200: + link_info = response.json() + return ( + link_info["head"]["repo"]["html_url"], + pr_commit if pr_commit else link_info["head"]["sha"], + link_info["head"]["ref"], + ) + + raise GithubRequestFailure( + f"The following url returned a non-200 status code: {api_url}" + ) + + +@comparator +class BenchmarkComparator(BasePerfComparator): + def _get_benchmark_info(self, arg_prefix): + # Get the flag from the comparator args + benchmark_info = {"repo": None, "branch": None, "revision": None, "link": None} + for arg in self.comparator_args: + if arg.startswith(arg_prefix): + _, settings = arg.split(arg_prefix) + setting, val = settings.split("=") + if setting not in benchmark_info: + raise BadComparatorArgs( + f"Unknown argument provided `{setting}`. Only the following " + f"are available (prefixed with `{arg_prefix}`): " + f"{list(benchmark_info.keys())}" + ) + benchmark_info[setting] = val + + # Parse the link for any required information + if benchmark_info.get("link", None) is not None: + ( + benchmark_info["repo"], + benchmark_info["revision"], + benchmark_info["branch"], + ) = get_github_pull_request_info(benchmark_info["link"]) + + return benchmark_info + + def _setup_benchmark_args(self, extra_args, benchmark_info): + # Setup the arguments for Raptor + extra_args.append(f"benchmark-repository={benchmark_info['repo']}") + extra_args.append(f"benchmark-revision={benchmark_info['revision']}") + + if benchmark_info.get("branch", None): + extra_args.append(f"benchmark-branch={benchmark_info['branch']}") + + def setup_base_revision(self, extra_args): + """Sets up the options for a base benchmark revision run. + + Checks for a `base-link` in the + command and adds the appropriate commands to the extra_args + which will be added to the PERF_FLAGS environment variable. + + If that isn't provided, then you must provide the repo, branch, + and revision directly through these (branch is optional): + + base-repo=https://github.com/mozilla-mobile/firefox-android + base-branch=main + base-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365 + + Otherwise, we'll use the default mach try perf + base behaviour. + + TODO: Get the information automatically from a commit link. Github + API doesn't provide the branch name from a link like that. + """ + base_info = self._get_benchmark_info("base-") + + # If no options were provided, use the default BasePerfComparator behaviour + if not any(v is not None for v in base_info.values()): + raise BadComparatorArgs( + f"Could not find the correct base-revision arguments in: {self.comparator_args}" + ) + + self._setup_benchmark_args(extra_args, base_info) + + def setup_new_revision(self, extra_args): + """Sets up the options for a new benchmark revision run. + + Same as `setup_base_revision`, except it uses + `new-` as the prefix instead of `base-`. + """ + new_info = self._get_benchmark_info("new-") + + # If no options were provided, use the default BasePerfComparator behaviour + if not any(v is not None for v in new_info.values()): + raise BadComparatorArgs( + f"Could not find the correct new-revision arguments in: {self.comparator_args}" + ) + + self._setup_benchmark_args(extra_args, new_info) + + +def get_comparator(comparator): + if comparator in BUILTIN_COMPARATORS: + return BUILTIN_COMPARATORS[comparator] + + file = pathlib.Path(comparator) + if not file.exists(): + raise ComparatorNotFound( + f"Expected either a path to a file containing a comparator, or a " + f"builtin comparator from this list: {BUILTIN_COMPARATORS.keys()}" + ) + + # Importing a source file directly + spec = importlib.util.spec_from_file_location(name=file.name, location=comparator) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + members = inspect.getmembers( + module, + lambda c: inspect.isclass(c) + and issubclass(c, BasePerfComparator) + and c != BasePerfComparator, + ) + + if not members: + raise ComparatorNotFound( + f"The path {comparator} was found but it was not a valid comparator. " + f"Ensure it is a subclass of BasePerfComparator and optionally contains the " + f"following methods: " + f"{', '.join(inspect.getmembers(BasePerfComparator, predicate=inspect.ismethod))}" + ) + + return members[0][-1] diff --git a/tools/tryselect/selectors/perfselector/utils.py b/tools/tryselect/selectors/perfselector/utils.py new file mode 100644 index 0000000000..105d003091 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/utils.py @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re +import sys + +REVISION_MATCHER = re.compile(r"remote:.*/try/rev/([\w]*)[ \t]*$") + + +class LogProcessor: + def __init__(self): + self.buf = "" + self.stdout = sys.__stdout__ + self._revision = None + + @property + def revision(self): + return self._revision + + def write(self, buf): + while buf: + try: + newline_index = buf.index("\n") + except ValueError: + # No newline, wait for next call + self.buf += buf + break + + # Get data up to next newline and combine with previously buffered data + data = self.buf + buf[: newline_index + 1] + buf = buf[newline_index + 1 :] + + # Reset buffer then output line + self.buf = "" + if data.strip() == "": + continue + self.stdout.write(data.strip("\n") + "\n") + + # Check if a temporary commit wa created + match = REVISION_MATCHER.match(data) + if match: + # Last line found is the revision we want + self._revision = match.group(1) |