258 lines
9.1 KiB
Python
258 lines
9.1 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
import importlib
|
|
import inspect
|
|
import pathlib
|
|
|
|
BUILTIN_COMPARATORS = {}
|
|
|
|
|
|
class ComparatorNotFound(Exception):
|
|
"""Raised when we can't find the specified comparator.
|
|
|
|
Triggered when either the comparator name is incorrect for a builtin one,
|
|
or when a path to a specified comparator cannot be found.
|
|
"""
|
|
|
|
pass
|
|
|
|
|
|
class GithubRequestFailure(Exception):
|
|
"""Raised when we hit a failure during PR link parsing."""
|
|
|
|
pass
|
|
|
|
|
|
class BadComparatorArgs(Exception):
|
|
"""Raised when the args given to the comparator are incorrect."""
|
|
|
|
pass
|
|
|
|
|
|
def comparator(comparator_klass):
|
|
BUILTIN_COMPARATORS[comparator_klass.__name__] = comparator_klass
|
|
return comparator_klass
|
|
|
|
|
|
@comparator
|
|
class BasePerfComparator:
|
|
def __init__(self, vcs, compare_commit, current_revision_ref, comparator_args):
|
|
"""Initialize the standard/default settings for Comparators.
|
|
|
|
:param vcs object: Used for updating the local repo.
|
|
:param compare_commit str: The base revision found for the local repo.
|
|
:param current_revision_ref str: The current revision of the local repo.
|
|
:param comparator_args list: List of comparator args in the format NAME=VALUE.
|
|
"""
|
|
self.vcs = vcs
|
|
self.compare_commit = compare_commit
|
|
self.current_revision_ref = current_revision_ref
|
|
self.comparator_args = comparator_args
|
|
|
|
# Used to ensure that the local repo gets cleaned up appropriately on failures
|
|
self._updated = False
|
|
|
|
def setup_base_revision(self, extra_args):
|
|
"""Setup the base try run/revision.
|
|
|
|
In this case, we update to the repo to the base revision and
|
|
push that to try. The extra_args can be used to set additional
|
|
arguments for Raptor (not available for other harnesses).
|
|
|
|
:param extra_args list: A list of extra arguments to pass to the try tasks.
|
|
"""
|
|
self.vcs.update(self.compare_commit)
|
|
self._updated = True
|
|
|
|
def teardown_base_revision(self):
|
|
"""Teardown the setup for the base revision."""
|
|
if self._updated:
|
|
self.vcs.update(self.current_revision_ref)
|
|
self._updated = False
|
|
|
|
def setup_new_revision(self, extra_args):
|
|
"""Setup the new try run/revision.
|
|
|
|
Note that the extra_args are reset between the base, and new revision runs.
|
|
|
|
:param extra_args list: A list of extra arguments to pass to the try tasks.
|
|
"""
|
|
pass
|
|
|
|
def teardown_new_revision(self):
|
|
"""Teardown the new run/revision setup."""
|
|
pass
|
|
|
|
def teardown(self):
|
|
"""Teardown for failures.
|
|
|
|
This method can be used for ensuring that the repo is cleaned up
|
|
when a failure is hit at any point in the process of doing the
|
|
new/base revision setups, or the pushes to try.
|
|
"""
|
|
self.teardown_base_revision()
|
|
|
|
|
|
def get_github_pull_request_info(link):
|
|
"""Returns information about a PR link.
|
|
|
|
This method accepts a Github link in either of these formats:
|
|
https://github.com/mozilla-mobile/firefox-android/pull/1627,
|
|
https://github.com/mozilla-mobile/firefox-android/pull/1876/commits/17c7350cc37a4a85cea140a7ce54e9fd037b5365 #noqa
|
|
|
|
and returns the Github link, branch, and revision of the commit.
|
|
"""
|
|
from urllib.parse import urlparse
|
|
|
|
import requests
|
|
|
|
# Parse the url, and get all the necessary info
|
|
parsed_url = urlparse(link)
|
|
path_parts = parsed_url.path.strip("/").split("/")
|
|
owner, repo = path_parts[0], path_parts[1]
|
|
pr_number = path_parts[-1]
|
|
|
|
if "/pull/" not in parsed_url.path:
|
|
raise GithubRequestFailure(
|
|
f"Link for Github PR is invalid (missing /pull/): {link}"
|
|
)
|
|
|
|
# Get the commit being targeted in the PR
|
|
pr_commit = None
|
|
if "/commits/" in parsed_url.path:
|
|
pr_commit = path_parts[-1]
|
|
pr_number = path_parts[-3]
|
|
|
|
# Make the request, and get the PR info, otherwise,
|
|
# raise an exception if the response code is not 200
|
|
api_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}"
|
|
response = requests.get(api_url)
|
|
if response.status_code == 200:
|
|
link_info = response.json()
|
|
return (
|
|
link_info["head"]["repo"]["html_url"],
|
|
pr_commit if pr_commit else link_info["head"]["sha"],
|
|
link_info["head"]["ref"],
|
|
)
|
|
|
|
raise GithubRequestFailure(
|
|
f"The following url returned a non-200 status code: {api_url}"
|
|
)
|
|
|
|
|
|
@comparator
|
|
class BenchmarkComparator(BasePerfComparator):
|
|
def _get_benchmark_info(self, arg_prefix):
|
|
# Get the flag from the comparator args
|
|
benchmark_info = {"repo": None, "branch": None, "revision": None, "link": None}
|
|
for arg in self.comparator_args:
|
|
if arg.startswith(arg_prefix):
|
|
_, settings = arg.split(arg_prefix)
|
|
setting, val = settings.split("=")
|
|
if setting not in benchmark_info:
|
|
raise BadComparatorArgs(
|
|
f"Unknown argument provided `{setting}`. Only the following "
|
|
f"are available (prefixed with `{arg_prefix}`): "
|
|
f"{list(benchmark_info.keys())}"
|
|
)
|
|
benchmark_info[setting] = val
|
|
|
|
# Parse the link for any required information
|
|
if benchmark_info.get("link", None) is not None:
|
|
(
|
|
benchmark_info["repo"],
|
|
benchmark_info["revision"],
|
|
benchmark_info["branch"],
|
|
) = get_github_pull_request_info(benchmark_info["link"])
|
|
|
|
return benchmark_info
|
|
|
|
def _setup_benchmark_args(self, extra_args, benchmark_info):
|
|
# Setup the arguments for Raptor
|
|
extra_args.append(f"benchmark-repository={benchmark_info['repo']}")
|
|
extra_args.append(f"benchmark-revision={benchmark_info['revision']}")
|
|
|
|
if benchmark_info.get("branch", None):
|
|
extra_args.append(f"benchmark-branch={benchmark_info['branch']}")
|
|
|
|
def setup_base_revision(self, extra_args):
|
|
"""Sets up the options for a base benchmark revision run.
|
|
|
|
Checks for a `base-link` in the
|
|
command and adds the appropriate commands to the extra_args
|
|
which will be added to the PERF_FLAGS environment variable.
|
|
|
|
If that isn't provided, then you must provide the repo, branch,
|
|
and revision directly through these (branch is optional):
|
|
|
|
base-repo=https://github.com/mozilla-mobile/firefox-android
|
|
base-branch=main
|
|
base-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365
|
|
|
|
Otherwise, we'll use the default mach try perf
|
|
base behaviour.
|
|
|
|
TODO: Get the information automatically from a commit link. Github
|
|
API doesn't provide the branch name from a link like that.
|
|
"""
|
|
base_info = self._get_benchmark_info("base-")
|
|
|
|
# If no options were provided, use the default BasePerfComparator behaviour
|
|
if not any(v is not None for v in base_info.values()):
|
|
raise BadComparatorArgs(
|
|
f"Could not find the correct base-revision arguments in: {self.comparator_args}"
|
|
)
|
|
|
|
self._setup_benchmark_args(extra_args, base_info)
|
|
|
|
def setup_new_revision(self, extra_args):
|
|
"""Sets up the options for a new benchmark revision run.
|
|
|
|
Same as `setup_base_revision`, except it uses
|
|
`new-` as the prefix instead of `base-`.
|
|
"""
|
|
new_info = self._get_benchmark_info("new-")
|
|
|
|
# If no options were provided, use the default BasePerfComparator behaviour
|
|
if not any(v is not None for v in new_info.values()):
|
|
raise BadComparatorArgs(
|
|
f"Could not find the correct new-revision arguments in: {self.comparator_args}"
|
|
)
|
|
|
|
self._setup_benchmark_args(extra_args, new_info)
|
|
|
|
|
|
def get_comparator(comparator):
|
|
if comparator in BUILTIN_COMPARATORS:
|
|
return BUILTIN_COMPARATORS[comparator]
|
|
|
|
file = pathlib.Path(comparator)
|
|
if not file.exists():
|
|
raise ComparatorNotFound(
|
|
f"Expected either a path to a file containing a comparator, or a "
|
|
f"builtin comparator from this list: {BUILTIN_COMPARATORS.keys()}"
|
|
)
|
|
|
|
# Importing a source file directly
|
|
spec = importlib.util.spec_from_file_location(name=file.name, location=comparator)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
|
|
members = inspect.getmembers(
|
|
module,
|
|
lambda c: inspect.isclass(c)
|
|
and issubclass(c, BasePerfComparator)
|
|
and c != BasePerfComparator,
|
|
)
|
|
|
|
if not members:
|
|
raise ComparatorNotFound(
|
|
f"The path {comparator} was found but it was not a valid comparator. "
|
|
f"Ensure it is a subclass of BasePerfComparator and optionally contains the "
|
|
f"following methods: "
|
|
f"{', '.join(inspect.getmembers(BasePerfComparator, predicate=inspect.ismethod))}"
|
|
)
|
|
|
|
return members[0][-1]
|