# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import copy import os from argparse import ArgumentParser, Namespace import mozlog here = os.path.abspath(os.path.dirname(__file__)) try: from mozbuild.base import MachCommandConditions as conditions from mozbuild.base import MozbuildObject build_obj = MozbuildObject.from_environment(cwd=here) except Exception: build_obj = None conditions = None from mozperftest.metrics import get_layers as metrics_layers # noqa from mozperftest.system import get_layers as system_layers # noqa from mozperftest.test import get_layers as test_layers # noqa from mozperftest.utils import convert_day # noqa FLAVORS = "desktop-browser", "mobile-browser", "doc", "xpcshell", "webpagetest" class Options: general_args = { "--flavor": { "choices": FLAVORS, "metavar": "{{{}}}".format(", ".join(FLAVORS)), "default": "desktop-browser", "help": "Only run tests of this flavor.", }, "tests": { "nargs": "*", "metavar": "TEST", "default": [], "help": "Test to run. Can be a single test file or URL or a directory" " of tests (to run recursively). If omitted, the entire suite is run.", }, "--test-iterations": { "type": int, "default": 1, "help": "Number of times the whole test is executed", }, "--output": { "type": str, "default": "artifacts", "help": "Path to where data will be stored, defaults to a top-level " "`artifacts` folder.", }, "--hooks": { "type": str, "default": None, "help": "Script containing hooks. Can be a path or a URL.", }, "--verbose": {"action": "store_true", "default": False, "help": "Verbose mode"}, "--push-to-try": { "action": "store_true", "default": False, "help": "Pushin the test to try", }, "--try-platform": { "nargs": "*", "type": str, "default": "linux", "help": "Platform to use on try", "choices": ["g5", "pixel2", "linux", "mac", "win"], }, "--on-try": { "action": "store_true", "default": False, "help": "Running the test on try", }, "--test-date": { "type": convert_day, "default": "today", "help": "Used in multi-commit testing, it specifies the day to get test builds from. " "Must follow the format `YYYY.MM.DD` or be `today` or `yesterday`.", }, } args = copy.deepcopy(general_args) for layer in system_layers() + test_layers() + metrics_layers(): if layer.activated: # add an option to deactivate it option_name = "--no-%s" % layer.name option_help = "Deactivates the %s layer" % layer.name else: option_name = "--%s" % layer.name option_help = "Activates the %s layer" % layer.name Options.args[option_name] = { "action": "store_true", "default": False, "help": option_help, } for option, value in layer.arguments.items(): option = "--%s-%s" % (layer.name, option.replace("_", "-")) if option in Options.args: raise KeyError("%s option already defined!" % option) Options.args[option] = value class PerftestArgumentParser(ArgumentParser): """%(prog)s [options] [test paths]""" def __init__(self, app=None, **kwargs): ArgumentParser.__init__( self, usage=self.__doc__, conflict_handler="resolve", **kwargs ) # XXX see if this list will vary depending on the flavor & app self.oldcwd = os.getcwd() self.app = app if not self.app and build_obj: if conditions.is_android(build_obj): self.app = "android" if not self.app: self.app = "generic" for name, options in Options.args.items(): self.add_argument(name, **options) mozlog.commandline.add_logging_group(self) self.set_by_user = [] def parse_helper(self, args): for arg in args: arg_part = arg.partition("--")[-1].partition("-") layer_name = f"--{arg_part[0]}" layer_exists = arg_part[1] and layer_name in Options.args if layer_exists: args.append(layer_name) def get_user_args(self, args): # suppress args that were not provided by the user. res = {} for key, value in args.items(): if key not in self.set_by_user: continue res[key] = value return res def _parse_known_args(self, arg_strings, namespace): # at this point, the namespace is filled with default values # defined in the args # let's parse what the user really gave us in the CLI # in a new namespace user_namespace, extras = super()._parse_known_args(arg_strings, Namespace()) self.set_by_user = list([name for name, value in user_namespace._get_kwargs()]) # we can now merge both for key, value in user_namespace._get_kwargs(): setattr(namespace, key, value) return namespace, extras def parse_args(self, args=None, namespace=None): self.parse_helper(args) return super().parse_args(args, namespace) def parse_known_args(self, args=None, namespace=None): self.parse_helper(args) return super().parse_known_args(args, namespace) class SideBySideOptions: args = [ [ ["-t", "--test-name"], { "type": str, "required": True, "dest": "test_name", "help": "The name of the test task to get videos from.", }, ], [ ["--new-test-name"], { "type": str, "default": None, "help": "The name of the test task to get videos from in the new revision.", }, ], [ ["--base-revision"], { "type": str, "required": True, "help": "The base revision to compare a new revision to.", }, ], [ ["--new-revision"], { "type": str, "required": True, "help": "The base revision to compare a new revision to.", }, ], [ ["--base-branch"], { "type": str, "default": "autoland", "help": "Branch to search for the base revision.", }, ], [ ["--new-branch"], { "type": str, "default": "autoland", "help": "Branch to search for the new revision.", }, ], [ ["--base-platform"], { "type": str, "required": True, "dest": "platform", "help": "Platform to return results for.", }, ], [ ["--new-platform"], { "type": str, "default": None, "help": "Platform to return results for in the new revision.", }, ], [ ["-o", "--overwrite"], { "action": "store_true", "default": False, "help": "If set, the downloaded task group data will be deleted before " + "it gets re-downloaded.", }, ], [ ["--cold"], { "action": "store_true", "default": False, "help": "If set, we'll only look at cold pageload tests.", }, ], [ ["--warm"], { "action": "store_true", "default": False, "help": "If set, we'll only look at warm pageload tests.", }, ], [ ["--most-similar"], { "action": "store_true", "default": False, "help": "If set, we'll search for a video pairing that is the most similar.", }, ], [ ["--search-crons"], { "action": "store_true", "default": False, "help": "If set, we will search for the tasks within the cron jobs as well. ", }, ], [ ["--skip-download"], { "action": "store_true", "default": False, "help": "If set, we won't try to download artifacts again and we'll " + "try using what already exists in the output folder.", }, ], [ ["--output"], { "type": str, "default": None, "help": "This is where the data will be saved. Defaults to CWD. " + "You can include a name for the file here, otherwise it will " + "default to side-by-side.mp4.", }, ], [ ["--metric"], { "type": str, "default": "speedindex", "help": "Metric to use for side-by-side comparison.", }, ], [ ["--vismetPath"], { "type": str, "default": False, "help": "Paths to visualmetrics.py for step chart generation.", }, ], [ ["--original"], { "action": "store_true", "default": False, "help": "If set, use the original videos in the side-by-side instead " + "of the postprocessed videos.", }, ], [ ["--skip-slow-gif"], { "action": "store_true", "default": False, "help": "If set, the slow-motion GIFs won't be produced.", }, ], ] class ChangeDetectorOptions: args = [ # TODO: Move the common tool arguments to a common # argument class. [ ["--task-name"], { "type": str, "nargs": "*", "default": [], "dest": "task_names", "help": "The full name of the test task to get data from e.g. " "test-android-hw-a51-11-0-aarch64-shippable-qr/opt-" "browsertime-tp6m-geckoview-sina-nofis.", }, ], [ ["-t", "--test-name"], { "type": str, "default": None, "dest": "test_name", "help": "The name of the test task to get data from e.g. " "browsertime-tp6m-geckoview-sina-nofis.", }, ], [ ["--platform"], { "type": str, "default": None, "help": "Platform to analyze e.g. " "test-android-hw-a51-11-0-aarch64-shippable-qr/opt.", }, ], [ ["--new-test-name"], { "type": str, "help": "The name of the test task to get data from in the " "base revision e.g. browsertime-tp6m-geckoview-sina-nofis.", }, ], [ ["--new-platform"], { "type": str, "help": "Platform to analyze in base revision e.g. " "test-android-hw-a51-11-0-aarch64-shippable-qr/opt.", }, ], [ ["--depth"], { "type": int, "default": None, "help": "This sets how the change detector should run. " "Default is None, which is a direct comparison between the " "revisions. -1 will autocompute the number of revisions to " "look at between the base, and new. Any other positive integer " "acts as a maximum number to look at.", }, ], [ ["--base-revision"], { "type": str, "required": True, "help": "The base revision to compare a new revision to.", }, ], [ ["--new-revision"], { "type": str, "required": True, "help": "The new revision to compare a base revision to.", }, ], [ ["--base-branch"], { "type": str, "default": "try", "help": "Branch to search for the base revision.", }, ], [ ["--new-branch"], { "type": str, "default": "try", "help": "Branch to search for the new revision.", }, ], [ ["--skip-download"], { "action": "store_true", "default": False, "help": "If set, we won't try to download artifacts again and we'll " + "try using what already exists in the output folder.", }, ], [ ["-o", "--overwrite"], { "action": "store_true", "default": False, "help": "If set, the downloaded task group data will be deleted before " + "it gets re-downloaded.", }, ], ] class ToolingOptions: args = { "side-by-side": SideBySideOptions.args, "change-detector": ChangeDetectorOptions.args, } class PerftestToolsArgumentParser(ArgumentParser): """%(prog)s [options] [test paths]""" tool = None def __init__(self, *args, **kwargs): ArgumentParser.__init__( self, usage=self.__doc__, conflict_handler="resolve", **kwargs ) if PerftestToolsArgumentParser.tool is None: raise SystemExit("No tool specified, cannot continue parsing") else: for name, options in ToolingOptions.args[PerftestToolsArgumentParser.tool]: self.add_argument(*name, **options)