diff options
Diffstat (limited to 'tools/tryselect')
78 files changed, 11853 insertions, 0 deletions
diff --git a/tools/tryselect/__init__.py b/tools/tryselect/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/cli.py b/tools/tryselect/cli.py new file mode 100644 index 0000000000..74c82c4bed --- /dev/null +++ b/tools/tryselect/cli.py @@ -0,0 +1,167 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import subprocess +import tempfile +from argparse import ArgumentParser + +from .task_config import all_task_configs + +COMMON_ARGUMENT_GROUPS = { + "push": [ + [ + ["-m", "--message"], + { + "const": "editor", + "default": "{msg}", + "nargs": "?", + "help": "Use the specified commit message, or create it in your " + "$EDITOR if blank. Defaults to computed message.", + }, + ], + [ + ["--closed-tree"], + { + "action": "store_true", + "default": False, + "help": "Push despite a closed try tree", + }, + ], + ], + "preset": [ + [ + ["--save"], + { + "default": None, + "help": "Save selection for future use with --preset.", + }, + ], + [ + ["--preset"], + { + "default": None, + "help": "Load a saved selection.", + }, + ], + [ + ["--list-presets"], + { + "action": "store_const", + "dest": "preset_action", + "const": "list", + "default": None, + "help": "List available preset selections.", + }, + ], + [ + ["--edit-presets"], + { + "action": "store_const", + "dest": "preset_action", + "const": "edit", + "default": None, + "help": "Edit the preset file.", + }, + ], + ], + "task": [ + [ + ["--full"], + { + "action": "store_true", + "default": False, + "help": "Use the full set of tasks as input to fzf (instead of " + "target tasks).", + }, + ], + [ + ["-p", "--parameters"], + { + "default": None, + "help": "Use the given parameters.yml to generate tasks, " + "defaults to a default set of parameters", + }, + ], + ], +} + +NO_PUSH_ARGUMENT_GROUP = [ + [ + ["--stage-changes"], + { + "dest": "stage_changes", + "action": "store_true", + "help": "Locally stage changes created by this command but do not " + "push to try.", + }, + ], + [ + ["--no-push"], + { + "dest": "dry_run", + "action": "store_true", + "help": "Do not push to try as a result of running this command (if " + "specified this command will only print calculated try " + "syntax and selection info and not change files).", + }, + ], +] + + +class BaseTryParser(ArgumentParser): + name = "try" + common_groups = ["push", "preset"] + arguments = [] + task_configs = [] + + def __init__(self, *args, **kwargs): + ArgumentParser.__init__(self, *args, **kwargs) + + group = self.add_argument_group("{} arguments".format(self.name)) + for cli, kwargs in self.arguments: + group.add_argument(*cli, **kwargs) + + for name in self.common_groups: + group = self.add_argument_group("{} arguments".format(name)) + arguments = COMMON_ARGUMENT_GROUPS[name] + + # Preset arguments are all mutually exclusive. + if name == "preset": + group = group.add_mutually_exclusive_group() + + for cli, kwargs in arguments: + group.add_argument(*cli, **kwargs) + + if name == "push": + group_no_push = group.add_mutually_exclusive_group() + arguments = NO_PUSH_ARGUMENT_GROUP + for cli, kwargs in arguments: + group_no_push.add_argument(*cli, **kwargs) + + group = self.add_argument_group("task configuration arguments") + self.task_configs = {c: all_task_configs[c]() for c in self.task_configs} + for cfg in self.task_configs.values(): + cfg.add_arguments(group) + + def validate(self, args): + if hasattr(args, "message"): + if args.message == "editor": + if "EDITOR" not in os.environ: + self.error( + "must set the $EDITOR environment variable to use blank --message" + ) + + with tempfile.NamedTemporaryFile(mode="r") as fh: + subprocess.call([os.environ["EDITOR"], fh.name]) + args.message = fh.read().strip() + + if "{msg}" not in args.message: + args.message = "{}\n\n{}".format(args.message, "{msg}") + + def parse_known_args(self, *args, **kwargs): + args, remainder = ArgumentParser.parse_known_args(self, *args, **kwargs) + self.validate(args) + return args, remainder diff --git a/tools/tryselect/docs/configuration.rst b/tools/tryselect/docs/configuration.rst new file mode 100644 index 0000000000..6743d5f385 --- /dev/null +++ b/tools/tryselect/docs/configuration.rst @@ -0,0 +1,30 @@ +Configuring Try +=============== + + +Getting Level 1 Commit Access +----------------------------- + +In order to push to try, `Level 1 Commit Access`_ is required. Please see `Becoming a Mozilla +Committer`_ for more information on how to obtain this. + + +Configuring Version Control +--------------------------- + +After you have level 1 access, you'll need to do a little bit of setup before you can push. Both +``hg`` and ``git`` are supported, move on to the appropriate section below. + + +Configuring Try with Mercurial / Git +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The recommended way to push to try is via the ``mach try`` command. This requires the +``push-to-try`` extension which can be installed by running: + +.. code-block:: shell + + $ mach vcs-setup + +.. _Level 1 Commit Access: https://www.mozilla.org/en-US/about/governance/policies/commit/access-policy/ +.. _Becoming a Mozilla Committer: https://www.mozilla.org/en-US/about/governance/policies/commit/ diff --git a/tools/tryselect/docs/img/add-new-jobs.png b/tools/tryselect/docs/img/add-new-jobs.png Binary files differnew file mode 100644 index 0000000000..513565cb14 --- /dev/null +++ b/tools/tryselect/docs/img/add-new-jobs.png diff --git a/tools/tryselect/docs/img/phab-treeherder-link.png b/tools/tryselect/docs/img/phab-treeherder-link.png Binary files differnew file mode 100644 index 0000000000..52b58b6231 --- /dev/null +++ b/tools/tryselect/docs/img/phab-treeherder-link.png diff --git a/tools/tryselect/docs/index.rst b/tools/tryselect/docs/index.rst new file mode 100644 index 0000000000..673b93a233 --- /dev/null +++ b/tools/tryselect/docs/index.rst @@ -0,0 +1,71 @@ +Pushing to Try +============== + +"Pushing to Try" allows developers to build and test their changes on Mozilla's automation servers +without requiring their code to be reviewed and landed. + +First, :doc:`ensure that you can push to Try <configuration>`. +Try knows how to run tasks that are defined in-tree, +such as ``build-linux64/opt`` (build Firefox for Linux). To manually select some tasks for +Try to process, run the following command: + +.. code-block:: shell + + ./mach try fuzzy + +After submitting your requested tasks, you'll be given a link to your "push" in Treeherder. +It may take a few minutes for your push to appear in Treeherder! Be patient, and it will automatically +update when Try begins processing your work. + +Another very useful Try command is ``./mach try auto``, which will automatically select the tasks +that are mostly likely to be affected by your changes. +See the :doc:`selectors page <selectors/index>` to view all the other ways to select which tasks to push. + +Resolving "<Try build> is damaged and can't be opened" error +------------------------------------------------------------ + +Apple automatically quarantines apps that are downloaded with a browser from an untrusted +location. This "quarantine status" can be cleared by doing ``xattr -c <Try build>`` after +downloading. You can avoid this "quarantine status" by downloading the build from the command +line instead, such as by using ``curl``: + +.. code-block:: shell + + curl -L <artifact-url> -o <file-name> + +.. _attach-job-review: + +Adding Try jobs to a Phabricator patch +-------------------------------------- + +For every patch submitted for review in Phabricator, a new Try run is automatically created. +A link called ``Treeherder Jobs`` can be found in the ``Diff Detail`` section of the review in +Phabricator. + +.. image:: img/phab-treeherder-link.png + +This run is created for static analysis, linting and other tasks. Attaching new jobs to the run is +easy and doesn't require more actions from the developer. +Click on the down-arrow to access the actions menu, select the relevant jobs +and, click on ``Trigger X new jobs`` (located on the top of the job). + +.. image:: img/add-new-jobs.png + +Table of Contents +----------------- + +.. toctree:: + :maxdepth: 2 + + configuration + selectors/index + presets + tasks + + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/tools/tryselect/docs/presets.rst b/tools/tryselect/docs/presets.rst new file mode 100644 index 0000000000..6e1008a456 --- /dev/null +++ b/tools/tryselect/docs/presets.rst @@ -0,0 +1,85 @@ +Presets +======= + +Some selectors, such as ``fuzzy`` and ``syntax``, allow saving and loading presets from a file. This is a +good way to re-use a selection, either at a later date or by sharing with others. Look for a +'preset' section in ``mach <selector> --help`` to determine whether the selector supports this +functionality. + +Using Presets +------------- + +To save a preset, run: + +.. code-block:: shell + + $ mach try <selector> --save <name> <args> + +For example, to save a preset that selects all Windows mochitests: + +.. code-block:: shell + + $ mach try fuzzy --save all-windows-mochitests --query "'win 'mochitest" + preset saved, run with: --preset=all-windows-mochitests + +Then run that saved preset like this: + +.. code-block:: shell + + $ mach try --preset all-windows-mochitests + +To see a list of all available presets run: + +.. code-block:: shell + + $ mach try --list-presets + + +Editing and Sharing Presets +--------------------------- + +Presets can be defined in one of two places, in your home directory or in a file checked into +mozilla-central. + +Local Presets +~~~~~~~~~~~~~ + +These are defined in your ``$MOZBUILD_STATE_DIR``, typically ``~/.mozbuild/try_presets.yml``. +Presets defined here are your own personal collection of presets. You can modify them by running: + +.. code-block:: shell + + $ ./mach try --edit-presets + + +Shared Presets +~~~~~~~~~~~~~~ + +You can also check presets into mozilla-central in `tools/tryselect/try_presets.yml`_. These presets +will be available to all users of ``mach try``, so please be mindful when editing this file. Make +sure the name of the preset is scoped appropriately (i.e doesn't contain any team or module specific +terminology). It is good practice to prefix the preset name with the name of the team or module that +will get the most use out of it. + + +Preset Format +~~~~~~~~~~~~~ + +Presets are simple key/value objects, with the name as the key and a metadata object as the value. +For example, the preset saved above would look something like this in ``try_presets.yml``: + +.. code-block:: yaml + + all-windows-mochitests: + selector: fuzzy + description: >- + Runs all windows mochitests. + query: + - "'win 'mochitest" + +The ``selector`` key (required) allows ``mach try`` to determine which subcommand to dispatch to. +The ``description`` key (optional in user presets but required for shared presets) is a human +readable string describing what the preset selects and when to use it. All other values in the +preset are forwarded to the specified selector as is. + +.. _tools/tryselect/try_presets.yml: https://searchfox.org/mozilla-central/source/tools/tryselect/try_presets.yml diff --git a/tools/tryselect/docs/selectors/again.rst b/tools/tryselect/docs/selectors/again.rst new file mode 100644 index 0000000000..da592b7a34 --- /dev/null +++ b/tools/tryselect/docs/selectors/again.rst @@ -0,0 +1,36 @@ +Again Selector +============== + +When you push to try, the computed ``try_task_config.json`` is saved in a +history file under ``~/.mozbuild/srcdirs/<srcdir hash>/history`` (note: the +``syntax`` selector does not use ``try_task_config.json`` yet so does not save +any history). You can then use the ``again`` selector to re-push any of your +previously generated task configs. + +In the simple case, you can re-run your last try push with: + +.. code-block:: shell + + $ mach try again + +If you want to re-push a task config a little further down the history stack, +first you need to figure out its index with: + +.. code-block:: shell + + $ mach try again --list + +Then run: + +.. code-block:: shell + + $ mach try again --index <index> + +Note that index ``0`` is always the most recent ``try_task_config.json`` in the +history. You can clear your history with: + +.. code-block:: shell + + $ mach try again --purge + +Only the 10 most recent pushes will be saved in your history. diff --git a/tools/tryselect/docs/selectors/auto.rst b/tools/tryselect/docs/selectors/auto.rst new file mode 100644 index 0000000000..39ab5b7450 --- /dev/null +++ b/tools/tryselect/docs/selectors/auto.rst @@ -0,0 +1,24 @@ +Auto Selector +============= + +This selector automatically determines the most efficient set of tests and +tasks to run against your push. It accomplishes this via a combination of +machine learning and manual heuristics. The tasks selected here should match +pretty closely to what would be scheduled if your patch were pushed to +autoland. + +It is the officially recommended selector to use when you are unsure of which +tasks should run on your push. + +To use: + +.. code-block:: + + $ mach try auto + +Unlike other try selectors, tasks are not chosen locally. Rather they will be +computed by the decision task. + +Like most other selectors, ``mach try auto`` supports many of the standard +templates such as ``--artifact`` or ``--env``. See ``mach try auto --help`` for +the full list of supported templates. diff --git a/tools/tryselect/docs/selectors/chooser.rst b/tools/tryselect/docs/selectors/chooser.rst new file mode 100644 index 0000000000..6c59d54009 --- /dev/null +++ b/tools/tryselect/docs/selectors/chooser.rst @@ -0,0 +1,32 @@ +Chooser Selector +================ + +When pushing to try, there are a very large amount of builds and tests to choose from. Often too +many to remember, making it easy to forget a set of tasks which should otherwise have been run. + +This selector allows you to select tasks from a web interface that lists all the possible build and +test tasks and allows you to select them from a list. It is similar in concept to the old `try +syntax chooser`_ page, except that the values are dynamically generated using the :ref:`taskgraph<TaskCluster Task-Graph Generation>` as an +input. This ensures that it will never be out of date. + +To use: + +.. code-block:: shell + + $ mach try chooser + +This will spin up a local web server (using Flask) which serves the chooser app. After making your +selection, simply press ``Push`` and the rest will be handled from there. No need to copy/paste any +syntax strings or the like. + +You can run: + +.. code-block:: shell + + $ mach try chooser --full + +To generate the interface using the full :ref:`taskgraph<TaskCluster Task-Graph Generation>` instead. This will include tasks that don't run +on mozilla-central. + + +.. _try syntax chooser: https://mozilla-releng.net/trychooser diff --git a/tools/tryselect/docs/selectors/compare.rst b/tools/tryselect/docs/selectors/compare.rst new file mode 100644 index 0000000000..a87b263030 --- /dev/null +++ b/tools/tryselect/docs/selectors/compare.rst @@ -0,0 +1,17 @@ +Compare Selector +================ + +When this command runs it pushes two identical try jobs to treeherder. The first +job is on the current commit you are on, and the second one is a commit +specified in the command line arguments. This selector is aimed at helping +engineers test performance enhancements or resolve performance regressions. + +Currently the only way you can select jobs is through fuzzy but we are +planning on expanding to other choosing frameworks also. + +You pass the commit you want to compare against as a commit hash as either +``-cc`` or ``--compare-commit``, an example is show below + +.. code-block:: shell + + $ mach try compare --compare-commit <commit-hash> diff --git a/tools/tryselect/docs/selectors/empty.rst b/tools/tryselect/docs/selectors/empty.rst new file mode 100644 index 0000000000..c3ea61b9ce --- /dev/null +++ b/tools/tryselect/docs/selectors/empty.rst @@ -0,0 +1,21 @@ +Empty Selector +============== + +The ``mach try empty`` subcommand is very simple, it won't schedule any additional tasks. You'll +still see lint tasks and python-unittest tasks if applicable, this is due to a configuration option +in taskcluster. + +Other than those, your try run will be empty. You can use treeherder's ``Add new jobs`` feature to +selectively add additional tasks after the fact. + +.. note:: + + To use ``Add new jobs`` you'll need to be logged in and have commit access level 1, just as if + you were pushing to try. + +To do this: + + 1. Click the drop-down arrow at the top right of your commit. + 2. Select ``Add new jobs`` (it may take a couple seconds to load). + 3. Choose your desired tasks by clicking them one at a time. + 4. At the top of your commit, select ``Trigger New Jobs``. diff --git a/tools/tryselect/docs/selectors/fuzzy.rst b/tools/tryselect/docs/selectors/fuzzy.rst new file mode 100644 index 0000000000..d50f801eb0 --- /dev/null +++ b/tools/tryselect/docs/selectors/fuzzy.rst @@ -0,0 +1,371 @@ +Fuzzy Selector +============== + +The fuzzy selector uses a tool called `fzf`_. It allows you to filter down all of the task labels +from a terminal based UI and an intelligent fuzzy finding algorithm. If the ``fzf`` binary is not +installed, you'll be prompted to bootstrap it on first run. + + +Understanding the Interface +--------------------------- + +When you run ``mach try fuzzy`` an interface similar to the one below will open. This is `fzf`_. + +.. image:: fzf.png + +There's a lot to unpack here, so let's examine each component a bit more closely. + + A. The set of tasks that match the currently typed-out query. In the above image only tasks that + match the query ``'linux64 mochibrochr`` are displayed. + + B. The set of selected tasks. These are the tasks that will be scheduled once you hit ``Enter``. + In other words, if the task you want does not appear here, *it won't be scheduled*. + + C. Count information of the form ``x/y (z)``, where ``x`` is the number of tasks that match the + current query, ``y`` is the total number of tasks and ``z`` is the number of tasks you have + selected. + + D. The input bar for entering queries. As you type you'll notice the list of tasks in ``A`` + starts to update immediately. In the image above, the query ``'linux64 mochibrochr`` is entered. + Correspondingly only tasks matching that query are displayed. + +In general terms, you first find tasks on the left. Then you move them over to the right by +selecting them. Once you are satisfied with your selection, press ``Enter`` to push to try. + + +Selecting Tasks +--------------- + +There are few ways you can select tasks. If you are feeling a bit overwhelmed, it might be best to +stick with the mouse to start: + + 1. Enter a query (e.g ``mochitest``) to reduce the task list a little. + 2. Scroll up and look for the task(s) you want. + 3. ``Right-Click`` as many tasks as desired to select them. + 4. Optionally delete your query, go back to step 1) and repeat. + 5. Press ``Enter`` to push (or ``Esc`` to cancel). + +.. note:: + + Dependencies are automatically filled in, so you can select a test task without needing + to select the build it depends on. + +As you ``Right-Click``, notice that a little arrow appears to the left of the task label. This +indicates that the task is selected and exists in the preview pane to the right. + +Once you are a bit more comfortable with the interface, using the keyboard is much better at quickly +selecting tasks. Here are the main shortcuts you'll need: + +.. code-block:: text + + Ctrl-K / Up => Move cursor up + Ctrl-J / Down => Move cursor down + Tab => Select task + move cursor down + Shift-Tab => Select task + move cursor up + Ctrl-A => Select all currently filtered tasks + Ctrl-T => Toggle select all currently filtered tasks + Ctrl-D => De-select all selected tasks (both filtered and not) + Alt-Bspace => Clear query from input bar + Enter => Accept selection and exit + Ctrl-C / Esc => Cancel selection and exit + ? => Toggle preview pane + + +The process for selecting tasks is otherwise the same as for a mouse. A particularly fast and +powerful way to select tasks is to: + +.. code-block:: text + + Write a precise query => Ctrl-A => Alt-Bspace => Repeat + +As before, when you are satisfied with your selection press ``Enter`` and all the tasks in the +preview pane will be pushed to try. If you change your mind you can press ``Esc`` or ``Ctrl-C`` to +exit the interface without pushing anything. + +.. note:: + + Initially ``fzf`` will automatically select whichever task is under your cursor. This is a + convenience feature for the case where you are only selecting a single task. This feature will be + turned off as soon as you *lock in* a selection with ``Right-Click``, ``Tab`` or ``Ctrl-A``. + + +Writing Queries +--------------- + +Queries are built from a series of terms, each separated by a space. Terms are logically joined by +the AND operator. For example: + +.. code-block:: text + + 'windows 'mochitest + +This query has two terms, and is the equivalent of saying: Give me all the tasks that match both the +term ``'windows'`` and the term ``'mochitest'``. In other words, this query matches all Windows +mochitest tasks. + +The single quote prefix before each term tells ``fzf`` to use exact substring matches, so only tasks +that contain both the literal string ``windows`` AND the literal string ``mochitest`` will be +matched. + +Another thing to note is that the order of the terms makes no difference, so ``'windows 'mochitest`` +and ``'mochitest 'windows`` are equivalent. + + +Fuzzy terms +~~~~~~~~~~~ + +If a term is *not* prefixed with a single quote, that makes it a fuzzy term. This means the +characters in the term need to show up in order, but not in sequence. E.g the fuzzy term ``max`` +would match the string ``mozilla firefox`` (as first there is an ``m``, then an ``a`` and finally an +``x``), but not the string ``firefox by mozilla`` (since the ``x`` is now out of order). Here's a +less contrived example: + +.. code-block:: text + + wndws mchtst + +Like the query above, this one would also select all Windows mochitest tasks. But it will +additionally select: + +.. code-block:: text + + test-macosx1014-64-shippable/opt-talos-sessionrestore-many-windows-e10s + +This is because both sequences of letters (``wndws`` and ``mchtst``) independently appear in order +somewhere in this string (remember the order of the terms makes no difference). + +At first fuzzy terms may not seem very useful, but they are actually extremely powerful! Let's use +the term from the interface image above, ``'linux64 mochibrochr``, as an example. First, just notice +how in the image ``fzf`` highlights the characters that constitute the match in green. Next, notice +how typing ``mochibrochr`` can quickly get us all mochitest browser-chrome tasks. The power of fuzzy +terms is that you don't need to memorize the exact task labels you are looking for. Just start +typing something you think is vaguely correct and chances are you'll see the task you're looking for. + + +Term Modifiers +~~~~~~~~~~~~~~ + +The following modifiers can be applied to a search term: + +.. code-block:: text + + 'word => exact match (line must contain the literal string "word") + ^word => exact prefix match (line must start with literal "word") + word$ => exact suffix match (line must end with literal "word") + !word => exact negation match (line must not contain literal "word") + 'a | 'b => OR operator (joins two exact match operators together) + +For example: + +.. code-block:: text + + ^start 'exact | 'other !ignore fuzzy end$ + +would match the string: + +.. code-block:: text + + starting to bake isn't exactly fun, but pizza is yummy in the end + +.. note:: + + The best way to learn how to write queries is to run ``mach try fuzzy --no-push`` and play + around with all of these modifiers! + + +Specifying Queries on the Command Line +-------------------------------------- + +Sometimes it's more convenient to skip the interactive interface and specify a query on the command +line with ``-q/--query``. This is equivalent to opening the interface then typing: +``<query><ctrl-a><enter>``. + +For example: + +.. code-block:: shell + + # selects all mochitest tasks + $ mach try fuzzy --query "mochitest" + +You can pass in multiple queries at once and the results of each will be joined together: + +.. code-block:: shell + + # selects all mochitest and reftest tasks + $ mach try fuzzy -q "mochitest" -q "reftest" + +If instead you want the intersection of queries, you can pass in ``-x/--and``: + +.. code-block:: shell + + # selects all windows mochitest tasks + $ mach try fuzzy --and -q "mochitest" -q "windows" + + +Modifying Presets +~~~~~~~~~~~~~~~~~ + +:doc:`Presets <../presets>` make it easy to run a pre-determined set of tasks. But sometimes you +might not want to run that set exactly as is, you may only want to use the preset as a starting +point then add or remove tasks as needed. This can be accomplished with ``-q/--query`` or +``-i/--interactive``. + +Here are some examples of adding tasks to a preset: + +.. code-block:: shell + + # selects all perf tasks plus all mochitest-chrome tasks + $ mach try fuzzy --preset perf -q "mochitest-chrome" + + # adds tasks to the perf preset interactively + $ mach try fuzzy --preset perf -i + +Similarly, ``-x/--and`` can be used to filter down a preset by taking the intersection of the two +sets: + +.. code-block:: shell + + # limits perf tasks to windows only + $ mach try fuzzy --preset perf -xq "windows" + + # limits perf tasks interactively + $ mach try fuzzy --preset perf -xi + + +Shell Conflicts +~~~~~~~~~~~~~~~ + +Unfortunately ``fzf``'s query language uses some characters (namely ``'``, ``!`` and ``$``) that can +interfere with your shell when using ``-q/--query``. Below are some tips for how to type out a query +on the command line. + +The ``!`` character is typically used for history expansion. If you don't use this feature, the +easiest way to specify queries on the command line is to disable it: + +.. code-block:: shell + + # bash + $ set +H + $ ./mach try fuzzy -q "'foo !bar" + + # zsh + $ setopt no_banghist + $ ./mach try fuzzy -q "'foo !bar" + +If using ``bash``, add ``set +H`` to your ``~/.bashrc``, ``~/.bash_profile`` or equivalent. If using +``zsh``, add ``setopt no_banghist`` to your ``~/.zshrc`` or equivalent. + +If you don't want to disable history expansion, you can escape your queries like this: + +.. code-block:: shell + + # bash + $ ./mach try fuzzy -q $'\'foo !bar' + + # zsh + $ ./mach try fuzzy -q "'foo \!bar" + + +The third option is to use ``-e/--exact`` which reverses the behaviour of the ``'`` character (see +:ref:`additional-arguments` for more details). Using this flag means you won't need to escape the +``'`` character as often and allows you to run your queries like this: + +.. code-block:: shell + + # bash and zsh + $ ./mach try fuzzy -eq 'foo !bar' + +This method is only useful if you find you almost always prefix terms with ``'`` (and rarely use +fuzzy terms). Otherwise as soon as you want to use a fuzzy match you'll run into the same problem as +before. + +.. note:: All the examples in these three approaches will select the same set of tasks. + +If you use ``fish`` shell, you won't need to escape ``!``, however you will need to escape ``$``: + +.. code-block:: shell + + # fish + $ ./mach try fuzzy -q "'foo !bar baz\$" + + +Test Paths +---------- + +One or more paths to a file or directory may be specified as positional arguments. When +specifying paths, the list of available tasks to choose from is filtered down such that +only suites that have tests in a specified path can be selected. Notably, only the first +chunk of each suite/platform appears. When the tasks are scheduled, only tests that live +under one of the specified paths will be run. + +.. note:: + + When using paths, be aware that all tests under the specified paths will run in the + same chunk. This might produce a different ordering from what gets run on production + branches, and may yield different results. + + For suites that restart the browser between each manifest (like mochitest), this + shouldn't be as big of a concern. + +Paths can be used with the interactive ``fzf`` window, or using the ``-q/--query`` argument. +For example, running: + +.. code-block:: shell + + $ mach try fuzzy layout/reftests/reftest-sanity -q "!pgo !cov !asan 'linux64" + +Would produce the following ``try_task_config.json``: + +.. code-block:: json + + { + "env":{ + "MOZHARNESS_TEST_PATHS":"{\"reftest\":\"layout/reftests/reftest-sanity\"}" + }, + "tasks":[ + "test-linux64-qr/debug-reftest-e10s-1", + "test-linux64-qr/opt-reftest-e10s-1", + "test-linux64/debug-reftest-e10s-1", + "test-linux64/debug-reftest-no-accel-e10s-1", + "test-linux64/opt-reftest-e10s-1", + "test-linux64/opt-reftest-no-accel-e10s-1", + ] + } + +Inside of these tasks, the reftest harness will only run tests that live under +``layout/reftests/reftest-sanity``. + + +.. _additional-arguments: + +Additional Arguments +-------------------- + +There are a few additional command line arguments you may wish to use: + +``-e/--exact`` +By default, ``fzf`` treats terms as a fuzzy match and prefixing a term with ``'`` turns it into an exact +match. If passing in ``--exact``, this behaviour is reversed. Non-prefixed terms become exact, and a +``'`` prefix makes a term fuzzy. + +``--full`` +By default, only target tasks (e.g tasks that would normally run on mozilla-central) +are generated. Passing in ``--full`` allows you to select from all tasks. This is useful for +things like nightly or release tasks. + +``-u/--update`` +Update the bootstrapped ``fzf`` binary to the latest version. + +For a full list of command line arguments, run: + +.. code-block:: shell + + $ mach try fuzzy --help + +For more information on using ``fzf``, run: + +.. code-block:: shell + + $ man fzf + +.. _fzf: https://github.com/junegunn/fzf diff --git a/tools/tryselect/docs/selectors/fzf.png b/tools/tryselect/docs/selectors/fzf.png Binary files differnew file mode 100644 index 0000000000..a64f4b04f3 --- /dev/null +++ b/tools/tryselect/docs/selectors/fzf.png diff --git a/tools/tryselect/docs/selectors/index.rst b/tools/tryselect/docs/selectors/index.rst new file mode 100644 index 0000000000..be618202d6 --- /dev/null +++ b/tools/tryselect/docs/selectors/index.rst @@ -0,0 +1,44 @@ +Selectors +========= + +These are the currently implemented try selectors: + +* :doc:`auto <auto>`: Have tasks chosen for you automatically. +* :doc:`fuzzy <fuzzy>`: Select tasks using a fuzzy finding algorithm and + a terminal interface. +* :doc:`chooser <chooser>`: Select tasks using a web interface. +* :doc:`again <again>`: Re-run a previous ``try_task_config.json`` based + push. +* :doc:`empty <empty>`: Don't select any tasks. Taskcluster will still run + some tasks automatically (like lint and python unittest tasks). Further tasks + can be chosen with treeherder's ``Add New Jobs`` feature. +* :doc:`syntax <syntax>`: Select tasks using classic try syntax. +* :doc:`release <release>`: Prepare a tree for doing a staging release. +* :doc:`scriptworker <scriptworker>`: Run scriptworker tasks against a recent release. +* :doc:`compare <compare>`: Push two identical try jobs, one on your current commit and another of your choice + +You can run them with: + +.. code-block:: shell + + $ mach try <selector> + +See selector specific options by running: + +.. code-block:: shell + + $ mach try <selector> --help + +.. toctree:: + :caption: Available Selectors + :maxdepth: 1 + :hidden: + + Auto <auto> + Fuzzy <fuzzy> + Chooser <chooser> + Again <again> + Empty <empty> + Syntax <syntax> + Release <release> + Scriptworker <scriptworker> diff --git a/tools/tryselect/docs/selectors/release.rst b/tools/tryselect/docs/selectors/release.rst new file mode 100644 index 0000000000..946266249b --- /dev/null +++ b/tools/tryselect/docs/selectors/release.rst @@ -0,0 +1,31 @@ +Release Selector +================ + +This command configures the tree in preparation for doing a staging release, +and pushes the result to try. The changes that that are made include: + +- Updating the version number. +- Applying the migrations that are done as part of merge day. +- Disabling repacking most locales. (This can be disabled by passing ``--no-limit-locales``). + +For staging a beta release, run the following (with an appropriate version number): + +.. code-block:: shell + + $ mach try release --version 64.0b5 --migration central-to-beta + +For staging a final release (rc or patch), run the following (with an appropriate version number) + +.. code-block:: shell + + $ mach try release --version 64.0 --migration central-to-beta --migration beta-to-release + +Once the decision task is on the push is complete, you can start the release +through `staging ship-it instance <https://shipit.staging.mozilla-releng.net/new>`_\ [#shipit]_. + +.. note:: + + If pushing from beta or release, the corresponding migration should not be + passed, as they have already been applied. + +.. [#shipit] This is only available to release engineering and release management (as of 2018-10-15). diff --git a/tools/tryselect/docs/selectors/scriptworker.rst b/tools/tryselect/docs/selectors/scriptworker.rst new file mode 100644 index 0000000000..a3cba08cbe --- /dev/null +++ b/tools/tryselect/docs/selectors/scriptworker.rst @@ -0,0 +1,31 @@ +Scriptworker Selector +===================== + +This command runs a selection of scriptworker tasks against builds from a +recent release. This is aimed at release engineering, to test changes to +scriptworker implementations. It currently requires being connected to +Mozilla's internal datacenter VPN with access to shipit\ [#shipit]_. + +There are a number of preset groups of tasks to run. To run a particular +set of tasks, pass the name of the set to ``mach try scriptworker``: + +.. code-block:: shell + + $ mach try scriptworker linux-signing + +To get the list of task sets, along with the list of tasks they will run: + +.. code-block:: shell + + $ mach try scriptworker list + +The selector defaults to using tasks from the most recent beta, to use tasks +from a different release, pass ``--release-type <release-type>``: + +.. code-block:: shell + + $ mach try scriptworker --release-type release linux-signing + + +.. [#shipit] The shipit API is not currently publicly available, and is used + to find the release graph to pull previous tasks from. diff --git a/tools/tryselect/docs/selectors/syntax.rst b/tools/tryselect/docs/selectors/syntax.rst new file mode 100644 index 0000000000..b64bb65ab7 --- /dev/null +++ b/tools/tryselect/docs/selectors/syntax.rst @@ -0,0 +1,41 @@ +Syntax Selector +=============== + +.. warning:: + + Try syntax is antiquated and hard to understand. If you aren't already + familiar with try syntax, you might want to use the ``fuzzy`` selector + instead. + +Try syntax is a command line string that goes into the commit message. Using +``mach try syntax`` will automatically create a temporary commit with your +chosen syntax and then delete it again after pushing to try. + +Try syntax can contain all kinds of different options parsed by various +places in various repos, but the majority are parsed by `try_option_syntax.py`_. +The most common arguments include: + + * ``-b/--build`` - One of ``d``, ``o`` or ``do``. This is the build type, + either opt, debug or both (required). + * ``-p/--platform`` - The platforms you want to build and/or run tests on + (required). + * ``-u/--unittests`` - The test tasks you want to run (optional). + * ``-t/--talos`` - The talos tasks you want to run (optional). + +Here are some examples: + +.. code-block:: shell + + $ mach try syntax -b do -p linux,macosx64 -u mochitest-e10s-1,crashtest -t none + $ mach try syntax -b d -p win64 -u all + $ mach try syntax -b o -p linux64 + +Unfortunately, knowing the magic strings that make it work can be a bit of a +guessing game. If you are unsure of what string will get your task to run, try +using :doc:`mach try fuzzy <fuzzy>` instead. + +While using ``mach try syntax -b do -p all -u all -t all`` will work, heavy use +of ``all`` is discouraged as it consumes a lot of unnecessary resources (some of +which are hardware constrained). + +.. _try_option_syntax.py: https://searchfox.org/mozilla-central/source/taskcluster/gecko_taskgraph/try_option_syntax.py diff --git a/tools/tryselect/docs/tasks.rst b/tools/tryselect/docs/tasks.rst new file mode 100644 index 0000000000..61de9ec9ed --- /dev/null +++ b/tools/tryselect/docs/tasks.rst @@ -0,0 +1,152 @@ +Task Generation +=============== + +Many selectors (including ``chooser``, ``coverage`` and ``fuzzy``) source their available tasks +directly from the :ref:`taskgraph <TaskCluster Task-Graph Generation>` module by building the taskgraph +locally. This means that the list of available tasks will never be stale. While this is very +powerful, it comes with a large enough performance cost to get annoying (around twenty seconds). + +The result of the taskgraph generation will be cached, so this penalty will only be incurred +whenever a file in the ``/taskcluster`` directory is modified. Unfortunately this directory changes +pretty frequently, so developers can expect to rebuild the cache each time they pull in +``mozilla-central``. Developers who regularly work on ``/taskcluster`` can expect to rebuild even +more frequently. + + +Configuring Watchman +-------------------- + +It's possible to bypass this penalty completely by using the file watching service `watchman`_. If +you use the ``fsmonitor`` mercurial extension, you already have ``watchman`` installed. + +.. note:: + + If you aren't using `fsmonitor`_ but end up installng watchman anyway, you + might as well enable it for a faster Mercurial experience. + +Otherwise, `install watchman`_. If using Linux you'll likely run into the `inotify limits`_ outlined +on that page due to the size of ``mozilla-central``. You can `read this page`_ for more information +on how to bump the limits permanently. + +Next run the following commands: + +.. code-block:: shell + + $ cd path/to/mozilla-central + $ watchman watch . + $ watchman -j < tools/tryselect/watchman.json + +You should see output like: + +.. code-block:: json + + { + "triggerid": "rebuild-taskgraph-cache", + "disposition": "created", + "version": "20200920.192359.0" + } + +That's it. Now anytime a file under ``/taskcluster`` is modified (either by your editor, or by +updating version control), the taskgraph cache will be rebuilt in the background, allowing you to +skip the wait the next time you run ``mach try``. + +.. note:: + + Watchman triggers are persistent and don't need to be added more than once. + See `Managing Triggers`_ for how to remove a trigger. + +You can test that everything is working by running these commands: + +.. code-block:: shell + + $ statedir=`mach python -c "from mach.util import get_state_dir; print(get_state_dir(specific_to_topsrcdir=True))"` + $ rm -rf $statedir/cache/taskgraph + $ touch taskcluster/mach_commands.py + # wait a minute for generation to trigger and finish + $ ls $statedir/cache/taskgraph + +If the ``target_task_set`` file exists, you are good to go. If not you can look at the ``watchman`` +log to see if there were any errors. This typically lives somewhere like +``/usr/local/var/run/watchman/$USER-state/log``. In this case please file a bug under ``Firefox +Build System :: Try`` and include the relevant portion of the log. + + +Running Watchman on Startup +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Watchman is both a client and a service all in one. When running a ``watchman`` command, the client +binary will start the service in the background if it isn't running. This means on reboot the +service won't be running and you'll need to start the service each time by invoking the client +binary (e.g by running ``watchman version``). + +If you'd like this to happen automatically, you can use your favourite platform specific way of +running commands at startup (``crontab``, ``rc.local``, etc.). Watchman stores separate state for +each user, so be sure you run the command as the user that set up the triggers. + +Setting up a systemd Service +++++++++++++++++++++++++++++ + +If ``systemd`` is an option you can create a service: + +.. code-block:: ini + + [Unit] + Description=Watchman for %i + After=network.target + + [Service] + Type=simple + User=%i + ExecStart=/usr/local/bin/watchman --log-level 1 watch-list -f + ExecStop=/usr/local/bin/watchman shutdown-server + + [Install] + WantedBy=multi-user.target + +Save this to a file called ``/etc/systemd/system/watchman@.service``. Then run: + +.. code-block:: shell + + $ sudo systemctl enable watchman@$USER.service + $ sudo systemctl start watchman@$USER.service + +The next time you reboot, the watchman service should start automatically. + + +Managing Triggers +~~~~~~~~~~~~~~~~~ + +When adding a trigger watchman writes it to disk. Typically it'll be a path similar to +``/usr/local/var/run/watchman/$USER-state/state``. While editing that file by hand would work, the +watchman binary provides an interface for managing your triggers. + +To see all directories you are currently watching: + +.. code-block:: shell + + $ watchman watch-list + +To view triggers that are active in a specified watch: + +.. code-block:: shell + + $ watchman trigger-list <path> + +To delete a trigger from a specified watch: + +.. code-block:: shell + + $ watchman trigger-del <path> <name> + +In the above two examples, replace ``<path>`` with the path of the watch, presumably +``mozilla-central``. Using ``.`` works as well if that is already your working directory. For more +information on managing triggers and a reference of other commands, see the `official docs`_. + + +.. _watchman: https://facebook.github.io/watchman/ +.. _fsmonitor: https://www.mercurial-scm.org/wiki/FsMonitorExtension +.. _install watchman: https://facebook.github.io/watchman/docs/install.html +.. _inotify limits: https://facebook.github.io/watchman/docs/install.html#linux-inotify-limits +.. _read this page: https://github.com/guard/listen/wiki/Increasing-the-amount-of-inotify-watchers +.. _this hint: https://github.com/facebook/watchman/commit/2985377eaf8c8538b28fae9add061b67991a87c2 +.. _official docs: https://facebook.github.io/watchman/docs/cmd/trigger.html diff --git a/tools/tryselect/mach_commands.py b/tools/tryselect/mach_commands.py new file mode 100644 index 0000000000..26c15a0990 --- /dev/null +++ b/tools/tryselect/mach_commands.py @@ -0,0 +1,528 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import argparse +import importlib +import os +import sys + +from mach.decorators import Command, SettingsProvider, SubCommand +from mach.util import get_state_dir +from mozbuild.base import BuildEnvironmentNotFoundException +from mozbuild.util import memoize + +CONFIG_ENVIRONMENT_NOT_FOUND = """ +No config environment detected. This means we are unable to properly +detect test files in the specified paths or tags. Please run: + + $ mach configure + +and try again. +""".lstrip() + + +class get_parser: + def __init__(self, selector): + self.selector = selector + + def __call__(self): + mod = importlib.import_module("tryselect.selectors.{}".format(self.selector)) + return getattr(mod, "{}Parser".format(self.selector.capitalize()))() + + +def generic_parser(): + from tryselect.cli import BaseTryParser + + parser = BaseTryParser() + parser.add_argument("argv", nargs=argparse.REMAINDER) + return parser + + +@SettingsProvider +class TryConfig: + @classmethod + def config_settings(cls): + from mach.registrar import Registrar + + desc = ( + "The default selector to use when running `mach try` without a subcommand." + ) + choices = Registrar.command_handlers["try"].subcommand_handlers.keys() + + return [ + ("try.default", "string", desc, "auto", {"choices": choices}), + ( + "try.maxhistory", + "int", + "Maximum number of pushes to save in history.", + 10, + ), + ] + + +def init(command_context): + from tryselect import push + + push.MAX_HISTORY = command_context._mach_context.settings["try"]["maxhistory"] + + +@memoize +def presets(command_context): + from tryselect.preset import MergedHandler + + # Create our handler using both local and in-tree presets. The first + # path in this list will be treated as the 'user' file for the purposes + # of saving and editing. All subsequent paths are 'read-only'. We check + # an environment variable first for testing purposes. + if os.environ.get("MACH_TRY_PRESET_PATHS"): + preset_paths = os.environ["MACH_TRY_PRESET_PATHS"].split(os.pathsep) + else: + preset_paths = [ + os.path.join(get_state_dir(), "try_presets.yml"), + os.path.join( + command_context.topsrcdir, "tools", "tryselect", "try_presets.yml" + ), + ] + + return MergedHandler(*preset_paths) + + +def handle_presets( + command_context, preset_action=None, save=None, preset=None, **kwargs +): + """Handle preset related arguments. + + This logic lives here so that the underlying selectors don't need + special preset handling. They can all save and load presets the same + way. + """ + from tryselect.util.dicttools import merge + + user_presets = presets(command_context).handlers[0] + if preset_action == "list": + presets(command_context).list() + sys.exit() + + if preset_action == "edit": + user_presets.edit() + sys.exit() + + parser = command_context._mach_context.handler.parser + subcommand = command_context._mach_context.handler.subcommand + if "preset" not in parser.common_groups: + return kwargs + + default = parser.get_default + if save: + selector = ( + subcommand or command_context._mach_context.settings["try"]["default"] + ) + + # Only save non-default values for simplicity. + kwargs = {k: v for k, v in kwargs.items() if v != default(k)} + user_presets.save(save, selector=selector, **kwargs) + print("preset saved, run with: --preset={}".format(save)) + sys.exit() + + if preset: + if preset not in presets(command_context): + command_context._mach_context.parser.error( + "preset '{}' does not exist".format(preset) + ) + + name = preset + preset = presets(command_context)[name] + selector = preset.pop("selector") + preset.pop("description", None) # description isn't used by any selectors + + if not subcommand: + subcommand = selector + elif subcommand != selector: + print( + "error: preset '{}' exists for a different selector " + "(did you mean to run 'mach try {}' instead?)".format(name, selector) + ) + sys.exit(1) + + # Order of precedence is defaults -> presets -> cli. Configuration + # from the right overwrites configuration from the left. + defaults = {} + nondefaults = {} + for k, v in kwargs.items(): + if v == default(k): + defaults[k] = v + else: + nondefaults[k] = v + + kwargs = merge(defaults, preset, nondefaults) + + return kwargs + + +def handle_try_config(command_context, **kwargs): + from tryselect.util.dicttools import merge + + to_validate = [] + kwargs.setdefault("try_config", {}) + for cls in command_context._mach_context.handler.parser.task_configs.values(): + try_config = cls.try_config(**kwargs) + if try_config is not None: + to_validate.append(cls) + kwargs["try_config"] = merge(kwargs["try_config"], try_config) + + for name in cls.dests: + del kwargs[name] + + # Validate task_configs after they have all been parsed to avoid + # depending on the order they were processed. + for cls in to_validate: + cls.validate(**kwargs) + return kwargs + + +def run(command_context, **kwargs): + kwargs = handle_presets(command_context, **kwargs) + + if command_context._mach_context.handler.parser.task_configs: + kwargs = handle_try_config(command_context, **kwargs) + + mod = importlib.import_module( + "tryselect.selectors.{}".format( + command_context._mach_context.handler.subcommand + ) + ) + return mod.run(**kwargs) + + +@Command( + "try", + category="ci", + description="Push selected tasks to the try server", + parser=generic_parser, +) +def try_default(command_context, argv=None, **kwargs): + """Push selected tests to the try server. + + The |mach try| command is a frontend for scheduling tasks to + run on try server using selectors. A selector is a subcommand + that provides its own set of command line arguments and are + listed below. + + If no subcommand is specified, the `auto` selector is run by + default. Run |mach try auto --help| for more information on + scheduling with the `auto` selector. + """ + init(command_context) + subcommand = command_context._mach_context.handler.subcommand + # We do special handling of presets here so that `./mach try --preset foo` + # works no matter what subcommand 'foo' was saved with. + preset = kwargs["preset"] + if preset: + if preset not in presets(command_context): + command_context._mach_context.handler.parser.error( + "preset '{}' does not exist".format(preset) + ) + + subcommand = presets(command_context)[preset]["selector"] + + sub = subcommand or command_context._mach_context.settings["try"]["default"] + return command_context._mach_context.commands.dispatch( + "try", command_context._mach_context, subcommand=sub, argv=argv, **kwargs + ) + + +@SubCommand( + "try", + "fuzzy", + description="Select tasks on try using a fuzzy finder", + parser=get_parser("fuzzy"), +) +def try_fuzzy(command_context, **kwargs): + """Select which tasks to run with a fuzzy finding interface (fzf). + + When entering the fzf interface you'll be confronted by two panes. The + one on the left contains every possible task you can schedule, the one + on the right contains the list of selected tasks. In other words, the + tasks that will be scheduled once you you press <enter>. + + At first fzf will automatically select whichever task is under your + cursor, which simplifies the case when you are looking for a single + task. But normally you'll want to select many tasks. To accomplish + you'll generally start by typing a query in the search bar to filter + down the list of tasks (see Extended Search below). Then you'll either: + + A) Move the cursor to each task you want and press <tab> to select it. + Notice it now shows up in the pane to the right. + + OR + + B) Press <ctrl-a> to select every task that matches your filter. + + You can delete your query, type a new one and select further tasks as + many times as you like. Once you are happy with your selection, press + <enter> to push the selected tasks to try. + + All selected task labels and their dependencies will be scheduled. This + means you can select a test task and its build will automatically be + filled in. + + + Keyboard Shortcuts + ------------------ + + When in the fuzzy finder interface, start typing to filter down the + task list. Then use the following keyboard shortcuts to select tasks: + + Ctrl-K / Up => Move cursor up + Ctrl-J / Down => Move cursor down + Tab => Select task + move cursor down + Shift-Tab => Select task + move cursor up + Ctrl-A => Select all currently filtered tasks + Ctrl-D => De-select all currently filtered tasks + Ctrl-T => Toggle select all currently filtered tasks + Alt-Bspace => Clear query from input bar + Enter => Accept selection and exit + Ctrl-C / Esc => Cancel selection and exit + ? => Toggle preview pane + + There are many more shortcuts enabled by default, you can also define + your own shortcuts by setting `--bind` in the $FZF_DEFAULT_OPTS + environment variable. See `man fzf` for more info. + + + Extended Search + --------------- + + When typing in search terms, the following modifiers can be applied: + + 'word: exact match (line must contain the literal string "word") + ^word: exact prefix match (line must start with literal "word") + word$: exact suffix match (line must end with literal "word") + !word: exact negation match (line must not contain literal "word") + 'a | 'b: OR operator (joins two exact match operators together) + + For example: + + ^start 'exact | !ignore fuzzy end$ + + + Documentation + ------------- + + For more detailed documentation, please see: + https://firefox-source-docs.mozilla.org/tools/try/selectors/fuzzy.html + """ + init(command_context) + if kwargs.pop("interactive"): + kwargs["query"].append("INTERACTIVE") + + if kwargs.pop("intersection"): + kwargs["intersect_query"] = kwargs["query"] + del kwargs["query"] + + if kwargs.get("save") and not kwargs.get("query"): + # If saving preset without -q/--query, allow user to use the + # interface to build the query. + kwargs_copy = kwargs.copy() + kwargs_copy["dry_run"] = True + kwargs_copy["save"] = None + kwargs["query"] = run(command_context, save_query=True, **kwargs_copy) + if not kwargs["query"]: + return + + if kwargs.get("paths"): + kwargs["test_paths"] = kwargs["paths"] + + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "chooser", + description="Schedule tasks by selecting them from a web interface.", + parser=get_parser("chooser"), +) +def try_chooser(command_context, **kwargs): + """Push tasks selected from a web interface to try. + + This selector will build the taskgraph and spin up a dynamically + created 'trychooser-like' web-page on the localhost. After a selection + has been made, pressing the 'Push' button will automatically push the + selection to try. + """ + init(command_context) + command_context.activate_virtualenv() + path = os.path.join( + "tools", "tryselect", "selectors", "chooser", "requirements.txt" + ) + command_context.virtualenv_manager.install_pip_requirements(path, quiet=True) + + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "auto", + description="Automatically determine which tasks to run. This runs the same " + "set of tasks that would be run on autoland. This " + "selector is EXPERIMENTAL.", + parser=get_parser("auto"), +) +def try_auto(command_context, **kwargs): + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "again", + description="Schedule a previously generated (non try syntax) push again.", + parser=get_parser("again"), +) +def try_again(command_context, **kwargs): + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "empty", + description="Push to try without scheduling any tasks.", + parser=get_parser("empty"), +) +def try_empty(command_context, **kwargs): + """Push to try, running no builds or tests + + This selector does not prompt you to run anything, it just pushes + your patches to try, running no builds or tests by default. After + the push finishes, you can manually add desired jobs to your push + via Treeherder's Add New Jobs feature, located in the per-push + menu. + """ + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "syntax", + description="Select tasks on try using try syntax", + parser=get_parser("syntax"), +) +def try_syntax(command_context, **kwargs): + """Push the current tree to try, with the specified syntax. + + Build options, platforms and regression tests may be selected + using the usual try options (-b, -p and -u respectively). In + addition, tests in a given directory may be automatically + selected by passing that directory as a positional argument to the + command. For example: + + mach try -b d -p linux64 dom testing/web-platform/tests/dom + + would schedule a try run for linux64 debug consisting of all + tests under dom/ and testing/web-platform/tests/dom. + + Test selection using positional arguments is available for + mochitests, reftests, xpcshell tests and web-platform-tests. + + Tests may be also filtered by passing --tag to the command, + which will run only tests marked as having the specified + tags e.g. + + mach try -b d -p win64 --tag media + + would run all tests tagged 'media' on Windows 64. + + If both positional arguments or tags and -u are supplied, the + suites in -u will be run in full. Where tests are selected by + positional argument they will be run in a single chunk. + + If no build option is selected, both debug and opt will be + scheduled. If no platform is selected a default is taken from + the AUTOTRY_PLATFORM_HINT environment variable, if set. + + The command requires either its own mercurial extension ("push-to-try", + installable from mach vcs-setup) or a git repo using git-cinnabar + (installable from mach vcs-setup). + + """ + init(command_context) + try: + if command_context.substs.get("MOZ_ARTIFACT_BUILDS"): + kwargs["local_artifact_build"] = True + except BuildEnvironmentNotFoundException: + # If we don't have a build locally, we can't tell whether + # an artifact build is desired, but we still want the + # command to succeed, if possible. + pass + + config_status = os.path.join(command_context.topobjdir, "config.status") + if (kwargs["paths"] or kwargs["tags"]) and not config_status: + print(CONFIG_ENVIRONMENT_NOT_FOUND) + sys.exit(1) + + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "coverage", + description="Select tasks on try using coverage data", + parser=get_parser("coverage"), +) +def try_coverage(command_context, **kwargs): + """Select which tasks to use using coverage data.""" + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "release", + description="Push the current tree to try, configured for a staging release.", + parser=get_parser("release"), +) +def try_release(command_context, **kwargs): + """Push the current tree to try, configured for a staging release.""" + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "scriptworker", + description="Run scriptworker tasks against a recent release.", + parser=get_parser("scriptworker"), +) +def try_scriptworker(command_context, **kwargs): + """Run scriptworker tasks against a recent release. + + Requires VPN and shipit access. + """ + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "compare", + description="Push two try jobs, one on your current commit and another on the one you specify", + parser=get_parser("compare"), +) +def try_compare(command_context, **kwargs): + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "perf", + description="Try selector for running performance tests.", + parser=get_parser("perf"), +) +def try_perf(command_context, **kwargs): + init(command_context) + return run(command_context, **kwargs) diff --git a/tools/tryselect/preset.py b/tools/tryselect/preset.py new file mode 100644 index 0000000000..dc8cba5c57 --- /dev/null +++ b/tools/tryselect/preset.py @@ -0,0 +1,107 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import shlex +import subprocess + +import yaml + + +# Ensure strings with ' like fzf query strings get double-quoted: +def represent_str(self, data): + if "'" in data: + return self.represent_scalar("tag:yaml.org,2002:str", data, style='"') + return self.represent_scalar("tag:yaml.org,2002:str", data) + + +yaml.SafeDumper.add_representer(str, represent_str) + + +class PresetHandler: + def __init__(self, path): + self.path = path + self._presets = {} + + @property + def presets(self): + if not self._presets and os.path.isfile(self.path): + with open(self.path) as fh: + self._presets = yaml.safe_load(fh) or {} + + return self._presets + + def __contains__(self, name): + return name in self.presets + + def __getitem__(self, name): + return self.presets[name] + + def __len__(self): + return len(self.presets) + + def __str__(self): + if not self.presets: + return "" + return yaml.safe_dump(self.presets, default_flow_style=False) + + def list(self): + if not self.presets: + print("no presets found") + else: + print(self) + + def edit(self): + if "EDITOR" not in os.environ: + print( + "error: must set the $EDITOR environment variable to use --edit-presets" + ) + return + + subprocess.call(shlex.split(os.environ["EDITOR"]) + [self.path]) + + def save(self, name, **data): + self.presets[name] = data + + with open(self.path, "w") as fh: + fh.write(str(self)) + + +class MergedHandler: + def __init__(self, *paths): + """Helper class for dealing with multiple preset files.""" + self.handlers = [PresetHandler(p) for p in paths] + + def __contains__(self, name): + return any(name in handler for handler in self.handlers) + + def __getitem__(self, name): + for handler in self.handlers: + if name in handler: + return handler[name] + raise KeyError(name) + + def __len__(self): + return sum(len(h) for h in self.handlers) + + def __str__(self): + all_presets = { + k: v for handler in self.handlers for k, v in handler.presets.items() + } + return yaml.safe_dump(all_presets, default_flow_style=False) + + def list(self): + if len(self) == 0: + print("no presets found") + return + + for handler in self.handlers: + val = str(handler) + if val: + val = "\n ".join( + [""] + val.splitlines() + [""] + ) # indent all lines by 2 spaces + print("Presets from {}:".format(handler.path)) + print(val) diff --git a/tools/tryselect/push.py b/tools/tryselect/push.py new file mode 100644 index 0000000000..cade05db9b --- /dev/null +++ b/tools/tryselect/push.py @@ -0,0 +1,239 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +import sys +import traceback + +import six +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozversioncontrol import MissingVCSExtension, get_repository_object + +from .util.estimates import duration_summary +from .util.manage_estimates import ( + download_task_history_data, + make_trimmed_taskgraph_cache, +) + +GIT_CINNABAR_NOT_FOUND = """ +Could not detect `git-cinnabar`. + +The `mach try` command requires git-cinnabar to be installed when +pushing from git. Please install it by running: + + $ ./mach vcs-setup +""".lstrip() + +HG_PUSH_TO_TRY_NOT_FOUND = """ +Could not detect `push-to-try`. + +The `mach try` command requires the push-to-try extension enabled +when pushing from hg. Please install it by running: + + $ ./mach vcs-setup +""".lstrip() + +VCS_NOT_FOUND = """ +Could not detect version control. Only `hg` or `git` are supported. +""".strip() + +UNCOMMITTED_CHANGES = """ +ERROR please commit changes before continuing +""".strip() + +MAX_HISTORY = 10 + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) +vcs = get_repository_object(build.topsrcdir) + +history_path = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "history", "try_task_configs.json" +) + + +def write_task_config(try_task_config): + config_path = os.path.join(vcs.path, "try_task_config.json") + with open(config_path, "w") as fh: + json.dump(try_task_config, fh, indent=4, separators=(",", ": "), sort_keys=True) + fh.write("\n") + return config_path + + +def write_task_config_history(msg, try_task_config): + if not os.path.isfile(history_path): + if not os.path.isdir(os.path.dirname(history_path)): + os.makedirs(os.path.dirname(history_path)) + history = [] + else: + with open(history_path) as fh: + history = fh.read().strip().splitlines() + + history.insert(0, json.dumps([msg, try_task_config])) + history = history[:MAX_HISTORY] + with open(history_path, "w") as fh: + fh.write("\n".join(history)) + + +def check_working_directory(push=True): + if not push: + return + + if not vcs.working_directory_clean(): + print(UNCOMMITTED_CHANGES) + sys.exit(1) + + +def generate_try_task_config(method, labels, try_config=None, routes=None): + try_task_config = try_config or {} + try_task_config.setdefault("env", {})["TRY_SELECTOR"] = method + try_task_config.update( + { + "version": 1, + "tasks": sorted(labels), + } + ) + if routes: + try_task_config["routes"] = routes + + return try_task_config + + +def task_labels_from_try_config(try_task_config): + if try_task_config["version"] == 2: + parameters = try_task_config.get("parameters", {}) + if parameters.get("try_mode") == "try_task_config": + return parameters["try_task_config"]["tasks"] + else: + return None + elif try_task_config["version"] == 1: + return try_task_config.get("tasks", list()) + else: + return None + + +def display_push_estimates(try_task_config): + task_labels = task_labels_from_try_config(try_task_config) + if task_labels is None: + return + + cache_dir = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph" + ) + + graph_cache = None + dep_cache = None + target_file = None + for graph_cache_file in ["target_task_graph", "full_task_graph"]: + graph_cache = os.path.join(cache_dir, graph_cache_file) + if os.path.isfile(graph_cache): + dep_cache = graph_cache.replace("task_graph", "task_dependencies") + target_file = graph_cache.replace("task_graph", "task_set") + break + + if not dep_cache: + return + + download_task_history_data(cache_dir=cache_dir) + make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_file) + + durations = duration_summary(dep_cache, task_labels, cache_dir) + + print( + "estimates: Runs {} tasks ({} selected, {} dependencies)".format( + durations["dependency_count"] + durations["selected_count"], + durations["selected_count"], + durations["dependency_count"], + ) + ) + print( + "estimates: Total task duration {}".format( + durations["dependency_duration"] + durations["selected_duration"] + ) + ) + if "percentile" in durations: + print( + "estimates: In the top {}% of durations".format( + 100 - durations["percentile"] + ) + ) + print( + "estimates: Should take about {} (Finished around {})".format( + durations["wall_duration_seconds"], + durations["eta_datetime"].strftime("%Y-%m-%d %H:%M"), + ) + ) + + +def push_to_try( + method, + msg, + try_task_config=None, + stage_changes=False, + dry_run=False, + closed_tree=False, + files_to_change=None, + allow_log_capture=False, +): + push = not stage_changes and not dry_run + check_working_directory(push) + + if try_task_config and method not in ("auto", "empty"): + try: + display_push_estimates(try_task_config) + except Exception: + traceback.print_exc() + print("warning: unable to display push estimates") + + # Format the commit message + closed_tree_string = " ON A CLOSED TREE" if closed_tree else "" + commit_message = "{}{}\n\nPushed via `mach try {}`".format( + msg, + closed_tree_string, + method, + ) + + config_path = None + changed_files = [] + if try_task_config: + if push and method not in ("again", "auto", "empty"): + write_task_config_history(msg, try_task_config) + config_path = write_task_config(try_task_config) + changed_files.append(config_path) + + if (push or stage_changes) and files_to_change: + for path, content in files_to_change.items(): + path = os.path.join(vcs.path, path) + with open(path, "wb") as fh: + fh.write(six.ensure_binary(content)) + changed_files.append(path) + + try: + if not push: + print("Commit message:") + print(commit_message) + if config_path: + print("Calculated try_task_config.json:") + with open(config_path) as fh: + print(fh.read()) + return + + vcs.add_remove_files(*changed_files) + + try: + vcs.push_to_try(commit_message, allow_log_capture=allow_log_capture) + except MissingVCSExtension as e: + if e.ext == "push-to-try": + print(HG_PUSH_TO_TRY_NOT_FOUND) + elif e.ext == "cinnabar": + print(GIT_CINNABAR_NOT_FOUND) + else: + raise + sys.exit(1) + finally: + if config_path and os.path.isfile(config_path): + os.remove(config_path) diff --git a/tools/tryselect/selectors/__init__.py b/tools/tryselect/selectors/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/selectors/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/selectors/again.py b/tools/tryselect/selectors/again.py new file mode 100644 index 0000000000..434aed7cc1 --- /dev/null +++ b/tools/tryselect/selectors/again.py @@ -0,0 +1,151 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os + +from ..cli import BaseTryParser +from ..push import history_path, push_to_try + + +class AgainParser(BaseTryParser): + name = "again" + arguments = [ + [ + ["--index"], + { + "default": 0, + "const": "list", + "nargs": "?", + "help": "Index of entry in the history to re-push, " + "where '0' is the most recent (default 0). " + "Use --index without a value to display indices.", + }, + ], + [ + ["--list"], + { + "default": False, + "action": "store_true", + "dest": "list_configs", + "help": "Display history and exit", + }, + ], + [ + ["--list-tasks"], + { + "default": 0, + "action": "count", + "dest": "list_tasks", + "help": "Like --list, but display selected tasks " + "for each history entry, up to 10. Repeat " + "to display all selected tasks.", + }, + ], + [ + ["--purge"], + { + "default": False, + "action": "store_true", + "help": "Remove all history and exit", + }, + ], + ] + common_groups = ["push"] + + +def run( + index=0, purge=False, list_configs=False, list_tasks=0, message="{msg}", **pushargs +): + if index == "list": + list_configs = True + else: + try: + index = int(index) + except ValueError: + print("error: '--index' must be an integer") + return 1 + + if purge: + os.remove(history_path) + return + + if not os.path.isfile(history_path): + print("error: history file not found: {}".format(history_path)) + return 1 + + with open(history_path) as fh: + history = fh.readlines() + + if list_configs or list_tasks > 0: + for i, data in enumerate(history): + msg, config = json.loads(data) + version = config.get("version", "1") + settings = {} + if version == 1: + tasks = config["tasks"] + settings = config + elif version == 2: + try_config = config.get("parameters", {}).get("try_task_config", {}) + tasks = try_config.get("tasks") + else: + tasks = None + + if tasks is not None: + # Select only the things that are of interest to display. + settings = settings.copy() + env = settings.pop("env", {}).copy() + env.pop("TRY_SELECTOR", None) + for name in ("tasks", "version"): + settings.pop(name, None) + + def pluralize(n, noun): + return "{n} {noun}{s}".format( + n=n, noun=noun, s="" if n == 1 else "s" + ) + + out = str(i) + ". (" + pluralize(len(tasks), "task") + if env: + out += ", " + pluralize(len(env), "env var") + if settings: + out += ", " + pluralize(len(settings), "setting") + out += ") " + msg + print(out) + + if list_tasks > 0: + indent = " " * 4 + if list_tasks > 1: + shown_tasks = tasks + else: + shown_tasks = tasks[:10] + print(indent + ("\n" + indent).join(shown_tasks)) + + num_hidden_tasks = len(tasks) - len(shown_tasks) + if num_hidden_tasks > 0: + print("{}... and {} more".format(indent, num_hidden_tasks)) + + if list_tasks and env: + for line in ("env: " + json.dumps(env, indent=2)).splitlines(): + print(" " + line) + + if list_tasks and settings: + for line in ( + "settings: " + json.dumps(settings, indent=2) + ).splitlines(): + print(" " + line) + else: + print( + "{index}. {msg}".format( + index=i, + msg=msg, + ) + ) + + return + + msg, try_task_config = json.loads(history[index]) + return push_to_try( + "again", message.format(msg=msg), try_task_config=try_task_config, **pushargs + ) diff --git a/tools/tryselect/selectors/auto.py b/tools/tryselect/selectors/auto.py new file mode 100644 index 0000000000..7b27b33616 --- /dev/null +++ b/tools/tryselect/selectors/auto.py @@ -0,0 +1,115 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.util.python_path import find_object + +from ..cli import BaseTryParser +from ..push import push_to_try + +TRY_AUTO_PARAMETERS = { + "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium", # noqa + "optimize_target_tasks": True, + "target_tasks_method": "try_auto", + "test_manifest_loader": "bugbug", + "try_mode": "try_auto", + "try_task_config": {}, +} + + +class AutoParser(BaseTryParser): + name = "auto" + common_groups = ["push"] + task_configs = [ + "artifact", + "env", + "chemspill-prio", + "disable-pgo", + "worker-overrides", + ] + arguments = [ + [ + ["--strategy"], + { + "default": None, + "help": "Override the default optimization strategy. Valid values " + "are the experimental strategies defined at the bottom of " + "`taskcluster/gecko_taskgraph/optimize/__init__.py`.", + }, + ], + [ + ["--tasks-regex"], + { + "default": [], + "action": "append", + "help": "Apply a regex filter to the tasks selected. Specifying " + "multiple times schedules the union of computed tasks.", + }, + ], + [ + ["--tasks-regex-exclude"], + { + "default": [], + "action": "append", + "help": "Apply a regex filter to the tasks selected. Specifying " + "multiple times excludes computed tasks matching any regex.", + }, + ], + ] + + def validate(self, args): + super().validate(args) + + if args.strategy: + if ":" not in args.strategy: + args.strategy = "gecko_taskgraph.optimize:tryselect.{}".format( + args.strategy + ) + + try: + obj = find_object(args.strategy) + except (ImportError, AttributeError): + self.error("invalid module path '{}'".format(args.strategy)) + + if not isinstance(obj, dict): + self.error("object at '{}' must be a dict".format(args.strategy)) + + +def run( + message="{msg}", + stage_changes=False, + dry_run=False, + closed_tree=False, + strategy=None, + tasks_regex=None, + tasks_regex_exclude=None, + try_config=None, + **ignored +): + msg = message.format(msg="Tasks automatically selected.") + + params = TRY_AUTO_PARAMETERS.copy() + if try_config: + params["try_task_config"] = try_config + + if strategy: + params["optimize_strategies"] = strategy + + if tasks_regex or tasks_regex_exclude: + params.setdefault("try_task_config", {})["tasks-regex"] = {} + params["try_task_config"]["tasks-regex"]["include"] = tasks_regex + params["try_task_config"]["tasks-regex"]["exclude"] = tasks_regex_exclude + + task_config = { + "version": 2, + "parameters": params, + } + return push_to_try( + "auto", + msg, + try_task_config=task_config, + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + ) diff --git a/tools/tryselect/selectors/chooser/.eslintrc.js b/tools/tryselect/selectors/chooser/.eslintrc.js new file mode 100644 index 0000000000..861d6bafc2 --- /dev/null +++ b/tools/tryselect/selectors/chooser/.eslintrc.js @@ -0,0 +1,16 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + env: { + jquery: true, + }, + globals: { + apply: true, + applyChunks: true, + tasks: true, + }, +}; diff --git a/tools/tryselect/selectors/chooser/__init__.py b/tools/tryselect/selectors/chooser/__init__.py new file mode 100644 index 0000000000..b71cf801ea --- /dev/null +++ b/tools/tryselect/selectors/chooser/__init__.py @@ -0,0 +1,97 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import webbrowser +from threading import Timer + +from gecko_taskgraph.target_tasks import filter_by_uncommon_try_tasks + +from tryselect.cli import BaseTryParser +from tryselect.push import ( + check_working_directory, + generate_try_task_config, + push_to_try, +) +from tryselect.tasks import generate_tasks + +here = os.path.abspath(os.path.dirname(__file__)) + + +class ChooserParser(BaseTryParser): + name = "chooser" + arguments = [] + common_groups = ["push", "task"] + task_configs = [ + "artifact", + "browsertime", + "chemspill-prio", + "disable-pgo", + "env", + "gecko-profile", + "path", + "pernosco", + "rebuild", + "worker-overrides", + ] + + +def run( + update=False, + query=None, + try_config=None, + full=False, + parameters=None, + save=False, + preset=None, + mod_presets=False, + stage_changes=False, + dry_run=False, + message="{msg}", + closed_tree=False, +): + from .app import create_application + + push = not stage_changes and not dry_run + check_working_directory(push) + + tg = generate_tasks(parameters, full) + + # Remove tasks that are not to be shown unless `--full` is specified. + if not full: + blacklisted_tasks = [ + label + for label in tg.tasks.keys() + if not filter_by_uncommon_try_tasks(label) + ] + for task in blacklisted_tasks: + tg.tasks.pop(task) + + app = create_application(tg) + + if os.environ.get("WERKZEUG_RUN_MAIN") == "true": + # we are in the reloader process, don't open the browser or do any try stuff + app.run() + return + + # give app a second to start before opening the browser + url = "http://127.0.0.1:5000" + Timer(1, lambda: webbrowser.open(url)).start() + print("Starting trychooser on {}".format(url)) + app.run() + + selected = app.tasks + if not selected: + print("no tasks selected") + return + + msg = "Try Chooser Enhanced ({} tasks selected)".format(len(selected)) + return push_to_try( + "chooser", + message.format(msg=msg), + try_task_config=generate_try_task_config("chooser", selected, try_config), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + ) diff --git a/tools/tryselect/selectors/chooser/app.py b/tools/tryselect/selectors/chooser/app.py new file mode 100644 index 0000000000..adbf6f33dd --- /dev/null +++ b/tools/tryselect/selectors/chooser/app.py @@ -0,0 +1,177 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from abc import ABCMeta, abstractproperty +from collections import defaultdict + +from flask import Flask, render_template, request + +SECTIONS = [] +SUPPORTED_KINDS = set() + + +def register_section(cls): + assert issubclass(cls, Section) + instance = cls() + SECTIONS.append(instance) + SUPPORTED_KINDS.update(instance.kind.split(",")) + + +class Section(object): + __metaclass__ = ABCMeta + + @abstractproperty + def name(self): + pass + + @abstractproperty + def kind(self): + pass + + @abstractproperty + def title(self): + pass + + @abstractproperty + def attrs(self): + pass + + def contains(self, task): + return task.kind in self.kind.split(",") + + def get_context(self, tasks): + labels = defaultdict(lambda: {"max_chunk": 0, "attrs": defaultdict(list)}) + + for task in tasks.values(): + if not self.contains(task): + continue + + task = task.attributes + label = labels[self.labelfn(task)] + for attr in self.attrs: + if attr in task and task[attr] not in label["attrs"][attr]: + label["attrs"][attr].append(task[attr]) + + if "test_chunk" in task: + label["max_chunk"] = max( + label["max_chunk"], int(task["test_chunk"]) + ) + + return { + "name": self.name, + "kind": self.kind, + "title": self.title, + "labels": labels, + } + + +@register_section +class Platform(Section): + name = "platform" + kind = "build" + title = "Platforms" + attrs = ["build_platform"] + + def labelfn(self, task): + return task["build_platform"] + + def contains(self, task): + if not Section.contains(self, task): + return False + + # android-stuff tasks aren't actual platforms + return task.task["tags"].get("android-stuff", False) != "true" + + +@register_section +class Test(Section): + name = "test" + kind = "test" + title = "Test Suites" + attrs = ["unittest_suite"] + + def labelfn(self, task): + suite = task["unittest_suite"].replace(" ", "-") + + if suite.endswith("-chunked"): + suite = suite[: -len("-chunked")] + + return suite + + def contains(self, task): + if not Section.contains(self, task): + return False + return task.attributes["unittest_suite"] not in ("raptor", "talos") + + +@register_section +class Perf(Section): + name = "perf" + kind = "test" + title = "Performance" + attrs = ["unittest_suite", "raptor_try_name", "talos_try_name"] + + def labelfn(self, task): + suite = task["unittest_suite"] + label = task["{}_try_name".format(suite)] + + if not label.startswith(suite): + label = "{}-{}".format(suite, label) + + if label.endswith("-e10s"): + label = label[: -len("-e10s")] + + return label + + def contains(self, task): + if not Section.contains(self, task): + return False + return task.attributes["unittest_suite"] in ("raptor", "talos") + + +@register_section +class Analysis(Section): + name = "analysis" + kind = "build,static-analysis-autotest" + title = "Analysis" + attrs = ["build_platform"] + + def labelfn(self, task): + return task["build_platform"] + + def contains(self, task): + if not Section.contains(self, task): + return False + if task.kind == "build": + return task.task["tags"].get("android-stuff", False) == "true" + return True + + +def create_application(tg): + tasks = {l: t for l, t in tg.tasks.items() if t.kind in SUPPORTED_KINDS} + sections = [s.get_context(tasks) for s in SECTIONS] + context = { + "tasks": {l: t.attributes for l, t in tasks.items()}, + "sections": sections, + } + + app = Flask(__name__) + app.env = "development" + app.tasks = [] + + @app.route("/", methods=["GET", "POST"]) + def chooser(): + if request.method == "GET": + return render_template("chooser.html", **context) + + if request.form["action"] == "Push": + labels = request.form["selected-tasks"].splitlines() + app.tasks.extend(labels) + + shutdown = request.environ.get("werkzeug.server.shutdown") + if shutdown: + shutdown() + return render_template("close.html") + + return app diff --git a/tools/tryselect/selectors/chooser/requirements.txt b/tools/tryselect/selectors/chooser/requirements.txt new file mode 100644 index 0000000000..966eae636f --- /dev/null +++ b/tools/tryselect/selectors/chooser/requirements.txt @@ -0,0 +1,44 @@ +Flask==1.1.4 \ + --hash=sha256:0fbeb6180d383a9186d0d6ed954e0042ad9f18e0e8de088b2b419d526927d196 \ + --hash=sha256:c34f04500f2cbbea882b1acb02002ad6fe6b7ffa64a6164577995657f50aed22 +click==7.1.2 \ + --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \ + --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc +Werkzeug==1.0.1 \ + --hash=sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43 \ + --hash=sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c +itsdangerous==1.1.0 \ + --hash=sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19 \ + --hash=sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749 +Jinja2==2.11.3 \ + --hash=sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419 \ + --hash=sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6 +MarkupSafe==1.1.1 \ + --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \ + --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \ + --hash=sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235 \ + --hash=sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5 \ + --hash=sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff \ + --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \ + --hash=sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1 \ + --hash=sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e \ + --hash=sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183 \ + --hash=sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66 \ + --hash=sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1 \ + --hash=sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1 \ + --hash=sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e \ + --hash=sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b \ + --hash=sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905 \ + --hash=sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735 \ + --hash=sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d \ + --hash=sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e \ + --hash=sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d \ + --hash=sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c \ + --hash=sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21 \ + --hash=sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2 \ + --hash=sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5 \ + --hash=sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b \ + --hash=sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6 \ + --hash=sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f \ + --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \ + --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 diff --git a/tools/tryselect/selectors/chooser/static/filter.js b/tools/tryselect/selectors/chooser/static/filter.js new file mode 100644 index 0000000000..2d8731e61f --- /dev/null +++ b/tools/tryselect/selectors/chooser/static/filter.js @@ -0,0 +1,116 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const selection = $("#selection")[0]; +const count = $("#selection-count")[0]; +const pluralize = (count, noun, suffix = "s") => + `${count} ${noun}${count !== 1 ? suffix : ""}`; + +var selected = []; + +var updateLabels = () => { + $(".tab-pane.active > .filter-label").each(function (index) { + let box = $("#" + this.htmlFor)[0]; + let method = box.checked ? "add" : "remove"; + $(this)[method + "Class"]("is-checked"); + }); +}; + +var apply = () => { + let filters = {}; + let kinds = []; + + $(".filter:checked").each(function (index) { + for (let kind of this.name.split(",")) { + if (!kinds.includes(kind)) { + kinds.push(kind); + } + } + + // Checkbox element values are generated by Section.get_context() in app.py + let attrs = JSON.parse(this.value); + for (let attr in attrs) { + if (!(attr in filters)) { + filters[attr] = []; + } + + let values = attrs[attr]; + filters[attr] = filters[attr].concat(values); + } + }); + updateLabels(); + + if ( + !Object.keys(filters).length || + (Object.keys(filters).length == 1 && "build_type" in filters) + ) { + selection.value = ""; + count.innerHTML = "0 tasks selected"; + return; + } + + var taskMatches = label => { + let task = tasks[label]; + + // If no box for the given kind has been checked, this task is + // automatically not selected. + if (!kinds.includes(task.kind)) { + return false; + } + + for (let attr in filters) { + let values = filters[attr]; + if (!(attr in task) || values.includes(task[attr])) { + continue; + } + return false; + } + return true; + }; + + selected = Object.keys(tasks).filter(taskMatches); + applyChunks(); +}; + +var applyChunks = () => { + // For tasks that have a chunk filter applied, we handle that here. + let filters = {}; + $(".filter:text").each(function (index) { + let value = $(this).val(); + if (value === "") { + return; + } + + let attrs = JSON.parse(this.name); + let key = `${attrs.unittest_suite}-${attrs.unittest_flavor}`; + if (!(key in filters)) { + filters[key] = []; + } + + // Parse the chunk strings. These are formatted like printer page setups, e.g: "1,4-6,9" + for (let item of value.split(",")) { + if (!item.includes("-")) { + filters[key].push(parseInt(item)); + continue; + } + + let [start, end] = item.split("-"); + for (let i = parseInt(start); i <= parseInt(end); ++i) { + filters[key].push(i); + } + } + }); + + let chunked = selected.filter(function (label) { + let task = tasks[label]; + let key = task.unittest_suite + "-" + task.unittest_flavor; + if (key in filters && !filters[key].includes(parseInt(task.test_chunk))) { + return false; + } + return true; + }); + + selection.value = chunked.join("\n"); + count.innerText = pluralize(chunked.length, "task") + " selected"; +}; diff --git a/tools/tryselect/selectors/chooser/static/select.js b/tools/tryselect/selectors/chooser/static/select.js new file mode 100644 index 0000000000..8a315c0a52 --- /dev/null +++ b/tools/tryselect/selectors/chooser/static/select.js @@ -0,0 +1,46 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const labels = $("label.multiselect"); +const boxes = $("label.multiselect input:checkbox"); +var lastChecked = {}; + +// implements shift+click +labels.click(function (e) { + if (e.target.tagName === "INPUT") { + return; + } + + let box = $("#" + this.htmlFor)[0]; + let activeSection = $("div.tab-pane.active")[0].id; + + if (activeSection in lastChecked) { + // Bug 559506 - In Firefox shift/ctrl/alt+clicking a label doesn't check the box. + let isFirefox = navigator.userAgent.toLowerCase().indexOf("firefox") > -1; + + if (e.shiftKey) { + if (isFirefox) { + box.checked = !box.checked; + } + + let start = boxes.index(box); + let end = boxes.index(lastChecked[activeSection]); + + boxes + .slice(Math.min(start, end), Math.max(start, end) + 1) + .prop("checked", box.checked); + apply(); + } + } + + lastChecked[activeSection] = box; +}); + +function selectAll(btn) { + let checked = !!btn.value; + $("div.active label.filter-label").each(function (index) { + $(this).find("input:checkbox")[0].checked = checked; + }); + apply(); +} diff --git a/tools/tryselect/selectors/chooser/static/style.css b/tools/tryselect/selectors/chooser/static/style.css new file mode 100644 index 0000000000..6b2f96935b --- /dev/null +++ b/tools/tryselect/selectors/chooser/static/style.css @@ -0,0 +1,107 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +body { + padding-top: 70px; +} + +/* Tabs */ + +#tabbar .nav-link { + color: #009570; + font-size: 18px; + padding-bottom: 15px; + padding-top: 15px; +} + +#tabbar .nav-link.active { + color: #212529; +} + +#tabbar .nav-link:hover { + color: #0f5a3a; +} + +/* Sections */ + +.tab-content button { + font-size: 14px; + margin-bottom: 5px; + margin-top: 10px; +} + +.filter-label { + display: block; + font-size: 16px; + position: relative; + padding-left: 15px; + padding-right: 15px; + padding-top: 10px; + padding-bottom: 10px; + margin-bottom: 0; + user-select: none; + vertical-align: middle; +} + +.filter-label span { + display: flex; + min-height: 34px; + align-items: center; + justify-content: space-between; +} + +.filter-label input[type="checkbox"] { + position: absolute; + opacity: 0; + height: 0; + width: 0; +} + +.filter-label input[type="text"] { + width: 50px; +} + +.filter-label:hover { + background-color: #91a0b0; +} + +.filter-label.is-checked:hover { + background-color: #91a0b0; +} + +.filter-label.is-checked { + background-color: #404c59; + color: white; +} + +/* Preview pane */ + +#preview { + position: fixed; + height: 100vh; + margin-left: 66%; + width: 100%; +} + +#submit-tasks { + display: flex; + flex-direction: column; + height: 80%; +} + +#buttons { + display: flex; + justify-content: space-between; +} + +#push { + background-color: #00e9b7; + margin-left: 5px; + width: 100%; +} + +#selection { + height: 100%; + width: 100%; +} diff --git a/tools/tryselect/selectors/chooser/templates/chooser.html b/tools/tryselect/selectors/chooser/templates/chooser.html new file mode 100644 index 0000000000..d89870ac77 --- /dev/null +++ b/tools/tryselect/selectors/chooser/templates/chooser.html @@ -0,0 +1,78 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this file, + - You can obtain one at http://mozilla.org/MPL/2.0/. --> + +{% extends 'layout.html' %} +{% block content %} +<div class="container-fluid"> + <div class="row"> + <div class="col-8"> + <div class="form-group form-inline"> + <span class="col-form-label col-md-2 pt-1">Build Type</span> + <div class="form-check form-check-inline"> + <input id="both" class="filter form-check-input" type="radio" name="buildtype" value='{}' onchange="apply();" checked> + <label for="both" class="form-check-label">both</label> + </div> + {% for type in ["opt", "debug"] %} + <div class="form-check form-check-inline"> + <input id="{{ type }}" class="filter form-check-input" type="radio" name="buildtype" value='{"build_type": "{{ type }}"}' onchange="apply();"> + <label for={{ type }} class="form-check-label">{{ type }}</label> + </div> + {% endfor %} + </div> + <ul class="nav nav-tabs" id="tabbar" role="tablist"> + {% for section in sections %} + <li class="nav-item"> + {% if loop.first %} + <a class="nav-link active" id="{{ section.name }}-tab" data-toggle="tab" href="#{{section.name }}" role="tab" aria-controls="{{ section.name }}" aria-selected="true">{{ section.title }}</a> + {% else %} + <a class="nav-link" id="{{ section.name }}-tab" data-toggle="tab" href="#{{section.name }}" role="tab" aria-controls="{{ section.name }}" aria-selected="false">{{ section.title }}</a> + {% endif %} + </li> + {% endfor %} + </ul> + <div class="tab-content"> + <button type="button" class="btn btn-secondary" value="true" onclick="selectAll(this);">Select All</button> + <button type="button" class="btn btn-secondary" onclick="selectAll(this);">Deselect All</button> + {% for section in sections %} + {% if loop.first %} + <div class="tab-pane show active" id="{{ section.name }}" role="tabpanel" aria-labelledby="{{ section.name }}-tab"> + {% else %} + <div class="tab-pane" id="{{ section.name }}" role="tabpanel" aria-labelledby="{{ section.name }}-tab"> + {% endif %} + {% for label, meta in section.labels|dictsort %} + <label class="multiselect filter-label" for={{ label }}> + <span> + {{ label }} + <input class="filter" type="checkbox" id={{ label }} name="{{ section.kind }}" value='{{ meta.attrs|tojson|safe }}' onchange="console.log('checkbox onchange triggered');apply();"> + {% if meta.max_chunk > 1 %} + <input class="filter" type="text" pattern="^[0-9][0-9,-]*$" placeholder="1-{{ meta.max_chunk }}" name='{{ meta.attrs|tojson|safe }}' oninput="applyChunks();"> + {% endif %} + </span> + </label> + {% endfor %} + </div> + {% endfor %} + </div> + </div> + <div class="col-4" id="preview"> + <form id="submit-tasks" action="" method="POST"> + <textarea id="selection" name="selected-tasks" wrap="off"></textarea> + <span id="selection-count">0 tasks selected</span><br> + <span id="buttons"> + <input id="cancel" class="btn btn-default" type="submit" name="action" value="Cancel"> + <input id="push" class="btn btn-default" type="submit" name="action" value="Push"> + </span> + </form> + </div> + </div> +</div> +{% endblock %} + +{% block scripts %} +<script> + const tasks = {{ tasks|tojson|safe }}; +</script> +<script src="{{ url_for('static', filename='filter.js') }}"></script> +<script src="{{ url_for('static', filename='select.js') }}"></script> +{% endblock %} diff --git a/tools/tryselect/selectors/chooser/templates/close.html b/tools/tryselect/selectors/chooser/templates/close.html new file mode 100644 index 0000000000..9dc0a161f3 --- /dev/null +++ b/tools/tryselect/selectors/chooser/templates/close.html @@ -0,0 +1,11 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this file, + - You can obtain one at http://mozilla.org/MPL/2.0/. --> + +{% extends 'layout.html' %} {% block content %} +<div class="container-fluid"> + <div class="alert alert-primary" role="alert"> + You may now close this page. + </div> +</div> +{% endblock %} diff --git a/tools/tryselect/selectors/chooser/templates/layout.html b/tools/tryselect/selectors/chooser/templates/layout.html new file mode 100644 index 0000000000..8553ae94df --- /dev/null +++ b/tools/tryselect/selectors/chooser/templates/layout.html @@ -0,0 +1,71 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this file, + - You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> + <head> + <meta charset="utf-8" /> + <title>Try Chooser Enhanced</title> + <link + rel="stylesheet" + href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" + /> + <link + rel="stylesheet" + href="{{ url_for('static', filename='style.css') }}" + /> + </head> + <body> + <nav class="navbar navbar-default fixed-top navbar-dark bg-dark"> + <div class="container-fluid"> + <span class="navbar-brand mb-0 h1">Try Chooser Enhanced</span> + <button + class="navbar-toggler" + type="button" + data-toggle="collapse" + data-target="#navbarSupportedContent" + aria-controls="navbarSupportedContent" + aria-expanded="false" + aria-label="Toggle navigation" + > + <span class="navbar-toggler-icon"></span> + </button> + <div class="collapse navbar-collapse" id="navbarSupportedContent"> + <ul class="navbar-nav mr-auto"> + <li class="nav-item"> + <a + class="nav-link" + href="https://firefox-source-docs.mozilla.org/tools/try/index.html" + >Documentation</a + > + </li> + <li class="nav-item"> + <a + class="nav-link" + href="https://treeherder.mozilla.org/#/jobs?repo=try" + >Treeherder</a + > + </li> + </ul> + </div> + </div> + </nav> + {% block content %}{% endblock %} + <script + src="https://code.jquery.com/jquery-3.3.1.slim.min.js" + integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" + crossorigin="anonymous" + ></script> + <script + src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" + integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" + crossorigin="anonymous" + ></script> + <script + src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" + integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" + crossorigin="anonymous" + ></script> + {% block scripts %}{% endblock %} + </body> +</html> diff --git a/tools/tryselect/selectors/compare.py b/tools/tryselect/selectors/compare.py new file mode 100644 index 0000000000..ac468e0974 --- /dev/null +++ b/tools/tryselect/selectors/compare.py @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +from mozbuild.base import MozbuildObject +from mozversioncontrol import get_repository_object + +from tryselect.cli import BaseTryParser + +from .again import run as again_run +from .fuzzy import run as fuzzy_run + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + + +class CompareParser(BaseTryParser): + name = "compare" + arguments = [ + [ + ["-cc", "--compare-commit"], + { + "default": None, + "help": "The commit that you want to compare your current revision with", + }, + ], + ] + common_groups = ["task"] + task_configs = [ + "rebuild", + ] + + def get_revisions_to_run(vcs, compare_commit): + if compare_commit is None: + compare_commit = vcs.base_ref + if vcs.branch: + current_revision_ref = vcs.branch + else: + current_revision_ref = vcs.head_ref + + return compare_commit, current_revision_ref + + +def run(compare_commit=None, **kwargs): + vcs = get_repository_object(build.topsrcdir) + compare_commit, current_revision_ref = CompareParser.get_revisions_to_run( + vcs, compare_commit + ) + print("********************************************") + print("* 2 commits are created with this command *") + print("********************************************") + + try: + fuzzy_run(**kwargs) + print("********************************************") + print("* The base commit can be found above *") + print("********************************************") + vcs.update(compare_commit) + again_run() + print("*****************************************") + print("* The compare commit can be found above *") + print("*****************************************") + finally: + vcs.update(current_revision_ref) diff --git a/tools/tryselect/selectors/coverage.py b/tools/tryselect/selectors/coverage.py new file mode 100644 index 0000000000..a871f35a11 --- /dev/null +++ b/tools/tryselect/selectors/coverage.py @@ -0,0 +1,447 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import collections +import datetime +import hashlib +import json +import os +import shutil +import sqlite3 +import subprocess + +import requests +import six +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozpack.files import FileFinder +from moztest.resolve import TestResolver +from mozversioncontrol import get_repository_object + +from ..cli import BaseTryParser +from ..push import generate_try_task_config, push_to_try +from ..tasks import filter_tasks_by_paths, generate_tasks, resolve_tests_by_suite + +here = os.path.abspath(os.path.dirname(__file__)) +build = None +vcs = None +CHUNK_MAPPING_FILE = None +CHUNK_MAPPING_TAG_FILE = None + + +def setup_globals(): + # Avoid incurring expensive computation on import. + global build, vcs, CHUNK_MAPPING_TAG_FILE, CHUNK_MAPPING_FILE + build = MozbuildObject.from_environment(cwd=here) + vcs = get_repository_object(build.topsrcdir) + + root_hash = hashlib.sha256( + six.ensure_binary(os.path.abspath(build.topsrcdir)) + ).hexdigest() + cache_dir = os.path.join(get_state_dir(), "cache", root_hash, "chunk_mapping") + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + CHUNK_MAPPING_FILE = os.path.join(cache_dir, "chunk_mapping.sqlite") + CHUNK_MAPPING_TAG_FILE = os.path.join(cache_dir, "chunk_mapping_tag.json") + + +# Maps from platform names in the chunk_mapping sqlite database to respective +# substrings in task names. +PLATFORM_MAP = { + "linux": "test-linux64/opt", + "windows": "test-windows10-64/opt", +} + +# List of platform/build type combinations that are included in pushes by |mach try coverage|. +OPT_TASK_PATTERNS = [ + "macosx64/opt", + "windows10-64/opt", + "windows7-32/opt", + "linux64/opt", +] + + +class CoverageParser(BaseTryParser): + name = "coverage" + arguments = [] + common_groups = ["push", "task"] + task_configs = [ + "artifact", + "env", + "rebuild", + "chemspill-prio", + "disable-pgo", + "worker-overrides", + ] + + +def read_test_manifests(): + """Uses TestResolver to read all test manifests in the tree. + + Returns a (tests, support_files_map) tuple that describes the tests in the tree: + tests - a set of test file paths + support_files_map - a dict that maps from each support file to a list with + test files that require them it + """ + test_resolver = TestResolver.from_environment(cwd=here) + file_finder = FileFinder(build.topsrcdir) + support_files_map = collections.defaultdict(list) + tests = set() + + for test in test_resolver.resolve_tests(build.topsrcdir): + tests.add(test["srcdir_relpath"]) + if "support-files" not in test: + continue + + for support_file_pattern in test["support-files"].split(): + # Get the pattern relative to topsrcdir. + if support_file_pattern.startswith("!/"): + support_file_pattern = support_file_pattern[2:] + elif support_file_pattern.startswith("/"): + support_file_pattern = support_file_pattern[1:] + else: + support_file_pattern = os.path.normpath( + os.path.join(test["dir_relpath"], support_file_pattern) + ) + + # If it doesn't have a glob, then it's a single file. + if "*" not in support_file_pattern: + # Simple case: single support file, just add it here. + support_files_map[support_file_pattern].append(test["srcdir_relpath"]) + continue + + for support_file, _ in file_finder.find(support_file_pattern): + support_files_map[support_file].append(test["srcdir_relpath"]) + + return tests, support_files_map + + +# TODO cache the output of this function +all_tests, all_support_files = read_test_manifests() + + +def download_coverage_mapping(base_revision): + try: + with open(CHUNK_MAPPING_TAG_FILE) as f: + tags = json.load(f) + if tags["target_revision"] == base_revision: + return + else: + print("Base revision changed.") + except (OSError, ValueError): + print("Chunk mapping file not found.") + + CHUNK_MAPPING_URL_TEMPLATE = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/project.relman.code-coverage.production.cron.{}/artifacts/public/chunk_mapping.tar.xz" # noqa + JSON_PUSHES_URL_TEMPLATE = "https://hg.mozilla.org/mozilla-central/json-pushes?version=2&tipsonly=1&startdate={}" # noqa + + # Get pushes from at most one month ago. + PUSH_HISTORY_DAYS = 30 + delta = datetime.timedelta(days=PUSH_HISTORY_DAYS) + start_time = (datetime.datetime.now() - delta).strftime("%Y-%m-%d") + pushes_url = JSON_PUSHES_URL_TEMPLATE.format(start_time) + pushes_data = requests.get(pushes_url + "&tochange={}".format(base_revision)).json() + if "error" in pushes_data: + if "unknown revision" in pushes_data["error"]: + print( + "unknown revision {}, trying with latest mozilla-central".format( + base_revision + ) + ) + pushes_data = requests.get(pushes_url).json() + + if "error" in pushes_data: + raise Exception(pushes_data["error"]) + + pushes = pushes_data["pushes"] + + print("Looking for coverage data. This might take a minute or two.") + print("Base revision:", base_revision) + for push_id in sorted(pushes.keys())[::-1]: + rev = pushes[push_id]["changesets"][0] + url = CHUNK_MAPPING_URL_TEMPLATE.format(rev) + print("push id: {},\trevision: {}".format(push_id, rev)) + + r = requests.head(url) + if not r.ok: + continue + + print("Chunk mapping found, downloading...") + r = requests.get(url, stream=True) + + CHUNK_MAPPING_ARCHIVE = os.path.join(build.topsrcdir, "chunk_mapping.tar.xz") + with open(CHUNK_MAPPING_ARCHIVE, "wb") as f: + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) + + subprocess.check_call( + [ + "tar", + "-xJf", + CHUNK_MAPPING_ARCHIVE, + "-C", + os.path.dirname(CHUNK_MAPPING_FILE), + ] + ) + os.remove(CHUNK_MAPPING_ARCHIVE) + assert os.path.isfile(CHUNK_MAPPING_FILE) + with open(CHUNK_MAPPING_TAG_FILE, "w") as f: + json.dump( + { + "target_revision": base_revision, + "chunk_mapping_revision": rev, + "download_date": start_time, + }, + f, + ) + return + raise Exception("Could not find suitable coverage data.") + + +def is_a_test(cursor, path): + """Checks the all_tests global and the chunk mapping database to see if a + given file is a test file. + """ + if path in all_tests: + return True + + cursor.execute("SELECT COUNT(*) from chunk_to_test WHERE path=?", (path,)) + if cursor.fetchone()[0]: + return True + + cursor.execute("SELECT COUNT(*) from file_to_test WHERE test=?", (path,)) + if cursor.fetchone()[0]: + return True + + return False + + +def tests_covering_file(cursor, path): + """Returns a set of tests that cover a given source file.""" + cursor.execute("SELECT test FROM file_to_test WHERE source=?", (path,)) + return {e[0] for e in cursor.fetchall()} + + +def tests_in_chunk(cursor, platform, chunk): + """Returns a set of tests that are contained in a given chunk.""" + cursor.execute( + "SELECT path FROM chunk_to_test WHERE platform=? AND chunk=?", (platform, chunk) + ) + # Because of bug 1480103, some entries in this table contain both a file name and a test name, + # separated by a space. With the split, only the file name is kept. + return {e[0].split(" ")[0] for e in cursor.fetchall()} + + +def chunks_covering_file(cursor, path): + """Returns a set of (platform, chunk) tuples with the chunks that cover a given source file.""" + cursor.execute("SELECT platform, chunk FROM file_to_chunk WHERE path=?", (path,)) + return set(cursor.fetchall()) + + +def tests_supported_by_file(path): + """Returns a set of tests that are using the given file as a support-file.""" + return set(all_support_files[path]) + + +def find_tests(changed_files): + """Finds both individual tests and test chunks that should be run to test code changes. + Argument: a list of file paths relative to the source checkout. + + Returns: a (test_files, test_chunks) tuple with two sets. + test_files - contains tests that should be run to verify changes to changed_files. + test_chunks - contains (platform, chunk) tuples with chunks that should be + run. These chunnks do not support running a subset of the tests (like + cppunit or gtest), so the whole chunk must be run. + """ + test_files = set() + test_chunks = set() + files_no_coverage = set() + + with sqlite3.connect(CHUNK_MAPPING_FILE) as conn: + c = conn.cursor() + for path in changed_files: + # If path is a test, add it to the list and continue. + if is_a_test(c, path): + test_files.add(path) + continue + + # Look at the chunk mapping and add all tests that cover this file. + tests = tests_covering_file(c, path) + chunks = chunks_covering_file(c, path) + # If we found tests covering this, then it's not a support-file, so + # save these and continue. + if tests or chunks: + test_files |= tests + test_chunks |= chunks + continue + + # Check if the path is a support-file for any test, by querying test manifests. + tests = tests_supported_by_file(path) + if tests: + test_files |= tests + continue + + # There is no coverage information for this file. + files_no_coverage.add(path) + + files_covered = set(changed_files) - files_no_coverage + test_files = {s.replace("\\", "/") for s in test_files} + + _print_found_tests(files_covered, files_no_coverage, test_files, test_chunks) + + remaining_test_chunks = set() + # For all test_chunks, try to find the tests contained by them in the + # chunk_to_test mapping. + for platform, chunk in test_chunks: + tests = tests_in_chunk(c, platform, chunk) + if tests: + for test in tests: + test_files.add(test.replace("\\", "/")) + else: + remaining_test_chunks.add((platform, chunk)) + + return test_files, remaining_test_chunks + + +def _print_found_tests(files_covered, files_no_coverage, test_files, test_chunks): + """Print a summary of what will be run to the user's terminal.""" + files_covered = sorted(files_covered) + files_no_coverage = sorted(files_no_coverage) + test_files = sorted(test_files) + test_chunks = sorted(test_chunks) + + if files_covered: + print( + "Found {} modified source files with test coverage:".format( + len(files_covered) + ) + ) + for covered in files_covered: + print("\t", covered) + + if files_no_coverage: + print( + "Found {} modified source files with no coverage:".format( + len(files_no_coverage) + ) + ) + for f in files_no_coverage: + print("\t", f) + + if not files_covered: + print("No modified source files are covered by tests.") + elif not files_no_coverage: + print("All modified source files are covered by tests.") + + if test_files: + print("Running {} individual test files.".format(len(test_files))) + else: + print("Could not find any individual tests to run.") + + if test_chunks: + print("Running {} test chunks.".format(len(test_chunks))) + for platform, chunk in test_chunks: + print("\t", platform, chunk) + else: + print("Could not find any test chunks to run.") + + +def filter_tasks_by_chunks(tasks, chunks): + """Find all tasks that will run the given chunks.""" + selected_tasks = set() + for platform, chunk in chunks: + platform = PLATFORM_MAP[platform] + + selected_task = None + for task in tasks: + if not task.startswith(platform): + continue + + if not any( + task[len(platform) + 1 :].endswith(c) for c in [chunk, chunk + "-e10s"] + ): + continue + + assert ( + selected_task is None + ), "Only one task should be selected for a given platform-chunk couple ({} - {}), {} and {} were selected".format( # noqa + platform, chunk, selected_task, task + ) + selected_task = task + + if selected_task is None: + print("Warning: no task found for chunk", platform, chunk) + else: + selected_tasks.add(selected_task) + + return list(selected_tasks) + + +def is_opt_task(task): + """True if the task runs on a supported platform and build type combination. + This is used to remove -ccov/asan/pgo tasks, along with all /debug tasks. + """ + return any(platform in task for platform in OPT_TASK_PATTERNS) + + +def run( + try_config={}, + full=False, + parameters=None, + stage_changes=False, + dry_run=False, + message="{msg}", + closed_tree=False, +): + setup_globals() + download_coverage_mapping(vcs.base_ref) + + changed_sources = vcs.get_outgoing_files() + test_files, test_chunks = find_tests(changed_sources) + if not test_files and not test_chunks: + print("ERROR Could not find any tests or chunks to run.") + return 1 + + tg = generate_tasks(parameters, full) + all_tasks = tg.tasks.keys() + + tasks_by_chunks = filter_tasks_by_chunks(all_tasks, test_chunks) + tasks_by_path = filter_tasks_by_paths(all_tasks, test_files) + tasks = filter(is_opt_task, set(tasks_by_path) | set(tasks_by_chunks)) + tasks = list(tasks) + + if not tasks: + print("ERROR Did not find any matching tasks after filtering.") + return 1 + test_count_message = ( + "{test_count} test file{test_plural} that " + + "cover{test_singular} these changes " + + "({task_count} task{task_plural} to be scheduled)" + ).format( + test_count=len(test_files), + test_plural="" if len(test_files) == 1 else "s", + test_singular="s" if len(test_files) == 1 else "", + task_count=len(tasks), + task_plural="" if len(tasks) == 1 else "s", + ) + print("Found " + test_count_message) + + # Set the test paths to be run by setting MOZHARNESS_TEST_PATHS. + path_env = { + "MOZHARNESS_TEST_PATHS": six.ensure_text( + json.dumps(resolve_tests_by_suite(test_files)) + ) + } + try_config.setdefault("env", {}).update(path_env) + + # Build commit message. + msg = "try coverage - " + test_count_message + return push_to_try( + "coverage", + message.format(msg=msg), + try_task_config=generate_try_task_config("coverage", tasks, try_config), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + ) diff --git a/tools/tryselect/selectors/empty.py b/tools/tryselect/selectors/empty.py new file mode 100644 index 0000000000..2465999c90 --- /dev/null +++ b/tools/tryselect/selectors/empty.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from ..cli import BaseTryParser +from ..push import generate_try_task_config, push_to_try + + +class EmptyParser(BaseTryParser): + name = "empty" + common_groups = ["push"] + + +def run(message="{msg}", stage_changes=False, dry_run=False, closed_tree=False): + msg = 'No try selector specified, use "Add New Jobs" to select tasks.' + return push_to_try( + "empty", + message.format(msg=msg), + try_task_config=generate_try_task_config("empty", []), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + ) diff --git a/tools/tryselect/selectors/fuzzy.py b/tools/tryselect/selectors/fuzzy.py new file mode 100644 index 0000000000..03fc0531d4 --- /dev/null +++ b/tools/tryselect/selectors/fuzzy.py @@ -0,0 +1,255 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import sys +from pathlib import PurePath + +from gecko_taskgraph.target_tasks import filter_by_uncommon_try_tasks +from mach.util import get_state_dir + +from ..cli import BaseTryParser +from ..push import check_working_directory, generate_try_task_config, push_to_try +from ..tasks import filter_tasks_by_paths, generate_tasks +from ..util.fzf import ( + FZF_NOT_FOUND, + PREVIEW_SCRIPT, + format_header, + fzf_bootstrap, + fzf_shortcuts, + run_fzf, +) +from ..util.manage_estimates import ( + download_task_history_data, + make_trimmed_taskgraph_cache, +) + + +class FuzzyParser(BaseTryParser): + name = "fuzzy" + arguments = [ + [ + ["-q", "--query"], + { + "metavar": "STR", + "action": "append", + "default": [], + "help": "Use the given query instead of entering the selection " + "interface. Equivalent to typing <query><ctrl-a><enter> " + "from the interface. Specifying multiple times schedules " + "the union of computed tasks.", + }, + ], + [ + ["-i", "--interactive"], + { + "action": "store_true", + "default": False, + "help": "Force running fzf interactively even when using presets or " + "queries with -q/--query.", + }, + ], + [ + ["-x", "--and"], + { + "dest": "intersection", + "action": "store_true", + "default": False, + "help": "When specifying queries on the command line with -q/--query, " + "use the intersection of tasks rather than the union. This is " + "especially useful for post filtering presets.", + }, + ], + [ + ["-e", "--exact"], + { + "action": "store_true", + "default": False, + "help": "Enable exact match mode. Terms will use an exact match " + "by default, and terms prefixed with ' will become fuzzy.", + }, + ], + [ + ["-u", "--update"], + { + "action": "store_true", + "default": False, + "help": "Update fzf before running.", + }, + ], + [ + ["-s", "--show-estimates"], + { + "action": "store_true", + "default": False, + "help": "Show task duration estimates.", + }, + ], + [ + ["--disable-target-task-filter"], + { + "action": "store_true", + "default": False, + "help": "Some tasks run on mozilla-central but are filtered out " + "of the default list due to resource constraints. This flag " + "disables this filtering.", + }, + ], + ] + common_groups = ["push", "task", "preset"] + task_configs = [ + "artifact", + "browsertime", + "chemspill-prio", + "disable-pgo", + "env", + "gecko-profile", + "path", + "pernosco", + "rebuild", + "routes", + "worker-overrides", + ] + + +def run( + update=False, + query=None, + intersect_query=None, + try_config=None, + full=False, + parameters=None, + save_query=False, + stage_changes=False, + dry_run=False, + message="{msg}", + test_paths=None, + exact=False, + closed_tree=False, + show_estimates=False, + disable_target_task_filter=False, +): + fzf = fzf_bootstrap(update) + + if not fzf: + print(FZF_NOT_FOUND) + return 1 + + push = not stage_changes and not dry_run + check_working_directory(push) + tg = generate_tasks( + parameters, full=full, disable_target_task_filter=disable_target_task_filter + ) + all_tasks = sorted(tg.tasks.keys()) + + # graph_Cache created by generate_tasks, recreate the path to that file. + cache_dir = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph" + ) + if full: + graph_cache = os.path.join(cache_dir, "full_task_graph") + dep_cache = os.path.join(cache_dir, "full_task_dependencies") + target_set = os.path.join(cache_dir, "full_task_set") + else: + graph_cache = os.path.join(cache_dir, "target_task_graph") + dep_cache = os.path.join(cache_dir, "target_task_dependencies") + target_set = os.path.join(cache_dir, "target_task_set") + + if show_estimates: + download_task_history_data(cache_dir=cache_dir) + make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_set) + + if not full and not disable_target_task_filter: + # Put all_tasks into a list because it's used multiple times, and "filter()" + # returns a consumable iterator. + all_tasks = list(filter(filter_by_uncommon_try_tasks, all_tasks)) + + if test_paths: + all_tasks = filter_tasks_by_paths(all_tasks, test_paths) + if not all_tasks: + return 1 + + key_shortcuts = [k + ":" + v for k, v in fzf_shortcuts.items()] + base_cmd = [ + fzf, + "-m", + "--bind", + ",".join(key_shortcuts), + "--header", + format_header(), + "--preview-window=right:30%", + "--print-query", + ] + + if show_estimates: + base_cmd.extend( + [ + "--preview", + '{} {} -g {} -s -c {} -t "{{+f}}"'.format( + str(PurePath(sys.executable)), PREVIEW_SCRIPT, dep_cache, cache_dir + ), + ] + ) + else: + base_cmd.extend( + [ + "--preview", + '{} {} -t "{{+f}}"'.format( + str(PurePath(sys.executable)), PREVIEW_SCRIPT + ), + ] + ) + + if exact: + base_cmd.append("--exact") + + selected = set() + queries = [] + + def get_tasks(query_arg=None, candidate_tasks=all_tasks): + cmd = base_cmd[:] + if query_arg and query_arg != "INTERACTIVE": + cmd.extend(["-f", query_arg]) + + query_str, tasks = run_fzf(cmd, sorted(candidate_tasks)) + queries.append(query_str) + return set(tasks) + + for q in query or []: + selected |= get_tasks(q) + + for q in intersect_query or []: + if not selected: + tasks = get_tasks(q) + selected |= tasks + else: + tasks = get_tasks(q, selected) + selected &= tasks + + if not queries: + selected = get_tasks() + + if not selected: + print("no tasks selected") + return + + if save_query: + return queries + + # build commit message + msg = "Fuzzy" + args = ["query={}".format(q) for q in queries] + if test_paths: + args.append("paths={}".format(":".join(test_paths))) + if args: + msg = "{} {}".format(msg, "&".join(args)) + return push_to_try( + "fuzzy", + message.format(msg=msg), + try_task_config=generate_try_task_config("fuzzy", selected, try_config), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + ) diff --git a/tools/tryselect/selectors/perf.py b/tools/tryselect/selectors/perf.py new file mode 100644 index 0000000000..b570c8d63b --- /dev/null +++ b/tools/tryselect/selectors/perf.py @@ -0,0 +1,1335 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import copy +import itertools +import json +import os +import pathlib +import shutil +import subprocess +from contextlib import redirect_stdout +from datetime import datetime, timedelta + +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozversioncontrol import get_repository_object + +from ..push import generate_try_task_config, push_to_try +from ..util.fzf import ( + FZF_NOT_FOUND, + build_base_cmd, + fzf_bootstrap, + run_fzf, + setup_tasks_for_fzf, +) +from .compare import CompareParser +from .perfselector.classification import ( + Apps, + ClassificationProvider, + Platforms, + Suites, + Variants, +) +from .perfselector.perfcomparators import get_comparator +from .perfselector.utils import LogProcessor + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) +cache_file = pathlib.Path(get_state_dir(), "try_perf_revision_cache.json") + +PERFHERDER_BASE_URL = ( + "https://treeherder.mozilla.org/perfherder/" + "compare?originalProject=try&originalRevision=%s&newProject=try&newRevision=%s" +) +TREEHERDER_TRY_BASE_URL = "https://treeherder.mozilla.org/jobs?repo=try&revision=%s" + +# Prevent users from running more than 300 tests at once. It's possible, but +# it's more likely that a query is broken and is selecting far too much. +MAX_PERF_TASKS = 300 + +# Name of the base category with no variants applied to it +BASE_CATEGORY_NAME = "base" + +# Add environment variable for firefox-android integration. +# This will let us find the APK to upload automatically. However, +# the following option will need to be supplied: +# --browsertime-upload-apk firefox-android +# OR --mozperftest-upload-apk firefox-android +MOZ_FIREFOX_ANDROID_APK_OUTPUT = os.getenv("MOZ_FIREFOX_ANDROID_APK_OUTPUT", None) + + +class InvalidCategoryException(Exception): + """Thrown when a category is found to be invalid. + + See the `PerfParser.run_category_checks()` method for more info. + """ + + pass + + +class APKNotFound(Exception): + """Raised when a user-supplied path to an APK is invalid.""" + + pass + + +class InvalidRegressionDetectorQuery(Exception): + """Thrown when the detector query produces anything other than 1 task.""" + + pass + + +class PerfParser(CompareParser): + name = "perf" + common_groups = ["push", "task"] + task_configs = [ + "artifact", + "browsertime", + "disable-pgo", + "env", + "gecko-profile", + "path", + "rebuild", + ] + + provider = ClassificationProvider() + platforms = provider.platforms + apps = provider.apps + variants = provider.variants + suites = provider.suites + categories = provider.categories + + arguments = [ + [ + ["--show-all"], + { + "action": "store_true", + "default": False, + "help": "Show all available tasks.", + }, + ], + [ + ["--android"], + { + "action": "store_true", + "default": False, + "help": "Show android test categories (disabled by default).", + }, + ], + [ + ["--chrome"], + { + "action": "store_true", + "default": False, + "help": "Show tests available for Chrome-based browsers " + "(disabled by default).", + }, + ], + [ + ["--custom-car"], + { + "action": "store_true", + "default": False, + "help": "Show tests available for Custom Chromium-as-Release (disabled by default).", + }, + ], + [ + ["--safari"], + { + "action": "store_true", + "default": False, + "help": "Show tests available for Safari (disabled by default).", + }, + ], + [ + ["--live-sites"], + { + "action": "store_true", + "default": False, + "help": "Run tasks with live sites (if possible). " + "You can also use the `live-sites` variant.", + }, + ], + [ + ["--profile"], + { + "action": "store_true", + "default": False, + "help": "Run tasks with profiling (if possible). " + "You can also use the `profiling` variant.", + }, + ], + [ + ["--single-run"], + { + "action": "store_true", + "default": False, + "help": "Run tasks without a comparison", + }, + ], + [ + ["-q", "--query"], + { + "type": str, + "default": None, + "help": "Query to run in either the perf-category selector, " + "or the fuzzy selector if --show-all is provided.", + }, + ], + [ + ["--browsertime-upload-apk"], + { + "type": str, + "default": None, + "help": "Path to an APK to upload. Note that this " + "will replace the APK installed in all Android Performance " + "tests. If the Activity, Binary Path, or Intents required " + "change at all relative to the existing GeckoView, and Fenix " + "tasks, then you will need to make fixes in the associated " + "taskcluster files (e.g. taskcluster/ci/test/browsertime-mobile.yml). " + "Alternatively, set MOZ_FIREFOX_ANDROID_APK_OUTPUT to a path to " + "an APK, and then run the command with --browsertime-upload-apk " + "firefox-android. This option will only copy the APK for browsertime, see " + "--mozperftest-upload-apk to upload APKs for startup tests.", + }, + ], + [ + ["--mozperftest-upload-apk"], + { + "type": str, + "default": None, + "help": "See --browsertime-upload-apk. This option does the same " + "thing except it's for mozperftest tests such as the startup ones. " + "Note that those tests only exist through --show-all, as they " + "aren't contained in any existing categories.", + }, + ], + [ + ["--detect-changes"], + { + "action": "store_true", + "default": False, + "help": "Adds a task that detects performance changes using MWU.", + }, + ], + [ + ["--comparator"], + { + "type": str, + "default": "BasePerfComparator", + "help": "Either a path to a file to setup a custom comparison, " + "or a builtin name. See the Firefox source docs for mach try perf for " + "examples of how to build your own, along with the interface.", + }, + ], + [ + ["--comparator-args"], + { + "nargs": "*", + "type": str, + "default": [], + "dest": "comparator_args", + "help": "Arguments provided to the base, and new revision setup stages " + "of the comparator.", + "metavar": "ARG=VALUE", + }, + ], + [ + ["--variants"], + { + "nargs": "*", + "type": str, + "default": [BASE_CATEGORY_NAME], + "dest": "requested_variants", + "choices": list(variants.keys()), + "help": "Select variants to display in the selector from: " + + ", ".join(list(variants.keys())), + "metavar": "", + }, + ], + [ + ["--platforms"], + { + "nargs": "*", + "type": str, + "default": [], + "dest": "requested_platforms", + "choices": list(platforms.keys()), + "help": "Select specific platforms to target. Android only " + "available with --android. Available platforms: " + + ", ".join(list(platforms.keys())), + "metavar": "", + }, + ], + [ + ["--apps"], + { + "nargs": "*", + "type": str, + "default": [], + "dest": "requested_apps", + "choices": list(apps.keys()), + "help": "Select specific applications to target from: " + + ", ".join(list(apps.keys())), + "metavar": "", + }, + ], + [ + ["--extra-args"], + { + "nargs": "*", + "type": str, + "default": [], + "dest": "extra_args", + "help": "Set the extra args " + "(e.x, --extra-args verbose post-startup-delay=1)", + "metavar": "", + }, + ], + ] + + def get_tasks(base_cmd, queries, query_arg=None, candidate_tasks=None): + cmd = base_cmd[:] + if query_arg: + cmd.extend(["-f", query_arg]) + + query_str, tasks = run_fzf(cmd, sorted(candidate_tasks)) + queries.append(query_str) + return set(tasks) + + def get_perf_tasks(base_cmd, all_tg_tasks, perf_categories, query=None): + # Convert the categories to tasks + selected_tasks = set() + queries = [] + + selected_categories = PerfParser.get_tasks( + base_cmd, queries, query, perf_categories + ) + + for category, category_info in perf_categories.items(): + if category not in selected_categories: + continue + print("Gathering tasks for %s category" % category) + + category_tasks = set() + for suite in PerfParser.suites: + # Either perform a query to get the tasks (recommended), or + # use a hardcoded task list + suite_queries = category_info["queries"].get(suite) + + category_suite_tasks = set() + if suite_queries: + print( + "Executing %s queries: %s" % (suite, ", ".join(suite_queries)) + ) + + for perf_query in suite_queries: + if not category_suite_tasks: + # Get all tasks selected with the first query + category_suite_tasks |= PerfParser.get_tasks( + base_cmd, queries, perf_query, all_tg_tasks + ) + else: + # Keep only those tasks that matched in all previous queries + category_suite_tasks &= PerfParser.get_tasks( + base_cmd, queries, perf_query, category_suite_tasks + ) + + if len(category_suite_tasks) == 0: + print("Failed to find any tasks for query: %s" % perf_query) + break + + if category_suite_tasks: + category_tasks |= category_suite_tasks + + if category_info["tasks"]: + category_tasks = set(category_info["tasks"]) & all_tg_tasks + if category_tasks != set(category_info["tasks"]): + print( + "Some expected tasks could not be found: %s" + % ", ".join(category_info["tasks"] - category_tasks) + ) + + if not category_tasks: + print("Could not find any tasks for category %s" % category) + else: + # Add the new tasks to the currently selected ones + selected_tasks |= category_tasks + + return selected_tasks, selected_categories, queries + + def _check_app(app, target): + """Checks if the app exists in the target.""" + if app.value in target: + return True + return False + + def _check_platform(platform, target): + """Checks if the platform, or it's type exists in the target.""" + if ( + platform.value in target + or PerfParser.platforms[platform.value]["platform"] in target + ): + return True + return False + + def _build_initial_decision_matrix(): + # Build first stage of matrix APPS X PLATFORMS + initial_decision_matrix = [] + for platform in Platforms: + platform_row = [] + for app in Apps: + if PerfParser._check_platform( + platform, PerfParser.apps[app.value]["platforms"] + ): + # This app can run on this platform + platform_row.append(True) + else: + platform_row.append(False) + initial_decision_matrix.append(platform_row) + return initial_decision_matrix + + def _build_intermediate_decision_matrix(): + # Second stage of matrix building applies the 2D matrix found above + # to each suite + initial_decision_matrix = PerfParser._build_initial_decision_matrix() + + intermediate_decision_matrix = [] + for suite in Suites: + suite_matrix = copy.deepcopy(initial_decision_matrix) + suite_info = PerfParser.suites[suite.value] + + # Restric the platforms for this suite now + for platform in Platforms: + for app in Apps: + runnable = False + if PerfParser._check_app( + app, suite_info["apps"] + ) and PerfParser._check_platform(platform, suite_info["platforms"]): + runnable = True + suite_matrix[platform][app] = ( + runnable and suite_matrix[platform][app] + ) + + intermediate_decision_matrix.append(suite_matrix) + return intermediate_decision_matrix + + def _build_variants_matrix(): + # Third stage is expanding the intermediate matrix + # across all the variants (non-expanded). Start with the + # intermediate matrix in the list since it provides our + # base case with no variants + intermediate_decision_matrix = PerfParser._build_intermediate_decision_matrix() + + variants_matrix = [] + for variant in Variants: + variant_matrix = copy.deepcopy(intermediate_decision_matrix) + + for suite in Suites: + if variant.value in PerfParser.suites[suite.value]["variants"]: + # Allow the variant through and set it's platforms and apps + # based on how it sets it -> only restrict, don't make allowances + # here + for platform in Platforms: + for app in Apps: + if not ( + PerfParser._check_platform( + platform, + PerfParser.variants[variant.value]["platforms"], + ) + and PerfParser._check_app( + app, PerfParser.variants[variant.value]["apps"] + ) + ): + variant_matrix[suite][platform][app] = False + else: + # This variant matrix needs to be completely False + variant_matrix[suite] = [ + [False] * len(platform_row) + for platform_row in variant_matrix[suite] + ] + + variants_matrix.append(variant_matrix) + + return variants_matrix, intermediate_decision_matrix + + def _build_decision_matrix(): + """Build the decision matrix. + + This method builds the decision matrix that is used + to determine what categories will be shown to the user. + This matrix has the following form (as lists): + - Variants + - Suites + - Platforms + - Apps + + Each element in the 4D Matrix is either True or False and tells us + whether the particular combination is "runnable" according to + the given specifications. This does not mean that the combination + exists, just that it's fully configured in this selector. + + The ("base",) variant combination found in the matrix has + no variants applied to it. At this stage, it's a catch-all for those + categories. The query it uses is reduced further in later stages. + """ + # Get the variants matrix (see methods above) and the intermediate decision + # matrix to act as the base category + ( + variants_matrix, + intermediate_decision_matrix, + ) = PerfParser._build_variants_matrix() + + # Get all possible combinations of the variants + expanded_variants = [ + variant_combination + for set_size in range(len(Variants) + 1) + for variant_combination in itertools.combinations(list(Variants), set_size) + ] + + # Final stage combines the intermediate matrix with the + # expanded variants and leaves a "base" category which + # doesn't have any variant specifications (it catches them all) + decision_matrix = {(BASE_CATEGORY_NAME,): intermediate_decision_matrix} + for variant_combination in expanded_variants: + expanded_variant_matrix = [] + + # Perform an AND operation on the combination of variants + # to determine where this particular combination can run + for suite in Suites: + suite_matrix = [] + suite_variants = PerfParser.suites[suite.value]["variants"] + + # Disable the variant combination if none of them + # are found in the suite + disable_variant = not any( + [variant.value in suite_variants for variant in variant_combination] + ) + + for platform in Platforms: + if disable_variant: + platform_row = [False for _ in Apps] + else: + platform_row = [ + all( + variants_matrix[variant][suite][platform][app] + for variant in variant_combination + if variant.value in suite_variants + ) + for app in Apps + ] + suite_matrix.append(platform_row) + + expanded_variant_matrix.append(suite_matrix) + decision_matrix[variant_combination] = expanded_variant_matrix + + return decision_matrix + + def _skip_with_restrictions(value, restrictions, requested=[]): + """Determines if we should skip an app, platform, or variant. + + We add base here since it's the base category variant that + would always be displayed and it won't affect the app, or + platform selections. + """ + if restrictions is not None and value not in restrictions + [ + BASE_CATEGORY_NAME + ]: + return True + if requested and value not in requested + [BASE_CATEGORY_NAME]: + return True + return False + + def build_category_matrix( + requested_variants=[BASE_CATEGORY_NAME], + requested_platforms=[], + requested_apps=[], + **kwargs, + ): + """Build a decision matrix for all the categories. + + It will have the form: + - Category + - Variants + - ... + """ + # Build the base decision matrix + decision_matrix = PerfParser._build_decision_matrix() + + # Here, the variants are further restricted by the category settings + # using the `_skip_with_restrictions` method. This part also handles + # explicitly requested platforms, apps, and variants. + category_decision_matrix = {} + for category, category_info in PerfParser.categories.items(): + category_matrix = copy.deepcopy(decision_matrix) + + for variant_combination, variant_matrix in decision_matrix.items(): + variant_runnable = True + if BASE_CATEGORY_NAME not in variant_combination: + # Make sure that all portions of the variant combination + # target at least one of the suites in the category + tmp_variant_combination = set( + [v.value for v in variant_combination] + ) + for suite in Suites: + if suite.value not in category_info["suites"]: + continue + tmp_variant_combination = tmp_variant_combination - set( + [ + variant.value + for variant in variant_combination + if variant.value + in PerfParser.suites[suite.value]["variants"] + ] + ) + if tmp_variant_combination: + # If it's not empty, then some variants + # are non-existent + variant_runnable = False + + for suite, platform, app in itertools.product(Suites, Platforms, Apps): + runnable = variant_runnable + + # Disable this combination if there are any variant + # restrictions for this suite, or if the user didn't request it + # (and did request some variants). The same is done below with + # the apps, and platforms. + if any( + PerfParser._skip_with_restrictions( + variant.value if not isinstance(variant, str) else variant, + category_info.get("variant-restrictions", {}).get( + suite.value, None + ), + requested_variants, + ) + for variant in variant_combination + ): + runnable = False + + if PerfParser._skip_with_restrictions( + platform.value, + category_info.get("platform-restrictions", None), + requested_platforms, + ): + runnable = False + + # If the platform is restricted, check if the appropriate + # flags were provided (or appropriate conditions hit). We do + # the same thing for apps below. + if ( + PerfParser.platforms[platform.value].get("restriction", None) + is not None + ): + runnable = runnable and PerfParser.platforms[platform.value][ + "restriction" + ](**kwargs) + + if PerfParser._skip_with_restrictions( + app.value, + category_info.get("app-restrictions", {}).get( + suite.value, None + ), + requested_apps, + ): + runnable = False + if PerfParser.apps[app.value].get("restriction", None) is not None: + runnable = runnable and PerfParser.apps[app.value][ + "restriction" + ](**kwargs) + + category_matrix[variant_combination][suite][platform][app] = ( + runnable and variant_matrix[suite][platform][app] + ) + + category_decision_matrix[category] = category_matrix + + return category_decision_matrix + + def _enable_restriction(restriction, **kwargs): + """Used to simplify checking a restriction.""" + return restriction is not None and restriction(**kwargs) + + def _category_suites(category_info): + """Returns all the suite enum entries in this category.""" + return [suite for suite in Suites if suite.value in category_info["suites"]] + + def _add_variant_queries( + category_info, variant_matrix, variant_combination, platform, queries, app=None + ): + """Used to add the variant queries to various categories.""" + for variant in variant_combination: + for suite in PerfParser._category_suites(category_info): + if (app is not None and variant_matrix[suite][platform][app]) or ( + app is None and any(variant_matrix[suite][platform]) + ): + queries[suite.value].append( + PerfParser.variants[variant.value]["query"] + ) + + def _build_categories(category, category_info, category_matrix): + """Builds the categories to display.""" + categories = {} + + for variant_combination, variant_matrix in category_matrix.items(): + base_category = BASE_CATEGORY_NAME in variant_combination + + for platform in Platforms: + if not any( + any(variant_matrix[suite][platform]) + for suite in PerfParser._category_suites(category_info) + ): + # There are no apps available on this platform in either + # of the requested suites + continue + + # This code has the effect of restricting all suites to + # a platform. This means categories with mixed suites will + # be available even if some suites will no longer run + # given this platform constraint. The reasoning for this is that + # it's unexpected to receive desktop tests when you explcitly + # request android. + platform_queries = { + suite: ( + category_info["query"][suite] + + [PerfParser.platforms[platform.value]["query"]] + ) + for suite in category_info["suites"] + } + + platform_category_name = f"{category} {platform.value}" + platform_category_info = { + "queries": platform_queries, + "tasks": category_info["tasks"], + "platform": platform, + "app": None, + "suites": category_info["suites"], + "base-category": base_category, + "base-category-name": category, + } + for app in Apps: + if not any( + variant_matrix[suite][platform][app] + for suite in PerfParser._category_suites(category_info) + ): + # This app is not available on the given platform + # for any of the suites + continue + + # Add the queries for the app for any suites that need it and + # the variant queries if needed + app_queries = copy.deepcopy(platform_queries) + for suite in Suites: + if suite.value not in app_queries: + continue + app_queries[suite.value].append( + PerfParser.apps[app.value]["query"] + ) + if not base_category: + PerfParser._add_variant_queries( + category_info, + variant_matrix, + variant_combination, + platform, + app_queries, + app=app, + ) + + app_category_name = f"{platform_category_name} {app.value}" + if not base_category: + app_category_name = ( + f"{app_category_name} " + f"{'+'.join([v.value for v in variant_combination])}" + ) + categories[app_category_name] = { + "queries": app_queries, + "tasks": category_info["tasks"], + "platform": platform, + "app": app, + "suites": category_info["suites"], + "base-category": base_category, + } + + if not base_category: + platform_category_name = ( + f"{platform_category_name} " + f"{'+'.join([v.value for v in variant_combination])}" + ) + PerfParser._add_variant_queries( + category_info, + variant_matrix, + variant_combination, + platform, + platform_queries, + ) + categories[platform_category_name] = platform_category_info + + return categories + + def _handle_variant_negations(category, category_info, **kwargs): + """Handle variant negations. + + The reason why we're negating variants here instead of where we add + them to the queries is because we need to iterate over all of the variants + but when we add them, we only look at the variants in the combination. It's + possible to combine these, but that increases the complexity of the code + by quite a bit so it's best to do it separately. + """ + for variant in Variants: + if category_info["base-category"] and variant.value in kwargs.get( + "requested_variants", [BASE_CATEGORY_NAME] + ): + # When some particular variant(s) are requested, and we are at a + # base category, don't negate it. Otherwise, if the variant + # wasn't requested negate it + continue + if variant.value in category: + # If this variant is in the category name, skip negations + continue + if not PerfParser._check_platform( + category_info["platform"], + PerfParser.variants[variant.value]["platforms"], + ): + # Make sure the variant applies to the platform + continue + + for suite in category_info["suites"]: + if variant.value not in PerfParser.suites[suite]["variants"]: + continue + category_info["queries"][suite].append( + PerfParser.variants[variant.value]["negation"] + ) + + def _handle_app_negations(category, category_info, **kwargs): + """Handle app negations. + + This is where the global chrome/safari negations get added. We use kwargs + along with the app restriction method to make this decision. + """ + for app in Apps: + if PerfParser.apps[app.value].get("negation", None) is None: + continue + elif any( + PerfParser.apps[app.value]["negation"] + in category_info["queries"][suite] + for suite in category_info["suites"] + ): + # Already added the negations + continue + if category_info.get("app", None) is not None: + # We only need to handle this for categories that + # don't specify an app + continue + if PerfParser._enable_restriction( + PerfParser.apps[app.value].get("restriction", None), **kwargs + ): + continue + + for suite in category_info["suites"]: + if app.value not in PerfParser.suites[suite]["apps"]: + continue + category_info["queries"][suite].append( + PerfParser.apps[app.value]["negation"] + ) + + def _handle_negations(category, category_info, **kwargs): + """This method handles negations. + + This method should only include things that should be globally applied + to all the queries. The apps are included as chrome is negated if + --chrome isn't provided, and the variants are negated here too. + """ + PerfParser._handle_variant_negations(category, category_info, **kwargs) + PerfParser._handle_app_negations(category, category_info, **kwargs) + + def get_categories(**kwargs): + """Get the categories to be displayed. + + The categories are built using the decision matrices from `build_category_matrix`. + The methods above provide more detail on how this is done. Here, we use + this matrix to determine if we should show a category to a user. + + We also apply the negations for restricted apps/platforms and variants + at the end before displaying the categories. + """ + categories = {} + + # Setup the restrictions, and ease-of-use variants requested (if any) + for variant in Variants: + if PerfParser._enable_restriction( + PerfParser.variants[variant.value].get("restriction", None), **kwargs + ): + kwargs.setdefault("requested_variants", []).append(variant.value) + + category_decision_matrix = PerfParser.build_category_matrix(**kwargs) + + # Now produce the categories by finding all the entries that are True + for category, category_matrix in category_decision_matrix.items(): + categories.update( + PerfParser._build_categories( + category, PerfParser.categories[category], category_matrix + ) + ) + + # Handle the restricted app queries, and variant negations + for category, category_info in categories.items(): + PerfParser._handle_negations(category, category_info, **kwargs) + + return categories + + def inject_change_detector(base_cmd, all_tasks, selected_tasks): + query = "'perftest 'mwu 'detect" + mwu_task = PerfParser.get_tasks(base_cmd, [], query, all_tasks) + + if len(mwu_task) > 1 or len(mwu_task) == 0: + raise InvalidRegressionDetectorQuery( + f"Expected 1 task from change detector " + f"query, but found {len(mwu_task)}" + ) + + selected_tasks |= set(mwu_task) + + def check_cached_revision(selected_tasks, base_commit=None): + """ + If the base_commit parameter does not exist, remove expired cache data. + Cache data format: + { + base_commit[str]: [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-12", + "tasks": ["a-task"], + }, + { + "base_revision_treeherder": "999998888", + "date": "2023-03-12", + "tasks": ["b-task"], + }, + ] + } + + The list represents different pushes with different task selections. + + TODO: See if we can request additional tests on a given base revision. + + :param selected_tasks list: The list of tasks selected by the user + :param base_commit str: The base commit to search + :return: The base_revision_treeherder if found, else None + """ + today = datetime.now() + expired_date = (today - timedelta(weeks=2)).strftime("%Y-%m-%d") + today = today.strftime("%Y-%m-%d") + + if not cache_file.is_file(): + return + + with cache_file.open("r") as f: + cache_data = json.load(f) + + # Remove expired cache data + if base_commit is None: + for cached_base_commit in list(cache_data): + if not isinstance(cache_data[cached_base_commit], list): + # TODO: Remove in the future, this is for backwards-compatibility + # with the previous cache structure + cache_data.pop(cached_base_commit) + else: + # Go through the pushes, and expire any that are too old + new_pushes = [] + for push in cache_data[cached_base_commit]: + if push["date"] > expired_date: + new_pushes.append(push) + # If no pushes are left after expiration, expire the base commit + if new_pushes: + cache_data[cached_base_commit] = new_pushes + else: + cache_data.pop(cached_base_commit) + with cache_file.open("w") as f: + json.dump(cache_data, f, indent=4) + + cached_base_commit = cache_data.get(base_commit, None) + if cached_base_commit: + for push in cached_base_commit: + if set(selected_tasks) <= set(push["tasks"]): + return push["base_revision_treeherder"] + + def save_revision_treeherder(selected_tasks, base_commit, base_revision_treeherder): + """ + Save the base revision of treeherder to the cache. + See "check_cached_revision" for more information about the data structure. + + :param selected_tasks list: The list of tasks selected by the user + :param base_commit str: The base commit to save + :param base_revision_treeherder str: The base revision of treeherder to save + :return: None + """ + today = datetime.now().strftime("%Y-%m-%d") + new_revision = { + "base_revision_treeherder": base_revision_treeherder, + "date": today, + "tasks": list(selected_tasks), + } + cache_data = {} + + if cache_file.is_file(): + with cache_file.open("r") as f: + cache_data = json.load(f) + cache_data.setdefault(base_commit, []).append(new_revision) + else: + cache_data[base_commit] = [new_revision] + + with cache_file.open(mode="w") as f: + json.dump(cache_data, f, indent=4) + + def setup_try_config(try_config, extra_args, base_revision_treeherder=None): + if try_config is None: + try_config = {} + if extra_args: + args = " ".join(extra_args) + try_config.setdefault("env", {})["PERF_FLAGS"] = args + if base_revision_treeherder: + # Reset updated since we no longer need to worry + # about failing while we're on a base commit + try_config.setdefault("env", {})[ + "PERF_BASE_REVISION" + ] = base_revision_treeherder + + def perf_push_to_try( + selected_tasks, + selected_categories, + queries, + try_config, + dry_run, + single_run, + extra_args, + comparator, + comparator_args, + ): + """Perf-specific push to try method. + + This makes use of logic from the CompareParser to do something + very similar except with log redirection. We get the comparison + revisions, then use the repository object to update between revisions + and the LogProcessor for parsing out the revisions that are used + to build the Perfherder links. + """ + vcs = get_repository_object(build.topsrcdir) + compare_commit, current_revision_ref = PerfParser.get_revisions_to_run( + vcs, None + ) + + # Build commit message + msg = "Perf selections={} (queries={})".format( + ",".join(selected_categories), + "&".join([q for q in queries if q is not None and len(q) > 0]), + ) + + # Get the comparator to run + comparator_klass = get_comparator(comparator) + comparator_obj = comparator_klass( + vcs, compare_commit, current_revision_ref, comparator_args + ) + base_comparator = True + if comparator_klass.__name__ != "BasePerfComparator": + base_comparator = False + + new_revision_treeherder = "" + base_revision_treeherder = "" + try: + # redirect_stdout allows us to feed each line into + # a processor that we can use to catch the revision + # while providing real-time output + log_processor = LogProcessor() + + # Push the base revision first. This lets the new revision appear + # first in the Treeherder view, and it also lets us enhance the new + # revision with information about the base run. + base_revision_treeherder = None + if base_comparator: + # Don't cache the base revision when a custom comparison is being performed + # since the base revision is now unique and not general to all pushes + base_revision_treeherder = PerfParser.check_cached_revision( + selected_tasks, compare_commit + ) + + if not (dry_run or single_run or base_revision_treeherder): + # Setup the base revision, and try config. This lets us change the options + # we run the tests with through the PERF_FLAGS environment variable. + base_extra_args = list(extra_args) + base_try_config = copy.deepcopy(try_config) + comparator_obj.setup_base_revision(base_extra_args) + PerfParser.setup_try_config(base_try_config, base_extra_args) + + with redirect_stdout(log_processor): + # XXX Figure out if we can use the `again` selector in some way + # Right now we would need to modify it to be able to do this. + # XXX Fix up the again selector for the perf selector (if it makes sense to) + push_to_try( + "perf-again", + "{msg}".format(msg=msg), + try_task_config=generate_try_task_config( + "fuzzy", selected_tasks, base_try_config + ), + stage_changes=False, + dry_run=dry_run, + closed_tree=False, + allow_log_capture=True, + ) + + base_revision_treeherder = log_processor.revision + if base_comparator: + PerfParser.save_revision_treeherder( + selected_tasks, compare_commit, base_revision_treeherder + ) + + comparator_obj.teardown_base_revision() + + new_extra_args = list(extra_args) + comparator_obj.setup_new_revision(new_extra_args) + PerfParser.setup_try_config( + try_config, + new_extra_args, + base_revision_treeherder=base_revision_treeherder, + ) + + with redirect_stdout(log_processor): + push_to_try( + "perf", + "{msg}".format(msg=msg), + # XXX Figure out if changing `fuzzy` to `perf` will break something + try_task_config=generate_try_task_config( + "fuzzy", selected_tasks, try_config + ), + stage_changes=False, + dry_run=dry_run, + closed_tree=False, + allow_log_capture=True, + ) + + new_revision_treeherder = log_processor.revision + comparator_obj.teardown_new_revision() + + finally: + comparator_obj.teardown() + + return base_revision_treeherder, new_revision_treeherder + + def run( + update=False, + show_all=False, + parameters=None, + try_config=None, + dry_run=False, + single_run=False, + query=None, + detect_changes=False, + rebuild=1, + **kwargs, + ): + # Setup fzf + fzf = fzf_bootstrap(update) + + if not fzf: + print(FZF_NOT_FOUND) + return 1 + + all_tasks, dep_cache, cache_dir = setup_tasks_for_fzf( + not dry_run, + parameters, + full=True, + disable_target_task_filter=False, + ) + base_cmd = build_base_cmd(fzf, dep_cache, cache_dir, show_estimates=False) + + # Perform the selection, then push to try and return the revisions + queries = [] + selected_categories = [] + if not show_all: + # Expand the categories first + categories = PerfParser.get_categories(**kwargs) + selected_tasks, selected_categories, queries = PerfParser.get_perf_tasks( + base_cmd, all_tasks, categories, query=query + ) + else: + selected_tasks = PerfParser.get_tasks(base_cmd, queries, query, all_tasks) + + if len(selected_tasks) == 0: + print("No tasks selected") + return None + + if (len(selected_tasks) * rebuild) > MAX_PERF_TASKS: + print( + "That's a lot of tests selected (%s)!\n" + "These tests won't be triggered. If this was unexpected, " + "please file a bug in Testing :: Performance." % MAX_PERF_TASKS + ) + return None + + if detect_changes: + PerfParser.inject_change_detector(base_cmd, all_tasks, selected_tasks) + + return PerfParser.perf_push_to_try( + selected_tasks, + selected_categories, + queries, + try_config, + dry_run, + single_run, + kwargs.get("extra_args", []), + kwargs.get("comparator", "BasePerfComparator"), + kwargs.get("comparator_args", []), + ) + + def run_category_checks(): + # XXX: Add a jsonschema check for the category definition + # Make sure the queries don't specify variants in them + variant_queries = { + suite: [ + PerfParser.variants[variant]["query"] + for variant in suite_info.get( + "variants", list(PerfParser.variants.keys()) + ) + ] + + [ + PerfParser.variants[variant]["negation"] + for variant in suite_info.get( + "variants", list(PerfParser.variants.keys()) + ) + ] + for suite, suite_info in PerfParser.suites.items() + } + + for category, category_info in PerfParser.categories.items(): + for suite, query in category_info["query"].items(): + if len(variant_queries[suite]) == 0: + # This suite has no variants + continue + if any(any(v in q for q in query) for v in variant_queries[suite]): + raise InvalidCategoryException( + f"The '{category}' category suite query for '{suite}' " + f"uses a variant in it's query '{query}'." + "If you don't want a particular variant use the " + "`variant-restrictions` field in the category." + ) + + return True + + def setup_apk_upload(framework, apk_upload_path): + """Setup the APK for uploading to test on try. + + There are two ways of performing the upload: + (1) Passing a path to an APK with: + --browsertime-upload-apk <PATH/FILE.APK> + --mozperftest-upload-apk <PATH/FILE.APK> + (2) Setting MOZ_FIREFOX_ANDROID_APK_OUTPUT to a path that will + always point to an APK (<PATH/FILE.APK>) that we can upload. + + The file is always copied to testing/raptor/raptor/user_upload.apk to + integrate with minimal changes for simpler cases when using raptor-browsertime. + + For mozperftest, the APK is always uploaded here for the same reasons: + python/mozperftest/mozperftest/user_upload.apk + """ + frameworks_to_locations = { + "browsertime": pathlib.Path( + build.topsrcdir, "testing", "raptor", "raptor", "user_upload.apk" + ), + "mozperftest": pathlib.Path( + build.topsrcdir, + "python", + "mozperftest", + "mozperftest", + "user_upload.apk", + ), + } + + print("Setting up custom APK upload") + if apk_upload_path in ("firefox-android"): + apk_upload_path = MOZ_FIREFOX_ANDROID_APK_OUTPUT + if apk_upload_path is None: + raise APKNotFound( + "MOZ_FIREFOX_ANDROID_APK_OUTPUT is not defined. It should " + "point to an APK to upload." + ) + apk_upload_path = pathlib.Path(apk_upload_path) + if not apk_upload_path.exists() or apk_upload_path.is_dir(): + raise APKNotFound( + "MOZ_FIREFOX_ANDROID_APK_OUTPUT needs to point to an APK." + ) + else: + apk_upload_path = pathlib.Path(apk_upload_path) + if not apk_upload_path.exists(): + raise APKNotFound(f"Path does not exist: {str(apk_upload_path)}") + + print("\nCopying file in-tree for upload...") + shutil.copyfile( + str(apk_upload_path), + frameworks_to_locations[framework], + ) + + hg_cmd = ["hg", "add", str(frameworks_to_locations[framework])] + print( + f"\nRunning the following hg command (RAM warnings are expected):\n" + f" {hg_cmd}" + ) + subprocess.check_output(hg_cmd) + print( + "\nAPK is setup for uploading. Please commit the changes, " + "and re-run this command. \nEnsure you supply the --android, " + "and select the correct tasks (fenix, geckoview) or use " + "--show-all for mozperftest task selection.\n" + ) + + +def run(**kwargs): + if ( + kwargs.get("browsertime_upload_apk") is not None + or kwargs.get("mozperftest_upload_apk") is not None + ): + framework = "browsertime" + upload_apk = kwargs.get("browsertime_upload_apk") + if upload_apk is None: + framework = "mozperftest" + upload_apk = kwargs.get("mozperftest_upload_apk") + + PerfParser.setup_apk_upload(framework, upload_apk) + return + + # Make sure the categories are following + # the rules we've setup + PerfParser.run_category_checks() + PerfParser.check_cached_revision([]) + + revisions = PerfParser.run( + profile=kwargs.get("try_config", {}).get("gecko-profile", False), + rebuild=kwargs.get("try_config", {}).get("rebuild", 1), + **kwargs, + ) + + if revisions is None: + return + + # Provide link to perfherder for comparisons now + if not kwargs.get("single_run", False): + perfcompare_url = PERFHERDER_BASE_URL % revisions + original_try_url = TREEHERDER_TRY_BASE_URL % revisions[0] + local_change_try_url = TREEHERDER_TRY_BASE_URL % revisions[1] + print( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison here " + "once the tests are complete (ensure you select the right " + "framework): %s\n" % perfcompare_url + ) + print("\n*******************************************************") + print("* 2 commits/try-runs are created... *") + print("*******************************************************") + print(f"Base revision's try run: {original_try_url}") + print(f"Local revision's try run: {local_change_try_url}\n") + print( + "If you need any help, you can find us in the #perf-help Matrix channel:\n" + "https://matrix.to/#/#perf-help:mozilla.org\n" + ) + print( + "For more information on the performance tests, see our PerfDocs here:\n" + "https://firefox-source-docs.mozilla.org/testing/perfdocs/" + ) diff --git a/tools/tryselect/selectors/perfselector/__init__.py b/tools/tryselect/selectors/perfselector/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/selectors/perfselector/classification.py b/tools/tryselect/selectors/perfselector/classification.py new file mode 100644 index 0000000000..4ed18ce3e5 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/classification.py @@ -0,0 +1,374 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import enum + + +class ClassificationEnum(enum.Enum): + """This class provides the ability to use Enums as array indices.""" + + @property + def value(self): + return self._value_["value"] + + def __index__(self): + return self._value_["index"] + + def __int__(self): + return self._value_["index"] + + +class Platforms(ClassificationEnum): + ANDROID_A51 = {"value": "android-a51", "index": 0} + ANDROID = {"value": "android", "index": 1} + WINDOWS = {"value": "windows", "index": 2} + LINUX = {"value": "linux", "index": 3} + MACOSX = {"value": "macosx", "index": 4} + DESKTOP = {"value": "desktop", "index": 5} + + +class Apps(ClassificationEnum): + FIREFOX = {"value": "firefox", "index": 0} + CHROME = {"value": "chrome", "index": 1} + CHROMIUM = {"value": "chromium", "index": 2} + GECKOVIEW = {"value": "geckoview", "index": 3} + FENIX = {"value": "fenix", "index": 4} + CHROME_M = {"value": "chrome-m", "index": 5} + SAFARI = {"value": "safari", "index": 6} + CHROMIUM_RELEASE = {"value": "custom-car", "index": 7} + + +class Suites(ClassificationEnum): + RAPTOR = {"value": "raptor", "index": 0} + TALOS = {"value": "talos", "index": 1} + AWSY = {"value": "awsy", "index": 2} + + +class Variants(ClassificationEnum): + NO_FISSION = {"value": "no-fission", "index": 0} + BYTECODE_CACHED = {"value": "bytecode-cached", "index": 1} + LIVE_SITES = {"value": "live-sites", "index": 2} + PROFILING = {"value": "profiling", "index": 3} + SWR = {"value": "swr", "index": 4} + + +""" +The following methods and constants are used for restricting +certain platforms and applications such as chrome, safari, and +android tests. These all require a flag such as --android to +enable (see build_category_matrix for more info). +""" + + +def check_for_android(android=False, **kwargs): + return android + + +def check_for_chrome(chrome=False, **kwargs): + return chrome + + +def check_for_custom_car(custom_car=False, **kwargs): + return custom_car + + +def check_for_safari(safari=False, **kwargs): + return safari + + +def check_for_live_sites(live_sites=False, **kwargs): + return live_sites + + +def check_for_profile(profile=False, **kwargs): + return profile + + +class ClassificationProvider: + @property + def platforms(self): + return { + Platforms.ANDROID_A51.value: { + "query": "'android 'a51 'shippable 'aarch64", + "restriction": check_for_android, + "platform": Platforms.ANDROID.value, + }, + Platforms.ANDROID.value: { + # The android, and android-a51 queries are expected to be the same, + # we don't want to run the tests on other mobile platforms. + "query": "'android 'a51 'shippable 'aarch64", + "restriction": check_for_android, + "platform": Platforms.ANDROID.value, + }, + Platforms.WINDOWS.value: { + "query": "!-32 'windows 'shippable", + "platform": Platforms.DESKTOP.value, + }, + Platforms.LINUX.value: { + "query": "!clang 'linux 'shippable", + "platform": Platforms.DESKTOP.value, + }, + Platforms.MACOSX.value: { + "query": "'osx 'shippable", + "platform": Platforms.DESKTOP.value, + }, + Platforms.DESKTOP.value: { + "query": "!android 'shippable !-32 !clang", + "platform": Platforms.DESKTOP.value, + }, + } + + @property + def apps(self): + return { + Apps.FIREFOX.value: { + "query": "!chrom !geckoview !fenix !safari !custom-car", + "platforms": [Platforms.DESKTOP.value], + }, + Apps.CHROME.value: { + "query": "'chrome", + "negation": "!chrom", + "restriction": check_for_chrome, + "platforms": [Platforms.DESKTOP.value], + }, + Apps.CHROMIUM.value: { + "query": "'chromium", + "negation": "!chrom", + "restriction": check_for_chrome, + "platforms": [Platforms.DESKTOP.value], + }, + Apps.GECKOVIEW.value: { + "query": "'geckoview", + "platforms": [Platforms.ANDROID.value], + }, + Apps.FENIX.value: { + "query": "'fenix", + "platforms": [Platforms.ANDROID.value], + }, + Apps.CHROME_M.value: { + "query": "'chrome-m", + "negation": "!chrom", + "restriction": check_for_chrome, + "platforms": [Platforms.ANDROID.value], + }, + Apps.SAFARI.value: { + "query": "'safari", + "negation": "!safari", + "restriction": check_for_safari, + "platforms": [Platforms.MACOSX.value], + }, + Apps.CHROMIUM_RELEASE.value: { + "query": "'custom-car", + "negation": "!custom-car", + "restriction": check_for_custom_car, + "platforms": [Platforms.LINUX.value, Platforms.WINDOWS.value], + }, + } + + @property + def variants(self): + return { + Variants.NO_FISSION.value: { + "query": "'nofis", + "negation": "!nofis", + "platforms": [Platforms.ANDROID.value], + "apps": [Apps.FENIX.value, Apps.GECKOVIEW.value], + }, + Variants.BYTECODE_CACHED.value: { + "query": "'bytecode", + "negation": "!bytecode", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, + Variants.LIVE_SITES.value: { + "query": "'live", + "negation": "!live", + "restriction": check_for_live_sites, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": [ # XXX No live CaR tests + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + Apps.SAFARI.value, + ], + }, + Variants.PROFILING.value: { + "query": "'profil", + "negation": "!profil", + "restriction": check_for_profile, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": [Apps.FIREFOX.value, Apps.GECKOVIEW.value, Apps.FENIX.value], + }, + Variants.SWR.value: { + "query": "'swr", + "negation": "!swr", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, + } + + @property + def suites(self): + return { + Suites.RAPTOR.value: { + "apps": list(self.apps.keys()), + "platforms": list(self.platforms.keys()), + "variants": [ + Variants.NO_FISSION.value, + Variants.LIVE_SITES.value, + Variants.PROFILING.value, + Variants.BYTECODE_CACHED.value, + ], + }, + Suites.TALOS.value: { + "apps": [Apps.FIREFOX.value], + "platforms": [Platforms.DESKTOP.value], + "variants": [ + Variants.PROFILING.value, + Variants.SWR.value, + ], + }, + Suites.AWSY.value: { + "apps": [Apps.FIREFOX.value], + "platforms": [Platforms.DESKTOP.value], + "variants": [], + }, + } + + """ + Here you can find the base categories that are defined for the perf + selector. The following fields are available: + * query: Set the queries to use for each suite you need. + * suites: The suites that are needed for this category. + * tasks: A hard-coded list of tasks to select. + * platforms: The platforms that it can run on. + * app-restrictions: A list of apps that the category can run. + * variant-restrictions: A list of variants available for each suite. + + Note that setting the App/Variant-Restriction fields should be used to + restrict the available apps and variants, not expand them. + """ + + @property + def categories(self): + return { + "Pageload": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'tp6"], + }, + "suites": [Suites.RAPTOR.value], + "tasks": [], + }, + "Pageload (essential)": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'tp6 'essential"], + }, + "variant-restrictions": { + Suites.RAPTOR.value: [Variants.NO_FISSION.value] + }, + "suites": [Suites.RAPTOR.value], + "app-restrictions": { + Suites.RAPTOR.value: [ + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + Apps.CHROMIUM_RELEASE.value, + ], + }, + "tasks": [], + }, + "Speedometer 3": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'speedometer3"], + }, + "variant-restrictions": { + Suites.RAPTOR.value: [Variants.NO_FISSION.value] + }, + "suites": [Suites.RAPTOR.value], + "app-restrictions": {}, + "tasks": [], + }, + "Responsiveness": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'responsive"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "app-restrictions": { + Suites.RAPTOR.value: [ + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + ], + }, + "tasks": [], + }, + "Benchmarks": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'benchmark"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "tasks": [], + }, + "DAMP (Devtools)": { + "query": { + Suites.TALOS.value: ["'talos 'damp"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + }, + "Talos PerfTests": { + "query": { + Suites.TALOS.value: ["'talos"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + }, + "Resource Usage": { + "query": { + Suites.TALOS.value: ["'talos 'xperf | 'tp5"], + Suites.RAPTOR.value: ["'power 'osx"], + Suites.AWSY.value: ["'awsy"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value, Suites.AWSY.value], + "platform-restrictions": [Platforms.DESKTOP.value], + "variant-restrictions": { + Suites.RAPTOR.value: [], + Suites.TALOS.value: [], + }, + "app-restrictions": { + Suites.RAPTOR.value: [Apps.FIREFOX.value], + Suites.TALOS.value: [Apps.FIREFOX.value], + }, + "tasks": [], + }, + "Graphics, & Media Playback": { + "query": { + # XXX This might not be an exhaustive list for talos atm + Suites.TALOS.value: ["'talos 'svgr | 'bcv | 'webgl"], + Suites.RAPTOR.value: ["'browsertime 'youtube-playback"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value], + "variant-restrictions": { + Suites.RAPTOR.value: [Variants.NO_FISSION.value] + }, + "app-restrictions": { + Suites.RAPTOR.value: [ + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + ], + }, + "tasks": [], + }, + } diff --git a/tools/tryselect/selectors/perfselector/perfcomparators.py b/tools/tryselect/selectors/perfselector/perfcomparators.py new file mode 100644 index 0000000000..fce35fe562 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/perfcomparators.py @@ -0,0 +1,258 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import importlib +import inspect +import pathlib + +BUILTIN_COMPARATORS = {} + + +class ComparatorNotFound(Exception): + """Raised when we can't find the specified comparator. + + Triggered when either the comparator name is incorrect for a builtin one, + or when a path to a specified comparator cannot be found. + """ + + pass + + +class GithubRequestFailure(Exception): + """Raised when we hit a failure during PR link parsing.""" + + pass + + +class BadComparatorArgs(Exception): + """Raised when the args given to the comparator are incorrect.""" + + pass + + +def comparator(comparator_klass): + BUILTIN_COMPARATORS[comparator_klass.__name__] = comparator_klass + return comparator_klass + + +@comparator +class BasePerfComparator: + def __init__(self, vcs, compare_commit, current_revision_ref, comparator_args): + """Initialize the standard/default settings for Comparators. + + :param vcs object: Used for updating the local repo. + :param compare_commit str: The base revision found for the local repo. + :param current_revision_ref str: The current revision of the local repo. + :param comparator_args list: List of comparator args in the format NAME=VALUE. + """ + self.vcs = vcs + self.compare_commit = compare_commit + self.current_revision_ref = current_revision_ref + self.comparator_args = comparator_args + + # Used to ensure that the local repo gets cleaned up appropriately on failures + self._updated = False + + def setup_base_revision(self, extra_args): + """Setup the base try run/revision. + + In this case, we update to the repo to the base revision and + push that to try. The extra_args can be used to set additional + arguments for Raptor (not available for other harnesses). + + :param extra_args list: A list of extra arguments to pass to the try tasks. + """ + self.vcs.update(self.compare_commit) + self._updated = True + + def teardown_base_revision(self): + """Teardown the setup for the base revision.""" + if self._updated: + self.vcs.update(self.current_revision_ref) + self._updated = False + + def setup_new_revision(self, extra_args): + """Setup the new try run/revision. + + Note that the extra_args are reset between the base, and new revision runs. + + :param extra_args list: A list of extra arguments to pass to the try tasks. + """ + pass + + def teardown_new_revision(self): + """Teardown the new run/revision setup.""" + pass + + def teardown(self): + """Teardown for failures. + + This method can be used for ensuring that the repo is cleaned up + when a failure is hit at any point in the process of doing the + new/base revision setups, or the pushes to try. + """ + self.teardown_base_revision() + + +def get_github_pull_request_info(link): + """Returns information about a PR link. + + This method accepts a Github link in either of these formats: + https://github.com/mozilla-mobile/firefox-android/pull/1627, + https://github.com/mozilla-mobile/firefox-android/pull/1876/commits/17c7350cc37a4a85cea140a7ce54e9fd037b5365 #noqa + + and returns the Github link, branch, and revision of the commit. + """ + from urllib.parse import urlparse + + import requests + + # Parse the url, and get all the necessary info + parsed_url = urlparse(link) + path_parts = parsed_url.path.strip("/").split("/") + owner, repo = path_parts[0], path_parts[1] + pr_number = path_parts[-1] + + if "/pull/" not in parsed_url.path: + raise GithubRequestFailure( + f"Link for Github PR is invalid (missing /pull/): {link}" + ) + + # Get the commit being targeted in the PR + pr_commit = None + if "/commits/" in parsed_url.path: + pr_commit = path_parts[-1] + pr_number = path_parts[-3] + + # Make the request, and get the PR info, otherwise, + # raise an exception if the response code is not 200 + api_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}" + response = requests.get(api_url) + if response.status_code == 200: + link_info = response.json() + return ( + link_info["head"]["repo"]["html_url"], + pr_commit if pr_commit else link_info["head"]["sha"], + link_info["head"]["ref"], + ) + + raise GithubRequestFailure( + f"The following url returned a non-200 status code: {api_url}" + ) + + +@comparator +class BenchmarkComparator(BasePerfComparator): + def _get_benchmark_info(self, arg_prefix): + # Get the flag from the comparator args + benchmark_info = {"repo": None, "branch": None, "revision": None, "link": None} + for arg in self.comparator_args: + if arg.startswith(arg_prefix): + _, settings = arg.split(arg_prefix) + setting, val = settings.split("=") + if setting not in benchmark_info: + raise BadComparatorArgs( + f"Unknown argument provided `{setting}`. Only the following " + f"are available (prefixed with `{arg_prefix}`): " + f"{list(benchmark_info.keys())}" + ) + benchmark_info[setting] = val + + # Parse the link for any required information + if benchmark_info.get("link", None) is not None: + ( + benchmark_info["repo"], + benchmark_info["revision"], + benchmark_info["branch"], + ) = get_github_pull_request_info(benchmark_info["link"]) + + return benchmark_info + + def _setup_benchmark_args(self, extra_args, benchmark_info): + # Setup the arguments for Raptor + extra_args.append(f"benchmark-repository={benchmark_info['repo']}") + extra_args.append(f"benchmark-revision={benchmark_info['revision']}") + + if benchmark_info.get("branch", None): + extra_args.append(f"benchmark-branch={benchmark_info['branch']}") + + def setup_base_revision(self, extra_args): + """Sets up the options for a base benchmark revision run. + + Checks for a `base-link` in the + command and adds the appropriate commands to the extra_args + which will be added to the PERF_FLAGS environment variable. + + If that isn't provided, then you must provide the repo, branch, + and revision directly through these (branch is optional): + + base-repo=https://github.com/mozilla-mobile/firefox-android + base-branch=main + base-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365 + + Otherwise, we'll use the default mach try perf + base behaviour. + + TODO: Get the information automatically from a commit link. Github + API doesn't provide the branch name from a link like that. + """ + base_info = self._get_benchmark_info("base-") + + # If no options were provided, use the default BasePerfComparator behaviour + if not any(v is not None for v in base_info.values()): + raise BadComparatorArgs( + f"Could not find the correct base-revision arguments in: {self.comparator_args}" + ) + + self._setup_benchmark_args(extra_args, base_info) + + def setup_new_revision(self, extra_args): + """Sets up the options for a new benchmark revision run. + + Same as `setup_base_revision`, except it uses + `new-` as the prefix instead of `base-`. + """ + new_info = self._get_benchmark_info("new-") + + # If no options were provided, use the default BasePerfComparator behaviour + if not any(v is not None for v in new_info.values()): + raise BadComparatorArgs( + f"Could not find the correct new-revision arguments in: {self.comparator_args}" + ) + + self._setup_benchmark_args(extra_args, new_info) + + +def get_comparator(comparator): + if comparator in BUILTIN_COMPARATORS: + return BUILTIN_COMPARATORS[comparator] + + file = pathlib.Path(comparator) + if not file.exists(): + raise ComparatorNotFound( + f"Expected either a path to a file containing a comparator, or a " + f"builtin comparator from this list: {BUILTIN_COMPARATORS.keys()}" + ) + + # Importing a source file directly + spec = importlib.util.spec_from_file_location(name=file.name, location=comparator) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + members = inspect.getmembers( + module, + lambda c: inspect.isclass(c) + and issubclass(c, BasePerfComparator) + and c != BasePerfComparator, + ) + + if not members: + raise ComparatorNotFound( + f"The path {comparator} was found but it was not a valid comparator. " + f"Ensure it is a subclass of BasePerfComparator and optionally contains the " + f"following methods: " + f"{', '.join(inspect.getmembers(BasePerfComparator, predicate=inspect.ismethod))}" + ) + + return members[0][-1] diff --git a/tools/tryselect/selectors/perfselector/utils.py b/tools/tryselect/selectors/perfselector/utils.py new file mode 100644 index 0000000000..105d003091 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/utils.py @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re +import sys + +REVISION_MATCHER = re.compile(r"remote:.*/try/rev/([\w]*)[ \t]*$") + + +class LogProcessor: + def __init__(self): + self.buf = "" + self.stdout = sys.__stdout__ + self._revision = None + + @property + def revision(self): + return self._revision + + def write(self, buf): + while buf: + try: + newline_index = buf.index("\n") + except ValueError: + # No newline, wait for next call + self.buf += buf + break + + # Get data up to next newline and combine with previously buffered data + data = self.buf + buf[: newline_index + 1] + buf = buf[newline_index + 1 :] + + # Reset buffer then output line + self.buf = "" + if data.strip() == "": + continue + self.stdout.write(data.strip("\n") + "\n") + + # Check if a temporary commit wa created + match = REVISION_MATCHER.match(data) + if match: + # Last line found is the revision we want + self._revision = match.group(1) diff --git a/tools/tryselect/selectors/preview.py b/tools/tryselect/selectors/preview.py new file mode 100644 index 0000000000..1d232af9e0 --- /dev/null +++ b/tools/tryselect/selectors/preview.py @@ -0,0 +1,102 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""This script is intended to be called through fzf as a preview formatter.""" + + +import argparse +import os +import sys + +here = os.path.abspath(os.path.dirname(__file__)) +sys.path.insert(0, os.path.join(os.path.dirname(here), "util")) +from estimates import duration_summary + + +def process_args(): + """Process preview arguments.""" + argparser = argparse.ArgumentParser() + argparser.add_argument( + "-s", + "--show-estimates", + action="store_true", + help="Show task duration estimates (default: False)", + ) + argparser.add_argument( + "-g", + "--graph-cache", + type=str, + default=None, + help="Filename of task graph dependencies", + ) + argparser.add_argument( + "-c", + "--cache_dir", + type=str, + default=None, + help="Path to cache directory containing task durations", + ) + argparser.add_argument( + "-t", + "--tasklist", + type=str, + default=None, + help="Path to temporary file containing the selected tasks", + ) + return argparser.parse_args() + + +def plain_display(taskfile): + """Original preview window display.""" + with open(taskfile) as f: + tasklist = [line.strip() for line in f] + print("\n".join(sorted(tasklist))) + + +def duration_display(graph_cache_file, taskfile, cache_dir): + """Preview window display with task durations + metadata.""" + with open(taskfile) as f: + tasklist = [line.strip() for line in f] + + durations = duration_summary(graph_cache_file, tasklist, cache_dir) + output = "" + max_columns = int(os.environ["FZF_PREVIEW_COLUMNS"]) + + output += "\nSelected tasks take {}\n".format(durations["selected_duration"]) + output += "+{} dependencies, total {}\n".format( + durations["dependency_count"], + durations["selected_duration"] + durations["dependency_duration"], + ) + + if durations.get("percentile"): + output += "This is in the top {}% of requests\n".format( + 100 - durations["percentile"] + ) + + output += "Estimated finish in {} at {}".format( + durations["wall_duration_seconds"], durations["eta_datetime"].strftime("%H:%M") + ) + + duration_width = 5 # show five numbers at most. + output += "{:>{width}}\n".format("Duration", width=max_columns) + for task in tasklist: + duration = durations["task_durations"].get(task, 0.0) + output += "{:{align}{width}} {:{nalign}{nwidth}}s\n".format( + task, + duration, + align="<", + width=max_columns - (duration_width + 2), # 2: space and 's' + nalign=">", + nwidth=duration_width, + ) + + print(output) + + +if __name__ == "__main__": + args = process_args() + if args.show_estimates and os.path.isdir(args.cache_dir): + duration_display(args.graph_cache, args.tasklist, args.cache_dir) + else: + plain_display(args.tasklist) diff --git a/tools/tryselect/selectors/release.py b/tools/tryselect/selectors/release.py new file mode 100644 index 0000000000..a828b3d038 --- /dev/null +++ b/tools/tryselect/selectors/release.py @@ -0,0 +1,159 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os + +import attr +import yaml +from mozilla_version.gecko import FirefoxVersion + +from ..cli import BaseTryParser +from ..push import push_to_try, vcs + +TARGET_TASKS = { + "staging": "staging_release_builds", + "release-sim": "release_simulation", +} + + +def read_file(path): + with open(path) as fh: + return fh.read() + + +class ReleaseParser(BaseTryParser): + name = "release" + arguments = [ + [ + ["-v", "--version"], + { + "metavar": "STR", + "required": True, + "action": "store", + "type": FirefoxVersion.parse, + "help": "The version number to use for the staging release.", + }, + ], + [ + ["--migration"], + { + "metavar": "STR", + "action": "append", + "dest": "migrations", + "choices": [ + "central-to-beta", + "beta-to-release", + "early-to-late-beta", + "release-to-esr", + ], + "help": "Migration to run for the release (can be specified multiple times).", + }, + ], + [ + ["--no-limit-locales"], + { + "action": "store_false", + "dest": "limit_locales", + "help": "Don't build a limited number of locales in the staging release.", + }, + ], + [ + ["--tasks"], + { + "choices": TARGET_TASKS.keys(), + "default": "staging", + "help": "Which tasks to run on-push.", + }, + ], + ] + common_groups = ["push"] + task_configs = ["disable-pgo", "worker-overrides"] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.set_defaults(migrations=[]) + + +def run( + version, + migrations, + limit_locales, + tasks, + try_config=None, + stage_changes=False, + dry_run=False, + message="{msg}", + closed_tree=False, +): + app_version = attr.evolve(version, beta_number=None, is_esr=False) + + files_to_change = { + "browser/config/version.txt": "{}\n".format(app_version), + "browser/config/version_display.txt": "{}\n".format(version), + "config/milestone.txt": "{}\n".format(app_version), + } + with open("browser/config/version.txt") as f: + current_version = FirefoxVersion.parse(f.read()) + format_options = { + "current_major_version": current_version.major_number, + "next_major_version": version.major_number, + "current_weave_version": current_version.major_number + 2, + "next_weave_version": version.major_number + 2, + } + + if "beta-to-release" in migrations and "early-to-late-beta" not in migrations: + migrations.append("early-to-late-beta") + + release_type = version.version_type.name.lower() + if release_type not in ("beta", "release", "esr"): + raise Exception( + "Can't do staging release for version: {} type: {}".format( + version, version.version_type + ) + ) + elif release_type == "esr": + release_type += str(version.major_number) + task_config = { + "version": 2, + "parameters": { + "target_tasks_method": TARGET_TASKS[tasks], + "optimize_target_tasks": True, + "release_type": release_type, + }, + } + if try_config: + task_config["parameters"]["try_task_config"] = try_config + + with open(os.path.join(vcs.path, "taskcluster/ci/config.yml")) as f: + migration_configs = yaml.safe_load(f) + for migration in migrations: + migration_config = migration_configs["merge-automation"]["behaviors"][migration] + for (path, from_, to) in migration_config["replacements"]: + if path in files_to_change: + contents = files_to_change[path] + else: + contents = read_file(path) + from_ = from_.format(**format_options) + to = to.format(**format_options) + files_to_change[path] = contents.replace(from_, to) + + if limit_locales: + files_to_change["browser/locales/l10n-changesets.json"] = read_file( + os.path.join(vcs.path, "browser/locales/l10n-onchange-changesets.json") + ) + files_to_change["browser/locales/shipped-locales"] = "en-US\n" + read_file( + os.path.join(vcs.path, "browser/locales/onchange-locales") + ) + + msg = "staging release: {}".format(version) + return push_to_try( + "release", + message.format(msg=msg), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + try_task_config=task_config, + files_to_change=files_to_change, + ) diff --git a/tools/tryselect/selectors/scriptworker.py b/tools/tryselect/selectors/scriptworker.py new file mode 100644 index 0000000000..984ad26a88 --- /dev/null +++ b/tools/tryselect/selectors/scriptworker.py @@ -0,0 +1,177 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import sys + +import requests +from gecko_taskgraph.util.taskgraph import find_existing_tasks +from taskgraph.parameters import Parameters +from taskgraph.util.taskcluster import find_task_id, get_artifact, get_session + +from ..cli import BaseTryParser +from ..push import push_to_try + +TASK_TYPES = { + "linux-signing": [ + "build-signing-linux-shippable/opt", + "build-signing-linux64-shippable/opt", + "build-signing-win64-shippable/opt", + "build-signing-win32-shippable/opt", + "repackage-signing-win64-shippable/opt", + "repackage-signing-win32-shippable/opt", + "repackage-signing-msi-win32-shippable/opt", + "repackage-signing-msi-win64-shippable/opt", + "mar-signing-linux64-shippable/opt", + ], + "linux-signing-partial": ["partials-signing-linux64-shippable/opt"], + "mac-signing": ["build-signing-macosx64-shippable/opt"], + "beetmover-candidates": ["beetmover-repackage-linux64-shippable/opt"], + "bouncer-submit": ["release-bouncer-sub-firefox"], + "balrog-submit": [ + "release-balrog-submit-toplevel-firefox", + "balrog-linux64-shippable/opt", + ], + "tree": ["release-early-tagging-firefox", "release-version-bump-firefox"], +} + +RELEASE_TO_BRANCH = { + "beta": "releases/mozilla-beta", + "release": "releases/mozilla-release", +} + + +class ScriptworkerParser(BaseTryParser): + name = "scriptworker" + arguments = [ + [ + ["task_type"], + { + "choices": ["list"] + list(TASK_TYPES.keys()), + "metavar": "TASK-TYPE", + "help": "Scriptworker task types to run. (Use `list` to show possibilities)", + }, + ], + [ + ["--release-type"], + { + "choices": ["nightly"] + list(RELEASE_TO_BRANCH.keys()), + "default": "beta", + "help": "Release type to run", + }, + ], + ] + + common_groups = ["push"] + task_configs = ["worker-overrides", "routes"] + + +def get_releases(branch): + response = requests.get( + "https://shipitapi-public.services.mozilla.com/releases", + params={"product": "firefox", "branch": branch, "status": "shipped"}, + headers={"Accept": "application/json"}, + ) + response.raise_for_status() + return response.json() + + +def get_release_graph(release): + for phase in release["phases"]: + if phase["name"] in ("ship_firefox",): + return phase["actionTaskId"] + raise Exception("No ship phase.") + + +def get_nightly_graph(): + return find_task_id( + "gecko.v2.mozilla-central.latest.taskgraph.decision-nightly-desktop" + ) + + +def print_available_task_types(): + print("Available task types:") + for task_type, tasks in TASK_TYPES.items(): + print(" " * 4 + "{}:".format(task_type)) + for task in tasks: + print(" " * 8 + "- {}".format(task)) + + +def get_hg_file(parameters, path): + session = get_session() + response = session.get(parameters.file_url(path)) + response.raise_for_status() + return response.content + + +def run( + task_type, + release_type, + try_config=None, + stage_changes=False, + dry_run=False, + message="{msg}", + closed_tree=False, +): + if task_type == "list": + print_available_task_types() + sys.exit(0) + + if release_type == "nightly": + previous_graph = get_nightly_graph() + else: + release = get_releases(RELEASE_TO_BRANCH[release_type])[-1] + previous_graph = get_release_graph(release) + existing_tasks = find_existing_tasks([previous_graph]) + + previous_parameters = Parameters( + strict=False, **get_artifact(previous_graph, "public/parameters.yml") + ) + + # Copy L10n configuration from the commit the release we are using was + # based on. This *should* ensure that the chunking of L10n tasks is the + # same between graphs. + files_to_change = { + path: get_hg_file(previous_parameters, path) + for path in [ + "browser/locales/l10n-changesets.json", + "browser/locales/shipped-locales", + ] + } + + try_config = try_config or {} + task_config = { + "version": 2, + "parameters": { + "existing_tasks": existing_tasks, + "try_task_config": try_config, + "try_mode": "try_task_config", + }, + } + for param in ( + "app_version", + "build_number", + "next_version", + "release_history", + "release_product", + "release_type", + "version", + ): + task_config["parameters"][param] = previous_parameters[param] + + try_config["tasks"] = TASK_TYPES[task_type] + for label in try_config["tasks"]: + if label in existing_tasks: + del existing_tasks[label] + + msg = "scriptworker tests: {}".format(task_type) + return push_to_try( + "scriptworker", + message.format(msg=msg), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + try_task_config=task_config, + files_to_change=files_to_change, + ) diff --git a/tools/tryselect/selectors/syntax.py b/tools/tryselect/selectors/syntax.py new file mode 100644 index 0000000000..ad643f8314 --- /dev/null +++ b/tools/tryselect/selectors/syntax.py @@ -0,0 +1,708 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import re +import sys +from collections import defaultdict + +import mozpack.path as mozpath +from moztest.resolve import TestResolver + +from ..cli import BaseTryParser +from ..push import build, push_to_try + +here = os.path.abspath(os.path.dirname(__file__)) + + +class SyntaxParser(BaseTryParser): + name = "syntax" + arguments = [ + [ + ["paths"], + { + "nargs": "*", + "default": [], + "help": "Paths to search for tests to run on try.", + }, + ], + [ + ["-b", "--build"], + { + "dest": "builds", + "default": "do", + "help": "Build types to run (d for debug, o for optimized).", + }, + ], + [ + ["-p", "--platform"], + { + "dest": "platforms", + "action": "append", + "help": "Platforms to run (required if not found in the environment as " + "AUTOTRY_PLATFORM_HINT).", + }, + ], + [ + ["-u", "--unittests"], + { + "dest": "tests", + "action": "append", + "help": "Test suites to run in their entirety.", + }, + ], + [ + ["-t", "--talos"], + { + "action": "append", + "help": "Talos suites to run.", + }, + ], + [ + ["-j", "--jobs"], + { + "action": "append", + "help": "Job tasks to run.", + }, + ], + [ + ["--tag"], + { + "dest": "tags", + "action": "append", + "help": "Restrict tests to the given tag (may be specified multiple times).", + }, + ], + [ + ["--and"], + { + "action": "store_true", + "dest": "intersection", + "help": "When -u and paths are supplied run only the intersection of the " + "tests specified by the two arguments.", + }, + ], + [ + ["--no-artifact"], + { + "action": "store_true", + "help": "Disable artifact builds even if --enable-artifact-builds is set " + "in the mozconfig.", + }, + ], + [ + ["-v", "--verbose"], + { + "dest": "verbose", + "action": "store_true", + "default": False, + "help": "Print detailed information about the resulting test selection " + "and commands performed.", + }, + ], + ] + + # Arguments we will accept on the command line and pass through to try + # syntax with no further intervention. The set is taken from + # http://trychooser.pub.build.mozilla.org with a few additions. + # + # Note that the meaning of store_false and store_true arguments is + # not preserved here, as we're only using these to echo the literal + # arguments to another consumer. Specifying either store_false or + # store_true here will have an equivalent effect. + pass_through_arguments = { + "--rebuild": { + "action": "store", + "dest": "rebuild", + "help": "Re-trigger all test jobs (up to 20 times)", + }, + "--rebuild-talos": { + "action": "store", + "dest": "rebuild_talos", + "help": "Re-trigger all talos jobs", + }, + "--interactive": { + "action": "store_true", + "dest": "interactive", + "help": "Allow ssh-like access to running test containers", + }, + "--no-retry": { + "action": "store_true", + "dest": "no_retry", + "help": "Do not retrigger failed tests", + }, + "--setenv": { + "action": "append", + "dest": "setenv", + "help": "Set the corresponding variable in the test environment for " + "applicable harnesses.", + }, + "-f": { + "action": "store_true", + "dest": "failure_emails", + "help": "Request failure emails only", + }, + "--failure-emails": { + "action": "store_true", + "dest": "failure_emails", + "help": "Request failure emails only", + }, + "-e": { + "action": "store_true", + "dest": "all_emails", + "help": "Request all emails", + }, + "--all-emails": { + "action": "store_true", + "dest": "all_emails", + "help": "Request all emails", + }, + "--artifact": { + "action": "store_true", + "dest": "artifact", + "help": "Force artifact builds where possible.", + }, + "--upload-xdbs": { + "action": "store_true", + "dest": "upload_xdbs", + "help": "Upload XDB compilation db files generated by hazard build", + }, + } + task_configs = [] + + def __init__(self, *args, **kwargs): + BaseTryParser.__init__(self, *args, **kwargs) + + group = self.add_argument_group("pass-through arguments") + for arg, opts in self.pass_through_arguments.items(): + group.add_argument(arg, **opts) + + +class TryArgumentTokenizer: + symbols = [ + ("separator", ","), + ("list_start", r"\["), + ("list_end", r"\]"), + ("item", r"([^,\[\]\s][^,\[\]]+)"), + ("space", r"\s+"), + ] + token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols)) + + def tokenize(self, data): + for match in self.token_re.finditer(data): + symbol = match.lastgroup + data = match.group(symbol) + if symbol == "space": + pass + else: + yield symbol, data + + +class TryArgumentParser: + """Simple three-state parser for handling expressions + of the from "foo[sub item, another], bar,baz". This takes + input from the TryArgumentTokenizer and runs through a small + state machine, returning a dictionary of {top-level-item:[sub_items]} + i.e. the above would result in + {"foo":["sub item", "another"], "bar": [], "baz": []} + In the case of invalid input a ValueError is raised.""" + + EOF = object() + + def __init__(self): + self.reset() + + def reset(self): + self.tokens = None + self.current_item = None + self.data = {} + self.token = None + self.state = None + + def parse(self, tokens): + self.reset() + self.tokens = tokens + self.consume() + self.state = self.item_state + while self.token[0] != self.EOF: + self.state() + return self.data + + def consume(self): + try: + self.token = next(self.tokens) + except StopIteration: + self.token = (self.EOF, None) + + def expect(self, *types): + if self.token[0] not in types: + raise ValueError( + "Error parsing try string, unexpected %s" % (self.token[0]) + ) + + def item_state(self): + self.expect("item") + value = self.token[1].strip() + if value not in self.data: + self.data[value] = [] + self.current_item = value + self.consume() + if self.token[0] == "separator": + self.consume() + elif self.token[0] == "list_start": + self.consume() + self.state = self.subitem_state + elif self.token[0] == self.EOF: + pass + else: + raise ValueError + + def subitem_state(self): + self.expect("item") + value = self.token[1].strip() + self.data[self.current_item].append(value) + self.consume() + if self.token[0] == "separator": + self.consume() + elif self.token[0] == "list_end": + self.consume() + self.state = self.after_list_end_state + else: + raise ValueError + + def after_list_end_state(self): + self.expect("separator") + self.consume() + self.state = self.item_state + + +def parse_arg(arg): + tokenizer = TryArgumentTokenizer() + parser = TryArgumentParser() + return parser.parse(tokenizer.tokenize(arg)) + + +class AutoTry: + + # Maps from flavors to the job names needed to run that flavour + flavor_jobs = { + "mochitest": ["mochitest-1", "mochitest-e10s-1"], + "xpcshell": ["xpcshell"], + "chrome": ["mochitest-o"], + "browser-a11y": ["mochitest-ba"], + "browser-media": ["mochitest-bmda"], + "browser-chrome": [ + "mochitest-browser-chrome-1", + "mochitest-e10s-browser-chrome-1", + "mochitest-browser-chrome-e10s-1", + ], + "devtools-chrome": [ + "mochitest-devtools-chrome-1", + "mochitest-e10s-devtools-chrome-1", + "mochitest-devtools-chrome-e10s-1", + ], + "crashtest": ["crashtest", "crashtest-e10s"], + "reftest": ["reftest", "reftest-e10s"], + "remote": ["mochitest-remote"], + "web-platform-tests": ["web-platform-tests-1"], + } + + flavor_suites = { + "mochitest": "mochitests", + "xpcshell": "xpcshell", + "chrome": "mochitest-o", + "browser-chrome": "mochitest-bc", + "browser-a11y": "mochitest-ba", + "browser-media": "mochitest-bmda", + "devtools-chrome": "mochitest-dt", + "crashtest": "crashtest", + "reftest": "reftest", + "web-platform-tests": "web-platform-tests", + } + + compiled_suites = [ + "cppunit", + "gtest", + "jittest", + ] + + common_suites = [ + "cppunit", + "crashtest", + "firefox-ui-functional", + "geckoview", + "geckoview-junit", + "gtest", + "jittest", + "jsreftest", + "marionette", + "marionette-e10s", + "mochitests", + "reftest", + "robocop", + "web-platform-tests", + "xpcshell", + ] + + def __init__(self): + self.topsrcdir = build.topsrcdir + self._resolver = None + + @property + def resolver(self): + if self._resolver is None: + self._resolver = TestResolver.from_environment(cwd=here) + return self._resolver + + @classmethod + def split_try_string(cls, data): + return re.findall(r"(?:\[.*?\]|\S)+", data) + + def paths_by_flavor(self, paths=None, tags=None): + paths_by_flavor = defaultdict(set) + + if not (paths or tags): + return dict(paths_by_flavor) + + tests = list(self.resolver.resolve_tests(paths=paths, tags=tags)) + + for t in tests: + if t["flavor"] in self.flavor_suites: + flavor = t["flavor"] + if "subsuite" in t and t["subsuite"] == "devtools": + flavor = "devtools-chrome" + + if "subsuite" in t and t["subsuite"] == "a11y": + flavor = "browser-a11y" + + if "subsuite" in t and t["subsuite"] == "media-bc": + flavor = "browser-media" + + if flavor in ["crashtest", "reftest"]: + manifest_relpath = os.path.relpath(t["manifest"], self.topsrcdir) + paths_by_flavor[flavor].add(os.path.dirname(manifest_relpath)) + elif "dir_relpath" in t: + paths_by_flavor[flavor].add(t["dir_relpath"]) + else: + file_relpath = os.path.relpath(t["path"], self.topsrcdir) + dir_relpath = os.path.dirname(file_relpath) + paths_by_flavor[flavor].add(dir_relpath) + + for flavor, path_set in paths_by_flavor.items(): + paths_by_flavor[flavor] = self.deduplicate_prefixes(path_set, paths) + + return dict(paths_by_flavor) + + def deduplicate_prefixes(self, path_set, input_paths): + # Removes paths redundant to test selection in the given path set. + # If a path was passed on the commandline that is the prefix of a + # path in our set, we only need to include the specified prefix to + # run the intended tests (every test in "layout/base" will run if + # "layout" is passed to the reftest harness). + removals = set() + additions = set() + + for path in path_set: + full_path = path + while path: + path, _ = os.path.split(path) + if path in input_paths: + removals.add(full_path) + additions.add(path) + + return additions | (path_set - removals) + + def remove_duplicates(self, paths_by_flavor, tests): + rv = {} + for item in paths_by_flavor: + if self.flavor_suites[item] not in tests: + rv[item] = paths_by_flavor[item].copy() + return rv + + def calc_try_syntax( + self, + platforms, + tests, + talos, + jobs, + builds, + paths_by_flavor, + tags, + extras, + intersection, + ): + parts = ["try:"] + + if platforms: + parts.extend(["-b", builds, "-p", ",".join(platforms)]) + + suites = tests if not intersection else {} + paths = set() + for flavor, flavor_tests in paths_by_flavor.items(): + suite = self.flavor_suites[flavor] + if suite not in suites and (not intersection or suite in tests): + for job_name in self.flavor_jobs[flavor]: + for test in flavor_tests: + paths.add("{}:{}".format(flavor, test)) + suites[job_name] = tests.get(suite, []) + + # intersection implies tests are expected + if intersection and not suites: + raise ValueError("No tests found matching filters") + + if extras.get("artifact") and any([p.endswith("-nightly") for p in platforms]): + print( + 'You asked for |--artifact| but "-nightly" platforms don\'t have artifacts. ' + "Running without |--artifact| instead." + ) + del extras["artifact"] + + if extras.get("artifact"): + rejected = [] + for suite in suites.keys(): + if any([suite.startswith(c) for c in self.compiled_suites]): + rejected.append(suite) + if rejected: + raise ValueError( + "You can't run {} with " + "--artifact option.".format(", ".join(rejected)) + ) + + if extras.get("artifact") and "all" in suites.keys(): + non_compiled_suites = set(self.common_suites) - set(self.compiled_suites) + message = ( + "You asked for |-u all| with |--artifact| but compiled-code tests ({tests})" + " can't run against an artifact build. Running (-u {non_compiled_suites}) " + "instead." + ) + string_format = { + "tests": ",".join(self.compiled_suites), + "non_compiled_suites": ",".join(non_compiled_suites), + } + print(message.format(**string_format)) + del suites["all"] + suites.update({suite_name: None for suite_name in non_compiled_suites}) + + if suites: + parts.append("-u") + parts.append( + ",".join( + "{}{}".format(k, "[%s]" % ",".join(v) if v else "") + for k, v in sorted(suites.items()) + ) + ) + + if talos: + parts.append("-t") + parts.append( + ",".join( + "{}{}".format(k, "[%s]" % ",".join(v) if v else "") + for k, v in sorted(talos.items()) + ) + ) + + if jobs: + parts.append("-j") + parts.append(",".join(jobs)) + + if tags: + parts.append(" ".join("--tag %s" % t for t in tags)) + + if paths: + parts.append("--try-test-paths %s" % " ".join(sorted(paths))) + + args_by_dest = { + v["dest"]: k for k, v in SyntaxParser.pass_through_arguments.items() + } + for dest, value in extras.items(): + assert dest in args_by_dest + arg = args_by_dest[dest] + action = SyntaxParser.pass_through_arguments[arg]["action"] + if action == "store": + parts.append(arg) + parts.append(value) + if action == "append": + for e in value: + parts.append(arg) + parts.append(e) + if action in ("store_true", "store_false"): + parts.append(arg) + + return " ".join(parts) + + def normalise_list(self, items, allow_subitems=False): + rv = defaultdict(list) + for item in items: + parsed = parse_arg(item) + for key, values in parsed.items(): + rv[key].extend(values) + + if not allow_subitems: + if not all(item == [] for item in rv.values()): + raise ValueError("Unexpected subitems in argument") + return rv.keys() + else: + return rv + + def validate_args(self, **kwargs): + tests_selected = kwargs["tests"] or kwargs["paths"] or kwargs["tags"] + if kwargs["platforms"] is None and (kwargs["jobs"] is None or tests_selected): + if "AUTOTRY_PLATFORM_HINT" in os.environ: + kwargs["platforms"] = [os.environ["AUTOTRY_PLATFORM_HINT"]] + elif tests_selected: + print("Must specify platform when selecting tests.") + sys.exit(1) + else: + print( + "Either platforms or jobs must be specified as an argument to autotry." + ) + sys.exit(1) + + try: + platforms = ( + self.normalise_list(kwargs["platforms"]) if kwargs["platforms"] else {} + ) + except ValueError as e: + print("Error parsing -p argument:\n%s" % e) + sys.exit(1) + + try: + tests = ( + self.normalise_list(kwargs["tests"], allow_subitems=True) + if kwargs["tests"] + else {} + ) + except ValueError as e: + print("Error parsing -u argument ({}):\n{}".format(kwargs["tests"], e)) + sys.exit(1) + + try: + talos = ( + self.normalise_list(kwargs["talos"], allow_subitems=True) + if kwargs["talos"] + else [] + ) + except ValueError as e: + print("Error parsing -t argument:\n%s" % e) + sys.exit(1) + + try: + jobs = self.normalise_list(kwargs["jobs"]) if kwargs["jobs"] else {} + except ValueError as e: + print("Error parsing -j argument:\n%s" % e) + sys.exit(1) + + paths = [] + for p in kwargs["paths"]: + p = mozpath.normpath(os.path.abspath(p)) + if not (os.path.isdir(p) and p.startswith(self.topsrcdir)): + print( + 'Specified path "%s" is not a directory under the srcdir,' + " unable to specify tests outside of the srcdir" % p + ) + sys.exit(1) + if len(p) <= len(self.topsrcdir): + print( + 'Specified path "%s" is at the top of the srcdir and would' + " select all tests." % p + ) + sys.exit(1) + paths.append(os.path.relpath(p, self.topsrcdir)) + + try: + tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else [] + except ValueError as e: + print("Error parsing --tags argument:\n%s" % e) + sys.exit(1) + + extra_values = {k["dest"] for k in SyntaxParser.pass_through_arguments.values()} + extra_args = {k: v for k, v in kwargs.items() if k in extra_values and v} + + return kwargs["builds"], platforms, tests, talos, jobs, paths, tags, extra_args + + def run(self, **kwargs): + if not any(kwargs[item] for item in ("paths", "tests", "tags")): + kwargs["paths"] = set() + kwargs["tags"] = set() + + builds, platforms, tests, talos, jobs, paths, tags, extra = self.validate_args( + **kwargs + ) + + if paths or tags: + paths = [ + os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir) + for item in paths + ] + paths_by_flavor = self.paths_by_flavor(paths=paths, tags=tags) + + if not paths_by_flavor and not tests: + print( + "No tests were found when attempting to resolve paths:\n\n\t%s" + % paths + ) + sys.exit(1) + + if not kwargs["intersection"]: + paths_by_flavor = self.remove_duplicates(paths_by_flavor, tests) + else: + paths_by_flavor = {} + + # No point in dealing with artifacts if we aren't running any builds + local_artifact_build = False + if platforms: + local_artifact_build = kwargs.get("local_artifact_build", False) + + # Add --artifact if --enable-artifact-builds is set ... + if local_artifact_build: + extra["artifact"] = True + # ... unless --no-artifact is explicitly given. + if kwargs["no_artifact"]: + if "artifact" in extra: + del extra["artifact"] + + try: + msg = self.calc_try_syntax( + platforms, + tests, + talos, + jobs, + builds, + paths_by_flavor, + tags, + extra, + kwargs["intersection"], + ) + except ValueError as e: + print(e) + sys.exit(1) + + if local_artifact_build and not kwargs["no_artifact"]: + print( + "mozconfig has --enable-artifact-builds; including " + "--artifact flag in try syntax (use --no-artifact " + "to override)" + ) + + if kwargs["verbose"] and paths_by_flavor: + print("The following tests will be selected: ") + for flavor, paths in paths_by_flavor.items(): + print("{}: {}".format(flavor, ",".join(paths))) + + if kwargs["verbose"]: + print("The following try syntax was calculated:\n%s" % msg) + + push_to_try( + "syntax", + kwargs["message"].format(msg=msg), + stage_changes=kwargs["stage_changes"], + dry_run=kwargs["dry_run"], + closed_tree=kwargs["closed_tree"], + ) + + +def run(**kwargs): + at = AutoTry() + return at.run(**kwargs) diff --git a/tools/tryselect/task_config.py b/tools/tryselect/task_config.py new file mode 100644 index 0000000000..07a39d39e9 --- /dev/null +++ b/tools/tryselect/task_config.py @@ -0,0 +1,530 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Templates provide a way of modifying the task definition of selected tasks. +They are added to 'try_task_config.json' and processed by the transforms. +""" + + +import json +import os +import subprocess +import sys +from abc import ABCMeta, abstractmethod, abstractproperty +from argparse import SUPPRESS, Action +from textwrap import dedent + +import mozpack.path as mozpath +import six +from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject + +from .tasks import resolve_tests_by_suite + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + + +class TryConfig: + __metaclass__ = ABCMeta + + def __init__(self): + self.dests = set() + + def add_arguments(self, parser): + for cli, kwargs in self.arguments: + action = parser.add_argument(*cli, **kwargs) + self.dests.add(action.dest) + + @abstractproperty + def arguments(self): + pass + + @abstractmethod + def try_config(self, **kwargs): + pass + + def validate(self, **kwargs): + pass + + +class Artifact(TryConfig): + + arguments = [ + [ + ["--artifact"], + {"action": "store_true", "help": "Force artifact builds where possible."}, + ], + [ + ["--no-artifact"], + { + "action": "store_true", + "help": "Disable artifact builds even if being used locally.", + }, + ], + ] + + def add_arguments(self, parser): + group = parser.add_mutually_exclusive_group() + return super().add_arguments(group) + + @classmethod + def is_artifact_build(cls): + try: + return build.substs.get("MOZ_ARTIFACT_BUILDS", False) + except BuildEnvironmentNotFoundException: + return False + + def try_config(self, artifact, no_artifact, **kwargs): + if artifact: + return {"use-artifact-builds": True, "disable-pgo": True} + + if no_artifact: + return + + if self.is_artifact_build(): + print("Artifact builds enabled, pass --no-artifact to disable") + return {"use-artifact-builds": True, "disable-pgo": True} + + +class Pernosco(TryConfig): + arguments = [ + [ + ["--pernosco"], + { + "action": "store_true", + "default": None, + "help": "Opt-in to analysis by the Pernosco debugging service.", + }, + ], + [ + ["--no-pernosco"], + { + "dest": "pernosco", + "action": "store_false", + "default": None, + "help": "Opt-out of the Pernosco debugging service (if you are on the whitelist).", + }, + ], + ] + + def add_arguments(self, parser): + group = parser.add_mutually_exclusive_group() + return super().add_arguments(group) + + def try_config(self, pernosco, **kwargs): + if pernosco is None: + return + + if pernosco: + try: + # The Pernosco service currently requires a Mozilla e-mail address to + # log in. Prevent people with non-Mozilla addresses from using this + # flag so they don't end up consuming time and resources only to + # realize they can't actually log in and see the reports. + cmd = ["ssh", "-G", "hg.mozilla.org"] + output = subprocess.check_output( + cmd, universal_newlines=True + ).splitlines() + address = [ + l.rsplit(" ", 1)[-1] for l in output if l.startswith("user") + ][0] + if not address.endswith("@mozilla.com"): + print( + dedent( + """\ + Pernosco requires a Mozilla e-mail address to view its reports. Please + push to try with an @mozilla.com address to use --pernosco. + + Current user: {} + """.format( + address + ) + ) + ) + sys.exit(1) + + except (subprocess.CalledProcessError, IndexError): + print("warning: failed to detect current user for 'hg.mozilla.org'") + print("Pernosco requires a Mozilla e-mail address to view its reports.") + while True: + answer = input( + "Do you have an @mozilla.com address? [Y/n]: " + ).lower() + if answer == "n": + sys.exit(1) + elif answer == "y": + break + + return { + "env": { + "PERNOSCO": str(int(pernosco)), + } + } + + def validate(self, **kwargs): + if kwargs["try_config"].get("use-artifact-builds"): + print( + "Pernosco does not support artifact builds at this time. " + "Please try again with '--no-artifact'." + ) + sys.exit(1) + + +class Path(TryConfig): + + arguments = [ + [ + ["paths"], + { + "nargs": "*", + "default": [], + "help": "Run tasks containing tests under the specified path(s).", + }, + ], + ] + + def try_config(self, paths, **kwargs): + if not paths: + return + + for p in paths: + if not os.path.exists(p): + print("error: '{}' is not a valid path.".format(p), file=sys.stderr) + sys.exit(1) + + paths = [ + mozpath.relpath(mozpath.join(os.getcwd(), p), build.topsrcdir) + for p in paths + ] + return { + "env": { + "MOZHARNESS_TEST_PATHS": six.ensure_text( + json.dumps(resolve_tests_by_suite(paths)) + ), + } + } + + +class Environment(TryConfig): + + arguments = [ + [ + ["--env"], + { + "action": "append", + "default": None, + "help": "Set an environment variable, of the form FOO=BAR. " + "Can be passed in multiple times.", + }, + ], + ] + + def try_config(self, env, **kwargs): + if not env: + return + return { + "env": dict(e.split("=", 1) for e in env), + } + + +class RangeAction(Action): + def __init__(self, min, max, *args, **kwargs): + self.min = min + self.max = max + kwargs["metavar"] = "[{}-{}]".format(self.min, self.max) + super().__init__(*args, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + name = option_string or self.dest + if values < self.min: + parser.error("{} can not be less than {}".format(name, self.min)) + if values > self.max: + parser.error("{} can not be more than {}".format(name, self.max)) + setattr(namespace, self.dest, values) + + +class Rebuild(TryConfig): + + arguments = [ + [ + ["--rebuild"], + { + "action": RangeAction, + "min": 2, + "max": 20, + "default": None, + "type": int, + "help": "Rebuild all selected tasks the specified number of times.", + }, + ], + ] + + def try_config(self, rebuild, **kwargs): + if not rebuild: + return + + if kwargs.get("full") and rebuild > 3: + print( + "warning: limiting --rebuild to 3 when using --full. " + "Use custom push actions to add more." + ) + rebuild = 3 + + return { + "rebuild": rebuild, + } + + +class Routes(TryConfig): + arguments = [ + [ + ["--route"], + { + "action": "append", + "dest": "routes", + "help": ( + "Additional route to add to the tasks " + "(note: these will not be added to the decision task)" + ), + }, + ], + ] + + def try_config(self, routes, **kwargs): + if routes: + return { + "routes": routes, + } + + +class ChemspillPrio(TryConfig): + + arguments = [ + [ + ["--chemspill-prio"], + { + "action": "store_true", + "help": "Run at a higher priority than most try jobs (chemspills only).", + }, + ], + ] + + def try_config(self, chemspill_prio, **kwargs): + if chemspill_prio: + return {"chemspill-prio": {}} + + +class GeckoProfile(TryConfig): + arguments = [ + [ + ["--gecko-profile"], + { + "dest": "profile", + "action": "store_true", + "default": False, + "help": "Create and upload a gecko profile during talos/raptor tasks.", + }, + ], + [ + ["--gecko-profile-interval"], + { + "dest": "gecko_profile_interval", + "type": float, + "help": "How frequently to take samples (ms)", + }, + ], + [ + ["--gecko-profile-entries"], + { + "dest": "gecko_profile_entries", + "type": int, + "help": "How many samples to take with the profiler", + }, + ], + [ + ["--gecko-profile-features"], + { + "dest": "gecko_profile_features", + "type": str, + "default": None, + "help": "Set the features enabled for the profiler.", + }, + ], + [ + ["--gecko-profile-threads"], + { + "dest": "gecko_profile_threads", + "type": str, + "help": "Comma-separated list of threads to sample.", + }, + ], + # For backwards compatibility + [ + ["--talos-profile"], + { + "dest": "profile", + "action": "store_true", + "default": False, + "help": SUPPRESS, + }, + ], + # This is added for consistency with the 'syntax' selector + [ + ["--geckoProfile"], + { + "dest": "profile", + "action": "store_true", + "default": False, + "help": SUPPRESS, + }, + ], + ] + + def try_config( + self, + profile, + gecko_profile_interval, + gecko_profile_entries, + gecko_profile_features, + gecko_profile_threads, + **kwargs + ): + if profile or not all( + s is None for s in (gecko_profile_features, gecko_profile_threads) + ): + cfg = { + "gecko-profile": True, + "gecko-profile-interval": gecko_profile_interval, + "gecko-profile-entries": gecko_profile_entries, + "gecko-profile-features": gecko_profile_features, + "gecko-profile-threads": gecko_profile_threads, + } + return {key: value for key, value in cfg.items() if value is not None} + + +class Browsertime(TryConfig): + arguments = [ + [ + ["--browsertime"], + { + "action": "store_true", + "help": "Use browsertime during Raptor tasks.", + }, + ], + ] + + def try_config(self, browsertime, **kwargs): + if browsertime: + return { + "browsertime": True, + } + + +class DisablePgo(TryConfig): + + arguments = [ + [ + ["--disable-pgo"], + { + "action": "store_true", + "help": "Don't run PGO builds", + }, + ], + ] + + def try_config(self, disable_pgo, **kwargs): + if disable_pgo: + return { + "disable-pgo": True, + } + + +class WorkerOverrides(TryConfig): + + arguments = [ + [ + ["--worker-override"], + { + "action": "append", + "dest": "worker_overrides", + "help": ( + "Override the worker pool used for a given taskgraph worker alias. " + "The argument should be `<alias>=<worker-pool>`. " + "Can be specified multiple times." + ), + }, + ], + [ + ["--worker-suffix"], + { + "action": "append", + "dest": "worker_suffixes", + "help": ( + "Override the worker pool used for a given taskgraph worker alias, " + "by appending a suffix to the work-pool. " + "The argument should be `<alias>=<suffix>`. " + "Can be specified multiple times." + ), + }, + ], + ] + + def try_config(self, worker_overrides, worker_suffixes, **kwargs): + from gecko_taskgraph.util.workertypes import get_worker_type + from taskgraph.config import load_graph_config + + overrides = {} + if worker_overrides: + for override in worker_overrides: + alias, worker_pool = override.split("=", 1) + if alias in overrides: + print( + "Can't override worker alias {alias} more than once. " + "Already set to use {previous}, but also asked to use {new}.".format( + alias=alias, previous=overrides[alias], new=worker_pool + ) + ) + sys.exit(1) + overrides[alias] = worker_pool + + if worker_suffixes: + root = build.topsrcdir + root = os.path.join(root, "taskcluster", "ci") + graph_config = load_graph_config(root) + for worker_suffix in worker_suffixes: + alias, suffix = worker_suffix.split("=", 1) + if alias in overrides: + print( + "Can't override worker alias {alias} more than once. " + "Already set to use {previous}, but also asked " + "to add suffix {suffix}.".format( + alias=alias, previous=overrides[alias], suffix=suffix + ) + ) + sys.exit(1) + provisioner, worker_type = get_worker_type( + graph_config, worker_type=alias, parameters={"level": "1"} + ) + overrides[alias] = "{provisioner}/{worker_type}{suffix}".format( + provisioner=provisioner, worker_type=worker_type, suffix=suffix + ) + + if overrides: + return {"worker-overrides": overrides} + + +all_task_configs = { + "artifact": Artifact, + "browsertime": Browsertime, + "chemspill-prio": ChemspillPrio, + "disable-pgo": DisablePgo, + "env": Environment, + "gecko-profile": GeckoProfile, + "path": Path, + "pernosco": Pernosco, + "rebuild": Rebuild, + "routes": Routes, + "worker-overrides": WorkerOverrides, +} diff --git a/tools/tryselect/tasks.py b/tools/tryselect/tasks.py new file mode 100644 index 0000000000..bca6b3de1b --- /dev/null +++ b/tools/tryselect/tasks.py @@ -0,0 +1,186 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +import re +import sys +from collections import defaultdict + +import mozpack.path as mozpath +import taskgraph +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozpack.files import FileFinder +from moztest.resolve import TestManifestLoader, TestResolver, get_suite_definition +from taskgraph.generator import TaskGraphGenerator +from taskgraph.parameters import ParameterMismatch, parameters_loader +from taskgraph.taskgraph import TaskGraph + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + +PARAMETER_MISMATCH = """ +ERROR - The parameters being used to generate tasks differ from those expected +by your working copy: + + {} + +To fix this, either rebase onto the latest mozilla-central or pass in +-p/--parameters. For more information on how to define parameters, see: +https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/mach.html#parameters +""" + + +def invalidate(cache): + try: + cmod = os.path.getmtime(cache) + except OSError as e: + # File does not exist. We catch OSError rather than use `isfile` + # because the recommended watchman hook could possibly invalidate the + # cache in-between the check to `isfile` and the call to `getmtime` + # below. + if e.errno == 2: + return + raise + + tc_dir = os.path.join(build.topsrcdir, "taskcluster") + tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir)) + + if tmod > cmod: + os.remove(cache) + + +def cache_key(attr, params, disable_target_task_filter): + key = attr + if params and params["project"] not in ("autoland", "mozilla-central"): + key += f"-{params['project']}" + + if disable_target_task_filter and "full" not in attr: + key += "-uncommon" + return key + + +def generate_tasks(params=None, full=False, disable_target_task_filter=False): + attr = "full_task_set" if full else "target_task_set" + target_tasks_method = ( + "try_select_tasks" + if not disable_target_task_filter + else "try_select_tasks_uncommon" + ) + params = parameters_loader( + params, + strict=False, + overrides={ + "try_mode": "try_select", + "target_tasks_method": target_tasks_method, + }, + ) + root = os.path.join(build.topsrcdir, "taskcluster", "ci") + taskgraph.fast = True + generator = TaskGraphGenerator(root_dir=root, parameters=params) + + cache_dir = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph" + ) + key = cache_key(attr, generator.parameters, disable_target_task_filter) + cache = os.path.join(cache_dir, key) + + invalidate(cache) + if os.path.isfile(cache): + with open(cache) as fh: + return TaskGraph.from_json(json.load(fh))[1] + + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + + print("Task configuration changed, generating {}".format(attr.replace("_", " "))) + + cwd = os.getcwd() + os.chdir(build.topsrcdir) + + def generate(attr): + try: + tg = getattr(generator, attr) + except ParameterMismatch as e: + print(PARAMETER_MISMATCH.format(e.args[0])) + sys.exit(1) + + # write cache + key = cache_key(attr, generator.parameters, disable_target_task_filter) + with open(os.path.join(cache_dir, key), "w") as fh: + json.dump(tg.to_json(), fh) + return tg + + # Cache both full_task_set and target_task_set regardless of whether or not + # --full was requested. Caching is cheap and can potentially save a lot of + # time. + tg_full = generate("full_task_set") + tg_target = generate("target_task_set") + + # discard results from these, we only need cache. + if full: + generate("full_task_graph") + generate("target_task_graph") + + os.chdir(cwd) + if full: + return tg_full + return tg_target + + +def filter_tasks_by_paths(tasks, paths): + resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader) + run_suites, run_tests = resolver.resolve_metadata(paths) + flavors = {(t["flavor"], t.get("subsuite")) for t in run_tests} + + task_regexes = set() + for flavor, subsuite in flavors: + _, suite = get_suite_definition(flavor, subsuite, strict=True) + if "task_regex" not in suite: + print( + "warning: no tasks could be resolved from flavor '{}'{}".format( + flavor, " and subsuite '{}'".format(subsuite) if subsuite else "" + ) + ) + continue + + task_regexes.update(suite["task_regex"]) + + def match_task(task): + return any(re.search(pattern, task) for pattern in task_regexes) + + return filter(match_task, tasks) + + +def resolve_tests_by_suite(paths): + resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader) + _, run_tests = resolver.resolve_metadata(paths) + + suite_to_tests = defaultdict(list) + + # A dictionary containing all the input paths that we haven't yet + # assigned to a specific test flavor. + remaining_paths_by_suite = defaultdict(lambda: set(paths)) + + for test in run_tests: + key, _ = get_suite_definition(test["flavor"], test.get("subsuite"), strict=True) + + test_path = test.get("srcdir_relpath") + if test_path is None: + continue + found_path = None + manifest_relpath = None + if "manifest_relpath" in test: + manifest_relpath = mozpath.normpath(test["manifest_relpath"]) + for path in remaining_paths_by_suite[key]: + if test_path.startswith(path) or manifest_relpath == path: + found_path = path + break + if found_path: + suite_to_tests[key].append(found_path) + remaining_paths_by_suite[key].remove(found_path) + + return suite_to_tests diff --git a/tools/tryselect/test/conftest.py b/tools/tryselect/test/conftest.py new file mode 100644 index 0000000000..7ffffc4b9b --- /dev/null +++ b/tools/tryselect/test/conftest.py @@ -0,0 +1,101 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from unittest.mock import MagicMock + +import pytest +import yaml +from moztest.resolve import TestResolver +from taskgraph.graph import Graph +from taskgraph.task import Task +from taskgraph.taskgraph import TaskGraph +from tryselect import push + + +@pytest.fixture +def tg(request): + if not hasattr(request.module, "TASKS"): + pytest.fail( + "'tg' fixture used from a module that didn't define the TASKS variable" + ) + + tasks = request.module.TASKS + for task in tasks: + task.setdefault("task", {}) + task["task"].setdefault("tags", {}) + + tasks = {t["label"]: Task(**t) for t in tasks} + return TaskGraph(tasks, Graph(tasks.keys(), set())) + + +@pytest.fixture +def patch_resolver(monkeypatch): + def inner(suites, tests): + def fake_test_metadata(*args, **kwargs): + return suites, tests + + monkeypatch.setattr(TestResolver, "resolve_metadata", fake_test_metadata) + + return inner + + +@pytest.fixture(autouse=True) +def patch_vcs(monkeypatch): + attrs = { + "path": push.vcs.path, + } + mock = MagicMock() + mock.configure_mock(**attrs) + monkeypatch.setattr(push, "vcs", mock) + + +@pytest.fixture(scope="session") +def run_mach(): + import mach_initialize + from mach.config import ConfigSettings + from tryselect.tasks import build + + mach = mach_initialize.initialize(build.topsrcdir) + + def inner(args): + mach.settings = ConfigSettings() + return mach.run(args) + + return inner + + +def pytest_generate_tests(metafunc): + if all( + fixture in metafunc.fixturenames + for fixture in ("task_config", "args", "expected") + ): + + def load_tests(): + for task_config, tests in metafunc.module.TASK_CONFIG_TESTS.items(): + for args, expected in tests: + yield (task_config, args, expected) + + tests = list(load_tests()) + ids = ["{} {}".format(t[0], " ".join(t[1])).strip() for t in tests] + metafunc.parametrize("task_config,args,expected", tests, ids=ids) + + elif all( + fixture in metafunc.fixturenames for fixture in ("shared_name", "shared_preset") + ): + preset_path = os.path.join( + push.build.topsrcdir, "tools", "tryselect", "try_presets.yml" + ) + with open(preset_path, "r") as fh: + presets = list(yaml.safe_load(fh).items()) + + ids = [p[0] for p in presets] + + # Mark fuzzy presets on Windows xfail due to fzf not being installed. + if os.name == "nt": + for i, preset in enumerate(presets): + if preset[1]["selector"] == "fuzzy": + presets[i] = pytest.param(*preset, marks=pytest.mark.xfail) + + metafunc.parametrize("shared_name,shared_preset", presets, ids=ids) diff --git a/tools/tryselect/test/cram.ini b/tools/tryselect/test/cram.ini new file mode 100644 index 0000000000..b1f0d8dfec --- /dev/null +++ b/tools/tryselect/test/cram.ini @@ -0,0 +1,5 @@ +[test_auto.t] +[test_empty.t] +[test_fuzzy.t] +[test_message.t] +[test_preset.t] diff --git a/tools/tryselect/test/python.ini b/tools/tryselect/test/python.ini new file mode 100644 index 0000000000..4b75ea6ab5 --- /dev/null +++ b/tools/tryselect/test/python.ini @@ -0,0 +1,19 @@ +[DEFAULT] +subsuite=try + +[test_again.py] +[test_auto.py] +[test_chooser.py] +requirements = tools/tryselect/selectors/chooser/requirements.txt +[test_fuzzy.py] +[test_mozharness_integration.py] +[test_perf.py] +[test_perfcomparators.py] +[test_presets.py] +# Modifies "task_duration_history.json" in .mozbuild. Since other tests depend on this file, this test +# shouldn't be run in parallel with those other tests. +sequential = true +[test_release.py] +[test_scriptworker.py] +[test_tasks.py] +[test_task_configs.py] diff --git a/tools/tryselect/test/setup.sh b/tools/tryselect/test/setup.sh new file mode 100644 index 0000000000..d9f3b62a2e --- /dev/null +++ b/tools/tryselect/test/setup.sh @@ -0,0 +1,99 @@ +export topsrcdir=$TESTDIR/../../../ +export MOZBUILD_STATE_PATH=$TMP/mozbuild +export MACH_TRY_PRESET_PATHS=$MOZBUILD_STATE_PATH/try_presets.yml + +# This helps to find fzf when running these tests locally, since normally fzf +# would be found via MOZBUILD_STATE_PATH pointing to $HOME/.mozbuild +export PATH="$PATH:$HOME/.mozbuild/fzf/bin" + +export MACHRC=$TMP/machrc +cat > $MACHRC << EOF +[try] +default=syntax +EOF + +cmd="$topsrcdir/mach python -c 'from mach.util import get_state_dir; print(get_state_dir(specific_to_topsrcdir=True))'" +# First run local state dir generation so it doesn't affect test output. +eval $cmd > /dev/null 2>&1 +# Now run it again to get the actual directory. +cachedir=$(eval $cmd)/cache/taskgraph +mkdir -p $cachedir + +cat > $cachedir/target_task_set << EOF +{ + "test/foo-opt": { + "kind": "test", + "label": "test/foo-opt", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "test/foo-debug": { + "kind": "test", + "label": "test/foo-debug", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "build-baz": { + "kind": "build", + "label": "build-baz", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + } +} +EOF + +cat > $cachedir/full_task_set << EOF +{ + "test/foo-opt": { + "kind": "test", + "label": "test/foo-opt", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "test/foo-debug": { + "kind": "test", + "label": "test/foo-debug", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "test/bar-opt": { + "kind": "test", + "label": "test/bar-opt", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "test/bar-debug": { + "kind": "test", + "label": "test/bar-debug", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "build-baz": { + "kind": "build", + "label": "build-baz", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + } +} +EOF + +# set mtime to the future so we don't re-generate tasks +find $cachedir -type f -exec touch -d "next day" {} + + +export testargs="--no-push --no-artifact" diff --git a/tools/tryselect/test/test_again.py b/tools/tryselect/test/test_again.py new file mode 100644 index 0000000000..0e5d9d6b6d --- /dev/null +++ b/tools/tryselect/test/test_again.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +import mozunit +import pytest +from six.moves import reload_module as reload +from tryselect import push +from tryselect.selectors import again + + +@pytest.fixture(autouse=True) +def patch_history_path(tmpdir, monkeypatch): + monkeypatch.setattr(push, "history_path", tmpdir.join("history.json").strpath) + reload(again) + + +def test_try_again(monkeypatch): + push.push_to_try( + "fuzzy", + "Fuzzy message", + try_task_config=push.generate_try_task_config( + "fuzzy", + ["foo", "bar"], + {"use-artifact-builds": True}, + ), + ) + + assert os.path.isfile(push.history_path) + with open(push.history_path, "r") as fh: + assert len(fh.readlines()) == 1 + + def fake_push_to_try(*args, **kwargs): + return args, kwargs + + monkeypatch.setattr(push, "push_to_try", fake_push_to_try) + reload(again) + + args, kwargs = again.run() + + assert args[0] == "again" + assert args[1] == "Fuzzy message" + + try_task_config = kwargs.pop("try_task_config") + assert sorted(try_task_config.get("tasks")) == sorted(["foo", "bar"]) + assert try_task_config.get("env") == {"TRY_SELECTOR": "fuzzy"} + assert try_task_config.get("use-artifact-builds") + + with open(push.history_path, "r") as fh: + assert len(fh.readlines()) == 1 + + +def test_no_push_does_not_generate_history(tmpdir): + assert not os.path.isfile(push.history_path) + + push.push_to_try( + "fuzzy", + "Fuzzy", + try_task_config=push.generate_try_task_config( + "fuzzy", + ["foo", "bar"], + {"use-artifact-builds": True}, + ), + dry_run=True, + ) + assert not os.path.isfile(push.history_path) + assert again.run() == 1 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_auto.py b/tools/tryselect/test/test_auto.py new file mode 100644 index 0000000000..63f0fe6bd7 --- /dev/null +++ b/tools/tryselect/test/test_auto.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit +import pytest +from tryselect.selectors.auto import AutoParser + + +def test_strategy_validation(): + parser = AutoParser() + args = parser.parse_args(["--strategy", "relevant_tests"]) + assert args.strategy == "gecko_taskgraph.optimize:tryselect.relevant_tests" + + args = parser.parse_args( + ["--strategy", "gecko_taskgraph.optimize:experimental.relevant_tests"] + ) + assert args.strategy == "gecko_taskgraph.optimize:experimental.relevant_tests" + + with pytest.raises(SystemExit): + parser.parse_args(["--strategy", "gecko_taskgraph.optimize:tryselect"]) + + with pytest.raises(SystemExit): + parser.parse_args(["--strategy", "foo"]) + + with pytest.raises(SystemExit): + parser.parse_args(["--strategy", "foo:bar"]) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_auto.t b/tools/tryselect/test/test_auto.t new file mode 100644 index 0000000000..c3fe797949 --- /dev/null +++ b/tools/tryselect/test/test_auto.t @@ -0,0 +1,61 @@ + + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test auto selector + + $ ./mach try auto $testargs + Commit message: + Tasks automatically selected. + + Pushed via `mach try auto` + Calculated try_task_config.json: + { + "parameters": { + "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium", + "optimize_target_tasks": true, + "target_tasks_method": "try_auto", + "test_manifest_loader": "bugbug", + "try_mode": "try_auto", + "try_task_config": {} + }, + "version": 2 + } + + + $ ./mach try auto $testargs --closed-tree + Commit message: + Tasks automatically selected. ON A CLOSED TREE + + Pushed via `mach try auto` + Calculated try_task_config.json: + { + "parameters": { + "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium", + "optimize_target_tasks": true, + "target_tasks_method": "try_auto", + "test_manifest_loader": "bugbug", + "try_mode": "try_auto", + "try_task_config": {} + }, + "version": 2 + } + + $ ./mach try auto $testargs --closed-tree -m "foo {msg} bar" + Commit message: + foo Tasks automatically selected. bar ON A CLOSED TREE + + Pushed via `mach try auto` + Calculated try_task_config.json: + { + "parameters": { + "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium", + "optimize_target_tasks": true, + "target_tasks_method": "try_auto", + "test_manifest_loader": "bugbug", + "try_mode": "try_auto", + "try_task_config": {} + }, + "version": 2 + } + diff --git a/tools/tryselect/test/test_chooser.py b/tools/tryselect/test/test_chooser.py new file mode 100644 index 0000000000..885ed41a78 --- /dev/null +++ b/tools/tryselect/test/test_chooser.py @@ -0,0 +1,77 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit +import pytest +from tryselect.selectors.chooser.app import create_application + +TASKS = [ + { + "kind": "build", + "label": "build-windows", + "attributes": { + "build_platform": "windows", + }, + }, + { + "kind": "test", + "label": "test-windows-mochitest-e10s", + "attributes": { + "unittest_suite": "mochitest-browser-chrome", + "mochitest_try_name": "mochitest-browser-chrome", + }, + }, +] + + +@pytest.fixture +def app(tg): + app = create_application(tg) + app.config["TESTING"] = True + + ctx = app.app_context() + ctx.push() + yield app + ctx.pop() + + +def test_try_chooser(app): + client = app.test_client() + + response = client.get("/") + assert response.status_code == 200 + + expected_output = [ + b"""<title>Try Chooser Enhanced</title>""", + b"""<input class="filter" type="checkbox" id=windows name="build" value='{"build_platform": ["windows"]}' onchange="console.log('checkbox onchange triggered');apply();">""", # noqa + b"""<input class="filter" type="checkbox" id=mochitest-browser-chrome name="test" value='{"unittest_suite": ["mochitest-browser-chrome"]}' onchange="console.log('checkbox onchange triggered');apply();">""", # noqa + ] + + for expected in expected_output: + assert expected in response.data + + response = client.post("/", data={"action": "Cancel"}) + assert response.status_code == 200 + assert b"You may now close this page" in response.data + assert app.tasks == [] + + response = client.post("/", data={"action": "Push", "selected-tasks": ""}) + assert response.status_code == 200 + assert b"You may now close this page" in response.data + assert app.tasks == [] + + response = client.post( + "/", + data={ + "action": "Push", + "selected-tasks": "build-windows\ntest-windows-mochitest-e10s", + }, + ) + assert response.status_code == 200 + assert b"You may now close this page" in response.data + assert set(app.tasks) == set(["build-windows", "test-windows-mochitest-e10s"]) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_empty.t b/tools/tryselect/test/test_empty.t new file mode 100644 index 0000000000..f6dea366c4 --- /dev/null +++ b/tools/tryselect/test/test_empty.t @@ -0,0 +1,47 @@ + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test empty selector + + $ ./mach try empty --no-push + Commit message: + No try selector specified, use "Add New Jobs" to select tasks. + + Pushed via `mach try empty` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "empty" + }, + "tasks": [], + "version": 1 + } + + $ ./mach try empty --no-push --closed-tree + Commit message: + No try selector specified, use "Add New Jobs" to select tasks. ON A CLOSED TREE + + Pushed via `mach try empty` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "empty" + }, + "tasks": [], + "version": 1 + } + + $ ./mach try empty --no-push --closed-tree -m "foo {msg} bar" + Commit message: + foo No try selector specified, use "Add New Jobs" to select tasks. bar ON A CLOSED TREE + + Pushed via `mach try empty` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "empty" + }, + "tasks": [], + "version": 1 + } + diff --git a/tools/tryselect/test/test_fuzzy.py b/tools/tryselect/test/test_fuzzy.py new file mode 100644 index 0000000000..330ba99825 --- /dev/null +++ b/tools/tryselect/test/test_fuzzy.py @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +import mozunit +import pytest + + +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +def test_query_paths(run_mach, capfd): + cmd = [ + "try", + "fuzzy", + "--no-push", + "-q", + "^test-linux '64-qr/debug-mochitest-chrome-1proc-", + "caps/tests/mochitest/test_addonMayLoad.html", + ] + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + # If there are more than one tasks here, it means that something went wrong + # with the path filtering. + expected = """ + "tasks": [ + "test-linux1804-64-qr/debug-mochitest-chrome-1proc-1" + ]""".lstrip() + + assert expected in output + + +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +def test_query(run_mach, capfd): + cmd = ["try", "fuzzy", "--no-push", "-q", "'source-test-python-taskgraph-tests-py3"] + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + # Should only ever mach one task exactly. + expected = """ + "tasks": [ + "source-test-python-taskgraph-tests-py3" + ]""".lstrip() + + assert expected in output + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_fuzzy.t b/tools/tryselect/test/test_fuzzy.t new file mode 100644 index 0000000000..5f53c07c97 --- /dev/null +++ b/tools/tryselect/test/test_fuzzy.t @@ -0,0 +1,201 @@ + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test fuzzy selector + + $ ./mach try fuzzy $testargs -q "'foo" + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + $ ./mach try fuzzy $testargs -q "'bar" + no tasks selected + $ ./mach try fuzzy $testargs --full -q "'bar" + Commit message: + Fuzzy query='bar + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/bar-debug", + "test/bar-opt" + ], + "version": 1 + } + + +Test multiple selectors + + $ ./mach try fuzzy $testargs --full -q "'foo" -q "'bar" + Commit message: + Fuzzy query='foo&query='bar + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/bar-debug", + "test/bar-opt", + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + +Test query intersection + + $ ./mach try fuzzy $testargs --and -q "'foo" -q "'opt" + Commit message: + Fuzzy query='foo&query='opt + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-opt" + ], + "version": 1 + } + + +Test intersection with preset containing multiple queries + + $ ./mach try fuzzy --save foo -q "'test" -q "'opt" + preset saved, run with: --preset=foo + + $ ./mach try fuzzy $testargs --preset foo -xq "'test" + Commit message: + Fuzzy query='test&query='opt&query='test + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + $ ./mach try $testargs --preset foo -xq "'test" + Commit message: + Fuzzy query='test&query='opt&query='test + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + +Test exact match + + $ ./mach try fuzzy $testargs --full -q "testfoo | 'testbar" + Commit message: + Fuzzy query=testfoo | 'testbar + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + $ ./mach try fuzzy $testargs --full --exact -q "testfoo | 'testbar" + Commit message: + Fuzzy query=testfoo | 'testbar + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/bar-debug", + "test/bar-opt" + ], + "version": 1 + } + + + +Test task config + + $ ./mach try fuzzy --no-push --artifact -q "'foo" + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "disable-pgo": true, + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "use-artifact-builds": true, + "version": 1 + } + + $ ./mach try fuzzy $testargs --env FOO=1 --env BAR=baz -q "'foo" + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "BAR": "baz", + "FOO": "1", + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + diff --git a/tools/tryselect/test/test_message.t b/tools/tryselect/test/test_message.t new file mode 100644 index 0000000000..25a104833c --- /dev/null +++ b/tools/tryselect/test/test_message.t @@ -0,0 +1,63 @@ + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test custom commit messages with fuzzy selector + + $ ./mach try fuzzy $testargs -q foo --message "Foobar" + Commit message: + Foobar + + Fuzzy query=foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + $ ./mach try fuzzy $testargs -q foo -m "Foobar: {msg}" + Commit message: + Foobar: Fuzzy query=foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + $ unset EDITOR + $ ./mach try fuzzy $testargs -q foo -m > /dev/null 2>&1 + [2] + + +Test custom commit messages with syntax selector + + $ ./mach try syntax $testargs -p linux -u mochitests --message "Foobar" + Commit message: + Foobar + + try: -b do -p linux -u mochitests + + Pushed via `mach try syntax` + $ ./mach try syntax $testargs -p linux -u mochitests -m "Foobar: {msg}" + Commit message: + Foobar: try: -b do -p linux -u mochitests + + Pushed via `mach try syntax` + $ unset EDITOR + $ ./mach try syntax $testargs -p linux -u mochitests -m > /dev/null 2>&1 + [2] diff --git a/tools/tryselect/test/test_mozharness_integration.py b/tools/tryselect/test/test_mozharness_integration.py new file mode 100644 index 0000000000..abeaaf370e --- /dev/null +++ b/tools/tryselect/test/test_mozharness_integration.py @@ -0,0 +1,145 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os + +import mozunit +import pytest +from mozfile import load_source +from tryselect.tasks import build, resolve_tests_by_suite + +MOZHARNESS_SCRIPTS = { + "android_emulator_unittest": { + "class_name": "AndroidEmulatorTest", + "configs": [ + "android/android_common.py", + ], + "xfail": [ + "cppunittest", + "crashtest-qr", + "gtest", + "geckoview-junit", + "jittest", + "jsreftest", + "reftest-qr", + ], + }, + "desktop_unittest": { + "class_name": "DesktopUnittest", + "configs": [ + "unittests/linux_unittest.py", + "unittests/mac_unittest.py", + "unittests/win_unittest.py", + ], + "xfail": [ + "cppunittest", + "gtest", + "jittest", + "jittest-chunked", + "jittest1", + "jittest2", + "jsreftest", + "mochitest-valgrind-plain", + "reftest-no-accel", + "reftest-snapshot", + "xpcshell-msix", + ], + }, +} +"""A suite being listed in a script's `xfail` list means it won't work +properly with MOZHARNESS_TEST_PATHS (the mechanism |mach try fuzzy <path>| +uses). +""" + + +def get_mozharness_test_paths(name): + scriptdir = os.path.join(build.topsrcdir, "testing", "mozharness") + mod = load_source( + "scripts." + name, os.path.join(scriptdir, "scripts", name + ".py") + ) + + class_name = MOZHARNESS_SCRIPTS[name]["class_name"] + cls = getattr(mod, class_name) + return cls(require_config_file=False)._get_mozharness_test_paths + + +@pytest.fixture(scope="module") +def all_suites(): + from moztest.resolve import _test_flavors, _test_subsuites + + all_suites = [] + for flavor in _test_flavors: + all_suites.append({"flavor": flavor, "srcdir_relpath": "test"}) + + for flavor, subsuite in _test_subsuites: + all_suites.append( + {"flavor": flavor, "subsuite": subsuite, "srcdir_relpath": "test"} + ) + + return all_suites + + +def generate_suites_from_config(path): + parent, name = os.path.split(path) + name = os.path.splitext(name)[0] + + configdir = os.path.join( + build.topsrcdir, "testing", "mozharness", "configs", parent + ) + + mod = load_source(name, os.path.join(configdir, name + ".py")) + + config = mod.config + + for category in sorted(config["suite_definitions"]): + key = "all_{}_suites".format(category) + if key not in config: + yield category, + continue + + for suite in sorted(config["all_{}_suites".format(category)]): + yield category, suite + + +def generate_suites(): + for name, script in MOZHARNESS_SCRIPTS.items(): + seen = set() + + for path in script["configs"]: + for suite in generate_suites_from_config(path): + if suite in seen: + continue + seen.add(suite) + + item = (name, suite) + + if suite[-1] in script["xfail"]: + item = pytest.param(item, marks=pytest.mark.xfail) + + yield item + + +def idfn(item): + name, suite = item + return "{}/{}".format(name, suite[-1]) + + +@pytest.mark.parametrize("item", generate_suites(), ids=idfn) +def test_suites(item, patch_resolver, all_suites): + """An integration test to make sure the suites returned by + `tasks.resolve_tests_by_suite` match up with the names defined in + mozharness. + """ + patch_resolver([], all_suites) + suites = resolve_tests_by_suite(["test"]) + os.environ["MOZHARNESS_TEST_PATHS"] = json.dumps(suites) + + name, suite = item + func = get_mozharness_test_paths(name) + assert func(*suite) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_perf.py b/tools/tryselect/test/test_perf.py new file mode 100644 index 0000000000..28b8d56acb --- /dev/null +++ b/tools/tryselect/test/test_perf.py @@ -0,0 +1,1245 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import pathlib +import shutil +import tempfile +from unittest import mock + +import mozunit +import pytest +from tryselect.selectors.perf import ( + MAX_PERF_TASKS, + Apps, + InvalidCategoryException, + InvalidRegressionDetectorQuery, + PerfParser, + Platforms, + Suites, + Variants, + run, +) +from tryselect.selectors.perfselector.classification import ( + check_for_live_sites, + check_for_profile, +) + +TASKS = [ + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-animometer", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-optimizing", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-webaudio", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-speedometer", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-jetstream2", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-ares6", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-optimizing", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-sunspider", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-matrix-react-bench", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-baseline", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-twitch-animation", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-assorted-dom", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-stylebench", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-baseline", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-htmlsuite", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-unity-webgl", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot", +] + +# The TEST_VARIANTS, and TEST_CATEGORIES are used to force +# a particular set of categories to show up in testing. Otherwise, +# every time someone adds a category, or a variant, we'll need +# to redo all the category counts. The platforms, and apps are +# not forced because they change infrequently. +TEST_VARIANTS = { + Variants.NO_FISSION.value: { + "query": "'nofis", + "negation": "!nofis", + "platforms": [Platforms.ANDROID.value], + "apps": [Apps.FENIX.value, Apps.GECKOVIEW.value], + }, + Variants.BYTECODE_CACHED.value: { + "query": "'bytecode", + "negation": "!bytecode", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, + Variants.LIVE_SITES.value: { + "query": "'live", + "negation": "!live", + "restriction": check_for_live_sites, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": list(PerfParser.apps.keys()), + }, + Variants.PROFILING.value: { + "query": "'profil", + "negation": "!profil", + "restriction": check_for_profile, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": [Apps.FIREFOX.value, Apps.GECKOVIEW.value, Apps.FENIX.value], + }, + Variants.SWR.value: { + "query": "'swr", + "negation": "!swr", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, +} + +TEST_CATEGORIES = { + "Pageload": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'tp6"], + }, + "suites": [Suites.RAPTOR.value], + "tasks": [], + }, + "Pageload (essential)": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'tp6 'essential"], + }, + "variant-restrictions": {Suites.RAPTOR.value: [Variants.NO_FISSION.value]}, + "suites": [Suites.RAPTOR.value], + "tasks": [], + }, + "Responsiveness": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'responsive"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "tasks": [], + }, + "Benchmarks": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'benchmark"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "tasks": [], + }, + "DAMP (Devtools)": { + "query": { + Suites.TALOS.value: ["'talos 'damp"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + }, + "Talos PerfTests": { + "query": { + Suites.TALOS.value: ["'talos"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + }, + "Resource Usage": { + "query": { + Suites.TALOS.value: ["'talos 'xperf | 'tp5"], + Suites.RAPTOR.value: ["'power 'osx"], + Suites.AWSY.value: ["'awsy"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value, Suites.AWSY.value], + "platform-restrictions": [Platforms.DESKTOP.value], + "variant-restrictions": { + Suites.RAPTOR.value: [], + Suites.TALOS.value: [], + }, + "app-restrictions": { + Suites.RAPTOR.value: [Apps.FIREFOX.value], + Suites.TALOS.value: [Apps.FIREFOX.value], + }, + "tasks": [], + }, + "Graphics, & Media Playback": { + "query": { + # XXX This might not be an exhaustive list for talos atm + Suites.TALOS.value: ["'talos 'svgr | 'bcv | 'webgl"], + Suites.RAPTOR.value: ["'browsertime 'youtube-playback"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: [Variants.NO_FISSION.value]}, + "tasks": [], + }, +} + + +@pytest.mark.parametrize( + "category_options, expected_counts, unique_categories, missing", + [ + # Default should show the premade live category, but no chrome or android + # The benchmark desktop category should be visible in all configurations + # except for when there are requested apps/variants/platforms + ( + {}, + 58, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ] + }, + "Pageload macosx": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "!bytecode", + "!live", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ] + }, + "Resource Usage desktop": { + "awsy": ["'awsy", "!android 'shippable !-32 !clang"], + "raptor": [ + "'power 'osx", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ], + "talos": [ + "'talos 'xperf | 'tp5", + "!android 'shippable !-32 !clang", + "!profil", + "!swr", + ], + }, + }, + [ + "Responsiveness android-p2 geckoview", + "Benchmarks desktop chromium", + ], + ), # Default settings + ( + {"live_sites": True}, + 66, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ] + }, + "Pageload macosx": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "!bytecode", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ] + }, + "Pageload macosx live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "'live", + "!bytecode", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ], + }, + }, + [ + "Responsiveness android-p2 geckoview", + "Benchmarks desktop chromium", + "Benchmarks desktop firefox profiling", + "Talos desktop live-sites", + "Talos desktop profiling+swr", + "Benchmarks desktop firefox live-sites+profiling" + "Benchmarks desktop firefox live-sites", + ], + ), + ( + {"live_sites": True, "safari": True}, + 72, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!profil", + "!chrom", + "!custom-car", + ] + }, + "Pageload macosx safari": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "'safari", + "!bytecode", + "!profil", + ] + }, + "Pageload macosx safari live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "'safari", + "'live", + "!bytecode", + "!profil", + ], + }, + }, + [ + "Pageload linux safari", + "Pageload desktop safari", + ], + ), + ( + {"live_sites": True, "chrome": True}, + 114, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!profil", + "!safari", + "!custom-car", + ] + }, + "Pageload macosx live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "'live", + "!bytecode", + "!profil", + "!safari", + "!custom-car", + ], + }, + "Benchmarks desktop chromium": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "'chromium", + "!bytecode", + "!profil", + ], + }, + }, + [ + "Responsiveness android-p2 geckoview", + "Firefox Pageload linux chrome", + "Talos PerfTests desktop swr", + ], + ), + ( + {"android": True}, + 88, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ], + }, + "Responsiveness android-a51 geckoview": { + "raptor": [ + "'browsertime 'responsive", + "'android 'a51 'shippable 'aarch64", + "'geckoview", + "!nofis", + "!live", + "!profil", + ], + }, + }, + [ + "Responsiveness android-a51 chrome-m", + "Firefox Pageload android", + ], + ), + ( + {"android": True, "chrome": True}, + 138, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!profil", + "!safari", + "!custom-car", + ], + }, + "Responsiveness android-a51 chrome-m": { + "raptor": [ + "'browsertime 'responsive", + "'android 'a51 'shippable 'aarch64", + "'chrome-m", + "!nofis", + "!live", + "!profil", + ], + }, + }, + ["Responsiveness android-p2 chrome-m", "Resource Usage android"], + ), + ( + {"android": True, "chrome": True, "profile": True}, + 176, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!safari", + "!custom-car", + ] + }, + "Talos PerfTests desktop profiling": { + "talos": [ + "'talos", + "!android 'shippable !-32 !clang", + "'profil", + "!swr", + ] + }, + }, + [ + "Resource Usage desktop profiling", + "DAMP (Devtools) desktop chrome", + "Resource Usage android", + "Resource Usage windows chromium", + ], + ), + # Show all available windows tests, no other platform should exist + # including the desktop catgeory + ( + {"requested_platforms": ["windows"]}, + 14, + { + "Benchmarks windows firefox": { + "raptor": [ + "'browsertime 'benchmark", + "!-32 'windows 'shippable", + "!chrom !geckoview !fenix !safari !custom-car", + "!bytecode", + "!live", + "!profil", + ] + }, + }, + [ + "Resource Usage desktop", + "Benchmarks desktop", + "Benchmarks linux firefox bytecode-cached+profiling", + ], + ), + # Can't have fenix on the windows platform + ( + {"requested_platforms": ["windows"], "requested_apps": ["fenix"]}, + 0, + {}, + ["Benchmarks desktop"], + ), + # Android flag also needs to be supplied + ( + {"requested_platforms": ["android"], "requested_apps": ["fenix"]}, + 0, + {}, + ["Benchmarks desktop"], + ), + # There should be no global categories available, only fenix + ( + { + "requested_platforms": ["android"], + "requested_apps": ["fenix"], + "android": True, + }, + 10, + { + "Pageload android fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!nofis", + "!live", + "!profil", + ], + } + }, + ["Benchmarks desktop", "Pageload (live) android"], + ), + # Test with multiple apps + ( + { + "requested_platforms": ["android"], + "requested_apps": ["fenix", "geckoview"], + "android": True, + }, + 15, + { + "Benchmarks android geckoview": { + "raptor": [ + "'browsertime 'benchmark", + "'android 'a51 'shippable 'aarch64", + "'geckoview", + "!nofis", + "!live", + "!profil", + ], + }, + "Pageload android fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!nofis", + "!live", + "!profil", + ], + }, + }, + [ + "Benchmarks desktop", + "Pageload android no-fission", + "Pageload android fenix live-sites", + ], + ), + # Variants are inclusive, so we'll see the variant alongside the + # base here for fenix + ( + { + "requested_variants": ["no-fission"], + "requested_apps": ["fenix"], + "android": True, + }, + 32, + { + "Pageload android-a51 fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!live", + "!profil", + ], + }, + "Pageload android-a51 fenix no-fission": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "!live", + "!profil", + ], + }, + "Pageload (essential) android fenix no-fission": { + "raptor": [ + "'browsertime 'tp6 'essential", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "!live", + "!profil", + ], + }, + }, + [ + "Benchmarks desktop", + "Pageload (live) android", + "Pageload android-p2 fenix live-sites", + ], + ), + # With multiple variants, we'll see the base variant (with no combinations) + # for each of them + ( + { + "requested_variants": ["no-fission", "live-sites"], + "requested_apps": ["fenix"], + "android": True, + }, + 40, + { + "Pageload android-a51 fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!profil", + ], + }, + "Pageload android-a51 fenix no-fission": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "!live", + "!profil", + ], + }, + "Pageload android-a51 fenix live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'live", + "!nofis", + "!profil", + ], + }, + "Pageload (essential) android fenix no-fission": { + "raptor": [ + "'browsertime 'tp6 'essential", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "!live", + "!profil", + ], + }, + "Pageload android fenix no-fission+live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "'live", + "!profil", + ], + }, + }, + [ + "Benchmarks desktop", + "Pageload (live) android", + "Pageload android-p2 fenix live-sites", + "Pageload (essential) android fenix no-fission+live-sites", + ], + ), + # Make sure that no no-fission tasks are selected when a variant cannot + # run on a requested platform + ( + { + "requested_variants": ["no-fission"], + "requested_platforms": ["windows"], + }, + 14, + { + "Responsiveness windows firefox": { + "raptor": [ + "'browsertime 'responsive", + "!-32 'windows 'shippable", + "!chrom !geckoview !fenix !safari !custom-car", + "!bytecode", + "!live", + "!profil", + ], + }, + }, + ["Benchmarks desktop", "Responsiveness windows firefox no-fisson"], + ), + # We should only see the base and the live-site variants here for windows + ( + { + "requested_variants": ["no-fission", "live-sites"], + "requested_platforms": ["windows"], + "android": True, + }, + 16, + { + "Responsiveness windows firefox": { + "raptor": [ + "'browsertime 'responsive", + "!-32 'windows 'shippable", + "!chrom !geckoview !fenix !safari !custom-car", + "!bytecode", + "!profil", + ], + }, + "Pageload windows live-sites": { + "raptor": [ + "'browsertime 'tp6", + "!-32 'windows 'shippable", + "'live", + "!bytecode", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ], + }, + "Graphics, & Media Playback windows": { + "raptor": [ + "'browsertime 'youtube-playback", + "!-32 'windows 'shippable", + "!bytecode", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ], + "talos": [ + "'talos 'svgr | 'bcv | 'webgl", + "!-32 'windows 'shippable", + "!profil", + "!swr", + ], + }, + }, + [ + "Benchmarks desktop", + "Responsiveness windows firefox no-fisson", + "Pageload (live) android", + "Talos desktop live-sites", + "Talos android", + "Graphics, & Media Playback windows live-sites", + "Graphics, & Media Playback android no-fission", + ], + ), + ], +) +def test_category_expansion( + category_options, expected_counts, unique_categories, missing +): + # Set the categories, and variants to expand + PerfParser.categories = TEST_CATEGORIES + PerfParser.variants = TEST_VARIANTS + + # Expand the categories, then either check if the unique_categories, + # exist or are missing from the categories + expanded_cats = PerfParser.get_categories(**category_options) + + assert len(expanded_cats) == expected_counts + assert not any([expanded_cats.get(ucat, None) is not None for ucat in missing]) + assert all( + [expanded_cats.get(ucat, None) is not None for ucat in unique_categories.keys()] + ) + + # Ensure that the queries are as expected + for cat_name, cat_query in unique_categories.items(): + # Don't use get here because these fields should always exist + assert cat_query == expanded_cats[cat_name]["queries"] + + +@pytest.mark.parametrize( + "options, call_counts, log_ind, expected_log_message", + [ + ( + {}, + [9, 2, 2, 10, 2, 1], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"query": "'Pageload 'linux 'firefox"}, + [9, 2, 2, 10, 2, 1], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"cached_revision": "cached_base_revision"}, + [9, 1, 1, 10, 2, 0], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=cached_base_revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"dry_run": True}, + [9, 1, 1, 10, 2, 0], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=&newProject=try&newRevision=revision\n" + ), + ), + ( + {"show_all": True}, + [1, 2, 2, 8, 2, 1], + 0, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"show_all": True, "query": "'shippable !32 speedometer 'firefox"}, + [1, 2, 2, 8, 2, 1], + 0, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"single_run": True}, + [9, 1, 1, 4, 2, 0], + 2, + ( + "If you need any help, you can find us in the #perf-help Matrix channel:\n" + "https://matrix.to/#/#perf-help:mozilla.org\n" + ), + ), + ( + {"detect_changes": True}, + [10, 2, 2, 10, 2, 1], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ], +) +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +def test_full_run(options, call_counts, log_ind, expected_log_message): + with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( + "tryselect.selectors.perf.run_fzf" + ) as fzf, mock.patch( + "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() + ), mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ) as logger, mock.patch( + "tryselect.selectors.perf.PerfParser.check_cached_revision", + ) as ccr, mock.patch( + "tryselect.selectors.perf.PerfParser.save_revision_treeherder" + ) as srt, mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print: + fzf.side_effect = [ + ["", ["Benchmarks linux"]], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", ["Perftest Change Detector"]], + ] + ccr.return_value = options.get("cached_revision", "") + + run(**options) + + assert fzf.call_count == call_counts[0] + assert ptt.call_count == call_counts[1] + assert logger.call_count == call_counts[2] + assert perf_print.call_count == call_counts[3] + assert ccr.call_count == call_counts[4] + assert srt.call_count == call_counts[5] + assert perf_print.call_args_list[log_ind][0][0] == expected_log_message + + +@pytest.mark.parametrize( + "options, call_counts, log_ind, expected_log_message, expected_failure", + [ + ( + {"detect_changes": True}, + [10, 0, 0, 2, 1], + 1, + ( + "Executing raptor queries: 'browsertime 'benchmark, !clang 'linux " + "'shippable, !bytecode, !live, !profil, !chrom, !safari, !custom-car" + ), + InvalidRegressionDetectorQuery, + ), + ], +) +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +def test_change_detection_task_injection_failure( + options, + call_counts, + log_ind, + expected_log_message, + expected_failure, +): + with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( + "tryselect.selectors.perf.run_fzf" + ) as fzf, mock.patch( + "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() + ), mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ) as logger, mock.patch( + "tryselect.selectors.perf.PerfParser.check_cached_revision" + ) as ccr, mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print: + fzf.side_effect = [ + ["", ["Benchmarks linux"]], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ] + + with pytest.raises(expected_failure): + run(**options) + + assert fzf.call_count == call_counts[0] + assert ptt.call_count == call_counts[1] + assert logger.call_count == call_counts[2] + assert perf_print.call_count == call_counts[3] + assert ccr.call_count == call_counts[4] + assert perf_print.call_args_list[log_ind][0][0] == expected_log_message + + +@pytest.mark.parametrize( + "query, should_fail", + [ + ( + { + "query": { + # Raptor has all variants available so it + # should fail on this category + "raptor": ["browsertime 'live 'no-fission"], + } + }, + True, + ), + ( + { + "query": { + # Awsy has no variants defined so it shouldn't fail + # on a query like this + "awsy": ["browsertime 'live 'no-fission"], + } + }, + False, + ), + ], +) +def test_category_rules(query, should_fail): + # Set the categories, and variants to expand + PerfParser.categories = {"test-live": query} + PerfParser.variants = TEST_VARIANTS + + if should_fail: + with pytest.raises(InvalidCategoryException): + PerfParser.run_category_checks() + else: + assert PerfParser.run_category_checks() + + # Reset the categories, and variants to expand + PerfParser.categories = TEST_CATEGORIES + PerfParser.variants = TEST_VARIANTS + + +@pytest.mark.parametrize( + "apk_name, apk_content, should_fail, failure_message", + [ + ( + "real-file", + "file-content", + False, + None, + ), + ("bad-file", None, True, "Path does not exist:"), + ], +) +def test_apk_upload(apk_name, apk_content, should_fail, failure_message): + with mock.patch("tryselect.selectors.perf.subprocess") as _, mock.patch( + "tryselect.selectors.perf.shutil" + ) as _: + temp_dir = None + try: + temp_dir = tempfile.mkdtemp() + sample_apk = pathlib.Path(temp_dir, apk_name) + if apk_content is not None: + with sample_apk.open("w") as f: + f.write(apk_content) + + if should_fail: + with pytest.raises(Exception) as exc_info: + PerfParser.setup_apk_upload("browsertime", str(sample_apk)) + assert failure_message in str(exc_info) + else: + PerfParser.setup_apk_upload("browsertime", str(sample_apk)) + finally: + if temp_dir is not None: + shutil.rmtree(temp_dir) + + +@pytest.mark.parametrize( + "args, load_data, return_value, call_counts, exists_cache_file", + [ + ( + ( + [], + "base_commit", + ), + { + "base_commit": [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-31", + "tasks": [], + }, + ], + }, + "2b04563b5", + [1, 0], + True, + ), + ( + ( + ["task-a"], + "subset_base_commit", + ), + { + "subset_base_commit": [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-31", + "tasks": ["task-a", "task-b"], + }, + ], + }, + "2b04563b5", + [1, 0], + True, + ), + ( + ([], "not_exist_cached_base_commit"), + { + "base_commit": [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-31", + "tasks": [], + }, + ], + }, + None, + [1, 0], + True, + ), + ( + ( + ["task-a", "task-b"], + "superset_base_commit", + ), + { + "superset_base_commit": [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-31", + "tasks": ["task-a"], + }, + ], + }, + None, + [1, 0], + True, + ), + ( + ([], None), + {}, + None, + [1, 1], + True, + ), + ( + ([], None), + {}, + None, + [0, 0], + False, + ), + ], +) +def test_check_cached_revision( + args, load_data, return_value, call_counts, exists_cache_file +): + with mock.patch("tryselect.selectors.perf.json.load") as load, mock.patch( + "tryselect.selectors.perf.json.dump" + ) as dump, mock.patch( + "tryselect.selectors.perf.pathlib.Path.is_file" + ) as is_file, mock.patch( + "tryselect.selectors.perf.pathlib.Path.open" + ): + load.return_value = load_data + is_file.return_value = exists_cache_file + result = PerfParser.check_cached_revision(*args) + + assert load.call_count == call_counts[0] + assert dump.call_count == call_counts[1] + assert result == return_value + + +@pytest.mark.parametrize( + "args, call_counts, exists_cache_file", + [ + ( + ["base_commit", "base_revision_treeherder"], + [0, 1], + False, + ), + ( + ["base_commit", "base_revision_treeherder"], + [1, 1], + True, + ), + ], +) +def test_save_revision_treeherder(args, call_counts, exists_cache_file): + with mock.patch("tryselect.selectors.perf.json.load") as load, mock.patch( + "tryselect.selectors.perf.json.dump" + ) as dump, mock.patch( + "tryselect.selectors.perf.pathlib.Path.is_file" + ) as is_file, mock.patch( + "tryselect.selectors.perf.pathlib.Path.open" + ): + is_file.return_value = exists_cache_file + PerfParser.save_revision_treeherder(TASKS, args[0], args[1]) + + assert load.call_count == call_counts[0] + assert dump.call_count == call_counts[1] + + +@pytest.mark.parametrize( + "total_tasks, options, call_counts, expected_log_message, expected_failure", + [ + ( + MAX_PERF_TASKS + 1, + {}, + [1, 0, 0, 1], + ( + "That's a lot of tests selected (300)!\n" + "These tests won't be triggered. If this was unexpected, " + "please file a bug in Testing :: Performance." + ), + True, + ), + ( + MAX_PERF_TASKS, + {"show_all": True}, + [9, 0, 0, 8], + ( + "For more information on the performance tests, see our " + "PerfDocs here:\nhttps://firefox-source-docs.mozilla.org/testing/perfdocs/" + ), + False, + ), + ( + int((MAX_PERF_TASKS + 2) / 2), + {"show_all": True, "try_config": {"rebuild": 2}}, + [1, 0, 0, 1], + ( + "That's a lot of tests selected (300)!\n" + "These tests won't be triggered. If this was unexpected, " + "please file a bug in Testing :: Performance." + ), + True, + ), + (0, {}, [1, 0, 0, 1], ("No tasks selected"), True), + ], +) +def test_max_perf_tasks( + total_tasks, + options, + call_counts, + expected_log_message, + expected_failure, +): + # Set the categories, and variants to expand + PerfParser.categories = TEST_CATEGORIES + PerfParser.variants = TEST_VARIANTS + + with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print, mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ), mock.patch( + "tryselect.selectors.perf.PerfParser.perf_push_to_try", + new_callable=mock.MagicMock, + return_value=("revision1", "revision2"), + ) as perf_push_to_try_mock, mock.patch( + "tryselect.selectors.perf.PerfParser.get_perf_tasks" + ) as get_perf_tasks_mock, mock.patch( + "tryselect.selectors.perf.PerfParser.get_tasks" + ) as get_tasks_mock, mock.patch( + "tryselect.selectors.perf.run_fzf" + ) as fzf, mock.patch( + "tryselect.selectors.perf.fzf_bootstrap", return_value=mock.MagicMock() + ): + tasks = ["a-task"] * total_tasks + get_tasks_mock.return_value = tasks + get_perf_tasks_mock.return_value = tasks, [], [] + + run(**options) + + assert perf_push_to_try_mock.call_count == 0 if expected_failure else 1 + assert ptt.call_count == call_counts[1] + assert perf_print.call_count == call_counts[3] + assert fzf.call_count == 0 + assert perf_print.call_args_list[-1][0][0] == expected_log_message + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_perfcomparators.py b/tools/tryselect/test/test_perfcomparators.py new file mode 100644 index 0000000000..51f0bdb287 --- /dev/null +++ b/tools/tryselect/test/test_perfcomparators.py @@ -0,0 +1,150 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import tempfile +from unittest import mock + +import mozunit +import pytest +from tryselect.selectors.perfselector.perfcomparators import ( + BadComparatorArgs, + BenchmarkComparator, + ComparatorNotFound, + get_comparator, +) + + +@pytest.mark.parametrize( + "test_link", + [ + "https://github.com/mozilla-mobile/firefox-android/pull/1627", + "https://github.com/mozilla-mobile/firefox-android/pull/1876/" + "commits/17c7350cc37a4a85cea140a7ce54e9fd037b5365", + ], +) +def test_benchmark_comparator(test_link): + def _verify_extra_args(extra_args): + assert len(extra_args) == 3 + if "commit" in test_link: + assert ( + "benchmark-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365" + in extra_args + ) + else: + assert "benchmark-revision=sha-for-link" in extra_args + assert "benchmark-repository=url-for-link" in extra_args + assert "benchmark-branch=ref-for-link" in extra_args + + comparator = BenchmarkComparator( + None, None, None, [f"base-link={test_link}", f"new-link={test_link}"] + ) + + with mock.patch("requests.get") as mocked_get: + magic_get = mock.MagicMock() + magic_get.json.return_value = { + "head": { + "repo": { + "html_url": "url-for-link", + }, + "sha": "sha-for-link", + "ref": "ref-for-link", + } + } + magic_get.status_code = 200 + mocked_get.return_value = magic_get + + extra_args = [] + comparator.setup_base_revision(extra_args) + _verify_extra_args(extra_args) + + extra_args = [] + comparator.setup_new_revision(extra_args) + _verify_extra_args(extra_args) + + +def test_benchmark_comparator_no_pr_links(): + def _verify_extra_args(extra_args): + assert len(extra_args) == 3 + assert "benchmark-revision=rev" in extra_args + assert "benchmark-repository=link" in extra_args + assert "benchmark-branch=fake" in extra_args + + comparator = BenchmarkComparator( + None, + None, + None, + [ + "base-repo=link", + "base-branch=fake", + "base-revision=rev", + "new-repo=link", + "new-branch=fake", + "new-revision=rev", + ], + ) + + with mock.patch("requests.get") as mocked_get: + magic_get = mock.MagicMock() + magic_get.json.return_value = { + "head": { + "repo": { + "html_url": "url-for-link", + }, + "sha": "sha-for-link", + "ref": "ref-for-link", + } + } + magic_get.status_code = 200 + mocked_get.return_value = magic_get + + extra_args = [] + comparator.setup_base_revision(extra_args) + _verify_extra_args(extra_args) + + extra_args = [] + comparator.setup_new_revision(extra_args) + _verify_extra_args(extra_args) + + +def test_benchmark_comparator_bad_args(): + comparator = BenchmarkComparator( + None, + None, + None, + [ + "base-bad-args=val", + ], + ) + + with pytest.raises(BadComparatorArgs): + comparator.setup_base_revision([]) + + +def test_get_comparator_bad_name(): + with pytest.raises(ComparatorNotFound): + get_comparator("BadName") + + +def test_get_comparator_bad_script(): + with pytest.raises(ComparatorNotFound): + with tempfile.NamedTemporaryFile() as tmpf: + tmpf.close() + get_comparator(tmpf.name) + + +def test_get_comparator_benchmark_name(): + comparator_klass = get_comparator("BenchmarkComparator") + assert comparator_klass.__name__ == "BenchmarkComparator" + + +def test_get_comparator_benchmark_script(): + # If the get_comparator method is working for scripts, then + # it should find the first defined class in this file, or the + # first imported class that matches it + comparator_klass = get_comparator(__file__) + assert comparator_klass.__name__ == "BenchmarkComparator" + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_preset.t b/tools/tryselect/test/test_preset.t new file mode 100644 index 0000000000..d67a5d6341 --- /dev/null +++ b/tools/tryselect/test/test_preset.t @@ -0,0 +1,360 @@ + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test preset with no subcommand + + $ ./mach try $testargs --save foo -b do -p linux -u mochitests -t none --tag foo + preset saved, run with: --preset=foo + + $ ./mach try $testargs --preset foo + Commit message: + try: -b do -p linux -u mochitests -t none --tag foo + + Pushed via `mach try syntax` + + $ ./mach try syntax $testargs --preset foo + Commit message: + try: -b do -p linux -u mochitests -t none --tag foo + + Pushed via `mach try syntax` + + $ ./mach try $testargs --list-presets + Presets from */mozbuild/try_presets.yml: (glob) + + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + + $ unset EDITOR + $ ./mach try $testargs --edit-presets + error: must set the $EDITOR environment variable to use --edit-presets + $ export EDITOR=cat + $ ./mach try $testargs --edit-presets + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + +Test preset with syntax subcommand + + $ ./mach try syntax $testargs --save bar -b do -p win32 -u none -t all --tag bar + preset saved, run with: --preset=bar + + $ ./mach try syntax $testargs --preset bar + Commit message: + try: -b do -p win32 -u none -t all --tag bar + + Pushed via `mach try syntax` + + $ ./mach try $testargs --preset bar + Commit message: + try: -b do -p win32 -u none -t all --tag bar + + Pushed via `mach try syntax` + + $ ./mach try syntax $testargs --list-presets + Presets from */mozbuild/try_presets.yml: (glob) + + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + + $ ./mach try syntax $testargs --edit-presets + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + +Test preset with fuzzy subcommand + + $ ./mach try fuzzy $testargs --save baz -q "'foo" --rebuild 5 + preset saved, run with: --preset=baz + + $ ./mach try fuzzy $testargs --preset baz + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "rebuild": 5, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + + $ ./mach try $testargs --preset baz + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "rebuild": 5, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + +Queries can be appended to presets + + $ ./mach try fuzzy $testargs --preset baz -q "'build" + Commit message: + Fuzzy query='foo&query='build + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "rebuild": 5, + "tasks": [ + "build-baz", + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + + $ ./mach try $testargs --preset baz -xq "'opt" + Commit message: + Fuzzy query='foo&query='opt + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "rebuild": 5, + "tasks": [ + "test/foo-opt" + ], + "version": 1 + } + + + $ ./mach try fuzzy $testargs --list-presets + Presets from */mozbuild/try_presets.yml: (glob) + + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + baz: + dry_run: true + no_artifact: true + query: + - "'foo" + rebuild: 5 + selector: fuzzy + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + + $ ./mach try fuzzy $testargs --edit-presets + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + baz: + dry_run: true + no_artifact: true + query: + - "'foo" + rebuild: 5 + selector: fuzzy + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + +Test gecko-profile argument handling. Add in profiling to a preset. + + $ ./mach try fuzzy $testargs --preset baz --gecko-profile-features=nostacksampling,cpu + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "gecko-profile": true, + "gecko-profile-features": "nostacksampling,cpu", + "rebuild": 5, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + +Check whether the gecko-profile flags can be used from a preset, and check +dashes vs underscores (presets save with underscores to match ArgumentParser +settings; everything else uses dashes.) + + $ ./mach try fuzzy $testargs --save profile -q "'foo" --rebuild 5 --gecko-profile-features=nostacksampling,cpu + preset saved, run with: --preset=profile + + $ ./mach try fuzzy $testargs --preset profile + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "gecko-profile": true, + "gecko-profile-features": "nostacksampling,cpu", + "rebuild": 5, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + $ EDITOR=cat ./mach try fuzzy $testargs --edit-preset profile + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + baz: + dry_run: true + no_artifact: true + query: + - "'foo" + rebuild: 5 + selector: fuzzy + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + profile: + dry_run: true + gecko_profile_features: nostacksampling,cpu + no_artifact: true + query: + - "'foo" + rebuild: 5 + selector: fuzzy + + $ rm $MOZBUILD_STATE_PATH/try_presets.yml diff --git a/tools/tryselect/test/test_presets.py b/tools/tryselect/test/test_presets.py new file mode 100644 index 0000000000..89cc810808 --- /dev/null +++ b/tools/tryselect/test/test_presets.py @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit +import pytest + +TASKS = [ + { + "kind": "build", + "label": "build-windows", + "attributes": { + "build_platform": "windows", + }, + }, + { + "kind": "test", + "label": "test-windows-mochitest-e10s", + "attributes": { + "unittest_suite": "mochitest", + "unittest_flavor": "browser-chrome", + "mochitest_try_name": "mochitest", + }, + }, +] + + +@pytest.fixture(autouse=True) +def skip_taskgraph_generation(monkeypatch, tg): + def fake_generate_tasks(*args, **kwargs): + return tg + + from tryselect import tasks + + monkeypatch.setattr(tasks, "generate_tasks", fake_generate_tasks) + + +@pytest.mark.xfail( + strict=False, reason="Bug 1635204: " "test_shared_presets[sample-suites] is flaky" +) +def test_shared_presets(run_mach, shared_name, shared_preset): + """This test makes sure that we don't break any of the in-tree presets when + renaming/removing variables in any of the selectors. + """ + assert "description" in shared_preset + assert "selector" in shared_preset + + selector = shared_preset["selector"] + if selector == "fuzzy": + assert "query" in shared_preset + assert isinstance(shared_preset["query"], list) + + # Run the preset and assert there were no exceptions. + assert run_mach(["try", "--no-push", "--preset", shared_name]) == 0 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_release.py b/tools/tryselect/test/test_release.py new file mode 100644 index 0000000000..a1a0d348b2 --- /dev/null +++ b/tools/tryselect/test/test_release.py @@ -0,0 +1,43 @@ +# Any copyright is dedicated to the Public Domain. +# https://creativecommons.org/publicdomain/zero/1.0/ + +from textwrap import dedent + +import mozunit + + +def test_release(run_mach, capfd): + cmd = [ + "try", + "release", + "--no-push", + "--version=97.0", + ] + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + expected = dedent( + """ + Commit message: + staging release: 97.0 + + Pushed via `mach try release` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": true, + "release_type": "release", + "target_tasks_method": "staging_release_builds" + }, + "version": 2 + } + + """ + ).lstrip() + assert expected in output + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_scriptworker.py b/tools/tryselect/test/test_scriptworker.py new file mode 100644 index 0000000000..e25279ace4 --- /dev/null +++ b/tools/tryselect/test/test_scriptworker.py @@ -0,0 +1,39 @@ +# Any copyright is dedicated to the Public Domain. +# https://creativecommons.org/publicdomain/zero/1.0/ + +import re +from textwrap import dedent + +import mozunit + + +def test_release(run_mach, capfd): + cmd = [ + "try", + "scriptworker", + "--no-push", + "tree", + ] + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + expected = re.compile( + dedent( + r""" + Pushed via `mach try scriptworker` + Calculated try_task_config.json: + { + "parameters": { + "app_version": "\d+\.\d+", + "build_number": \d+, + """ + ).lstrip(), + re.MULTILINE, + ) + assert expected.search(output) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_task_configs.py b/tools/tryselect/test/test_task_configs.py new file mode 100644 index 0000000000..4865a1bfca --- /dev/null +++ b/tools/tryselect/test/test_task_configs.py @@ -0,0 +1,144 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import inspect +import subprocess +from argparse import ArgumentParser +from textwrap import dedent + +import mozunit +import pytest +from tryselect.task_config import Pernosco, all_task_configs + +# task configs have a list of tests of the form (input, expected) +TASK_CONFIG_TESTS = { + "artifact": [ + (["--no-artifact"], None), + (["--artifact"], {"use-artifact-builds": True, "disable-pgo": True}), + ], + "chemspill-prio": [ + ([], None), + (["--chemspill-prio"], {"chemspill-prio": {}}), + ], + "env": [ + ([], None), + (["--env", "foo=bar", "--env", "num=10"], {"env": {"foo": "bar", "num": "10"}}), + ], + "path": [ + ([], None), + ( + ["dom/indexedDB"], + {"env": {"MOZHARNESS_TEST_PATHS": '{"xpcshell": ["dom/indexedDB"]}'}}, + ), + ( + ["dom/indexedDB", "testing"], + { + "env": { + "MOZHARNESS_TEST_PATHS": '{"xpcshell": ["dom/indexedDB", "testing"]}' + } + }, + ), + (["invalid/path"], SystemExit), + ], + "pernosco": [ + ([], None), + ], + "rebuild": [ + ([], None), + (["--rebuild", "10"], {"rebuild": 10}), + (["--rebuild", "1"], SystemExit), + (["--rebuild", "21"], SystemExit), + ], + "worker-overrides": [ + ([], None), + ( + ["--worker-override", "alias=worker/pool"], + {"worker-overrides": {"alias": "worker/pool"}}, + ), + ( + [ + "--worker-override", + "alias=worker/pool", + "--worker-override", + "alias=other/pool", + ], + SystemExit, + ), + ( + ["--worker-suffix", "b-linux=-dev"], + {"worker-overrides": {"b-linux": "gecko-1/b-linux-dev"}}, + ), + ( + [ + "--worker-override", + "b-linux=worker/pool" "--worker-suffix", + "b-linux=-dev", + ], + SystemExit, + ), + ], +} + + +@pytest.fixture +def config_patch_resolver(patch_resolver): + def inner(paths): + patch_resolver( + [], [{"flavor": "xpcshell", "srcdir_relpath": path} for path in paths] + ) + + return inner + + +def test_task_configs(config_patch_resolver, task_config, args, expected): + parser = ArgumentParser() + + cfg = all_task_configs[task_config]() + cfg.add_arguments(parser) + + if inspect.isclass(expected) and issubclass(expected, BaseException): + with pytest.raises(expected): + args = parser.parse_args(args) + if task_config == "path": + config_patch_resolver(**vars(args)) + + cfg.try_config(**vars(args)) + else: + args = parser.parse_args(args) + if task_config == "path": + config_patch_resolver(**vars(args)) + assert cfg.try_config(**vars(args)) == expected + + +@pytest.fixture +def patch_pernosco_email_check(monkeypatch): + def inner(val): + def fake_check_output(*args, **kwargs): + return val + + monkeypatch.setattr(subprocess, "check_output", fake_check_output) + + return inner + + +def test_pernosco(patch_pernosco_email_check): + patch_pernosco_email_check( + dedent( + """ + user foobar@mozilla.com + hostname hg.mozilla.com + """ + ) + ) + + parser = ArgumentParser() + + cfg = Pernosco() + cfg.add_arguments(parser) + args = parser.parse_args(["--pernosco"]) + assert cfg.try_config(**vars(args)) == {"env": {"PERNOSCO": "1"}} + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_tasks.py b/tools/tryselect/test/test_tasks.py new file mode 100644 index 0000000000..cdf8cf84c1 --- /dev/null +++ b/tools/tryselect/test/test_tasks.py @@ -0,0 +1,93 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +import mozunit +import pytest +from tryselect.tasks import cache_key, filter_tasks_by_paths, resolve_tests_by_suite + + +def test_filter_tasks_by_paths(patch_resolver): + tasks = ["foobar/xpcshell-1", "foobar/mochitest", "foobar/xpcshell"] + + patch_resolver(["xpcshell"], {}) + assert list(filter_tasks_by_paths(tasks, "dummy")) == [] + + patch_resolver([], [{"flavor": "xpcshell"}]) + assert list(filter_tasks_by_paths(tasks, "dummy")) == [ + "foobar/xpcshell-1", + "foobar/xpcshell", + ] + + +@pytest.mark.parametrize( + "input, tests, expected", + ( + pytest.param( + ["xpcshell.js"], + [{"flavor": "xpcshell", "srcdir_relpath": "xpcshell.js"}], + {"xpcshell": ["xpcshell.js"]}, + id="single test", + ), + pytest.param( + ["xpcshell.ini"], + [ + { + "flavor": "xpcshell", + "srcdir_relpath": "xpcshell.js", + "manifest_relpath": "xpcshell.ini", + }, + ], + {"xpcshell": ["xpcshell.ini"]}, + id="single manifest", + ), + pytest.param( + ["xpcshell.js", "mochitest.js"], + [ + {"flavor": "xpcshell", "srcdir_relpath": "xpcshell.js"}, + {"flavor": "mochitest", "srcdir_relpath": "mochitest.js"}, + ], + { + "xpcshell": ["xpcshell.js"], + "mochitest-plain": ["mochitest.js"], + }, + id="two tests", + ), + pytest.param( + ["test/xpcshell.ini"], + [ + { + "flavor": "xpcshell", + "srcdir_relpath": "test/xpcshell.js", + "manifest_relpath": os.path.join("test", "xpcshell.ini"), + }, + ], + {"xpcshell": ["test/xpcshell.ini"]}, + id="mismatched path separators", + ), + ), +) +def test_resolve_tests_by_suite(patch_resolver, input, tests, expected): + patch_resolver([], tests) + assert resolve_tests_by_suite(input) == expected + + +@pytest.mark.parametrize( + "attr,params,disable_target_task_filter,expected", + ( + ("target_task_set", None, False, "target_task_set"), + ("target_task_set", {"project": "autoland"}, False, "target_task_set"), + ("target_task_set", {"project": "mozilla-central"}, False, "target_task_set"), + ("target_task_set", None, True, "target_task_set-uncommon"), + ("full_task_set", {"project": "pine"}, False, "full_task_set-pine"), + ("full_task_set", None, True, "full_task_set"), + ), +) +def test_cache_key(attr, params, disable_target_task_filter, expected): + assert cache_key(attr, params, disable_target_task_filter) == expected + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/try_presets.yml b/tools/tryselect/try_presets.yml new file mode 100644 index 0000000000..4b689ec0d5 --- /dev/null +++ b/tools/tryselect/try_presets.yml @@ -0,0 +1,266 @@ +--- +# Presets defined here will be available to all users. Run them with: +# $ mach try --preset <name> +# +# If editing this file, make sure to run: +# $ mach python-test tools/tryselect/test/test_presets.py +# +# Descriptions are required. Please keep this in alphabetical order. + +# yamllint disable rule:line-length + +builds: + selector: fuzzy + description: >- + Run builds without any of the extras. + query: + - "^build- !fuzzing !notarization !reproduced !rusttests !signing !upload-symbols" + +builds-debug: + selector: fuzzy + description: >- + Run the bare minimum of debug build jobs to ensure builds work on + all tier-1 platforms. + query: + - "^build- 'debug !fuzzing !rusttests !signing !plain !asan !tsan !noopt !toolchain !upload-symbols" + +builds-debugopt: + selector: fuzzy + description: >- + Run the bare minimum of debug and opt build jobs to ensure builds work on + all tier-1 platforms. + query: + - "^build- !fuzzing !rusttests !signing !plain !asan !tsan !noopt !toolchain !upload-symbols" + +desktop-frontend: + description: >- + Run mochitest-browser, xpcshell, mochitest-chrome, mochitest-a11y, + marionette, firefox-ui-functional on all desktop platforms. + Excludes non-shipped/duplicate configurations like asan/tsan/msix + to reduce the runtime of the push as well as infra load. + Use with --artifact to speed up your trypush. + If this is green, you can be 99% sure that any frontend change will + stick on central. + selector: fuzzy + query: + # Runs 64-bit frontend-tests, plus win7. Tries to avoid running + # asan/tsan because they're not available as artifact builds, and + # rarely offer different results from debug/opt. It also avoids running + # msix/swr/a11y-checks/gpu/nofis/headless variants of otherwise + # identical tests, as again those are unlikely to show different + # results for frontend-only changes. + # This won't run 32-bit debug tests, which seems an acceptable + # trade-off for query complexity + runtime on infrastructure. + - "'browser-chrome 'windows7 | '64 !spi !asan !tsan !msix !a11y !swr | 'linux" + - "'mochitest-chrome 'windows7 | '64 !spi !asan !tsan !swr !gpu" + - "'xpcshell 'windows7 | '64 !spi !asan !tsan !msix !nofis !condprof" + - "'browser-a11y | 'mochitest-a11y 'windows7 | '64 !spi !asan !tsan !no-cache !swr" + - "'marionette 'windows7 | '64 !asan !source !headless !swr" + - "'firefox-ui-functional 'windows7 | '64 !asan !tsan" + +devtools: + selector: fuzzy + description: >- + Runs the tests relevant to the Firefox Devtools + query: + - "'node-debugger | 'node-devtools" + - "'mozlint-eslint" + # Windows: skip jobs on asan and 32 bits platforms + - "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'windows !asan !-32" + # macos: no extra platform to filter out + - "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'macosx" + # Linux is being named "linux1804" and may change over time, so use a more flexible search + - "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'linux '64-qr/ !swr" + - "'xpcshell 'linux !nofis '64-qr/" + +devtools-linux: + selector: fuzzy + description: >- + Runs the tests relevant to the Firefox Devtools, on Linux only. + query: + - "'node-debugger | 'node-devtools" + - "'mozlint-eslint" + - "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'linux '64-qr/ !swr" + - "'xpcshell 'linux !nofis '64-qr/" + +fpush-linux-android: + selector: fuzzy + description: >- + Runs correctness test suites on Linux and Android emulator platforms, as + well as builds across main platforms. The resulting jobs on TreeHerder + used to end up looking like a "F" shape (not so much these days) and so + this is typically referred to as an F-push. This is useful to do as a + general sanity check on changes to cross-platform Gecko code where you + unsure of what tests might be affected. Linux and Android (emulator) + test coverage are relatively cheap to run and cover a lot of the + codebase, while the builds on other platforms catch compilation problems + that might result from only building locally on one platform. + query: + - "'test-linux1804 'debug- !-shippable !-asan" + - "'test-android-em 'debug" + - "^build !-shippable !-signing !-asan !-fuzzing !-rusttests !-base-toolchain !-aar-" + +geckodriver: + selector: fuzzy + description: >- + Runs the tests relevant to geckodriver, which implements the WebDriver + specification. This preset can be filtered down further to limit it to + a specific platform or other tasks only. For example: + |mach try --preset geckodriver -xq "'linux"| + query: + - "'rusttests" + - "'platform 'wdspec 'debug 'nofis" + - "'browsertime 'amazon 'shippable 'firefox 'nofis" + +media-full: + selector: fuzzy + description: >- + Runs tests that exercise media playback and WebRTC code. + query: + - "mochitest-media !dfpi !nofis" + - "mochitest-media android !spi !swr !lite" + - "mochitest-browser-chrome !dfpi !nofis !a11y" + - "mochitest-browser-media" + - "web-platform-tests !dfpi !nofis !shippable" + - "web-platform-tests android !wdspec !spi !swr !lite" + - "crashtest !wdspec !nofis" + - "crashtest android !wdspec !spi !swr !lite" + - "'gtest" + +mochitest-bc: + description: >- + Runs mochitest-browser-chrome on all Desktop platforms in both opt + and debug. Excludes jobs that require non-artifact builds (asan, + tsan, msix, etc.) and some non-default configurations. For frontend + only changes, use this with --artifact to speed up your trypushes. + query: + - "'browser-chrome 'windows7 | '64 !tsan !asan !msix !spi !a11y !swr | 'linux" + selector: fuzzy + +perf: + selector: fuzzy + description: >- + Runs all performance (raptor and talos) tasks across all platforms. + This preset can be filtered down further (e.g to limit it to a specific + platform) via |mach try --preset perf -xq "'windows"|. + + Android hardware platforms are excluded due to resource limitations. + query: + - "^test- !android-hw 'raptor | 'talos" + rebuild: 5 + +perf-chrome: + description: >- + Runs the talos tests most likely to change when making a change to + the browser chrome. This skips a number of talos jobs that are unlikely + to be affected in order to conserve resources. + query: + - "opt-talos- 'chrome | 'svg | 'session | 'tabswitch | 'other | 'g5" + rebuild: 6 + selector: fuzzy + +remote-protocol: + selector: fuzzy + description: >- + Runs the tests relevant to the Remote protocol, which underpins + many test harnesses as well as our CDP and WebDriver implementations. + This preset can be filtered down further to limit it to a specific + platform or to opt/debug tasks only. For example: + |mach try --preset remote-protocol -xq "'linux 'opt"| + query: + - "'awsy-base" + - "'firefox-ui" + - "'marionette !swr | harness" + - "'mochitest-browser !spi !swr !nofis '-1$" + - "'mochitest-remote !spi !swr" + - "'platform 'reftest !swr !nofis | 'android !-lite -1$" + - "'platform 'wdspec !swr" + - "'platform !reftest !wdspec !swr !nofis | 'android !-lite -1$" + - "'puppeteer" + - "'reftest !platform !gpu !swr !no-accel !nofis | 'android !-lite -1$" + - "'xpcshell !spi !tsan !-lite" + +sample-suites: + selector: fuzzy + description: >- + Runs one chunk of every test suite plus all suites that aren't chunked. + It is useful for testing infrastructure changes that can affect the + harnesses themselves but are unlikely to break specific tests. + query: + - ^test- -1$ + # Only run a single talos + raptor suite per platform + - ^test- !1$ !2$ !3$ !4$ !5$ !6$ !7$ !8$ !9$ !0$ !raptor !talos + - ^test- 'raptor-speedometer | 'talos-g1 + +sm-shell-all: + selector: fuzzy + description: <- + Runs a set of tests aimed to give a reasonable level of confidence for + basic SpiderMonkey changes (shell only), all platforms + query: + - "'spidermonkey | 'shell-haz" + - "!shippable !android 'jittest" # macosx64 jittests + +sm-shell: + selector: fuzzy + description: <- + Runs a set of tests aimed to give a reasonable level of confidence for + basic SpiderMonkey changes (shell only) (linux only) + query: + - "!win !osx 'spidermonkey | 'shell-haz" + + +sm-all: + selector: fuzzy + description: <- + Runs a set of tests aimed to give a reasonable level of confidence for + basic SpiderMonkey changes, including those that would require a + browser build. + query: + - "'spidermonkey | 'hazard" + - "!android !asan !shippable 'xpcshell" + - "!android !asan !shippable 'jsreftest" + - "!shippable !android 'jittest" # macosx64 jittests + +webgpu: + selector: fuzzy + description: >- + Runs the tests relevant to WebGPU. + query: + - "'webgpu" + - "source-test-mozlint-updatebot" + - "source-test-vendor-rust" + +webrender: + selector: fuzzy + description: >- + Runs the conformance tests relevant to WebRender. + query: + - "!talos !raptor !shippable !asan '-qr" + - "^webrender-" + +webrender-reftests: + selector: fuzzy + description: >- + Runs the reftests relevant to WebRender. + query: + - "!talos !raptor !shippable !asan !nofis 'reftest" + +webrender-reftests-linux: + selector: fuzzy + description: >- + Runs the reftests relevant to WebRender on linux only. + query: + - "!talos !raptor !shippable !asan !nofis 'linux 'reftest" + +webrender-perf: + selector: fuzzy + description: >- + Runs the performance tests relevant to WebRender. + query: + - "'-qr 'svgr" + - "'-qr 'g1" + - "'-qr 'g4" + - "'-qr 'tp5" + - "'-qr 'talos-webgl" + - "'-qr 'motionmark-animometer" diff --git a/tools/tryselect/util/__init__.py b/tools/tryselect/util/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/util/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/util/dicttools.py b/tools/tryselect/util/dicttools.py new file mode 100644 index 0000000000..465e4a43de --- /dev/null +++ b/tools/tryselect/util/dicttools.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import copy + + +def merge_to(source, dest): + """ + Merge dict and arrays (override scalar values) + + Keys from source override keys from dest, and elements from lists in source + are appended to lists in dest. + + :param dict source: to copy from + :param dict dest: to copy to (modified in place) + """ + + for key, value in source.items(): + # Override mismatching or empty types + if type(value) != type(dest.get(key)): # noqa + dest[key] = source[key] + continue + + # Merge dict + if isinstance(value, dict): + merge_to(value, dest[key]) + continue + + if isinstance(value, list): + dest[key] = dest[key] + source[key] + continue + + dest[key] = source[key] + + return dest + + +def merge(*objects): + """ + Merge the given objects, using the semantics described for merge_to, with + objects later in the list taking precedence. From an inheritance + perspective, "parents" should be listed before "children". + + Returns the result without modifying any arguments. + """ + if len(objects) == 1: + return copy.deepcopy(objects[0]) + return merge_to(objects[-1], merge(*objects[:-1])) diff --git a/tools/tryselect/util/estimates.py b/tools/tryselect/util/estimates.py new file mode 100644 index 0000000000..634b1de2a2 --- /dev/null +++ b/tools/tryselect/util/estimates.py @@ -0,0 +1,126 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +from datetime import datetime, timedelta + +TASK_DURATION_CACHE = "task_duration_history.json" +GRAPH_QUANTILE_CACHE = "graph_quantile_cache.csv" +TASK_DURATION_TAG_FILE = "task_duration_tag.json" + + +def find_all_dependencies(graph, tasklist): + all_dependencies = dict() + + def find_dependencies(task): + dependencies = set() + if task in all_dependencies: + return all_dependencies[task] + if task not in graph: + # Don't add tasks (and so durations) for + # things optimised out. + return dependencies + dependencies.add(task) + for dep in graph.get(task, list()): + all_dependencies[dep] = find_dependencies(dep) + dependencies.update(all_dependencies[dep]) + return dependencies + + full_deps = set() + for task in tasklist: + full_deps.update(find_dependencies(task)) + + # Since these have been asked for, they're not inherited dependencies. + return sorted(full_deps - set(tasklist)) + + +def find_longest_path(graph, tasklist, duration_data): + + dep_durations = dict() + + def find_dependency_durations(task): + if task in dep_durations: + return dep_durations[task] + + durations = [find_dependency_durations(dep) for dep in graph.get(task, list())] + durations.append(0.0) + md = max(durations) + duration_data.get(task, 0.0) + dep_durations[task] = md + return md + + longest_paths = [find_dependency_durations(task) for task in tasklist] + # Default in case there are no tasks + if longest_paths: + return max(longest_paths) + else: + return 0 + + +def determine_percentile(quantiles_file, duration): + + duration = duration.total_seconds() + + with open(quantiles_file) as f: + f.readline() # skip header + boundaries = [float(l.strip()) for l in f.readlines()] + + boundaries.sort() + for i, v in enumerate(boundaries): + if duration < v: + break + # Estimate percentile from len(boundaries)-quantile + return int(100 * i / len(boundaries)) + + +def task_duration_data(cache_dir): + with open(os.path.join(cache_dir, TASK_DURATION_CACHE)) as f: + return json.load(f) + + +def duration_summary(graph_cache_file, tasklist, cache_dir): + durations = task_duration_data(cache_dir) + + graph = dict() + if graph_cache_file: + with open(graph_cache_file) as f: + graph = json.load(f) + dependencies = find_all_dependencies(graph, tasklist) + longest_path = find_longest_path(graph, tasklist, durations) + dependency_duration = 0.0 + for task in dependencies: + dependency_duration += int(durations.get(task, 0.0)) + + total_requested_duration = 0.0 + for task in tasklist: + duration = int(durations.get(task, 0.0)) + total_requested_duration += duration + output = dict() + + total_requested_duration = timedelta(seconds=total_requested_duration) + total_dependency_duration = timedelta(seconds=dependency_duration) + + output["selected_duration"] = total_requested_duration + output["dependency_duration"] = total_dependency_duration + output["dependency_count"] = len(dependencies) + output["selected_count"] = len(tasklist) + + percentile = None + graph_quantile_cache = os.path.join(cache_dir, GRAPH_QUANTILE_CACHE) + if os.path.isfile(graph_quantile_cache): + percentile = determine_percentile( + graph_quantile_cache, total_dependency_duration + total_requested_duration + ) + if percentile: + output["percentile"] = percentile + + output["wall_duration_seconds"] = timedelta(seconds=int(longest_path)) + output["eta_datetime"] = datetime.now() + timedelta(seconds=longest_path) + + output["task_durations"] = { + task: int(durations.get(task, 0.0)) for task in tasklist + } + + return output diff --git a/tools/tryselect/util/fzf.py b/tools/tryselect/util/fzf.py new file mode 100644 index 0000000000..66894ffb67 --- /dev/null +++ b/tools/tryselect/util/fzf.py @@ -0,0 +1,423 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import platform +import shutil +import subprocess +import sys +from distutils.spawn import find_executable + +import mozfile +import six +from gecko_taskgraph.target_tasks import filter_by_uncommon_try_tasks +from mach.util import get_state_dir +from mozboot.util import http_download_and_save +from mozbuild.base import MozbuildObject +from mozterm import Terminal +from packaging.version import Version + +from ..push import check_working_directory +from ..tasks import generate_tasks +from ..util.manage_estimates import ( + download_task_history_data, + make_trimmed_taskgraph_cache, +) + +terminal = Terminal() + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + +PREVIEW_SCRIPT = os.path.join(build.topsrcdir, "tools/tryselect/selectors/preview.py") + +FZF_MIN_VERSION = "0.20.0" +FZF_CURRENT_VERSION = "0.29.0" + +# It would make more sense to have the full filename be the key; but that makes +# the line too long and ./mach lint and black can't agree about what to about that. +# You can get these from the github release, e.g. +# https://github.com/junegunn/fzf/releases/download/0.24.1/fzf_0.24.1_checksums.txt +# However the darwin releases may not be included, so double check you have everything +FZF_CHECKSUMS = { + "linux_armv5.tar.gz": "61d3c2aa77b977ba694836fd1134da9272bd97ee490ececaf87959b985820111", + "linux_armv6.tar.gz": "db6b30fcbbd99ac4cf7e3ff6c5db1d3c0afcbe37d10ec3961bdc43e8c4f2e4f9", + "linux_armv7.tar.gz": "ed86f0e91e41d2cea7960a78e3eb175dc2a5fc1510380c195d0c3559bfdc701c", + "linux_arm64.tar.gz": "47988d8b68905541cbc26587db3ed1cfa8bc3aa8da535120abb4229b988f259e", + "linux_amd64.tar.gz": "0106f458b933be65edb0e8f0edb9a16291a79167836fd26a77ff5496269b5e9a", + "windows_armv5.zip": "08eaac45b3600d82608d292c23e7312696e7e11b6278b292feba25e8eb91c712", + "windows_armv6.zip": "8b6618726a9d591a45120fddebc29f4164e01ce6639ed9aa8fc79ab03eefcfed", + "windows_armv7.zip": "c167117b4c08f4f098446291115871ce5f14a8a8b22f0ca70e1b4342452ab5d7", + "windows_arm64.zip": "0cda7bf68850a3e867224a05949612405e63a4421d52396c1a6c9427d4304d72", + "windows_amd64.zip": "f0797ceee089017108c80b09086c71b8eec43d4af11ce939b78b1d5cfd202540", + "darwin_arm64.zip": "2571b4d381f1fc691e7603bbc8113a67116da2404751ebb844818d512dd62b4b", + "darwin_amd64.zip": "bc541e8ae0feb94efa96424bfe0b944f746db04e22f5cccfe00709925839a57f", + "openbsd_amd64.tar.gz": "b62343827ff83949c09d5e2c8ca0c1198d05f733c9a779ec37edd840541ccdab", + "freebsd_amd64.tar.gz": "f0367f2321c070d103589c7c7eb6a771bc7520820337a6c2fbb75be37ff783a9", +} + +FZF_INSTALL_MANUALLY = """ +The `mach try fuzzy` command depends on fzf. Please install it following the +appropriate instructions for your platform: + + https://github.com/junegunn/fzf#installation + +Only the binary is required, if you do not wish to install the shell and +editor integrations, download the appropriate binary and put it on your $PATH: + + https://github.com/junegunn/fzf/releases +""".lstrip() + +FZF_COULD_NOT_DETERMINE_PLATFORM = ( + """ +Could not automatically obtain the `fzf` binary because we could not determine +your Operating System. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_COULD_NOT_DETERMINE_MACHINE = ( + """ +Could not automatically obtain the `fzf` binary because we could not determine +your machine type. It's reported as '%s' and we don't handle that case; but fzf +may still be available as a prebuilt binary. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_NOT_SUPPORTED_X86 = ( + """ +We don't believe that a prebuilt binary for `fzf` if available on %s, but we +could be wrong. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_NOT_FOUND = ( + """ +Could not find the `fzf` binary. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_VERSION_FAILED = ( + """ +Could not obtain the 'fzf' version; we require version > 0.20.0 for some of +the features. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_INSTALL_FAILED = ( + """ +Failed to install fzf. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_HEADER = """ +For more shortcuts, see {t.italic_white}mach help try fuzzy{t.normal} and {t.italic_white}man fzf +{shortcuts} +""".strip() + +fzf_shortcuts = { + "ctrl-a": "select-all", + "ctrl-d": "deselect-all", + "ctrl-t": "toggle-all", + "alt-bspace": "beginning-of-line+kill-line", + "?": "toggle-preview", +} + +fzf_header_shortcuts = [ + ("select", "tab"), + ("accept", "enter"), + ("cancel", "ctrl-c"), + ("select-all", "ctrl-a"), + ("cursor-up", "up"), + ("cursor-down", "down"), +] + + +def get_fzf_platform(): + if platform.machine() in ["i386", "i686"]: + print(FZF_NOT_SUPPORTED_X86 % platform.machine()) + sys.exit(1) + + if platform.system().lower() == "windows": + if platform.machine().lower() in ["x86_64", "amd64"]: + return "windows_amd64.zip" + elif platform.machine().lower() == "arm64": + return "windows_arm64.zip" + else: + print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine()) + sys.exit(1) + elif platform.system().lower() == "darwin": + if platform.machine().lower() in ["x86_64", "amd64"]: + return "darwin_amd64.zip" + elif platform.machine().lower() == "arm64": + return "darwin_arm64.zip" + else: + print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine()) + sys.exit(1) + elif platform.system().lower() == "linux": + if platform.machine().lower() in ["x86_64", "amd64"]: + return "linux_amd64.tar.gz" + elif platform.machine().lower() == "arm64": + return "linux_arm64.tar.gz" + else: + print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine()) + sys.exit(1) + else: + print(FZF_COULD_NOT_DETERMINE_PLATFORM) + sys.exit(1) + + +def get_fzf_state_dir(): + return os.path.join(get_state_dir(), "fzf") + + +def get_fzf_filename(): + return "fzf-%s-%s" % (FZF_CURRENT_VERSION, get_fzf_platform()) + + +def get_fzf_download_link(): + return "https://github.com/junegunn/fzf/releases/download/%s/%s" % ( + FZF_CURRENT_VERSION, + get_fzf_filename(), + ) + + +def clean_up_state_dir(): + """ + We used to have a checkout of fzf that we would update. + Now we only download the bin and cpin the hash; so if + we find the old git checkout, wipe it + """ + + fzf_path = os.path.join(get_state_dir(), "fzf") + git_path = os.path.join(fzf_path, ".git") + if os.path.isdir(git_path): + shutil.rmtree(fzf_path, ignore_errors=True) + + # Also delete any existing fzf binary + fzf_bin = find_executable("fzf", fzf_path) + if fzf_bin: + mozfile.remove(fzf_bin) + + # Make sure the state dir is present + if not os.path.isdir(fzf_path): + os.makedirs(fzf_path) + + +def download_and_install_fzf(): + clean_up_state_dir() + download_link = get_fzf_download_link() + download_file = get_fzf_filename() + download_destination_path = get_fzf_state_dir() + download_destination_file = os.path.join(download_destination_path, download_file) + http_download_and_save( + download_link, download_destination_file, FZF_CHECKSUMS[get_fzf_platform()] + ) + + mozfile.extract(download_destination_file, download_destination_path) + mozfile.remove(download_destination_file) + + +def get_fzf_version(fzf_bin): + cmd = [fzf_bin, "--version"] + try: + fzf_version = subprocess.check_output(cmd) + except subprocess.CalledProcessError: + print(FZF_VERSION_FAILED) + sys.exit(1) + + # Some fzf versions have extra, e.g 0.18.0 (ff95134) + fzf_version = six.ensure_text(fzf_version.split()[0]) + + return fzf_version + + +def should_force_fzf_update(fzf_bin): + fzf_version = get_fzf_version(fzf_bin) + + # 0.20.0 introduced passing selections through a temporary file, + # which is good for large ctrl-a actions. + if Version(fzf_version) < Version(FZF_MIN_VERSION): + print("fzf version is old, you must update to use ./mach try fuzzy.") + return True + return False + + +def fzf_bootstrap(update=False): + """ + Bootstrap fzf if necessary and return path to the executable. + + This function is a bit complicated. We fetch a new version of fzf if: + 1) an existing fzf is too outdated + 2) the user says --update and we are behind the recommended version + 3) no fzf can be found and + 3a) user passes --update + 3b) user agrees to a prompt + + """ + fzf_path = get_fzf_state_dir() + + fzf_bin = find_executable("fzf") + if not fzf_bin: + fzf_bin = find_executable("fzf", fzf_path) + + if fzf_bin and should_force_fzf_update(fzf_bin): # Case (1) + update = True + + if fzf_bin and not update: + return fzf_bin + + elif fzf_bin and update: + # Case 2 + fzf_version = get_fzf_version(fzf_bin) + if Version(fzf_version) < Version(FZF_CURRENT_VERSION) and update: + # Bug 1623197: We only want to run fzf's `install` if it's not in the $PATH + # Swap to os.path.commonpath when we're not on Py2 + if fzf_bin and update and not fzf_bin.startswith(fzf_path): + print( + "fzf installed somewhere other than {}, please update manually".format( + fzf_path + ) + ) + sys.exit(1) + + download_and_install_fzf() + print("Updated fzf to {}".format(FZF_CURRENT_VERSION)) + else: + print("fzf is the recommended version and does not need an update") + + else: # not fzf_bin: + if not update: + # Case 3b + install = input("Could not detect fzf, install it now? [y/n]: ") + if install.lower() != "y": + return + + # Case 3a and 3b-fall-through + download_and_install_fzf() + fzf_bin = find_executable("fzf", fzf_path) + print("Installed fzf to {}".format(fzf_path)) + + return fzf_bin + + +def format_header(): + shortcuts = [] + for action, key in fzf_header_shortcuts: + shortcuts.append( + "{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}".format( + t=terminal, action=action, key=key + ) + ) + return FZF_HEADER.format(shortcuts=", ".join(shortcuts), t=terminal) + + +def run_fzf(cmd, tasks): + env = dict(os.environ) + env.update( + {"PYTHONPATH": os.pathsep.join([p for p in sys.path if "requests" in p])} + ) + # Make sure fzf uses Windows' shell rather than MozillaBuild bash or + # whatever our caller uses, since it doesn't quote the arguments properly + # and thus windows paths like: C:\moz\foo end up as C:mozfoo... + if platform.system() == "Windows": + env["SHELL"] = env["COMSPEC"] + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + env=env, + universal_newlines=True, + ) + out = proc.communicate("\n".join(tasks))[0].splitlines() + + selected = [] + query = None + if out: + query = out[0] + selected = out[1:] + return query, selected + + +def setup_tasks_for_fzf( + push, + parameters, + full=False, + disable_target_task_filter=False, + show_estimates=True, +): + check_working_directory(push) + tg = generate_tasks( + parameters, full=full, disable_target_task_filter=disable_target_task_filter + ) + all_tasks = sorted(tg.tasks.keys()) + + # graph_Cache created by generate_tasks, recreate the path to that file. + cache_dir = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph" + ) + if full: + graph_cache = os.path.join(cache_dir, "full_task_graph") + dep_cache = os.path.join(cache_dir, "full_task_dependencies") + target_set = os.path.join(cache_dir, "full_task_set") + else: + graph_cache = os.path.join(cache_dir, "target_task_graph") + dep_cache = os.path.join(cache_dir, "target_task_dependencies") + target_set = os.path.join(cache_dir, "target_task_set") + + if show_estimates: + download_task_history_data(cache_dir=cache_dir) + make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_set) + + if not full and not disable_target_task_filter: + # Put all_tasks into a list because it's used multiple times, and "filter()" + # returns a consumable iterator. + all_tasks = list(filter(filter_by_uncommon_try_tasks, all_tasks)) + + return all_tasks, dep_cache, cache_dir + + +def build_base_cmd(fzf, dep_cache, cache_dir, show_estimates=True): + key_shortcuts = [k + ":" + v for k, v in fzf_shortcuts.items()] + base_cmd = [ + fzf, + "-m", + "--bind", + ",".join(key_shortcuts), + "--header", + format_header(), + "--preview-window=right:30%", + "--print-query", + ] + + if show_estimates: + base_cmd.extend( + [ + "--preview", + '{} {} -g {} -s -c {} -t "{{+f}}"'.format( + sys.executable, PREVIEW_SCRIPT, dep_cache, cache_dir + ), + ] + ) + else: + base_cmd.extend( + [ + "--preview", + '{} {} -t "{{+f}}"'.format(sys.executable, PREVIEW_SCRIPT), + ] + ) + + return base_cmd diff --git a/tools/tryselect/util/manage_estimates.py b/tools/tryselect/util/manage_estimates.py new file mode 100644 index 0000000000..23fa481228 --- /dev/null +++ b/tools/tryselect/util/manage_estimates.py @@ -0,0 +1,132 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +from datetime import datetime, timedelta + +import requests +import six + +TASK_DURATION_URL = ( + "https://storage.googleapis.com/mozilla-mach-data/task_duration_history.json" +) +GRAPH_QUANTILES_URL = ( + "https://storage.googleapis.com/mozilla-mach-data/machtry_quantiles.csv" +) +from .estimates import GRAPH_QUANTILE_CACHE, TASK_DURATION_CACHE, TASK_DURATION_TAG_FILE + + +def check_downloaded_history(tag_file, duration_cache, quantile_cache): + if not os.path.isfile(tag_file): + return False + + try: + with open(tag_file) as f: + duration_tags = json.load(f) + download_date = datetime.strptime( + duration_tags.get("download_date"), "%Y-%M-%d" + ) + if download_date < datetime.now() - timedelta(days=7): + return False + except (OSError, ValueError): + return False + + if not os.path.isfile(duration_cache): + return False + # Check for old format version of file. + with open(duration_cache) as f: + data = json.load(f) + if isinstance(data, list): + return False + if not os.path.isfile(quantile_cache): + return False + + return True + + +def download_task_history_data(cache_dir): + """Fetch task duration data exported from BigQuery.""" + task_duration_cache = os.path.join(cache_dir, TASK_DURATION_CACHE) + task_duration_tag_file = os.path.join(cache_dir, TASK_DURATION_TAG_FILE) + graph_quantile_cache = os.path.join(cache_dir, GRAPH_QUANTILE_CACHE) + + if check_downloaded_history( + task_duration_tag_file, task_duration_cache, graph_quantile_cache + ): + return + + try: + os.unlink(task_duration_tag_file) + os.unlink(task_duration_cache) + os.unlink(graph_quantile_cache) + except OSError: + print("No existing task history to clean up.") + + try: + r = requests.get(TASK_DURATION_URL, stream=True) + r.raise_for_status() + except requests.exceptions.RequestException as exc: + # This is fine, the durations just won't be in the preview window. + print( + "Error fetching task duration cache from {}: {}".format( + TASK_DURATION_URL, exc + ) + ) + return + + # The data retrieved from google storage is a newline-separated + # list of json entries, which Python's json module can't parse. + duration_data = list() + for line in r.text.splitlines(): + duration_data.append(json.loads(line)) + + # Reformat duration data to avoid list of dicts, as this is slow in the preview window + duration_data = {d["name"]: d["mean_duration_seconds"] for d in duration_data} + + with open(task_duration_cache, "w") as f: + json.dump(duration_data, f, indent=4) + + try: + r = requests.get(GRAPH_QUANTILES_URL, stream=True) + r.raise_for_status() + except requests.exceptions.RequestException as exc: + # This is fine, the percentile just won't be in the preview window. + print( + "Error fetching task group percentiles from {}: {}".format( + GRAPH_QUANTILES_URL, exc + ) + ) + return + + with open(graph_quantile_cache, "w") as f: + f.write(six.ensure_text(r.content)) + + with open(task_duration_tag_file, "w") as f: + json.dump({"download_date": datetime.now().strftime("%Y-%m-%d")}, f, indent=4) + + +def make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=None): + """Trim the taskgraph cache used for dependencies. + + Speeds up the fzf preview window to less human-perceptible + ranges.""" + if not os.path.isfile(graph_cache): + return + + target_task_set = set() + if target_file and os.path.isfile(target_file): + with open(target_file) as f: + target_task_set = set(json.load(f).keys()) + + with open(graph_cache) as f: + graph = json.load(f) + graph = { + name: list(defn["dependencies"].values()) + for name, defn in graph.items() + if name in target_task_set + } + with open(dep_cache, "w") as f: + json.dump(graph, f, indent=4) diff --git a/tools/tryselect/watchman.json b/tools/tryselect/watchman.json new file mode 100644 index 0000000000..a41b1829d7 --- /dev/null +++ b/tools/tryselect/watchman.json @@ -0,0 +1,15 @@ +[ + "trigger", + ".", + { + "name": "rebuild-taskgraph-cache", + "expression": ["match", "taskcluster/**", "wholename"], + "command": [ + "./mach", + "python", + "-c", + "from tryselect.tasks import generate_tasks; generate_tasks()" + ], + "append_files": false + } +] |
