summaryrefslogtreecommitdiffstats
path: root/tools/tryselect/selectors
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-28 14:29:10 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-28 14:29:10 +0000
commit2aa4a82499d4becd2284cdb482213d541b8804dd (patch)
treeb80bf8bf13c3766139fbacc530efd0dd9d54394c /tools/tryselect/selectors
parentInitial commit. (diff)
downloadfirefox-upstream.tar.xz
firefox-upstream.zip
Adding upstream version 86.0.1.upstream/86.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'tools/tryselect/selectors')
-rw-r--r--tools/tryselect/selectors/__init__.py3
-rw-r--r--tools/tryselect/selectors/again.py118
-rw-r--r--tools/tryselect/selectors/auto.py109
-rw-r--r--tools/tryselect/selectors/chooser/.eslintrc.js16
-rw-r--r--tools/tryselect/selectors/chooser/__init__.py96
-rw-r--r--tools/tryselect/selectors/chooser/app.py183
-rw-r--r--tools/tryselect/selectors/chooser/requirements.txt44
-rw-r--r--tools/tryselect/selectors/chooser/static/filter.js116
-rw-r--r--tools/tryselect/selectors/chooser/static/select.js46
-rw-r--r--tools/tryselect/selectors/chooser/static/style.css108
-rw-r--r--tools/tryselect/selectors/chooser/templates/chooser.html78
-rw-r--r--tools/tryselect/selectors/chooser/templates/close.html12
-rw-r--r--tools/tryselect/selectors/chooser/templates/layout.html37
-rw-r--r--tools/tryselect/selectors/coverage.py447
-rw-r--r--tools/tryselect/selectors/empty.py24
-rw-r--r--tools/tryselect/selectors/fuzzy.py456
-rw-r--r--tools/tryselect/selectors/preview.py101
-rw-r--r--tools/tryselect/selectors/release.py160
-rw-r--r--tools/tryselect/selectors/scriptworker.py177
-rw-r--r--tools/tryselect/selectors/syntax.py699
20 files changed, 3030 insertions, 0 deletions
diff --git a/tools/tryselect/selectors/__init__.py b/tools/tryselect/selectors/__init__.py
new file mode 100644
index 0000000000..c580d191c1
--- /dev/null
+++ b/tools/tryselect/selectors/__init__.py
@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
diff --git a/tools/tryselect/selectors/again.py b/tools/tryselect/selectors/again.py
new file mode 100644
index 0000000000..f76869a19a
--- /dev/null
+++ b/tools/tryselect/selectors/again.py
@@ -0,0 +1,118 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import os
+
+from ..cli import BaseTryParser
+from ..push import push_to_try, history_path
+
+
+class AgainParser(BaseTryParser):
+ name = "again"
+ arguments = [
+ [
+ ["--index"],
+ {
+ "default": 0,
+ "type": int,
+ "help": "Index of entry in the history to re-push, "
+ "where '0' is the most recent (default 0). "
+ "Use --list to display indices.",
+ },
+ ],
+ [
+ ["--list"],
+ {
+ "default": False,
+ "action": "store_true",
+ "dest": "list_configs",
+ "help": "Display history and exit",
+ },
+ ],
+ [
+ ["--list-tasks"],
+ {
+ "default": 0,
+ "action": "count",
+ "dest": "list_tasks",
+ "help": "Like --list, but display selected tasks "
+ "for each history entry, up to 10. Repeat "
+ "to display all selected tasks.",
+ },
+ ],
+ [
+ ["--purge"],
+ {
+ "default": False,
+ "action": "store_true",
+ "help": "Remove all history and exit",
+ },
+ ],
+ ]
+ common_groups = ["push"]
+
+
+def run(
+ index=0, purge=False, list_configs=False, list_tasks=0, message="{msg}", **pushargs
+):
+ if purge:
+ os.remove(history_path)
+ return
+
+ if not os.path.isfile(history_path):
+ print("error: history file not found: {}".format(history_path))
+ return 1
+
+ with open(history_path, "r") as fh:
+ history = fh.readlines()
+
+ if list_configs or list_tasks > 0:
+ for i, data in enumerate(history):
+ msg, config = json.loads(data)
+ version = config.get("version", "1")
+ if version == 1:
+ tasks = config["tasks"]
+ elif version == 2:
+ try_config = config.get("parameters", {}).get("try_task_config", {})
+ tasks = try_config.get("tasks")
+ else:
+ tasks = None
+
+ if tasks is not None:
+ n = len(tasks)
+
+ print(
+ "{index}. ({n} task{s}) {msg}".format(
+ index=i, msg=msg, n=n, s="" if n == 1 else "s"
+ )
+ )
+
+ if list_tasks > 0:
+ indent = " " * 4
+ if list_tasks > 1:
+ shown_tasks = tasks
+ else:
+ shown_tasks = tasks[:10]
+ print(indent + ("\n" + indent).join(shown_tasks))
+
+ num_hidden_tasks = len(tasks) - len(shown_tasks)
+ if num_hidden_tasks > 0:
+ print("{}... and {} more".format(indent, num_hidden_tasks))
+ else:
+ print(
+ "{index}. {msg}".format(
+ index=i,
+ msg=msg,
+ )
+ )
+
+ return
+
+ msg, try_task_config = json.loads(history[index])
+ return push_to_try(
+ "again", message.format(msg=msg), try_task_config=try_task_config, **pushargs
+ )
diff --git a/tools/tryselect/selectors/auto.py b/tools/tryselect/selectors/auto.py
new file mode 100644
index 0000000000..d194f09c8c
--- /dev/null
+++ b/tools/tryselect/selectors/auto.py
@@ -0,0 +1,109 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.util.python_path import find_object
+
+from ..cli import BaseTryParser
+from ..push import push_to_try
+
+
+TRY_AUTO_PARAMETERS = {
+ "optimize_strategies": "taskgraph.optimize:tryselect.bugbug_debug_disperse",
+ "optimize_target_tasks": True,
+ "target_tasks_method": "try_auto",
+ "test_manifest_loader": "bugbug",
+ "try_mode": "try_auto",
+ "try_task_config": {},
+}
+
+
+class AutoParser(BaseTryParser):
+ name = "auto"
+ common_groups = ["push"]
+ task_configs = [
+ "artifact",
+ "env",
+ "chemspill-prio",
+ "disable-pgo",
+ "worker-overrides",
+ ]
+ arguments = [
+ [
+ ["--strategy"],
+ {
+ "default": None,
+ "help": "Override the default optimization strategy. Valid values "
+ "are the experimental strategies defined at the bottom of "
+ "`taskcluster/taskgraph/optimize/__init__.py`.",
+ },
+ ],
+ [
+ ["--tasks-regex"],
+ {
+ "default": [],
+ "action": "append",
+ "help": "Apply a regex filter to the tasks selected. Specifying "
+ "multiple times schedules the union of computed tasks.",
+ },
+ ],
+ [
+ ["--tasks-regex-exclude"],
+ {
+ "default": [],
+ "action": "append",
+ "help": "Apply a regex filter to the tasks selected. Specifying "
+ "multiple times excludes computed tasks matching any regex.",
+ },
+ ],
+ ]
+
+ def validate(self, args):
+ super(AutoParser, self).validate(args)
+
+ if args.strategy:
+ if ":" not in args.strategy:
+ args.strategy = "taskgraph.optimize:tryselect.{}".format(args.strategy)
+
+ try:
+ obj = find_object(args.strategy)
+ except (ImportError, AttributeError):
+ self.error("invalid module path '{}'".format(args.strategy))
+
+ if not isinstance(obj, dict):
+ self.error("object at '{}' must be a dict".format(args.strategy))
+
+
+def run(
+ message="{msg}",
+ push=True,
+ closed_tree=False,
+ strategy=None,
+ tasks_regex=None,
+ tasks_regex_exclude=None,
+ try_config=None,
+ **ignored
+):
+ msg = message.format(msg="Tasks automatically selected.")
+
+ params = TRY_AUTO_PARAMETERS.copy()
+ if try_config:
+ params["try_task_config"] = try_config
+
+ if strategy:
+ params["optimize_strategies"] = strategy
+
+ if tasks_regex or tasks_regex_exclude:
+ params.setdefault("try_task_config", {})["tasks-regex"] = {}
+ params["try_task_config"]["tasks-regex"]["include"] = tasks_regex
+ params["try_task_config"]["tasks-regex"]["exclude"] = tasks_regex_exclude
+
+ task_config = {
+ "version": 2,
+ "parameters": params,
+ }
+ return push_to_try(
+ "auto", msg, try_task_config=task_config, push=push, closed_tree=closed_tree
+ )
diff --git a/tools/tryselect/selectors/chooser/.eslintrc.js b/tools/tryselect/selectors/chooser/.eslintrc.js
new file mode 100644
index 0000000000..861d6bafc2
--- /dev/null
+++ b/tools/tryselect/selectors/chooser/.eslintrc.js
@@ -0,0 +1,16 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+module.exports = {
+ env: {
+ jquery: true,
+ },
+ globals: {
+ apply: true,
+ applyChunks: true,
+ tasks: true,
+ },
+};
diff --git a/tools/tryselect/selectors/chooser/__init__.py b/tools/tryselect/selectors/chooser/__init__.py
new file mode 100644
index 0000000000..91c0b55712
--- /dev/null
+++ b/tools/tryselect/selectors/chooser/__init__.py
@@ -0,0 +1,96 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import webbrowser
+from threading import Timer
+
+from tryselect.cli import BaseTryParser
+from tryselect.push import (
+ check_working_directory,
+ generate_try_task_config,
+ push_to_try,
+)
+from tryselect.tasks import generate_tasks
+
+from taskgraph.target_tasks import filter_by_uncommon_try_tasks
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+class ChooserParser(BaseTryParser):
+ name = "chooser"
+ arguments = []
+ common_groups = ["push", "task"]
+ task_configs = [
+ "artifact",
+ "browsertime",
+ "chemspill-prio",
+ "disable-pgo",
+ "env",
+ "gecko-profile",
+ "path",
+ "pernosco",
+ "rebuild",
+ "worker-overrides",
+ ]
+
+
+def run(
+ update=False,
+ query=None,
+ try_config=None,
+ full=False,
+ parameters=None,
+ save=False,
+ preset=None,
+ mod_presets=False,
+ push=True,
+ message="{msg}",
+ closed_tree=False,
+):
+ from .app import create_application
+
+ check_working_directory(push)
+
+ tg = generate_tasks(parameters, full)
+
+ # Remove tasks that are not to be shown unless `--full` is specified.
+ if not full:
+ blacklisted_tasks = [
+ label
+ for label in tg.tasks.keys()
+ if not filter_by_uncommon_try_tasks(label)
+ ]
+ for task in blacklisted_tasks:
+ tg.tasks.pop(task)
+
+ app = create_application(tg)
+
+ if os.environ.get("WERKZEUG_RUN_MAIN") == "true":
+ # we are in the reloader process, don't open the browser or do any try stuff
+ app.run()
+ return
+
+ # give app a second to start before opening the browser
+ url = "http://127.0.0.1:5000"
+ Timer(1, lambda: webbrowser.open(url)).start()
+ print("Starting trychooser on {}".format(url))
+ app.run()
+
+ selected = app.tasks
+ if not selected:
+ print("no tasks selected")
+ return
+
+ msg = "Try Chooser Enhanced ({} tasks selected)".format(len(selected))
+ return push_to_try(
+ "chooser",
+ message.format(msg=msg),
+ try_task_config=generate_try_task_config("chooser", selected, try_config),
+ push=push,
+ closed_tree=closed_tree,
+ )
diff --git a/tools/tryselect/selectors/chooser/app.py b/tools/tryselect/selectors/chooser/app.py
new file mode 100644
index 0000000000..d09e620517
--- /dev/null
+++ b/tools/tryselect/selectors/chooser/app.py
@@ -0,0 +1,183 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+from abc import ABCMeta, abstractproperty
+from collections import defaultdict
+
+from flask import (
+ Flask,
+ render_template,
+ request,
+)
+
+SECTIONS = []
+SUPPORTED_KINDS = set()
+
+
+def register_section(cls):
+ assert issubclass(cls, Section)
+ instance = cls()
+ SECTIONS.append(instance)
+ SUPPORTED_KINDS.update(instance.kind.split(","))
+
+
+class Section(object):
+ __metaclass__ = ABCMeta
+
+ @abstractproperty
+ def name(self):
+ pass
+
+ @abstractproperty
+ def kind(self):
+ pass
+
+ @abstractproperty
+ def title(self):
+ pass
+
+ @abstractproperty
+ def attrs(self):
+ pass
+
+ def contains(self, task):
+ return task.kind in self.kind.split(",")
+
+ def get_context(self, tasks):
+ labels = defaultdict(lambda: {"max_chunk": 0, "attrs": defaultdict(list)})
+
+ for task in tasks.values():
+ if not self.contains(task):
+ continue
+
+ task = task.attributes
+ label = labels[self.labelfn(task)]
+ for attr in self.attrs:
+ if attr in task and task[attr] not in label["attrs"][attr]:
+ label["attrs"][attr].append(task[attr])
+
+ if "test_chunk" in task:
+ label["max_chunk"] = max(
+ label["max_chunk"], int(task["test_chunk"])
+ )
+
+ return {
+ "name": self.name,
+ "kind": self.kind,
+ "title": self.title,
+ "labels": labels,
+ }
+
+
+@register_section
+class Platform(Section):
+ name = "platform"
+ kind = "build"
+ title = "Platforms"
+ attrs = ["build_platform"]
+
+ def labelfn(self, task):
+ return task["build_platform"]
+
+ def contains(self, task):
+ if not Section.contains(self, task):
+ return False
+
+ # android-stuff tasks aren't actual platforms
+ return task.task["tags"].get("android-stuff", False) != "true"
+
+
+@register_section
+class Test(Section):
+ name = "test"
+ kind = "test"
+ title = "Test Suites"
+ attrs = ["unittest_suite"]
+
+ def labelfn(self, task):
+ suite = task["unittest_suite"].replace(" ", "-")
+
+ if suite.endswith("-chunked"):
+ suite = suite[: -len("-chunked")]
+
+ return suite
+
+ def contains(self, task):
+ if not Section.contains(self, task):
+ return False
+ return task.attributes["unittest_suite"] not in ("raptor", "talos")
+
+
+@register_section
+class Perf(Section):
+ name = "perf"
+ kind = "test"
+ title = "Performance"
+ attrs = ["unittest_suite", "raptor_try_name", "talos_try_name"]
+
+ def labelfn(self, task):
+ suite = task["unittest_suite"]
+ label = task["{}_try_name".format(suite)]
+
+ if not label.startswith(suite):
+ label = "{}-{}".format(suite, label)
+
+ if label.endswith("-e10s"):
+ label = label[: -len("-e10s")]
+
+ return label
+
+ def contains(self, task):
+ if not Section.contains(self, task):
+ return False
+ return task.attributes["unittest_suite"] in ("raptor", "talos")
+
+
+@register_section
+class Analysis(Section):
+ name = "analysis"
+ kind = "build,static-analysis-autotest"
+ title = "Analysis"
+ attrs = ["build_platform"]
+
+ def labelfn(self, task):
+ return task["build_platform"]
+
+ def contains(self, task):
+ if not Section.contains(self, task):
+ return False
+ if task.kind == "build":
+ return task.task["tags"].get("android-stuff", False) == "true"
+ return True
+
+
+def create_application(tg):
+ tasks = {l: t for l, t in tg.tasks.items() if t.kind in SUPPORTED_KINDS}
+ sections = [s.get_context(tasks) for s in SECTIONS]
+ context = {
+ "tasks": {l: t.attributes for l, t in tasks.items()},
+ "sections": sections,
+ }
+
+ app = Flask(__name__)
+ app.env = "development"
+ app.tasks = []
+
+ @app.route("/", methods=["GET", "POST"])
+ def chooser():
+ if request.method == "GET":
+ return render_template("chooser.html", **context)
+
+ if request.form["action"] == "Push":
+ labels = request.form["selected-tasks"].splitlines()
+ app.tasks.extend(labels)
+
+ shutdown = request.environ.get("werkzeug.server.shutdown")
+ if shutdown:
+ shutdown()
+ return render_template("close.html")
+
+ return app
diff --git a/tools/tryselect/selectors/chooser/requirements.txt b/tools/tryselect/selectors/chooser/requirements.txt
new file mode 100644
index 0000000000..642274f337
--- /dev/null
+++ b/tools/tryselect/selectors/chooser/requirements.txt
@@ -0,0 +1,44 @@
+Flask==1.0.2 \
+ --hash=sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48 \
+ --hash=sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05
+click==7.1.2 \
+ --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \
+ --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc
+Werkzeug==0.14.1 \
+ --hash=sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c \
+ --hash=sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b
+itsdangerous==1.1.0 \
+ --hash=sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19 \
+ --hash=sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749
+Jinja2==2.10 \
+ --hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd \
+ --hash=sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4
+MarkupSafe==1.1.1 \
+ --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
+ --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
+ --hash=sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235 \
+ --hash=sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5 \
+ --hash=sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff \
+ --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \
+ --hash=sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1 \
+ --hash=sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e \
+ --hash=sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183 \
+ --hash=sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66 \
+ --hash=sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1 \
+ --hash=sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1 \
+ --hash=sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e \
+ --hash=sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b \
+ --hash=sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905 \
+ --hash=sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735 \
+ --hash=sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d \
+ --hash=sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e \
+ --hash=sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d \
+ --hash=sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c \
+ --hash=sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21 \
+ --hash=sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2 \
+ --hash=sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5 \
+ --hash=sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b \
+ --hash=sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6 \
+ --hash=sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f \
+ --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \
+ --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7
diff --git a/tools/tryselect/selectors/chooser/static/filter.js b/tools/tryselect/selectors/chooser/static/filter.js
new file mode 100644
index 0000000000..132372665b
--- /dev/null
+++ b/tools/tryselect/selectors/chooser/static/filter.js
@@ -0,0 +1,116 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const selection = $("#selection")[0];
+const count = $("#selection-count")[0];
+const pluralize = (count, noun, suffix = "s") =>
+ `${count} ${noun}${count !== 1 ? suffix : ""}`;
+
+var selected = [];
+
+var updateLabels = () => {
+ $(".tab-pane.active > .filter-label").each(function(index) {
+ let box = $("#" + this.htmlFor)[0];
+ let method = box.checked ? "add" : "remove";
+ $(this)[method + "Class"]("is-checked");
+ });
+};
+
+var apply = () => {
+ let filters = {};
+ let kinds = [];
+
+ $(".filter:checked").each(function(index) {
+ for (let kind of this.name.split(",")) {
+ if (!kinds.includes(kind)) {
+ kinds.push(kind);
+ }
+ }
+
+ // Checkbox element values are generated by Section.get_context() in app.py
+ let attrs = JSON.parse(this.value);
+ for (let attr in attrs) {
+ if (!(attr in filters)) {
+ filters[attr] = [];
+ }
+
+ let values = attrs[attr];
+ filters[attr] = filters[attr].concat(values);
+ }
+ });
+ updateLabels();
+
+ if (
+ Object.keys(filters).length == 0 ||
+ (Object.keys(filters).length == 1 && "build_type" in filters)
+ ) {
+ selection.value = "";
+ count.innerHTML = "0 tasks selected";
+ return;
+ }
+
+ var taskMatches = label => {
+ let task = tasks[label];
+
+ // If no box for the given kind has been checked, this task is
+ // automatically not selected.
+ if (!kinds.includes(task.kind)) {
+ return false;
+ }
+
+ for (let attr in filters) {
+ let values = filters[attr];
+ if (!(attr in task) || values.includes(task[attr])) {
+ continue;
+ }
+ return false;
+ }
+ return true;
+ };
+
+ selected = Object.keys(tasks).filter(taskMatches);
+ applyChunks();
+};
+
+var applyChunks = () => {
+ // For tasks that have a chunk filter applied, we handle that here.
+ let filters = {};
+ $(".filter:text").each(function(index) {
+ let value = $(this).val();
+ if (value === "") {
+ return;
+ }
+
+ let attrs = JSON.parse(this.name);
+ let key = `${attrs.unittest_suite}-${attrs.unittest_flavor}`;
+ if (!(key in filters)) {
+ filters[key] = [];
+ }
+
+ // Parse the chunk strings. These are formatted like printer page setups, e.g: "1,4-6,9"
+ for (let item of value.split(",")) {
+ if (!item.includes("-")) {
+ filters[key].push(parseInt(item));
+ continue;
+ }
+
+ let [start, end] = item.split("-");
+ for (let i = parseInt(start); i <= parseInt(end); ++i) {
+ filters[key].push(i);
+ }
+ }
+ });
+
+ let chunked = selected.filter(function(label) {
+ let task = tasks[label];
+ let key = task.unittest_suite + "-" + task.unittest_flavor;
+ if (key in filters && !filters[key].includes(parseInt(task.test_chunk))) {
+ return false;
+ }
+ return true;
+ });
+
+ selection.value = chunked.join("\n");
+ count.innerText = pluralize(chunked.length, "task") + " selected";
+};
diff --git a/tools/tryselect/selectors/chooser/static/select.js b/tools/tryselect/selectors/chooser/static/select.js
new file mode 100644
index 0000000000..439f979457
--- /dev/null
+++ b/tools/tryselect/selectors/chooser/static/select.js
@@ -0,0 +1,46 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const labels = $("label.multiselect");
+const boxes = $("label.multiselect input:checkbox");
+var lastChecked = {};
+
+// implements shift+click
+labels.click(function(e) {
+ if (e.target.tagName === "INPUT") {
+ return;
+ }
+
+ let box = $("#" + this.htmlFor)[0];
+ let activeSection = $("div.tab-pane.active")[0].id;
+
+ if (activeSection in lastChecked) {
+ // Bug 559506 - In Firefox shift/ctrl/alt+clicking a label doesn't check the box.
+ let isFirefox = navigator.userAgent.toLowerCase().indexOf("firefox") > -1;
+
+ if (e.shiftKey) {
+ if (isFirefox) {
+ box.checked = !box.checked;
+ }
+
+ let start = boxes.index(box);
+ let end = boxes.index(lastChecked[activeSection]);
+
+ boxes
+ .slice(Math.min(start, end), Math.max(start, end) + 1)
+ .prop("checked", box.checked);
+ apply();
+ }
+ }
+
+ lastChecked[activeSection] = box;
+});
+
+function selectAll(btn) {
+ let checked = !!btn.value;
+ $("div.active label.filter-label").each(function(index) {
+ $(this).find("input:checkbox")[0].checked = checked;
+ });
+ apply();
+}
diff --git a/tools/tryselect/selectors/chooser/static/style.css b/tools/tryselect/selectors/chooser/static/style.css
new file mode 100644
index 0000000000..2945e4ee12
--- /dev/null
+++ b/tools/tryselect/selectors/chooser/static/style.css
@@ -0,0 +1,108 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+body {
+ padding-top: 70px;
+}
+
+/* Tabs */
+
+#tabbar .nav-link {
+ color: #009570;
+ font-size: 18px;
+ padding-bottom: 15px;
+ padding-top: 15px;
+}
+
+#tabbar .nav-link.active {
+ color: #212529;
+}
+
+#tabbar .nav-link:hover {
+ color: #0f5a3a;
+}
+
+/* Sections */
+
+.tab-content button {
+ font-size: 14px;
+ margin-bottom: 5px;
+ margin-top: 10px;
+}
+
+.filter-label {
+ display: block;
+ font-size: 16px;
+ position: relative;
+ padding-left: 15px;
+ padding-right: 15px;
+ padding-top: 10px;
+ padding-bottom: 10px;
+ margin-bottom: 0;
+ user-select: none;
+ user-select: none;
+ vertical-align: middle;
+}
+
+.filter-label span {
+ display: flex;
+ min-height: 34px;
+ align-items: center;
+ justify-content: space-between;
+}
+
+.filter-label input[type="checkbox"] {
+ position: absolute;
+ opacity: 0;
+ height: 0;
+ width: 0;
+}
+
+.filter-label input[type="text"] {
+ width: 50px;
+}
+
+.filter-label:hover {
+ background-color: #91a0b0;
+}
+
+.filter-label.is-checked:hover {
+ background-color: #91a0b0;
+}
+
+.filter-label.is-checked {
+ background-color: #404c59;
+ color: white;
+}
+
+/* Preview pane */
+
+#preview {
+ position: fixed;
+ height: 100vh;
+ margin-left: 66%;
+ width: 100%;
+}
+
+#submit-tasks {
+ display: flex;
+ flex-direction: column;
+ height: 80%;
+}
+
+#buttons {
+ display: flex;
+ justify-content: space-between;
+}
+
+#push {
+ background-color: #00e9b7;
+ margin-left: 5px;
+ width: 100%;
+}
+
+#selection {
+ height: 100%;
+ width: 100%;
+}
diff --git a/tools/tryselect/selectors/chooser/templates/chooser.html b/tools/tryselect/selectors/chooser/templates/chooser.html
new file mode 100644
index 0000000000..d89870ac77
--- /dev/null
+++ b/tools/tryselect/selectors/chooser/templates/chooser.html
@@ -0,0 +1,78 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ - You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+{% extends 'layout.html' %}
+{% block content %}
+<div class="container-fluid">
+ <div class="row">
+ <div class="col-8">
+ <div class="form-group form-inline">
+ <span class="col-form-label col-md-2 pt-1">Build Type</span>
+ <div class="form-check form-check-inline">
+ <input id="both" class="filter form-check-input" type="radio" name="buildtype" value='{}' onchange="apply();" checked>
+ <label for="both" class="form-check-label">both</label>
+ </div>
+ {% for type in ["opt", "debug"] %}
+ <div class="form-check form-check-inline">
+ <input id="{{ type }}" class="filter form-check-input" type="radio" name="buildtype" value='{"build_type": "{{ type }}"}' onchange="apply();">
+ <label for={{ type }} class="form-check-label">{{ type }}</label>
+ </div>
+ {% endfor %}
+ </div>
+ <ul class="nav nav-tabs" id="tabbar" role="tablist">
+ {% for section in sections %}
+ <li class="nav-item">
+ {% if loop.first %}
+ <a class="nav-link active" id="{{ section.name }}-tab" data-toggle="tab" href="#{{section.name }}" role="tab" aria-controls="{{ section.name }}" aria-selected="true">{{ section.title }}</a>
+ {% else %}
+ <a class="nav-link" id="{{ section.name }}-tab" data-toggle="tab" href="#{{section.name }}" role="tab" aria-controls="{{ section.name }}" aria-selected="false">{{ section.title }}</a>
+ {% endif %}
+ </li>
+ {% endfor %}
+ </ul>
+ <div class="tab-content">
+ <button type="button" class="btn btn-secondary" value="true" onclick="selectAll(this);">Select All</button>
+ <button type="button" class="btn btn-secondary" onclick="selectAll(this);">Deselect All</button>
+ {% for section in sections %}
+ {% if loop.first %}
+ <div class="tab-pane show active" id="{{ section.name }}" role="tabpanel" aria-labelledby="{{ section.name }}-tab">
+ {% else %}
+ <div class="tab-pane" id="{{ section.name }}" role="tabpanel" aria-labelledby="{{ section.name }}-tab">
+ {% endif %}
+ {% for label, meta in section.labels|dictsort %}
+ <label class="multiselect filter-label" for={{ label }}>
+ <span>
+ {{ label }}
+ <input class="filter" type="checkbox" id={{ label }} name="{{ section.kind }}" value='{{ meta.attrs|tojson|safe }}' onchange="console.log('checkbox onchange triggered');apply();">
+ {% if meta.max_chunk > 1 %}
+ <input class="filter" type="text" pattern="^[0-9][0-9,-]*$" placeholder="1-{{ meta.max_chunk }}" name='{{ meta.attrs|tojson|safe }}' oninput="applyChunks();">
+ {% endif %}
+ </span>
+ </label>
+ {% endfor %}
+ </div>
+ {% endfor %}
+ </div>
+ </div>
+ <div class="col-4" id="preview">
+ <form id="submit-tasks" action="" method="POST">
+ <textarea id="selection" name="selected-tasks" wrap="off"></textarea>
+ <span id="selection-count">0 tasks selected</span><br>
+ <span id="buttons">
+ <input id="cancel" class="btn btn-default" type="submit" name="action" value="Cancel">
+ <input id="push" class="btn btn-default" type="submit" name="action" value="Push">
+ </span>
+ </form>
+ </div>
+ </div>
+</div>
+{% endblock %}
+
+{% block scripts %}
+<script>
+ const tasks = {{ tasks|tojson|safe }};
+</script>
+<script src="{{ url_for('static', filename='filter.js') }}"></script>
+<script src="{{ url_for('static', filename='select.js') }}"></script>
+{% endblock %}
diff --git a/tools/tryselect/selectors/chooser/templates/close.html b/tools/tryselect/selectors/chooser/templates/close.html
new file mode 100644
index 0000000000..622f9f38b2
--- /dev/null
+++ b/tools/tryselect/selectors/chooser/templates/close.html
@@ -0,0 +1,12 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ - You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+{% extends 'layout.html' %}
+{% block content %}
+<div class="container-fluid">
+ <div class="alert alert-primary" role="alert">
+ You may now close this page.
+ </div>
+</div>
+{% endblock %}
diff --git a/tools/tryselect/selectors/chooser/templates/layout.html b/tools/tryselect/selectors/chooser/templates/layout.html
new file mode 100644
index 0000000000..4dfd27b46b
--- /dev/null
+++ b/tools/tryselect/selectors/chooser/templates/layout.html
@@ -0,0 +1,37 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ - You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+ <head>
+ <meta charset="utf-8">
+ <title>Try Chooser Enhanced</title>
+ <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css">
+ <link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
+ </head>
+ <body>
+ <nav class="navbar navbar-default fixed-top navbar-dark bg-dark">
+ <div class="container-fluid">
+ <span class="navbar-brand mb-0 h1">Try Chooser Enhanced</span>
+ <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+ <span class="navbar-toggler-icon"></span>
+ </button>
+ <div class="collapse navbar-collapse" id="navbarSupportedContent">
+ <ul class="navbar-nav mr-auto">
+ <li class="nav-item">
+ <a class="nav-link" href="https://firefox-source-docs.mozilla.org/tools/try/index.html">Documentation</a>
+ </li>
+ <li class="nav-item">
+ <a class="nav-link" href="https://treeherder.mozilla.org/#/jobs?repo=try">Treeherder</a>
+ </li>
+ </ul>
+ </div>
+ </div>
+ </nav>
+ {% block content %}{% endblock %}
+ <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
+ <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script>
+ {% block scripts %}{% endblock %}
+ </body>
+</html>
diff --git a/tools/tryselect/selectors/coverage.py b/tools/tryselect/selectors/coverage.py
new file mode 100644
index 0000000000..88f53528e7
--- /dev/null
+++ b/tools/tryselect/selectors/coverage.py
@@ -0,0 +1,447 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import collections
+import json
+import hashlib
+import os
+import shutil
+import six
+import sqlite3
+import subprocess
+import requests
+import datetime
+
+
+from mozboot.util import get_state_dir
+from mozbuild.base import MozbuildObject
+from mozpack.files import FileFinder
+from moztest.resolve import TestResolver
+from mozversioncontrol import get_repository_object
+
+from ..cli import BaseTryParser
+from ..tasks import generate_tasks, filter_tasks_by_paths, resolve_tests_by_suite
+from ..push import push_to_try, generate_try_task_config
+
+here = os.path.abspath(os.path.dirname(__file__))
+build = None
+vcs = None
+CHUNK_MAPPING_FILE = None
+CHUNK_MAPPING_TAG_FILE = None
+
+
+def setup_globals():
+ # Avoid incurring expensive computation on import.
+ global build, vcs, CHUNK_MAPPING_TAG_FILE, CHUNK_MAPPING_FILE
+ build = MozbuildObject.from_environment(cwd=here)
+ vcs = get_repository_object(build.topsrcdir)
+
+ root_hash = hashlib.sha256(
+ six.ensure_binary(os.path.abspath(build.topsrcdir))
+ ).hexdigest()
+ cache_dir = os.path.join(get_state_dir(), "cache", root_hash, "chunk_mapping")
+ if not os.path.isdir(cache_dir):
+ os.makedirs(cache_dir)
+ CHUNK_MAPPING_FILE = os.path.join(cache_dir, "chunk_mapping.sqlite")
+ CHUNK_MAPPING_TAG_FILE = os.path.join(cache_dir, "chunk_mapping_tag.json")
+
+
+# Maps from platform names in the chunk_mapping sqlite database to respective
+# substrings in task names.
+PLATFORM_MAP = {
+ "linux": "test-linux64/opt",
+ "windows": "test-windows10-64/opt",
+}
+
+# List of platform/build type combinations that are included in pushes by |mach try coverage|.
+OPT_TASK_PATTERNS = [
+ "macosx64/opt",
+ "windows10-64/opt",
+ "windows7-32/opt",
+ "linux64/opt",
+]
+
+
+class CoverageParser(BaseTryParser):
+ name = "coverage"
+ arguments = []
+ common_groups = ["push", "task"]
+ task_configs = [
+ "artifact",
+ "env",
+ "rebuild",
+ "chemspill-prio",
+ "disable-pgo",
+ "worker-overrides",
+ ]
+
+
+def read_test_manifests():
+ """Uses TestResolver to read all test manifests in the tree.
+
+ Returns a (tests, support_files_map) tuple that describes the tests in the tree:
+ tests - a set of test file paths
+ support_files_map - a dict that maps from each support file to a list with
+ test files that require them it
+ """
+ test_resolver = TestResolver.from_environment(cwd=here)
+ file_finder = FileFinder(build.topsrcdir)
+ support_files_map = collections.defaultdict(list)
+ tests = set()
+
+ for test in test_resolver.resolve_tests(build.topsrcdir):
+ tests.add(test["srcdir_relpath"])
+ if "support-files" not in test:
+ continue
+
+ for support_file_pattern in test["support-files"].split():
+ # Get the pattern relative to topsrcdir.
+ if support_file_pattern.startswith("!/"):
+ support_file_pattern = support_file_pattern[2:]
+ elif support_file_pattern.startswith("/"):
+ support_file_pattern = support_file_pattern[1:]
+ else:
+ support_file_pattern = os.path.normpath(
+ os.path.join(test["dir_relpath"], support_file_pattern)
+ )
+
+ # If it doesn't have a glob, then it's a single file.
+ if "*" not in support_file_pattern:
+ # Simple case: single support file, just add it here.
+ support_files_map[support_file_pattern].append(test["srcdir_relpath"])
+ continue
+
+ for support_file, _ in file_finder.find(support_file_pattern):
+ support_files_map[support_file].append(test["srcdir_relpath"])
+
+ return tests, support_files_map
+
+
+# TODO cache the output of this function
+all_tests, all_support_files = read_test_manifests()
+
+
+def download_coverage_mapping(base_revision):
+ try:
+ with open(CHUNK_MAPPING_TAG_FILE, "r") as f:
+ tags = json.load(f)
+ if tags["target_revision"] == base_revision:
+ return
+ else:
+ print("Base revision changed.")
+ except (IOError, ValueError):
+ print("Chunk mapping file not found.")
+
+ CHUNK_MAPPING_URL_TEMPLATE = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/project.relman.code-coverage.production.cron.{}/artifacts/public/chunk_mapping.tar.xz" # noqa
+ JSON_PUSHES_URL_TEMPLATE = "https://hg.mozilla.org/mozilla-central/json-pushes?version=2&tipsonly=1&startdate={}" # noqa
+
+ # Get pushes from at most one month ago.
+ PUSH_HISTORY_DAYS = 30
+ delta = datetime.timedelta(days=PUSH_HISTORY_DAYS)
+ start_time = (datetime.datetime.now() - delta).strftime("%Y-%m-%d")
+ pushes_url = JSON_PUSHES_URL_TEMPLATE.format(start_time)
+ pushes_data = requests.get(pushes_url + "&tochange={}".format(base_revision)).json()
+ if "error" in pushes_data:
+ if "unknown revision" in pushes_data["error"]:
+ print(
+ "unknown revision {}, trying with latest mozilla-central".format(
+ base_revision
+ )
+ )
+ pushes_data = requests.get(pushes_url).json()
+
+ if "error" in pushes_data:
+ raise Exception(pushes_data["error"])
+
+ pushes = pushes_data["pushes"]
+
+ print("Looking for coverage data. This might take a minute or two.")
+ print("Base revision:", base_revision)
+ for push_id in sorted(pushes.keys())[::-1]:
+ rev = pushes[push_id]["changesets"][0]
+ url = CHUNK_MAPPING_URL_TEMPLATE.format(rev)
+ print("push id: {},\trevision: {}".format(push_id, rev))
+
+ r = requests.head(url)
+ if not r.ok:
+ continue
+
+ print("Chunk mapping found, downloading...")
+ r = requests.get(url, stream=True)
+
+ CHUNK_MAPPING_ARCHIVE = os.path.join(build.topsrcdir, "chunk_mapping.tar.xz")
+ with open(CHUNK_MAPPING_ARCHIVE, "wb") as f:
+ r.raw.decode_content = True
+ shutil.copyfileobj(r.raw, f)
+
+ subprocess.check_call(
+ [
+ "tar",
+ "-xJf",
+ CHUNK_MAPPING_ARCHIVE,
+ "-C",
+ os.path.dirname(CHUNK_MAPPING_FILE),
+ ]
+ )
+ os.remove(CHUNK_MAPPING_ARCHIVE)
+ assert os.path.isfile(CHUNK_MAPPING_FILE)
+ with open(CHUNK_MAPPING_TAG_FILE, "w") as f:
+ json.dump(
+ {
+ "target_revision": base_revision,
+ "chunk_mapping_revision": rev,
+ "download_date": start_time,
+ },
+ f,
+ )
+ return
+ raise Exception("Could not find suitable coverage data.")
+
+
+def is_a_test(cursor, path):
+ """Checks the all_tests global and the chunk mapping database to see if a
+ given file is a test file.
+ """
+ if path in all_tests:
+ return True
+
+ cursor.execute("SELECT COUNT(*) from chunk_to_test WHERE path=?", (path,))
+ if cursor.fetchone()[0]:
+ return True
+
+ cursor.execute("SELECT COUNT(*) from file_to_test WHERE test=?", (path,))
+ if cursor.fetchone()[0]:
+ return True
+
+ return False
+
+
+def tests_covering_file(cursor, path):
+ """Returns a set of tests that cover a given source file."""
+ cursor.execute("SELECT test FROM file_to_test WHERE source=?", (path,))
+ return set(e[0] for e in cursor.fetchall())
+
+
+def tests_in_chunk(cursor, platform, chunk):
+ """Returns a set of tests that are contained in a given chunk."""
+ cursor.execute(
+ "SELECT path FROM chunk_to_test WHERE platform=? AND chunk=?", (platform, chunk)
+ )
+ # Because of bug 1480103, some entries in this table contain both a file name and a test name,
+ # separated by a space. With the split, only the file name is kept.
+ return set(e[0].split(" ")[0] for e in cursor.fetchall())
+
+
+def chunks_covering_file(cursor, path):
+ """Returns a set of (platform, chunk) tuples with the chunks that cover a given source file."""
+ cursor.execute("SELECT platform, chunk FROM file_to_chunk WHERE path=?", (path,))
+ return set(cursor.fetchall())
+
+
+def tests_supported_by_file(path):
+ """Returns a set of tests that are using the given file as a support-file."""
+ return set(all_support_files[path])
+
+
+def find_tests(changed_files):
+ """Finds both individual tests and test chunks that should be run to test code changes.
+ Argument: a list of file paths relative to the source checkout.
+
+ Returns: a (test_files, test_chunks) tuple with two sets.
+ test_files - contains tests that should be run to verify changes to changed_files.
+ test_chunks - contains (platform, chunk) tuples with chunks that should be
+ run. These chunnks do not support running a subset of the tests (like
+ cppunit or gtest), so the whole chunk must be run.
+ """
+ test_files = set()
+ test_chunks = set()
+ files_no_coverage = set()
+
+ with sqlite3.connect(CHUNK_MAPPING_FILE) as conn:
+ c = conn.cursor()
+ for path in changed_files:
+ # If path is a test, add it to the list and continue.
+ if is_a_test(c, path):
+ test_files.add(path)
+ continue
+
+ # Look at the chunk mapping and add all tests that cover this file.
+ tests = tests_covering_file(c, path)
+ chunks = chunks_covering_file(c, path)
+ # If we found tests covering this, then it's not a support-file, so
+ # save these and continue.
+ if tests or chunks:
+ test_files |= tests
+ test_chunks |= chunks
+ continue
+
+ # Check if the path is a support-file for any test, by querying test manifests.
+ tests = tests_supported_by_file(path)
+ if tests:
+ test_files |= tests
+ continue
+
+ # There is no coverage information for this file.
+ files_no_coverage.add(path)
+
+ files_covered = set(changed_files) - files_no_coverage
+ test_files = set(s.replace("\\", "/") for s in test_files)
+
+ _print_found_tests(files_covered, files_no_coverage, test_files, test_chunks)
+
+ remaining_test_chunks = set()
+ # For all test_chunks, try to find the tests contained by them in the
+ # chunk_to_test mapping.
+ for platform, chunk in test_chunks:
+ tests = tests_in_chunk(c, platform, chunk)
+ if tests:
+ for test in tests:
+ test_files.add(test.replace("\\", "/"))
+ else:
+ remaining_test_chunks.add((platform, chunk))
+
+ return test_files, remaining_test_chunks
+
+
+def _print_found_tests(files_covered, files_no_coverage, test_files, test_chunks):
+ """Print a summary of what will be run to the user's terminal."""
+ files_covered = sorted(files_covered)
+ files_no_coverage = sorted(files_no_coverage)
+ test_files = sorted(test_files)
+ test_chunks = sorted(test_chunks)
+
+ if files_covered:
+ print(
+ "Found {} modified source files with test coverage:".format(
+ len(files_covered)
+ )
+ )
+ for covered in files_covered:
+ print("\t", covered)
+
+ if files_no_coverage:
+ print(
+ "Found {} modified source files with no coverage:".format(
+ len(files_no_coverage)
+ )
+ )
+ for f in files_no_coverage:
+ print("\t", f)
+
+ if not files_covered:
+ print("No modified source files are covered by tests.")
+ elif not files_no_coverage:
+ print("All modified source files are covered by tests.")
+
+ if test_files:
+ print("Running {} individual test files.".format(len(test_files)))
+ else:
+ print("Could not find any individual tests to run.")
+
+ if test_chunks:
+ print("Running {} test chunks.".format(len(test_chunks)))
+ for platform, chunk in test_chunks:
+ print("\t", platform, chunk)
+ else:
+ print("Could not find any test chunks to run.")
+
+
+def filter_tasks_by_chunks(tasks, chunks):
+ """Find all tasks that will run the given chunks."""
+ selected_tasks = set()
+ for platform, chunk in chunks:
+ platform = PLATFORM_MAP[platform]
+
+ selected_task = None
+ for task in tasks:
+ if not task.startswith(platform):
+ continue
+
+ if not any(
+ task[len(platform) + 1 :].endswith(c) for c in [chunk, chunk + "-e10s"]
+ ):
+ continue
+
+ assert (
+ selected_task is None
+ ), "Only one task should be selected for a given platform-chunk couple ({} - {}), {} and {} were selected".format( # noqa
+ platform, chunk, selected_task, task
+ )
+ selected_task = task
+
+ if selected_task is None:
+ print("Warning: no task found for chunk", platform, chunk)
+ else:
+ selected_tasks.add(selected_task)
+
+ return list(selected_tasks)
+
+
+def is_opt_task(task):
+ """True if the task runs on a supported platform and build type combination.
+ This is used to remove -ccov/asan/pgo tasks, along with all /debug tasks.
+ """
+ return any(platform in task for platform in OPT_TASK_PATTERNS)
+
+
+def run(
+ try_config={},
+ full=False,
+ parameters=None,
+ push=True,
+ message="{msg}",
+ closed_tree=False,
+):
+ setup_globals()
+ download_coverage_mapping(vcs.base_ref)
+
+ changed_sources = vcs.get_outgoing_files()
+ test_files, test_chunks = find_tests(changed_sources)
+ if not test_files and not test_chunks:
+ print("ERROR Could not find any tests or chunks to run.")
+ return 1
+
+ tg = generate_tasks(parameters, full)
+ all_tasks = tg.tasks.keys()
+
+ tasks_by_chunks = filter_tasks_by_chunks(all_tasks, test_chunks)
+ tasks_by_path = filter_tasks_by_paths(all_tasks, test_files)
+ tasks = filter(is_opt_task, set(tasks_by_path) | set(tasks_by_chunks))
+ tasks = list(tasks)
+
+ if not tasks:
+ print("ERROR Did not find any matching tasks after filtering.")
+ return 1
+ test_count_message = (
+ "{test_count} test file{test_plural} that "
+ + "cover{test_singular} these changes "
+ + "({task_count} task{task_plural} to be scheduled)"
+ ).format(
+ test_count=len(test_files),
+ test_plural="" if len(test_files) == 1 else "s",
+ test_singular="s" if len(test_files) == 1 else "",
+ task_count=len(tasks),
+ task_plural="" if len(tasks) == 1 else "s",
+ )
+ print("Found " + test_count_message)
+
+ # Set the test paths to be run by setting MOZHARNESS_TEST_PATHS.
+ path_env = {
+ "MOZHARNESS_TEST_PATHS": six.ensure_text(
+ json.dumps(resolve_tests_by_suite(test_files))
+ )
+ }
+ try_config.setdefault("env", {}).update(path_env)
+
+ # Build commit message.
+ msg = "try coverage - " + test_count_message
+ return push_to_try(
+ "coverage",
+ message.format(msg=msg),
+ try_task_config=generate_try_task_config("coverage", tasks, try_config),
+ push=push,
+ closed_tree=closed_tree,
+ )
diff --git a/tools/tryselect/selectors/empty.py b/tools/tryselect/selectors/empty.py
new file mode 100644
index 0000000000..6f83851696
--- /dev/null
+++ b/tools/tryselect/selectors/empty.py
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from ..cli import BaseTryParser
+from ..push import push_to_try, generate_try_task_config
+
+
+class EmptyParser(BaseTryParser):
+ name = "empty"
+ common_groups = ["push"]
+
+
+def run(message="{msg}", push=True, closed_tree=False):
+ msg = 'No try selector specified, use "Add New Jobs" to select tasks.'
+ return push_to_try(
+ "empty",
+ message.format(msg=msg),
+ try_task_config=generate_try_task_config("empty", []),
+ push=push,
+ closed_tree=closed_tree,
+ )
diff --git a/tools/tryselect/selectors/fuzzy.py b/tools/tryselect/selectors/fuzzy.py
new file mode 100644
index 0000000000..fe71f7ff39
--- /dev/null
+++ b/tools/tryselect/selectors/fuzzy.py
@@ -0,0 +1,456 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import platform
+import subprocess
+import six
+import sys
+from distutils.spawn import find_executable
+from distutils.version import StrictVersion
+from six.moves import input
+
+from mozbuild.base import MozbuildObject
+from mozbuild.util import ensure_subprocess_env
+from mozboot.util import get_state_dir
+from mozterm import Terminal
+
+from ..cli import BaseTryParser
+from ..tasks import generate_tasks, filter_tasks_by_paths
+from ..push import check_working_directory, push_to_try, generate_try_task_config
+from ..util.manage_estimates import (
+ download_task_history_data,
+ make_trimmed_taskgraph_cache,
+)
+
+from taskgraph.target_tasks import filter_by_uncommon_try_tasks
+
+terminal = Terminal()
+
+here = os.path.abspath(os.path.dirname(__file__))
+build = MozbuildObject.from_environment(cwd=here)
+
+PREVIEW_SCRIPT = os.path.join(build.topsrcdir, "tools/tryselect/selectors/preview.py")
+
+FZF_NOT_FOUND = """
+Could not find the `fzf` binary.
+
+The `mach try fuzzy` command depends on fzf. Please install it following the
+appropriate instructions for your platform:
+
+ https://github.com/junegunn/fzf#installation
+
+Only the binary is required, if you do not wish to install the shell and
+editor integrations, download the appropriate binary and put it on your $PATH:
+
+ https://github.com/junegunn/fzf/releases
+""".lstrip()
+
+FZF_VERSION_FAILED = """
+Could not obtain the 'fzf' version.
+
+The 'mach try fuzzy' command depends on fzf, and requires version > 0.20.0
+for some of the features. Please install it following the appropriate
+instructions for your platform:
+
+ https://github.com/junegunn/fzf#installation
+
+Only the binary is required, if you do not wish to install the shell and
+editor integrations, download the appropriate binary and put it on your $PATH:
+
+ https://github.com/junegunn/fzf/releases
+""".lstrip()
+
+FZF_INSTALL_FAILED = """
+Failed to install fzf.
+
+Please install fzf manually following the appropriate instructions for your
+platform:
+
+ https://github.com/junegunn/fzf#installation
+
+Only the binary is required, if you do not wish to install the shell and
+editor integrations, download the appropriate binary and put it on your $PATH:
+
+ https://github.com/junegunn/fzf/releases
+""".lstrip()
+
+FZF_HEADER = """
+For more shortcuts, see {t.italic_white}mach help try fuzzy{t.normal} and {t.italic_white}man fzf
+{shortcuts}
+""".strip()
+
+fzf_shortcuts = {
+ "ctrl-a": "select-all",
+ "ctrl-d": "deselect-all",
+ "ctrl-t": "toggle-all",
+ "alt-bspace": "beginning-of-line+kill-line",
+ "?": "toggle-preview",
+}
+
+fzf_header_shortcuts = [
+ ("select", "tab"),
+ ("accept", "enter"),
+ ("cancel", "ctrl-c"),
+ ("select-all", "ctrl-a"),
+ ("cursor-up", "up"),
+ ("cursor-down", "down"),
+]
+
+
+class FuzzyParser(BaseTryParser):
+ name = "fuzzy"
+ arguments = [
+ [
+ ["-q", "--query"],
+ {
+ "metavar": "STR",
+ "action": "append",
+ "default": [],
+ "help": "Use the given query instead of entering the selection "
+ "interface. Equivalent to typing <query><ctrl-a><enter> "
+ "from the interface. Specifying multiple times schedules "
+ "the union of computed tasks.",
+ },
+ ],
+ [
+ ["-i", "--interactive"],
+ {
+ "action": "store_true",
+ "default": False,
+ "help": "Force running fzf interactively even when using presets or "
+ "queries with -q/--query.",
+ },
+ ],
+ [
+ ["-x", "--and"],
+ {
+ "dest": "intersection",
+ "action": "store_true",
+ "default": False,
+ "help": "When specifying queries on the command line with -q/--query, "
+ "use the intersection of tasks rather than the union. This is "
+ "especially useful for post filtering presets.",
+ },
+ ],
+ [
+ ["-e", "--exact"],
+ {
+ "action": "store_true",
+ "default": False,
+ "help": "Enable exact match mode. Terms will use an exact match "
+ "by default, and terms prefixed with ' will become fuzzy.",
+ },
+ ],
+ [
+ ["-u", "--update"],
+ {
+ "action": "store_true",
+ "default": False,
+ "help": "Update fzf before running.",
+ },
+ ],
+ [
+ ["-s", "--show-estimates"],
+ {
+ "action": "store_true",
+ "default": False,
+ "help": "Show task duration estimates.",
+ },
+ ],
+ [
+ ["--disable-target-task-filter"],
+ {
+ "action": "store_true",
+ "default": False,
+ "help": "Some tasks run on mozilla-central but are filtered out "
+ "of the default list due to resource constraints. This flag "
+ "disables this filtering.",
+ },
+ ],
+ ]
+ common_groups = ["push", "task", "preset"]
+ task_configs = [
+ "artifact",
+ "browsertime",
+ "chemspill-prio",
+ "disable-pgo",
+ "env",
+ "gecko-profile",
+ "path",
+ "pernosco",
+ "rebuild",
+ "routes",
+ "worker-overrides",
+ ]
+
+
+def run_cmd(cmd, cwd=None):
+ is_win = platform.system() == "Windows"
+ return subprocess.call(cmd, cwd=cwd, shell=True if is_win else False)
+
+
+def run_fzf_install_script(fzf_path):
+ if platform.system() == "Windows":
+ cmd = ["bash", "-c", "./install --bin"]
+ else:
+ cmd = ["./install", "--bin"]
+
+ if run_cmd(cmd, cwd=fzf_path):
+ print(FZF_INSTALL_FAILED)
+ sys.exit(1)
+
+
+def should_force_fzf_update(fzf_bin):
+ cmd = [fzf_bin, "--version"]
+ try:
+ fzf_version = subprocess.check_output(cmd)
+ except subprocess.CalledProcessError:
+ print(FZF_VERSION_FAILED)
+ sys.exit(1)
+
+ # Some fzf versions have extra, e.g 0.18.0 (ff95134)
+ fzf_version = six.ensure_text(fzf_version.split()[0])
+
+ # 0.20.0 introduced passing selections through a temporary file,
+ # which is good for large ctrl-a actions.
+ if StrictVersion(fzf_version) < StrictVersion("0.20.0"):
+ print("fzf version is old, forcing update.")
+ return True
+ return False
+
+
+def fzf_bootstrap(update=False):
+ """Bootstrap fzf if necessary and return path to the executable.
+
+ The bootstrap works by cloning the fzf repository and running the included
+ `install` script. If update is True, we will pull the repository and re-run
+ the install script.
+ """
+ fzf_bin = find_executable("fzf")
+ if fzf_bin and should_force_fzf_update(fzf_bin):
+ update = True
+
+ if fzf_bin and not update:
+ return fzf_bin
+
+ fzf_path = os.path.join(get_state_dir(), "fzf")
+
+ # Bug 1623197: We only want to run fzf's `install` if it's not in the $PATH
+ # Swap to os.path.commonpath when we're not on Py2
+ if fzf_bin and update and not fzf_bin.startswith(fzf_path):
+ print(
+ "fzf installed somewhere other than {}, please update manually".format(
+ fzf_path
+ )
+ )
+ sys.exit(1)
+
+ def get_fzf():
+ return find_executable("fzf", os.path.join(fzf_path, "bin"))
+
+ if os.path.isdir(fzf_path):
+ if update:
+ ret = run_cmd(["git", "pull"], cwd=fzf_path)
+ if ret:
+ print("Update fzf failed.")
+ sys.exit(1)
+
+ run_fzf_install_script(fzf_path)
+ return get_fzf()
+
+ fzf_bin = get_fzf()
+ if not fzf_bin or should_force_fzf_update(fzf_bin):
+ return fzf_bootstrap(update=True)
+
+ return fzf_bin
+
+ if not update:
+ install = input("Could not detect fzf, install it now? [y/n]: ")
+ if install.lower() != "y":
+ return
+
+ if not find_executable("git"):
+ print("Git not found.")
+ print(FZF_INSTALL_FAILED)
+ sys.exit(1)
+
+ cmd = ["git", "clone", "--depth", "1", "https://github.com/junegunn/fzf.git"]
+ if subprocess.call(cmd, cwd=os.path.dirname(fzf_path)):
+ print(FZF_INSTALL_FAILED)
+ sys.exit(1)
+
+ run_fzf_install_script(fzf_path)
+
+ print("Installed fzf to {}".format(fzf_path))
+ return get_fzf()
+
+
+def format_header():
+ shortcuts = []
+ for action, key in fzf_header_shortcuts:
+ shortcuts.append(
+ "{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}".format(
+ t=terminal, action=action, key=key
+ )
+ )
+ return FZF_HEADER.format(shortcuts=", ".join(shortcuts), t=terminal)
+
+
+def run_fzf(cmd, tasks):
+ env = dict(os.environ)
+ env.update(
+ {"PYTHONPATH": os.pathsep.join([p for p in sys.path if "requests" in p])}
+ )
+ proc = subprocess.Popen(
+ cmd,
+ stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE,
+ env=ensure_subprocess_env(env),
+ universal_newlines=True,
+ )
+ out = proc.communicate("\n".join(tasks))[0].splitlines()
+
+ selected = []
+ query = None
+ if out:
+ query = out[0]
+ selected = out[1:]
+ return query, selected
+
+
+def run(
+ update=False,
+ query=None,
+ intersect_query=None,
+ try_config=None,
+ full=False,
+ parameters=None,
+ save_query=False,
+ push=True,
+ message="{msg}",
+ test_paths=None,
+ exact=False,
+ closed_tree=False,
+ show_estimates=False,
+ disable_target_task_filter=False,
+):
+ fzf = fzf_bootstrap(update)
+
+ if not fzf:
+ print(FZF_NOT_FOUND)
+ return 1
+
+ check_working_directory(push)
+ tg = generate_tasks(
+ parameters, full=full, disable_target_task_filter=disable_target_task_filter
+ )
+ all_tasks = sorted(tg.tasks.keys())
+
+ # graph_Cache created by generate_tasks, recreate the path to that file.
+ cache_dir = os.path.join(get_state_dir(srcdir=True), "cache", "taskgraph")
+ if full:
+ graph_cache = os.path.join(cache_dir, "full_task_graph")
+ dep_cache = os.path.join(cache_dir, "full_task_dependencies")
+ target_set = os.path.join(cache_dir, "full_task_set")
+ else:
+ graph_cache = os.path.join(cache_dir, "target_task_graph")
+ dep_cache = os.path.join(cache_dir, "target_task_dependencies")
+ target_set = os.path.join(cache_dir, "target_task_set")
+
+ if show_estimates:
+ download_task_history_data(cache_dir=cache_dir)
+ make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_set)
+
+ if not full and not disable_target_task_filter:
+ # Put all_tasks into a list because it's used multiple times, and "filter()"
+ # returns a consumable iterator.
+ all_tasks = list(filter(filter_by_uncommon_try_tasks, all_tasks))
+
+ if test_paths:
+ all_tasks = filter_tasks_by_paths(all_tasks, test_paths)
+ if not all_tasks:
+ return 1
+
+ key_shortcuts = [k + ":" + v for k, v in six.iteritems(fzf_shortcuts)]
+ base_cmd = [
+ fzf,
+ "-m",
+ "--bind",
+ ",".join(key_shortcuts),
+ "--header",
+ format_header(),
+ "--preview-window=right:30%",
+ "--print-query",
+ ]
+
+ if show_estimates:
+ base_cmd.extend(
+ [
+ "--preview",
+ '{} {} -g {} -s -c {} -t "{{+f}}"'.format(
+ sys.executable, PREVIEW_SCRIPT, dep_cache, cache_dir
+ ),
+ ]
+ )
+ else:
+ base_cmd.extend(
+ [
+ "--preview",
+ '{} {} -t "{{+f}}"'.format(sys.executable, PREVIEW_SCRIPT),
+ ]
+ )
+
+ if exact:
+ base_cmd.append("--exact")
+
+ selected = set()
+ queries = []
+
+ def get_tasks(query_arg=None, candidate_tasks=all_tasks):
+ cmd = base_cmd[:]
+ if query_arg and query_arg != "INTERACTIVE":
+ cmd.extend(["-f", query_arg])
+
+ query_str, tasks = run_fzf(cmd, sorted(candidate_tasks))
+ queries.append(query_str)
+ return set(tasks)
+
+ for q in query or []:
+ selected |= get_tasks(q)
+
+ for q in intersect_query or []:
+ if not selected:
+ tasks = get_tasks(q)
+ selected |= tasks
+ else:
+ tasks = get_tasks(q, selected)
+ selected &= tasks
+
+ if not queries:
+ selected = get_tasks()
+
+ if not selected:
+ print("no tasks selected")
+ return
+
+ if save_query:
+ return queries
+
+ # build commit message
+ msg = "Fuzzy"
+ args = ["query={}".format(q) for q in queries]
+ if test_paths:
+ args.append("paths={}".format(":".join(test_paths)))
+ if args:
+ msg = "{} {}".format(msg, "&".join(args))
+ return push_to_try(
+ "fuzzy",
+ message.format(msg=msg),
+ try_task_config=generate_try_task_config("fuzzy", selected, try_config),
+ push=push,
+ closed_tree=closed_tree,
+ )
diff --git a/tools/tryselect/selectors/preview.py b/tools/tryselect/selectors/preview.py
new file mode 100644
index 0000000000..de476475f0
--- /dev/null
+++ b/tools/tryselect/selectors/preview.py
@@ -0,0 +1,101 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""This script is intended to be called through fzf as a preview formatter."""
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import os
+import sys
+
+here = os.path.abspath(os.path.dirname(__file__))
+sys.path.insert(0, os.path.join(os.path.dirname(here), "util"))
+from estimates import duration_summary
+
+
+def process_args():
+ """Process preview arguments."""
+ argparser = argparse.ArgumentParser()
+ argparser.add_argument(
+ "-s",
+ "--show-estimates",
+ action="store_true",
+ help="Show task duration estimates (default: False)",
+ )
+ argparser.add_argument(
+ "-g",
+ "--graph-cache",
+ type=str,
+ default=None,
+ help="Filename of task graph dependencies",
+ )
+ argparser.add_argument(
+ "-c",
+ "--cache_dir",
+ type=str,
+ default=None,
+ help="Path to cache directory containing task durations",
+ )
+ argparser.add_argument(
+ "-t",
+ "--tasklist",
+ type=str,
+ default=None,
+ help="Path to temporary file containing the selected tasks",
+ )
+ return argparser.parse_args()
+
+
+def plain_display(taskfile):
+ """Original preview window display."""
+ with open(taskfile, "r") as f:
+ tasklist = [line.strip() for line in f]
+ print("\n".join(sorted(tasklist)))
+
+
+def duration_display(graph_cache_file, taskfile, cache_dir):
+ """Preview window display with task durations + metadata."""
+ with open(taskfile, "r") as f:
+ tasklist = [line.strip() for line in f]
+
+ durations = duration_summary(graph_cache_file, tasklist, cache_dir)
+ output = ""
+ max_columns = int(os.environ["FZF_PREVIEW_COLUMNS"])
+
+ output += "\nSelected tasks take {}\n".format(durations["selected_duration"])
+ output += "+{} dependencies, total {}\n".format(
+ durations["dependency_count"],
+ durations["selected_duration"] + durations["dependency_duration"],
+ )
+
+ if durations.get("quantile"):
+ output += "This is in the top {}% of requests\n".format(durations["quantile"])
+
+ output += "Estimated finish in {} at {}".format(
+ durations["wall_duration_seconds"], durations["eta_datetime"].strftime("%H:%M")
+ )
+
+ duration_width = 5 # show five numbers at most.
+ output += "{:>{width}}\n".format("Duration", width=max_columns)
+ for task in tasklist:
+ duration = durations["task_durations"].get(task, 0.0)
+ output += "{:{align}{width}} {:{nalign}{nwidth}}s\n".format(
+ task,
+ duration,
+ align="<",
+ width=max_columns - (duration_width + 2), # 2: space and 's'
+ nalign=">",
+ nwidth=duration_width,
+ )
+
+ print(output)
+
+
+if __name__ == "__main__":
+ args = process_args()
+ if args.show_estimates and os.path.isdir(args.cache_dir):
+ duration_display(args.graph_cache, args.tasklist, args.cache_dir)
+ else:
+ plain_display(args.tasklist)
diff --git a/tools/tryselect/selectors/release.py b/tools/tryselect/selectors/release.py
new file mode 100644
index 0000000000..78573ff011
--- /dev/null
+++ b/tools/tryselect/selectors/release.py
@@ -0,0 +1,160 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+import attr
+from mozilla_version.gecko import FirefoxVersion
+
+from ..cli import BaseTryParser
+from ..push import push_to_try, vcs
+
+TARGET_TASKS = {
+ "staging": "staging_release_builds",
+ "release-sim": "release_simulation",
+}
+
+
+def read_file(path):
+ with open(path) as fh:
+ return fh.read()
+
+
+class ReleaseParser(BaseTryParser):
+ name = "release"
+ arguments = [
+ [
+ ["-v", "--version"],
+ {
+ "metavar": "STR",
+ "required": True,
+ "action": "store",
+ "type": FirefoxVersion.parse,
+ "help": "The version number to use for the staging release.",
+ },
+ ],
+ [
+ ["--migration"],
+ {
+ "metavar": "STR",
+ "action": "append",
+ "dest": "migrations",
+ "choices": [
+ "central-to-beta",
+ "beta-to-release",
+ "early-to-late-beta",
+ "release-to-esr",
+ ],
+ "help": "Migration to run for the release (can be specified multiple times).",
+ },
+ ],
+ [
+ ["--no-limit-locales"],
+ {
+ "action": "store_false",
+ "dest": "limit_locales",
+ "help": "Don't build a limited number of locales in the staging release.",
+ },
+ ],
+ [
+ ["--tasks"],
+ {
+ "choices": TARGET_TASKS.keys(),
+ "default": "staging",
+ "help": "Which tasks to run on-push.",
+ },
+ ],
+ ]
+ common_groups = ["push"]
+ task_configs = ["disable-pgo", "worker-overrides"]
+
+ def __init__(self, *args, **kwargs):
+ super(ReleaseParser, self).__init__(*args, **kwargs)
+ self.set_defaults(migrations=[])
+
+
+_UNSET_EARLY_BETA_OR_EARLIER = """\
+# Define indicating that this build is prior to one of the early betas. To be
+# unset mid-way through the beta cycle.
+EARLY_BETA_OR_EARLIER=
+"""
+
+
+def run(
+ version,
+ migrations,
+ limit_locales,
+ tasks,
+ try_config=None,
+ push=True,
+ message="{msg}",
+ closed_tree=False,
+):
+ app_version = attr.evolve(version, beta_number=None, is_esr=False)
+
+ files_to_change = {
+ "browser/config/version.txt": "{}\n".format(app_version),
+ "browser/config/version_display.txt": "{}\n".format(version),
+ "config/milestone.txt": "{}\n".format(app_version),
+ }
+
+ if "beta-to-release" in migrations:
+ files_to_change["build/defines.sh"] = _UNSET_EARLY_BETA_OR_EARLIER
+
+ release_type = version.version_type.name.lower()
+ if release_type not in ("beta", "release", "esr"):
+ raise Exception(
+ "Can't do staging release for version: {} type: {}".format(
+ version, version.version_type
+ )
+ )
+ elif release_type == "esr":
+ release_type += str(version.major_number)
+ task_config = {
+ "version": 2,
+ "parameters": {
+ "target_tasks_method": TARGET_TASKS[tasks],
+ "optimize_target_tasks": True,
+ "release_type": release_type,
+ },
+ }
+ if try_config:
+ task_config["parameters"]["try_task_config"] = try_config
+
+ for migration in migrations:
+ migration_path = os.path.join(
+ vcs.path,
+ "testing/mozharness/configs/merge_day",
+ "{}.py".format(migration.replace("-", "_")),
+ )
+ migration_config = {}
+ with open(migration_path) as f:
+ code = compile(f.read(), migration_path, "exec")
+ exec(code, migration_config, migration_config)
+ for (path, from_, to) in migration_config["config"]["replacements"]:
+ if path in files_to_change:
+ contents = files_to_change[path]
+ else:
+ contents = read_file(path)
+ files_to_change[path] = contents.replace(from_, to)
+
+ if limit_locales:
+ files_to_change["browser/locales/l10n-changesets.json"] = read_file(
+ os.path.join(vcs.path, "browser/locales/l10n-onchange-changesets.json")
+ )
+ files_to_change["browser/locales/shipped-locales"] = "en-US\n" + read_file(
+ os.path.join(vcs.path, "browser/locales/onchange-locales")
+ )
+
+ msg = "staging release: {}".format(version)
+ return push_to_try(
+ "release",
+ message.format(msg=msg),
+ push=push,
+ closed_tree=closed_tree,
+ try_task_config=task_config,
+ files_to_change=files_to_change,
+ )
diff --git a/tools/tryselect/selectors/scriptworker.py b/tools/tryselect/selectors/scriptworker.py
new file mode 100644
index 0000000000..fd47ee59af
--- /dev/null
+++ b/tools/tryselect/selectors/scriptworker.py
@@ -0,0 +1,177 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import sys
+
+import requests
+
+from taskgraph.parameters import Parameters
+from taskgraph.util.taskcluster import find_task_id, get_artifact, get_session
+from taskgraph.util.taskgraph import find_existing_tasks
+
+from ..cli import BaseTryParser
+from ..push import push_to_try
+
+TASK_TYPES = {
+ "linux-signing": [
+ "build-signing-linux-shippable/opt",
+ "build-signing-linux64-shippable/opt",
+ "build-signing-win64-shippable/opt",
+ "build-signing-win32-shippable/opt",
+ "repackage-signing-win64-shippable/opt",
+ "repackage-signing-win32-shippable/opt",
+ "repackage-signing-msi-win32-shippable/opt",
+ "repackage-signing-msi-win64-shippable/opt",
+ "mar-signing-linux64-shippable/opt",
+ ],
+ "linux-signing-partial": ["partials-signing-linux64-shippable/opt"],
+ "mac-signing": ["build-signing-macosx64-shippable/opt"],
+ "beetmover-candidates": ["beetmover-repackage-linux64-shippable/opt"],
+ "bouncer-submit": ["release-bouncer-sub-firefox"],
+ "balrog-submit": [
+ "release-balrog-submit-toplevel-firefox",
+ "balrog-linux64-shippable/opt",
+ ],
+ "tree": ["release-early-tagging-firefox", "release-version-bump-firefox"],
+}
+
+RELEASE_TO_BRANCH = {
+ "beta": "releases/mozilla-beta",
+ "release": "releases/mozilla-release",
+}
+
+
+class ScriptworkerParser(BaseTryParser):
+ name = "scriptworker"
+ arguments = [
+ [
+ ["task_type"],
+ {
+ "choices": ["list"] + list(TASK_TYPES.keys()),
+ "metavar": "TASK-TYPE",
+ "help": "Scriptworker task types to run. (Use `list` to show possibilities)",
+ },
+ ],
+ [
+ ["--release-type"],
+ {
+ "choices": ["nightly"] + list(RELEASE_TO_BRANCH.keys()),
+ "default": "beta",
+ "help": "Release type to run",
+ },
+ ],
+ ]
+
+ common_groups = ["push"]
+ task_configs = ["worker-overrides", "routes"]
+
+
+def get_releases(branch):
+ response = requests.get(
+ "https://shipitapi-public.services.mozilla.com/releases",
+ params={"product": "firefox", "branch": branch, "status": "shipped"},
+ headers={"Accept": "application/json"},
+ )
+ response.raise_for_status()
+ return response.json()
+
+
+def get_release_graph(release):
+ for phase in release["phases"]:
+ if phase["name"] in ("ship_firefox",):
+ return phase["actionTaskId"]
+ raise Exception("No ship phase.")
+
+
+def get_nightly_graph():
+ return find_task_id(
+ "gecko.v2.mozilla-central.latest.taskgraph.decision-nightly-desktop"
+ )
+
+
+def print_available_task_types():
+ print("Available task types:")
+ for task_type, tasks in TASK_TYPES.items():
+ print(" " * 4 + "{}:".format(task_type))
+ for task in tasks:
+ print(" " * 8 + "- {}".format(task))
+
+
+def get_hg_file(parameters, path):
+ session = get_session()
+ response = session.get(parameters.file_url(path))
+ response.raise_for_status()
+ return response.content
+
+
+def run(
+ task_type,
+ release_type,
+ try_config=None,
+ push=True,
+ message="{msg}",
+ closed_tree=False,
+):
+ if task_type == "list":
+ print_available_task_types()
+ sys.exit(0)
+
+ if release_type == "nightly":
+ previous_graph = get_nightly_graph()
+ else:
+ release = get_releases(RELEASE_TO_BRANCH[release_type])[-1]
+ previous_graph = get_release_graph(release)
+ existing_tasks = find_existing_tasks([previous_graph])
+
+ previous_parameters = Parameters(
+ strict=False, **get_artifact(previous_graph, "public/parameters.yml")
+ )
+
+ # Copy L10n configuration from the commit the release we are using was
+ # based on. This *should* ensure that the chunking of L10n tasks is the
+ # same between graphs.
+ files_to_change = {
+ path: get_hg_file(previous_parameters, path)
+ for path in [
+ "browser/locales/l10n-changesets.json",
+ "browser/locales/shipped-locales",
+ ]
+ }
+
+ try_config = try_config or {}
+ task_config = {
+ "version": 2,
+ "parameters": {
+ "existing_tasks": existing_tasks,
+ "try_task_config": try_config,
+ "try_mode": "try_task_config",
+ },
+ }
+ for param in (
+ "app_version",
+ "build_number",
+ "next_version",
+ "release_history",
+ "release_product",
+ "release_type",
+ "version",
+ ):
+ task_config["parameters"][param] = previous_parameters[param]
+
+ try_config["tasks"] = TASK_TYPES[task_type]
+ for label in try_config["tasks"]:
+ if label in existing_tasks:
+ del existing_tasks[label]
+
+ msg = "scriptworker tests: {}".format(task_type)
+ return push_to_try(
+ "scriptworker",
+ message.format(msg=msg),
+ push=push,
+ closed_tree=closed_tree,
+ try_task_config=task_config,
+ files_to_change=files_to_change,
+ )
diff --git a/tools/tryselect/selectors/syntax.py b/tools/tryselect/selectors/syntax.py
new file mode 100644
index 0000000000..d0af69e100
--- /dev/null
+++ b/tools/tryselect/selectors/syntax.py
@@ -0,0 +1,699 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import re
+import sys
+from collections import defaultdict
+
+import mozpack.path as mozpath
+import six
+from moztest.resolve import TestResolver
+
+from ..cli import BaseTryParser
+from ..push import build, push_to_try
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+class SyntaxParser(BaseTryParser):
+ name = "syntax"
+ arguments = [
+ [
+ ["paths"],
+ {
+ "nargs": "*",
+ "default": [],
+ "help": "Paths to search for tests to run on try.",
+ },
+ ],
+ [
+ ["-b", "--build"],
+ {
+ "dest": "builds",
+ "default": "do",
+ "help": "Build types to run (d for debug, o for optimized).",
+ },
+ ],
+ [
+ ["-p", "--platform"],
+ {
+ "dest": "platforms",
+ "action": "append",
+ "help": "Platforms to run (required if not found in the environment as "
+ "AUTOTRY_PLATFORM_HINT).",
+ },
+ ],
+ [
+ ["-u", "--unittests"],
+ {
+ "dest": "tests",
+ "action": "append",
+ "help": "Test suites to run in their entirety.",
+ },
+ ],
+ [
+ ["-t", "--talos"],
+ {
+ "action": "append",
+ "help": "Talos suites to run.",
+ },
+ ],
+ [
+ ["-j", "--jobs"],
+ {
+ "action": "append",
+ "help": "Job tasks to run.",
+ },
+ ],
+ [
+ ["--tag"],
+ {
+ "dest": "tags",
+ "action": "append",
+ "help": "Restrict tests to the given tag (may be specified multiple times).",
+ },
+ ],
+ [
+ ["--and"],
+ {
+ "action": "store_true",
+ "dest": "intersection",
+ "help": "When -u and paths are supplied run only the intersection of the "
+ "tests specified by the two arguments.",
+ },
+ ],
+ [
+ ["--no-artifact"],
+ {
+ "action": "store_true",
+ "help": "Disable artifact builds even if --enable-artifact-builds is set "
+ "in the mozconfig.",
+ },
+ ],
+ [
+ ["-v", "--verbose"],
+ {
+ "dest": "verbose",
+ "action": "store_true",
+ "default": False,
+ "help": "Print detailed information about the resulting test selection "
+ "and commands performed.",
+ },
+ ],
+ ]
+
+ # Arguments we will accept on the command line and pass through to try
+ # syntax with no further intervention. The set is taken from
+ # http://trychooser.pub.build.mozilla.org with a few additions.
+ #
+ # Note that the meaning of store_false and store_true arguments is
+ # not preserved here, as we're only using these to echo the literal
+ # arguments to another consumer. Specifying either store_false or
+ # store_true here will have an equivalent effect.
+ pass_through_arguments = {
+ "--rebuild": {
+ "action": "store",
+ "dest": "rebuild",
+ "help": "Re-trigger all test jobs (up to 20 times)",
+ },
+ "--rebuild-talos": {
+ "action": "store",
+ "dest": "rebuild_talos",
+ "help": "Re-trigger all talos jobs",
+ },
+ "--interactive": {
+ "action": "store_true",
+ "dest": "interactive",
+ "help": "Allow ssh-like access to running test containers",
+ },
+ "--no-retry": {
+ "action": "store_true",
+ "dest": "no_retry",
+ "help": "Do not retrigger failed tests",
+ },
+ "--setenv": {
+ "action": "append",
+ "dest": "setenv",
+ "help": "Set the corresponding variable in the test environment for "
+ "applicable harnesses.",
+ },
+ "-f": {
+ "action": "store_true",
+ "dest": "failure_emails",
+ "help": "Request failure emails only",
+ },
+ "--failure-emails": {
+ "action": "store_true",
+ "dest": "failure_emails",
+ "help": "Request failure emails only",
+ },
+ "-e": {
+ "action": "store_true",
+ "dest": "all_emails",
+ "help": "Request all emails",
+ },
+ "--all-emails": {
+ "action": "store_true",
+ "dest": "all_emails",
+ "help": "Request all emails",
+ },
+ "--artifact": {
+ "action": "store_true",
+ "dest": "artifact",
+ "help": "Force artifact builds where possible.",
+ },
+ "--upload-xdbs": {
+ "action": "store_true",
+ "dest": "upload_xdbs",
+ "help": "Upload XDB compilation db files generated by hazard build",
+ },
+ }
+ task_configs = []
+
+ def __init__(self, *args, **kwargs):
+ BaseTryParser.__init__(self, *args, **kwargs)
+
+ group = self.add_argument_group("pass-through arguments")
+ for arg, opts in self.pass_through_arguments.items():
+ group.add_argument(arg, **opts)
+
+
+class TryArgumentTokenizer(object):
+ symbols = [
+ ("separator", ","),
+ ("list_start", "\["),
+ ("list_end", "\]"),
+ ("item", "([^,\[\]\s][^,\[\]]+)"),
+ ("space", "\s+"),
+ ]
+ token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols))
+
+ def tokenize(self, data):
+ for match in self.token_re.finditer(data):
+ symbol = match.lastgroup
+ data = match.group(symbol)
+ if symbol == "space":
+ pass
+ else:
+ yield symbol, data
+
+
+class TryArgumentParser(object):
+ """Simple three-state parser for handling expressions
+ of the from "foo[sub item, another], bar,baz". This takes
+ input from the TryArgumentTokenizer and runs through a small
+ state machine, returning a dictionary of {top-level-item:[sub_items]}
+ i.e. the above would result in
+ {"foo":["sub item", "another"], "bar": [], "baz": []}
+ In the case of invalid input a ValueError is raised."""
+
+ EOF = object()
+
+ def __init__(self):
+ self.reset()
+
+ def reset(self):
+ self.tokens = None
+ self.current_item = None
+ self.data = {}
+ self.token = None
+ self.state = None
+
+ def parse(self, tokens):
+ self.reset()
+ self.tokens = tokens
+ self.consume()
+ self.state = self.item_state
+ while self.token[0] != self.EOF:
+ self.state()
+ return self.data
+
+ def consume(self):
+ try:
+ self.token = next(self.tokens)
+ except StopIteration:
+ self.token = (self.EOF, None)
+
+ def expect(self, *types):
+ if self.token[0] not in types:
+ raise ValueError(
+ "Error parsing try string, unexpected %s" % (self.token[0])
+ )
+
+ def item_state(self):
+ self.expect("item")
+ value = self.token[1].strip()
+ if value not in self.data:
+ self.data[value] = []
+ self.current_item = value
+ self.consume()
+ if self.token[0] == "separator":
+ self.consume()
+ elif self.token[0] == "list_start":
+ self.consume()
+ self.state = self.subitem_state
+ elif self.token[0] == self.EOF:
+ pass
+ else:
+ raise ValueError
+
+ def subitem_state(self):
+ self.expect("item")
+ value = self.token[1].strip()
+ self.data[self.current_item].append(value)
+ self.consume()
+ if self.token[0] == "separator":
+ self.consume()
+ elif self.token[0] == "list_end":
+ self.consume()
+ self.state = self.after_list_end_state
+ else:
+ raise ValueError
+
+ def after_list_end_state(self):
+ self.expect("separator")
+ self.consume()
+ self.state = self.item_state
+
+
+def parse_arg(arg):
+ tokenizer = TryArgumentTokenizer()
+ parser = TryArgumentParser()
+ return parser.parse(tokenizer.tokenize(arg))
+
+
+class AutoTry(object):
+
+ # Maps from flavors to the job names needed to run that flavour
+ flavor_jobs = {
+ "mochitest": ["mochitest-1", "mochitest-e10s-1"],
+ "xpcshell": ["xpcshell"],
+ "chrome": ["mochitest-o"],
+ "browser-chrome": [
+ "mochitest-browser-chrome-1",
+ "mochitest-e10s-browser-chrome-1",
+ "mochitest-browser-chrome-e10s-1",
+ ],
+ "devtools-chrome": [
+ "mochitest-devtools-chrome-1",
+ "mochitest-e10s-devtools-chrome-1",
+ "mochitest-devtools-chrome-e10s-1",
+ ],
+ "crashtest": ["crashtest", "crashtest-e10s"],
+ "reftest": ["reftest", "reftest-e10s"],
+ "remote": ["mochitest-remote"],
+ "web-platform-tests": ["web-platform-tests-1"],
+ }
+
+ flavor_suites = {
+ "mochitest": "mochitests",
+ "xpcshell": "xpcshell",
+ "chrome": "mochitest-o",
+ "browser-chrome": "mochitest-bc",
+ "devtools-chrome": "mochitest-dt",
+ "crashtest": "crashtest",
+ "reftest": "reftest",
+ "web-platform-tests": "web-platform-tests",
+ }
+
+ compiled_suites = [
+ "cppunit",
+ "gtest",
+ "jittest",
+ ]
+
+ common_suites = [
+ "cppunit",
+ "crashtest",
+ "firefox-ui-functional",
+ "geckoview",
+ "geckoview-junit",
+ "gtest",
+ "jittest",
+ "jsreftest",
+ "marionette",
+ "marionette-e10s",
+ "mochitests",
+ "reftest",
+ "robocop",
+ "web-platform-tests",
+ "xpcshell",
+ ]
+
+ def __init__(self):
+ self.topsrcdir = build.topsrcdir
+ self._resolver = None
+
+ @property
+ def resolver(self):
+ if self._resolver is None:
+ self._resolver = TestResolver.from_environment(cwd=here)
+ return self._resolver
+
+ @classmethod
+ def split_try_string(cls, data):
+ return re.findall(r"(?:\[.*?\]|\S)+", data)
+
+ def paths_by_flavor(self, paths=None, tags=None):
+ paths_by_flavor = defaultdict(set)
+
+ if not (paths or tags):
+ return dict(paths_by_flavor)
+
+ tests = list(self.resolver.resolve_tests(paths=paths, tags=tags))
+
+ for t in tests:
+ if t["flavor"] in self.flavor_suites:
+ flavor = t["flavor"]
+ if "subsuite" in t and t["subsuite"] == "devtools":
+ flavor = "devtools-chrome"
+
+ if flavor in ["crashtest", "reftest"]:
+ manifest_relpath = os.path.relpath(t["manifest"], self.topsrcdir)
+ paths_by_flavor[flavor].add(os.path.dirname(manifest_relpath))
+ elif "dir_relpath" in t:
+ paths_by_flavor[flavor].add(t["dir_relpath"])
+ else:
+ file_relpath = os.path.relpath(t["path"], self.topsrcdir)
+ dir_relpath = os.path.dirname(file_relpath)
+ paths_by_flavor[flavor].add(dir_relpath)
+
+ for flavor, path_set in paths_by_flavor.items():
+ paths_by_flavor[flavor] = self.deduplicate_prefixes(path_set, paths)
+
+ return dict(paths_by_flavor)
+
+ def deduplicate_prefixes(self, path_set, input_paths):
+ # Removes paths redundant to test selection in the given path set.
+ # If a path was passed on the commandline that is the prefix of a
+ # path in our set, we only need to include the specified prefix to
+ # run the intended tests (every test in "layout/base" will run if
+ # "layout" is passed to the reftest harness).
+ removals = set()
+ additions = set()
+
+ for path in path_set:
+ full_path = path
+ while path:
+ path, _ = os.path.split(path)
+ if path in input_paths:
+ removals.add(full_path)
+ additions.add(path)
+
+ return additions | (path_set - removals)
+
+ def remove_duplicates(self, paths_by_flavor, tests):
+ rv = {}
+ for item in paths_by_flavor:
+ if self.flavor_suites[item] not in tests:
+ rv[item] = paths_by_flavor[item].copy()
+ return rv
+
+ def calc_try_syntax(
+ self,
+ platforms,
+ tests,
+ talos,
+ jobs,
+ builds,
+ paths_by_flavor,
+ tags,
+ extras,
+ intersection,
+ ):
+ parts = ["try:"]
+
+ if platforms:
+ parts.extend(["-b", builds, "-p", ",".join(platforms)])
+
+ suites = tests if not intersection else {}
+ paths = set()
+ for flavor, flavor_tests in six.iteritems(paths_by_flavor):
+ suite = self.flavor_suites[flavor]
+ if suite not in suites and (not intersection or suite in tests):
+ for job_name in self.flavor_jobs[flavor]:
+ for test in flavor_tests:
+ paths.add("%s:%s" % (flavor, test))
+ suites[job_name] = tests.get(suite, [])
+
+ # intersection implies tests are expected
+ if intersection and not suites:
+ raise ValueError("No tests found matching filters")
+
+ if extras.get("artifact") and any([p.endswith("-nightly") for p in platforms]):
+ print(
+ 'You asked for |--artifact| but "-nightly" platforms don\'t have artifacts. '
+ "Running without |--artifact| instead."
+ )
+ del extras["artifact"]
+
+ if extras.get("artifact"):
+ rejected = []
+ for suite in suites.keys():
+ if any([suite.startswith(c) for c in self.compiled_suites]):
+ rejected.append(suite)
+ if rejected:
+ raise ValueError(
+ "You can't run {} with "
+ "--artifact option.".format(", ".join(rejected))
+ )
+
+ if extras.get("artifact") and "all" in suites.keys():
+ non_compiled_suites = set(self.common_suites) - set(self.compiled_suites)
+ message = (
+ "You asked for |-u all| with |--artifact| but compiled-code tests ({tests})"
+ " can't run against an artifact build. Running (-u {non_compiled_suites}) "
+ "instead."
+ )
+ string_format = {
+ "tests": ",".join(self.compiled_suites),
+ "non_compiled_suites": ",".join(non_compiled_suites),
+ }
+ print(message.format(**string_format))
+ del suites["all"]
+ suites.update({suite_name: None for suite_name in non_compiled_suites})
+
+ if suites:
+ parts.append("-u")
+ parts.append(
+ ",".join(
+ "%s%s" % (k, "[%s]" % ",".join(v) if v else "")
+ for k, v in sorted(suites.items())
+ )
+ )
+
+ if talos:
+ parts.append("-t")
+ parts.append(
+ ",".join(
+ "%s%s" % (k, "[%s]" % ",".join(v) if v else "")
+ for k, v in sorted(talos.items())
+ )
+ )
+
+ if jobs:
+ parts.append("-j")
+ parts.append(",".join(jobs))
+
+ if tags:
+ parts.append(" ".join("--tag %s" % t for t in tags))
+
+ if paths:
+ parts.append("--try-test-paths %s" % " ".join(sorted(paths)))
+
+ args_by_dest = {
+ v["dest"]: k for k, v in SyntaxParser.pass_through_arguments.items()
+ }
+ for dest, value in six.iteritems(extras):
+ assert dest in args_by_dest
+ arg = args_by_dest[dest]
+ action = SyntaxParser.pass_through_arguments[arg]["action"]
+ if action == "store":
+ parts.append(arg)
+ parts.append(value)
+ if action == "append":
+ for e in value:
+ parts.append(arg)
+ parts.append(e)
+ if action in ("store_true", "store_false"):
+ parts.append(arg)
+
+ return " ".join(parts)
+
+ def normalise_list(self, items, allow_subitems=False):
+ rv = defaultdict(list)
+ for item in items:
+ parsed = parse_arg(item)
+ for key, values in six.iteritems(parsed):
+ rv[key].extend(values)
+
+ if not allow_subitems:
+ if not all(item == [] for item in six.itervalues(rv)):
+ raise ValueError("Unexpected subitems in argument")
+ return rv.keys()
+ else:
+ return rv
+
+ def validate_args(self, **kwargs):
+ tests_selected = kwargs["tests"] or kwargs["paths"] or kwargs["tags"]
+ if kwargs["platforms"] is None and (kwargs["jobs"] is None or tests_selected):
+ if "AUTOTRY_PLATFORM_HINT" in os.environ:
+ kwargs["platforms"] = [os.environ["AUTOTRY_PLATFORM_HINT"]]
+ elif tests_selected:
+ print("Must specify platform when selecting tests.")
+ sys.exit(1)
+ else:
+ print(
+ "Either platforms or jobs must be specified as an argument to autotry."
+ )
+ sys.exit(1)
+
+ try:
+ platforms = (
+ self.normalise_list(kwargs["platforms"]) if kwargs["platforms"] else {}
+ )
+ except ValueError as e:
+ print("Error parsing -p argument:\n%s" % e.message)
+ sys.exit(1)
+
+ try:
+ tests = (
+ self.normalise_list(kwargs["tests"], allow_subitems=True)
+ if kwargs["tests"]
+ else {}
+ )
+ except ValueError as e:
+ print("Error parsing -u argument (%s):\n%s" % (kwargs["tests"], e.message))
+ sys.exit(1)
+
+ try:
+ talos = (
+ self.normalise_list(kwargs["talos"], allow_subitems=True)
+ if kwargs["talos"]
+ else []
+ )
+ except ValueError as e:
+ print("Error parsing -t argument:\n%s" % e.message)
+ sys.exit(1)
+
+ try:
+ jobs = self.normalise_list(kwargs["jobs"]) if kwargs["jobs"] else {}
+ except ValueError as e:
+ print("Error parsing -j argument:\n%s" % e.message)
+ sys.exit(1)
+
+ paths = []
+ for p in kwargs["paths"]:
+ p = mozpath.normpath(os.path.abspath(p))
+ if not (os.path.isdir(p) and p.startswith(self.topsrcdir)):
+ print(
+ 'Specified path "%s" is not a directory under the srcdir,'
+ " unable to specify tests outside of the srcdir" % p
+ )
+ sys.exit(1)
+ if len(p) <= len(self.topsrcdir):
+ print(
+ 'Specified path "%s" is at the top of the srcdir and would'
+ " select all tests." % p
+ )
+ sys.exit(1)
+ paths.append(os.path.relpath(p, self.topsrcdir))
+
+ try:
+ tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else []
+ except ValueError as e:
+ print("Error parsing --tags argument:\n%s" % e.message)
+ sys.exit(1)
+
+ extra_values = {k["dest"] for k in SyntaxParser.pass_through_arguments.values()}
+ extra_args = {k: v for k, v in kwargs.items() if k in extra_values and v}
+
+ return kwargs["builds"], platforms, tests, talos, jobs, paths, tags, extra_args
+
+ def run(self, **kwargs):
+ if not any(kwargs[item] for item in ("paths", "tests", "tags")):
+ kwargs["paths"] = set()
+ kwargs["tags"] = set()
+
+ builds, platforms, tests, talos, jobs, paths, tags, extra = self.validate_args(
+ **kwargs
+ )
+
+ if paths or tags:
+ paths = [
+ os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
+ for item in paths
+ ]
+ paths_by_flavor = self.paths_by_flavor(paths=paths, tags=tags)
+
+ if not paths_by_flavor and not tests:
+ print(
+ "No tests were found when attempting to resolve paths:\n\n\t%s"
+ % paths
+ )
+ sys.exit(1)
+
+ if not kwargs["intersection"]:
+ paths_by_flavor = self.remove_duplicates(paths_by_flavor, tests)
+ else:
+ paths_by_flavor = {}
+
+ # No point in dealing with artifacts if we aren't running any builds
+ local_artifact_build = False
+ if platforms:
+ local_artifact_build = kwargs.get("local_artifact_build", False)
+
+ # Add --artifact if --enable-artifact-builds is set ...
+ if local_artifact_build:
+ extra["artifact"] = True
+ # ... unless --no-artifact is explicitly given.
+ if kwargs["no_artifact"]:
+ if "artifact" in extra:
+ del extra["artifact"]
+
+ try:
+ msg = self.calc_try_syntax(
+ platforms,
+ tests,
+ talos,
+ jobs,
+ builds,
+ paths_by_flavor,
+ tags,
+ extra,
+ kwargs["intersection"],
+ )
+ except ValueError as e:
+ print(e.message)
+ sys.exit(1)
+
+ if local_artifact_build and not kwargs["no_artifact"]:
+ print(
+ "mozconfig has --enable-artifact-builds; including "
+ "--artifact flag in try syntax (use --no-artifact "
+ "to override)"
+ )
+
+ if kwargs["verbose"] and paths_by_flavor:
+ print("The following tests will be selected: ")
+ for flavor, paths in six.iteritems(paths_by_flavor):
+ print("%s: %s" % (flavor, ",".join(paths)))
+
+ if kwargs["verbose"]:
+ print("The following try syntax was calculated:\n%s" % msg)
+
+ push_to_try(
+ "syntax",
+ kwargs["message"].format(msg=msg),
+ push=kwargs["push"],
+ closed_tree=kwargs["closed_tree"],
+ )
+
+
+def run(**kwargs):
+ at = AutoTry()
+ return at.run(**kwargs)