summaryrefslogtreecommitdiffstats
path: root/tools/tryselect/task_config.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--tools/tryselect/task_config.py530
1 files changed, 530 insertions, 0 deletions
diff --git a/tools/tryselect/task_config.py b/tools/tryselect/task_config.py
new file mode 100644
index 0000000000..07a39d39e9
--- /dev/null
+++ b/tools/tryselect/task_config.py
@@ -0,0 +1,530 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Templates provide a way of modifying the task definition of selected tasks.
+They are added to 'try_task_config.json' and processed by the transforms.
+"""
+
+
+import json
+import os
+import subprocess
+import sys
+from abc import ABCMeta, abstractmethod, abstractproperty
+from argparse import SUPPRESS, Action
+from textwrap import dedent
+
+import mozpack.path as mozpath
+import six
+from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
+
+from .tasks import resolve_tests_by_suite
+
+here = os.path.abspath(os.path.dirname(__file__))
+build = MozbuildObject.from_environment(cwd=here)
+
+
+class TryConfig:
+ __metaclass__ = ABCMeta
+
+ def __init__(self):
+ self.dests = set()
+
+ def add_arguments(self, parser):
+ for cli, kwargs in self.arguments:
+ action = parser.add_argument(*cli, **kwargs)
+ self.dests.add(action.dest)
+
+ @abstractproperty
+ def arguments(self):
+ pass
+
+ @abstractmethod
+ def try_config(self, **kwargs):
+ pass
+
+ def validate(self, **kwargs):
+ pass
+
+
+class Artifact(TryConfig):
+
+ arguments = [
+ [
+ ["--artifact"],
+ {"action": "store_true", "help": "Force artifact builds where possible."},
+ ],
+ [
+ ["--no-artifact"],
+ {
+ "action": "store_true",
+ "help": "Disable artifact builds even if being used locally.",
+ },
+ ],
+ ]
+
+ def add_arguments(self, parser):
+ group = parser.add_mutually_exclusive_group()
+ return super().add_arguments(group)
+
+ @classmethod
+ def is_artifact_build(cls):
+ try:
+ return build.substs.get("MOZ_ARTIFACT_BUILDS", False)
+ except BuildEnvironmentNotFoundException:
+ return False
+
+ def try_config(self, artifact, no_artifact, **kwargs):
+ if artifact:
+ return {"use-artifact-builds": True, "disable-pgo": True}
+
+ if no_artifact:
+ return
+
+ if self.is_artifact_build():
+ print("Artifact builds enabled, pass --no-artifact to disable")
+ return {"use-artifact-builds": True, "disable-pgo": True}
+
+
+class Pernosco(TryConfig):
+ arguments = [
+ [
+ ["--pernosco"],
+ {
+ "action": "store_true",
+ "default": None,
+ "help": "Opt-in to analysis by the Pernosco debugging service.",
+ },
+ ],
+ [
+ ["--no-pernosco"],
+ {
+ "dest": "pernosco",
+ "action": "store_false",
+ "default": None,
+ "help": "Opt-out of the Pernosco debugging service (if you are on the whitelist).",
+ },
+ ],
+ ]
+
+ def add_arguments(self, parser):
+ group = parser.add_mutually_exclusive_group()
+ return super().add_arguments(group)
+
+ def try_config(self, pernosco, **kwargs):
+ if pernosco is None:
+ return
+
+ if pernosco:
+ try:
+ # The Pernosco service currently requires a Mozilla e-mail address to
+ # log in. Prevent people with non-Mozilla addresses from using this
+ # flag so they don't end up consuming time and resources only to
+ # realize they can't actually log in and see the reports.
+ cmd = ["ssh", "-G", "hg.mozilla.org"]
+ output = subprocess.check_output(
+ cmd, universal_newlines=True
+ ).splitlines()
+ address = [
+ l.rsplit(" ", 1)[-1] for l in output if l.startswith("user")
+ ][0]
+ if not address.endswith("@mozilla.com"):
+ print(
+ dedent(
+ """\
+ Pernosco requires a Mozilla e-mail address to view its reports. Please
+ push to try with an @mozilla.com address to use --pernosco.
+
+ Current user: {}
+ """.format(
+ address
+ )
+ )
+ )
+ sys.exit(1)
+
+ except (subprocess.CalledProcessError, IndexError):
+ print("warning: failed to detect current user for 'hg.mozilla.org'")
+ print("Pernosco requires a Mozilla e-mail address to view its reports.")
+ while True:
+ answer = input(
+ "Do you have an @mozilla.com address? [Y/n]: "
+ ).lower()
+ if answer == "n":
+ sys.exit(1)
+ elif answer == "y":
+ break
+
+ return {
+ "env": {
+ "PERNOSCO": str(int(pernosco)),
+ }
+ }
+
+ def validate(self, **kwargs):
+ if kwargs["try_config"].get("use-artifact-builds"):
+ print(
+ "Pernosco does not support artifact builds at this time. "
+ "Please try again with '--no-artifact'."
+ )
+ sys.exit(1)
+
+
+class Path(TryConfig):
+
+ arguments = [
+ [
+ ["paths"],
+ {
+ "nargs": "*",
+ "default": [],
+ "help": "Run tasks containing tests under the specified path(s).",
+ },
+ ],
+ ]
+
+ def try_config(self, paths, **kwargs):
+ if not paths:
+ return
+
+ for p in paths:
+ if not os.path.exists(p):
+ print("error: '{}' is not a valid path.".format(p), file=sys.stderr)
+ sys.exit(1)
+
+ paths = [
+ mozpath.relpath(mozpath.join(os.getcwd(), p), build.topsrcdir)
+ for p in paths
+ ]
+ return {
+ "env": {
+ "MOZHARNESS_TEST_PATHS": six.ensure_text(
+ json.dumps(resolve_tests_by_suite(paths))
+ ),
+ }
+ }
+
+
+class Environment(TryConfig):
+
+ arguments = [
+ [
+ ["--env"],
+ {
+ "action": "append",
+ "default": None,
+ "help": "Set an environment variable, of the form FOO=BAR. "
+ "Can be passed in multiple times.",
+ },
+ ],
+ ]
+
+ def try_config(self, env, **kwargs):
+ if not env:
+ return
+ return {
+ "env": dict(e.split("=", 1) for e in env),
+ }
+
+
+class RangeAction(Action):
+ def __init__(self, min, max, *args, **kwargs):
+ self.min = min
+ self.max = max
+ kwargs["metavar"] = "[{}-{}]".format(self.min, self.max)
+ super().__init__(*args, **kwargs)
+
+ def __call__(self, parser, namespace, values, option_string=None):
+ name = option_string or self.dest
+ if values < self.min:
+ parser.error("{} can not be less than {}".format(name, self.min))
+ if values > self.max:
+ parser.error("{} can not be more than {}".format(name, self.max))
+ setattr(namespace, self.dest, values)
+
+
+class Rebuild(TryConfig):
+
+ arguments = [
+ [
+ ["--rebuild"],
+ {
+ "action": RangeAction,
+ "min": 2,
+ "max": 20,
+ "default": None,
+ "type": int,
+ "help": "Rebuild all selected tasks the specified number of times.",
+ },
+ ],
+ ]
+
+ def try_config(self, rebuild, **kwargs):
+ if not rebuild:
+ return
+
+ if kwargs.get("full") and rebuild > 3:
+ print(
+ "warning: limiting --rebuild to 3 when using --full. "
+ "Use custom push actions to add more."
+ )
+ rebuild = 3
+
+ return {
+ "rebuild": rebuild,
+ }
+
+
+class Routes(TryConfig):
+ arguments = [
+ [
+ ["--route"],
+ {
+ "action": "append",
+ "dest": "routes",
+ "help": (
+ "Additional route to add to the tasks "
+ "(note: these will not be added to the decision task)"
+ ),
+ },
+ ],
+ ]
+
+ def try_config(self, routes, **kwargs):
+ if routes:
+ return {
+ "routes": routes,
+ }
+
+
+class ChemspillPrio(TryConfig):
+
+ arguments = [
+ [
+ ["--chemspill-prio"],
+ {
+ "action": "store_true",
+ "help": "Run at a higher priority than most try jobs (chemspills only).",
+ },
+ ],
+ ]
+
+ def try_config(self, chemspill_prio, **kwargs):
+ if chemspill_prio:
+ return {"chemspill-prio": {}}
+
+
+class GeckoProfile(TryConfig):
+ arguments = [
+ [
+ ["--gecko-profile"],
+ {
+ "dest": "profile",
+ "action": "store_true",
+ "default": False,
+ "help": "Create and upload a gecko profile during talos/raptor tasks.",
+ },
+ ],
+ [
+ ["--gecko-profile-interval"],
+ {
+ "dest": "gecko_profile_interval",
+ "type": float,
+ "help": "How frequently to take samples (ms)",
+ },
+ ],
+ [
+ ["--gecko-profile-entries"],
+ {
+ "dest": "gecko_profile_entries",
+ "type": int,
+ "help": "How many samples to take with the profiler",
+ },
+ ],
+ [
+ ["--gecko-profile-features"],
+ {
+ "dest": "gecko_profile_features",
+ "type": str,
+ "default": None,
+ "help": "Set the features enabled for the profiler.",
+ },
+ ],
+ [
+ ["--gecko-profile-threads"],
+ {
+ "dest": "gecko_profile_threads",
+ "type": str,
+ "help": "Comma-separated list of threads to sample.",
+ },
+ ],
+ # For backwards compatibility
+ [
+ ["--talos-profile"],
+ {
+ "dest": "profile",
+ "action": "store_true",
+ "default": False,
+ "help": SUPPRESS,
+ },
+ ],
+ # This is added for consistency with the 'syntax' selector
+ [
+ ["--geckoProfile"],
+ {
+ "dest": "profile",
+ "action": "store_true",
+ "default": False,
+ "help": SUPPRESS,
+ },
+ ],
+ ]
+
+ def try_config(
+ self,
+ profile,
+ gecko_profile_interval,
+ gecko_profile_entries,
+ gecko_profile_features,
+ gecko_profile_threads,
+ **kwargs
+ ):
+ if profile or not all(
+ s is None for s in (gecko_profile_features, gecko_profile_threads)
+ ):
+ cfg = {
+ "gecko-profile": True,
+ "gecko-profile-interval": gecko_profile_interval,
+ "gecko-profile-entries": gecko_profile_entries,
+ "gecko-profile-features": gecko_profile_features,
+ "gecko-profile-threads": gecko_profile_threads,
+ }
+ return {key: value for key, value in cfg.items() if value is not None}
+
+
+class Browsertime(TryConfig):
+ arguments = [
+ [
+ ["--browsertime"],
+ {
+ "action": "store_true",
+ "help": "Use browsertime during Raptor tasks.",
+ },
+ ],
+ ]
+
+ def try_config(self, browsertime, **kwargs):
+ if browsertime:
+ return {
+ "browsertime": True,
+ }
+
+
+class DisablePgo(TryConfig):
+
+ arguments = [
+ [
+ ["--disable-pgo"],
+ {
+ "action": "store_true",
+ "help": "Don't run PGO builds",
+ },
+ ],
+ ]
+
+ def try_config(self, disable_pgo, **kwargs):
+ if disable_pgo:
+ return {
+ "disable-pgo": True,
+ }
+
+
+class WorkerOverrides(TryConfig):
+
+ arguments = [
+ [
+ ["--worker-override"],
+ {
+ "action": "append",
+ "dest": "worker_overrides",
+ "help": (
+ "Override the worker pool used for a given taskgraph worker alias. "
+ "The argument should be `<alias>=<worker-pool>`. "
+ "Can be specified multiple times."
+ ),
+ },
+ ],
+ [
+ ["--worker-suffix"],
+ {
+ "action": "append",
+ "dest": "worker_suffixes",
+ "help": (
+ "Override the worker pool used for a given taskgraph worker alias, "
+ "by appending a suffix to the work-pool. "
+ "The argument should be `<alias>=<suffix>`. "
+ "Can be specified multiple times."
+ ),
+ },
+ ],
+ ]
+
+ def try_config(self, worker_overrides, worker_suffixes, **kwargs):
+ from gecko_taskgraph.util.workertypes import get_worker_type
+ from taskgraph.config import load_graph_config
+
+ overrides = {}
+ if worker_overrides:
+ for override in worker_overrides:
+ alias, worker_pool = override.split("=", 1)
+ if alias in overrides:
+ print(
+ "Can't override worker alias {alias} more than once. "
+ "Already set to use {previous}, but also asked to use {new}.".format(
+ alias=alias, previous=overrides[alias], new=worker_pool
+ )
+ )
+ sys.exit(1)
+ overrides[alias] = worker_pool
+
+ if worker_suffixes:
+ root = build.topsrcdir
+ root = os.path.join(root, "taskcluster", "ci")
+ graph_config = load_graph_config(root)
+ for worker_suffix in worker_suffixes:
+ alias, suffix = worker_suffix.split("=", 1)
+ if alias in overrides:
+ print(
+ "Can't override worker alias {alias} more than once. "
+ "Already set to use {previous}, but also asked "
+ "to add suffix {suffix}.".format(
+ alias=alias, previous=overrides[alias], suffix=suffix
+ )
+ )
+ sys.exit(1)
+ provisioner, worker_type = get_worker_type(
+ graph_config, worker_type=alias, parameters={"level": "1"}
+ )
+ overrides[alias] = "{provisioner}/{worker_type}{suffix}".format(
+ provisioner=provisioner, worker_type=worker_type, suffix=suffix
+ )
+
+ if overrides:
+ return {"worker-overrides": overrides}
+
+
+all_task_configs = {
+ "artifact": Artifact,
+ "browsertime": Browsertime,
+ "chemspill-prio": ChemspillPrio,
+ "disable-pgo": DisablePgo,
+ "env": Environment,
+ "gecko-profile": GeckoProfile,
+ "path": Path,
+ "pernosco": Pernosco,
+ "rebuild": Rebuild,
+ "routes": Routes,
+ "worker-overrides": WorkerOverrides,
+}