summaryrefslogtreecommitdiffstats
path: root/taskcluster/gecko_taskgraph/actions/backfill.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--taskcluster/gecko_taskgraph/actions/backfill.py426
1 files changed, 426 insertions, 0 deletions
diff --git a/taskcluster/gecko_taskgraph/actions/backfill.py b/taskcluster/gecko_taskgraph/actions/backfill.py
new file mode 100644
index 0000000000..81f29394d6
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/backfill.py
@@ -0,0 +1,426 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import logging
+import re
+import sys
+from functools import partial
+
+from taskgraph.util.taskcluster import get_task_definition
+
+from .registry import register_callback_action
+from .util import (
+ combine_task_graph_files,
+ create_tasks,
+ fetch_graph_and_labels,
+ get_decision_task_id,
+ get_pushes,
+ get_pushes_from_params_input,
+ trigger_action,
+)
+
+logger = logging.getLogger(__name__)
+SYMBOL_REGEX = re.compile("^(.*)-[a-z0-9]{11}-bk$")
+GROUP_SYMBOL_REGEX = re.compile("^(.*)-bk$")
+
+
+def input_for_support_action(revision, task, times=1, retrigger=True):
+ """Generate input for action to be scheduled.
+
+ Define what label to schedule with 'label'.
+ If it is a test task that uses explicit manifests add that information.
+ """
+ input = {
+ "label": task["metadata"]["name"],
+ "revision": revision,
+ "times": times,
+ # We want the backfilled tasks to share the same symbol as the originating task
+ "symbol": task["extra"]["treeherder"]["symbol"],
+ "retrigger": retrigger,
+ }
+
+ # Support tasks that are using manifest based scheduling
+ if task["payload"].get("env", {}).get("MOZHARNESS_TEST_PATHS"):
+ input["test_manifests"] = json.loads(
+ task["payload"]["env"]["MOZHARNESS_TEST_PATHS"]
+ )
+
+ return input
+
+
+@register_callback_action(
+ title="Backfill",
+ name="backfill",
+ permission="backfill",
+ symbol="Bk",
+ description=("Given a task schedule it on previous pushes in the same project."),
+ order=200,
+ context=[{}], # This will be available for all tasks
+ schema={
+ "type": "object",
+ "properties": {
+ "depth": {
+ "type": "integer",
+ "default": 19,
+ "minimum": 1,
+ "maximum": 25,
+ "title": "Depth",
+ "description": (
+ "The number of previous pushes before the current "
+ "push to attempt to trigger this task on."
+ ),
+ },
+ "inclusive": {
+ "type": "boolean",
+ "default": False,
+ "title": "Inclusive Range",
+ "description": (
+ "If true, the backfill will also retrigger the task "
+ "on the selected push."
+ ),
+ },
+ "times": {
+ "type": "integer",
+ "default": 1,
+ "minimum": 1,
+ "maximum": 10,
+ "title": "Times",
+ "description": (
+ "The number of times to execute each job you are backfilling."
+ ),
+ },
+ "retrigger": {
+ "type": "boolean",
+ "default": True,
+ "title": "Retrigger",
+ "description": (
+ "If False, the task won't retrigger on pushes that have already "
+ "ran it."
+ ),
+ },
+ },
+ "additionalProperties": False,
+ },
+ available=lambda parameters: True,
+)
+def backfill_action(parameters, graph_config, input, task_group_id, task_id):
+ """
+ This action takes a task ID and schedules it on previous pushes (via support action).
+
+ To execute this action locally follow the documentation here:
+ https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally
+ """
+ task = get_task_definition(task_id)
+ pushes = get_pushes_from_params_input(parameters, input)
+ failed = False
+ input_for_action = input_for_support_action(
+ revision=parameters["head_rev"],
+ task=task,
+ times=input.get("times", 1),
+ retrigger=input.get("retrigger", True),
+ )
+
+ for push_id in pushes:
+ try:
+ # The Gecko decision task can sometimes fail on a push and we need to handle
+ # the exception that this call will produce
+ push_decision_task_id = get_decision_task_id(parameters["project"], push_id)
+ except Exception:
+ logger.warning(f"Could not find decision task for push {push_id}")
+ # The decision task may have failed, this is common enough that we
+ # don't want to report an error for it.
+ continue
+
+ try:
+ trigger_action(
+ action_name="backfill-task",
+ # This lets the action know on which push we want to add a new task
+ decision_task_id=push_decision_task_id,
+ input=input_for_action,
+ )
+ except Exception:
+ logger.exception(f"Failed to trigger action for {push_id}")
+ failed = True
+
+ if failed:
+ sys.exit(1)
+
+
+def add_backfill_suffix(regex, symbol, suffix):
+ m = regex.match(symbol)
+ if m is None:
+ symbol += suffix
+ return symbol
+
+
+def backfill_modifier(task, input):
+ if task.label != input["label"]:
+ return task
+
+ logger.debug(f"Modifying test_manifests for {task.label}")
+ times = input.get("times", 1)
+
+ # Set task duplicates based on 'times' value.
+ if times > 1:
+ task.attributes["task_duplicates"] = times
+
+ # If the original task has defined test paths
+ test_manifests = input.get("test_manifests")
+ if test_manifests:
+ revision = input.get("revision")
+
+ task.attributes["test_manifests"] = test_manifests
+ task.task["payload"]["env"]["MOZHARNESS_TEST_PATHS"] = json.dumps(
+ test_manifests
+ )
+ # The name/label might have been modify in new_label, thus, change it here as well
+ task.task["metadata"]["name"] = task.label
+ th_info = task.task["extra"]["treeherder"]
+ # Use a job symbol of the originating task as defined in the backfill action
+ th_info["symbol"] = add_backfill_suffix(
+ SYMBOL_REGEX, th_info["symbol"], f"-{revision[0:11]}-bk"
+ )
+ if th_info.get("groupSymbol"):
+ # Group all backfilled tasks together
+ th_info["groupSymbol"] = add_backfill_suffix(
+ GROUP_SYMBOL_REGEX, th_info["groupSymbol"], "-bk"
+ )
+ task.task["tags"]["action"] = "backfill-task"
+ return task
+
+
+def do_not_modify(task):
+ return task
+
+
+def new_label(label, tasks):
+ """This is to handle the case when a previous push does not contain a specific task label
+ and we try to find a label we can reuse.
+
+ For instance, we try to backfill chunk #3, however, a previous push does not contain such
+ chunk, thus, we try to reuse another task/label.
+ """
+ begining_label, ending = label.rsplit("-", 1)
+ if ending.isdigit():
+ # We assume that the taskgraph has chunk #1 OR unnumbered chunk and we hijack it
+ if begining_label in tasks:
+ return begining_label
+ if begining_label + "-1" in tasks:
+ return begining_label + "-1"
+ raise Exception(f"New label ({label}) was not found in the task-graph")
+ else:
+ raise Exception(f"{label} was not found in the task-graph")
+
+
+@register_callback_action(
+ name="backfill-task",
+ title="Backfill task on a push.",
+ permission="backfill",
+ symbol="backfill-task",
+ description="This action is normally scheduled by the backfill action. "
+ "The intent is to schedule a task on previous pushes.",
+ order=500,
+ context=[],
+ schema={
+ "type": "object",
+ "properties": {
+ "label": {"type": "string", "description": "A task label"},
+ "revision": {
+ "type": "string",
+ "description": "Revision of the original push from where we backfill.",
+ },
+ "symbol": {
+ "type": "string",
+ "description": "Symbol to be used by the scheduled task.",
+ },
+ "test_manifests": {
+ "type": "array",
+ "default": [],
+ "description": "An array of test manifest paths",
+ "items": {"type": "string"},
+ },
+ "times": {
+ "type": "integer",
+ "default": 1,
+ "minimum": 1,
+ "maximum": 10,
+ "title": "Times",
+ "description": (
+ "The number of times to execute each job " "you are backfilling."
+ ),
+ },
+ "retrigger": {
+ "type": "boolean",
+ "default": True,
+ "title": "Retrigger",
+ "description": (
+ "If False, the task won't retrigger on pushes that have already "
+ "ran it."
+ ),
+ },
+ },
+ },
+)
+def add_task_with_original_manifests(
+ parameters, graph_config, input, task_group_id, task_id
+):
+ """
+ This action is normally scheduled by the backfill action. The intent is to schedule a test
+ task with the test manifests from the original task (if available).
+
+ The push in which we want to schedule a new task is defined by the parameters object.
+
+ To execute this action locally follow the documentation here:
+ https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally
+ """
+ # This step takes a lot of time when executed locally
+ logger.info("Retreving the full task graph and labels.")
+ decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
+ parameters, graph_config
+ )
+
+ label = input.get("label")
+ if not input.get("retrigger") and label in label_to_taskid:
+ logger.info(
+ f"Skipping push with decision task ID {decision_task_id} as it already has this test."
+ )
+ return
+
+ if label not in full_task_graph.tasks:
+ label = new_label(label, full_task_graph.tasks)
+
+ to_run = [label]
+
+ logger.info("Creating tasks...")
+ create_tasks(
+ graph_config,
+ to_run,
+ full_task_graph,
+ label_to_taskid,
+ parameters,
+ decision_task_id,
+ suffix="0",
+ modifier=partial(backfill_modifier, input=input),
+ )
+
+ # TODO Implement a way to write out artifacts without assuming there's
+ # multiple sets of them so we can stop passing in "suffix".
+ combine_task_graph_files(["0"])
+
+
+@register_callback_action(
+ title="Backfill all browsertime",
+ name="backfill-all-browsertime",
+ permission="backfill",
+ symbol="baB",
+ description=(
+ "Schedule all browsertime tests for the current and previous push in the same project."
+ ),
+ order=800,
+ context=[], # This will be available for all tasks
+ available=lambda parameters: True,
+)
+def backfill_all_browsertime(parameters, graph_config, input, task_group_id, task_id):
+ """
+ This action takes a revision and schedules it on previous pushes (via support action).
+
+ To execute this action locally follow the documentation here:
+ https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally
+ """
+ pushes = get_pushes(
+ project=parameters["head_repository"],
+ end_id=int(parameters["pushlog_id"]),
+ depth=2,
+ )
+
+ for push_id in pushes:
+ try:
+ # The Gecko decision task can sometimes fail on a push and we need to handle
+ # the exception that this call will produce
+ push_decision_task_id = get_decision_task_id(parameters["project"], push_id)
+ except Exception:
+ logger.warning(f"Could not find decision task for push {push_id}")
+ # The decision task may have failed, this is common enough that we
+ # don't want to report an error for it.
+ continue
+
+ try:
+ trigger_action(
+ action_name="add-all-browsertime",
+ # This lets the action know on which push we want to add a new task
+ decision_task_id=push_decision_task_id,
+ )
+ except Exception:
+ logger.exception(f"Failed to trigger action for {push_id}")
+ sys.exit(1)
+
+
+def filter_raptor_jobs(full_task_graph, label_to_taskid):
+ to_run = []
+ for label, entry in full_task_graph.tasks.items():
+ if entry.kind != "test":
+ continue
+ if entry.task.get("extra", {}).get("suite", "") != "raptor":
+ continue
+ if "browsertime" not in entry.attributes.get("raptor_try_name", ""):
+ continue
+ if not entry.attributes.get("test_platform", "").endswith("shippable-qr/opt"):
+ continue
+ if "android" in entry.attributes.get("test_platform", ""):
+ # Bug 1786254 - The backfill bot is scheduling too many tests atm
+ continue
+ exceptions = ("live", "profiling", "youtube-playback")
+ if any(e in entry.attributes.get("raptor_try_name", "") for e in exceptions):
+ continue
+ if "firefox" in entry.attributes.get(
+ "raptor_try_name", ""
+ ) and entry.attributes.get("test_platform", "").endswith("64-shippable-qr/opt"):
+ # add the browsertime test
+ if label not in label_to_taskid:
+ to_run.append(label)
+ if "geckoview" in entry.attributes.get("raptor_try_name", ""):
+ # add the pageload test
+ if label not in label_to_taskid:
+ to_run.append(label)
+ return to_run
+
+
+@register_callback_action(
+ name="add-all-browsertime",
+ title="Add All Browsertime Tests.",
+ permission="backfill",
+ symbol="aaB",
+ description="This action is normally scheduled by the backfill-all-browsertime action. "
+ "The intent is to schedule all browsertime tests on a specific pushe.",
+ order=900,
+ context=[],
+)
+def add_all_browsertime(parameters, graph_config, input, task_group_id, task_id):
+ """
+ This action is normally scheduled by the backfill-all-browsertime action. The intent is to
+ trigger all browsertime tasks for the current revision.
+
+ The push in which we want to schedule a new task is defined by the parameters object.
+
+ To execute this action locally follow the documentation here:
+ https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally
+ """
+ logger.info("Retreving the full task graph and labels.")
+ decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
+ parameters, graph_config
+ )
+
+ to_run = filter_raptor_jobs(full_task_graph, label_to_taskid)
+
+ create_tasks(
+ graph_config,
+ to_run,
+ full_task_graph,
+ label_to_taskid,
+ parameters,
+ decision_task_id,
+ )
+ logger.info(f"Scheduled {len(to_run)} raptor tasks (time 1)")