Adding upstream version 124.0.1.upstream/124.0.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

22 files changed, 3571 insertions, 0 deletions

diff --git a/taskcluster/gecko_taskgraph/actions/__init__.py b/taskcluster/gecko_taskgraph/actions/__init__.py
new file mode 100644
index 0000000000..590a957282
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/__init__.py
@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from .registry import (
+    register_callback_action,
+    render_actions_json,
+    trigger_action_callback,
+)
+
+__all__ = [
+    "register_callback_action",
+    "render_actions_json",
+    "trigger_action_callback",
+]
diff --git a/taskcluster/gecko_taskgraph/actions/add_new_jobs.py b/taskcluster/gecko_taskgraph/actions/add_new_jobs.py
new file mode 100644
index 0000000000..05ca5cb7d4
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/add_new_jobs.py
@@ -0,0 +1,59 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from .registry import register_callback_action
+from .util import combine_task_graph_files, create_tasks, fetch_graph_and_labels
+
+
+@register_callback_action(
+    name="add-new-jobs",
+    title="Add new jobs",
+    symbol="add-new",
+    description="Add new jobs using task labels.",
+    order=100,
+    context=[],
+    schema={
+        "type": "object",
+        "properties": {
+            "tasks": {
+                "type": "array",
+                "description": "An array of task labels",
+                "items": {"type": "string"},
+            },
+            "times": {
+                "type": "integer",
+                "default": 1,
+                "minimum": 1,
+                "maximum": 100,
+                "title": "Times",
+                "description": "How many times to run each task.",
+            },
+        },
+    },
+)
+def add_new_jobs_action(parameters, graph_config, input, task_group_id, task_id):
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+
+    to_run = []
+    for elem in input["tasks"]:
+        if elem in full_task_graph.tasks:
+            to_run.append(elem)
+        else:
+            raise Exception(f"{elem} was not found in the task-graph")
+
+    times = input.get("times", 1)
+    for i in range(times):
+        create_tasks(
+            graph_config,
+            to_run,
+            full_task_graph,
+            label_to_taskid,
+            parameters,
+            decision_task_id,
+            i,
+        )
+    combine_task_graph_files(list(range(times)))
diff --git a/taskcluster/gecko_taskgraph/actions/add_talos.py b/taskcluster/gecko_taskgraph/actions/add_talos.py
new file mode 100644
index 0000000000..ed3980713b
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/add_talos.py
@@ -0,0 +1,59 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+
+from ..target_tasks import standard_filter
+from .registry import register_callback_action
+from .util import create_tasks, fetch_graph_and_labels
+
+logger = logging.getLogger(__name__)
+
+
+@register_callback_action(
+    name="run-all-talos",
+    title="Run All Talos Tests",
+    symbol="raT",
+    description="Add all Talos tasks to a push.",
+    order=150,
+    context=[],
+    schema={
+        "type": "object",
+        "properties": {
+            "times": {
+                "type": "integer",
+                "default": 1,
+                "minimum": 1,
+                "maximum": 6,
+                "title": "Times",
+                "description": "How many times to run each task.",
+            }
+        },
+        "additionalProperties": False,
+    },
+)
+def add_all_talos(parameters, graph_config, input, task_group_id, task_id):
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+
+    times = input.get("times", 1)
+    for i in range(times):
+        to_run = [
+            label
+            for label, entry in full_task_graph.tasks.items()
+            if "talos_try_name" in entry.attributes
+            and standard_filter(entry, parameters)
+        ]
+
+        create_tasks(
+            graph_config,
+            to_run,
+            full_task_graph,
+            label_to_taskid,
+            parameters,
+            decision_task_id,
+        )
+        logger.info(f"Scheduled {len(to_run)} talos tasks (time {i + 1}/{times})")
diff --git a/taskcluster/gecko_taskgraph/actions/backfill.py b/taskcluster/gecko_taskgraph/actions/backfill.py
new file mode 100644
index 0000000000..b5ee66b54c
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/backfill.py
@@ -0,0 +1,441 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import logging
+import re
+import sys
+from functools import partial
+
+from taskgraph.util.taskcluster import get_task_definition
+
+from .registry import register_callback_action
+from .util import (
+    combine_task_graph_files,
+    create_tasks,
+    fetch_graph_and_labels,
+    get_decision_task_id,
+    get_pushes,
+    get_pushes_from_params_input,
+    trigger_action,
+)
+
+logger = logging.getLogger(__name__)
+SYMBOL_REGEX = re.compile("^(.*)-[a-z0-9]{11}-bk$")
+GROUP_SYMBOL_REGEX = re.compile("^(.*)-bk$")
+
+
+def input_for_support_action(revision, task, times=1, retrigger=True):
+    """Generate input for action to be scheduled.
+
+    Define what label to schedule with 'label'.
+    If it is a test task that uses explicit manifests add that information.
+    """
+    input = {
+        "label": task["metadata"]["name"],
+        "revision": revision,
+        "times": times,
+        # We want the backfilled tasks to share the same symbol as the originating task
+        "symbol": task["extra"]["treeherder"]["symbol"],
+        "retrigger": retrigger,
+    }
+
+    # Support tasks that are using manifest based scheduling
+    if task["payload"].get("env", {}).get("MOZHARNESS_TEST_PATHS"):
+        input["test_manifests"] = json.loads(
+            task["payload"]["env"]["MOZHARNESS_TEST_PATHS"]
+        )
+
+    return input
+
+
+@register_callback_action(
+    title="Backfill",
+    name="backfill",
+    permission="backfill",
+    symbol="Bk",
+    description=("Given a task schedule it on previous pushes in the same project."),
+    order=200,
+    context=[{}],  # This will be available for all tasks
+    schema={
+        "type": "object",
+        "properties": {
+            "depth": {
+                "type": "integer",
+                "default": 19,
+                "minimum": 1,
+                "maximum": 25,
+                "title": "Depth",
+                "description": (
+                    "The number of previous pushes before the current "
+                    "push to attempt to trigger this task on."
+                ),
+            },
+            "inclusive": {
+                "type": "boolean",
+                "default": False,
+                "title": "Inclusive Range",
+                "description": (
+                    "If true, the backfill will also retrigger the task "
+                    "on the selected push."
+                ),
+            },
+            "times": {
+                "type": "integer",
+                "default": 1,
+                "minimum": 1,
+                "maximum": 10,
+                "title": "Times",
+                "description": (
+                    "The number of times to execute each job you are backfilling."
+                ),
+            },
+            "retrigger": {
+                "type": "boolean",
+                "default": True,
+                "title": "Retrigger",
+                "description": (
+                    "If False, the task won't retrigger on pushes that have already "
+                    "ran it."
+                ),
+            },
+        },
+        "additionalProperties": False,
+    },
+    available=lambda parameters: True,
+)
+def backfill_action(parameters, graph_config, input, task_group_id, task_id):
+    """
+    This action takes a task ID and schedules it on previous pushes (via support action).
+
+    To execute this action locally follow the documentation here:
+    https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally
+    """
+    task = get_task_definition(task_id)
+    pushes = get_pushes_from_params_input(parameters, input)
+    failed = False
+    input_for_action = input_for_support_action(
+        revision=parameters["head_rev"],
+        task=task,
+        times=input.get("times", 1),
+        retrigger=input.get("retrigger", True),
+    )
+
+    for push_id in pushes:
+        try:
+            # The Gecko decision task can sometimes fail on a push and we need to handle
+            # the exception that this call will produce
+            push_decision_task_id = get_decision_task_id(parameters["project"], push_id)
+        except Exception:
+            logger.warning(f"Could not find decision task for push {push_id}")
+            # The decision task may have failed, this is common enough that we
+            # don't want to report an error for it.
+            continue
+
+        try:
+            trigger_action(
+                action_name="backfill-task",
+                # This lets the action know on which push we want to add a new task
+                decision_task_id=push_decision_task_id,
+                input=input_for_action,
+            )
+        except Exception:
+            logger.exception(f"Failed to trigger action for {push_id}")
+            failed = True
+
+    if failed:
+        sys.exit(1)
+
+
+def add_backfill_suffix(regex, symbol, suffix):
+    m = regex.match(symbol)
+    if m is None:
+        symbol += suffix
+    return symbol
+
+
+def backfill_modifier(task, input):
+    if task.label != input["label"]:
+        return task
+
+    logger.debug(f"Modifying test_manifests for {task.label}")
+    times = input.get("times", 1)
+
+    # Set task duplicates based on 'times' value.
+    if times > 1:
+        task.attributes["task_duplicates"] = times
+
+    # If the original task has defined test paths
+    test_manifests = input.get("test_manifests")
+    if test_manifests:
+        revision = input.get("revision")
+
+        task.attributes["test_manifests"] = test_manifests
+        task.task["payload"]["env"]["MOZHARNESS_TEST_PATHS"] = json.dumps(
+            test_manifests
+        )
+        # The name/label might have been modify in new_label, thus, change it here as well
+        task.task["metadata"]["name"] = task.label
+        th_info = task.task["extra"]["treeherder"]
+        # Use a job symbol of the originating task as defined in the backfill action
+        th_info["symbol"] = add_backfill_suffix(
+            SYMBOL_REGEX, th_info["symbol"], f"-{revision[0:11]}-bk"
+        )
+        if th_info.get("groupSymbol"):
+            # Group all backfilled tasks together
+            th_info["groupSymbol"] = add_backfill_suffix(
+                GROUP_SYMBOL_REGEX, th_info["groupSymbol"], "-bk"
+            )
+        task.task["tags"]["action"] = "backfill-task"
+    return task
+
+
+def do_not_modify(task):
+    return task
+
+
+def new_label(label, tasks):
+    """This is to handle the case when a previous push does not contain a specific task label
+    and we try to find a label we can reuse.
+
+    For instance, we try to backfill chunk #3, however, a previous push does not contain such
+    chunk, thus, we try to reuse another task/label.
+    """
+    logger.info(f"Extracting new label for {label}")
+
+    if "-" not in label:
+        raise Exception(
+            f"Expected '-' was not found in label {label}, cannot extract new label."
+        )
+
+    begining_label, ending = label.rsplit("-", 1)
+
+    if ending.isdigit():
+        # We assume that the taskgraph has chunk #1 OR unnumbered chunk and we hijack it
+        if begining_label in tasks:
+            return begining_label
+        if begining_label + "-1" in tasks:
+            return begining_label + "-1"
+        raise Exception(f"New label ({label}) was not found in the task-graph")
+    else:
+        raise Exception(f"{label} was not found in the task-graph")
+
+
+@register_callback_action(
+    name="backfill-task",
+    title="Backfill task on a push.",
+    permission="backfill",
+    symbol="backfill-task",
+    description="This action is normally scheduled by the backfill action. "
+    "The intent is to schedule a task on previous pushes.",
+    order=500,
+    context=[],
+    schema={
+        "type": "object",
+        "properties": {
+            "label": {"type": "string", "description": "A task label"},
+            "revision": {
+                "type": "string",
+                "description": "Revision of the original push from where we backfill.",
+            },
+            "symbol": {
+                "type": "string",
+                "description": "Symbol to be used by the scheduled task.",
+            },
+            "test_manifests": {
+                "type": "array",
+                "default": [],
+                "description": "An array of test manifest paths",
+                "items": {"type": "string"},
+            },
+            "times": {
+                "type": "integer",
+                "default": 1,
+                "minimum": 1,
+                "maximum": 10,
+                "title": "Times",
+                "description": (
+                    "The number of times to execute each job " "you are backfilling."
+                ),
+            },
+            "retrigger": {
+                "type": "boolean",
+                "default": True,
+                "title": "Retrigger",
+                "description": (
+                    "If False, the task won't retrigger on pushes that have already "
+                    "ran it."
+                ),
+            },
+        },
+    },
+)
+def add_task_with_original_manifests(
+    parameters, graph_config, input, task_group_id, task_id
+):
+    """
+    This action is normally scheduled by the backfill action. The intent is to schedule a test
+    task with the test manifests from the original task (if available).
+
+    The push in which we want to schedule a new task is defined by the parameters object.
+
+    To execute this action locally follow the documentation here:
+    https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally
+    """
+    # This step takes a lot of time when executed locally
+    logger.info("Retreving the full task graph and labels.")
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+
+    label = input.get("label")
+    if not input.get("retrigger") and label in label_to_taskid:
+        logger.info(
+            f"Skipping push with decision task ID {decision_task_id} as it already has this test."
+        )
+        return
+
+    if label not in full_task_graph.tasks:
+        label = new_label(label, full_task_graph.tasks)
+
+    to_run = [label]
+
+    logger.info("Creating tasks...")
+    create_tasks(
+        graph_config,
+        to_run,
+        full_task_graph,
+        label_to_taskid,
+        parameters,
+        decision_task_id,
+        suffix="0",
+        modifier=partial(backfill_modifier, input=input),
+    )
+
+    # TODO Implement a way to write out artifacts without assuming there's
+    # multiple sets of them so we can stop passing in "suffix".
+    combine_task_graph_files(["0"])
+
+
+@register_callback_action(
+    title="Backfill all browsertime",
+    name="backfill-all-browsertime",
+    permission="backfill",
+    symbol="baB",
+    description=(
+        "Schedule all browsertime tests for the current and previous push in the same project."
+    ),
+    order=800,
+    context=[],  # This will be available for all tasks
+    available=lambda parameters: True,
+)
+def backfill_all_browsertime(parameters, graph_config, input, task_group_id, task_id):
+    """
+    This action takes a revision and schedules it on previous pushes (via support action).
+
+    To execute this action locally follow the documentation here:
+    https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally
+    """
+    pushes = get_pushes(
+        project=parameters["head_repository"],
+        end_id=int(parameters["pushlog_id"]),
+        depth=2,
+    )
+
+    for push_id in pushes:
+        try:
+            # The Gecko decision task can sometimes fail on a push and we need to handle
+            # the exception that this call will produce
+            push_decision_task_id = get_decision_task_id(parameters["project"], push_id)
+        except Exception:
+            logger.warning(f"Could not find decision task for push {push_id}")
+            # The decision task may have failed, this is common enough that we
+            # don't want to report an error for it.
+            continue
+
+        try:
+            trigger_action(
+                action_name="add-all-browsertime",
+                # This lets the action know on which push we want to add a new task
+                decision_task_id=push_decision_task_id,
+            )
+        except Exception:
+            logger.exception(f"Failed to trigger action for {push_id}")
+            sys.exit(1)
+
+
+def filter_raptor_jobs(full_task_graph, label_to_taskid, project):
+    # Late import to prevent impacting other backfill action tasks
+    from ..util.attributes import match_run_on_projects
+
+    to_run = []
+    for label, entry in full_task_graph.tasks.items():
+        if entry.kind != "test":
+            continue
+        if entry.task.get("extra", {}).get("suite", "") != "raptor":
+            continue
+        if not match_run_on_projects(
+            project, entry.attributes.get("run_on_projects", [])
+        ):
+            continue
+        if "browsertime" not in entry.attributes.get("raptor_try_name", ""):
+            continue
+        if not entry.attributes.get("test_platform", "").endswith("shippable-qr/opt"):
+            continue
+        if "android" in entry.attributes.get("test_platform", ""):
+            # Bug 1786254 - The backfill bot is scheduling too many tests atm
+            continue
+        exceptions = ("live", "profiling", "youtube-playback")
+        if any(e in entry.attributes.get("raptor_try_name", "") for e in exceptions):
+            continue
+        if "firefox" in entry.attributes.get(
+            "raptor_try_name", ""
+        ) and entry.attributes.get("test_platform", "").endswith("64-shippable-qr/opt"):
+            # add the browsertime test
+            if label not in label_to_taskid:
+                to_run.append(label)
+        if "geckoview" in entry.attributes.get("raptor_try_name", ""):
+            # add the pageload test
+            if label not in label_to_taskid:
+                to_run.append(label)
+    return to_run
+
+
+@register_callback_action(
+    name="add-all-browsertime",
+    title="Add All Browsertime Tests.",
+    permission="backfill",
+    symbol="aaB",
+    description="This action is normally scheduled by the backfill-all-browsertime action. "
+    "The intent is to schedule all browsertime tests on a specific pushe.",
+    order=900,
+    context=[],
+)
+def add_all_browsertime(parameters, graph_config, input, task_group_id, task_id):
+    """
+    This action is normally scheduled by the backfill-all-browsertime action. The intent is to
+    trigger all browsertime tasks for the current revision.
+
+    The push in which we want to schedule a new task is defined by the parameters object.
+
+    To execute this action locally follow the documentation here:
+    https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally
+    """
+    logger.info("Retreving the full task graph and labels.")
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+
+    to_run = filter_raptor_jobs(full_task_graph, label_to_taskid, parameters["project"])
+
+    create_tasks(
+        graph_config,
+        to_run,
+        full_task_graph,
+        label_to_taskid,
+        parameters,
+        decision_task_id,
+    )
+    logger.info(f"Scheduled {len(to_run)} raptor tasks (time 1)")
diff --git a/taskcluster/gecko_taskgraph/actions/cancel.py b/taskcluster/gecko_taskgraph/actions/cancel.py
new file mode 100644
index 0000000000..d895781395
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/cancel.py
@@ -0,0 +1,36 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+
+import requests
+from taskgraph.util.taskcluster import cancel_task
+
+from .registry import register_callback_action
+
+logger = logging.getLogger(__name__)
+
+
+@register_callback_action(
+    title="Cancel Task",
+    name="cancel",
+    symbol="cx",
+    description=("Cancel the given task"),
+    order=350,
+    context=[{}],
+)
+def cancel_action(parameters, graph_config, input, task_group_id, task_id):
+    # Note that this is limited by the scopes afforded to generic actions to
+    # only cancel tasks with the level-specific schedulerId.
+    try:
+        cancel_task(task_id, use_proxy=True)
+    except requests.HTTPError as e:
+        if e.response.status_code == 409:
+            # A 409 response indicates that this task is past its deadline.  It
+            # cannot be cancelled at this time, but it's also not running
+            # anymore, so we can ignore this error.
+            logger.info(f"Task {task_id} is past its deadline and cannot be cancelled.")
+            return
+        raise
diff --git a/taskcluster/gecko_taskgraph/actions/cancel_all.py b/taskcluster/gecko_taskgraph/actions/cancel_all.py
new file mode 100644
index 0000000000..d74b83b7d8
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/cancel_all.py
@@ -0,0 +1,60 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import concurrent.futures as futures
+import logging
+import os
+
+import requests
+from taskgraph.util.taskcluster import CONCURRENCY, cancel_task
+
+from gecko_taskgraph.util.taskcluster import list_task_group_incomplete_task_ids
+
+from .registry import register_callback_action
+
+logger = logging.getLogger(__name__)
+
+
+@register_callback_action(
+    title="Cancel All",
+    name="cancel-all",
+    symbol="cAll",
+    description=(
+        "Cancel all running and pending tasks created by the decision task "
+        "this action task is associated with."
+    ),
+    order=400,
+    context=[],
+)
+def cancel_all_action(parameters, graph_config, input, task_group_id, task_id):
+    def do_cancel_task(task_id):
+        logger.info(f"Cancelling task {task_id}")
+        try:
+            cancel_task(task_id, use_proxy=True)
+        except requests.HTTPError as e:
+            if e.response.status_code == 409:
+                # A 409 response indicates that this task is past its deadline.  It
+                # cannot be cancelled at this time, but it's also not running
+                # anymore, so we can ignore this error.
+                logger.info(
+                    "Task {} is past its deadline and cannot be cancelled.".format(
+                        task_id
+                    )
+                )
+                return
+            raise
+
+    own_task_id = os.environ.get("TASK_ID", "")
+    to_cancel = [
+        t
+        for t in list_task_group_incomplete_task_ids(task_group_id)
+        if t != own_task_id
+    ]
+
+    logger.info(f"Cancelling {len(to_cancel)} tasks")
+    with futures.ThreadPoolExecutor(CONCURRENCY) as e:
+        cancel_futs = [e.submit(do_cancel_task, t) for t in to_cancel]
+        for f in futures.as_completed(cancel_futs):
+            f.result()
diff --git a/taskcluster/gecko_taskgraph/actions/confirm_failure.py b/taskcluster/gecko_taskgraph/actions/confirm_failure.py
new file mode 100644
index 0000000000..84dbda2997
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/confirm_failure.py
@@ -0,0 +1,268 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import logging
+from functools import partial
+
+from taskgraph.util.taskcluster import get_artifact, get_task_definition, list_artifacts
+
+from .registry import register_callback_action
+from .retrigger import retrigger_action
+from .util import add_args_to_command, create_tasks, fetch_graph_and_labels
+
+logger = logging.getLogger(__name__)
+
+
+def get_failures(task_id, task_definition):
+    """Returns a dict containing properties containing a list of
+    directories containing test failures and a separate list of
+    individual test failures from the errorsummary.log artifact for
+    the task.
+
+    Find test path to pass to the task in
+    MOZHARNESS_TEST_PATHS.  If no appropriate test path can be
+    determined, nothing is returned.
+    """
+
+    def fix_wpt_name(test):
+        # TODO: find other cases to handle
+        if ".any." in test:
+            test = "%s.any.js" % test.split(".any.")[0]
+        if ".window.html" in test:
+            test = test.replace(".window.html", ".window.js")
+
+        if test.startswith("/_mozilla"):
+            test = "testing/web-platform/mozilla/tests" + test[len("_mozilla") :]
+        else:
+            test = "testing/web-platform/tests/" + test.strip("/")
+        # some wpt tests have params, those are not supported
+        test = test.split("?")[0]
+
+        return test
+
+    # collect dirs that don't have a specific manifest
+    dirs = []
+    tests = []
+
+    artifacts = list_artifacts(task_id)
+    for artifact in artifacts:
+        if "name" not in artifact or not artifact["name"].endswith("errorsummary.log"):
+            continue
+
+        stream = get_artifact(task_id, artifact["name"])
+        if not stream:
+            continue
+
+        # We handle the stream as raw bytes because it may contain invalid
+        # UTF-8 characters in portions other than those containing the error
+        # messages we're looking for.
+        for line in stream.read().split(b"\n"):
+            if not line.strip():
+                continue
+
+            l = json.loads(line)
+            if "group_results" in l.keys() and l["status"] != "OK":
+                dirs.append(l["group_results"].group())
+
+            elif "test" in l.keys():
+                if not l["test"]:
+                    print("Warning: no testname in errorsummary line: %s" % l)
+                    continue
+
+                test_path = l["test"].split(" ")[0]
+                found_path = False
+
+                # tests with url params (wpt), will get confused here
+                if "?" not in test_path:
+                    test_path = test_path.split(":")[-1]
+
+                # edge case where a crash on shutdown has a "test" name == group name
+                if (
+                    test_path.endswith(".toml")
+                    or test_path.endswith(".ini")
+                    or test_path.endswith(".list")
+                ):
+                    # TODO: consider running just the manifest
+                    continue
+
+                # edge cases with missing test names
+                if (
+                    test_path is None
+                    or test_path == "None"
+                    or "SimpleTest" in test_path
+                ):
+                    continue
+
+                if "signature" in l.keys():
+                    # dealing with a crash
+                    found_path = True
+                    if "web-platform" in task_definition["extra"]["suite"]:
+                        test_path = fix_wpt_name(test_path)
+                else:
+                    if "status" not in l and "expected" not in l:
+                        continue
+
+                    if l["status"] != l["expected"]:
+                        if l["status"] not in l.get("known_intermittent", []):
+                            found_path = True
+                            if "web-platform" in task_definition["extra"]["suite"]:
+                                test_path = fix_wpt_name(test_path)
+
+                if found_path and test_path:
+                    fpath = test_path.replace("\\", "/")
+                    tval = {"path": fpath, "group": l["group"]}
+                    # only store one failure per test
+                    if not [t for t in tests if t["path"] == fpath]:
+                        tests.append(tval)
+
+                # only run the failing test not both test + dir
+                if l["group"] in dirs:
+                    dirs.remove(l["group"])
+
+            # TODO: 10 is too much; how to get only NEW failures?
+            if len(tests) > 10:
+                break
+
+    dirs = [{"path": "", "group": d} for d in list(set(dirs))]
+    return {"dirs": dirs, "tests": tests}
+
+
+def get_repeat_args(task_definition, failure_group):
+    task_name = task_definition["metadata"]["name"]
+    repeatable_task = False
+    if (
+        "crashtest" in task_name
+        or "mochitest" in task_name
+        or "reftest" in task_name
+        or "xpcshell" in task_name
+        or "web-platform" in task_name
+        and "jsreftest" not in task_name
+    ):
+        repeatable_task = True
+
+    repeat_args = ""
+    if not repeatable_task:
+        return repeat_args
+
+    if failure_group == "dirs":
+        # execute 3 total loops
+        repeat_args = ["--repeat=2"] if repeatable_task else []
+    elif failure_group == "tests":
+        # execute 5 total loops
+        repeat_args = ["--repeat=4"] if repeatable_task else []
+
+    return repeat_args
+
+
+def confirm_modifier(task, input):
+    if task.label != input["label"]:
+        return task
+
+    logger.debug(f"Modifying paths for {task.label}")
+
+    # If the original task has defined test paths
+    suite = input.get("suite")
+    test_path = input.get("test_path")
+    test_group = input.get("test_group")
+    if test_path or test_group:
+        repeat_args = input.get("repeat_args")
+
+        if repeat_args:
+            task.task["payload"]["command"] = add_args_to_command(
+                task.task["payload"]["command"], extra_args=repeat_args
+            )
+
+        # TODO: do we need this attribute?
+        task.attributes["test_path"] = test_path
+
+        task.task["payload"]["env"]["MOZHARNESS_TEST_PATHS"] = json.dumps(
+            {suite: [test_group]}, sort_keys=True
+        )
+        task.task["payload"]["env"]["MOZHARNESS_CONFIRM_PATHS"] = json.dumps(
+            {suite: [test_path]}, sort_keys=True
+        )
+        task.task["payload"]["env"]["MOZLOG_DUMP_ALL_TESTS"] = "1"
+
+        task.task["metadata"]["name"] = task.label
+        task.task["tags"]["action"] = "confirm-failure"
+    return task
+
+
+@register_callback_action(
+    name="confirm-failures",
+    title="Confirm failures in job",
+    symbol="cf",
+    description="Re-run Tests for original manifest, directories or tests for failing tests.",
+    order=150,
+    context=[{"kind": "test"}],
+    schema={
+        "type": "object",
+        "properties": {
+            "label": {"type": "string", "description": "A task label"},
+            "suite": {"type": "string", "description": "Test suite"},
+            "test_path": {"type": "string", "description": "A full path to test"},
+            "test_group": {
+                "type": "string",
+                "description": "A full path to group name",
+            },
+            "repeat_args": {
+                "type": "string",
+                "description": "args to pass to test harness",
+            },
+        },
+        "additionalProperties": False,
+    },
+)
+def confirm_failures(parameters, graph_config, input, task_group_id, task_id):
+    task_definition = get_task_definition(task_id)
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+
+    # create -cf label; ideally make this a common function
+    task_definition["metadata"]["name"].split("-")
+    cfname = "%s-cf" % task_definition["metadata"]["name"]
+
+    if cfname not in full_task_graph.tasks:
+        raise Exception(f"{cfname} was not found in the task-graph")
+
+    to_run = [cfname]
+
+    suite = task_definition["extra"]["suite"]
+    if "-coverage" in suite:
+        suite = suite[: suite.index("-coverage")]
+    if "-qr" in suite:
+        suite = suite[: suite.index("-qr")]
+    failures = get_failures(task_id, task_definition)
+
+    if failures["dirs"] == [] and failures["tests"] == []:
+        logger.info("need to retrigger task as no specific test failures found")
+        retrigger_action(parameters, graph_config, input, decision_task_id, task_id)
+        return
+
+    # for each unique failure, create a new confirm failure job
+    for failure_group in failures:
+        for failure_path in failures[failure_group]:
+            repeat_args = get_repeat_args(task_definition, failure_group)
+
+            input = {
+                "label": cfname,
+                "suite": suite,
+                "test_path": failure_path["path"],
+                "test_group": failure_path["group"],
+                "repeat_args": repeat_args,
+            }
+
+            logger.info("confirm_failures: %s" % failures)
+            create_tasks(
+                graph_config,
+                to_run,
+                full_task_graph,
+                label_to_taskid,
+                parameters,
+                decision_task_id,
+                modifier=partial(confirm_modifier, input=input),
+            )
diff --git a/taskcluster/gecko_taskgraph/actions/create_interactive.py b/taskcluster/gecko_taskgraph/actions/create_interactive.py
new file mode 100644
index 0000000000..27ec3e78df
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/create_interactive.py
@@ -0,0 +1,188 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+import os
+import re
+
+import taskcluster_urls
+from taskgraph.util.taskcluster import get_root_url, get_task_definition
+
+from gecko_taskgraph.actions.registry import register_callback_action
+from gecko_taskgraph.actions.util import create_tasks, fetch_graph_and_labels
+
+logger = logging.getLogger(__name__)
+
+EMAIL_SUBJECT = "Your Interactive Task for {label}"
+EMAIL_CONTENT = """\
+As you requested, Firefox CI has created an interactive task to run {label}
+on revision {revision} in {repo}. Click the button below to connect to the
+task. You may need to wait for it to begin running.
+"""
+
+###
+# Security Concerns
+#
+# An "interactive task" is, quite literally, shell access to a worker. That
+# is limited by being in a Docker container, but we assume that Docker has
+# bugs so we do not want to rely on container isolation exclusively.
+#
+# Interactive tasks should never be allowed on hosts that build binaries
+# leading to a release -- level 3 builders.
+#
+# Users must not be allowed to create interactive tasks for tasks above
+# their own level.
+#
+# Interactive tasks must not have any routes that might make them appear
+# in the index to be used by other production tasks.
+#
+# Interactive tasks should not be able to write to any docker-worker caches.
+
+SCOPE_WHITELIST = [
+    # these are not actually secrets, and just about everything needs them
+    re.compile(r"^secrets:get:project/taskcluster/gecko/(hgfingerprint|hgmointernal)$"),
+    # public downloads are OK
+    re.compile(r"^docker-worker:relengapi-proxy:tooltool.download.public$"),
+    re.compile(r"^project:releng:services/tooltool/api/download/public$"),
+    # internal downloads are OK
+    re.compile(r"^docker-worker:relengapi-proxy:tooltool.download.internal$"),
+    re.compile(r"^project:releng:services/tooltool/api/download/internal$"),
+    # private toolchain artifacts from tasks
+    re.compile(r"^queue:get-artifact:project/gecko/.*$"),
+    # level-appropriate secrets are generally necessary to run a task; these
+    # also are "not that secret" - most of them are built into the resulting
+    # binary and could be extracted by someone with `strings`.
+    re.compile(r"^secrets:get:project/releng/gecko/build/level-[0-9]/\*"),
+    # ptracing is generally useful for interactive tasks, too!
+    re.compile(r"^docker-worker:feature:allowPtrace$"),
+    # docker-worker capabilities include loopback devices
+    re.compile(r"^docker-worker:capability:device:.*$"),
+    re.compile(r"^docker-worker:capability:privileged$"),
+    re.compile(r"^docker-worker:cache:gecko-level-1-checkouts.*$"),
+    re.compile(r"^docker-worker:cache:gecko-level-1-tooltool-cache.*$"),
+]
+
+
+def context(params):
+    # available for any docker-worker tasks at levels 1, 2; and for
+    # test tasks on level 3 (level-3 builders are firewalled off)
+    if int(params["level"]) < 3:
+        return [{"worker-implementation": "docker-worker"}]
+    return [{"worker-implementation": "docker-worker", "kind": "test"}]
+    # Windows is not supported by one-click loaners yet. See
+    # https://wiki.mozilla.org/ReleaseEngineering/How_To/Self_Provision_a_TaskCluster_Windows_Instance
+    # for instructions for using them.
+
+
+@register_callback_action(
+    title="Create Interactive Task",
+    name="create-interactive",
+    symbol="create-inter",
+    description=("Create a a copy of the task that you can interact with"),
+    order=50,
+    context=context,
+    schema={
+        "type": "object",
+        "properties": {
+            "notify": {
+                "type": "string",
+                "format": "email",
+                "title": "Who to notify of the pending interactive task",
+                "description": (
+                    "Enter your email here to get an email containing a link "
+                    "to interact with the task"
+                ),
+                # include a default for ease of users' editing
+                "default": "noreply@noreply.mozilla.org",
+            },
+        },
+        "additionalProperties": False,
+    },
+)
+def create_interactive_action(parameters, graph_config, input, task_group_id, task_id):
+    # fetch the original task definition from the taskgraph, to avoid
+    # creating interactive copies of unexpected tasks.  Note that this only applies
+    # to docker-worker tasks, so we can assume the docker-worker payload format.
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+    task = get_task_definition(task_id)
+    label = task["metadata"]["name"]
+
+    def edit(task):
+        if task.label != label:
+            return task
+        task_def = task.task
+
+        # drop task routes (don't index this!)
+        task_def["routes"] = []
+
+        # only try this once
+        task_def["retries"] = 0
+
+        # short expirations, at least 3 hour maxRunTime
+        task_def["deadline"] = {"relative-datestamp": "12 hours"}
+        task_def["created"] = {"relative-datestamp": "0 hours"}
+        task_def["expires"] = {"relative-datestamp": "1 day"}
+
+        # filter scopes with the SCOPE_WHITELIST
+        task.task["scopes"] = [
+            s
+            for s in task.task.get("scopes", [])
+            if any(p.match(s) for p in SCOPE_WHITELIST)
+        ]
+
+        payload = task_def["payload"]
+
+        # make sure the task runs for long enough..
+        payload["maxRunTime"] = max(3600 * 3, payload.get("maxRunTime", 0))
+
+        # no caches or artifacts
+        payload["cache"] = {}
+        payload["artifacts"] = {}
+
+        # enable interactive mode
+        payload.setdefault("features", {})["interactive"] = True
+        payload.setdefault("env", {})["TASKCLUSTER_INTERACTIVE"] = "true"
+
+        for key in task_def["payload"]["env"].keys():
+            payload["env"][key] = task_def["payload"]["env"].get(key, "")
+
+        # add notification
+        email = input.get("notify")
+        # no point sending to a noreply address!
+        if email and email != "noreply@noreply.mozilla.org":
+            info = {
+                "url": taskcluster_urls.ui(
+                    get_root_url(False), "tasks/${status.taskId}/connect"
+                ),
+                "label": label,
+                "revision": parameters["head_rev"],
+                "repo": parameters["head_repository"],
+            }
+            task_def.setdefault("extra", {}).setdefault("notify", {})["email"] = {
+                "subject": EMAIL_SUBJECT.format(**info),
+                "content": EMAIL_CONTENT.format(**info),
+                "link": {"text": "Connect", "href": info["url"]},
+            }
+            task_def["routes"].append(f"notify.email.{email}.on-pending")
+
+        return task
+
+    # Create the task and any of its dependencies. This uses a new taskGroupId to avoid
+    # polluting the existing taskGroup with interactive tasks.
+    action_task_id = os.environ.get("TASK_ID")
+    label_to_taskid = create_tasks(
+        graph_config,
+        [label],
+        full_task_graph,
+        label_to_taskid,
+        parameters,
+        decision_task_id=action_task_id,
+        modifier=edit,
+    )
+
+    taskId = label_to_taskid[label]
+    logger.info(f"Created interactive task {taskId}")
diff --git a/taskcluster/gecko_taskgraph/actions/gecko_profile.py b/taskcluster/gecko_taskgraph/actions/gecko_profile.py
new file mode 100644
index 0000000000..ce4394e77c
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/gecko_profile.py
@@ -0,0 +1,138 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+
+import requests
+from requests.exceptions import HTTPError
+from taskgraph.taskgraph import TaskGraph
+from taskgraph.util.taskcluster import get_artifact_from_index, get_task_definition
+
+from gecko_taskgraph.util.taskgraph import find_decision_task
+
+from .registry import register_callback_action
+from .util import combine_task_graph_files, create_tasks
+
+PUSHLOG_TMPL = "{}/json-pushes?version=2&startID={}&endID={}"
+INDEX_TMPL = "gecko.v2.{}.pushlog-id.{}.decision"
+
+logger = logging.getLogger(__name__)
+
+
+@register_callback_action(
+    title="GeckoProfile",
+    name="geckoprofile",
+    symbol="Gp",
+    description=(
+        "Take the label of the current task, "
+        "and trigger the task with that label "
+        "on previous pushes in the same project "
+        "while adding the --gecko-profile cmd arg."
+    ),
+    order=200,
+    context=[{"test-type": "talos"}, {"test-type": "raptor"}],
+    schema={},
+    available=lambda parameters: True,
+)
+def geckoprofile_action(parameters, graph_config, input, task_group_id, task_id):
+    task = get_task_definition(task_id)
+    label = task["metadata"]["name"]
+    pushes = []
+    depth = 2
+    end_id = int(parameters["pushlog_id"])
+
+    while True:
+        start_id = max(end_id - depth, 0)
+        pushlog_url = PUSHLOG_TMPL.format(
+            parameters["head_repository"], start_id, end_id
+        )
+        r = requests.get(pushlog_url)
+        r.raise_for_status()
+        pushes = pushes + list(r.json()["pushes"].keys())
+        if len(pushes) >= depth:
+            break
+
+        end_id = start_id - 1
+        start_id -= depth
+        if start_id < 0:
+            break
+
+    pushes = sorted(pushes)[-depth:]
+    backfill_pushes = []
+
+    for push in pushes:
+        try:
+            full_task_graph = get_artifact_from_index(
+                INDEX_TMPL.format(parameters["project"], push),
+                "public/full-task-graph.json",
+            )
+            _, full_task_graph = TaskGraph.from_json(full_task_graph)
+            label_to_taskid = get_artifact_from_index(
+                INDEX_TMPL.format(parameters["project"], push),
+                "public/label-to-taskid.json",
+            )
+            push_params = get_artifact_from_index(
+                INDEX_TMPL.format(parameters["project"], push), "public/parameters.yml"
+            )
+            push_decision_task_id = find_decision_task(push_params, graph_config)
+        except HTTPError as e:
+            logger.info(f"Skipping {push} due to missing index artifacts! Error: {e}")
+            continue
+
+        if label in full_task_graph.tasks.keys():
+
+            def modifier(task):
+                if task.label != label:
+                    return task
+
+                cmd = task.task["payload"]["command"]
+                task.task["payload"]["command"] = add_args_to_perf_command(
+                    cmd, ["--gecko-profile"]
+                )
+                task.task["extra"]["treeherder"]["symbol"] += "-p"
+                task.task["extra"]["treeherder"]["groupName"] += " (profiling)"
+                return task
+
+            create_tasks(
+                graph_config,
+                [label],
+                full_task_graph,
+                label_to_taskid,
+                push_params,
+                push_decision_task_id,
+                push,
+                modifier=modifier,
+            )
+            backfill_pushes.append(push)
+        else:
+            logging.info(f"Could not find {label} on {push}. Skipping.")
+    combine_task_graph_files(backfill_pushes)
+
+
+def add_args_to_perf_command(payload_commands, extra_args=[]):
+    """
+    Add custom command line args to a given command.
+    args:
+      payload_commands: the raw command as seen by taskcluster
+      extra_args: array of args we want to inject
+    """
+    perf_command_idx = -1  # currently, it's the last (or only) command
+    perf_command = payload_commands[perf_command_idx]
+
+    command_form = "default"
+    if isinstance(perf_command, str):
+        # windows has a single command, in long string form
+        perf_command = perf_command.split(" ")
+        command_form = "string"
+    # osx & linux have an array of subarrays
+
+    perf_command.extend(extra_args)
+
+    if command_form == "string":
+        # pack it back to list
+        perf_command = " ".join(perf_command)
+
+    payload_commands[perf_command_idx] = perf_command
+    return payload_commands
diff --git a/taskcluster/gecko_taskgraph/actions/merge_automation.py b/taskcluster/gecko_taskgraph/actions/merge_automation.py
new file mode 100644
index 0000000000..264383dd66
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/merge_automation.py
@@ -0,0 +1,98 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from taskgraph.parameters import Parameters
+
+from gecko_taskgraph.actions.registry import register_callback_action
+from gecko_taskgraph.decision import taskgraph_decision
+from gecko_taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS
+
+
+def is_release_promotion_available(parameters):
+    return parameters["project"] in RELEASE_PROMOTION_PROJECTS
+
+
+@register_callback_action(
+    name="merge-automation",
+    title="Merge Day Automation",
+    symbol="${input.behavior}",
+    description="Merge repository branches.",
+    permission="merge-automation",
+    order=500,
+    context=[],
+    available=is_release_promotion_available,
+    schema=lambda graph_config: {
+        "type": "object",
+        "properties": {
+            "force-dry-run": {
+                "type": "boolean",
+                "description": "Override other options and do not push changes",
+                "default": True,
+            },
+            "push": {
+                "type": "boolean",
+                "description": "Push changes using to_repo and to_branch",
+                "default": False,
+            },
+            "behavior": {
+                "type": "string",
+                "description": "The type of release promotion to perform.",
+                "enum": sorted(graph_config["merge-automation"]["behaviors"].keys()),
+                "default": "central-to-beta",
+            },
+            "from-repo": {
+                "type": "string",
+                "description": "The URI of the source repository",
+            },
+            "to-repo": {
+                "type": "string",
+                "description": "The push URI of the target repository",
+            },
+            "from-branch": {
+                "type": "string",
+                "description": "The fx head of the source, such as central",
+            },
+            "to-branch": {
+                "type": "string",
+                "description": "The fx head of the target, such as beta",
+            },
+            "ssh-user-alias": {
+                "type": "string",
+                "description": "The alias of an ssh account to use when pushing changes.",
+            },
+            "fetch-version-from": {
+                "type": "string",
+                "description": "Path to file used when querying current version.",
+            },
+        },
+        "required": ["behavior"],
+    },
+)
+def merge_automation_action(parameters, graph_config, input, task_group_id, task_id):
+    # make parameters read-write
+    parameters = dict(parameters)
+
+    parameters["target_tasks_method"] = "merge_automation"
+    parameters["merge_config"] = {
+        "force-dry-run": input.get("force-dry-run", False),
+        "behavior": input["behavior"],
+    }
+
+    for field in [
+        "from-repo",
+        "from-branch",
+        "to-repo",
+        "to-branch",
+        "ssh-user-alias",
+        "push",
+        "fetch-version-from",
+    ]:
+        if input.get(field):
+            parameters["merge_config"][field] = input[field]
+    parameters["tasks_for"] = "action"
+
+    # make parameters read-only
+    parameters = Parameters(**parameters)
+
+    taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
diff --git a/taskcluster/gecko_taskgraph/actions/openh264.py b/taskcluster/gecko_taskgraph/actions/openh264.py
new file mode 100644
index 0000000000..046d5910d2
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/openh264.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from .registry import register_callback_action
+from .util import create_tasks, fetch_graph_and_labels
+
+
+@register_callback_action(
+    name="openh264",
+    title="OpenH264 Binaries",
+    symbol="h264",
+    description="Action to prepare openh264 binaries for shipping",
+    context=[],
+)
+def openh264_action(parameters, graph_config, input, task_group_id, task_id):
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+    to_run = [
+        label
+        for label, entry in full_task_graph.tasks.items()
+        if "openh264" in entry.kind
+    ]
+    create_tasks(
+        graph_config,
+        to_run,
+        full_task_graph,
+        label_to_taskid,
+        parameters,
+        decision_task_id,
+    )
diff --git a/taskcluster/gecko_taskgraph/actions/purge_caches.py b/taskcluster/gecko_taskgraph/actions/purge_caches.py
new file mode 100644
index 0000000000..4905526f6c
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/purge_caches.py
@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+
+from taskgraph.util.taskcluster import get_task_definition, purge_cache
+
+from .registry import register_callback_action
+
+logger = logging.getLogger(__name__)
+
+
+@register_callback_action(
+    title="Purge Worker Caches",
+    name="purge-cache",
+    symbol="purge-cache",
+    description=(
+        "Purge any caches associated with this task "
+        "across all workers of the same workertype as the task."
+    ),
+    order=450,
+    context=[{"worker-implementation": "docker-worker"}],
+)
+def purge_caches_action(parameters, graph_config, input, task_group_id, task_id):
+    task = get_task_definition(task_id)
+    if task["payload"].get("cache"):
+        for cache in task["payload"]["cache"]:
+            purge_cache(
+                task["provisionerId"], task["workerType"], cache, use_proxy=True
+            )
+    else:
+        logger.info("Task has no caches. Will not clear anything!")
diff --git a/taskcluster/gecko_taskgraph/actions/raptor_extra_options.py b/taskcluster/gecko_taskgraph/actions/raptor_extra_options.py
new file mode 100644
index 0000000000..c8a7753319
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/raptor_extra_options.py
@@ -0,0 +1,77 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+
+from taskgraph.util.taskcluster import get_task_definition
+
+from .registry import register_callback_action
+from .util import create_tasks, fetch_graph_and_labels
+
+logger = logging.getLogger(__name__)
+
+
+@register_callback_action(
+    title="Raptor Extra Options",
+    name="raptor-extra-options",
+    symbol="rxo",
+    description=(
+        "Allows the user to rerun raptor-browsertime tasks with additional arguments."
+    ),
+    order=200,
+    context=[{"test-type": "raptor"}],
+    schema={
+        "type": "object",
+        "properties": {
+            "extra_options": {
+                "type": "string",
+                "default": "",
+                "description": "A space-delimited string of extra options "
+                "to be passed into a raptor-browsertime test."
+                "This also works with options with values, where the values "
+                "should be set as an assignment e.g. browser-cycles=3 "
+                "Passing multiple extra options could look something this:  "
+                "`verbose browser-cycles=3` where the test runs with verbose "
+                "mode on and the browser cycles only 3 times.",
+            }
+        },
+    },
+    available=lambda parameters: True,
+)
+def raptor_extra_options_action(
+    parameters, graph_config, input, task_group_id, task_id
+):
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+    task = get_task_definition(task_id)
+    label = task["metadata"]["name"]
+
+    def modifier(task):
+        if task.label != label:
+            return task
+
+        if task.task["payload"]["env"].get("PERF_FLAGS"):
+            task.task["payload"]["env"]["PERF_FLAGS"] += " " + input.get(
+                "extra_options"
+            )
+        else:
+            task.task["payload"]["env"].setdefault(
+                "PERF_FLAGS", input.get("extra_options")
+            )
+
+        task.task["extra"]["treeherder"]["symbol"] += "-rxo"
+        task.task["extra"]["treeherder"]["groupName"] += " (extra options run)"
+        return task
+
+    create_tasks(
+        graph_config,
+        [label],
+        full_task_graph,
+        label_to_taskid,
+        parameters,
+        decision_task_id,
+        modifier=modifier,
+    )
diff --git a/taskcluster/gecko_taskgraph/actions/rebuild_cached_tasks.py b/taskcluster/gecko_taskgraph/actions/rebuild_cached_tasks.py
new file mode 100644
index 0000000000..612da374ad
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/rebuild_cached_tasks.py
@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from .registry import register_callback_action
+from .util import create_tasks, fetch_graph_and_labels
+
+
+@register_callback_action(
+    name="rebuild-cached-tasks",
+    title="Rebuild Cached Tasks",
+    symbol="rebuild-cached",
+    description="Rebuild cached tasks.",
+    order=1000,
+    context=[],
+)
+def rebuild_cached_tasks_action(
+    parameters, graph_config, input, task_group_id, task_id
+):
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+    cached_tasks = [
+        label
+        for label, task in full_task_graph.tasks.items()
+        if task.attributes.get("cached_task", False)
+    ]
+    if cached_tasks:
+        create_tasks(
+            graph_config,
+            cached_tasks,
+            full_task_graph,
+            label_to_taskid,
+            parameters,
+            decision_task_id,
+        )
diff --git a/taskcluster/gecko_taskgraph/actions/registry.py b/taskcluster/gecko_taskgraph/actions/registry.py
new file mode 100644
index 0000000000..0c99e68d20
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/registry.py
@@ -0,0 +1,371 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import re
+from collections import namedtuple
+from types import FunctionType
+
+from mozbuild.util import memoize
+from taskgraph import create
+from taskgraph.config import load_graph_config
+from taskgraph.parameters import Parameters
+from taskgraph.util import taskcluster, yaml
+from taskgraph.util.python_path import import_sibling_modules
+
+from gecko_taskgraph.util import hash
+
+actions = []
+callbacks = {}
+
+Action = namedtuple("Action", ["order", "cb_name", "permission", "action_builder"])
+
+
+def is_json(data):
+    """Return ``True``, if ``data`` is a JSON serializable data structure."""
+    try:
+        json.dumps(data)
+    except ValueError:
+        return False
+    return True
+
+
+@memoize
+def read_taskcluster_yml(filename):
+    """Load and parse .taskcluster.yml, memoized to save some time"""
+    return yaml.load_yaml(filename)
+
+
+@memoize
+def hash_taskcluster_yml(filename):
+    """
+    Generate a hash of the given .taskcluster.yml.  This is the first 10 digits
+    of the sha256 of the file's content, and is used by administrative scripts
+    to create a hook based on this content.
+    """
+    return hash.hash_path(filename)[:10]
+
+
+def register_callback_action(
+    name,
+    title,
+    symbol,
+    description,
+    order=10000,
+    context=[],
+    available=lambda parameters: True,
+    schema=None,
+    permission="generic",
+    cb_name=None,
+):
+    """
+    Register an action callback that can be triggered from supporting
+    user interfaces, such as Treeherder.
+
+    This function is to be used as a decorator for a callback that takes
+    parameters as follows:
+
+    ``parameters``:
+        Decision task parameters, see ``taskgraph.parameters.Parameters``.
+    ``input``:
+        Input matching specified JSON schema, ``None`` if no ``schema``
+        parameter is given to ``register_callback_action``.
+    ``task_group_id``:
+        The id of the task-group this was triggered for.
+    ``task_id`` and `task``:
+        task identifier and task definition for task the action was triggered
+        for, ``None`` if no ``context`` parameters was given to
+        ``register_callback_action``.
+
+    Parameters
+    ----------
+    name : str
+        An identifier for this action, used by UIs to find the action.
+    title : str
+        A human readable title for the action to be used as label on a button
+        or text on a link for triggering the action.
+    symbol : str
+        Treeherder symbol for the action callback, this is the symbol that the
+        task calling your callback will be displayed as. This is usually 1-3
+        letters abbreviating the action title.
+    description : str
+        A human readable description of the action in **markdown**.
+        This will be display as tooltip and in dialog window when the action
+        is triggered. This is a good place to describe how to use the action.
+    order : int
+        Order of the action in menus, this is relative to the ``order`` of
+        other actions declared.
+    context : list of dict
+        List of tag-sets specifying which tasks the action is can take as input.
+        If no tag-sets is specified as input the action is related to the
+        entire task-group, and won't be triggered with a given task.
+
+        Otherwise, if ``context = [{'k': 'b', 'p': 'l'}, {'k': 't'}]`` will only
+        be displayed in the context menu for tasks that has
+        ``task.tags.k == 'b' && task.tags.p = 'l'`` or ``task.tags.k = 't'``.
+        Esentially, this allows filtering on ``task.tags``.
+
+        If this is a function, it is given the decision parameters and must return
+        a value of the form described above.
+    available : function
+        An optional function that given decision parameters decides if the
+        action is available. Defaults to a function that always returns ``True``.
+    schema : dict
+        JSON schema specifying input accepted by the action.
+        This is optional and can be left ``null`` if no input is taken.
+    permission : string
+        This defaults to ``generic`` and needs to be set for actions that need
+        additional permissions. It appears appears in ci-configuration and
+        various role and hook
+        names.
+    cb_name : string
+        The name under which this function should be registered, defaulting to
+        `name`.  Unlike `name`, which can appear multiple times, cb_name must be
+        unique among all registered callbacks.
+
+    Returns
+    -------
+    function
+        To be used as decorator for the callback function.
+    """
+    mem = {"registered": False}  # workaround nonlocal missing in 2.x
+
+    assert isinstance(title, str), "title must be a string"
+    assert isinstance(description, str), "description must be a string"
+    title = title.strip()
+    description = description.strip()
+
+    if not cb_name:
+        cb_name = name
+
+    # ensure that context is callable
+    if not callable(context):
+        context_value = context
+
+        # Because of the same name as param it must be redefined
+        # pylint: disable=E0102
+        def context(params):
+            return context_value  # noqa
+
+    def register_callback(cb):
+        assert isinstance(name, str), "name must be a string"
+        assert isinstance(order, int), "order must be an integer"
+        assert callable(schema) or is_json(
+            schema
+        ), "schema must be a JSON compatible object"
+        assert isinstance(cb, FunctionType), "callback must be a function"
+        # Allow for json-e > 25 chars in the symbol.
+        if "$" not in symbol:
+            assert 1 <= len(symbol) <= 25, "symbol must be between 1 and 25 characters"
+        assert isinstance(symbol, str), "symbol must be a string"
+
+        assert not mem[
+            "registered"
+        ], "register_callback_action must be used as decorator"
+        assert cb_name not in callbacks, "callback name {} is not unique".format(
+            cb_name
+        )
+
+        def action_builder(parameters, graph_config, decision_task_id):
+            if not available(parameters):
+                return None
+
+            # gather up the common decision-task-supplied data for this action
+            repo_param = "{}head_repository".format(
+                graph_config["project-repo-param-prefix"]
+            )
+            repository = {
+                "url": parameters[repo_param],
+                "project": parameters["project"],
+                "level": parameters["level"],
+            }
+
+            revision = parameters[
+                "{}head_rev".format(graph_config["project-repo-param-prefix"])
+            ]
+            base_revision = parameters[
+                "{}base_rev".format(graph_config["project-repo-param-prefix"])
+            ]
+            push = {
+                "owner": "mozilla-taskcluster-maintenance@mozilla.com",
+                "pushlog_id": parameters["pushlog_id"],
+                "revision": revision,
+                "base_revision": base_revision,
+            }
+
+            match = re.match(
+                r"https://(hg.mozilla.org)/(.*?)/?$", parameters[repo_param]
+            )
+            if not match:
+                raise Exception(f"Unrecognized {repo_param}")
+            action = {
+                "name": name,
+                "title": title,
+                "description": description,
+                # target taskGroupId (the task group this decision task is creating)
+                "taskGroupId": decision_task_id,
+                "cb_name": cb_name,
+                "symbol": symbol,
+            }
+
+            rv = {
+                "name": name,
+                "title": title,
+                "description": description,
+                "context": context(parameters),
+            }
+            if schema:
+                rv["schema"] = (
+                    schema(graph_config=graph_config) if callable(schema) else schema
+                )
+
+            trustDomain = graph_config["trust-domain"]
+            level = parameters["level"]
+            tcyml_hash = hash_taskcluster_yml(graph_config.taskcluster_yml)
+
+            # the tcyml_hash is prefixed with `/` in the hookId, so users will be granted
+            # hooks:trigger-hook:project-gecko/in-tree-action-3-myaction/*; if another
+            # action was named `myaction/release`, then the `*` in the scope would also
+            # match that action.  To prevent such an accident, we prohibit `/` in hook
+            # names.
+            if "/" in permission:
+                raise Exception("`/` is not allowed in action names; use `-`")
+
+            rv.update(
+                {
+                    "kind": "hook",
+                    "hookGroupId": f"project-{trustDomain}",
+                    "hookId": "in-tree-action-{}-{}/{}".format(
+                        level, permission, tcyml_hash
+                    ),
+                    "hookPayload": {
+                        # provide the decision-task parameters as context for triggerHook
+                        "decision": {
+                            "action": action,
+                            "repository": repository,
+                            "push": push,
+                        },
+                        # and pass everything else through from our own context
+                        "user": {
+                            "input": {"$eval": "input"},
+                            "taskId": {"$eval": "taskId"},  # target taskId (or null)
+                            "taskGroupId": {
+                                "$eval": "taskGroupId"
+                            },  # target task group
+                        },
+                    },
+                    "extra": {
+                        "actionPerm": permission,
+                    },
+                }
+            )
+
+            return rv
+
+        actions.append(Action(order, cb_name, permission, action_builder))
+
+        mem["registered"] = True
+        callbacks[cb_name] = cb
+        return cb
+
+    return register_callback
+
+
+def render_actions_json(parameters, graph_config, decision_task_id):
+    """
+    Render JSON object for the ``public/actions.json`` artifact.
+
+    Parameters
+    ----------
+    parameters : taskgraph.parameters.Parameters
+        Decision task parameters.
+
+    Returns
+    -------
+    dict
+        JSON object representation of the ``public/actions.json`` artifact.
+    """
+    assert isinstance(parameters, Parameters), "requires instance of Parameters"
+    actions = []
+    for action in sorted(_get_actions(graph_config), key=lambda action: action.order):
+        action = action.action_builder(parameters, graph_config, decision_task_id)
+        if action:
+            assert is_json(action), "action must be a JSON compatible object"
+            actions.append(action)
+    return {
+        "version": 1,
+        "variables": {},
+        "actions": actions,
+    }
+
+
+def sanity_check_task_scope(callback, parameters, graph_config):
+    """
+    If this action is not generic, then verify that this task has the necessary
+    scope to run the action. This serves as a backstop preventing abuse by
+    running non-generic actions using generic hooks. While scopes should
+    prevent serious damage from such abuse, it's never a valid thing to do.
+    """
+    for action in _get_actions(graph_config):
+        if action.cb_name == callback:
+            break
+    else:
+        raise Exception(f"No action with cb_name {callback}")
+
+    repo_param = "{}head_repository".format(graph_config["project-repo-param-prefix"])
+    head_repository = parameters[repo_param]
+    assert head_repository.startswith("https://hg.mozilla.org/")
+    expected_scope = "assume:repo:{}:action:{}".format(
+        head_repository[8:], action.permission
+    )
+
+    # the scope should appear literally; no need for a satisfaction check. The use of
+    # get_current_scopes here calls the auth service through the Taskcluster Proxy, giving
+    # the precise scopes available to this task.
+    if expected_scope not in taskcluster.get_current_scopes():
+        raise Exception(f"Expected task scope {expected_scope} for this action")
+
+
+def trigger_action_callback(
+    task_group_id, task_id, input, callback, parameters, root, test=False
+):
+    """
+    Trigger action callback with the given inputs. If `test` is true, then run
+    the action callback in testing mode, without actually creating tasks.
+    """
+    graph_config = load_graph_config(root)
+    graph_config.register()
+    callbacks = _get_callbacks(graph_config)
+    cb = callbacks.get(callback, None)
+    if not cb:
+        raise Exception(
+            "Unknown callback: {}. Known callbacks: {}".format(
+                callback, ", ".join(callbacks)
+            )
+        )
+
+    if test:
+        create.testing = True
+        taskcluster.testing = True
+
+    if not test:
+        sanity_check_task_scope(callback, parameters, graph_config)
+
+    cb(Parameters(**parameters), graph_config, input, task_group_id, task_id)
+
+
+def _load(graph_config):
+    # Load all modules from this folder, relying on the side-effects of register_
+    # functions to populate the action registry.
+    import_sibling_modules(exceptions=("util.py",))
+    return callbacks, actions
+
+
+def _get_callbacks(graph_config):
+    return _load(graph_config)[0]
+
+
+def _get_actions(graph_config):
+    return _load(graph_config)[1]
diff --git a/taskcluster/gecko_taskgraph/actions/release_promotion.py b/taskcluster/gecko_taskgraph/actions/release_promotion.py
new file mode 100644
index 0000000000..0d3c8f3e04
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/release_promotion.py
@@ -0,0 +1,427 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import os
+
+import requests
+from taskgraph.parameters import Parameters
+from taskgraph.taskgraph import TaskGraph
+from taskgraph.util.taskcluster import get_artifact, list_task_group_incomplete_tasks
+
+from gecko_taskgraph.actions.registry import register_callback_action
+from gecko_taskgraph.decision import taskgraph_decision
+from gecko_taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS, release_level
+from gecko_taskgraph.util.partials import populate_release_history
+from gecko_taskgraph.util.partners import (
+    fix_partner_config,
+    get_partner_config_by_url,
+    get_partner_url_config,
+    get_token,
+)
+from gecko_taskgraph.util.taskgraph import (
+    find_decision_task,
+    find_existing_tasks_from_previous_kinds,
+)
+
+RELEASE_PROMOTION_SIGNOFFS = ("mar-signing",)
+
+
+def is_release_promotion_available(parameters):
+    return parameters["project"] in RELEASE_PROMOTION_PROJECTS
+
+
+def get_partner_config(partner_url_config, github_token):
+    partner_config = {}
+    for kind, url in partner_url_config.items():
+        if url:
+            partner_config[kind] = get_partner_config_by_url(url, kind, github_token)
+    return partner_config
+
+
+def get_signoff_properties():
+    props = {}
+    for signoff in RELEASE_PROMOTION_SIGNOFFS:
+        props[signoff] = {
+            "type": "string",
+        }
+    return props
+
+
+def get_required_signoffs(input, parameters):
+    input_signoffs = set(input.get("required_signoffs", []))
+    params_signoffs = set(parameters["required_signoffs"] or [])
+    return sorted(list(input_signoffs | params_signoffs))
+
+
+def get_signoff_urls(input, parameters):
+    signoff_urls = parameters["signoff_urls"]
+    signoff_urls.update(input.get("signoff_urls", {}))
+    return signoff_urls
+
+
+def get_flavors(graph_config, param):
+    """
+    Get all flavors with the given parameter enabled.
+    """
+    promotion_flavors = graph_config["release-promotion"]["flavors"]
+    return sorted(
+        flavor
+        for (flavor, config) in promotion_flavors.items()
+        if config.get(param, False)
+    )
+
+
+@register_callback_action(
+    name="release-promotion",
+    title="Release Promotion",
+    symbol="${input.release_promotion_flavor}",
+    description="Promote a release.",
+    permission="release-promotion",
+    order=500,
+    context=[],
+    available=is_release_promotion_available,
+    schema=lambda graph_config: {
+        "type": "object",
+        "properties": {
+            "build_number": {
+                "type": "integer",
+                "default": 1,
+                "minimum": 1,
+                "title": "The release build number",
+                "description": (
+                    "The release build number. Starts at 1 per "
+                    "release version, and increments on rebuild."
+                ),
+            },
+            "do_not_optimize": {
+                "type": "array",
+                "description": (
+                    "Optional: a list of labels to avoid optimizing out "
+                    "of the graph (to force a rerun of, say, "
+                    "funsize docker-image tasks)."
+                ),
+                "items": {
+                    "type": "string",
+                },
+            },
+            "revision": {
+                "type": "string",
+                "title": "Optional: revision to promote",
+                "description": (
+                    "Optional: the revision to promote. If specified, "
+                    "and `previous_graph_kinds is not specified, find the "
+                    "push graph to promote based on the revision."
+                ),
+            },
+            "release_promotion_flavor": {
+                "type": "string",
+                "description": "The flavor of release promotion to perform.",
+                "default": "FILL ME OUT",
+                "enum": sorted(graph_config["release-promotion"]["flavors"].keys()),
+            },
+            "rebuild_kinds": {
+                "type": "array",
+                "description": (
+                    "Optional: an array of kinds to ignore from the previous "
+                    "graph(s)."
+                ),
+                "default": graph_config["release-promotion"].get("rebuild-kinds", []),
+                "items": {
+                    "type": "string",
+                },
+            },
+            "previous_graph_ids": {
+                "type": "array",
+                "description": (
+                    "Optional: an array of taskIds of decision or action "
+                    "tasks from the previous graph(s) to use to populate "
+                    "our `previous_graph_kinds`."
+                ),
+                "items": {
+                    "type": "string",
+                },
+            },
+            "version": {
+                "type": "string",
+                "description": (
+                    "Optional: override the version for release promotion. "
+                    "Occasionally we'll land a taskgraph fix in a later "
+                    "commit, but want to act on a build from a previous "
+                    "commit. If a version bump has landed in the meantime, "
+                    "relying on the in-tree version will break things."
+                ),
+                "default": "",
+            },
+            "next_version": {
+                "type": "string",
+                "description": (
+                    "Next version. Required in the following flavors: "
+                    "{}".format(get_flavors(graph_config, "version-bump"))
+                ),
+                "default": "",
+            },
+            # Example:
+            #   'partial_updates': {
+            #       '38.0': {
+            #           'buildNumber': 1,
+            #           'locales': ['de', 'en-GB', 'ru', 'uk', 'zh-TW']
+            #       },
+            #       '37.0': {
+            #           'buildNumber': 2,
+            #           'locales': ['de', 'en-GB', 'ru', 'uk']
+            #       }
+            #   }
+            "partial_updates": {
+                "type": "object",
+                "description": (
+                    "Partial updates. Required in the following flavors: "
+                    "{}".format(get_flavors(graph_config, "partial-updates"))
+                ),
+                "default": {},
+                "additionalProperties": {
+                    "type": "object",
+                    "properties": {
+                        "buildNumber": {
+                            "type": "number",
+                        },
+                        "locales": {
+                            "type": "array",
+                            "items": {
+                                "type": "string",
+                            },
+                        },
+                    },
+                    "required": [
+                        "buildNumber",
+                        "locales",
+                    ],
+                    "additionalProperties": False,
+                },
+            },
+            "release_eta": {
+                "type": "string",
+                "default": "",
+            },
+            "release_enable_partner_repack": {
+                "type": "boolean",
+                "default": False,
+                "description": "Toggle for creating partner repacks",
+            },
+            "release_enable_partner_attribution": {
+                "type": "boolean",
+                "default": False,
+                "description": "Toggle for creating partner attribution",
+            },
+            "release_partner_build_number": {
+                "type": "integer",
+                "default": 1,
+                "minimum": 1,
+                "description": (
+                    "The partner build number. This translates to, e.g. "
+                    "`v1` in the path. We generally only have to "
+                    "bump this on off-cycle partner rebuilds."
+                ),
+            },
+            "release_partners": {
+                "type": "array",
+                "description": (
+                    "A list of partners to repack, or if null or empty then use "
+                    "the current full set"
+                ),
+                "items": {
+                    "type": "string",
+                },
+            },
+            "release_partner_config": {
+                "type": "object",
+                "description": "Partner configuration to use for partner repacks.",
+                "properties": {},
+                "additionalProperties": True,
+            },
+            "release_enable_emefree": {
+                "type": "boolean",
+                "default": False,
+                "description": "Toggle for creating EME-free repacks",
+            },
+            "required_signoffs": {
+                "type": "array",
+                "description": ("The flavor of release promotion to perform."),
+                "items": {
+                    "enum": RELEASE_PROMOTION_SIGNOFFS,
+                },
+            },
+            "signoff_urls": {
+                "type": "object",
+                "default": {},
+                "additionalProperties": False,
+                "properties": get_signoff_properties(),
+            },
+        },
+        "required": ["release_promotion_flavor", "build_number"],
+    },
+)
+def release_promotion_action(parameters, graph_config, input, task_group_id, task_id):
+    release_promotion_flavor = input["release_promotion_flavor"]
+    promotion_config = graph_config["release-promotion"]["flavors"][
+        release_promotion_flavor
+    ]
+    release_history = {}
+    product = promotion_config["product"]
+
+    next_version = str(input.get("next_version") or "")
+    if promotion_config.get("version-bump", False):
+        # We force str() the input, hence the 'None'
+        if next_version in ["", "None"]:
+            raise Exception(
+                "`next_version` property needs to be provided for `{}` "
+                "target.".format(release_promotion_flavor)
+            )
+
+    if promotion_config.get("partial-updates", False):
+        partial_updates = input.get("partial_updates", {})
+        if not partial_updates and release_level(parameters["project"]) == "production":
+            raise Exception(
+                "`partial_updates` property needs to be provided for `{}`"
+                "target.".format(release_promotion_flavor)
+            )
+        balrog_prefix = product.title()
+        os.environ["PARTIAL_UPDATES"] = json.dumps(partial_updates, sort_keys=True)
+        release_history = populate_release_history(
+            balrog_prefix, parameters["project"], partial_updates=partial_updates
+        )
+
+    target_tasks_method = promotion_config["target-tasks-method"].format(
+        project=parameters["project"]
+    )
+    rebuild_kinds = input.get(
+        "rebuild_kinds", promotion_config.get("rebuild-kinds", [])
+    )
+    do_not_optimize = input.get(
+        "do_not_optimize", promotion_config.get("do-not-optimize", [])
+    )
+
+    # Make sure no pending tasks remain from a previous run
+    own_task_id = os.environ.get("TASK_ID", "")
+    try:
+        for t in list_task_group_incomplete_tasks(own_task_id):
+            if t == own_task_id:
+                continue
+            raise Exception(
+                "task group has unexpected pre-existing incomplete tasks (e.g. {})".format(
+                    t
+                )
+            )
+    except requests.exceptions.HTTPError as e:
+        # 404 means the task group doesn't exist yet, and we're fine
+        if e.response.status_code != 404:
+            raise
+
+    # Build previous_graph_ids from ``previous_graph_ids``, ``revision``,
+    # or the action parameters.
+    previous_graph_ids = input.get("previous_graph_ids")
+    if not previous_graph_ids:
+        revision = input.get("revision")
+        if revision:
+            head_rev_param = "{}head_rev".format(
+                graph_config["project-repo-param-prefix"]
+            )
+            push_parameters = {
+                head_rev_param: revision,
+                "project": parameters["project"],
+            }
+        else:
+            push_parameters = parameters
+        previous_graph_ids = [find_decision_task(push_parameters, graph_config)]
+
+    # Download parameters from the first decision task
+    parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
+    # Download and combine full task graphs from each of the previous_graph_ids.
+    # Sometimes previous relpro action tasks will add tasks, like partials,
+    # that didn't exist in the first full_task_graph, so combining them is
+    # important. The rightmost graph should take precedence in the case of
+    # conflicts.
+    combined_full_task_graph = {}
+    for graph_id in previous_graph_ids:
+        full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
+        combined_full_task_graph.update(full_task_graph)
+    _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
+    parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds(
+        combined_full_task_graph, previous_graph_ids, rebuild_kinds
+    )
+    parameters["do_not_optimize"] = do_not_optimize
+    parameters["target_tasks_method"] = target_tasks_method
+    parameters["build_number"] = int(input["build_number"])
+    parameters["next_version"] = next_version
+    parameters["release_history"] = release_history
+    if promotion_config.get("is-rc"):
+        parameters["release_type"] += "-rc"
+    parameters["release_eta"] = input.get("release_eta", "")
+    parameters["release_product"] = product
+    # When doing staging releases on try, we still want to re-use tasks from
+    # previous graphs.
+    parameters["optimize_target_tasks"] = True
+
+    if release_promotion_flavor == "promote_firefox_partner_repack":
+        release_enable_partner_repack = True
+        release_enable_partner_attribution = False
+        release_enable_emefree = False
+    elif release_promotion_flavor == "promote_firefox_partner_attribution":
+        release_enable_partner_repack = False
+        release_enable_partner_attribution = True
+        release_enable_emefree = False
+    else:
+        # for promotion or ship phases, we use the action input to turn the repacks/attribution off
+        release_enable_partner_repack = input["release_enable_partner_repack"]
+        release_enable_partner_attribution = input["release_enable_partner_attribution"]
+        release_enable_emefree = input["release_enable_emefree"]
+
+    partner_url_config = get_partner_url_config(parameters, graph_config)
+    if (
+        release_enable_partner_repack
+        and not partner_url_config["release-partner-repack"]
+    ):
+        raise Exception("Can't enable partner repacks when no config url found")
+    if (
+        release_enable_partner_attribution
+        and not partner_url_config["release-partner-attribution"]
+    ):
+        raise Exception("Can't enable partner attribution when no config url found")
+    if release_enable_emefree and not partner_url_config["release-eme-free-repack"]:
+        raise Exception("Can't enable EMEfree repacks when no config url found")
+    parameters["release_enable_partner_repack"] = release_enable_partner_repack
+    parameters[
+        "release_enable_partner_attribution"
+    ] = release_enable_partner_attribution
+    parameters["release_enable_emefree"] = release_enable_emefree
+
+    partner_config = input.get("release_partner_config")
+    if not partner_config and any(
+        [
+            release_enable_partner_repack,
+            release_enable_partner_attribution,
+            release_enable_emefree,
+        ]
+    ):
+        github_token = get_token(parameters)
+        partner_config = get_partner_config(partner_url_config, github_token)
+    if partner_config:
+        parameters["release_partner_config"] = fix_partner_config(partner_config)
+    parameters["release_partners"] = input.get("release_partners")
+    if input.get("release_partner_build_number"):
+        parameters["release_partner_build_number"] = input[
+            "release_partner_build_number"
+        ]
+
+    if input["version"]:
+        parameters["version"] = input["version"]
+
+    parameters["required_signoffs"] = get_required_signoffs(input, parameters)
+    parameters["signoff_urls"] = get_signoff_urls(input, parameters)
+
+    # make parameters read-only
+    parameters = Parameters(**parameters)
+
+    taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
diff --git a/taskcluster/gecko_taskgraph/actions/retrigger.py b/taskcluster/gecko_taskgraph/actions/retrigger.py
new file mode 100644
index 0000000000..bb4dfa8f89
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/retrigger.py
@@ -0,0 +1,311 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import itertools
+import logging
+import sys
+import textwrap
+
+from taskgraph.util.taskcluster import get_task_definition, rerun_task
+
+from gecko_taskgraph.util.taskcluster import state_task
+
+from .registry import register_callback_action
+from .util import (
+    combine_task_graph_files,
+    create_task_from_def,
+    create_tasks,
+    fetch_graph_and_labels,
+    get_tasks_with_downstream,
+    relativize_datestamps,
+)
+
+logger = logging.getLogger(__name__)
+
+RERUN_STATES = ("exception", "failed")
+
+
+def _should_retrigger(task_graph, label):
+    """
+    Return whether a given task in the taskgraph should be retriggered.
+
+    This handles the case where the task isn't there by assuming it should not be.
+    """
+    if label not in task_graph:
+        logger.info(
+            "Task {} not in full taskgraph, assuming task should not be retriggered.".format(
+                label
+            )
+        )
+        return False
+    return task_graph[label].attributes.get("retrigger", False)
+
+
+@register_callback_action(
+    title="Retrigger",
+    name="retrigger",
+    symbol="rt",
+    cb_name="retrigger-decision",
+    description=textwrap.dedent(
+        """\
+        Create a clone of the task (retriggering decision, action, and cron tasks requires
+        special scopes)."""
+    ),
+    order=11,
+    context=[
+        {"kind": "decision-task"},
+        {"kind": "action-callback"},
+        {"kind": "cron-task"},
+        {"action": "backfill-task"},
+    ],
+)
+def retrigger_decision_action(parameters, graph_config, input, task_group_id, task_id):
+    """For a single task, we try to just run exactly the same task once more.
+    It's quite possible that we don't have the scopes to do so (especially for
+    an action), but this is best-effort."""
+
+    # make all of the timestamps relative; they will then be turned back into
+    # absolute timestamps relative to the current time.
+    task = get_task_definition(task_id)
+    task = relativize_datestamps(task)
+    create_task_from_def(
+        task, parameters["level"], action_tag="retrigger-decision-task"
+    )
+
+
+@register_callback_action(
+    title="Retrigger",
+    name="retrigger",
+    symbol="rt",
+    description=("Create a clone of the task."),
+    order=19,  # must be greater than other orders in this file, as this is the fallback version
+    context=[{"retrigger": "true"}],
+    schema={
+        "type": "object",
+        "properties": {
+            "downstream": {
+                "type": "boolean",
+                "description": (
+                    "If true, downstream tasks from this one will be cloned as well. "
+                    "The dependencies will be updated to work with the new task at the root."
+                ),
+                "default": False,
+            },
+            "times": {
+                "type": "integer",
+                "default": 1,
+                "minimum": 1,
+                "maximum": 100,
+                "title": "Times",
+                "description": "How many times to run each task.",
+            },
+        },
+    },
+)
+@register_callback_action(
+    title="Retrigger (disabled)",
+    name="retrigger",
+    cb_name="retrigger-disabled",
+    symbol="rt",
+    description=(
+        "Create a clone of the task.\n\n"
+        "This type of task should typically be re-run instead of re-triggered."
+    ),
+    order=20,  # must be greater than other orders in this file, as this is the fallback version
+    context=[{}],
+    schema={
+        "type": "object",
+        "properties": {
+            "downstream": {
+                "type": "boolean",
+                "description": (
+                    "If true, downstream tasks from this one will be cloned as well. "
+                    "The dependencies will be updated to work with the new task at the root."
+                ),
+                "default": False,
+            },
+            "times": {
+                "type": "integer",
+                "default": 1,
+                "minimum": 1,
+                "maximum": 100,
+                "title": "Times",
+                "description": "How many times to run each task.",
+            },
+            "force": {
+                "type": "boolean",
+                "default": False,
+                "description": (
+                    "This task should not be re-triggered. "
+                    "This can be overridden by passing `true` here."
+                ),
+            },
+        },
+    },
+)
+def retrigger_action(parameters, graph_config, input, task_group_id, task_id):
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+
+    task = get_task_definition(task_id)
+    label = task["metadata"]["name"]
+
+    with_downstream = " "
+    to_run = [label]
+
+    if not input.get("force", None) and not _should_retrigger(full_task_graph, label):
+        logger.info(
+            "Not retriggering task {}, task should not be retrigged "
+            "and force not specified.".format(label)
+        )
+        sys.exit(1)
+
+    if input.get("downstream"):
+        to_run = get_tasks_with_downstream(to_run, full_task_graph, label_to_taskid)
+        with_downstream = " (with downstream) "
+
+    times = input.get("times", 1)
+    for i in range(times):
+        create_tasks(
+            graph_config,
+            to_run,
+            full_task_graph,
+            label_to_taskid,
+            parameters,
+            decision_task_id,
+            i,
+            action_tag="retrigger-task",
+        )
+
+        logger.info(f"Scheduled {label}{with_downstream}(time {i + 1}/{times})")
+    combine_task_graph_files(list(range(times)))
+
+
+@register_callback_action(
+    title="Rerun",
+    name="rerun",
+    symbol="rr",
+    description=(
+        "Rerun a task.\n\n"
+        "This only works on failed or exception tasks in the original taskgraph,"
+        " and is CoT friendly."
+    ),
+    order=300,
+    context=[{}],
+    schema={"type": "object", "properties": {}},
+)
+def rerun_action(parameters, graph_config, input, task_group_id, task_id):
+    task = get_task_definition(task_id)
+    parameters = dict(parameters)
+    (
+        decision_task_id,
+        full_task_graph,
+        label_to_taskid,
+        label_to_taskids,
+    ) = fetch_graph_and_labels(parameters, graph_config)
+    label = task["metadata"]["name"]
+    if task_id not in itertools.chain(*label_to_taskid.values()):
+        # XXX the error message is wrong, we're also looking at label_to_taskid
+        #     from action and cron tasks on that revision
+        logger.error(
+            "Refusing to rerun {}: taskId {} not in decision task {} label_to_taskid!".format(
+                label, task_id, decision_task_id
+            )
+        )
+
+    _rerun_task(task_id, label)
+
+
+def _rerun_task(task_id, label):
+    state = state_task(task_id)
+    if state not in RERUN_STATES:
+        logger.warning(
+            "No need to rerun {}: state '{}' not in {}!".format(
+                label, state, RERUN_STATES
+            )
+        )
+        return
+    rerun_task(task_id)
+    logger.info(f"Reran {label}")
+
+
+@register_callback_action(
+    title="Retrigger",
+    name="retrigger-multiple",
+    symbol="rt",
+    description=("Create a clone of the task."),
+    context=[],
+    schema={
+        "type": "object",
+        "properties": {
+            "requests": {
+                "type": "array",
+                "items": {
+                    "tasks": {
+                        "type": "array",
+                        "description": "An array of task labels",
+                        "items": {"type": "string"},
+                    },
+                    "times": {
+                        "type": "integer",
+                        "minimum": 1,
+                        "maximum": 100,
+                        "title": "Times",
+                        "description": "How many times to run each task.",
+                    },
+                    "additionalProperties": False,
+                },
+            },
+            "additionalProperties": False,
+        },
+    },
+)
+def retrigger_multiple(parameters, graph_config, input, task_group_id, task_id):
+    (
+        decision_task_id,
+        full_task_graph,
+        label_to_taskid,
+        label_to_taskids,
+    ) = fetch_graph_and_labels(parameters, graph_config)
+
+    suffixes = []
+    for i, request in enumerate(input.get("requests", [])):
+        times = request.get("times", 1)
+        rerun_tasks = [
+            label
+            for label in request.get("tasks")
+            if not _should_retrigger(full_task_graph, label)
+        ]
+        retrigger_tasks = [
+            label
+            for label in request.get("tasks")
+            if _should_retrigger(full_task_graph, label)
+        ]
+
+        for label in rerun_tasks:
+            # XXX we should not re-run tasks pulled in from other pushes
+            # In practice, this shouldn't matter, as only completed tasks
+            # are pulled in from other pushes and treeherder won't pass
+            # those labels.
+            for rerun_taskid in label_to_taskids[label]:
+                _rerun_task(rerun_taskid, label)
+
+        for j in range(times):
+            suffix = f"{i}-{j}"
+            suffixes.append(suffix)
+            create_tasks(
+                graph_config,
+                retrigger_tasks,
+                full_task_graph,
+                label_to_taskid,
+                parameters,
+                decision_task_id,
+                suffix,
+                action_tag="retrigger-multiple-task",
+            )
+
+    if suffixes:
+        combine_task_graph_files(suffixes)
diff --git a/taskcluster/gecko_taskgraph/actions/retrigger_custom.py b/taskcluster/gecko_taskgraph/actions/retrigger_custom.py
new file mode 100644
index 0000000000..a217009e82
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/retrigger_custom.py
@@ -0,0 +1,185 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import logging
+
+from taskgraph.util.parameterization import resolve_task_references
+from taskgraph.util.taskcluster import get_task_definition
+
+from .registry import register_callback_action
+from .util import create_task_from_def, fetch_graph_and_labels
+
+logger = logging.getLogger(__name__)
+
+# Properties available for custom retrigger of any supported test suites
+basic_properties = {
+    "path": {
+        "type": "string",
+        "maxLength": 255,
+        "default": "",
+        "title": "Path name",
+        "description": "Path of test(s) to retrigger",
+    },
+    "logLevel": {
+        "type": "string",
+        "enum": ["debug", "info", "warning", "error", "critical"],
+        "default": "info",
+        "title": "Log level",
+        "description": "Log level for output (INFO is normal, DEBUG gives more detail)",
+    },
+    "environment": {
+        "type": "object",
+        "default": {"MOZ_LOG": ""},
+        "title": "Extra environment variables",
+        "description": "Extra environment variables to use for this run",
+        "additionalProperties": {"type": "string"},
+    },
+}
+
+# Additional properties available for custom retrigger of some additional test suites
+extended_properties = basic_properties.copy()
+extended_properties.update(
+    {
+        "runUntilFail": {
+            "type": "boolean",
+            "default": False,
+            "title": "Run until failure",
+            "description": (
+                "Runs the specified set of tests repeatedly "
+                "until failure (up to REPEAT times)"
+            ),
+        },
+        "repeat": {
+            "type": "integer",
+            "default": 0,
+            "minimum": 0,
+            "title": "Repeat test(s) N times",
+            "description": (
+                "Run test(s) repeatedly (usually used in "
+                "conjunction with runUntilFail)"
+            ),
+        },
+        "preferences": {
+            "type": "object",
+            "default": {"remote.log.level": "Info"},
+            "title": "Extra gecko (about:config) preferences",
+            "description": "Extra gecko (about:config) preferences to use for this run",
+            "additionalProperties": {"type": "string"},
+        },
+    }
+)
+
+
+@register_callback_action(
+    name="retrigger-custom",
+    title="Retrigger task with custom parameters",
+    symbol="rt",
+    description="Retriggers the specified task with custom environment and parameters",
+    context=[
+        {"test-type": "mochitest", "worker-implementation": "docker-worker"},
+        {"test-type": "reftest", "worker-implementation": "docker-worker"},
+        {"test-type": "geckoview-junit", "worker-implementation": "docker-worker"},
+    ],
+    order=10,
+    schema={
+        "type": "object",
+        "properties": extended_properties,
+        "additionalProperties": False,
+        "required": ["path"],
+    },
+)
+def extended_custom_retrigger_action(
+    parameters, graph_config, input, task_group_id, task_id
+):
+    handle_custom_retrigger(parameters, graph_config, input, task_group_id, task_id)
+
+
+@register_callback_action(
+    name="retrigger-custom (gtest)",
+    title="Retrigger gtest task with custom parameters",
+    symbol="rt",
+    description="Retriggers the specified task with custom environment and parameters",
+    context=[{"test-type": "gtest", "worker-implementation": "docker-worker"}],
+    order=10,
+    schema={
+        "type": "object",
+        "properties": basic_properties,
+        "additionalProperties": False,
+        "required": ["path"],
+    },
+)
+def basic_custom_retrigger_action_basic(
+    parameters, graph_config, input, task_group_id, task_id
+):
+    handle_custom_retrigger(parameters, graph_config, input, task_group_id, task_id)
+
+
+def handle_custom_retrigger(parameters, graph_config, input, task_group_id, task_id):
+    task = get_task_definition(task_id)
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+
+    pre_task = full_task_graph.tasks[task["metadata"]["name"]]
+
+    # fix up the task's dependencies, similar to how optimization would
+    # have done in the decision
+    dependencies = {
+        name: label_to_taskid[label] for name, label in pre_task.dependencies.items()
+    }
+    new_task_definition = resolve_task_references(
+        pre_task.label, pre_task.task, task_id, decision_task_id, dependencies
+    )
+    new_task_definition.setdefault("dependencies", []).extend(dependencies.values())
+
+    # don't want to run mozharness tests, want a custom mach command instead
+    new_task_definition["payload"]["command"] += ["--no-run-tests"]
+
+    custom_mach_command = [task["tags"]["test-type"]]
+
+    # mochitests may specify a flavor
+    if new_task_definition["payload"]["env"].get("MOCHITEST_FLAVOR"):
+        custom_mach_command += [
+            "--keep-open=false",
+            "-f",
+            new_task_definition["payload"]["env"]["MOCHITEST_FLAVOR"],
+        ]
+
+    enable_e10s = json.loads(
+        new_task_definition["payload"]["env"].get("ENABLE_E10S", "true")
+    )
+    if not enable_e10s:
+        custom_mach_command += ["--disable-e10s"]
+
+    custom_mach_command += [
+        "--log-tbpl=-",
+        "--log-tbpl-level={}".format(input.get("logLevel", "debug")),
+    ]
+    if input.get("runUntilFail"):
+        custom_mach_command += ["--run-until-failure"]
+    if input.get("repeat"):
+        custom_mach_command += ["--repeat", str(input.get("repeat", 30))]
+
+    # add any custom gecko preferences
+    for key, val in input.get("preferences", {}).items():
+        custom_mach_command += ["--setpref", f"{key}={val}"]
+
+    custom_mach_command += [input["path"]]
+    new_task_definition["payload"]["env"]["CUSTOM_MACH_COMMAND"] = " ".join(
+        custom_mach_command
+    )
+
+    # update environment
+    new_task_definition["payload"]["env"].update(input.get("environment", {}))
+
+    # tweak the treeherder symbol
+    new_task_definition["extra"]["treeherder"]["symbol"] += "-custom"
+
+    logging.info("New task definition: %s", new_task_definition)
+
+    create_task_from_def(
+        new_task_definition, parameters["level"], action_tag="retrigger-custom-task"
+    )
diff --git a/taskcluster/gecko_taskgraph/actions/run_missing_tests.py b/taskcluster/gecko_taskgraph/actions/run_missing_tests.py
new file mode 100644
index 0000000000..b30bc0370a
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/run_missing_tests.py
@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+
+from taskgraph.util.taskcluster import get_artifact
+
+from .registry import register_callback_action
+from .util import create_tasks, fetch_graph_and_labels
+
+logger = logging.getLogger(__name__)
+
+
+@register_callback_action(
+    name="run-missing-tests",
+    title="Run Missing Tests",
+    symbol="rmt",
+    description=(
+        "Run tests in the selected push that were optimized away, usually by SETA."
+        "\n"
+        "This action is for use on pushes that will be merged into another branch,"
+        "to check that optimization hasn't hidden any failures."
+    ),
+    order=250,
+    context=[],  # Applies to decision task
+)
+def run_missing_tests(parameters, graph_config, input, task_group_id, task_id):
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+    target_tasks = get_artifact(decision_task_id, "public/target-tasks.json")
+
+    # The idea here is to schedule all tasks of the `test` kind that were
+    # targetted but did not appear in the final task-graph -- those were the
+    # optimized tasks.
+    to_run = []
+    already_run = 0
+    for label in target_tasks:
+        task = full_task_graph.tasks[label]
+        if task.kind != "test":
+            continue  # not a test
+        if label in label_to_taskid:
+            already_run += 1
+            continue
+        to_run.append(label)
+
+    create_tasks(
+        graph_config,
+        to_run,
+        full_task_graph,
+        label_to_taskid,
+        parameters,
+        decision_task_id,
+    )
+
+    logger.info(
+        "Out of {} test tasks, {} already existed and the action created {}".format(
+            already_run + len(to_run), already_run, len(to_run)
+        )
+    )
diff --git a/taskcluster/gecko_taskgraph/actions/scriptworker_canary.py b/taskcluster/gecko_taskgraph/actions/scriptworker_canary.py
new file mode 100644
index 0000000000..e0057da9a6
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/scriptworker_canary.py
@@ -0,0 +1,45 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from taskgraph.parameters import Parameters
+
+from gecko_taskgraph.actions.registry import register_callback_action
+from gecko_taskgraph.decision import taskgraph_decision
+
+
+@register_callback_action(
+    title="Push scriptworker canaries.",
+    name="scriptworker-canary",
+    symbol="scriptworker-canary",
+    description="Trigger scriptworker-canary pushes for the given scriptworkers.",
+    schema={
+        "type": "object",
+        "properties": {
+            "scriptworkers": {
+                "type": "array",
+                "description": "List of scriptworker types to run canaries for.",
+                "items": {"type": "string"},
+            },
+        },
+    },
+    order=1000,
+    permission="scriptworker-canary",
+    context=[],
+)
+def scriptworker_canary(parameters, graph_config, input, task_group_id, task_id):
+    scriptworkers = input["scriptworkers"]
+
+    # make parameters read-write
+    parameters = dict(parameters)
+
+    parameters["target_tasks_method"] = "scriptworker_canary"
+    parameters["try_task_config"] = {
+        "scriptworker-canary-workers": scriptworkers,
+    }
+    parameters["tasks_for"] = "action"
+
+    # make parameters read-only
+    parameters = Parameters(**parameters)
+
+    taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
diff --git a/taskcluster/gecko_taskgraph/actions/side_by_side.py b/taskcluster/gecko_taskgraph/actions/side_by_side.py
new file mode 100644
index 0000000000..0880c37760
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/side_by_side.py
@@ -0,0 +1,189 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+import os
+import sys
+from functools import partial
+
+from taskgraph.util.taskcluster import get_artifact, get_task_definition
+
+from ..util.taskcluster import list_task_group_complete_tasks
+from .registry import register_callback_action
+from .util import create_tasks, fetch_graph_and_labels, get_decision_task_id, get_pushes
+
+logger = logging.getLogger(__name__)
+
+
+def input_for_support_action(revision, base_revision, base_branch, task):
+    """Generate input for action to be scheduled.
+
+    Define what label to schedule with 'label'.
+    If it is a test task that uses explicit manifests add that information.
+    """
+    platform, test_name = task["metadata"]["name"].split("/opt-")
+    new_branch = os.environ.get("GECKO_HEAD_REPOSITORY", "/try").split("/")[-1]
+    symbol = task["extra"]["treeherder"]["symbol"]
+    input = {
+        "label": "perftest-linux-side-by-side",
+        "symbol": symbol,
+        "new_revision": revision,
+        "base_revision": base_revision,
+        "test_name": test_name,
+        "platform": platform,
+        "base_branch": base_branch,
+        "new_branch": new_branch,
+    }
+
+    return input
+
+
+def side_by_side_modifier(task, input):
+    if task.label != input["label"]:
+        return task
+
+    # Make side-by-side job searchable by the platform, test name, and revisions
+    # it was triggered for
+    task.task["metadata"][
+        "name"
+    ] = f"{input['platform']} {input['test_name']} {input['base_revision'][:12]} {input['new_revision'][:12]}"
+    # Use a job symbol to include the symbol of the job the side-by-side
+    # is running for
+    task.task["extra"]["treeherder"]["symbol"] += f"-{input['symbol']}"
+
+    cmd = task.task["payload"]["command"]
+    task.task["payload"]["command"][1][-1] = cmd[1][-1].format(**input)
+
+    return task
+
+
+@register_callback_action(
+    title="Side by side",
+    name="side-by-side",
+    symbol="gen-sxs",
+    description=(
+        "Given a performance test pageload job generate a side-by-side comparison against"
+        "the pageload job from the revision at the input."
+    ),
+    order=200,
+    context=[{"test-type": "raptor"}],
+    schema={
+        "type": "object",
+        "properties": {
+            "revision": {
+                "type": "string",
+                "default": "",
+                "description": "Revision of the push against the comparison is wanted.",
+            },
+            "project": {
+                "type": "string",
+                "default": "autoland",
+                "description": "Revision of the push against the comparison is wanted.",
+            },
+        },
+        "additionalProperties": False,
+    },
+)
+def side_by_side_action(parameters, graph_config, input, task_group_id, task_id):
+    """
+    This action does a side-by-side comparison between current revision and
+    the revision entered manually or the latest revision that ran the
+    pageload job (via support action).
+
+    To execute this action locally follow the documentation here:
+    https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally
+    """
+    task = get_task_definition(task_id)
+    decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels(
+        parameters, graph_config
+    )
+    # TODO: find another way to detect side-by-side comparable jobs
+    # (potentially lookig at the visual metrics flag)
+    if not (
+        "browsertime-tp6" in task["metadata"]["name"]
+        or "welcome" in task["metadata"]["name"]
+    ):
+        logger.exception(
+            f"Task {task['metadata']['name']} is not side-by-side comparable."
+        )
+        return
+
+    failed = False
+    input_for_action = {}
+
+    if input.get("revision"):
+        # If base_revision was introduced manually, use that
+        input_for_action = input_for_support_action(
+            revision=parameters["head_rev"],
+            base_revision=input.get("revision"),
+            base_branch=input.get("project"),
+            task=task,
+        )
+    else:
+        current_push_id = int(parameters["pushlog_id"]) - 1
+        # Go decrementally through pushlog_id, get push data, decision task id,
+        # full task graph and everything needed to find which of the past revisions
+        # ran the pageload job to compare against
+        while int(parameters["pushlog_id"]) - current_push_id < 30:
+            pushes = get_pushes(
+                project=parameters["head_repository"],
+                end_id=current_push_id,
+                depth=1,
+                full_response=True,
+            )
+            try:
+                # Get label-to-taskid.json artifact + the tasks triggered
+                # by the action tasks at a later time than the decision task
+                current_decision_task_id = get_decision_task_id(
+                    parameters["project"], current_push_id
+                )
+                current_task_group_id = get_task_definition(current_decision_task_id)[
+                    "taskGroupId"
+                ]
+                current_label_to_taskid = get_artifact(
+                    current_decision_task_id, "public/label-to-taskid.json"
+                )
+                current_full_label_to_taskid = current_label_to_taskid.copy()
+                action_task_triggered = list_task_group_complete_tasks(
+                    current_task_group_id
+                )
+                current_full_label_to_taskid.update(action_task_triggered)
+                if task["metadata"]["name"] in current_full_label_to_taskid.keys():
+                    input_for_action = input_for_support_action(
+                        revision=parameters["head_rev"],
+                        base_revision=pushes[str(current_push_id)]["changesets"][-1],
+                        base_branch=input.get("project", parameters["project"]),
+                        task=task,
+                    )
+                    break
+            except Exception:
+                logger.warning(
+                    f"Could not find decision task for push {current_push_id}"
+                )
+                # The decision task may have failed, this is common enough that we
+                # don't want to report an error for it.
+                continue
+            current_push_id -= 1
+        if not input_for_action:
+            raise Exception(
+                "Could not find a side-by-side comparable task within a depth of 30 revisions."
+            )
+
+    try:
+        create_tasks(
+            graph_config,
+            [input_for_action["label"]],
+            full_task_graph,
+            label_to_taskid,
+            parameters,
+            decision_task_id,
+            modifier=partial(side_by_side_modifier, input=input_for_action),
+        )
+    except Exception as e:
+        logger.exception(f"Failed to trigger action: {e}.")
+        failed = True
+
+    if failed:
+        sys.exit(1)
diff --git a/taskcluster/gecko_taskgraph/actions/util.py b/taskcluster/gecko_taskgraph/actions/util.py
new file mode 100644
index 0000000000..0a18b146cb
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/actions/util.py
@@ -0,0 +1,437 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import concurrent.futures as futures
+import copy
+import logging
+import os
+import re
+from functools import reduce
+
+import jsone
+import requests
+from requests.exceptions import HTTPError
+from slugid import nice as slugid
+from taskgraph import create
+from taskgraph.optimize.base import optimize_task_graph
+from taskgraph.taskgraph import TaskGraph
+from taskgraph.util.taskcluster import (
+    CONCURRENCY,
+    find_task_id,
+    get_artifact,
+    get_session,
+    get_task_definition,
+    list_tasks,
+    parse_time,
+)
+
+from gecko_taskgraph.decision import read_artifact, rename_artifact, write_artifact
+from gecko_taskgraph.util.taskcluster import trigger_hook
+from gecko_taskgraph.util.taskgraph import find_decision_task
+
+logger = logging.getLogger(__name__)
+
+INDEX_TMPL = "gecko.v2.{}.pushlog-id.{}.decision"
+PUSHLOG_TMPL = "{}/json-pushes?version=2&startID={}&endID={}"
+
+
+def _tags_within_context(tags, context=[]):
+    """A context of [] means that it *only* applies to a task group"""
+    return any(
+        all(tag in tags and tags[tag] == tag_set[tag] for tag in tag_set.keys())
+        for tag_set in context
+    )
+
+
+def _extract_applicable_action(actions_json, action_name, task_group_id, task_id):
+    """Extract action that applies to the given task or task group.
+
+    A task (as defined by its tags) is said to match a tag-set if its
+    tags are a super-set of the tag-set. A tag-set is a set of key-value pairs.
+
+    An action (as defined by its context) is said to be relevant for
+    a given task, if the task's tags match one of the tag-sets given
+    in the context property of the action.
+
+    The order of the actions is significant. When multiple actions apply to a
+    task the first one takes precedence.
+
+    For more details visit:
+    https://docs.taskcluster.net/docs/manual/design/conventions/actions/spec
+    """
+    if task_id:
+        tags = get_task_definition(task_id).get("tags")
+
+    for _action in actions_json["actions"]:
+        if action_name != _action["name"]:
+            continue
+
+        context = _action.get("context", [])
+        # Ensure the task is within the context of the action
+        if task_id and tags and _tags_within_context(tags, context):
+            return _action
+        if context == []:
+            return _action
+
+    available_actions = ", ".join(sorted({a["name"] for a in actions_json["actions"]}))
+    raise LookupError(
+        "{} action is not available for this task. Available: {}".format(
+            action_name, available_actions
+        )
+    )
+
+
+def trigger_action(action_name, decision_task_id, task_id=None, input={}):
+    if not decision_task_id:
+        raise ValueError("No decision task. We can't find the actions artifact.")
+    actions_json = get_artifact(decision_task_id, "public/actions.json")
+    if actions_json["version"] != 1:
+        raise RuntimeError("Wrong version of actions.json, unable to continue")
+
+    # These values substitute $eval in the template
+    context = {
+        "input": input,
+        "taskId": task_id,
+        "taskGroupId": decision_task_id,
+    }
+    # https://docs.taskcluster.net/docs/manual/design/conventions/actions/spec#variables
+    context.update(actions_json["variables"])
+    action = _extract_applicable_action(
+        actions_json, action_name, decision_task_id, task_id
+    )
+    kind = action["kind"]
+    if create.testing:
+        logger.info(f"Skipped triggering action for {kind} as testing is enabled")
+    elif kind == "hook":
+        hook_payload = jsone.render(action["hookPayload"], context)
+        trigger_hook(action["hookGroupId"], action["hookId"], hook_payload)
+    else:
+        raise NotImplementedError(f"Unable to submit actions with {kind} kind.")
+
+
+def get_pushes_from_params_input(parameters, input):
+    inclusive_tweak = 1 if input.get("inclusive") else 0
+    return get_pushes(
+        project=parameters["head_repository"],
+        end_id=int(parameters["pushlog_id"]) - (1 - inclusive_tweak),
+        depth=input.get("depth", 9) + inclusive_tweak,
+    )
+
+
+def get_pushes(project, end_id, depth, full_response=False):
+    pushes = []
+    while True:
+        start_id = max(end_id - depth, 0)
+        pushlog_url = PUSHLOG_TMPL.format(project, start_id, end_id)
+        logger.debug(pushlog_url)
+        r = requests.get(pushlog_url)
+        r.raise_for_status()
+        pushes = pushes + list(r.json()["pushes"].keys())
+        if len(pushes) >= depth:
+            break
+
+        end_id = start_id - 1
+        start_id -= depth
+        if start_id < 0:
+            break
+
+    pushes = sorted(pushes)[-depth:]
+    push_dict = {push: r.json()["pushes"][push] for push in pushes}
+    return push_dict if full_response else pushes
+
+
+def get_decision_task_id(project, push_id):
+    return find_task_id(INDEX_TMPL.format(project, push_id))
+
+
+def get_parameters(decision_task_id):
+    return get_artifact(decision_task_id, "public/parameters.yml")
+
+
+def get_tasks_with_downstream(labels, full_task_graph, label_to_taskid):
+    # Used to gather tasks when downstream tasks need to run as well
+    return full_task_graph.graph.transitive_closure(
+        set(labels), reverse=True
+    ).nodes & set(label_to_taskid.keys())
+
+
+def fetch_graph_and_labels(parameters, graph_config):
+    decision_task_id = find_decision_task(parameters, graph_config)
+
+    # First grab the graph and labels generated during the initial decision task
+    full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json")
+    logger.info("Load taskgraph from JSON.")
+    _, full_task_graph = TaskGraph.from_json(full_task_graph)
+    label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json")
+    label_to_taskids = {label: [task_id] for label, task_id in label_to_taskid.items()}
+
+    logger.info("Fetching additional tasks from action and cron tasks.")
+    # fetch everything in parallel; this avoids serializing any delay in downloading
+    # each artifact (such as waiting for the artifact to be mirrored locally)
+    with futures.ThreadPoolExecutor(CONCURRENCY) as e:
+        fetches = []
+
+        # fetch any modifications made by action tasks and add the new tasks
+        def fetch_action(task_id):
+            logger.info(f"fetching label-to-taskid.json for action task {task_id}")
+            try:
+                run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json")
+                label_to_taskid.update(run_label_to_id)
+                for label, task_id in run_label_to_id.items():
+                    label_to_taskids.setdefault(label, []).append(task_id)
+            except HTTPError as e:
+                if e.response.status_code != 404:
+                    raise
+                logger.debug(f"No label-to-taskid.json found for {task_id}: {e}")
+
+        head_rev_param = "{}head_rev".format(graph_config["project-repo-param-prefix"])
+
+        namespace = "{}.v2.{}.revision.{}.taskgraph.actions".format(
+            graph_config["trust-domain"],
+            parameters["project"],
+            parameters[head_rev_param],
+        )
+        for task_id in list_tasks(namespace):
+            fetches.append(e.submit(fetch_action, task_id))
+
+        # Similarly for cron tasks..
+        def fetch_cron(task_id):
+            logger.info(f"fetching label-to-taskid.json for cron task {task_id}")
+            try:
+                run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json")
+                label_to_taskid.update(run_label_to_id)
+                for label, task_id in run_label_to_id.items():
+                    label_to_taskids.setdefault(label, []).append(task_id)
+            except HTTPError as e:
+                if e.response.status_code != 404:
+                    raise
+                logger.debug(f"No label-to-taskid.json found for {task_id}: {e}")
+
+        namespace = "{}.v2.{}.revision.{}.cron".format(
+            graph_config["trust-domain"],
+            parameters["project"],
+            parameters[head_rev_param],
+        )
+        for task_id in list_tasks(namespace):
+            fetches.append(e.submit(fetch_cron, task_id))
+
+        # now wait for each fetch to complete, raising an exception if there
+        # were any issues
+        for f in futures.as_completed(fetches):
+            f.result()
+
+    return (decision_task_id, full_task_graph, label_to_taskid, label_to_taskids)
+
+
+def create_task_from_def(task_def, level, action_tag=None):
+    """Create a new task from a definition rather than from a label
+    that is already in the full-task-graph. The task definition will
+    have {relative-datestamp': '..'} rendered just like in a decision task.
+    Use this for entirely new tasks or ones that change internals of the task.
+    It is useful if you want to "edit" the full_task_graph and then hand
+    it to this function. No dependencies will be scheduled. You must handle
+    this yourself. Seeing how create_tasks handles it might prove helpful."""
+    task_def["schedulerId"] = f"gecko-level-{level}"
+    label = task_def["metadata"]["name"]
+    task_id = slugid()
+    session = get_session()
+    if action_tag:
+        task_def.setdefault("tags", {}).setdefault("action", action_tag)
+    create.create_task(session, task_id, label, task_def)
+
+
+def update_parent(task, graph):
+    task.task.setdefault("extra", {})["parent"] = os.environ.get("TASK_ID", "")
+    return task
+
+
+def update_action_tag(task, graph, action_tag):
+    task.task.setdefault("tags", {}).setdefault("action", action_tag)
+    return task
+
+
+def update_dependencies(task, graph):
+    if os.environ.get("TASK_ID"):
+        task.task.setdefault("dependencies", []).append(os.environ["TASK_ID"])
+    return task
+
+
+def create_tasks(
+    graph_config,
+    to_run,
+    full_task_graph,
+    label_to_taskid,
+    params,
+    decision_task_id,
+    suffix="",
+    modifier=lambda t: t,
+    action_tag=None,
+):
+    """Create new tasks.  The task definition will have {relative-datestamp':
+    '..'} rendered just like in a decision task.  Action callbacks should use
+    this function to create new tasks,
+    allowing easy debugging with `mach taskgraph action-callback --test`.
+    This builds up all required tasks to run in order to run the tasks requested.
+
+    Optionally this function takes a `modifier` function that is passed in each
+    task before it is put into a new graph. It should return a valid task. Note
+    that this is passed _all_ tasks in the graph, not just the set in to_run. You
+    may want to skip modifying tasks not in your to_run list.
+
+    If `suffix` is given, then it is used to give unique names to the resulting
+    artifacts.  If you call this function multiple times in the same action,
+    pass a different suffix each time to avoid overwriting artifacts.
+
+    If you wish to create the tasks in a new group, leave out decision_task_id.
+
+    Returns an updated label_to_taskid containing the new tasks"""
+    import gecko_taskgraph.optimize  # noqa: triggers registration of strategies
+
+    if suffix != "":
+        suffix = f"-{suffix}"
+    to_run = set(to_run)
+
+    #  Copy to avoid side-effects later
+    full_task_graph = copy.deepcopy(full_task_graph)
+    label_to_taskid = label_to_taskid.copy()
+
+    target_graph = full_task_graph.graph.transitive_closure(to_run)
+    target_task_graph = TaskGraph(
+        {l: modifier(full_task_graph[l]) for l in target_graph.nodes}, target_graph
+    )
+    target_task_graph.for_each_task(update_parent)
+    if action_tag:
+        target_task_graph.for_each_task(update_action_tag, action_tag)
+    if decision_task_id and decision_task_id != os.environ.get("TASK_ID"):
+        target_task_graph.for_each_task(update_dependencies)
+    optimized_task_graph, label_to_taskid = optimize_task_graph(
+        target_task_graph,
+        to_run,
+        params,
+        to_run,
+        decision_task_id,
+        existing_tasks=label_to_taskid,
+    )
+    write_artifact(f"task-graph{suffix}.json", optimized_task_graph.to_json())
+    write_artifact(f"label-to-taskid{suffix}.json", label_to_taskid)
+    write_artifact(f"to-run{suffix}.json", list(to_run))
+    create.create_tasks(
+        graph_config,
+        optimized_task_graph,
+        label_to_taskid,
+        params,
+        decision_task_id,
+    )
+    return label_to_taskid
+
+
+def _update_reducer(accumulator, new_value):
+    "similar to set or dict `update` method, but returning the modified object"
+    accumulator.update(new_value)
+    return accumulator
+
+
+def combine_task_graph_files(suffixes):
+    """Combine task-graph-{suffix}.json files into a single task-graph.json file.
+
+    Since Chain of Trust verification requires a task-graph.json file that
+    contains all children tasks, we can combine the various task-graph-0.json
+    type files into a master task-graph.json file at the end.
+
+    Actions also look for various artifacts, so we combine those in a similar
+    fashion.
+
+    In the case where there is only one suffix, we simply rename it to avoid the
+    additional cost of uploading two copies of the same data.
+    """
+
+    if len(suffixes) == 1:
+        for filename in ["task-graph", "label-to-taskid", "to-run"]:
+            rename_artifact(f"{filename}-{suffixes[0]}.json", f"{filename}.json")
+        return
+
+    def combine(file_contents, base):
+        return reduce(_update_reducer, file_contents, base)
+
+    files = [read_artifact(f"task-graph-{suffix}.json") for suffix in suffixes]
+    write_artifact("task-graph.json", combine(files, dict()))
+
+    files = [read_artifact(f"label-to-taskid-{suffix}.json") for suffix in suffixes]
+    write_artifact("label-to-taskid.json", combine(files, dict()))
+
+    files = [read_artifact(f"to-run-{suffix}.json") for suffix in suffixes]
+    write_artifact("to-run.json", list(combine(files, set())))
+
+
+def relativize_datestamps(task_def):
+    """
+    Given a task definition as received from the queue, convert all datestamps
+    to {relative_datestamp: ..} format, with the task creation time as "now".
+    The result is useful for handing to ``create_task``.
+    """
+    base = parse_time(task_def["created"])
+    # borrowed from https://github.com/epoberezkin/ajv/blob/master/lib/compile/formats.js
+    ts_pattern = re.compile(
+        r"^\d\d\d\d-[0-1]\d-[0-3]\d[t\s]"
+        r"(?:[0-2]\d:[0-5]\d:[0-5]\d|23:59:60)(?:\.\d+)?"
+        r"(?:z|[+-]\d\d:\d\d)$",
+        re.I,
+    )
+
+    def recurse(value):
+        if isinstance(value, str):
+            if ts_pattern.match(value):
+                value = parse_time(value)
+                diff = value - base
+                return {"relative-datestamp": f"{int(diff.total_seconds())} seconds"}
+        if isinstance(value, list):
+            return [recurse(e) for e in value]
+        if isinstance(value, dict):
+            return {k: recurse(v) for k, v in value.items()}
+        return value
+
+    return recurse(task_def)
+
+
+def add_args_to_command(cmd_parts, extra_args=[]):
+    """
+    Add custom command line args to a given command.
+    args:
+      cmd_parts: the raw command as seen by taskcluster
+      extra_args: array of args we want to add
+    """
+    # Prevent modification of the caller's copy of cmd_parts
+    cmd_parts = copy.deepcopy(cmd_parts)
+    cmd_type = "default"
+    if len(cmd_parts) == 1 and isinstance(cmd_parts[0], dict):
+        # windows has single cmd part as dict: 'task-reference', with long string
+        cmd_parts = cmd_parts[0]["task-reference"].split(" ")
+        cmd_type = "dict"
+    elif len(cmd_parts) == 1 and isinstance(cmd_parts[0], str):
+        # windows has single cmd part as a long string
+        cmd_parts = cmd_parts[0].split(" ")
+        cmd_type = "unicode"
+    elif len(cmd_parts) == 1 and isinstance(cmd_parts[0], list):
+        # osx has an single value array with an array inside
+        cmd_parts = cmd_parts[0]
+        cmd_type = "subarray"
+    elif len(cmd_parts) == 2 and isinstance(cmd_parts[1], list):
+        # osx has an double value array with an array inside each element.
+        # The first element is a pre-requisite command while the second
+        # is the actual test command.
+        cmd_type = "subarray2"
+
+    if cmd_type == "subarray2":
+        cmd_parts[1].extend(extra_args)
+    else:
+        cmd_parts.extend(extra_args)
+
+    if cmd_type == "dict":
+        cmd_parts = [{"task-reference": " ".join(cmd_parts)}]
+    elif cmd_type == "unicode":
+        cmd_parts = [" ".join(cmd_parts)]
+    elif cmd_type == "subarray":
+        cmd_parts = [cmd_parts]
+    return cmd_parts