diff options
Diffstat (limited to 'taskcluster/gecko_taskgraph')
221 files changed, 38375 insertions, 0 deletions
diff --git a/taskcluster/gecko_taskgraph/.ruff.toml b/taskcluster/gecko_taskgraph/.ruff.toml new file mode 100644 index 0000000000..0cd744c1cb --- /dev/null +++ b/taskcluster/gecko_taskgraph/.ruff.toml @@ -0,0 +1,4 @@ +extend = "../../pyproject.toml" + +[isort] +known-first-party = ["gecko_taskgraph"] diff --git a/taskcluster/gecko_taskgraph/__init__.py b/taskcluster/gecko_taskgraph/__init__.py new file mode 100644 index 0000000000..1346bf3c33 --- /dev/null +++ b/taskcluster/gecko_taskgraph/__init__.py @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +from taskgraph import config as taskgraph_config +from taskgraph import morph as taskgraph_morph +from taskgraph.util import schema +from taskgraph.util import taskcluster as tc_util + +from gecko_taskgraph.config import graph_config_schema + +GECKO = os.path.normpath(os.path.realpath(os.path.join(__file__, "..", "..", ".."))) + +# Maximum number of dependencies a single task can have +# https://firefox-ci-tc.services.mozilla.com/docs/reference/platform/queue/task-schema +# specifies 100, but we also optionally add the decision task id as a dep in +# taskgraph.create, so let's set this to 99. +MAX_DEPENDENCIES = 99 + +# Overwrite Taskgraph's default graph_config_schema with a custom one. +taskgraph_config.graph_config_schema = graph_config_schema + +# Don't use any of the upstream morphs. +# TODO Investigate merging our morphs with upstream. +taskgraph_morph.registered_morphs = [] + +# Default rootUrl to use if none is given in the environment; this should point +# to the production Taskcluster deployment used for CI. +tc_util.PRODUCTION_TASKCLUSTER_ROOT_URL = "https://firefox-ci-tc.services.mozilla.com" + +# Schemas for YAML files should use dashed identifiers by default. If there are +# components of the schema for which there is a good reason to use another format, +# exceptions can be added here. +schema.EXCEPTED_SCHEMA_IDENTIFIERS.extend( + [ + "test_name", + "json_location", + "video_location", + ] +) + + +def register(graph_config): + """Used to register Gecko specific extensions. + + Args: + graph_config: The graph configuration object. + """ + from taskgraph import generator + + from gecko_taskgraph import ( # noqa: trigger target task method registration + morph, # noqa: trigger morph registration + target_tasks, + ) + from gecko_taskgraph.parameters import register_parameters + from gecko_taskgraph.util.verify import verifications + + # Don't use the upstream verifications, and replace them with our own. + # TODO Investigate merging our verifications with upstream. + generator.verifications = verifications + + register_parameters() diff --git a/taskcluster/gecko_taskgraph/actions/__init__.py b/taskcluster/gecko_taskgraph/actions/__init__.py new file mode 100644 index 0000000000..590a957282 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/__init__.py @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from .registry import ( + register_callback_action, + render_actions_json, + trigger_action_callback, +) + +__all__ = [ + "register_callback_action", + "render_actions_json", + "trigger_action_callback", +] diff --git a/taskcluster/gecko_taskgraph/actions/add_new_jobs.py b/taskcluster/gecko_taskgraph/actions/add_new_jobs.py new file mode 100644 index 0000000000..39200cff68 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/add_new_jobs.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from .registry import register_callback_action +from .util import combine_task_graph_files, create_tasks, fetch_graph_and_labels + + +@register_callback_action( + name="add-new-jobs", + title="Add new jobs", + symbol="add-new", + description="Add new jobs using task labels.", + order=100, + context=[], + schema={ + "type": "object", + "properties": { + "tasks": { + "type": "array", + "description": "An array of task labels", + "items": {"type": "string"}, + }, + "times": { + "type": "integer", + "default": 1, + "minimum": 1, + "maximum": 100, + "title": "Times", + "description": "How many times to run each task.", + }, + }, + }, +) +def add_new_jobs_action(parameters, graph_config, input, task_group_id, task_id): + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + + to_run = [] + for elem in input["tasks"]: + if elem in full_task_graph.tasks: + to_run.append(elem) + else: + raise Exception(f"{elem} was not found in the task-graph") + + times = input.get("times", 1) + for i in range(times): + create_tasks( + graph_config, + to_run, + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + i, + ) + combine_task_graph_files(list(range(times))) diff --git a/taskcluster/gecko_taskgraph/actions/add_talos.py b/taskcluster/gecko_taskgraph/actions/add_talos.py new file mode 100644 index 0000000000..56b0c49cc9 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/add_talos.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +from ..target_tasks import standard_filter +from .registry import register_callback_action +from .util import create_tasks, fetch_graph_and_labels + +logger = logging.getLogger(__name__) + + +@register_callback_action( + name="run-all-talos", + title="Run All Talos Tests", + symbol="raT", + description="Add all Talos tasks to a push.", + order=150, + context=[], + schema={ + "type": "object", + "properties": { + "times": { + "type": "integer", + "default": 1, + "minimum": 1, + "maximum": 6, + "title": "Times", + "description": "How many times to run each task.", + } + }, + "additionalProperties": False, + }, +) +def add_all_talos(parameters, graph_config, input, task_group_id, task_id): + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + + times = input.get("times", 1) + for i in range(times): + to_run = [ + label + for label, entry in full_task_graph.tasks.items() + if "talos_try_name" in entry.attributes + and standard_filter(entry, parameters) + ] + + create_tasks( + graph_config, + to_run, + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + ) + logger.info(f"Scheduled {len(to_run)} talos tasks (time {i + 1}/{times})") diff --git a/taskcluster/gecko_taskgraph/actions/backfill.py b/taskcluster/gecko_taskgraph/actions/backfill.py new file mode 100644 index 0000000000..81f29394d6 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/backfill.py @@ -0,0 +1,426 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import logging +import re +import sys +from functools import partial + +from taskgraph.util.taskcluster import get_task_definition + +from .registry import register_callback_action +from .util import ( + combine_task_graph_files, + create_tasks, + fetch_graph_and_labels, + get_decision_task_id, + get_pushes, + get_pushes_from_params_input, + trigger_action, +) + +logger = logging.getLogger(__name__) +SYMBOL_REGEX = re.compile("^(.*)-[a-z0-9]{11}-bk$") +GROUP_SYMBOL_REGEX = re.compile("^(.*)-bk$") + + +def input_for_support_action(revision, task, times=1, retrigger=True): + """Generate input for action to be scheduled. + + Define what label to schedule with 'label'. + If it is a test task that uses explicit manifests add that information. + """ + input = { + "label": task["metadata"]["name"], + "revision": revision, + "times": times, + # We want the backfilled tasks to share the same symbol as the originating task + "symbol": task["extra"]["treeherder"]["symbol"], + "retrigger": retrigger, + } + + # Support tasks that are using manifest based scheduling + if task["payload"].get("env", {}).get("MOZHARNESS_TEST_PATHS"): + input["test_manifests"] = json.loads( + task["payload"]["env"]["MOZHARNESS_TEST_PATHS"] + ) + + return input + + +@register_callback_action( + title="Backfill", + name="backfill", + permission="backfill", + symbol="Bk", + description=("Given a task schedule it on previous pushes in the same project."), + order=200, + context=[{}], # This will be available for all tasks + schema={ + "type": "object", + "properties": { + "depth": { + "type": "integer", + "default": 19, + "minimum": 1, + "maximum": 25, + "title": "Depth", + "description": ( + "The number of previous pushes before the current " + "push to attempt to trigger this task on." + ), + }, + "inclusive": { + "type": "boolean", + "default": False, + "title": "Inclusive Range", + "description": ( + "If true, the backfill will also retrigger the task " + "on the selected push." + ), + }, + "times": { + "type": "integer", + "default": 1, + "minimum": 1, + "maximum": 10, + "title": "Times", + "description": ( + "The number of times to execute each job you are backfilling." + ), + }, + "retrigger": { + "type": "boolean", + "default": True, + "title": "Retrigger", + "description": ( + "If False, the task won't retrigger on pushes that have already " + "ran it." + ), + }, + }, + "additionalProperties": False, + }, + available=lambda parameters: True, +) +def backfill_action(parameters, graph_config, input, task_group_id, task_id): + """ + This action takes a task ID and schedules it on previous pushes (via support action). + + To execute this action locally follow the documentation here: + https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally + """ + task = get_task_definition(task_id) + pushes = get_pushes_from_params_input(parameters, input) + failed = False + input_for_action = input_for_support_action( + revision=parameters["head_rev"], + task=task, + times=input.get("times", 1), + retrigger=input.get("retrigger", True), + ) + + for push_id in pushes: + try: + # The Gecko decision task can sometimes fail on a push and we need to handle + # the exception that this call will produce + push_decision_task_id = get_decision_task_id(parameters["project"], push_id) + except Exception: + logger.warning(f"Could not find decision task for push {push_id}") + # The decision task may have failed, this is common enough that we + # don't want to report an error for it. + continue + + try: + trigger_action( + action_name="backfill-task", + # This lets the action know on which push we want to add a new task + decision_task_id=push_decision_task_id, + input=input_for_action, + ) + except Exception: + logger.exception(f"Failed to trigger action for {push_id}") + failed = True + + if failed: + sys.exit(1) + + +def add_backfill_suffix(regex, symbol, suffix): + m = regex.match(symbol) + if m is None: + symbol += suffix + return symbol + + +def backfill_modifier(task, input): + if task.label != input["label"]: + return task + + logger.debug(f"Modifying test_manifests for {task.label}") + times = input.get("times", 1) + + # Set task duplicates based on 'times' value. + if times > 1: + task.attributes["task_duplicates"] = times + + # If the original task has defined test paths + test_manifests = input.get("test_manifests") + if test_manifests: + revision = input.get("revision") + + task.attributes["test_manifests"] = test_manifests + task.task["payload"]["env"]["MOZHARNESS_TEST_PATHS"] = json.dumps( + test_manifests + ) + # The name/label might have been modify in new_label, thus, change it here as well + task.task["metadata"]["name"] = task.label + th_info = task.task["extra"]["treeherder"] + # Use a job symbol of the originating task as defined in the backfill action + th_info["symbol"] = add_backfill_suffix( + SYMBOL_REGEX, th_info["symbol"], f"-{revision[0:11]}-bk" + ) + if th_info.get("groupSymbol"): + # Group all backfilled tasks together + th_info["groupSymbol"] = add_backfill_suffix( + GROUP_SYMBOL_REGEX, th_info["groupSymbol"], "-bk" + ) + task.task["tags"]["action"] = "backfill-task" + return task + + +def do_not_modify(task): + return task + + +def new_label(label, tasks): + """This is to handle the case when a previous push does not contain a specific task label + and we try to find a label we can reuse. + + For instance, we try to backfill chunk #3, however, a previous push does not contain such + chunk, thus, we try to reuse another task/label. + """ + begining_label, ending = label.rsplit("-", 1) + if ending.isdigit(): + # We assume that the taskgraph has chunk #1 OR unnumbered chunk and we hijack it + if begining_label in tasks: + return begining_label + if begining_label + "-1" in tasks: + return begining_label + "-1" + raise Exception(f"New label ({label}) was not found in the task-graph") + else: + raise Exception(f"{label} was not found in the task-graph") + + +@register_callback_action( + name="backfill-task", + title="Backfill task on a push.", + permission="backfill", + symbol="backfill-task", + description="This action is normally scheduled by the backfill action. " + "The intent is to schedule a task on previous pushes.", + order=500, + context=[], + schema={ + "type": "object", + "properties": { + "label": {"type": "string", "description": "A task label"}, + "revision": { + "type": "string", + "description": "Revision of the original push from where we backfill.", + }, + "symbol": { + "type": "string", + "description": "Symbol to be used by the scheduled task.", + }, + "test_manifests": { + "type": "array", + "default": [], + "description": "An array of test manifest paths", + "items": {"type": "string"}, + }, + "times": { + "type": "integer", + "default": 1, + "minimum": 1, + "maximum": 10, + "title": "Times", + "description": ( + "The number of times to execute each job " "you are backfilling." + ), + }, + "retrigger": { + "type": "boolean", + "default": True, + "title": "Retrigger", + "description": ( + "If False, the task won't retrigger on pushes that have already " + "ran it." + ), + }, + }, + }, +) +def add_task_with_original_manifests( + parameters, graph_config, input, task_group_id, task_id +): + """ + This action is normally scheduled by the backfill action. The intent is to schedule a test + task with the test manifests from the original task (if available). + + The push in which we want to schedule a new task is defined by the parameters object. + + To execute this action locally follow the documentation here: + https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally + """ + # This step takes a lot of time when executed locally + logger.info("Retreving the full task graph and labels.") + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + + label = input.get("label") + if not input.get("retrigger") and label in label_to_taskid: + logger.info( + f"Skipping push with decision task ID {decision_task_id} as it already has this test." + ) + return + + if label not in full_task_graph.tasks: + label = new_label(label, full_task_graph.tasks) + + to_run = [label] + + logger.info("Creating tasks...") + create_tasks( + graph_config, + to_run, + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + suffix="0", + modifier=partial(backfill_modifier, input=input), + ) + + # TODO Implement a way to write out artifacts without assuming there's + # multiple sets of them so we can stop passing in "suffix". + combine_task_graph_files(["0"]) + + +@register_callback_action( + title="Backfill all browsertime", + name="backfill-all-browsertime", + permission="backfill", + symbol="baB", + description=( + "Schedule all browsertime tests for the current and previous push in the same project." + ), + order=800, + context=[], # This will be available for all tasks + available=lambda parameters: True, +) +def backfill_all_browsertime(parameters, graph_config, input, task_group_id, task_id): + """ + This action takes a revision and schedules it on previous pushes (via support action). + + To execute this action locally follow the documentation here: + https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally + """ + pushes = get_pushes( + project=parameters["head_repository"], + end_id=int(parameters["pushlog_id"]), + depth=2, + ) + + for push_id in pushes: + try: + # The Gecko decision task can sometimes fail on a push and we need to handle + # the exception that this call will produce + push_decision_task_id = get_decision_task_id(parameters["project"], push_id) + except Exception: + logger.warning(f"Could not find decision task for push {push_id}") + # The decision task may have failed, this is common enough that we + # don't want to report an error for it. + continue + + try: + trigger_action( + action_name="add-all-browsertime", + # This lets the action know on which push we want to add a new task + decision_task_id=push_decision_task_id, + ) + except Exception: + logger.exception(f"Failed to trigger action for {push_id}") + sys.exit(1) + + +def filter_raptor_jobs(full_task_graph, label_to_taskid): + to_run = [] + for label, entry in full_task_graph.tasks.items(): + if entry.kind != "test": + continue + if entry.task.get("extra", {}).get("suite", "") != "raptor": + continue + if "browsertime" not in entry.attributes.get("raptor_try_name", ""): + continue + if not entry.attributes.get("test_platform", "").endswith("shippable-qr/opt"): + continue + if "android" in entry.attributes.get("test_platform", ""): + # Bug 1786254 - The backfill bot is scheduling too many tests atm + continue + exceptions = ("live", "profiling", "youtube-playback") + if any(e in entry.attributes.get("raptor_try_name", "") for e in exceptions): + continue + if "firefox" in entry.attributes.get( + "raptor_try_name", "" + ) and entry.attributes.get("test_platform", "").endswith("64-shippable-qr/opt"): + # add the browsertime test + if label not in label_to_taskid: + to_run.append(label) + if "geckoview" in entry.attributes.get("raptor_try_name", ""): + # add the pageload test + if label not in label_to_taskid: + to_run.append(label) + return to_run + + +@register_callback_action( + name="add-all-browsertime", + title="Add All Browsertime Tests.", + permission="backfill", + symbol="aaB", + description="This action is normally scheduled by the backfill-all-browsertime action. " + "The intent is to schedule all browsertime tests on a specific pushe.", + order=900, + context=[], +) +def add_all_browsertime(parameters, graph_config, input, task_group_id, task_id): + """ + This action is normally scheduled by the backfill-all-browsertime action. The intent is to + trigger all browsertime tasks for the current revision. + + The push in which we want to schedule a new task is defined by the parameters object. + + To execute this action locally follow the documentation here: + https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally + """ + logger.info("Retreving the full task graph and labels.") + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + + to_run = filter_raptor_jobs(full_task_graph, label_to_taskid) + + create_tasks( + graph_config, + to_run, + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + ) + logger.info(f"Scheduled {len(to_run)} raptor tasks (time 1)") diff --git a/taskcluster/gecko_taskgraph/actions/cancel.py b/taskcluster/gecko_taskgraph/actions/cancel.py new file mode 100644 index 0000000000..d895781395 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/cancel.py @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +import requests +from taskgraph.util.taskcluster import cancel_task + +from .registry import register_callback_action + +logger = logging.getLogger(__name__) + + +@register_callback_action( + title="Cancel Task", + name="cancel", + symbol="cx", + description=("Cancel the given task"), + order=350, + context=[{}], +) +def cancel_action(parameters, graph_config, input, task_group_id, task_id): + # Note that this is limited by the scopes afforded to generic actions to + # only cancel tasks with the level-specific schedulerId. + try: + cancel_task(task_id, use_proxy=True) + except requests.HTTPError as e: + if e.response.status_code == 409: + # A 409 response indicates that this task is past its deadline. It + # cannot be cancelled at this time, but it's also not running + # anymore, so we can ignore this error. + logger.info(f"Task {task_id} is past its deadline and cannot be cancelled.") + return + raise diff --git a/taskcluster/gecko_taskgraph/actions/cancel_all.py b/taskcluster/gecko_taskgraph/actions/cancel_all.py new file mode 100644 index 0000000000..d74b83b7d8 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/cancel_all.py @@ -0,0 +1,60 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import concurrent.futures as futures +import logging +import os + +import requests +from taskgraph.util.taskcluster import CONCURRENCY, cancel_task + +from gecko_taskgraph.util.taskcluster import list_task_group_incomplete_task_ids + +from .registry import register_callback_action + +logger = logging.getLogger(__name__) + + +@register_callback_action( + title="Cancel All", + name="cancel-all", + symbol="cAll", + description=( + "Cancel all running and pending tasks created by the decision task " + "this action task is associated with." + ), + order=400, + context=[], +) +def cancel_all_action(parameters, graph_config, input, task_group_id, task_id): + def do_cancel_task(task_id): + logger.info(f"Cancelling task {task_id}") + try: + cancel_task(task_id, use_proxy=True) + except requests.HTTPError as e: + if e.response.status_code == 409: + # A 409 response indicates that this task is past its deadline. It + # cannot be cancelled at this time, but it's also not running + # anymore, so we can ignore this error. + logger.info( + "Task {} is past its deadline and cannot be cancelled.".format( + task_id + ) + ) + return + raise + + own_task_id = os.environ.get("TASK_ID", "") + to_cancel = [ + t + for t in list_task_group_incomplete_task_ids(task_group_id) + if t != own_task_id + ] + + logger.info(f"Cancelling {len(to_cancel)} tasks") + with futures.ThreadPoolExecutor(CONCURRENCY) as e: + cancel_futs = [e.submit(do_cancel_task, t) for t in to_cancel] + for f in futures.as_completed(cancel_futs): + f.result() diff --git a/taskcluster/gecko_taskgraph/actions/confirm_failure.py b/taskcluster/gecko_taskgraph/actions/confirm_failure.py new file mode 100644 index 0000000000..3d0b572a2e --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/confirm_failure.py @@ -0,0 +1,238 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import logging +import re + +from taskgraph.util.parameterization import resolve_task_references +from taskgraph.util.taskcluster import get_artifact, get_task_definition, list_artifacts + +from gecko_taskgraph.util.copy_task import copy_task + +from .registry import register_callback_action +from .util import add_args_to_command, create_task_from_def, fetch_graph_and_labels + +logger = logging.getLogger(__name__) + + +def get_failures(task_id): + """Returns a dict containing properties containing a list of + directories containing test failures and a separate list of + individual test failures from the errorsummary.log artifact for + the task. + + Calls the helper function munge_test_path to attempt to find an + appropriate test path to pass to the task in + MOZHARNESS_TEST_PATHS. If no appropriate test path can be + determined, nothing is returned. + """ + + def re_compile_list(*lst): + # Ideally we'd just use rb"" literals and avoid the encode, but + # this file needs to be importable in python2 for now. + return [re.compile(s.encode("utf-8")) for s in lst] + + re_bad_tests = re_compile_list( + r"Last test finished", + r"LeakSanitizer", + r"Main app process exited normally", + r"ShutdownLeaks", + r"[(]SimpleTest/TestRunner.js[)]", + r"automation.py", + r"https?://localhost:\d+/\d+/\d+/.*[.]html", + r"jsreftest", + r"leakcheck", + r"mozrunner-startup", + r"pid: ", + r"RemoteProcessMonitor", + r"unknown test url", + ) + re_extract_tests = re_compile_list( + r'"test": "(?:[^:]+:)?(?:https?|file):[^ ]+/reftest/tests/([^ "]+)', + r'"test": "(?:[^:]+:)?(?:https?|file):[^:]+:[0-9]+/tests/([^ "]+)', + r'xpcshell-?[^ "]*\.ini:([^ "]+)', + r'/tests/([^ "]+) - finished .*', + r'"test": "([^ "]+)"', + r'"message": "Error running command run_test with arguments ' + r"[(]<wptrunner[.]wpttest[.]TestharnessTest ([^>]+)>", + r'"message": "TEST-[^ ]+ [|] ([^ "]+)[^|]*[|]', + ) + + def munge_test_path(line): + test_path = None + for r in re_bad_tests: + if r.search(line): + return None + for r in re_extract_tests: + m = r.search(line) + if m: + test_path = m.group(1) + break + return test_path + + # collect dirs that don't have a specific manifest + dirs = set() + tests = set() + artifacts = list_artifacts(task_id) + for artifact in artifacts: + if "name" not in artifact or not artifact["name"].endswith("errorsummary.log"): + continue + + stream = get_artifact(task_id, artifact["name"]) + if not stream: + continue + + # We handle the stream as raw bytes because it may contain invalid + # UTF-8 characters in portions other than those containing the error + # messages we're looking for. + for line in stream.read().split(b"\n"): + if not line.strip(): + continue + + l = json.loads(line) + if "group_results" in l.keys() and l["status"] != "OK": + dirs.add(l["group_results"].group()) + + elif "test" in l.keys(): + test_path = munge_test_path(line.strip()) + tests.add(test_path.decode("utf-8")) + + # only run the failing test not both test + dir + if l["group"] in dirs: + dirs.remove(l["group"]) + + if len(tests) > 4: + break + + # turn group into dir by stripping off leafname + dirs = set([d.split("/")[0:-1] for d in dirs]) + + return {"dirs": sorted(dirs), "tests": sorted(tests)} + + +def create_confirm_failure_tasks(task_definition, failures, level): + """ + Create tasks to re-run the original task plus tasks to test + each failing test directory and individual path. + + """ + logger.info(f"Confirm Failures task:\n{json.dumps(task_definition, indent=2)}") + + # Operate on a copy of the original task_definition + task_definition = copy_task(task_definition) + + task_name = task_definition["metadata"]["name"] + repeatable_task = False + if ( + "crashtest" in task_name + or "mochitest" in task_name + or "reftest" in task_name + or "xpcshell" in task_name + and "jsreftest" not in task_name + ): + repeatable_task = True + + th_dict = task_definition["extra"]["treeherder"] + symbol = th_dict["symbol"] + is_windows = "windows" in th_dict["machine"]["platform"] + + suite = task_definition["extra"]["suite"] + if "-coverage" in suite: + suite = suite[: suite.index("-coverage")] + is_wpt = "web-platform-tests" in suite + + # command is a copy of task_definition['payload']['command'] from the original task. + # It is used to create the new version containing the + # task_definition['payload']['command'] with repeat_args which is updated every time + # through the failure_group loop. + + command = copy_task(task_definition["payload"]["command"]) + + th_dict["groupSymbol"] = th_dict["groupSymbol"] + "-cf" + th_dict["tier"] = 3 + + if repeatable_task: + task_definition["payload"]["maxRunTime"] = 3600 * 3 + + for failure_group in failures: + if len(failures[failure_group]) == 0: + continue + if failure_group == "dirs": + failure_group_suffix = "-d" + # execute 5 total loops + repeat_args = ["--repeat=4"] if repeatable_task else [] + elif failure_group == "tests": + failure_group_suffix = "-t" + # execute 10 total loops + repeat_args = ["--repeat=9"] if repeatable_task else [] + else: + logger.error( + "create_confirm_failure_tasks: Unknown failure_group {}".format( + failure_group + ) + ) + continue + + if repeat_args: + task_definition["payload"]["command"] = add_args_to_command( + command, extra_args=repeat_args + ) + else: + task_definition["payload"]["command"] = command + + for failure_path in failures[failure_group]: + th_dict["symbol"] = symbol + failure_group_suffix + if is_wpt: + failure_path = "testing/web-platform/tests" + failure_path + if is_windows: + failure_path = "\\".join(failure_path.split("/")) + task_definition["payload"]["env"]["MOZHARNESS_TEST_PATHS"] = json.dumps( + {suite: [failure_path]}, sort_keys=True + ) + + logger.info( + "Creating task for path {} with command {}".format( + failure_path, task_definition["payload"]["command"] + ) + ) + create_task_from_def(task_definition, level) + + +@register_callback_action( + name="confirm-failures", + title="Confirm failures in job", + symbol="cf", + description="Re-run Tests for original manifest, directories or tests for failing tests.", + order=150, + context=[{"kind": "test"}], + schema={ + "type": "object", + "properties": {}, + "additionalProperties": False, + }, +) +def confirm_failures(parameters, graph_config, input, task_group_id, task_id): + task = get_task_definition(task_id) + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + + pre_task = full_task_graph.tasks[task["metadata"]["name"]] + + # fix up the task's dependencies, similar to how optimization would + # have done in the decision + dependencies = { + name: label_to_taskid[label] for name, label in pre_task.dependencies.items() + } + + task_definition = resolve_task_references( + pre_task.label, pre_task.task, task_id, decision_task_id, dependencies + ) + task_definition.setdefault("dependencies", []).extend(dependencies.values()) + + failures = get_failures(task_id) + logger.info("confirm_failures: %s" % failures) + create_confirm_failure_tasks(task_definition, failures, parameters["level"]) diff --git a/taskcluster/gecko_taskgraph/actions/create_interactive.py b/taskcluster/gecko_taskgraph/actions/create_interactive.py new file mode 100644 index 0000000000..c8d196782e --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/create_interactive.py @@ -0,0 +1,192 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging +import os +import re + +import taskcluster_urls +from taskgraph.util.taskcluster import get_root_url, get_task_definition, send_email + +from gecko_taskgraph.actions.registry import register_callback_action +from gecko_taskgraph.actions.util import create_tasks, fetch_graph_and_labels + +logger = logging.getLogger(__name__) + +EMAIL_SUBJECT = "Your Interactive Task for {label}" +EMAIL_CONTENT = """\ +As you requested, Firefox CI has created an interactive task to run {label} +on revision {revision} in {repo}. Click the button below to connect to the +task. You may need to wait for it to begin running. +""" + +### +# Security Concerns +# +# An "interactive task" is, quite literally, shell access to a worker. That +# is limited by being in a Docker container, but we assume that Docker has +# bugs so we do not want to rely on container isolation exclusively. +# +# Interactive tasks should never be allowed on hosts that build binaries +# leading to a release -- level 3 builders. +# +# Users must not be allowed to create interactive tasks for tasks above +# their own level. +# +# Interactive tasks must not have any routes that might make them appear +# in the index to be used by other production tasks. +# +# Interactive tasks should not be able to write to any docker-worker caches. + +SCOPE_WHITELIST = [ + # these are not actually secrets, and just about everything needs them + re.compile(r"^secrets:get:project/taskcluster/gecko/(hgfingerprint|hgmointernal)$"), + # public downloads are OK + re.compile(r"^docker-worker:relengapi-proxy:tooltool.download.public$"), + re.compile(r"^project:releng:services/tooltool/api/download/public$"), + # internal downloads are OK + re.compile(r"^docker-worker:relengapi-proxy:tooltool.download.internal$"), + re.compile(r"^project:releng:services/tooltool/api/download/internal$"), + # private toolchain artifacts from tasks + re.compile(r"^queue:get-artifact:project/gecko/.*$"), + # level-appropriate secrets are generally necessary to run a task; these + # also are "not that secret" - most of them are built into the resulting + # binary and could be extracted by someone with `strings`. + re.compile(r"^secrets:get:project/releng/gecko/build/level-[0-9]/\*"), + # ptracing is generally useful for interactive tasks, too! + re.compile(r"^docker-worker:feature:allowPtrace$"), + # docker-worker capabilities include loopback devices + re.compile(r"^docker-worker:capability:device:.*$"), + re.compile(r"^docker-worker:capability:privileged$"), + re.compile(r"^docker-worker:cache:gecko-level-1-checkouts.*$"), + re.compile(r"^docker-worker:cache:gecko-level-1-tooltool-cache.*$"), +] + + +def context(params): + # available for any docker-worker tasks at levels 1, 2; and for + # test tasks on level 3 (level-3 builders are firewalled off) + if int(params["level"]) < 3: + return [{"worker-implementation": "docker-worker"}] + return [{"worker-implementation": "docker-worker", "kind": "test"}] + # Windows is not supported by one-click loaners yet. See + # https://wiki.mozilla.org/ReleaseEngineering/How_To/Self_Provision_a_TaskCluster_Windows_Instance + # for instructions for using them. + + +@register_callback_action( + title="Create Interactive Task", + name="create-interactive", + symbol="create-inter", + description=("Create a a copy of the task that you can interact with"), + order=50, + context=context, + schema={ + "type": "object", + "properties": { + "notify": { + "type": "string", + "format": "email", + "title": "Who to notify of the pending interactive task", + "description": ( + "Enter your email here to get an email containing a link " + "to interact with the task" + ), + # include a default for ease of users' editing + "default": "noreply@noreply.mozilla.org", + }, + }, + "additionalProperties": False, + }, +) +def create_interactive_action(parameters, graph_config, input, task_group_id, task_id): + # fetch the original task definition from the taskgraph, to avoid + # creating interactive copies of unexpected tasks. Note that this only applies + # to docker-worker tasks, so we can assume the docker-worker payload format. + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + task = get_task_definition(task_id) + label = task["metadata"]["name"] + + def edit(task): + if task.label != label: + return task + task_def = task.task + + # drop task routes (don't index this!) + task_def["routes"] = [] + + # only try this once + task_def["retries"] = 0 + + # short expirations, at least 3 hour maxRunTime + task_def["deadline"] = {"relative-datestamp": "12 hours"} + task_def["created"] = {"relative-datestamp": "0 hours"} + task_def["expires"] = {"relative-datestamp": "1 day"} + + # filter scopes with the SCOPE_WHITELIST + task.task["scopes"] = [ + s + for s in task.task.get("scopes", []) + if any(p.match(s) for p in SCOPE_WHITELIST) + ] + + payload = task_def["payload"] + + # make sure the task runs for long enough.. + payload["maxRunTime"] = max(3600 * 3, payload.get("maxRunTime", 0)) + + # no caches or artifacts + payload["cache"] = {} + payload["artifacts"] = {} + + # enable interactive mode + payload.setdefault("features", {})["interactive"] = True + payload.setdefault("env", {})["TASKCLUSTER_INTERACTIVE"] = "true" + + for key in task_def["payload"]["env"].keys(): + payload["env"][key] = task_def["payload"]["env"].get(key, "") + + return task + + # Create the task and any of its dependencies. This uses a new taskGroupId to avoid + # polluting the existing taskGroup with interactive tasks. + action_task_id = os.environ.get("TASK_ID") + label_to_taskid = create_tasks( + graph_config, + [label], + full_task_graph, + label_to_taskid, + parameters, + decision_task_id=action_task_id, + modifier=edit, + ) + + taskId = label_to_taskid[label] + logger.info(f"Created interactive task {taskId}; sending notification") + + if input and "notify" in input: + email = input["notify"] + # no point sending to a noreply address! + if email == "noreply@noreply.mozilla.org": + return + + info = { + "url": taskcluster_urls.ui(get_root_url(False), f"tasks/{taskId}/connect"), + "label": label, + "revision": parameters["head_rev"], + "repo": parameters["head_repository"], + } + send_email( + email, + subject=EMAIL_SUBJECT.format(**info), + content=EMAIL_CONTENT.format(**info), + link={ + "text": "Connect", + "href": info["url"], + }, + use_proxy=True, + ) diff --git a/taskcluster/gecko_taskgraph/actions/gecko_profile.py b/taskcluster/gecko_taskgraph/actions/gecko_profile.py new file mode 100644 index 0000000000..ce4394e77c --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/gecko_profile.py @@ -0,0 +1,138 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +import requests +from requests.exceptions import HTTPError +from taskgraph.taskgraph import TaskGraph +from taskgraph.util.taskcluster import get_artifact_from_index, get_task_definition + +from gecko_taskgraph.util.taskgraph import find_decision_task + +from .registry import register_callback_action +from .util import combine_task_graph_files, create_tasks + +PUSHLOG_TMPL = "{}/json-pushes?version=2&startID={}&endID={}" +INDEX_TMPL = "gecko.v2.{}.pushlog-id.{}.decision" + +logger = logging.getLogger(__name__) + + +@register_callback_action( + title="GeckoProfile", + name="geckoprofile", + symbol="Gp", + description=( + "Take the label of the current task, " + "and trigger the task with that label " + "on previous pushes in the same project " + "while adding the --gecko-profile cmd arg." + ), + order=200, + context=[{"test-type": "talos"}, {"test-type": "raptor"}], + schema={}, + available=lambda parameters: True, +) +def geckoprofile_action(parameters, graph_config, input, task_group_id, task_id): + task = get_task_definition(task_id) + label = task["metadata"]["name"] + pushes = [] + depth = 2 + end_id = int(parameters["pushlog_id"]) + + while True: + start_id = max(end_id - depth, 0) + pushlog_url = PUSHLOG_TMPL.format( + parameters["head_repository"], start_id, end_id + ) + r = requests.get(pushlog_url) + r.raise_for_status() + pushes = pushes + list(r.json()["pushes"].keys()) + if len(pushes) >= depth: + break + + end_id = start_id - 1 + start_id -= depth + if start_id < 0: + break + + pushes = sorted(pushes)[-depth:] + backfill_pushes = [] + + for push in pushes: + try: + full_task_graph = get_artifact_from_index( + INDEX_TMPL.format(parameters["project"], push), + "public/full-task-graph.json", + ) + _, full_task_graph = TaskGraph.from_json(full_task_graph) + label_to_taskid = get_artifact_from_index( + INDEX_TMPL.format(parameters["project"], push), + "public/label-to-taskid.json", + ) + push_params = get_artifact_from_index( + INDEX_TMPL.format(parameters["project"], push), "public/parameters.yml" + ) + push_decision_task_id = find_decision_task(push_params, graph_config) + except HTTPError as e: + logger.info(f"Skipping {push} due to missing index artifacts! Error: {e}") + continue + + if label in full_task_graph.tasks.keys(): + + def modifier(task): + if task.label != label: + return task + + cmd = task.task["payload"]["command"] + task.task["payload"]["command"] = add_args_to_perf_command( + cmd, ["--gecko-profile"] + ) + task.task["extra"]["treeherder"]["symbol"] += "-p" + task.task["extra"]["treeherder"]["groupName"] += " (profiling)" + return task + + create_tasks( + graph_config, + [label], + full_task_graph, + label_to_taskid, + push_params, + push_decision_task_id, + push, + modifier=modifier, + ) + backfill_pushes.append(push) + else: + logging.info(f"Could not find {label} on {push}. Skipping.") + combine_task_graph_files(backfill_pushes) + + +def add_args_to_perf_command(payload_commands, extra_args=[]): + """ + Add custom command line args to a given command. + args: + payload_commands: the raw command as seen by taskcluster + extra_args: array of args we want to inject + """ + perf_command_idx = -1 # currently, it's the last (or only) command + perf_command = payload_commands[perf_command_idx] + + command_form = "default" + if isinstance(perf_command, str): + # windows has a single command, in long string form + perf_command = perf_command.split(" ") + command_form = "string" + # osx & linux have an array of subarrays + + perf_command.extend(extra_args) + + if command_form == "string": + # pack it back to list + perf_command = " ".join(perf_command) + + payload_commands[perf_command_idx] = perf_command + return payload_commands diff --git a/taskcluster/gecko_taskgraph/actions/merge_automation.py b/taskcluster/gecko_taskgraph/actions/merge_automation.py new file mode 100644 index 0000000000..8b9455536b --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/merge_automation.py @@ -0,0 +1,99 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.parameters import Parameters + +from gecko_taskgraph.actions.registry import register_callback_action +from gecko_taskgraph.decision import taskgraph_decision +from gecko_taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS + + +def is_release_promotion_available(parameters): + return parameters["project"] in RELEASE_PROMOTION_PROJECTS + + +@register_callback_action( + name="merge-automation", + title="Merge Day Automation", + symbol="${input.behavior}", + description="Merge repository branches.", + permission="merge-automation", + order=500, + context=[], + available=is_release_promotion_available, + schema=lambda graph_config: { + "type": "object", + "properties": { + "force-dry-run": { + "type": "boolean", + "description": "Override other options and do not push changes", + "default": True, + }, + "push": { + "type": "boolean", + "description": "Push changes using to_repo and to_branch", + "default": False, + }, + "behavior": { + "type": "string", + "description": "The type of release promotion to perform.", + "enum": sorted(graph_config["merge-automation"]["behaviors"].keys()), + "default": "central-to-beta", + }, + "from-repo": { + "type": "string", + "description": "The URI of the source repository", + }, + "to-repo": { + "type": "string", + "description": "The push URI of the target repository", + }, + "from-branch": { + "type": "string", + "description": "The fx head of the source, such as central", + }, + "to-branch": { + "type": "string", + "description": "The fx head of the target, such as beta", + }, + "ssh-user-alias": { + "type": "string", + "description": "The alias of an ssh account to use when pushing changes.", + }, + "fetch-version-from": { + "type": "string", + "description": "Path to file used when querying current version.", + }, + }, + "required": ["behavior"], + }, +) +def merge_automation_action(parameters, graph_config, input, task_group_id, task_id): + + # make parameters read-write + parameters = dict(parameters) + + parameters["target_tasks_method"] = "merge_automation" + parameters["merge_config"] = { + "force-dry-run": input.get("force-dry-run", False), + "behavior": input["behavior"], + } + + for field in [ + "from-repo", + "from-branch", + "to-repo", + "to-branch", + "ssh-user-alias", + "push", + "fetch-version-from", + ]: + if input.get(field): + parameters["merge_config"][field] = input[field] + parameters["tasks_for"] = "action" + + # make parameters read-only + parameters = Parameters(**parameters) + + taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters) diff --git a/taskcluster/gecko_taskgraph/actions/openh264.py b/taskcluster/gecko_taskgraph/actions/openh264.py new file mode 100644 index 0000000000..226817f696 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/openh264.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from .registry import register_callback_action +from .util import create_tasks, fetch_graph_and_labels + + +@register_callback_action( + name="openh264", + title="OpenH264 Binaries", + symbol="h264", + description="Action to prepare openh264 binaries for shipping", + context=[], +) +def openh264_action(parameters, graph_config, input, task_group_id, task_id): + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + to_run = [ + label + for label, entry in full_task_graph.tasks.items() + if "openh264" in entry.kind + ] + create_tasks( + graph_config, + to_run, + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + ) diff --git a/taskcluster/gecko_taskgraph/actions/purge_caches.py b/taskcluster/gecko_taskgraph/actions/purge_caches.py new file mode 100644 index 0000000000..4905526f6c --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/purge_caches.py @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +from taskgraph.util.taskcluster import get_task_definition, purge_cache + +from .registry import register_callback_action + +logger = logging.getLogger(__name__) + + +@register_callback_action( + title="Purge Worker Caches", + name="purge-cache", + symbol="purge-cache", + description=( + "Purge any caches associated with this task " + "across all workers of the same workertype as the task." + ), + order=450, + context=[{"worker-implementation": "docker-worker"}], +) +def purge_caches_action(parameters, graph_config, input, task_group_id, task_id): + task = get_task_definition(task_id) + if task["payload"].get("cache"): + for cache in task["payload"]["cache"]: + purge_cache( + task["provisionerId"], task["workerType"], cache, use_proxy=True + ) + else: + logger.info("Task has no caches. Will not clear anything!") diff --git a/taskcluster/gecko_taskgraph/actions/raptor_extra_options.py b/taskcluster/gecko_taskgraph/actions/raptor_extra_options.py new file mode 100644 index 0000000000..43b26d284d --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/raptor_extra_options.py @@ -0,0 +1,77 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +from taskgraph.util.taskcluster import get_task_definition + +from .registry import register_callback_action +from .util import create_tasks, fetch_graph_and_labels + +logger = logging.getLogger(__name__) + + +@register_callback_action( + title="Raptor Extra Options", + name="raptor-extra-options", + symbol="rxo", + description=( + "Allows the user to rerun raptor-browsertime tasks with additional arguments." + ), + order=200, + context=[{"test-type": "raptor"}], + schema={ + "type": "object", + "properties": { + "extra_options": { + "type": "string", + "default": "", + "description": "A space-delimited string of extra options " + "to be passed into a raptor-browsertime test." + "This also works with options with values, where the values " + "should be set as an assignment e.g. browser-cycles=3 " + "Passing multiple extra options could look something this: " + "`verbose browser-cycles=3` where the test runs with verbose " + "mode on and the browser cycles only 3 times.", + } + }, + }, + available=lambda parameters: True, +) +def raptor_extra_options_action( + parameters, graph_config, input, task_group_id, task_id +): + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + task = get_task_definition(task_id) + label = task["metadata"]["name"] + + def modifier(task): + if task.label != label: + return task + + if task.task["payload"]["env"].get("PERF_FLAGS"): + task.task["payload"]["env"]["PERF_FLAGS"] += " " + input.get( + "extra_options" + ) + else: + task.task["payload"]["env"].setdefault( + "PERF_FLAGS", input.get("extra_options") + ) + + task.task["extra"]["treeherder"]["symbol"] += "-rxo" + task.task["extra"]["treeherder"]["groupName"] += " (extra options run)" + return task + + create_tasks( + graph_config, + [label], + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + modifier=modifier, + ) diff --git a/taskcluster/gecko_taskgraph/actions/rebuild_cached_tasks.py b/taskcluster/gecko_taskgraph/actions/rebuild_cached_tasks.py new file mode 100644 index 0000000000..9d8906caf1 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/rebuild_cached_tasks.py @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from .registry import register_callback_action +from .util import create_tasks, fetch_graph_and_labels + + +@register_callback_action( + name="rebuild-cached-tasks", + title="Rebuild Cached Tasks", + symbol="rebuild-cached", + description="Rebuild cached tasks.", + order=1000, + context=[], +) +def rebuild_cached_tasks_action( + parameters, graph_config, input, task_group_id, task_id +): + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + cached_tasks = [ + label + for label, task in full_task_graph.tasks.items() + if task.attributes.get("cached_task", False) + ] + if cached_tasks: + create_tasks( + graph_config, + cached_tasks, + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + ) diff --git a/taskcluster/gecko_taskgraph/actions/registry.py b/taskcluster/gecko_taskgraph/actions/registry.py new file mode 100644 index 0000000000..0c99e68d20 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/registry.py @@ -0,0 +1,371 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import re +from collections import namedtuple +from types import FunctionType + +from mozbuild.util import memoize +from taskgraph import create +from taskgraph.config import load_graph_config +from taskgraph.parameters import Parameters +from taskgraph.util import taskcluster, yaml +from taskgraph.util.python_path import import_sibling_modules + +from gecko_taskgraph.util import hash + +actions = [] +callbacks = {} + +Action = namedtuple("Action", ["order", "cb_name", "permission", "action_builder"]) + + +def is_json(data): + """Return ``True``, if ``data`` is a JSON serializable data structure.""" + try: + json.dumps(data) + except ValueError: + return False + return True + + +@memoize +def read_taskcluster_yml(filename): + """Load and parse .taskcluster.yml, memoized to save some time""" + return yaml.load_yaml(filename) + + +@memoize +def hash_taskcluster_yml(filename): + """ + Generate a hash of the given .taskcluster.yml. This is the first 10 digits + of the sha256 of the file's content, and is used by administrative scripts + to create a hook based on this content. + """ + return hash.hash_path(filename)[:10] + + +def register_callback_action( + name, + title, + symbol, + description, + order=10000, + context=[], + available=lambda parameters: True, + schema=None, + permission="generic", + cb_name=None, +): + """ + Register an action callback that can be triggered from supporting + user interfaces, such as Treeherder. + + This function is to be used as a decorator for a callback that takes + parameters as follows: + + ``parameters``: + Decision task parameters, see ``taskgraph.parameters.Parameters``. + ``input``: + Input matching specified JSON schema, ``None`` if no ``schema`` + parameter is given to ``register_callback_action``. + ``task_group_id``: + The id of the task-group this was triggered for. + ``task_id`` and `task``: + task identifier and task definition for task the action was triggered + for, ``None`` if no ``context`` parameters was given to + ``register_callback_action``. + + Parameters + ---------- + name : str + An identifier for this action, used by UIs to find the action. + title : str + A human readable title for the action to be used as label on a button + or text on a link for triggering the action. + symbol : str + Treeherder symbol for the action callback, this is the symbol that the + task calling your callback will be displayed as. This is usually 1-3 + letters abbreviating the action title. + description : str + A human readable description of the action in **markdown**. + This will be display as tooltip and in dialog window when the action + is triggered. This is a good place to describe how to use the action. + order : int + Order of the action in menus, this is relative to the ``order`` of + other actions declared. + context : list of dict + List of tag-sets specifying which tasks the action is can take as input. + If no tag-sets is specified as input the action is related to the + entire task-group, and won't be triggered with a given task. + + Otherwise, if ``context = [{'k': 'b', 'p': 'l'}, {'k': 't'}]`` will only + be displayed in the context menu for tasks that has + ``task.tags.k == 'b' && task.tags.p = 'l'`` or ``task.tags.k = 't'``. + Esentially, this allows filtering on ``task.tags``. + + If this is a function, it is given the decision parameters and must return + a value of the form described above. + available : function + An optional function that given decision parameters decides if the + action is available. Defaults to a function that always returns ``True``. + schema : dict + JSON schema specifying input accepted by the action. + This is optional and can be left ``null`` if no input is taken. + permission : string + This defaults to ``generic`` and needs to be set for actions that need + additional permissions. It appears appears in ci-configuration and + various role and hook + names. + cb_name : string + The name under which this function should be registered, defaulting to + `name`. Unlike `name`, which can appear multiple times, cb_name must be + unique among all registered callbacks. + + Returns + ------- + function + To be used as decorator for the callback function. + """ + mem = {"registered": False} # workaround nonlocal missing in 2.x + + assert isinstance(title, str), "title must be a string" + assert isinstance(description, str), "description must be a string" + title = title.strip() + description = description.strip() + + if not cb_name: + cb_name = name + + # ensure that context is callable + if not callable(context): + context_value = context + + # Because of the same name as param it must be redefined + # pylint: disable=E0102 + def context(params): + return context_value # noqa + + def register_callback(cb): + assert isinstance(name, str), "name must be a string" + assert isinstance(order, int), "order must be an integer" + assert callable(schema) or is_json( + schema + ), "schema must be a JSON compatible object" + assert isinstance(cb, FunctionType), "callback must be a function" + # Allow for json-e > 25 chars in the symbol. + if "$" not in symbol: + assert 1 <= len(symbol) <= 25, "symbol must be between 1 and 25 characters" + assert isinstance(symbol, str), "symbol must be a string" + + assert not mem[ + "registered" + ], "register_callback_action must be used as decorator" + assert cb_name not in callbacks, "callback name {} is not unique".format( + cb_name + ) + + def action_builder(parameters, graph_config, decision_task_id): + if not available(parameters): + return None + + # gather up the common decision-task-supplied data for this action + repo_param = "{}head_repository".format( + graph_config["project-repo-param-prefix"] + ) + repository = { + "url": parameters[repo_param], + "project": parameters["project"], + "level": parameters["level"], + } + + revision = parameters[ + "{}head_rev".format(graph_config["project-repo-param-prefix"]) + ] + base_revision = parameters[ + "{}base_rev".format(graph_config["project-repo-param-prefix"]) + ] + push = { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": parameters["pushlog_id"], + "revision": revision, + "base_revision": base_revision, + } + + match = re.match( + r"https://(hg.mozilla.org)/(.*?)/?$", parameters[repo_param] + ) + if not match: + raise Exception(f"Unrecognized {repo_param}") + action = { + "name": name, + "title": title, + "description": description, + # target taskGroupId (the task group this decision task is creating) + "taskGroupId": decision_task_id, + "cb_name": cb_name, + "symbol": symbol, + } + + rv = { + "name": name, + "title": title, + "description": description, + "context": context(parameters), + } + if schema: + rv["schema"] = ( + schema(graph_config=graph_config) if callable(schema) else schema + ) + + trustDomain = graph_config["trust-domain"] + level = parameters["level"] + tcyml_hash = hash_taskcluster_yml(graph_config.taskcluster_yml) + + # the tcyml_hash is prefixed with `/` in the hookId, so users will be granted + # hooks:trigger-hook:project-gecko/in-tree-action-3-myaction/*; if another + # action was named `myaction/release`, then the `*` in the scope would also + # match that action. To prevent such an accident, we prohibit `/` in hook + # names. + if "/" in permission: + raise Exception("`/` is not allowed in action names; use `-`") + + rv.update( + { + "kind": "hook", + "hookGroupId": f"project-{trustDomain}", + "hookId": "in-tree-action-{}-{}/{}".format( + level, permission, tcyml_hash + ), + "hookPayload": { + # provide the decision-task parameters as context for triggerHook + "decision": { + "action": action, + "repository": repository, + "push": push, + }, + # and pass everything else through from our own context + "user": { + "input": {"$eval": "input"}, + "taskId": {"$eval": "taskId"}, # target taskId (or null) + "taskGroupId": { + "$eval": "taskGroupId" + }, # target task group + }, + }, + "extra": { + "actionPerm": permission, + }, + } + ) + + return rv + + actions.append(Action(order, cb_name, permission, action_builder)) + + mem["registered"] = True + callbacks[cb_name] = cb + return cb + + return register_callback + + +def render_actions_json(parameters, graph_config, decision_task_id): + """ + Render JSON object for the ``public/actions.json`` artifact. + + Parameters + ---------- + parameters : taskgraph.parameters.Parameters + Decision task parameters. + + Returns + ------- + dict + JSON object representation of the ``public/actions.json`` artifact. + """ + assert isinstance(parameters, Parameters), "requires instance of Parameters" + actions = [] + for action in sorted(_get_actions(graph_config), key=lambda action: action.order): + action = action.action_builder(parameters, graph_config, decision_task_id) + if action: + assert is_json(action), "action must be a JSON compatible object" + actions.append(action) + return { + "version": 1, + "variables": {}, + "actions": actions, + } + + +def sanity_check_task_scope(callback, parameters, graph_config): + """ + If this action is not generic, then verify that this task has the necessary + scope to run the action. This serves as a backstop preventing abuse by + running non-generic actions using generic hooks. While scopes should + prevent serious damage from such abuse, it's never a valid thing to do. + """ + for action in _get_actions(graph_config): + if action.cb_name == callback: + break + else: + raise Exception(f"No action with cb_name {callback}") + + repo_param = "{}head_repository".format(graph_config["project-repo-param-prefix"]) + head_repository = parameters[repo_param] + assert head_repository.startswith("https://hg.mozilla.org/") + expected_scope = "assume:repo:{}:action:{}".format( + head_repository[8:], action.permission + ) + + # the scope should appear literally; no need for a satisfaction check. The use of + # get_current_scopes here calls the auth service through the Taskcluster Proxy, giving + # the precise scopes available to this task. + if expected_scope not in taskcluster.get_current_scopes(): + raise Exception(f"Expected task scope {expected_scope} for this action") + + +def trigger_action_callback( + task_group_id, task_id, input, callback, parameters, root, test=False +): + """ + Trigger action callback with the given inputs. If `test` is true, then run + the action callback in testing mode, without actually creating tasks. + """ + graph_config = load_graph_config(root) + graph_config.register() + callbacks = _get_callbacks(graph_config) + cb = callbacks.get(callback, None) + if not cb: + raise Exception( + "Unknown callback: {}. Known callbacks: {}".format( + callback, ", ".join(callbacks) + ) + ) + + if test: + create.testing = True + taskcluster.testing = True + + if not test: + sanity_check_task_scope(callback, parameters, graph_config) + + cb(Parameters(**parameters), graph_config, input, task_group_id, task_id) + + +def _load(graph_config): + # Load all modules from this folder, relying on the side-effects of register_ + # functions to populate the action registry. + import_sibling_modules(exceptions=("util.py",)) + return callbacks, actions + + +def _get_callbacks(graph_config): + return _load(graph_config)[0] + + +def _get_actions(graph_config): + return _load(graph_config)[1] diff --git a/taskcluster/gecko_taskgraph/actions/release_promotion.py b/taskcluster/gecko_taskgraph/actions/release_promotion.py new file mode 100644 index 0000000000..9d6b7ad0b7 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/release_promotion.py @@ -0,0 +1,426 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os + +import requests +from taskgraph.parameters import Parameters +from taskgraph.taskgraph import TaskGraph +from taskgraph.util.taskcluster import get_artifact, list_task_group_incomplete_tasks + +from gecko_taskgraph.actions.registry import register_callback_action +from gecko_taskgraph.decision import taskgraph_decision +from gecko_taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS, release_level +from gecko_taskgraph.util.partials import populate_release_history +from gecko_taskgraph.util.partners import ( + fix_partner_config, + get_partner_config_by_url, + get_partner_url_config, + get_token, +) +from gecko_taskgraph.util.taskgraph import ( + find_decision_task, + find_existing_tasks_from_previous_kinds, +) + +RELEASE_PROMOTION_SIGNOFFS = ("mar-signing",) + + +def is_release_promotion_available(parameters): + return parameters["project"] in RELEASE_PROMOTION_PROJECTS + + +def get_partner_config(partner_url_config, github_token): + partner_config = {} + for kind, url in partner_url_config.items(): + if url: + partner_config[kind] = get_partner_config_by_url(url, kind, github_token) + return partner_config + + +def get_signoff_properties(): + props = {} + for signoff in RELEASE_PROMOTION_SIGNOFFS: + props[signoff] = { + "type": "string", + } + return props + + +def get_required_signoffs(input, parameters): + input_signoffs = set(input.get("required_signoffs", [])) + params_signoffs = set(parameters["required_signoffs"] or []) + return sorted(list(input_signoffs | params_signoffs)) + + +def get_signoff_urls(input, parameters): + signoff_urls = parameters["signoff_urls"] + signoff_urls.update(input.get("signoff_urls", {})) + return signoff_urls + + +def get_flavors(graph_config, param): + """ + Get all flavors with the given parameter enabled. + """ + promotion_flavors = graph_config["release-promotion"]["flavors"] + return sorted( + flavor + for (flavor, config) in promotion_flavors.items() + if config.get(param, False) + ) + + +@register_callback_action( + name="release-promotion", + title="Release Promotion", + symbol="${input.release_promotion_flavor}", + description="Promote a release.", + permission="release-promotion", + order=500, + context=[], + available=is_release_promotion_available, + schema=lambda graph_config: { + "type": "object", + "properties": { + "build_number": { + "type": "integer", + "default": 1, + "minimum": 1, + "title": "The release build number", + "description": ( + "The release build number. Starts at 1 per " + "release version, and increments on rebuild." + ), + }, + "do_not_optimize": { + "type": "array", + "description": ( + "Optional: a list of labels to avoid optimizing out " + "of the graph (to force a rerun of, say, " + "funsize docker-image tasks)." + ), + "items": { + "type": "string", + }, + }, + "revision": { + "type": "string", + "title": "Optional: revision to promote", + "description": ( + "Optional: the revision to promote. If specified, " + "and `previous_graph_kinds is not specified, find the " + "push graph to promote based on the revision." + ), + }, + "release_promotion_flavor": { + "type": "string", + "description": "The flavor of release promotion to perform.", + "default": "FILL ME OUT", + "enum": sorted(graph_config["release-promotion"]["flavors"].keys()), + }, + "rebuild_kinds": { + "type": "array", + "description": ( + "Optional: an array of kinds to ignore from the previous " + "graph(s)." + ), + "items": { + "type": "string", + }, + }, + "previous_graph_ids": { + "type": "array", + "description": ( + "Optional: an array of taskIds of decision or action " + "tasks from the previous graph(s) to use to populate " + "our `previous_graph_kinds`." + ), + "items": { + "type": "string", + }, + }, + "version": { + "type": "string", + "description": ( + "Optional: override the version for release promotion. " + "Occasionally we'll land a taskgraph fix in a later " + "commit, but want to act on a build from a previous " + "commit. If a version bump has landed in the meantime, " + "relying on the in-tree version will break things." + ), + "default": "", + }, + "next_version": { + "type": "string", + "description": ( + "Next version. Required in the following flavors: " + "{}".format(get_flavors(graph_config, "version-bump")) + ), + "default": "", + }, + # Example: + # 'partial_updates': { + # '38.0': { + # 'buildNumber': 1, + # 'locales': ['de', 'en-GB', 'ru', 'uk', 'zh-TW'] + # }, + # '37.0': { + # 'buildNumber': 2, + # 'locales': ['de', 'en-GB', 'ru', 'uk'] + # } + # } + "partial_updates": { + "type": "object", + "description": ( + "Partial updates. Required in the following flavors: " + "{}".format(get_flavors(graph_config, "partial-updates")) + ), + "default": {}, + "additionalProperties": { + "type": "object", + "properties": { + "buildNumber": { + "type": "number", + }, + "locales": { + "type": "array", + "items": { + "type": "string", + }, + }, + }, + "required": [ + "buildNumber", + "locales", + ], + "additionalProperties": False, + }, + }, + "release_eta": { + "type": "string", + "default": "", + }, + "release_enable_partner_repack": { + "type": "boolean", + "default": False, + "description": "Toggle for creating partner repacks", + }, + "release_enable_partner_attribution": { + "type": "boolean", + "default": False, + "description": "Toggle for creating partner attribution", + }, + "release_partner_build_number": { + "type": "integer", + "default": 1, + "minimum": 1, + "description": ( + "The partner build number. This translates to, e.g. " + "`v1` in the path. We generally only have to " + "bump this on off-cycle partner rebuilds." + ), + }, + "release_partners": { + "type": "array", + "description": ( + "A list of partners to repack, or if null or empty then use " + "the current full set" + ), + "items": { + "type": "string", + }, + }, + "release_partner_config": { + "type": "object", + "description": "Partner configuration to use for partner repacks.", + "properties": {}, + "additionalProperties": True, + }, + "release_enable_emefree": { + "type": "boolean", + "default": False, + "description": "Toggle for creating EME-free repacks", + }, + "required_signoffs": { + "type": "array", + "description": ("The flavor of release promotion to perform."), + "items": { + "enum": RELEASE_PROMOTION_SIGNOFFS, + }, + }, + "signoff_urls": { + "type": "object", + "default": {}, + "additionalProperties": False, + "properties": get_signoff_properties(), + }, + }, + "required": ["release_promotion_flavor", "build_number"], + }, +) +def release_promotion_action(parameters, graph_config, input, task_group_id, task_id): + release_promotion_flavor = input["release_promotion_flavor"] + promotion_config = graph_config["release-promotion"]["flavors"][ + release_promotion_flavor + ] + release_history = {} + product = promotion_config["product"] + + next_version = str(input.get("next_version") or "") + if promotion_config.get("version-bump", False): + # We force str() the input, hence the 'None' + if next_version in ["", "None"]: + raise Exception( + "`next_version` property needs to be provided for `{}` " + "target.".format(release_promotion_flavor) + ) + + if promotion_config.get("partial-updates", False): + partial_updates = input.get("partial_updates", {}) + if not partial_updates and release_level(parameters["project"]) == "production": + raise Exception( + "`partial_updates` property needs to be provided for `{}`" + "target.".format(release_promotion_flavor) + ) + balrog_prefix = product.title() + os.environ["PARTIAL_UPDATES"] = json.dumps(partial_updates, sort_keys=True) + release_history = populate_release_history( + balrog_prefix, parameters["project"], partial_updates=partial_updates + ) + + target_tasks_method = promotion_config["target-tasks-method"].format( + project=parameters["project"] + ) + rebuild_kinds = input.get( + "rebuild_kinds", promotion_config.get("rebuild-kinds", []) + ) + do_not_optimize = input.get( + "do_not_optimize", promotion_config.get("do-not-optimize", []) + ) + + # Make sure no pending tasks remain from a previous run + own_task_id = os.environ.get("TASK_ID", "") + try: + for t in list_task_group_incomplete_tasks(own_task_id): + if t == own_task_id: + continue + raise Exception( + "task group has unexpected pre-existing incomplete tasks (e.g. {})".format( + t + ) + ) + except requests.exceptions.HTTPError as e: + # 404 means the task group doesn't exist yet, and we're fine + if e.response.status_code != 404: + raise + + # Build previous_graph_ids from ``previous_graph_ids``, ``revision``, + # or the action parameters. + previous_graph_ids = input.get("previous_graph_ids") + if not previous_graph_ids: + revision = input.get("revision") + if revision: + head_rev_param = "{}head_rev".format( + graph_config["project-repo-param-prefix"] + ) + push_parameters = { + head_rev_param: revision, + "project": parameters["project"], + } + else: + push_parameters = parameters + previous_graph_ids = [find_decision_task(push_parameters, graph_config)] + + # Download parameters from the first decision task + parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml") + # Download and combine full task graphs from each of the previous_graph_ids. + # Sometimes previous relpro action tasks will add tasks, like partials, + # that didn't exist in the first full_task_graph, so combining them is + # important. The rightmost graph should take precedence in the case of + # conflicts. + combined_full_task_graph = {} + for graph_id in previous_graph_ids: + full_task_graph = get_artifact(graph_id, "public/full-task-graph.json") + combined_full_task_graph.update(full_task_graph) + _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph) + parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds( + combined_full_task_graph, previous_graph_ids, rebuild_kinds + ) + parameters["do_not_optimize"] = do_not_optimize + parameters["target_tasks_method"] = target_tasks_method + parameters["build_number"] = int(input["build_number"]) + parameters["next_version"] = next_version + parameters["release_history"] = release_history + if promotion_config.get("is-rc"): + parameters["release_type"] += "-rc" + parameters["release_eta"] = input.get("release_eta", "") + parameters["release_product"] = product + # When doing staging releases on try, we still want to re-use tasks from + # previous graphs. + parameters["optimize_target_tasks"] = True + + if release_promotion_flavor == "promote_firefox_partner_repack": + release_enable_partner_repack = True + release_enable_partner_attribution = False + release_enable_emefree = False + elif release_promotion_flavor == "promote_firefox_partner_attribution": + release_enable_partner_repack = False + release_enable_partner_attribution = True + release_enable_emefree = False + else: + # for promotion or ship phases, we use the action input to turn the repacks/attribution off + release_enable_partner_repack = input["release_enable_partner_repack"] + release_enable_partner_attribution = input["release_enable_partner_attribution"] + release_enable_emefree = input["release_enable_emefree"] + + partner_url_config = get_partner_url_config(parameters, graph_config) + if ( + release_enable_partner_repack + and not partner_url_config["release-partner-repack"] + ): + raise Exception("Can't enable partner repacks when no config url found") + if ( + release_enable_partner_attribution + and not partner_url_config["release-partner-attribution"] + ): + raise Exception("Can't enable partner attribution when no config url found") + if release_enable_emefree and not partner_url_config["release-eme-free-repack"]: + raise Exception("Can't enable EMEfree repacks when no config url found") + parameters["release_enable_partner_repack"] = release_enable_partner_repack + parameters[ + "release_enable_partner_attribution" + ] = release_enable_partner_attribution + parameters["release_enable_emefree"] = release_enable_emefree + + partner_config = input.get("release_partner_config") + if not partner_config and any( + [ + release_enable_partner_repack, + release_enable_partner_attribution, + release_enable_emefree, + ] + ): + github_token = get_token(parameters) + partner_config = get_partner_config(partner_url_config, github_token) + if partner_config: + parameters["release_partner_config"] = fix_partner_config(partner_config) + parameters["release_partners"] = input.get("release_partners") + if input.get("release_partner_build_number"): + parameters["release_partner_build_number"] = input[ + "release_partner_build_number" + ] + + if input["version"]: + parameters["version"] = input["version"] + + parameters["required_signoffs"] = get_required_signoffs(input, parameters) + parameters["signoff_urls"] = get_signoff_urls(input, parameters) + + # make parameters read-only + parameters = Parameters(**parameters) + + taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters) diff --git a/taskcluster/gecko_taskgraph/actions/retrigger.py b/taskcluster/gecko_taskgraph/actions/retrigger.py new file mode 100644 index 0000000000..fc8c05c83f --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/retrigger.py @@ -0,0 +1,301 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging +import sys +import textwrap + +from taskgraph.util.taskcluster import get_task_definition, rerun_task + +from gecko_taskgraph.util.taskcluster import state_task + +from .registry import register_callback_action +from .util import ( + combine_task_graph_files, + create_task_from_def, + create_tasks, + fetch_graph_and_labels, + get_tasks_with_downstream, + relativize_datestamps, +) + +logger = logging.getLogger(__name__) + +RERUN_STATES = ("exception", "failed") + + +def _should_retrigger(task_graph, label): + """ + Return whether a given task in the taskgraph should be retriggered. + + This handles the case where the task isn't there by assuming it should not be. + """ + if label not in task_graph: + logger.info( + "Task {} not in full taskgraph, assuming task should not be retriggered.".format( + label + ) + ) + return False + return task_graph[label].attributes.get("retrigger", False) + + +@register_callback_action( + title="Retrigger", + name="retrigger", + symbol="rt", + cb_name="retrigger-decision", + description=textwrap.dedent( + """\ + Create a clone of the task (retriggering decision, action, and cron tasks requires + special scopes).""" + ), + order=11, + context=[ + {"kind": "decision-task"}, + {"kind": "action-callback"}, + {"kind": "cron-task"}, + {"action": "backfill-task"}, + ], +) +def retrigger_decision_action(parameters, graph_config, input, task_group_id, task_id): + """For a single task, we try to just run exactly the same task once more. + It's quite possible that we don't have the scopes to do so (especially for + an action), but this is best-effort.""" + + # make all of the timestamps relative; they will then be turned back into + # absolute timestamps relative to the current time. + task = get_task_definition(task_id) + task = relativize_datestamps(task) + create_task_from_def( + task, parameters["level"], action_tag="retrigger-decision-task" + ) + + +@register_callback_action( + title="Retrigger", + name="retrigger", + symbol="rt", + description=("Create a clone of the task."), + order=19, # must be greater than other orders in this file, as this is the fallback version + context=[{"retrigger": "true"}], + schema={ + "type": "object", + "properties": { + "downstream": { + "type": "boolean", + "description": ( + "If true, downstream tasks from this one will be cloned as well. " + "The dependencies will be updated to work with the new task at the root." + ), + "default": False, + }, + "times": { + "type": "integer", + "default": 1, + "minimum": 1, + "maximum": 100, + "title": "Times", + "description": "How many times to run each task.", + }, + }, + }, +) +@register_callback_action( + title="Retrigger (disabled)", + name="retrigger", + cb_name="retrigger-disabled", + symbol="rt", + description=( + "Create a clone of the task.\n\n" + "This type of task should typically be re-run instead of re-triggered." + ), + order=20, # must be greater than other orders in this file, as this is the fallback version + context=[{}], + schema={ + "type": "object", + "properties": { + "downstream": { + "type": "boolean", + "description": ( + "If true, downstream tasks from this one will be cloned as well. " + "The dependencies will be updated to work with the new task at the root." + ), + "default": False, + }, + "times": { + "type": "integer", + "default": 1, + "minimum": 1, + "maximum": 100, + "title": "Times", + "description": "How many times to run each task.", + }, + "force": { + "type": "boolean", + "default": False, + "description": ( + "This task should not be re-triggered. " + "This can be overridden by passing `true` here." + ), + }, + }, + }, +) +def retrigger_action(parameters, graph_config, input, task_group_id, task_id): + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + + task = get_task_definition(task_id) + label = task["metadata"]["name"] + + with_downstream = " " + to_run = [label] + + if not input.get("force", None) and not _should_retrigger(full_task_graph, label): + logger.info( + "Not retriggering task {}, task should not be retrigged " + "and force not specified.".format(label) + ) + sys.exit(1) + + if input.get("downstream"): + to_run = get_tasks_with_downstream(to_run, full_task_graph, label_to_taskid) + with_downstream = " (with downstream) " + + times = input.get("times", 1) + for i in range(times): + create_tasks( + graph_config, + to_run, + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + i, + action_tag="retrigger-task", + ) + + logger.info(f"Scheduled {label}{with_downstream}(time {i + 1}/{times})") + combine_task_graph_files(list(range(times))) + + +@register_callback_action( + title="Rerun", + name="rerun", + symbol="rr", + description=( + "Rerun a task.\n\n" + "This only works on failed or exception tasks in the original taskgraph," + " and is CoT friendly." + ), + order=300, + context=[{}], + schema={"type": "object", "properties": {}}, +) +def rerun_action(parameters, graph_config, input, task_group_id, task_id): + task = get_task_definition(task_id) + parameters = dict(parameters) + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + label = task["metadata"]["name"] + if task_id not in label_to_taskid.values(): + logger.error( + "Refusing to rerun {}: taskId {} not in decision task {} label_to_taskid!".format( + label, task_id, decision_task_id + ) + ) + + _rerun_task(task_id, label) + + +def _rerun_task(task_id, label): + state = state_task(task_id) + if state not in RERUN_STATES: + logger.warning( + "No need to rerun {}: state '{}' not in {}!".format( + label, state, RERUN_STATES + ) + ) + return + rerun_task(task_id) + logger.info(f"Reran {label}") + + +@register_callback_action( + title="Retrigger", + name="retrigger-multiple", + symbol="rt", + description=("Create a clone of the task."), + context=[], + schema={ + "type": "object", + "properties": { + "requests": { + "type": "array", + "items": { + "tasks": { + "type": "array", + "description": "An array of task labels", + "items": {"type": "string"}, + }, + "times": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "title": "Times", + "description": "How many times to run each task.", + }, + "additionalProperties": False, + }, + }, + "additionalProperties": False, + }, + }, +) +def retrigger_multiple(parameters, graph_config, input, task_group_id, task_id): + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + + suffixes = [] + for i, request in enumerate(input.get("requests", [])): + times = request.get("times", 1) + rerun_tasks = [ + label + for label in request.get("tasks") + if not _should_retrigger(full_task_graph, label) + ] + retrigger_tasks = [ + label + for label in request.get("tasks") + if _should_retrigger(full_task_graph, label) + ] + + for label in rerun_tasks: + # XXX we should not re-run tasks pulled in from other pushes + # In practice, this shouldn't matter, as only completed tasks + # are pulled in from other pushes and treeherder won't pass + # those labels. + _rerun_task(label_to_taskid[label], label) + + for j in range(times): + suffix = f"{i}-{j}" + suffixes.append(suffix) + create_tasks( + graph_config, + retrigger_tasks, + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + suffix, + action_tag="retrigger-multiple-task", + ) + + if suffixes: + combine_task_graph_files(suffixes) diff --git a/taskcluster/gecko_taskgraph/actions/retrigger_custom.py b/taskcluster/gecko_taskgraph/actions/retrigger_custom.py new file mode 100644 index 0000000000..677cb00a11 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/retrigger_custom.py @@ -0,0 +1,185 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import logging + +from taskgraph.util.parameterization import resolve_task_references +from taskgraph.util.taskcluster import get_task_definition + +from .registry import register_callback_action +from .util import create_task_from_def, fetch_graph_and_labels + +logger = logging.getLogger(__name__) + +# Properties available for custom retrigger of any supported test suites +basic_properties = { + "path": { + "type": "string", + "maxLength": 255, + "default": "", + "title": "Path name", + "description": "Path of test(s) to retrigger", + }, + "logLevel": { + "type": "string", + "enum": ["debug", "info", "warning", "error", "critical"], + "default": "info", + "title": "Log level", + "description": "Log level for output (INFO is normal, DEBUG gives more detail)", + }, + "environment": { + "type": "object", + "default": {"MOZ_LOG": ""}, + "title": "Extra environment variables", + "description": "Extra environment variables to use for this run", + "additionalProperties": {"type": "string"}, + }, +} + +# Additional properties available for custom retrigger of some additional test suites +extended_properties = basic_properties.copy() +extended_properties.update( + { + "runUntilFail": { + "type": "boolean", + "default": False, + "title": "Run until failure", + "description": ( + "Runs the specified set of tests repeatedly " + "until failure (up to REPEAT times)" + ), + }, + "repeat": { + "type": "integer", + "default": 0, + "minimum": 0, + "title": "Repeat test(s) N times", + "description": ( + "Run test(s) repeatedly (usually used in " + "conjunction with runUntilFail)" + ), + }, + "preferences": { + "type": "object", + "default": {"remote.log.level": "Info"}, + "title": "Extra gecko (about:config) preferences", + "description": "Extra gecko (about:config) preferences to use for this run", + "additionalProperties": {"type": "string"}, + }, + } +) + + +@register_callback_action( + name="retrigger-custom", + title="Retrigger task with custom parameters", + symbol="rt", + description="Retriggers the specified task with custom environment and parameters", + context=[ + {"test-type": "mochitest", "worker-implementation": "docker-worker"}, + {"test-type": "reftest", "worker-implementation": "docker-worker"}, + {"test-type": "geckoview-junit", "worker-implementation": "docker-worker"}, + ], + order=10, + schema={ + "type": "object", + "properties": extended_properties, + "additionalProperties": False, + "required": ["path"], + }, +) +def extended_custom_retrigger_action( + parameters, graph_config, input, task_group_id, task_id +): + handle_custom_retrigger(parameters, graph_config, input, task_group_id, task_id) + + +@register_callback_action( + name="retrigger-custom (gtest)", + title="Retrigger gtest task with custom parameters", + symbol="rt", + description="Retriggers the specified task with custom environment and parameters", + context=[{"test-type": "gtest", "worker-implementation": "docker-worker"}], + order=10, + schema={ + "type": "object", + "properties": basic_properties, + "additionalProperties": False, + "required": ["path"], + }, +) +def basic_custom_retrigger_action_basic( + parameters, graph_config, input, task_group_id, task_id +): + handle_custom_retrigger(parameters, graph_config, input, task_group_id, task_id) + + +def handle_custom_retrigger(parameters, graph_config, input, task_group_id, task_id): + task = get_task_definition(task_id) + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + + pre_task = full_task_graph.tasks[task["metadata"]["name"]] + + # fix up the task's dependencies, similar to how optimization would + # have done in the decision + dependencies = { + name: label_to_taskid[label] for name, label in pre_task.dependencies.items() + } + new_task_definition = resolve_task_references( + pre_task.label, pre_task.task, task_id, decision_task_id, dependencies + ) + new_task_definition.setdefault("dependencies", []).extend(dependencies.values()) + + # don't want to run mozharness tests, want a custom mach command instead + new_task_definition["payload"]["command"] += ["--no-run-tests"] + + custom_mach_command = [task["tags"]["test-type"]] + + # mochitests may specify a flavor + if new_task_definition["payload"]["env"].get("MOCHITEST_FLAVOR"): + custom_mach_command += [ + "--keep-open=false", + "-f", + new_task_definition["payload"]["env"]["MOCHITEST_FLAVOR"], + ] + + enable_e10s = json.loads( + new_task_definition["payload"]["env"].get("ENABLE_E10S", "true") + ) + if not enable_e10s: + custom_mach_command += ["--disable-e10s"] + + custom_mach_command += [ + "--log-tbpl=-", + "--log-tbpl-level={}".format(input.get("logLevel", "debug")), + ] + if input.get("runUntilFail"): + custom_mach_command += ["--run-until-failure"] + if input.get("repeat"): + custom_mach_command += ["--repeat", str(input.get("repeat", 30))] + + # add any custom gecko preferences + for (key, val) in input.get("preferences", {}).items(): + custom_mach_command += ["--setpref", f"{key}={val}"] + + custom_mach_command += [input["path"]] + new_task_definition["payload"]["env"]["CUSTOM_MACH_COMMAND"] = " ".join( + custom_mach_command + ) + + # update environment + new_task_definition["payload"]["env"].update(input.get("environment", {})) + + # tweak the treeherder symbol + new_task_definition["extra"]["treeherder"]["symbol"] += "-custom" + + logging.info("New task definition: %s", new_task_definition) + + create_task_from_def( + new_task_definition, parameters["level"], action_tag="retrigger-custom-task" + ) diff --git a/taskcluster/gecko_taskgraph/actions/run_missing_tests.py b/taskcluster/gecko_taskgraph/actions/run_missing_tests.py new file mode 100644 index 0000000000..92310445f3 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/run_missing_tests.py @@ -0,0 +1,62 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +from taskgraph.util.taskcluster import get_artifact + +from .registry import register_callback_action +from .util import create_tasks, fetch_graph_and_labels + +logger = logging.getLogger(__name__) + + +@register_callback_action( + name="run-missing-tests", + title="Run Missing Tests", + symbol="rmt", + description=( + "Run tests in the selected push that were optimized away, usually by SETA." + "\n" + "This action is for use on pushes that will be merged into another branch," + "to check that optimization hasn't hidden any failures." + ), + order=250, + context=[], # Applies to decision task +) +def run_missing_tests(parameters, graph_config, input, task_group_id, task_id): + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + target_tasks = get_artifact(decision_task_id, "public/target-tasks.json") + + # The idea here is to schedule all tasks of the `test` kind that were + # targetted but did not appear in the final task-graph -- those were the + # optimized tasks. + to_run = [] + already_run = 0 + for label in target_tasks: + task = full_task_graph.tasks[label] + if task.kind != "test": + continue # not a test + if label in label_to_taskid: + already_run += 1 + continue + to_run.append(label) + + create_tasks( + graph_config, + to_run, + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + ) + + logger.info( + "Out of {} test tasks, {} already existed and the action created {}".format( + already_run + len(to_run), already_run, len(to_run) + ) + ) diff --git a/taskcluster/gecko_taskgraph/actions/scriptworker_canary.py b/taskcluster/gecko_taskgraph/actions/scriptworker_canary.py new file mode 100644 index 0000000000..e0057da9a6 --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/scriptworker_canary.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.parameters import Parameters + +from gecko_taskgraph.actions.registry import register_callback_action +from gecko_taskgraph.decision import taskgraph_decision + + +@register_callback_action( + title="Push scriptworker canaries.", + name="scriptworker-canary", + symbol="scriptworker-canary", + description="Trigger scriptworker-canary pushes for the given scriptworkers.", + schema={ + "type": "object", + "properties": { + "scriptworkers": { + "type": "array", + "description": "List of scriptworker types to run canaries for.", + "items": {"type": "string"}, + }, + }, + }, + order=1000, + permission="scriptworker-canary", + context=[], +) +def scriptworker_canary(parameters, graph_config, input, task_group_id, task_id): + scriptworkers = input["scriptworkers"] + + # make parameters read-write + parameters = dict(parameters) + + parameters["target_tasks_method"] = "scriptworker_canary" + parameters["try_task_config"] = { + "scriptworker-canary-workers": scriptworkers, + } + parameters["tasks_for"] = "action" + + # make parameters read-only + parameters = Parameters(**parameters) + + taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters) diff --git a/taskcluster/gecko_taskgraph/actions/side_by_side.py b/taskcluster/gecko_taskgraph/actions/side_by_side.py new file mode 100644 index 0000000000..816e716fde --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/side_by_side.py @@ -0,0 +1,189 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging +import os +import sys +from functools import partial + +from taskgraph.util.taskcluster import get_artifact, get_task_definition + +from ..util.taskcluster import list_task_group_complete_tasks +from .registry import register_callback_action +from .util import create_tasks, fetch_graph_and_labels, get_decision_task_id, get_pushes + +logger = logging.getLogger(__name__) + + +def input_for_support_action(revision, base_revision, base_branch, task): + """Generate input for action to be scheduled. + + Define what label to schedule with 'label'. + If it is a test task that uses explicit manifests add that information. + """ + platform, test_name = task["metadata"]["name"].split("/opt-") + new_branch = os.environ.get("GECKO_HEAD_REPOSITORY", "/try").split("/")[-1] + symbol = task["extra"]["treeherder"]["symbol"] + input = { + "label": "perftest-linux-side-by-side", + "symbol": symbol, + "new_revision": revision, + "base_revision": base_revision, + "test_name": test_name, + "platform": platform, + "base_branch": base_branch, + "new_branch": new_branch, + } + + return input + + +def side_by_side_modifier(task, input): + if task.label != input["label"]: + return task + + # Make side-by-side job searchable by the platform, test name, and revisions + # it was triggered for + task.task["metadata"][ + "name" + ] = f"{input['platform']} {input['test_name']} {input['base_revision'][:12]} {input['new_revision'][:12]}" + # Use a job symbol to include the symbol of the job the side-by-side + # is running for + task.task["extra"]["treeherder"]["symbol"] += f"-{input['symbol']}" + + cmd = task.task["payload"]["command"] + task.task["payload"]["command"][1][-1] = cmd[1][-1].format(**input) + + return task + + +@register_callback_action( + title="Side by side", + name="side-by-side", + symbol="gen-sxs", + description=( + "Given a performance test pageload job generate a side-by-side comparison against" + "the pageload job from the revision at the input." + ), + order=200, + context=[{"test-type": "raptor"}], + schema={ + "type": "object", + "properties": { + "revision": { + "type": "string", + "default": "", + "description": "Revision of the push against the comparison is wanted.", + }, + "project": { + "type": "string", + "default": "autoland", + "description": "Revision of the push against the comparison is wanted.", + }, + }, + "additionalProperties": False, + }, +) +def side_by_side_action(parameters, graph_config, input, task_group_id, task_id): + """ + This action does a side-by-side comparison between current revision and + the revision entered manually or the latest revision that ran the + pageload job (via support action). + + To execute this action locally follow the documentation here: + https://firefox-source-docs.mozilla.org/taskcluster/actions.html#testing-the-action-locally + """ + task = get_task_definition(task_id) + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + # TODO: find another way to detect side-by-side comparable jobs + # (potentially lookig at the visual metrics flag) + if not ( + "browsertime-tp6" in task["metadata"]["name"] + or "welcome" in task["metadata"]["name"] + ): + logger.exception( + f"Task {task['metadata']['name']} is not side-by-side comparable." + ) + return + + failed = False + input_for_action = {} + + if input.get("revision"): + # If base_revision was introduced manually, use that + input_for_action = input_for_support_action( + revision=parameters["head_rev"], + base_revision=input.get("revision"), + base_branch=input.get("project"), + task=task, + ) + else: + current_push_id = int(parameters["pushlog_id"]) - 1 + # Go decrementally through pushlog_id, get push data, decision task id, + # full task graph and everything needed to find which of the past revisions + # ran the pageload job to compare against + while int(parameters["pushlog_id"]) - current_push_id < 30: + pushes = get_pushes( + project=parameters["head_repository"], + end_id=current_push_id, + depth=1, + full_response=True, + ) + try: + # Get label-to-taskid.json artifact + the tasks triggered + # by the action tasks at a later time than the decision task + current_decision_task_id = get_decision_task_id( + parameters["project"], current_push_id + ) + current_task_group_id = get_task_definition(current_decision_task_id)[ + "taskGroupId" + ] + current_label_to_taskid = get_artifact( + current_decision_task_id, "public/label-to-taskid.json" + ) + current_full_label_to_taskid = current_label_to_taskid.copy() + action_task_triggered = list_task_group_complete_tasks( + current_task_group_id + ) + current_full_label_to_taskid.update(action_task_triggered) + if task["metadata"]["name"] in current_full_label_to_taskid.keys(): + input_for_action = input_for_support_action( + revision=parameters["head_rev"], + base_revision=pushes[str(current_push_id)]["changesets"][-1], + base_branch=input.get("project", parameters["project"]), + task=task, + ) + break + except Exception: + logger.warning( + f"Could not find decision task for push {current_push_id}" + ) + # The decision task may have failed, this is common enough that we + # don't want to report an error for it. + continue + current_push_id -= 1 + if not input_for_action: + raise Exception( + "Could not find a side-by-side comparable task within a depth of 30 revisions." + ) + + try: + create_tasks( + graph_config, + [input_for_action["label"]], + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + modifier=partial(side_by_side_modifier, input=input_for_action), + ) + except Exception as e: + logger.exception(f"Failed to trigger action: {e}.") + failed = True + + if failed: + sys.exit(1) diff --git a/taskcluster/gecko_taskgraph/actions/util.py b/taskcluster/gecko_taskgraph/actions/util.py new file mode 100644 index 0000000000..76181dae2b --- /dev/null +++ b/taskcluster/gecko_taskgraph/actions/util.py @@ -0,0 +1,433 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import concurrent.futures as futures +import copy +import logging +import os +import re +from functools import reduce + +import jsone +import requests +from requests.exceptions import HTTPError +from slugid import nice as slugid +from taskgraph import create +from taskgraph.optimize.base import optimize_task_graph +from taskgraph.taskgraph import TaskGraph +from taskgraph.util.taskcluster import ( + CONCURRENCY, + find_task_id, + get_artifact, + get_session, + get_task_definition, + list_tasks, + parse_time, +) + +from gecko_taskgraph.decision import read_artifact, rename_artifact, write_artifact +from gecko_taskgraph.util.taskcluster import trigger_hook +from gecko_taskgraph.util.taskgraph import find_decision_task + +logger = logging.getLogger(__name__) + +INDEX_TMPL = "gecko.v2.{}.pushlog-id.{}.decision" +PUSHLOG_TMPL = "{}/json-pushes?version=2&startID={}&endID={}" + + +def _tags_within_context(tags, context=[]): + """A context of [] means that it *only* applies to a task group""" + return any( + all(tag in tags and tags[tag] == tag_set[tag] for tag in tag_set.keys()) + for tag_set in context + ) + + +def _extract_applicable_action(actions_json, action_name, task_group_id, task_id): + """Extract action that applies to the given task or task group. + + A task (as defined by its tags) is said to match a tag-set if its + tags are a super-set of the tag-set. A tag-set is a set of key-value pairs. + + An action (as defined by its context) is said to be relevant for + a given task, if the task's tags match one of the tag-sets given + in the context property of the action. + + The order of the actions is significant. When multiple actions apply to a + task the first one takes precedence. + + For more details visit: + https://docs.taskcluster.net/docs/manual/design/conventions/actions/spec + """ + if task_id: + tags = get_task_definition(task_id).get("tags") + + for _action in actions_json["actions"]: + if action_name != _action["name"]: + continue + + context = _action.get("context", []) + # Ensure the task is within the context of the action + if task_id and tags and _tags_within_context(tags, context): + return _action + if context == []: + return _action + + available_actions = ", ".join(sorted({a["name"] for a in actions_json["actions"]})) + raise LookupError( + "{} action is not available for this task. Available: {}".format( + action_name, available_actions + ) + ) + + +def trigger_action(action_name, decision_task_id, task_id=None, input={}): + if not decision_task_id: + raise ValueError("No decision task. We can't find the actions artifact.") + actions_json = get_artifact(decision_task_id, "public/actions.json") + if actions_json["version"] != 1: + raise RuntimeError("Wrong version of actions.json, unable to continue") + + # These values substitute $eval in the template + context = { + "input": input, + "taskId": task_id, + "taskGroupId": decision_task_id, + } + # https://docs.taskcluster.net/docs/manual/design/conventions/actions/spec#variables + context.update(actions_json["variables"]) + action = _extract_applicable_action( + actions_json, action_name, decision_task_id, task_id + ) + kind = action["kind"] + if create.testing: + logger.info(f"Skipped triggering action for {kind} as testing is enabled") + elif kind == "hook": + hook_payload = jsone.render(action["hookPayload"], context) + trigger_hook(action["hookGroupId"], action["hookId"], hook_payload) + else: + raise NotImplementedError(f"Unable to submit actions with {kind} kind.") + + +def get_pushes_from_params_input(parameters, input): + inclusive_tweak = 1 if input.get("inclusive") else 0 + return get_pushes( + project=parameters["head_repository"], + end_id=int(parameters["pushlog_id"]) - (1 - inclusive_tweak), + depth=input.get("depth", 9) + inclusive_tweak, + ) + + +def get_pushes(project, end_id, depth, full_response=False): + pushes = [] + while True: + start_id = max(end_id - depth, 0) + pushlog_url = PUSHLOG_TMPL.format(project, start_id, end_id) + logger.debug(pushlog_url) + r = requests.get(pushlog_url) + r.raise_for_status() + pushes = pushes + list(r.json()["pushes"].keys()) + if len(pushes) >= depth: + break + + end_id = start_id - 1 + start_id -= depth + if start_id < 0: + break + + pushes = sorted(pushes)[-depth:] + push_dict = {push: r.json()["pushes"][push] for push in pushes} + return push_dict if full_response else pushes + + +def get_decision_task_id(project, push_id): + return find_task_id(INDEX_TMPL.format(project, push_id)) + + +def get_parameters(decision_task_id): + return get_artifact(decision_task_id, "public/parameters.yml") + + +def get_tasks_with_downstream(labels, full_task_graph, label_to_taskid): + # Used to gather tasks when downstream tasks need to run as well + return full_task_graph.graph.transitive_closure( + set(labels), reverse=True + ).nodes & set(label_to_taskid.keys()) + + +def fetch_graph_and_labels(parameters, graph_config): + decision_task_id = find_decision_task(parameters, graph_config) + + # First grab the graph and labels generated during the initial decision task + full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") + logger.info("Load taskgraph from JSON.") + _, full_task_graph = TaskGraph.from_json(full_task_graph) + label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") + + logger.info("Fetching additional tasks from action and cron tasks.") + # fetch everything in parallel; this avoids serializing any delay in downloading + # each artifact (such as waiting for the artifact to be mirrored locally) + with futures.ThreadPoolExecutor(CONCURRENCY) as e: + fetches = [] + + # fetch any modifications made by action tasks and swap out new tasks + # for old ones + def fetch_action(task_id): + logger.info(f"fetching label-to-taskid.json for action task {task_id}") + try: + run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json") + label_to_taskid.update(run_label_to_id) + except HTTPError as e: + if e.response.status_code != 404: + raise + logger.debug(f"No label-to-taskid.json found for {task_id}: {e}") + + head_rev_param = "{}head_rev".format(graph_config["project-repo-param-prefix"]) + + namespace = "{}.v2.{}.revision.{}.taskgraph.actions".format( + graph_config["trust-domain"], + parameters["project"], + parameters[head_rev_param], + ) + for task_id in list_tasks(namespace): + fetches.append(e.submit(fetch_action, task_id)) + + # Similarly for cron tasks.. + def fetch_cron(task_id): + logger.info(f"fetching label-to-taskid.json for cron task {task_id}") + try: + run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json") + label_to_taskid.update(run_label_to_id) + except HTTPError as e: + if e.response.status_code != 404: + raise + logger.debug(f"No label-to-taskid.json found for {task_id}: {e}") + + namespace = "{}.v2.{}.revision.{}.cron".format( + graph_config["trust-domain"], + parameters["project"], + parameters[head_rev_param], + ) + for task_id in list_tasks(namespace): + fetches.append(e.submit(fetch_cron, task_id)) + + # now wait for each fetch to complete, raising an exception if there + # were any issues + for f in futures.as_completed(fetches): + f.result() + + return (decision_task_id, full_task_graph, label_to_taskid) + + +def create_task_from_def(task_def, level, action_tag=None): + """Create a new task from a definition rather than from a label + that is already in the full-task-graph. The task definition will + have {relative-datestamp': '..'} rendered just like in a decision task. + Use this for entirely new tasks or ones that change internals of the task. + It is useful if you want to "edit" the full_task_graph and then hand + it to this function. No dependencies will be scheduled. You must handle + this yourself. Seeing how create_tasks handles it might prove helpful.""" + task_def["schedulerId"] = f"gecko-level-{level}" + label = task_def["metadata"]["name"] + task_id = slugid() + session = get_session() + if action_tag: + task_def.setdefault("tags", {}).setdefault("action", action_tag) + create.create_task(session, task_id, label, task_def) + + +def update_parent(task, graph): + task.task.setdefault("extra", {})["parent"] = os.environ.get("TASK_ID", "") + return task + + +def update_action_tag(task, graph, action_tag): + task.task.setdefault("tags", {}).setdefault("action", action_tag) + return task + + +def update_dependencies(task, graph): + if os.environ.get("TASK_ID"): + task.task.setdefault("dependencies", []).append(os.environ["TASK_ID"]) + return task + + +def create_tasks( + graph_config, + to_run, + full_task_graph, + label_to_taskid, + params, + decision_task_id, + suffix="", + modifier=lambda t: t, + action_tag=None, +): + """Create new tasks. The task definition will have {relative-datestamp': + '..'} rendered just like in a decision task. Action callbacks should use + this function to create new tasks, + allowing easy debugging with `mach taskgraph action-callback --test`. + This builds up all required tasks to run in order to run the tasks requested. + + Optionally this function takes a `modifier` function that is passed in each + task before it is put into a new graph. It should return a valid task. Note + that this is passed _all_ tasks in the graph, not just the set in to_run. You + may want to skip modifying tasks not in your to_run list. + + If `suffix` is given, then it is used to give unique names to the resulting + artifacts. If you call this function multiple times in the same action, + pass a different suffix each time to avoid overwriting artifacts. + + If you wish to create the tasks in a new group, leave out decision_task_id. + + Returns an updated label_to_taskid containing the new tasks""" + import gecko_taskgraph.optimize # noqa: triggers registration of strategies + + if suffix != "": + suffix = f"-{suffix}" + to_run = set(to_run) + + # Copy to avoid side-effects later + full_task_graph = copy.deepcopy(full_task_graph) + label_to_taskid = label_to_taskid.copy() + + target_graph = full_task_graph.graph.transitive_closure(to_run) + target_task_graph = TaskGraph( + {l: modifier(full_task_graph[l]) for l in target_graph.nodes}, target_graph + ) + target_task_graph.for_each_task(update_parent) + if action_tag: + target_task_graph.for_each_task(update_action_tag, action_tag) + if decision_task_id and decision_task_id != os.environ.get("TASK_ID"): + target_task_graph.for_each_task(update_dependencies) + optimized_task_graph, label_to_taskid = optimize_task_graph( + target_task_graph, + to_run, + params, + to_run, + decision_task_id, + existing_tasks=label_to_taskid, + ) + write_artifact(f"task-graph{suffix}.json", optimized_task_graph.to_json()) + write_artifact(f"label-to-taskid{suffix}.json", label_to_taskid) + write_artifact(f"to-run{suffix}.json", list(to_run)) + create.create_tasks( + graph_config, + optimized_task_graph, + label_to_taskid, + params, + decision_task_id, + ) + return label_to_taskid + + +def _update_reducer(accumulator, new_value): + "similar to set or dict `update` method, but returning the modified object" + accumulator.update(new_value) + return accumulator + + +def combine_task_graph_files(suffixes): + """Combine task-graph-{suffix}.json files into a single task-graph.json file. + + Since Chain of Trust verification requires a task-graph.json file that + contains all children tasks, we can combine the various task-graph-0.json + type files into a master task-graph.json file at the end. + + Actions also look for various artifacts, so we combine those in a similar + fashion. + + In the case where there is only one suffix, we simply rename it to avoid the + additional cost of uploading two copies of the same data. + """ + + if len(suffixes) == 1: + for filename in ["task-graph", "label-to-taskid", "to-run"]: + rename_artifact(f"{filename}-{suffixes[0]}.json", f"{filename}.json") + return + + def combine(file_contents, base): + return reduce(_update_reducer, file_contents, base) + + files = [read_artifact(f"task-graph-{suffix}.json") for suffix in suffixes] + write_artifact("task-graph.json", combine(files, dict())) + + files = [read_artifact(f"label-to-taskid-{suffix}.json") for suffix in suffixes] + write_artifact("label-to-taskid.json", combine(files, dict())) + + files = [read_artifact(f"to-run-{suffix}.json") for suffix in suffixes] + write_artifact("to-run.json", list(combine(files, set()))) + + +def relativize_datestamps(task_def): + """ + Given a task definition as received from the queue, convert all datestamps + to {relative_datestamp: ..} format, with the task creation time as "now". + The result is useful for handing to ``create_task``. + """ + base = parse_time(task_def["created"]) + # borrowed from https://github.com/epoberezkin/ajv/blob/master/lib/compile/formats.js + ts_pattern = re.compile( + r"^\d\d\d\d-[0-1]\d-[0-3]\d[t\s]" + r"(?:[0-2]\d:[0-5]\d:[0-5]\d|23:59:60)(?:\.\d+)?" + r"(?:z|[+-]\d\d:\d\d)$", + re.I, + ) + + def recurse(value): + if isinstance(value, str): + if ts_pattern.match(value): + value = parse_time(value) + diff = value - base + return {"relative-datestamp": f"{int(diff.total_seconds())} seconds"} + if isinstance(value, list): + return [recurse(e) for e in value] + if isinstance(value, dict): + return {k: recurse(v) for k, v in value.items()} + return value + + return recurse(task_def) + + +def add_args_to_command(cmd_parts, extra_args=[]): + """ + Add custom command line args to a given command. + args: + cmd_parts: the raw command as seen by taskcluster + extra_args: array of args we want to add + """ + # Prevent modification of the caller's copy of cmd_parts + cmd_parts = copy.deepcopy(cmd_parts) + cmd_type = "default" + if len(cmd_parts) == 1 and isinstance(cmd_parts[0], dict): + # windows has single cmd part as dict: 'task-reference', with long string + cmd_parts = cmd_parts[0]["task-reference"].split(" ") + cmd_type = "dict" + elif len(cmd_parts) == 1 and isinstance(cmd_parts[0], str): + # windows has single cmd part as a long string + cmd_parts = cmd_parts[0].split(" ") + cmd_type = "unicode" + elif len(cmd_parts) == 1 and isinstance(cmd_parts[0], list): + # osx has an single value array with an array inside + cmd_parts = cmd_parts[0] + cmd_type = "subarray" + elif len(cmd_parts) == 2 and isinstance(cmd_parts[1], list): + # osx has an double value array with an array inside each element. + # The first element is a pre-requisite command while the second + # is the actual test command. + cmd_type = "subarray2" + + if cmd_type == "subarray2": + cmd_parts[1].extend(extra_args) + else: + cmd_parts.extend(extra_args) + + if cmd_type == "dict": + cmd_parts = [{"task-reference": " ".join(cmd_parts)}] + elif cmd_type == "unicode": + cmd_parts = [" ".join(cmd_parts)] + elif cmd_type == "subarray": + cmd_parts = [cmd_parts] + return cmd_parts diff --git a/taskcluster/gecko_taskgraph/config.py b/taskcluster/gecko_taskgraph/config.py new file mode 100644 index 0000000000..62c261647a --- /dev/null +++ b/taskcluster/gecko_taskgraph/config.py @@ -0,0 +1,122 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.util.schema import Schema, optionally_keyed_by +from voluptuous import Any, Optional, Required + +graph_config_schema = Schema( + { + # The trust-domain for this graph. + # (See https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/taskgraph.html#taskgraph-trust-domain) # noqa + Required("trust-domain"): str, + # This specifes the prefix for repo parameters that refer to the project being built. + # This selects between `head_rev` and `comm_head_rev` and related paramters. + # (See http://firefox-source-docs.mozilla.org/taskcluster/taskcluster/parameters.html#push-information # noqa + # and http://firefox-source-docs.mozilla.org/taskcluster/taskcluster/parameters.html#comm-push-information) # noqa + Required("project-repo-param-prefix"): str, + # This specifies the top level directory of the application being built. + # ie. "browser/" for Firefox, "comm/mail/" for Thunderbird. + Required("product-dir"): str, + Required("treeherder"): { + # Mapping of treeherder group symbols to descriptive names + Required("group-names"): {str: str} + }, + Required("index"): {Required("products"): [str]}, + Required("try"): { + # We have a few platforms for which we want to do some "extra" builds, or at + # least build-ish things. Sort of. Anyway, these other things are implemented + # as different "platforms". These do *not* automatically ride along with "-p + # all" + Required("ridealong-builds"): {str: [str]}, + }, + Required("release-promotion"): { + Required("products"): [str], + Required("flavors"): { + str: { + Required("product"): str, + Required("target-tasks-method"): str, + Optional("is-rc"): bool, + Optional("rebuild-kinds"): [str], + Optional("version-bump"): bool, + Optional("partial-updates"): bool, + } + }, + }, + Required("merge-automation"): { + Required("behaviors"): { + str: { + Optional("from-branch"): str, + Required("to-branch"): str, + Optional("from-repo"): str, + Required("to-repo"): str, + Required("version-files"): [ + { + Required("filename"): str, + Optional("new-suffix"): str, + Optional("version-bump"): Any("major", "minor"), + } + ], + Required("replacements"): [[str]], + Required("merge-old-head"): bool, + Optional("base-tag"): str, + Optional("end-tag"): str, + Optional("fetch-version-from"): str, + } + }, + }, + Required("scriptworker"): { + # Prefix to add to scopes controlling scriptworkers + Required("scope-prefix"): str, + }, + Required("task-priority"): optionally_keyed_by( + "project", + Any( + "highest", + "very-high", + "high", + "medium", + "low", + "very-low", + "lowest", + ), + ), + Required("partner-urls"): { + Required("release-partner-repack"): optionally_keyed_by( + "release-product", "release-level", "release-type", Any(str, None) + ), + Optional("release-partner-attribution"): optionally_keyed_by( + "release-product", "release-level", "release-type", Any(str, None) + ), + Required("release-eme-free-repack"): optionally_keyed_by( + "release-product", "release-level", "release-type", Any(str, None) + ), + }, + Required("workers"): { + Required("aliases"): { + str: { + Required("provisioner"): optionally_keyed_by("level", str), + Required("implementation"): str, + Required("os"): str, + Required("worker-type"): optionally_keyed_by( + "level", "release-level", "project", str + ), + } + }, + }, + Required("mac-notarization"): { + Required("mac-entitlements"): optionally_keyed_by( + "platform", "release-level", str + ), + Required("mac-requirements"): optionally_keyed_by("platform", str), + }, + Required("taskgraph"): { + Optional( + "register", + description="Python function to call to register extensions.", + ): str, + Optional("decision-parameters"): str, + }, + Required("expiration-policy"): optionally_keyed_by("project", {str: str}), + } +) diff --git a/taskcluster/gecko_taskgraph/decision.py b/taskcluster/gecko_taskgraph/decision.py new file mode 100644 index 0000000000..1849770481 --- /dev/null +++ b/taskcluster/gecko_taskgraph/decision.py @@ -0,0 +1,565 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import logging +import os +import shutil +import sys +import time +from collections import defaultdict + +import yaml +from redo import retry +from taskgraph import create +from taskgraph.create import create_tasks + +# TODO: Let standalone taskgraph generate parameters instead of calling internals +from taskgraph.decision import ( + _determine_more_accurate_base_ref, + _determine_more_accurate_base_rev, + _get_env_prefix, +) +from taskgraph.generator import TaskGraphGenerator +from taskgraph.parameters import Parameters +from taskgraph.taskgraph import TaskGraph +from taskgraph.util.python_path import find_object +from taskgraph.util.schema import Schema, validate_schema +from taskgraph.util.taskcluster import get_artifact +from taskgraph.util.vcs import get_repository +from taskgraph.util.yaml import load_yaml +from voluptuous import Any, Optional, Required + +from . import GECKO +from .actions import render_actions_json +from .parameters import get_app_version, get_version +from .try_option_syntax import parse_message +from .util.backstop import BACKSTOP_INDEX, is_backstop +from .util.bugbug import push_schedules +from .util.chunking import resolver +from .util.hg import get_hg_commit_message, get_hg_revision_branch +from .util.partials import populate_release_history +from .util.taskcluster import insert_index +from .util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds + +logger = logging.getLogger(__name__) + +ARTIFACTS_DIR = "artifacts" + +# For each project, this gives a set of parameters specific to the project. +# See `taskcluster/docs/parameters.rst` for information on parameters. +PER_PROJECT_PARAMETERS = { + "try": { + "enable_always_target": True, + "target_tasks_method": "try_tasks", + }, + "kaios-try": { + "target_tasks_method": "try_tasks", + }, + "ash": { + "target_tasks_method": "default", + }, + "cedar": { + "target_tasks_method": "default", + }, + "holly": { + "enable_always_target": True, + "target_tasks_method": "holly_tasks", + }, + "oak": { + "target_tasks_method": "default", + "release_type": "nightly-oak", + }, + "graphics": { + "target_tasks_method": "graphics_tasks", + }, + "autoland": { + "optimize_strategies": "gecko_taskgraph.optimize:project.autoland", + "target_tasks_method": "autoland_tasks", + "test_manifest_loader": "bugbug", # Remove this line to disable "manifest scheduling". + }, + "mozilla-central": { + "target_tasks_method": "mozilla_central_tasks", + "release_type": "nightly", + }, + "mozilla-beta": { + "target_tasks_method": "mozilla_beta_tasks", + "release_type": "beta", + }, + "mozilla-release": { + "target_tasks_method": "mozilla_release_tasks", + "release_type": "release", + }, + "mozilla-esr102": { + "target_tasks_method": "mozilla_esr102_tasks", + "release_type": "esr102", + }, + "mozilla-esr115": { + "target_tasks_method": "mozilla_esr115_tasks", + "release_type": "esr115", + }, + "pine": { + "target_tasks_method": "pine_tasks", + }, + "kaios": { + "target_tasks_method": "kaios_tasks", + }, + "toolchains": { + "target_tasks_method": "mozilla_central_tasks", + }, + # the default parameters are used for projects that do not match above. + "default": { + "target_tasks_method": "default", + }, +} + +try_task_config_schema = Schema( + { + Required("tasks"): [str], + Optional("browsertime"): bool, + Optional("chemspill-prio"): bool, + Optional("disable-pgo"): bool, + Optional("env"): {str: str}, + Optional("gecko-profile"): bool, + Optional("gecko-profile-interval"): float, + Optional("gecko-profile-entries"): int, + Optional("gecko-profile-features"): str, + Optional("gecko-profile-threads"): str, + Optional( + "perftest-options", + description="Options passed from `mach perftest` to try.", + ): object, + Optional( + "optimize-strategies", + description="Alternative optimization strategies to use instead of the default. " + "A module path pointing to a dict to be use as the `strategy_override` " + "argument in `taskgraph.optimize.base.optimize_task_graph`.", + ): str, + Optional("rebuild"): int, + Optional("tasks-regex"): { + "include": Any(None, [str]), + "exclude": Any(None, [str]), + }, + Optional("use-artifact-builds"): bool, + Optional( + "worker-overrides", + description="Mapping of worker alias to worker pools to use for those aliases.", + ): {str: str}, + Optional("routes"): [str], + } +) +""" +Schema for try_task_config.json files. +""" + +try_task_config_schema_v2 = Schema( + { + Optional("parameters"): {str: object}, + } +) + + +def full_task_graph_to_runnable_jobs(full_task_json): + runnable_jobs = {} + for label, node in full_task_json.items(): + if not ("extra" in node["task"] and "treeherder" in node["task"]["extra"]): + continue + + th = node["task"]["extra"]["treeherder"] + runnable_jobs[label] = {"symbol": th["symbol"]} + + for i in ("groupName", "groupSymbol", "collection"): + if i in th: + runnable_jobs[label][i] = th[i] + if th.get("machine", {}).get("platform"): + runnable_jobs[label]["platform"] = th["machine"]["platform"] + return runnable_jobs + + +def full_task_graph_to_manifests_by_task(full_task_json): + manifests_by_task = defaultdict(list) + for label, node in full_task_json.items(): + manifests = node["attributes"].get("test_manifests") + if not manifests: + continue + + manifests_by_task[label].extend(manifests) + return manifests_by_task + + +def try_syntax_from_message(message): + """ + Parse the try syntax out of a commit message, returning '' if none is + found. + """ + try_idx = message.find("try:") + if try_idx == -1: + return "" + return message[try_idx:].split("\n", 1)[0] + + +def taskgraph_decision(options, parameters=None): + """ + Run the decision task. This function implements `mach taskgraph decision`, + and is responsible for + + * processing decision task command-line options into parameters + * running task-graph generation exactly the same way the other `mach + taskgraph` commands do + * generating a set of artifacts to memorialize the graph + * calling TaskCluster APIs to create the graph + """ + + parameters = parameters or ( + lambda graph_config: get_decision_parameters(graph_config, options) + ) + + decision_task_id = os.environ["TASK_ID"] + + # create a TaskGraphGenerator instance + tgg = TaskGraphGenerator( + root_dir=options.get("root"), + parameters=parameters, + decision_task_id=decision_task_id, + write_artifacts=True, + ) + + if not create.testing: + # set additional index paths for the decision task + set_decision_indexes(decision_task_id, tgg.parameters, tgg.graph_config) + + # write out the parameters used to generate this graph + write_artifact("parameters.yml", dict(**tgg.parameters)) + + # write out the public/actions.json file + write_artifact( + "actions.json", + render_actions_json(tgg.parameters, tgg.graph_config, decision_task_id), + ) + + # write out the full graph for reference + full_task_json = tgg.full_task_graph.to_json() + write_artifact("full-task-graph.json", full_task_json) + + # write out the public/runnable-jobs.json file + write_artifact( + "runnable-jobs.json", full_task_graph_to_runnable_jobs(full_task_json) + ) + + # write out the public/manifests-by-task.json file + write_artifact( + "manifests-by-task.json.gz", + full_task_graph_to_manifests_by_task(full_task_json), + ) + + # write out the public/tests-by-manifest.json file + write_artifact("tests-by-manifest.json.gz", resolver.tests_by_manifest) + + # this is just a test to check whether the from_json() function is working + _, _ = TaskGraph.from_json(full_task_json) + + # write out the target task set to allow reproducing this as input + write_artifact("target-tasks.json", list(tgg.target_task_set.tasks.keys())) + + # write out the optimized task graph to describe what will actually happen, + # and the map of labels to taskids + write_artifact("task-graph.json", tgg.morphed_task_graph.to_json()) + write_artifact("label-to-taskid.json", tgg.label_to_taskid) + + # write bugbug scheduling information if it was invoked + if len(push_schedules) > 0: + write_artifact("bugbug-push-schedules.json", push_schedules.popitem()[1]) + + # cache run-task & misc/fetch-content + scripts_root_dir = os.path.join( + "/builds/worker/checkouts/gecko/taskcluster/scripts" + ) + run_task_file_path = os.path.join(scripts_root_dir, "run-task") + fetch_content_file_path = os.path.join(scripts_root_dir, "misc/fetch-content") + shutil.copy2(run_task_file_path, ARTIFACTS_DIR) + shutil.copy2(fetch_content_file_path, ARTIFACTS_DIR) + + # actually create the graph + create_tasks( + tgg.graph_config, + tgg.morphed_task_graph, + tgg.label_to_taskid, + tgg.parameters, + decision_task_id=decision_task_id, + ) + + +def get_decision_parameters(graph_config, options): + """ + Load parameters from the command-line options for 'taskgraph decision'. + This also applies per-project parameters, based on the given project. + + """ + product_dir = graph_config["product-dir"] + + parameters = { + n: options[n] + for n in [ + "base_repository", + "base_ref", + "base_rev", + "head_repository", + "head_rev", + "head_ref", + "head_tag", + "project", + "pushlog_id", + "pushdate", + "owner", + "level", + "repository_type", + "target_tasks_method", + "tasks_for", + ] + if n in options + } + + commit_message = get_hg_commit_message(os.path.join(GECKO, product_dir)) + + repo_path = os.getcwd() + repo = get_repository(repo_path) + parameters["base_ref"] = _determine_more_accurate_base_ref( + repo, + candidate_base_ref=options.get("base_ref"), + head_ref=options.get("head_ref"), + base_rev=options.get("base_rev"), + ) + + parameters["base_rev"] = _determine_more_accurate_base_rev( + repo, + base_ref=parameters["base_ref"], + candidate_base_rev=options.get("base_rev"), + head_rev=options.get("head_rev"), + env_prefix=_get_env_prefix(graph_config), + ) + + # Define default filter list, as most configurations shouldn't need + # custom filters. + parameters["filters"] = [ + "target_tasks_method", + ] + parameters["enable_always_target"] = False + parameters["existing_tasks"] = {} + parameters["do_not_optimize"] = [] + parameters["build_number"] = 1 + parameters["version"] = get_version(product_dir) + parameters["app_version"] = get_app_version(product_dir) + parameters["message"] = try_syntax_from_message(commit_message) + parameters["hg_branch"] = get_hg_revision_branch( + GECKO, revision=parameters["head_rev"] + ) + parameters["next_version"] = None + parameters["optimize_strategies"] = None + parameters["optimize_target_tasks"] = True + parameters["phabricator_diff"] = None + parameters["release_type"] = "" + parameters["release_eta"] = "" + parameters["release_enable_partner_repack"] = False + parameters["release_enable_partner_attribution"] = False + parameters["release_partners"] = [] + parameters["release_partner_config"] = {} + parameters["release_partner_build_number"] = 1 + parameters["release_enable_emefree"] = False + parameters["release_product"] = None + parameters["required_signoffs"] = [] + parameters["signoff_urls"] = {} + parameters["test_manifest_loader"] = "default" + parameters["try_mode"] = None + parameters["try_task_config"] = {} + parameters["try_options"] = None + + # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which + # case, fake it + if "@" not in parameters["owner"]: + parameters["owner"] += "@noreply.mozilla.org" + + # use the pushdate as build_date if given, else use current time + parameters["build_date"] = parameters["pushdate"] or int(time.time()) + # moz_build_date is the build identifier based on build_date + parameters["moz_build_date"] = time.strftime( + "%Y%m%d%H%M%S", time.gmtime(parameters["build_date"]) + ) + + project = parameters["project"] + try: + parameters.update(PER_PROJECT_PARAMETERS[project]) + except KeyError: + logger.warning( + "using default project parameters; add {} to " + "PER_PROJECT_PARAMETERS in {} to customize behavior " + "for this project".format(project, __file__) + ) + parameters.update(PER_PROJECT_PARAMETERS["default"]) + + # `target_tasks_method` has higher precedence than `project` parameters + if options.get("target_tasks_method"): + parameters["target_tasks_method"] = options["target_tasks_method"] + + # ..but can be overridden by the commit message: if it contains the special + # string "DONTBUILD" and this is an on-push decision task, then use the + # special 'nothing' target task method. + if "DONTBUILD" in commit_message and options["tasks_for"] == "hg-push": + parameters["target_tasks_method"] = "nothing" + + if options.get("include_push_tasks"): + get_existing_tasks(options.get("rebuild_kinds", []), parameters, graph_config) + + # If the target method is nightly, we should build partials. This means + # knowing what has been released previously. + # An empty release_history is fine, it just means no partials will be built + parameters.setdefault("release_history", dict()) + if "nightly" in parameters.get("target_tasks_method", ""): + parameters["release_history"] = populate_release_history("Firefox", project) + + if options.get("try_task_config_file"): + task_config_file = os.path.abspath(options.get("try_task_config_file")) + else: + # if try_task_config.json is present, load it + task_config_file = os.path.join(os.getcwd(), "try_task_config.json") + + # load try settings + if "try" in project and options["tasks_for"] == "hg-push": + set_try_config(parameters, task_config_file) + + if options.get("optimize_target_tasks") is not None: + parameters["optimize_target_tasks"] = options["optimize_target_tasks"] + + # Determine if this should be a backstop push. + parameters["backstop"] = is_backstop(parameters) + + if "decision-parameters" in graph_config["taskgraph"]: + find_object(graph_config["taskgraph"]["decision-parameters"])( + graph_config, parameters + ) + + result = Parameters(**parameters) + result.check() + return result + + +def get_existing_tasks(rebuild_kinds, parameters, graph_config): + """ + Find the decision task corresponding to the on-push graph, and return + a mapping of labels to task-ids from it. This will skip the kinds specificed + by `rebuild_kinds`. + """ + try: + decision_task = retry( + find_decision_task, + args=(parameters, graph_config), + attempts=4, + sleeptime=5 * 60, + ) + except Exception: + logger.exception("Didn't find existing push task.") + sys.exit(1) + _, task_graph = TaskGraph.from_json( + get_artifact(decision_task, "public/full-task-graph.json") + ) + parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds( + task_graph, [decision_task], rebuild_kinds + ) + + +def set_try_config(parameters, task_config_file): + if os.path.isfile(task_config_file): + logger.info(f"using try tasks from {task_config_file}") + with open(task_config_file) as fh: + task_config = json.load(fh) + task_config_version = task_config.pop("version", 1) + if task_config_version == 1: + validate_schema( + try_task_config_schema, + task_config, + "Invalid v1 `try_task_config.json`.", + ) + parameters["try_mode"] = "try_task_config" + parameters["try_task_config"] = task_config + elif task_config_version == 2: + validate_schema( + try_task_config_schema_v2, + task_config, + "Invalid v2 `try_task_config.json`.", + ) + parameters.update(task_config["parameters"]) + return + else: + raise Exception( + f"Unknown `try_task_config.json` version: {task_config_version}" + ) + + if "try:" in parameters["message"]: + parameters["try_mode"] = "try_option_syntax" + parameters.update(parse_message(parameters["message"])) + else: + parameters["try_options"] = None + + if parameters["try_mode"] == "try_task_config": + # The user has explicitly requested a set of jobs, so run them all + # regardless of optimization. Their dependencies can be optimized, + # though. + parameters["optimize_target_tasks"] = False + else: + # For a try push with no task selection, apply the default optimization + # process to all of the tasks. + parameters["optimize_target_tasks"] = True + + +def set_decision_indexes(decision_task_id, params, graph_config): + index_paths = [] + if params["backstop"]: + index_paths.append(BACKSTOP_INDEX) + + subs = params.copy() + subs["trust-domain"] = graph_config["trust-domain"] + + index_paths = [i.format(**subs) for i in index_paths] + for index_path in index_paths: + insert_index(index_path, decision_task_id, use_proxy=True) + + +def write_artifact(filename, data): + logger.info(f"writing artifact file `{filename}`") + if not os.path.isdir(ARTIFACTS_DIR): + os.mkdir(ARTIFACTS_DIR) + path = os.path.join(ARTIFACTS_DIR, filename) + if filename.endswith(".yml"): + with open(path, "w") as f: + yaml.safe_dump(data, f, allow_unicode=True, default_flow_style=False) + elif filename.endswith(".json"): + with open(path, "w") as f: + json.dump(data, f, sort_keys=True, indent=2, separators=(",", ": ")) + elif filename.endswith(".json.gz"): + import gzip + + with gzip.open(path, "wb") as f: + f.write(json.dumps(data).encode("utf-8")) + else: + raise TypeError(f"Don't know how to write to {filename}") + + +def read_artifact(filename): + path = os.path.join(ARTIFACTS_DIR, filename) + if filename.endswith(".yml"): + return load_yaml(path, filename) + if filename.endswith(".json"): + with open(path) as f: + return json.load(f) + if filename.endswith(".json.gz"): + import gzip + + with gzip.open(path, "rb") as f: + return json.load(f.decode("utf-8")) + else: + raise TypeError(f"Don't know how to read {filename}") + + +def rename_artifact(src, dest): + os.rename(os.path.join(ARTIFACTS_DIR, src), os.path.join(ARTIFACTS_DIR, dest)) diff --git a/taskcluster/gecko_taskgraph/docker.py b/taskcluster/gecko_taskgraph/docker.py new file mode 100644 index 0000000000..a1952f278c --- /dev/null +++ b/taskcluster/gecko_taskgraph/docker.py @@ -0,0 +1,198 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +import tarfile +from io import BytesIO + +from taskgraph.generator import load_tasks_for_kind +from taskgraph.parameters import Parameters +from taskgraph.util.taskcluster import get_artifact_url, get_session + +from gecko_taskgraph.optimize.strategies import IndexSearch +from gecko_taskgraph.util import docker + +from . import GECKO + + +def get_image_digest(image_name): + params = Parameters( + level=os.environ.get("MOZ_SCM_LEVEL", "3"), + strict=False, + ) + tasks = load_tasks_for_kind(params, "docker-image") + task = tasks[f"docker-image-{image_name}"] + return task.attributes["cached_task"]["digest"] + + +def load_image_by_name(image_name, tag=None): + params = {"level": os.environ.get("MOZ_SCM_LEVEL", "3")} + tasks = load_tasks_for_kind(params, "docker-image") + task = tasks[f"docker-image-{image_name}"] + deadline = None + task_id = IndexSearch().should_replace_task( + task, {}, deadline, task.optimization.get("index-search", []) + ) + + if task_id in (True, False): + print( + "Could not find artifacts for a docker image " + "named `{image_name}`. Local commits and other changes " + "in your checkout may cause this error. Try " + "updating to a fresh checkout of mozilla-central " + "to download image.".format(image_name=image_name) + ) + return False + + return load_image_by_task_id(task_id, tag) + + +def load_image_by_task_id(task_id, tag=None): + artifact_url = get_artifact_url(task_id, "public/image.tar.zst") + result = load_image(artifact_url, tag) + print("Found docker image: {}:{}".format(result["image"], result["tag"])) + if tag: + print(f"Re-tagged as: {tag}") + else: + tag = "{}:{}".format(result["image"], result["tag"]) + print(f"Try: docker run -ti --rm {tag} bash") + return True + + +def build_context(name, outputFile, args=None): + """Build a context.tar for image with specified name.""" + if not name: + raise ValueError("must provide a Docker image name") + if not outputFile: + raise ValueError("must provide a outputFile") + + image_dir = docker.image_path(name) + if not os.path.isdir(image_dir): + raise Exception("image directory does not exist: %s" % image_dir) + + docker.create_context_tar(GECKO, image_dir, outputFile, image_name=name, args=args) + + +def build_image(name, tag, args=None): + """Build a Docker image of specified name. + + Output from image building process will be printed to stdout. + """ + if not name: + raise ValueError("must provide a Docker image name") + + image_dir = docker.image_path(name) + if not os.path.isdir(image_dir): + raise Exception("image directory does not exist: %s" % image_dir) + + tag = tag or docker.docker_image(name, by_tag=True) + + buf = BytesIO() + docker.stream_context_tar(GECKO, image_dir, buf, name, args) + docker.post_to_docker(buf.getvalue(), "/build", nocache=1, t=tag) + + print(f"Successfully built {name} and tagged with {tag}") + + if tag.endswith(":latest"): + print("*" * 50) + print("WARNING: no VERSION file found in image directory.") + print("Image is not suitable for deploying/pushing.") + print("Create an image suitable for deploying/pushing by creating") + print("a VERSION file in the image directory.") + print("*" * 50) + + +def load_image(url, imageName=None, imageTag=None): + """ + Load docker image from URL as imageName:tag, if no imageName or tag is given + it will use whatever is inside the zstd compressed tarball. + + Returns an object with properties 'image', 'tag' and 'layer'. + """ + import zstandard as zstd + + # If imageName is given and we don't have an imageTag + # we parse out the imageTag from imageName, or default it to 'latest' + # if no imageName and no imageTag is given, 'repositories' won't be rewritten + if imageName and not imageTag: + if ":" in imageName: + imageName, imageTag = imageName.split(":", 1) + else: + imageTag = "latest" + + info = {} + + def download_and_modify_image(): + # This function downloads and edits the downloaded tar file on the fly. + # It emits chunked buffers of the editted tar file, as a generator. + print(f"Downloading from {url}") + # get_session() gets us a requests.Session set to retry several times. + req = get_session().get(url, stream=True) + req.raise_for_status() + + with zstd.ZstdDecompressor().stream_reader(req.raw) as ifh: + + tarin = tarfile.open( + mode="r|", + fileobj=ifh, + bufsize=zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE, + ) + + # Stream through each member of the downloaded tar file individually. + for member in tarin: + # Non-file members only need a tar header. Emit one. + if not member.isfile(): + yield member.tobuf(tarfile.GNU_FORMAT) + continue + + # Open stream reader for the member + reader = tarin.extractfile(member) + + # If member is `repositories`, we parse and possibly rewrite the + # image tags. + if member.name == "repositories": + # Read and parse repositories + repos = json.loads(reader.read()) + reader.close() + + # If there is more than one image or tag, we can't handle it + # here. + if len(repos.keys()) > 1: + raise Exception("file contains more than one image") + info["image"] = image = list(repos.keys())[0] + if len(repos[image].keys()) > 1: + raise Exception("file contains more than one tag") + info["tag"] = tag = list(repos[image].keys())[0] + info["layer"] = layer = repos[image][tag] + + # Rewrite the repositories file + data = json.dumps({imageName or image: {imageTag or tag: layer}}) + reader = BytesIO(data.encode("utf-8")) + member.size = len(data) + + # Emit the tar header for this member. + yield member.tobuf(tarfile.GNU_FORMAT) + # Then emit its content. + remaining = member.size + while remaining: + length = min(remaining, zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE) + buf = reader.read(length) + remaining -= len(buf) + yield buf + # Pad to fill a 512 bytes block, per tar format. + remainder = member.size % 512 + if remainder: + yield ("\0" * (512 - remainder)).encode("utf-8") + + reader.close() + + docker.post_to_docker(download_and_modify_image(), "/images/load", quiet=0) + + # Check that we found a repositories file + if not info.get("image") or not info.get("tag") or not info.get("layer"): + raise Exception("No repositories file found!") + + return info diff --git a/taskcluster/gecko_taskgraph/files_changed.py b/taskcluster/gecko_taskgraph/files_changed.py new file mode 100644 index 0000000000..c814df0806 --- /dev/null +++ b/taskcluster/gecko_taskgraph/files_changed.py @@ -0,0 +1,95 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Support for optimizing tasks based on the set of files that have changed. +""" + +import logging +import os +from subprocess import CalledProcessError + +from mozbuild.util import memoize +from mozpack.path import join as join_path +from mozpack.path import match as mozpackmatch +from mozversioncontrol import InvalidRepoPath, get_repository_object + +from gecko_taskgraph import GECKO +from gecko_taskgraph.util.hg import get_json_automationrelevance + +logger = logging.getLogger(__name__) + + +@memoize +def get_changed_files(repository, revision): + """ + Get the set of files changed in the push headed by the given revision. + Responses are cached, so multiple calls with the same arguments are OK. + """ + contents = get_json_automationrelevance(repository, revision) + try: + changesets = contents["changesets"] + except KeyError: + # We shouldn't hit this error in CI. + if os.environ.get("MOZ_AUTOMATION"): + raise + + # We're likely on an unpublished commit, grab changed files from + # version control. + return get_locally_changed_files(GECKO) + + logger.debug("{} commits influencing task scheduling:".format(len(changesets))) + changed_files = set() + for c in changesets: + desc = "" # Support empty desc + if c["desc"]: + desc = c["desc"].splitlines()[0].encode("ascii", "ignore") + logger.debug(" {cset} {desc}".format(cset=c["node"][0:12], desc=desc)) + changed_files |= set(c["files"]) + + return changed_files + + +def check(params, file_patterns): + """Determine whether any of the files changed in the indicated push to + https://hg.mozilla.org match any of the given file patterns.""" + repository = params.get("head_repository") + revision = params.get("head_rev") + if not repository or not revision: + logger.warning( + "Missing `head_repository` or `head_rev` parameters; " + "assuming all files have changed" + ) + return True + + changed_files = get_changed_files(repository, revision) + + if "comm_head_repository" in params: + repository = params.get("comm_head_repository") + revision = params.get("comm_head_rev") + if not revision: + logger.warning( + "Missing `comm_head_rev` parameters; " "assuming all files have changed" + ) + return True + + changed_files |= { + join_path("comm", file) for file in get_changed_files(repository, revision) + } + + for pattern in file_patterns: + for path in changed_files: + if mozpackmatch(path, pattern): + return True + + return False + + +@memoize +def get_locally_changed_files(repo): + try: + vcs = get_repository_object(repo) + return set(vcs.get_outgoing_files("AM")) + except (InvalidRepoPath, CalledProcessError): + return set() diff --git a/taskcluster/gecko_taskgraph/loader/__init__.py b/taskcluster/gecko_taskgraph/loader/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/taskcluster/gecko_taskgraph/loader/__init__.py diff --git a/taskcluster/gecko_taskgraph/loader/multi_dep.py b/taskcluster/gecko_taskgraph/loader/multi_dep.py new file mode 100644 index 0000000000..04fda04415 --- /dev/null +++ b/taskcluster/gecko_taskgraph/loader/multi_dep.py @@ -0,0 +1,275 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.task import Task +from taskgraph.util.schema import Schema +from voluptuous import Required + +from gecko_taskgraph.util.copy_task import copy_task + +from ..util.attributes import sorted_unique_list + +schema = Schema( + { + Required("primary-dependency", "primary dependency task"): Task, + Required( + "dependent-tasks", + "dictionary of dependent tasks, keyed by kind", + ): {str: Task}, + } +) + + +# Define a collection of group_by functions +GROUP_BY_MAP = {} + + +def group_by(name): + def wrapper(func): + GROUP_BY_MAP[name] = func + return func + + return wrapper + + +def loader(kind, path, config, params, loaded_tasks): + """ + Load tasks based on the jobs dependant kinds, designed for use as + multiple-dependent needs. + + Required ``group-by-fn`` is used to define how we coalesce the + multiple deps together to pass to transforms, e.g. all kinds specified get + collapsed by platform with `platform` + + Optional ``primary-dependency`` (ordered list or string) is used to determine + which upstream kind to inherit attrs from. See ``get_primary_dep``. + + The `only-for-build-platforms` kind configuration, if specified, will limit + the build platforms for which a job will be created. Alternatively there is + 'not-for-build-platforms' kind configuration which will be consulted only after + 'only-for-build-platforms' is checked (if present), and omit any jobs where the + build platform matches. + + Optional ``job-template`` kind configuration value, if specified, will be used to + pass configuration down to the specified transforms used. + """ + job_template = config.get("job-template") + + for dep_tasks in group_tasks(config, loaded_tasks): + job = {"dependent-tasks": dep_tasks} + job["primary-dependency"] = get_primary_dep(config, dep_tasks) + if job_template: + job.update(copy_task(job_template)) + # copy shipping_product from upstream + product = job["primary-dependency"].attributes.get( + "shipping_product", job["primary-dependency"].task.get("shipping-product") + ) + if product: + job.setdefault("shipping-product", product) + job.setdefault("attributes", {})["required_signoffs"] = sorted_unique_list( + *( + task.attributes.get("required_signoffs", []) + for task in dep_tasks.values() + ) + ) + + yield job + + +def skip_only_or_not(config, task): + """Return True if we should skip this task based on `only_` or `not_` config.""" + only_platforms = config.get("only-for-build-platforms") + not_platforms = config.get("not-for-build-platforms") + only_attributes = config.get("only-for-attributes") + not_attributes = config.get("not-for-attributes") + task_attrs = task.attributes + if only_platforms or not_platforms: + platform = task_attrs.get("build_platform") + build_type = task_attrs.get("build_type") + if not platform or not build_type: + return True + combined_platform = f"{platform}/{build_type}" + if only_platforms and combined_platform not in only_platforms: + return True + if not_platforms and combined_platform in not_platforms: + return True + if only_attributes: + if not set(only_attributes) & set(task_attrs): + # make sure any attribute exists + return True + if not_attributes: + if set(not_attributes) & set(task_attrs): + return True + return False + + +def group_tasks(config, tasks): + group_by_fn = GROUP_BY_MAP[config["group-by"]] + + groups = group_by_fn(config, tasks) + + for combinations in groups.values(): + kinds = [f.kind for f in combinations] + assert_unique_members( + kinds, + error_msg=("Multi_dep.py should have filtered down to one task per kind"), + ) + dependencies = {t.kind: copy_task(t) for t in combinations} + yield dependencies + + +@group_by("platform") +def platform_grouping(config, tasks): + groups = {} + for task in tasks: + if task.kind not in config.get("kind-dependencies", []): + continue + if skip_only_or_not(config, task): + continue + platform = task.attributes.get("build_platform") + build_type = task.attributes.get("build_type") + product = task.attributes.get( + "shipping_product", task.task.get("shipping-product") + ) + + groups.setdefault((platform, build_type, product), []).append(task) + return groups + + +@group_by("single-locale") +def single_locale_grouping(config, tasks): + """Split by a single locale (but also by platform, build-type, product) + + The locale can be `None` (en-US build/signing/repackage), a single locale, + or multiple locales per task, e.g. for l10n chunking. In the case of a task + with, say, five locales, the task will show up in all five locale groupings. + + This grouping is written for non-partner-repack beetmover, but might also + be useful elsewhere. + + """ + groups = {} + + for task in tasks: + if task.kind not in config.get("kind-dependencies", []): + continue + if skip_only_or_not(config, task): + continue + platform = task.attributes.get("build_platform") + build_type = task.attributes.get("build_type") + product = task.attributes.get( + "shipping_product", task.task.get("shipping-product") + ) + task_locale = task.attributes.get("locale") + chunk_locales = task.attributes.get("chunk_locales") + locales = chunk_locales or [task_locale] + + for locale in locales: + locale_key = (platform, build_type, product, locale) + groups.setdefault(locale_key, []) + if task not in groups[locale_key]: + groups[locale_key].append(task) + + return groups + + +@group_by("chunk-locales") +def chunk_locale_grouping(config, tasks): + """Split by a chunk_locale (but also by platform, build-type, product) + + This grouping is written for mac signing with notarization, but might also + be useful elsewhere. + + """ + groups = {} + + for task in tasks: + if task.kind not in config.get("kind-dependencies", []): + continue + if skip_only_or_not(config, task): + continue + platform = task.attributes.get("build_platform") + build_type = task.attributes.get("build_type") + product = task.attributes.get( + "shipping_product", task.task.get("shipping-product") + ) + chunk_locales = tuple(sorted(task.attributes.get("chunk_locales", []))) + + chunk_locale_key = (platform, build_type, product, chunk_locales) + groups.setdefault(chunk_locale_key, []) + if task not in groups[chunk_locale_key]: + groups[chunk_locale_key].append(task) + + return groups + + +@group_by("partner-repack-ids") +def partner_repack_ids_grouping(config, tasks): + """Split by partner_repack_ids (but also by platform, build-type, product) + + This grouping is written for release-{eme-free,partner}-repack-signing. + + """ + groups = {} + + for task in tasks: + if task.kind not in config.get("kind-dependencies", []): + continue + if skip_only_or_not(config, task): + continue + platform = task.attributes.get("build_platform") + build_type = task.attributes.get("build_type") + product = task.attributes.get( + "shipping_product", task.task.get("shipping-product") + ) + partner_repack_ids = tuple( + sorted(task.task.get("extra", {}).get("repack_ids", [])) + ) + + partner_repack_ids_key = (platform, build_type, product, partner_repack_ids) + groups.setdefault(partner_repack_ids_key, []) + if task not in groups[partner_repack_ids_key]: + groups[partner_repack_ids_key].append(task) + + return groups + + +def assert_unique_members(kinds, error_msg=None): + if len(kinds) != len(set(kinds)): + raise Exception(error_msg) + + +def get_primary_dep(config, dep_tasks): + """Find the dependent task to inherit attributes from. + + If ``primary-dependency`` is defined in ``kind.yml`` and is a string, + then find the first dep with that task kind and return it. If it is + defined and is a list, the first kind in that list with a matching dep + is the primary dependency. If it's undefined, return the first dep. + + """ + primary_dependencies = config.get("primary-dependency") + if isinstance(primary_dependencies, str): + primary_dependencies = [primary_dependencies] + if not primary_dependencies: + assert len(dep_tasks) == 1, "Must define a primary-dependency!" + return list(dep_tasks.values())[0] + primary_dep = None + for primary_kind in primary_dependencies: + for dep_kind in dep_tasks: + if dep_kind == primary_kind: + assert ( + primary_dep is None + ), "Too many primary dependent tasks in dep_tasks: {}!".format( + [t.label for t in dep_tasks] + ) + primary_dep = dep_tasks[dep_kind] + if primary_dep is None: + raise Exception( + "Can't find dependency of {}: {}".format( + config["primary-dependency"], config + ) + ) + return primary_dep diff --git a/taskcluster/gecko_taskgraph/loader/single_dep.py b/taskcluster/gecko_taskgraph/loader/single_dep.py new file mode 100644 index 0000000000..caea4b85d0 --- /dev/null +++ b/taskcluster/gecko_taskgraph/loader/single_dep.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.task import Task +from taskgraph.util.schema import Schema +from voluptuous import Required + +from gecko_taskgraph.util.copy_task import copy_task + +schema = Schema( + { + Required("primary-dependency", "primary dependency task"): Task, + } +) + + +def loader(kind, path, config, params, loaded_tasks): + """ + Load tasks based on the jobs dependant kinds. + + The `only-for-build-platforms` kind configuration, if specified, will limit + the build platforms for which a job will be created. Alternatively there is + 'not-for-build-platforms' kind configuration which will be consulted only after + 'only-for-build-platforms' is checked (if present), and omit any jobs where the + build platform matches. + + Optional `only-for-attributes` kind configuration, if specified, will limit + the jobs chosen to ones which have the specified attribute, with the specified + value. + + Optional `job-template` kind configuration value, if specified, will be used to + pass configuration down to the specified transforms used. + """ + only_platforms = config.get("only-for-build-platforms") + not_platforms = config.get("not-for-build-platforms") + only_attributes = config.get("only-for-attributes") + job_template = config.get("job-template") + + for task in loaded_tasks: + if task.kind not in config.get("kind-dependencies", []): + continue + + if only_platforms or not_platforms: + build_platform = task.attributes.get("build_platform") + build_type = task.attributes.get("build_type") + if not build_platform or not build_type: + continue + platform = f"{build_platform}/{build_type}" + if only_platforms and platform not in only_platforms: + continue + elif not_platforms and platform in not_platforms: + continue + + if only_attributes: + config_attrs = set(only_attributes) + if not config_attrs & set(task.attributes): + # make sure any attribute exists + continue + + job = { + "primary-dependency": task, + } + + if job_template: + job.update(copy_task(job_template)) + + # copy shipping_product from upstream + product = task.attributes.get( + "shipping_product", task.task.get("shipping-product") + ) + if product: + job.setdefault("shipping-product", product) + + yield job diff --git a/taskcluster/gecko_taskgraph/loader/test.py b/taskcluster/gecko_taskgraph/loader/test.py new file mode 100644 index 0000000000..c97acecd1a --- /dev/null +++ b/taskcluster/gecko_taskgraph/loader/test.py @@ -0,0 +1,142 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +from taskgraph.util.yaml import load_yaml + +from gecko_taskgraph.util.copy_task import copy_task + +from .transform import loader as transform_loader + +logger = logging.getLogger(__name__) + + +def loader(kind, path, config, params, loaded_tasks): + """ + Generate tasks implementing Gecko tests. + """ + + builds_by_platform = get_builds_by_platform( + dep_kind="build", loaded_tasks=loaded_tasks + ) + signed_builds_by_platform = get_builds_by_platform( + dep_kind="build-signing", loaded_tasks=loaded_tasks + ) + + # get the test platforms for those build tasks + test_platforms_cfg = load_yaml(path, "test-platforms.yml") + test_platforms = get_test_platforms( + test_platforms_cfg, builds_by_platform, signed_builds_by_platform + ) + + # expand the test sets for each of those platforms + test_sets_cfg = load_yaml(path, "test-sets.yml") + test_platforms = expand_tests(test_sets_cfg, test_platforms) + + # load the test descriptions + tests = transform_loader(kind, path, config, params, loaded_tasks) + test_descriptions = {t.pop("name"): t for t in tests} + + # generate all tests for all test platforms + for test_platform_name, test_platform in test_platforms.items(): + for test_name in test_platform["test-names"]: + test = copy_task(test_descriptions[test_name]) + test["build-platform"] = test_platform["build-platform"] + test["test-platform"] = test_platform_name + test["build-label"] = test_platform["build-label"] + if test_platform.get("build-signing-label", None): + test["build-signing-label"] = test_platform["build-signing-label"] + + test["build-attributes"] = test_platform["build-attributes"] + test["test-name"] = test_name + if test_platform.get("shippable"): + test.setdefault("attributes", {})["shippable"] = True + test["attributes"]["shipping_product"] = test_platform[ + "shipping_product" + ] + + logger.debug( + "Generating tasks for test {} on platform {}".format( + test_name, test["test-platform"] + ) + ) + yield test + + +def get_builds_by_platform(dep_kind, loaded_tasks): + """Find the build tasks on which tests will depend, keyed by + platform/type. Returns a dictionary mapping build platform to task.""" + builds_by_platform = {} + for task in loaded_tasks: + if task.kind != dep_kind: + continue + + build_platform = task.attributes.get("build_platform") + build_type = task.attributes.get("build_type") + if not build_platform or not build_type: + continue + platform = f"{build_platform}/{build_type}" + if platform in builds_by_platform: + raise Exception("multiple build jobs for " + platform) + builds_by_platform[platform] = task + return builds_by_platform + + +def get_test_platforms( + test_platforms_cfg, builds_by_platform, signed_builds_by_platform={} +): + """Get the test platforms for which test tasks should be generated, + based on the available build platforms. Returns a dictionary mapping + test platform to {test-set, build-platform, build-label}.""" + test_platforms = {} + for test_platform, cfg in test_platforms_cfg.items(): + build_platform = cfg["build-platform"] + if build_platform not in builds_by_platform: + logger.warning( + "No build task with platform {}; ignoring test platform {}".format( + build_platform, test_platform + ) + ) + continue + test_platforms[test_platform] = { + "build-platform": build_platform, + "build-label": builds_by_platform[build_platform].label, + "build-attributes": builds_by_platform[build_platform].attributes, + } + + if builds_by_platform[build_platform].attributes.get("shippable"): + test_platforms[test_platform]["shippable"] = builds_by_platform[ + build_platform + ].attributes["shippable"] + test_platforms[test_platform]["shipping_product"] = builds_by_platform[ + build_platform + ].attributes["shipping_product"] + + test_platforms[test_platform].update(cfg) + + return test_platforms + + +def expand_tests(test_sets_cfg, test_platforms): + """Expand the test sets in `test_platforms` out to sets of test names. + Returns a dictionary like `get_test_platforms`, with an additional + `test-names` key for each test platform, containing a set of test + names.""" + rv = {} + for test_platform, cfg in test_platforms.items(): + test_sets = cfg["test-sets"] + if not set(test_sets) <= set(test_sets_cfg): + raise Exception( + "Test sets {} for test platform {} are not defined".format( + ", ".join(test_sets), test_platform + ) + ) + test_names = set() + for test_set in test_sets: + test_names.update(test_sets_cfg[test_set]) + rv[test_platform] = cfg.copy() + rv[test_platform]["test-names"] = test_names + return rv diff --git a/taskcluster/gecko_taskgraph/loader/transform.py b/taskcluster/gecko_taskgraph/loader/transform.py new file mode 100644 index 0000000000..1e513bcb73 --- /dev/null +++ b/taskcluster/gecko_taskgraph/loader/transform.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +from taskgraph.util.yaml import load_yaml + +from ..util.templates import merge + +logger = logging.getLogger(__name__) + + +def loader(kind, path, config, params, loaded_tasks): + """ + Get the input elements that will be transformed into tasks in a generic + way. The elements themselves are free-form, and become the input to the + first transform. + + By default, this reads jobs from the `jobs` key, or from yaml files + named by `jobs-from`. The entities are read from mappings, and the + keys to those mappings are added in the `name` key of each entity. + + If there is a `job-defaults` config, then every job is merged with it. + This provides a simple way to set default values for all jobs of a kind. + The `job-defaults` key can also be specified in a yaml file pointed to by + `jobs-from`. In this case it will only apply to tasks defined in the same + file. + + Other kind implementations can use a different loader function to + produce inputs and hand them to `transform_inputs`. + """ + + def jobs(): + defaults = config.get("job-defaults") + for name, job in config.get("jobs", {}).items(): + if defaults: + job = merge(defaults, job) + job["job-from"] = "kind.yml" + yield name, job + + for filename in config.get("jobs-from", []): + tasks = load_yaml(path, filename) + + file_defaults = tasks.pop("job-defaults", None) + if defaults: + file_defaults = merge(defaults, file_defaults or {}) + + for name, job in tasks.items(): + if file_defaults: + job = merge(file_defaults, job) + job["job-from"] = filename + yield name, job + + for name, job in jobs(): + job["name"] = name + logger.debug(f"Generating tasks for {kind} {name}") + yield job diff --git a/taskcluster/gecko_taskgraph/main.py b/taskcluster/gecko_taskgraph/main.py new file mode 100644 index 0000000000..bb49ba6000 --- /dev/null +++ b/taskcluster/gecko_taskgraph/main.py @@ -0,0 +1,765 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import atexit +import json +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile +import traceback +from collections import namedtuple +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path +from typing import Any, List + +import appdirs +import yaml + +Command = namedtuple("Command", ["func", "args", "kwargs", "defaults"]) +commands = {} + + +def command(*args, **kwargs): + defaults = kwargs.pop("defaults", {}) + + def decorator(func): + commands[args[0]] = Command(func, args, kwargs, defaults) + return func + + return decorator + + +def argument(*args, **kwargs): + def decorator(func): + if not hasattr(func, "args"): + func.args = [] + func.args.append((args, kwargs)) + return func + + return decorator + + +def format_taskgraph_labels(taskgraph): + return "\n".join( + sorted( + taskgraph.tasks[index].label for index in taskgraph.graph.visit_postorder() + ) + ) + + +def format_taskgraph_json(taskgraph): + return json.dumps( + taskgraph.to_json(), sort_keys=True, indent=2, separators=(",", ": ") + ) + + +def format_taskgraph_yaml(taskgraph): + return yaml.safe_dump(taskgraph.to_json(), default_flow_style=False) + + +def get_filtered_taskgraph(taskgraph, tasksregex): + """ + Filter all the tasks on basis of a regular expression + and returns a new TaskGraph object + """ + from taskgraph.graph import Graph + from taskgraph.taskgraph import TaskGraph + + # return original taskgraph if no regular expression is passed + if not tasksregex: + return taskgraph + named_links_dict = taskgraph.graph.named_links_dict() + filteredtasks = {} + filterededges = set() + regexprogram = re.compile(tasksregex) + + for key in taskgraph.graph.visit_postorder(): + task = taskgraph.tasks[key] + if regexprogram.match(task.label): + filteredtasks[key] = task + for depname, dep in named_links_dict[key].items(): + if regexprogram.match(dep): + filterededges.add((key, dep, depname)) + filtered_taskgraph = TaskGraph( + filteredtasks, Graph(set(filteredtasks), filterededges) + ) + return filtered_taskgraph + + +FORMAT_METHODS = { + "labels": format_taskgraph_labels, + "json": format_taskgraph_json, + "yaml": format_taskgraph_yaml, +} + + +def get_taskgraph_generator(root, parameters): + """Helper function to make testing a little easier.""" + from taskgraph.generator import TaskGraphGenerator + + return TaskGraphGenerator(root_dir=root, parameters=parameters) + + +def format_taskgraph(options, parameters, logfile=None): + import taskgraph + from taskgraph.parameters import parameters_loader + + if logfile: + handler = logging.FileHandler(logfile, mode="w") + if logging.root.handlers: + oldhandler = logging.root.handlers[-1] + logging.root.removeHandler(oldhandler) + handler.setFormatter(oldhandler.formatter) + logging.root.addHandler(handler) + + if options["fast"]: + taskgraph.fast = True + + if isinstance(parameters, str): + parameters = parameters_loader( + parameters, + overrides={"target-kind": options.get("target_kind")}, + strict=False, + ) + + tgg = get_taskgraph_generator(options.get("root"), parameters) + + tg = getattr(tgg, options["graph_attr"]) + tg = get_filtered_taskgraph(tg, options["tasks_regex"]) + format_method = FORMAT_METHODS[options["format"] or "labels"] + return format_method(tg) + + +def dump_output(out, path=None, params_spec=None): + from taskgraph.parameters import Parameters + + params_name = Parameters.format_spec(params_spec) + fh = None + if path: + # Substitute params name into file path if necessary + if params_spec and "{params}" not in path: + name, ext = os.path.splitext(path) + name += "_{params}" + path = name + ext + + path = path.format(params=params_name) + fh = open(path, "w") + else: + print( + "Dumping result with parameters from {}:".format(params_name), + file=sys.stderr, + ) + print(out + "\n", file=fh) + + +def generate_taskgraph(options, parameters, logdir): + from taskgraph.parameters import Parameters + + def logfile(spec): + """Determine logfile given a parameters specification.""" + if logdir is None: + return None + return os.path.join( + logdir, + "{}_{}.log".format(options["graph_attr"], Parameters.format_spec(spec)), + ) + + # Don't bother using futures if there's only one parameter. This can make + # tracebacks a little more readable and avoids additional process overhead. + if len(parameters) == 1: + spec = parameters[0] + out = format_taskgraph(options, spec, logfile(spec)) + dump_output(out, options["output_file"]) + return + + futures = {} + with ProcessPoolExecutor() as executor: + for spec in parameters: + f = executor.submit(format_taskgraph, options, spec, logfile(spec)) + futures[f] = spec + + for future in as_completed(futures): + output_file = options["output_file"] + spec = futures[future] + e = future.exception() + if e: + out = "".join(traceback.format_exception(type(e), e, e.__traceback__)) + if options["diff"]: + # Dump to console so we don't accidentally diff the tracebacks. + output_file = None + else: + out = future.result() + + dump_output( + out, + path=output_file, + params_spec=spec if len(parameters) > 1 else None, + ) + + +@command( + "tasks", + help="Show all tasks in the taskgraph.", + defaults={"graph_attr": "full_task_set"}, +) +@command( + "full", help="Show the full taskgraph.", defaults={"graph_attr": "full_task_graph"} +) +@command( + "target", + help="Show the set of target tasks.", + defaults={"graph_attr": "target_task_set"}, +) +@command( + "target-graph", + help="Show the target graph.", + defaults={"graph_attr": "target_task_graph"}, +) +@command( + "optimized", + help="Show the optimized graph.", + defaults={"graph_attr": "optimized_task_graph"}, +) +@command( + "morphed", + help="Show the morphed graph.", + defaults={"graph_attr": "morphed_task_graph"}, +) +@argument("--root", "-r", help="root of the taskgraph definition relative to topsrcdir") +@argument("--quiet", "-q", action="store_true", help="suppress all logging output") +@argument( + "--verbose", "-v", action="store_true", help="include debug-level logging output" +) +@argument( + "--json", + "-J", + action="store_const", + dest="format", + const="json", + help="Output task graph as a JSON object", +) +@argument( + "--yaml", + "-Y", + action="store_const", + dest="format", + const="yaml", + help="Output task graph as a YAML object", +) +@argument( + "--labels", + "-L", + action="store_const", + dest="format", + const="labels", + help="Output the label for each task in the task graph (default)", +) +@argument( + "--parameters", + "-p", + default=None, + action="append", + help="Parameters to use for the generation. Can be a path to file (.yml or " + ".json; see `taskcluster/docs/parameters.rst`), a directory (containing " + "parameters files), a url, of the form `project=mozilla-central` to download " + "latest parameters file for the specified project from CI, or of the form " + "`task-id=<decision task id>` to download parameters from the specified " + "decision task. Can be specified multiple times, in which case multiple " + "generations will happen from the same invocation (one per parameters " + "specified).", +) +@argument( + "--no-optimize", + dest="optimize", + action="store_false", + default="true", + help="do not remove tasks from the graph that are found in the " + "index (a.k.a. optimize the graph)", +) +@argument( + "-o", + "--output-file", + default=None, + help="file path to store generated output.", +) +@argument( + "--tasks-regex", + "--tasks", + default=None, + help="only return tasks with labels matching this regular " "expression.", +) +@argument( + "--target-kind", + default=None, + help="only return tasks that are of the given kind, or their dependencies.", +) +@argument( + "-F", + "--fast", + default=False, + action="store_true", + help="enable fast task generation for local debugging.", +) +@argument( + "--diff", + const="default", + nargs="?", + default=None, + help="Generate and diff the current taskgraph against another revision. " + "Without args the base revision will be used. A revision specifier such as " + "the hash or `.~1` (hg) or `HEAD~1` (git) can be used as well.", +) +def show_taskgraph(options): + from mozversioncontrol import get_repository_object as get_repository + from taskgraph.parameters import Parameters, parameters_loader + + if options.pop("verbose", False): + logging.root.setLevel(logging.DEBUG) + + repo = None + cur_ref = None + diffdir = None + output_file = options["output_file"] + + if options["diff"]: + repo = get_repository(os.getcwd()) + + if not repo.working_directory_clean(): + print( + "abort: can't diff taskgraph with dirty working directory", + file=sys.stderr, + ) + return 1 + + # We want to return the working directory to the current state + # as best we can after we're done. In all known cases, using + # branch or bookmark (which are both available on the VCS object) + # as `branch` is preferable to a specific revision. + cur_ref = repo.branch or repo.head_ref[:12] + + diffdir = tempfile.mkdtemp() + atexit.register( + shutil.rmtree, diffdir + ) # make sure the directory gets cleaned up + options["output_file"] = os.path.join( + diffdir, f"{options['graph_attr']}_{cur_ref}" + ) + print(f"Generating {options['graph_attr']} @ {cur_ref}", file=sys.stderr) + + parameters: List[Any[str, Parameters]] = options.pop("parameters") + if not parameters: + overrides = { + "target-kind": options.get("target_kind"), + } + parameters = [ + parameters_loader(None, strict=False, overrides=overrides) + ] # will use default values + + for param in parameters[:]: + if isinstance(param, str) and os.path.isdir(param): + parameters.remove(param) + parameters.extend( + [ + p.as_posix() + for p in Path(param).iterdir() + if p.suffix in (".yml", ".json") + ] + ) + + logdir = None + if len(parameters) > 1: + # Log to separate files for each process instead of stderr to + # avoid interleaving. + basename = os.path.basename(os.getcwd()) + logdir = os.path.join(appdirs.user_log_dir("taskgraph"), basename) + if not os.path.isdir(logdir): + os.makedirs(logdir) + else: + # Only setup logging if we have a single parameter spec. Otherwise + # logging will go to files. This is also used as a hook for Gecko + # to setup its `mach` based logging. + setup_logging() + + generate_taskgraph(options, parameters, logdir) + + if options["diff"]: + assert diffdir is not None + assert repo is not None + + # Reload taskgraph modules to pick up changes and clear global state. + for mod in sys.modules.copy(): + if mod != __name__ and mod.split(".", 1)[0].endswith( + ("taskgraph", "mozbuild") + ): + del sys.modules[mod] + + # Ensure gecko_taskgraph is ahead of taskcluster_taskgraph in sys.path. + # Without this, we may end up validating some things against the wrong + # schema. + import gecko_taskgraph # noqa + + if options["diff"] == "default": + base_ref = repo.base_ref + else: + base_ref = options["diff"] + + try: + repo.update(base_ref) + base_ref = repo.head_ref[:12] + options["output_file"] = os.path.join( + diffdir, f"{options['graph_attr']}_{base_ref}" + ) + print(f"Generating {options['graph_attr']} @ {base_ref}", file=sys.stderr) + generate_taskgraph(options, parameters, logdir) + finally: + repo.update(cur_ref) + + # Generate diff(s) + diffcmd = [ + "diff", + "-U20", + "--report-identical-files", + f"--label={options['graph_attr']}@{base_ref}", + f"--label={options['graph_attr']}@{cur_ref}", + ] + + non_fatal_failures = [] + for spec in parameters: + base_path = os.path.join(diffdir, f"{options['graph_attr']}_{base_ref}") + cur_path = os.path.join(diffdir, f"{options['graph_attr']}_{cur_ref}") + + params_name = None + if len(parameters) > 1: + params_name = Parameters.format_spec(spec) + base_path += f"_{params_name}" + cur_path += f"_{params_name}" + + # If the base or cur files are missing it means that generation + # failed. If one of them failed but not the other, the failure is + # likely due to the patch making changes to taskgraph in modules + # that don't get reloaded (safe to ignore). If both generations + # failed, there's likely a real issue. + base_missing = not os.path.isfile(base_path) + cur_missing = not os.path.isfile(cur_path) + if base_missing != cur_missing: # != is equivalent to XOR for booleans + non_fatal_failures.append(os.path.basename(base_path)) + continue + + try: + # If the output file(s) are missing, this command will raise + # CalledProcessError with a returncode > 1. + proc = subprocess.run( + diffcmd + [base_path, cur_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + check=True, + ) + diff_output = proc.stdout + returncode = 0 + except subprocess.CalledProcessError as e: + # returncode 1 simply means diffs were found + if e.returncode != 1: + print(e.stderr, file=sys.stderr) + raise + diff_output = e.output + returncode = e.returncode + + dump_output( + diff_output, + # Don't bother saving file if no diffs were found. Log to + # console in this case instead. + path=None if returncode == 0 else output_file, + params_spec=spec if len(parameters) > 1 else None, + ) + + if non_fatal_failures: + failstr = "\n ".join(sorted(non_fatal_failures)) + print( + "WARNING: Diff skipped for the following generation{s} " + "due to failures:\n {failstr}".format( + s="s" if len(non_fatal_failures) > 1 else "", failstr=failstr + ), + file=sys.stderr, + ) + + if options["format"] != "json": + print( + "If you were expecting differences in task bodies " + 'you should pass "-J"\n', + file=sys.stderr, + ) + + if len(parameters) > 1: + print("See '{}' for logs".format(logdir), file=sys.stderr) + + +@command("build-image", help="Build a Docker image") +@argument("image_name", help="Name of the image to build") +@argument( + "-t", "--tag", help="tag that the image should be built as.", metavar="name:tag" +) +@argument( + "--context-only", + help="File name the context tarball should be written to." + "with this option it will only build the context.tar.", + metavar="context.tar", +) +def build_image(args): + from gecko_taskgraph.docker import build_context, build_image + + if args["context_only"] is None: + build_image(args["image_name"], args["tag"], os.environ) + else: + build_context(args["image_name"], args["context_only"], os.environ) + + +@command( + "load-image", + help="Load a pre-built Docker image. Note that you need to " + "have docker installed and running for this to work.", +) +@argument( + "--task-id", + help="Load the image at public/image.tar.zst in this task, " + "rather than searching the index", +) +@argument( + "-t", + "--tag", + help="tag that the image should be loaded as. If not " + "image will be loaded with tag from the tarball", + metavar="name:tag", +) +@argument( + "image_name", + nargs="?", + help="Load the image of this name based on the current " + "contents of the tree (as built for mozilla-central " + "or mozilla-inbound)", +) +def load_image(args): + from gecko_taskgraph.docker import load_image_by_name, load_image_by_task_id + + if not args.get("image_name") and not args.get("task_id"): + print("Specify either IMAGE-NAME or TASK-ID") + sys.exit(1) + try: + if args["task_id"]: + ok = load_image_by_task_id(args["task_id"], args.get("tag")) + else: + ok = load_image_by_name(args["image_name"], args.get("tag")) + if not ok: + sys.exit(1) + except Exception: + traceback.print_exc() + sys.exit(1) + + +@command("image-digest", help="Print the digest of a docker image.") +@argument( + "image_name", + help="Print the digest of the image of this name based on the current " + "contents of the tree.", +) +def image_digest(args): + from gecko_taskgraph.docker import get_image_digest + + try: + digest = get_image_digest(args["image_name"]) + print(digest) + except Exception: + traceback.print_exc() + sys.exit(1) + + +@command("decision", help="Run the decision task") +@argument("--root", "-r", help="root of the taskgraph definition relative to topsrcdir") +@argument( + "--message", + required=False, + help=argparse.SUPPRESS, +) +@argument( + "--project", + required=True, + help="Project to use for creating task graph. Example: --project=try", +) +@argument("--pushlog-id", dest="pushlog_id", required=True, default="0") +@argument("--pushdate", dest="pushdate", required=True, type=int, default=0) +@argument("--owner", required=True, help="email address of who owns this graph") +@argument("--level", required=True, help="SCM level of this repository") +@argument( + "--target-tasks-method", help="method for selecting the target tasks to generate" +) +@argument( + "--repository-type", + required=True, + help='Type of repository, either "hg" or "git"', +) +@argument("--base-repository", required=True, help='URL for "base" repository to clone') +@argument( + "--base-ref", default="", help='Reference of the revision in the "base" repository' +) +@argument( + "--base-rev", + default="", + help="Taskgraph decides what to do based on the revision range between " + "`--base-rev` and `--head-rev`. Value is determined automatically if not provided", +) +@argument( + "--head-repository", + required=True, + help='URL for "head" repository to fetch revision from', +) +@argument( + "--head-ref", required=True, help="Reference (this is same as rev usually for hg)" +) +@argument( + "--head-rev", required=True, help="Commit revision to use from head repository" +) +@argument("--head-tag", help="Tag attached to the revision", default="") +@argument( + "--tasks-for", required=True, help="the tasks_for value used to generate this task" +) +@argument("--try-task-config-file", help="path to try task configuration file") +def decision(options): + from gecko_taskgraph.decision import taskgraph_decision + + taskgraph_decision(options) + + +@command("action-callback", description="Run action callback used by action tasks") +@argument( + "--root", + "-r", + default="taskcluster/ci", + help="root of the taskgraph definition relative to topsrcdir", +) +def action_callback(options): + from gecko_taskgraph.actions import trigger_action_callback + from gecko_taskgraph.actions.util import get_parameters + + try: + # the target task for this action (or null if it's a group action) + task_id = json.loads(os.environ.get("ACTION_TASK_ID", "null")) + # the target task group for this action + task_group_id = os.environ.get("ACTION_TASK_GROUP_ID", None) + input = json.loads(os.environ.get("ACTION_INPUT", "null")) + callback = os.environ.get("ACTION_CALLBACK", None) + root = options["root"] + + parameters = get_parameters(task_group_id) + + return trigger_action_callback( + task_group_id=task_group_id, + task_id=task_id, + input=input, + callback=callback, + parameters=parameters, + root=root, + test=False, + ) + except Exception: + traceback.print_exc() + sys.exit(1) + + +@command("test-action-callback", description="Run an action callback in a testing mode") +@argument( + "--root", + "-r", + default="taskcluster/ci", + help="root of the taskgraph definition relative to topsrcdir", +) +@argument( + "--parameters", + "-p", + default="", + help="parameters file (.yml or .json; see " "`taskcluster/docs/parameters.rst`)`", +) +@argument("--task-id", default=None, help="TaskId to which the action applies") +@argument( + "--task-group-id", default=None, help="TaskGroupId to which the action applies" +) +@argument("--input", default=None, help="Action input (.yml or .json)") +@argument("callback", default=None, help="Action callback name (Python function name)") +def test_action_callback(options): + import taskgraph.parameters + from taskgraph.config import load_graph_config + from taskgraph.util import yaml + + import gecko_taskgraph.actions + + def load_data(filename): + with open(filename) as f: + if filename.endswith(".yml"): + return yaml.load_stream(f) + if filename.endswith(".json"): + return json.load(f) + raise Exception(f"unknown filename {filename}") + + try: + task_id = options["task_id"] + + if options["input"]: + input = load_data(options["input"]) + else: + input = None + + root = options["root"] + graph_config = load_graph_config(root) + trust_domain = graph_config["trust-domain"] + graph_config.register() + + parameters = taskgraph.parameters.load_parameters_file( + options["parameters"], strict=False, trust_domain=trust_domain + ) + parameters.check() + + return gecko_taskgraph.actions.trigger_action_callback( + task_group_id=options["task_group_id"], + task_id=task_id, + input=input, + callback=options["callback"], + parameters=parameters, + root=root, + test=True, + ) + except Exception: + traceback.print_exc() + sys.exit(1) + + +def create_parser(): + parser = argparse.ArgumentParser(description="Interact with taskgraph") + subparsers = parser.add_subparsers() + for _, (func, args, kwargs, defaults) in commands.items(): + subparser = subparsers.add_parser(*args, **kwargs) + for arg in func.args: + subparser.add_argument(*arg[0], **arg[1]) + subparser.set_defaults(command=func, **defaults) + return parser + + +def setup_logging(): + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO + ) + + +def main(args=sys.argv[1:]): + setup_logging() + parser = create_parser() + args = parser.parse_args(args) + try: + args.command(vars(args)) + except Exception: + traceback.print_exc() + sys.exit(1) diff --git a/taskcluster/gecko_taskgraph/manifests/fennec_geckoview.yml b/taskcluster/gecko_taskgraph/manifests/fennec_geckoview.yml new file mode 100644 index 0000000000..18974d3c19 --- /dev/null +++ b/taskcluster/gecko_taskgraph/manifests/fennec_geckoview.yml @@ -0,0 +1,210 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + - maven2 +default_locales: # Ignored for geckoview + - en-US +tasktype_map: # Map task reference to task type. + build: build + build-fat-aar: build + build-signing: signing + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + locale_prefix: '' + source_path_modifier: maven/org/mozilla/geckoview/${artifact_id}/${major_version}.${minor_version}.${build_date} + description: "TO_BE_OVERRIDDEN" + destinations: # locale_prefix is appended + - org/mozilla/geckoview/${artifact_id}/${major_version}.${minor_version}.${build_date} + +mapping: + ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar + ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.asc: + <<: *default + from: ['build-signing'] + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.asc + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.asc + ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.md5: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.md5 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.md5 + ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha1: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha1 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha1 + ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha256: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha256 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha256 + ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha512: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha512 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.aar.sha512 + ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom + ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.asc: + <<: *default + from: ['build-signing'] + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.asc + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.asc + ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.md5: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.md5 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.md5 + ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha1 + ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha256: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha256 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha256 + ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha512: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha512 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.pom.sha512 + ${artifact_id}-${major_version}.${minor_version}.${build_date}.module: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.module + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.module + ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.asc: + <<: *default + from: ['build-signing'] + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.asc + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.asc + ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.md5: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.md5 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.md5 + ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.sha1: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.sha1 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.sha1 + ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.sha256: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.sha256 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.sha256 + ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.sha512: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.sha512 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}.module.sha512 + ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar + ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.asc: + <<: *default + from: ['build-signing'] + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.asc + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.asc + ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.md5 + ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha1: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha1 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha1 + ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha256: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha256 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha256 + ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha512: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha512 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-javadoc.jar.sha512 + ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar + ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.asc: + <<: *default + from: ['build-signing'] + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.asc + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.asc + ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.md5: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.md5 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.md5 + ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.sha1: + <<: *default + from: + - build + - build-fat-aar + pretty_name: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.sha1 + checksums_path: ${artifact_id}-${major_version}.${minor_version}.${build_date}-sources.jar.sha1 diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml b/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml new file mode 100644 index 0000000000..e8d7061c78 --- /dev/null +++ b/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml @@ -0,0 +1,430 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file contains exhaustive information about all the release artifacs that +# are needed within a type of release. +# +# Structure +# -------- +# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets +# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of +# destinations. If given an empty locale, it uses these locales instead. +# `tasktype_map` -- mapping between task reference and task type, particularly usefule when +# composing the upstreamArtifacts for scriptworker. +# `platform_names` -- various platform mappings used in reckoning artifacts or other paths +# `default` -- a default entry, which the mappings extend and override in such a way that +# final path full-destinations will be a concatenation of the following: +# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name` +# `from` -- specifies the dependency(ies) from which to expect the particular artifact +# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected +# for all locales or just the default one +# `description` -- brief summary of what that artifact is +# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not +# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location +# for example `public/build` vs `public/build/ach/`. +# `destinations` -- final list of directories where to push the artifacts in S3 +# `pretty_name` -- the final name the artifact will have at destination +# `checksums_path` -- the name to identify one artifact within the checksums file +# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform +# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform +# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present +# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest +# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest + +s3_bucket_paths: + by-platform: + .*devedition.*: + - pub/devedition/candidates + default: + - pub/firefox/candidates +default_locales: + - en-US +tasktype_map: + build: build + signing: signing + mar-signing: signing + partials-signing: signing + repackage: repackage + repackage-deb: repackage + repackage-deb-l10n: repackage + repackage-signing: repackage + repackage-signing-msi: repackage + repackage-signing-shippable-l10n-msix: signing + langpack-copy: scriptworker + attribution: build + attribution-l10n: build +platform_names: + path_platform: + by-platform: + linux-shippable: 'linux-i686' + linux-devedition: 'linux-i686' + linux64-shippable: 'linux-x86_64' + linux64-devedition: 'linux-x86_64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'mac' + macosx64-devedition: 'mac' + win32-shippable: 'win32' + win32-devedition: 'win32' + win64-shippable: 'win64' + win64-devedition: 'win64' + win64-aarch64-shippable: 'win64-aarch64' + win64-aarch64-devedition: 'win64-aarch64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + tools_platform: + by-platform: + linux-shippable: 'linux' + linux-devedition: 'linux-devedition' + linux64-shippable: 'linux64' + linux64-devedition: 'linux64-devedition' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'macosx64' + macosx64-devedition: 'macosx64-devedition' + win32-shippable: 'win32' + win32-devedition: 'win32-devedition' + win64-shippable: 'win64' + win64-devedition: 'win64-devedition' + win64-aarch64-shippable: 'win64-aarch64' + win64-aarch64-devedition: 'win64-aarch64-devedition' + win64-asan-reporter-shippable: 'win64-asan-reporter' + filename_platform: + by-platform: + linux-shippable: 'linux' + linux-devedition: 'linux' + linux64-shippable: 'linux64' + linux64-devedition: 'linux64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'macosx64' + macosx64-devedition: 'macosx64' + win32-shippable: 'win32' + win32-devedition: 'win32' + win64-shippable: 'win64' + win64-devedition: 'win64' + win64-aarch64-shippable: 'win64_aarch64' + win64-aarch64-devedition: 'win64_aarch64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + +default: &default + from: + - build + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '${locale}/' + source_path_modifier: + by-locale: + default: '${locale}' + en-US: '' + destinations: + - ${version}-candidates/build${build_number}/${path_platform} + +mapping: + buildhub.json: + <<: *default + all_locales: false + description: "Build related information to be consumed by Buildhub service" + pretty_name: buildhub.json + checksums_path: ${path_platform}/${locale}/buildhub.json + target.common.tests.tar.gz: + <<: *default + description: "Mixture of reftests, mochitests, UI and others, commonly bundled together in a test suite" + pretty_name: firefox-${version}.common.tests.tar.gz + checksums_path: ${path_platform}/${locale}/firefox-${version}.common.tests.tar.gz + target.cppunittest.tests.tar.gz: + <<: *default + description: "C++ unittests related in-tree test infrastructure" + pretty_name: firefox-${version}.cppunittest.tests.tar.gz + checksums_path: ${path_platform}/${locale}/firefox-${version}.cppunittest.tests.tar.gz + target.crashreporter-symbols.zip: + <<: *default + description: "Crashreporter symbols to be consumed by Socorro" + pretty_name: firefox-${version}.crashreporter-symbols.zip + checksums_path: ${path_platform}/${locale}/firefox-${version}.crashreporter-symbols.zip + target.json: + <<: *default + description: "Various compile and moz_app flags baked together in a json file" + pretty_name: firefox-${version}.json + checksums_path: ${path_platform}/${locale}/firefox-${version}.json + target.mochitest.tests.tar.gz: + <<: *default + description: "Results for running the mochitest testing framework via Javascript function calls" + pretty_name: firefox-${version}.mochitest.tests.tar.gz + checksums_path: ${path_platform}/${locale}/firefox-${version}.mochitest.tests.tar.gz + target.mozinfo.json: + <<: *default + description: "Various compile and moz_app flags baked together in a json file" + pretty_name: firefox-${version}.mozinfo.json + checksums_path: ${path_platform}/${locale}/firefox-${version}.mozinfo.json + target.reftest.tests.tar.gz: + <<: *default + description: "Results for running the reftest testing framework via display of two Web pages comparison" + pretty_name: firefox-${version}.reftest.tests.tar.gz + checksums_path: ${path_platform}/${locale}/firefox-${version}.reftest.tests.tar.gz + target.talos.tests.tar.gz: + <<: *default + description: "Results for running the talos testing framework to measure performance" + pretty_name: firefox-${version}.talos.tests.tar.gz + checksums_path: ${path_platform}/${locale}/firefox-${version}.talos.tests.tar.gz + target.awsy.tests.tar.gz: + <<: *default + description: "Results for running the awsy testing framework to track memory usage" + pretty_name: firefox-${version}.awsy.tests.tar.gz + checksums_path: ${path_platform}/${locale}/firefox-${version}.awsy.tests.tar.gz + target.test_packages.json: + <<: *default + description: "File containing metadata about all other files and testing harnesses specifics" + pretty_name: firefox-${version}.test_packages.json + checksums_path: ${path_platform}/${locale}/firefox-${version}.test_packages.json + target.web-platform.tests.tar.gz: + <<: *default + description: "Results for running the webplatform testing framework to cover standard Web platform features" + pretty_name: firefox-${version}.web-platform.tests.tar.gz + checksums_path: ${path_platform}/${locale}/firefox-${version}.web-platform.tests.tar.gz + target.xpcshell.tests.tar.gz: + <<: *default + description: "Results for running the xpcshell testing framework to enable XPConnect console application" + pretty_name: firefox-${version}.xpcshell.tests.tar.gz + checksums_path: ${path_platform}/${locale}/firefox-${version}.xpcshell.tests.tar.gz + target_info.txt: + <<: *default + description: "File containing the buildID" + locale_prefix: '' + pretty_name: ${filename_platform}_info.txt + checksums_path: ${filename_platform}_info.txt + destinations: + - ${version}-candidates/build${build_number} + mozharness.zip: + <<: *default + description: "File containing the mozharness set of scripts and configuration used by various automation tools" + pretty_name: mozharness.zip + checksums_path: ${path_platform}/${locale}/mozharness.zip + target.jsshell.zip: + <<: *default + description: "Set of shells to allow test snippets of Javascript code without needing to reload the page" + locale_prefix: '' + pretty_name: jsshell-${path_platform}.zip + checksums_path: jsshell/jsshell-${path_platform}.zip + destinations: + - ${version}-candidates/build${build_number}/jsshell + target.langpack.xpi: + <<: *default + all_locales: true + description: "Localized repack that grabs a packaged en-US Firefox and repackages it as locale-specific Firefox" + locale_prefix: '' + from: + - langpack-copy + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux64-devedition + - macosx64-shippable + - win32-shippable + - win64-shippable + pretty_name: ${locale}.xpi + checksums_path: ${path_platform}/xpi/${locale}.xpi + destinations: + - ${version}-candidates/build${build_number}/${path_platform}/xpi + target.langpack.deb: + <<: *default + all_locales: true + description: "langpack.xpi repackaged as a .deb" + locale_prefix: '' + from: + - repackage-deb-l10n + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux-devedition + - linux64-devedition + pretty_name: ${locale}.deb + checksums_path: ${path_platform}/deb-l10n/${locale}.deb + destinations: + - ${version}-candidates/build${build_number}/${path_platform}/deb-l10n + update_balrog_manifest: false + mar: + <<: *default + description: "Alongside `mbsdiff`, a tool used to generate partials" + locale_prefix: '' + source_path_modifier: 'host/bin' + pretty_name: ${tools_platform}/mar + checksums_path: mar-tools/${tools_platform}/mar + not_for_platforms: + - win32-shippable + - win64-shippable + - win64-aarch64-shippable + - win32-devedition + - win64-devedition + - win64-aarch64-devedition + destinations: + - ${version}-candidates/build${build_number}/mar-tools + mbsdiff: + <<: *default + description: "Alongside `mar`, a tool used to generate partials" + locale_prefix: '' + source_path_modifier: 'host/bin' + pretty_name: ${tools_platform}/mbsdiff + checksums_path: mar-tools/${tools_platform}/mbsdiff + not_for_platforms: + - win32-shippable + - win64-shippable + - win64-aarch64-shippable + - win32-devedition + - win64-devedition + - win64-aarch64-devedition + destinations: + - ${version}-candidates/build${build_number}/mar-tools + target.tar.bz2: + <<: *default + description: "Main installer for Linux platforms" + all_locales: true + from: + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux-devedition + - linux64-devedition + pretty_name: firefox-${version}.tar.bz2 + checksums_path: ${path_platform}/${locale}/firefox-${version}.tar.bz2 + target.tar.bz2.asc: + <<: *default + description: "Detached signature for the checksums file" + all_locales: true + from: + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux-devedition + - linux64-devedition + pretty_name: firefox-${version}.tar.bz2.asc + checksums_path: ${path_platform}/${locale}/firefox-${version}.tar.bz2.asc + target.pkg: + <<: *default + description: "Main package installer for Mac OS X platforms" + all_locales: true + from: + - signing + only_for_platforms: + - macosx64-shippable + pretty_name: Firefox ${version}.pkg + checksums_path: ${path_platform}/${locale}/Firefox ${version}.pkg + target.dmg: + <<: *default + description: "Main package disk image for Mac OS X platforms" + all_locales: true + from: + - repackage + only_for_platforms: + - macosx64-shippable + - macosx64-devedition + pretty_name: Firefox ${version}.dmg + checksums_path: ${path_platform}/${locale}/Firefox ${version}.dmg + target.zip: + <<: *default + description: "Main package installer for Windows platforms" + all_locales: true + from: + - signing + only_for_platforms: + - win64-shippable + - win32-shippable + - win64-aarch64-shippable + - win64-devedition + - win32-devedition + - win64-aarch64-devedition + pretty_name: firefox-${version}.zip + checksums_path: ${path_platform}/${locale}/firefox-${version}.zip + target.installer.exe: + <<: *default + description: "Main installer for Windows platforms" + all_locales: true + source_path_modifier: '' + from: + - attribution + - attribution-l10n + only_for_platforms: + - win64-shippable + - win32-shippable + - win64-aarch64-shippable + - win64-devedition + - win32-devedition + - win64-aarch64-devedition + pretty_name: Firefox Setup ${version}.exe + checksums_path: ${path_platform}/${locale}/Firefox Setup ${version}.exe + target.stub-installer.exe: + <<: *default + description: "Stub installer for Win32 platforms" + all_locales: true + source_path_modifier: '' + from: + - attribution + - attribution-l10n + only_for_platforms: + - win32-shippable + - win32-devedition + pretty_name: Firefox Installer.exe + checksums_path: ${path_platform}/${locale}/Firefox Installer.exe + target.installer.msi: + <<: *default + description: "Windows installer for MSI platform" + all_locales: true + from: + - repackage-signing-msi + only_for_platforms: + - win64-shippable + - win32-shippable + - win64-devedition + - win32-devedition + pretty_name: Firefox Setup ${version}.msi + checksums_path: ${path_platform}/${locale}/Firefox Setup ${version}.msi + target.installer.msix: + <<: *default + description: "Windows MSIX installer" + from: + - repackage-signing-shippable-l10n-msix + only_for_platforms: + - win64-shippable + - win32-shippable + - win64-devedition + - win32-devedition + locale_prefix: 'multi/' + pretty_name: Firefox Setup ${version}.msix + checksums_path: ${path_platform}/multi/Firefox Setup ${version}.msix + target.complete.mar: + <<: *default + description: "Complete MAR to serve as updates" + all_locales: true + from: + - mar-signing + pretty_name: firefox-${version}.complete.mar + checksums_path: update/${path_platform}/${locale}/firefox-${version}.complete.mar + update_balrog_manifest: true + destinations: + - ${version}-candidates/build${build_number}/update/${path_platform} + target.deb: + <<: *default + description: "Firefox as a .deb package" + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux-devedition + - linux64-devedition + pretty_name: firefox-${version}.deb + checksums_path: ${path_platform}/${locale}/firefox-${version}.deb + from: + - repackage-deb + update_balrog_manifest: false + ${partial}: + <<: *default + description: "Partials MAR files to serve as updates" + all_locales: true + from: + - partials-signing + partials_only: true + pretty_name: firefox-${previous_version}-${version}.partial.mar + checksums_path: update/${path_platform}/${locale}/firefox-${previous_version}-${version}.partial.mar + update_balrog_manifest: true + from_buildid: ${from_buildid} + destinations: + - ${version}-candidates/build${build_number}/update/${path_platform} diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml b/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml new file mode 100644 index 0000000000..43ba4cbf15 --- /dev/null +++ b/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml @@ -0,0 +1,94 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file contains exhaustive information about all the release artifacs that +# are needed within a type of release. +# +# Structure +# -------- +# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets +# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of +# destinations. If given an empty locale, it uses these locales instead. +# `tasktype_map` -- mapping between task reference and task type, particularly usefule when +# composing the upstreamArtifacts for scriptworker. +# `platform_names` -- various platform mappings used in reckoning artifacts or other paths +# `default` -- a default entry, which the mappings extend and override in such a way that +# final path full-destinations will be a concatenation of the following: +# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name` +# `from` -- specifies the dependency(ies) from which to expect the particular artifact +# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected +# for all locales or just the default one +# `description` -- brief summary of what that artifact is +# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not +# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location +# for example `public/build` vs `public/build/ach/`. +# `destinations` -- final list of directories where to push the artifacts in S3 +# `pretty_name` -- the final name the artifact will have at destination +# `checksums_path` -- the name to identify one artifact within the checksums file +# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform +# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform +# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present +# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest +# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest + +s3_bucket_paths: + by-platform: + .*devedition.*: + - pub/devedition/candidates + default: + - pub/firefox/candidates +default_locales: + - en-US +tasktype_map: + beetmover-repackage: beetmover + release-beetmover-signed-langpacks: signing +platform_names: + path_platform: + by-platform: + linux-shippable: 'linux-i686' + linux-devedition: 'linux-i686' + linux64-shippable: 'linux-x86_64' + linux64-devedition: 'linux-x86_64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'mac' + macosx64-devedition: 'mac' + win32-shippable: 'win32' + win32-devedition: 'win32' + win64-shippable: 'win64' + win64-devedition: 'win64' + win64-aarch64-shippable: 'win64-aarch64' + win64-aarch64-devedition: 'win64-aarch64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + linux: 'linux-i686' + linux64: 'linux-x86_64' + macosx64: 'mac' + win32: 'win32' + win64: 'win64' + +default: &default + from: + - beetmover-repackage + all_locales: true + description: "TO_BE_OVERRIDDEN" + locale_prefix: '${locale}/' + source_path_modifier: '' + destinations: + - ${version}-candidates/build${build_number}/beetmover-checksums/${path_platform} + +mapping: + target.checksums: + <<: *default + description: "Checksums file containing size, hash, sha algorithm and filename" + pretty_name: firefox-${version}.checksums.beet + checksums_path: beetmover-checksums/${path_platform}/${locale}/firefox-${version}.checksums.beet + target-langpack.checksums: + <<: *default + description: "Checksums file containing size, hash, sha algorithm and filename for the langpack" + locale_prefix: '' + from: + - release-beetmover-signed-langpacks + pretty_name: ${locale}.checksums.beet + checksums_path: beetmover-checksums/${path_platform}/xpi/${locale}.checksums.beet + destinations: + - ${version}-candidates/build${build_number}/beetmover-checksums/${path_platform}/xpi diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml b/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml new file mode 100644 index 0000000000..d2e4f6adf0 --- /dev/null +++ b/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml @@ -0,0 +1,520 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file contains exhaustive information about all the release artifacs that +# are needed within a type of release. +# +# Structure +# -------- +# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets +# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of +# destinations. If given an empty locale, it uses these locales instead. +# `tasktype_map` -- mapping between task reference and task type, particularly usefule when +# composing the upstreamArtifacts for scriptworker. +# `platform_names` -- various platform mappings used in reckoning artifacts or other paths +# `default` -- a default entry, which the mappings extend and override in such a way that +# final path full-destinations will be a concatenation of the following: +# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name` +# `from` -- specifies the dependency(ies) from which to expect the particular artifact +# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected +# for all locales or just the default one +# `description` -- brief summary of what that artifact is +# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not +# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location +# for example `public/build` vs `public/build/ach/`. +# `destinations` -- final list of directories where to push the artifacts in S3 +# `pretty_name` -- the final name the artifact will have at destination +# `checksums_path` -- the name to identify one artifact within the checksums file +# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform +# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform +# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present +# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest +# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest + +s3_bucket_paths: + - pub/firefox/nightly +default_locales: + - en-US +tasktype_map: + build: build + signing: signing + mar-signing: signing + partials-signing: signing + repackage: repackage + repackage-deb: repackage + repackage-deb-l10n: repackage + repackage-signing: repackage + repackage-signing-msi: repackage + repackage-signing-shippable-l10n-msix: signing + langpack-copy: signing + attribution: build + attribution-l10n: build +platform_names: + filename_platform: + by-platform: + linux-shippable: 'linux-i686' + linux-devedition: 'linux-i686' + linux64-shippable: 'linux-x86_64' + linux64-devedition: 'linux-x86_64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'mac' + macosx64-devedition: 'mac' + win32-shippable: 'win32' + win32-devedition: 'win32' + win64-shippable: 'win64' + win64-devedition: 'win64' + win64-aarch64-shippable: 'win64-aarch64' + win64-aarch64-devedition: 'win64-aarch64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + stage_platform: + by-platform: + linux-shippable: 'linux' + linux-devedition: 'linux' + linux64-asan-reporter-shippable: 'linux64-asan-reporter' + linux64-shippable: 'linux64' + linux64-devedition: 'linux64' + macosx64-shippable: 'macosx64' + macosx64-devedition: 'macosx64' + win32-shippable: 'win32' + win32-devedition: 'win32' + win64-shippable: 'win64' + win64-devedition: 'win64' + win64-aarch64-shippable: 'win64-aarch64' + win64-aarch64-devedition: 'win64-aarch64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + +default: &default + from: + - build + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: + by-locale: + default: '${locale}' + en-US: '' + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + +mapping: + buildhub.json: + <<: *default + description: "Build related information to be consumed by Buildhub service" + pretty_name: firefox-${version}.${locale}.${filename_platform}.buildhub.json + checksums_path: firefox-${version}.${locale}.${filename_platform}.buildhub.json + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + KEY: + <<: *default + from: + - signing + description: "Public GPG Key" + pretty_name: KEY + checksums_path: KEY + only_for_platforms: + - linux64-shippable + destinations: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + target.common.tests.tar.gz: + <<: *default + description: "Mixture of reftests, mochitests, UI and others, commonly bundled together in a test suite" + pretty_name: firefox-${version}.${locale}.${filename_platform}.common.tests.tar.gz + checksums_path: firefox-${version}.${locale}.${filename_platform}.common.tests.tar.gz + target.cppunittest.tests.tar.gz: + <<: *default + description: "C++ unittests related in-tree test infrastructure" + pretty_name: firefox-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz + checksums_path: firefox-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz + target.crashreporter-symbols.zip: + <<: *default + description: "Crashreporter symbols to be consumed by Socorro" + pretty_name: firefox-${version}.${locale}.${filename_platform}.crashreporter-symbols.zip + checksums_path: firefox-${version}.${locale}.${filename_platform}.crashreporter-symbols.zip + not_for_platforms: + - linux64-asan-reporter-shippable + - win64-asan-reporter-shippable + target.json: + <<: *default + description: "Various compile and moz_app flags baked together in a json file" + pretty_name: firefox-${version}.${locale}.${filename_platform}.json + checksums_path: firefox-${version}.${locale}.${filename_platform}.json + target.mochitest.tests.tar.gz: + <<: *default + description: "Results for running the mochitest testing framework via Javascript function calls" + pretty_name: firefox-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz + checksums_path: firefox-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz + target.mozinfo.json: + <<: *default + description: "Various compile and moz_app flags baked together in a json file" + pretty_name: firefox-${version}.${locale}.${filename_platform}.mozinfo.json + checksums_path: firefox-${version}.${locale}.${filename_platform}.mozinfo.json + target.reftest.tests.tar.gz: + <<: *default + description: "Results for running the reftest testing framework via display of two Web pages comparison" + pretty_name: firefox-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz + checksums_path: firefox-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz + target.talos.tests.tar.gz: + <<: *default + description: "Results for running the talos testing framework to measure performance" + pretty_name: firefox-${version}.${locale}.${filename_platform}.talos.tests.tar.gz + checksums_path: firefox-${version}.${locale}.${filename_platform}.talos.tests.tar.gz + target.awsy.tests.tar.gz: + <<: *default + description: "Results for running the awsy testing framework to track memory usage" + pretty_name: firefox-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz + checksums_path: firefox-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz + target.test_packages.json: + <<: *default + description: "File containing metadata about all other files and testing harnesses specifics" + pretty_name: firefox-${version}.${locale}.${filename_platform}.test_packages.json + checksums_path: firefox-${version}.${locale}.${filename_platform}.test_packages.json + target.txt: + <<: *default + description: "File containing buildid and revision" + pretty_name: firefox-${version}.${locale}.${filename_platform}.txt + checksums_path: firefox-${version}.${locale}.${filename_platform}.txt + target.web-platform.tests.tar.gz: + <<: *default + description: "Results for running the webplatform testing framework to cover standard Web platform features" + pretty_name: firefox-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz + checksums_path: firefox-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz + target.xpcshell.tests.tar.gz: + <<: *default + description: "Results for running the xpcshell testing framework to enable XPConnect console application" + pretty_name: firefox-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz + checksums_path: firefox-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz + target_info.txt: + <<: *default + description: "File containing the buildID" + pretty_name: firefox-${version}.${locale}.${filename_platform}_info.txt + checksums_path: firefox-${version}.${locale}.${filename_platform}_info.txt + mozharness.zip: + <<: *default + description: "File containing the mozharness set of scripts and configuration used by various automation tools" + pretty_name: mozharness.zip + checksums_path: mozharness.zip + target.jsshell.zip: + <<: *default + description: "Set of shells to allow test snippets of Javascript code without needing to reload the page" + pretty_name: jsshell-${filename_platform}.zip + checksums_path: jsshell-${filename_platform}.zip + not_for_platforms: + - linux64-asan-reporter-shippable + - win64-asan-reporter-shippable + target.langpack.xpi: + <<: *default + all_locales: true + description: "Localized repack that grabs a packaged en-US Firefox and repackages it as locale-specific Firefox" + from: + - langpack-copy + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + - macosx64-shippable + - win64-shippable + - win32-shippable + - win64-shippable + - win64-aarch64-shippable + - win64-asan-reporter-shippable + - linux64-asan-reporter-shippable + pretty_name: firefox-${version}.${locale}.langpack.xpi + checksums_path: firefox-${version}.${locale}.langpack.xpi + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + default: + - ${year}/${month}/${upload_date}-${branch}-l10n/${filename_platform}/xpi + - latest-${branch}-l10n/${filename_platform}/xpi + target.langpack.deb: + <<: *default + all_locales: true + description: "langpack.xpi repackaged as a .deb" + from: + - repackage-deb-l10n + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux-devedition + - linux64-devedition + pretty_name: firefox-${version}.${locale}.langpack.deb + checksums_path: firefox-${version}.${locale}.langpack.deb + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + default: + - ${year}/${month}/${upload_date}-${branch}-l10n/${filename_platform}/deb-l10n + - latest-${branch}-l10n/${filename_platform}/deb-l10n + mar: + <<: *default + description: "Alongside `mbsdiff`, a tool used to generate partials" + source_path_modifier: 'host/bin' + pretty_name: mar + checksums_path: mar + not_for_platforms: + - win32-shippable + - win64-shippable + - win64-aarch64-shippable + - win64-asan-reporter-shippable + destinations: + - ${year}/${month}/${upload_date}-${branch}/mar-tools/${stage_platform} + - latest-${branch}/mar-tools/${stage_platform} + mbsdiff: + <<: *default + description: "Alongside `mar`, a tool used to generate partials" + source_path_modifier: 'host/bin' + pretty_name: mbsdiff + checksums_path: mbsdiff + not_for_platforms: + - win32-shippable + - win64-shippable + - win64-aarch64-shippable + - win64-asan-reporter-shippable + destinations: + - ${year}/${month}/${upload_date}-${branch}/mar-tools/${stage_platform} + - latest-${branch}/mar-tools/${stage_platform} + target.tar.bz2: + <<: *default + description: "Main installer for Linux platforms" + all_locales: true + from: + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux64-asan-reporter-shippable + pretty_name: firefox-${version}.${locale}.${filename_platform}.tar.bz2 + checksums_path: firefox-${version}.${locale}.${filename_platform}.tar.bz2 + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.tar.bz2.asc: + <<: *default + description: "Detached signature for the checksums file" + all_locales: true + from: + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux64-asan-reporter-shippable + pretty_name: firefox-${version}.${locale}.${filename_platform}.tar.bz2.asc + checksums_path: firefox-${version}.${locale}.${filename_platform}.tar.bz2.asc + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.pkg: + <<: *default + description: "Main package installer for Mac OS X platforms" + all_locales: true + from: + - signing + only_for_platforms: + - macosx64-shippable + pretty_name: firefox-${version}.${locale}.${filename_platform}.pkg + checksums_path: firefox-${version}.${locale}.${filename_platform}.pkg + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.dmg: + <<: *default + description: "Main package disk image for Mac OS X platforms" + all_locales: true + from: + - repackage + only_for_platforms: + - macosx64-shippable + pretty_name: firefox-${version}.${locale}.${filename_platform}.dmg + checksums_path: firefox-${version}.${locale}.${filename_platform}.dmg + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.zip: + <<: *default + description: "Main package installer for Windows platforms" + all_locales: true + from: + - signing + only_for_platforms: + - win64-shippable + - win32-shippable + - win64-aarch64-shippable + - win64-asan-reporter-shippable + pretty_name: firefox-${version}.${locale}.${filename_platform}.zip + checksums_path: firefox-${version}.${locale}.${filename_platform}.zip + target.installer.exe: + <<: *default + description: "Main installer for Windows platforms" + all_locales: true + source_path_modifier: '' + from: + by-platform: + win64-asan-reporter-shippable: + - repackage-signing + default: + - attribution + - attribution-l10n + only_for_platforms: + - win64-shippable + - win32-shippable + - win64-aarch64-shippable + - win64-asan-reporter-shippable + pretty_name: firefox-${version}.${locale}.${filename_platform}.installer.exe + checksums_path: firefox-${version}.${locale}.${filename_platform}.installer.exe + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.stub-installer.exe: + <<: *default + description: "Stub installer for Win32 platforms" + all_locales: true + source_path_modifier: '' + from: + - attribution + - attribution-l10n + only_for_platforms: + - win32-shippable + pretty_name: Firefox Installer.${locale}.exe + checksums_path: Firefox Installer.${locale}.exe + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.installer.msi: + <<: *default + description: "Windows installer for MSI platform" + all_locales: true + from: + - repackage-signing-msi + only_for_platforms: + - win64-shippable + - win32-shippable + pretty_name: firefox-${version}.${locale}.${filename_platform}.installer.msi + checksums_path: firefox-${version}.${locale}.${filename_platform}.installer.msi + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.installer.msix: + <<: *default + description: "Windows MSIX installer" + all_locales: true + from: + - repackage-signing-shippable-l10n-msix + only_for_platforms: + - win64-shippable + - win32-shippable + pretty_name: firefox-${version}.multi.${filename_platform}.installer.msix + checksums_path: firefox-${version}.multi.${filename_platform}.installer.msix + destinations: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + target.complete.mar: + <<: *default + description: "The main installer we ship our products baked within" + all_locales: true + from: + - mar-signing + pretty_name: firefox-${version}.${locale}.${filename_platform}.complete.mar + checksums_path: firefox-${version}.${locale}.${filename_platform}.complete.mar + update_balrog_manifest: true + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.deb: + <<: *default + description: "Firefox as a .deb package" + from: + - repackage-deb + only_for_platforms: + - linux-shippable + - linux64-shippable + pretty_name: firefox-${version}.${locale}.${filename_platform}.deb + checksums_path: firefox-${version}.${locale}.${filename_platform}.deb + update_balrog_manifest: false + destinations: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + ${partial}: + <<: *default + description: "Partials MAR files to serve as updates" + all_locales: true + from: + - partials-signing + partials_only: true + pretty_name: firefox-${branch}-${version}-${filename_platform}-${locale}-${from_buildid}-${buildid}.partial.mar + checksums_path: firefox-${branch}-${version}-${filename_platform}-${locale}-${from_buildid}-${buildid}.partial.mar + update_balrog_manifest: true + from_buildid: ${from_buildid} + destinations: + by-locale: + en-US: + - partials/${year}/${month}/${upload_date}-${branch} + default: + - partials/${year}/${month}/${upload_date}-${branch}-l10n diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml b/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml new file mode 100644 index 0000000000..f1b81572ab --- /dev/null +++ b/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + by-platform: + .*devedition.*: + - pub/devedition/nightly + default: + - pub/firefox/nightly +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + beetmover-repackage: beetmover +platform_names: + filename_platform: + by-platform: + linux-shippable: 'linux-i686' + linux-devedition: 'linux-i686' + linux64-shippable: 'linux-x86_64' + linux64-devedition: 'linux-x86_64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'mac' + macosx64-devedition: 'mac' + win32-shippable: 'win32' + win32-devedition: 'win32' + win64-shippable: 'win64' + win64-devedition: 'win64' + win64-aarch64-shippable: 'win64-aarch64' + win64-aarch64-devedition: 'win64-aarch64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - beetmover-repackage + all_locales: true + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + +# Configuration for individual files. Extends 'default', above. +mapping: + target.checksums: + <<: *default + description: "Checksums file containing size, hash, sha algorithm and filename" + pretty_name: firefox-${version}.${locale}.${filename_platform}.checksums + checksums_path: firefox-${version}.${locale}.${filename_platform}.checksums diff --git a/taskcluster/gecko_taskgraph/manifests/release_checksums.yml b/taskcluster/gecko_taskgraph/manifests/release_checksums.yml new file mode 100644 index 0000000000..c11e339958 --- /dev/null +++ b/taskcluster/gecko_taskgraph/manifests/release_checksums.yml @@ -0,0 +1,70 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + by-platform: + devedition-release: + - pub/devedition/candidates + firefox-release: + - pub/firefox/candidates +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + release-generate-checksums: build + release-generate-checksums-signing: signing + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - release-generate-checksums-signing + all_locales: true + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + - ${version}-candidates/build${build_number} + +# Configuration for individual files. Extends 'default', above. +mapping: + SHA256SUMMARY: + <<: *default + description: "Merkle-tree for the release artifacts with sha 256 hashes" + from: + - release-generate-checksums + pretty_name: SHA256SUMMARY + checksums_path: SHA256SUMMARY + SHA512SUMMARY: + <<: *default + description: "Merkle-tree for the release artifacts with sha 512 hashes" + from: + - release-generate-checksums + pretty_name: SHA512SUMMARY + checksums_path: SHA512SUMMARY + KEY: + <<: *default + description: "Public side of the key that was used to sign the release artifacts" + pretty_name: KEY + checksums_path: KEY + SHA256SUMS: + <<: *default + description: "Aggregated checksums with main installers details per platform in sha512 hashes" + pretty_name: SHA256SUMS + checksums_path: SHA256SUMS + SHA256SUMS.asc: + <<: *default + description: "Detached signature for the checksums file" + pretty_name: SHA256SUMS.asc + checksums_path: SHA256SUMS.asc + SHA512SUMS: + <<: *default + description: "Aggregated checksums with main installers details per platform in sha256 hashes" + pretty_name: SHA512SUMS + checksums_path: SHA512SUMS + SHA512SUMS.asc: + <<: *default + description: "Detached signature for the checksums file" + pretty_name: SHA512SUMS.asc + checksums_path: SHA512SUMS.asc diff --git a/taskcluster/gecko_taskgraph/manifests/source_checksums.yml b/taskcluster/gecko_taskgraph/manifests/source_checksums.yml new file mode 100644 index 0000000000..0789bcfa93 --- /dev/null +++ b/taskcluster/gecko_taskgraph/manifests/source_checksums.yml @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + by-platform: + devedition-source: + - pub/devedition/candidates + firefox-source: + - pub/firefox/candidates +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + release-source-checksums-signing: signing + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - release-source-checksums-signing + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + - ${version}-candidates/build${build_number}/beetmover-checksums/source + +# Configuration for individual files. Extends 'default', above. +mapping: + target-source.checksums: + <<: *default + description: "Checksums file for the source zip files" + pretty_name: + by-platform: + firefox-source: firefox-${version}.checksums.beet + devedition-source: firefox-${version}.checksums.beet + checksums_path: + by-platform: + firefox-source: firefox-${version}.checksums.beet + devedition-source: firefox-${version}.checksums.beet + target-source.checksums.asc: + <<: *default + description: "Detached signature for the checksums file" + pretty_name: + by-platform: + firefox-source: firefox-${version}.checksums.asc + devedition-source: firefox-${version}.checksums.asc + checksums_path: + by-platform: + firefox-source: firefox-${version}.checksums.asc + devedition-source: firefox-${version}.checksums.asc diff --git a/taskcluster/gecko_taskgraph/manifests/source_files.yml b/taskcluster/gecko_taskgraph/manifests/source_files.yml new file mode 100644 index 0000000000..0f5f5b5250 --- /dev/null +++ b/taskcluster/gecko_taskgraph/manifests/source_files.yml @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + by-platform: + devedition-source: + - pub/devedition/candidates + firefox-source: + - pub/firefox/candidates +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + release-source-signing: signing + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - release-source-signing + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + - ${version}-candidates/build${build_number}/source + +# Configuration for individual files. Extends 'default', above. +mapping: + source.tar.xz: + <<: *default + description: "Source file with the in-tree code archived" + pretty_name: + by-platform: + firefox-source: firefox-${version}.source.tar.xz + devedition-source: firefox-${version}.source.tar.xz + checksums_path: + by-platform: + firefox-source: source/firefox-${version}.source.tar.xz + devedition-source: source/firefox-${version}.source.tar.xz + source.tar.xz.asc: + <<: *default + description: "Detached signature for the source file" + pretty_name: + by-platform: + firefox-source: firefox-${version}.source.tar.xz.asc + devedition-source: firefox-${version}.source.tar.xz.asc + checksums_path: + by-platform: + firefox-source: source/firefox-${version}.source.tar.xz.asc + devedition-source: source/firefox-${version}.source.tar.xz.asc diff --git a/taskcluster/gecko_taskgraph/morph.py b/taskcluster/gecko_taskgraph/morph.py new file mode 100644 index 0000000000..1d03ddaab6 --- /dev/null +++ b/taskcluster/gecko_taskgraph/morph.py @@ -0,0 +1,263 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Graph morphs are modifications to task-graphs that take place *after* the +optimization phase. + +These graph morphs are largely invisible to developers running `./mach` +locally, so they should be limited to changes that do not modify the meaning of +the graph. +""" + +# Note that the translation of `{'task-reference': '..'}` and +# `artifact-reference` are handled in the optimization phase (since +# optimization involves dealing with taskIds directly). Similarly, +# `{'relative-datestamp': '..'}` is handled at the last possible moment during +# task creation. + + +import copy +import logging +import os +import re + +from slugid import nice as slugid +from taskgraph.graph import Graph +from taskgraph.morph import register_morph +from taskgraph.task import Task +from taskgraph.taskgraph import TaskGraph + +from .util.workertypes import get_worker_type + +here = os.path.abspath(os.path.dirname(__file__)) +logger = logging.getLogger(__name__) +MAX_ROUTES = 10 + + +def amend_taskgraph(taskgraph, label_to_taskid, to_add): + """Add the given tasks to the taskgraph, returning a new taskgraph""" + new_tasks = taskgraph.tasks.copy() + new_edges = set(taskgraph.graph.edges) + for task in to_add: + new_tasks[task.task_id] = task + assert task.label not in label_to_taskid + label_to_taskid[task.label] = task.task_id + for depname, dep in task.dependencies.items(): + new_edges.add((task.task_id, dep, depname)) + + taskgraph = TaskGraph(new_tasks, Graph(set(new_tasks), new_edges)) + return taskgraph, label_to_taskid + + +def derive_misc_task( + target_task, + purpose, + image, + taskgraph, + label_to_taskid, + parameters, + graph_config, + dependencies, +): + """Create the shell of a task that depends on `dependencies` and on the given docker + image.""" + label = f"{purpose}-{target_task.label}" + + # this is why all docker image tasks are included in the target task graph: we + # need to find them in label_to_taskid, even if nothing else required them + image_taskid = label_to_taskid["docker-image-" + image] + + provisioner_id, worker_type = get_worker_type( + graph_config, + parameters, + "misc", + ) + + deps = copy.copy(dependencies) + deps["docker-image"] = image_taskid + + task_def = { + "provisionerId": provisioner_id, + "workerType": worker_type, + "dependencies": [d for d in deps.values()], + "created": {"relative-datestamp": "0 seconds"}, + "deadline": target_task.task["deadline"], + # no point existing past the parent task's deadline + "expires": target_task.task["deadline"], + "metadata": { + "name": label, + "description": f"{purpose} for {target_task.description}", + "owner": target_task.task["metadata"]["owner"], + "source": target_task.task["metadata"]["source"], + }, + "scopes": [], + "payload": { + "image": { + "path": "public/image.tar.zst", + "taskId": image_taskid, + "type": "task-image", + }, + "features": {"taskclusterProxy": True}, + "maxRunTime": 600, + }, + } + + if image_taskid not in taskgraph.tasks: + # The task above depends on the replaced docker-image not one in + # this current graph. + del deps["docker-image"] + + task = Task( + kind="misc", + label=label, + attributes={}, + task=task_def, + dependencies=deps, + ) + task.task_id = slugid() + return task + + +# these regular expressions capture route prefixes for which we have a star +# scope, allowing them to be summarized. Each should correspond to a star scope +# in each Gecko `assume:repo:hg.mozilla.org/...` role. +SCOPE_SUMMARY_REGEXPS = [ + re.compile(r"(index:insert-task:docker\.images\.v1\.[^.]*\.).*"), + re.compile(r"(index:insert-task:gecko\.v2\.[^.]*\.).*"), + re.compile(r"(index:insert-task:comm\.v2\.[^.]*\.).*"), +] + + +def make_index_task( + parent_task, + taskgraph, + label_to_taskid, + parameters, + graph_config, + index_paths, + index_rank, + purpose, + dependencies, +): + task = derive_misc_task( + parent_task, + purpose, + "index-task", + taskgraph, + label_to_taskid, + parameters, + graph_config, + dependencies, + ) + + # we need to "summarize" the scopes, otherwise a particularly + # namespace-heavy index task might have more scopes than can fit in a + # temporary credential. + scopes = set() + for path in index_paths: + scope = f"index:insert-task:{path}" + for summ_re in SCOPE_SUMMARY_REGEXPS: + match = summ_re.match(scope) + if match: + scope = match.group(1) + "*" + break + scopes.add(scope) + task.task["scopes"] = sorted(scopes) + + task.task["payload"]["command"] = ["insert-indexes.js"] + index_paths + task.task["payload"]["env"] = { + "TARGET_TASKID": parent_task.task_id, + "INDEX_RANK": index_rank, + } + return task + + +@register_morph +def add_index_tasks(taskgraph, label_to_taskid, parameters, graph_config): + """ + The TaskCluster queue only allows 10 routes on a task, but we have tasks + with many more routes, for purposes of indexing. This graph morph adds + "index tasks" that depend on such tasks and do the index insertions + directly, avoiding the limits on task.routes. + """ + logger.debug("Morphing: adding index tasks") + + # Add indexes for tasks that exceed MAX_ROUTES. + added = [] + for label, task in taskgraph.tasks.items(): + if len(task.task.get("routes", [])) <= MAX_ROUTES: + continue + index_paths = [ + r.split(".", 1)[1] for r in task.task["routes"] if r.startswith("index.") + ] + task.task["routes"] = [ + r for r in task.task["routes"] if not r.startswith("index.") + ] + added.append( + make_index_task( + task, + taskgraph, + label_to_taskid, + parameters, + graph_config, + index_paths=index_paths, + index_rank=task.task.get("extra", {}).get("index", {}).get("rank", 0), + purpose="index-task", + dependencies={"parent": task.task_id}, + ) + ) + + if added: + taskgraph, label_to_taskid = amend_taskgraph(taskgraph, label_to_taskid, added) + logger.info(f"Added {len(added)} index tasks") + + return taskgraph, label_to_taskid + + +@register_morph +def add_eager_cache_index_tasks(taskgraph, label_to_taskid, parameters, graph_config): + """ + Some tasks (e.g. cached tasks) we want to exist in the index before they even + run/complete. Our current use is to allow us to depend on an unfinished cached + task in future pushes. This graph morph adds "eager-index tasks" that depend on + the decision task and do the index insertions directly, which does not need to + wait on the pointed at task to complete. + """ + logger.debug("Morphing: Adding eager cached index's") + + added = [] + for label, task in taskgraph.tasks.items(): + if "eager_indexes" not in task.attributes: + continue + eager_indexes = task.attributes["eager_indexes"] + added.append( + make_index_task( + task, + taskgraph, + label_to_taskid, + parameters, + graph_config, + index_paths=eager_indexes, + index_rank=0, # Be sure complete tasks get priority + purpose="eager-index", + dependencies={}, + ) + ) + + if added: + taskgraph, label_to_taskid = amend_taskgraph(taskgraph, label_to_taskid, added) + logger.info(f"Added {len(added)} eager index tasks") + return taskgraph, label_to_taskid + + +@register_morph +def add_try_task_duplicates(taskgraph, label_to_taskid, parameters, graph_config): + try_config = parameters["try_task_config"] + rebuild = try_config.get("rebuild") + if rebuild: + for task in taskgraph.tasks.values(): + if task.label in try_config.get("tasks", []): + task.attributes["task_duplicates"] = rebuild + return taskgraph, label_to_taskid diff --git a/taskcluster/gecko_taskgraph/optimize/__init__.py b/taskcluster/gecko_taskgraph/optimize/__init__.py new file mode 100644 index 0000000000..c2d3fdf839 --- /dev/null +++ b/taskcluster/gecko_taskgraph/optimize/__init__.py @@ -0,0 +1,287 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +The objective of optimization is to remove as many tasks from the graph as +possible, as efficiently as possible, thereby delivering useful results as +quickly as possible. For example, ideally if only a test script is modified in +a push, then the resulting graph contains only the corresponding test suite +task. + +See ``taskcluster/docs/optimization.rst`` for more information. +""" + +from taskgraph.optimize.base import Alias, All, Any, Not, register_strategy, registry +from taskgraph.util.python_path import import_sibling_modules + +# Use the gecko_taskgraph version of 'skip-unless-changed' for now. +registry.pop("skip-unless-changed", None) + +# Trigger registration in sibling modules. +import_sibling_modules() + + +def split_bugbug_arg(arg, substrategies): + """Split args for bugbug based strategies. + + Many bugbug based optimizations require passing an empty dict by reference + to communicate to downstream strategies. This function passes the provided + arg to the first (non bugbug) strategies and a shared empty dict to the + bugbug strategy and all substrategies after it. + """ + from gecko_taskgraph.optimize.bugbug import BugBugPushSchedules + + index = [ + i + for i, strategy in enumerate(substrategies) + if isinstance(strategy, BugBugPushSchedules) + ][0] + + return [arg] * index + [{}] * (len(substrategies) - index) + + +# Register composite strategies. +register_strategy("build", args=("skip-unless-schedules",))(Alias) +register_strategy("test", args=("skip-unless-schedules",))(Alias) +register_strategy("test-inclusive", args=("skip-unless-schedules",))(Alias) +register_strategy("test-verify", args=("skip-unless-schedules",))(Alias) +register_strategy("upload-symbols", args=("never",))(Alias) +register_strategy("reprocess-symbols", args=("never",))(Alias) + + +# Strategy overrides used to tweak the default strategies. These are referenced +# by the `optimize_strategies` parameter. + + +class project: + """Strategies that should be applied per-project.""" + + autoland = { + "test": Any( + # This `Any` strategy implements bi-modal behaviour. It allows different + # strategies on expanded pushes vs regular pushes. + # This first `All` handles "expanded" pushes. + All( + # There are three substrategies in this `All`, the first two act as barriers + # that help determine when to apply the third: + # 1. On backstop pushes, `skip-unless-backstop` returns False. Therefore + # the overall composite strategy is False and we don't optimize. + # 2. On regular pushes, `Not('skip-unless-expanded')` returns False. Therefore + # the overall composite strategy is False and we don't optimize. + # 3. On expanded pushes, the third strategy will determine whether or + # not to optimize each individual task. + # The barrier strategies. + "skip-unless-backstop", + Not("skip-unless-expanded"), + # The actual test strategy applied to "expanded" pushes. + Any( + "skip-unless-schedules", + "bugbug-reduced-manifests-fallback-last-10-pushes", + "platform-disperse", + split_args=split_bugbug_arg, + ), + ), + # This second `All` handles regular (aka not expanded or backstop) + # pushes. + All( + # There are two substrategies in this `All`, the first acts as a barrier + # that determines when to apply the second: + # 1. On expanded pushes (which includes backstops), `skip-unless-expanded` + # returns False. Therefore the overall composite strategy is False and we + # don't optimize. + # 2. On regular pushes, the second strategy will determine whether or + # not to optimize each individual task. + # The barrier strategy. + "skip-unless-expanded", + # The actual test strategy applied to regular pushes. + Any( + "skip-unless-schedules", + "bugbug-reduced-manifests-fallback-low", + "platform-disperse", + split_args=split_bugbug_arg, + ), + ), + ), + "build": All( + "skip-unless-expanded", + Any( + "skip-unless-schedules", + "bugbug-reduced-fallback", + split_args=split_bugbug_arg, + ), + ), + } + """Strategy overrides that apply to autoland.""" + + +class experimental: + """Experimental strategies either under development or used as benchmarks. + + These run as "shadow-schedulers" on each autoland push (tier 3) and/or can be used + with `./mach try auto`. E.g: + + ./mach try auto --strategy relevant_tests + """ + + bugbug_tasks_medium = { + "test": Any( + "skip-unless-schedules", "bugbug-tasks-medium", split_args=split_bugbug_arg + ), + } + """Doesn't limit platforms, medium confidence threshold.""" + + bugbug_tasks_high = { + "test": Any( + "skip-unless-schedules", "bugbug-tasks-high", split_args=split_bugbug_arg + ), + } + """Doesn't limit platforms, high confidence threshold.""" + + bugbug_debug_disperse = { + "test": Any( + "skip-unless-schedules", + "bugbug-low", + "platform-debug", + "platform-disperse", + split_args=split_bugbug_arg, + ), + } + """Restricts tests to debug platforms.""" + + bugbug_disperse_low = { + "test": Any( + "skip-unless-schedules", + "bugbug-low", + "platform-disperse", + split_args=split_bugbug_arg, + ), + } + """Disperse tests across platforms, low confidence threshold.""" + + bugbug_disperse_medium = { + "test": Any( + "skip-unless-schedules", + "bugbug-medium", + "platform-disperse", + split_args=split_bugbug_arg, + ), + } + """Disperse tests across platforms, medium confidence threshold.""" + + bugbug_disperse_reduced_medium = { + "test": Any( + "skip-unless-schedules", + "bugbug-reduced-manifests", + "platform-disperse", + split_args=split_bugbug_arg, + ), + } + """Disperse tests across platforms, medium confidence threshold with reduced tasks.""" + + bugbug_reduced_manifests_config_selection_low = { + "test": Any( + "skip-unless-schedules", + "bugbug-reduced-manifests-config-selection-low", + split_args=split_bugbug_arg, + ), + } + """Choose configs selected by bugbug, low confidence threshold with reduced tasks.""" + + bugbug_reduced_manifests_config_selection_medium = { + "test": Any( + "skip-unless-schedules", + "bugbug-reduced-manifests-config-selection", + split_args=split_bugbug_arg, + ), + } + """Choose configs selected by bugbug, medium confidence threshold with reduced tasks.""" + + bugbug_disperse_medium_no_unseen = { + "test": Any( + "skip-unless-schedules", + "bugbug-medium", + "platform-disperse-no-unseen", + split_args=split_bugbug_arg, + ), + } + """Disperse tests across platforms (no modified for unseen configurations), medium confidence + threshold.""" + + bugbug_disperse_medium_only_one = { + "test": Any( + "skip-unless-schedules", + "bugbug-medium", + "platform-disperse-only-one", + split_args=split_bugbug_arg, + ), + } + """Disperse tests across platforms (one platform per group), medium confidence threshold.""" + + bugbug_disperse_high = { + "test": Any( + "skip-unless-schedules", + "bugbug-high", + "platform-disperse", + split_args=split_bugbug_arg, + ), + } + """Disperse tests across platforms, high confidence threshold.""" + + bugbug_reduced = { + "test": Any( + "skip-unless-schedules", "bugbug-reduced", split_args=split_bugbug_arg + ), + } + """Use the reduced set of tasks (and no groups) chosen by bugbug.""" + + bugbug_reduced_high = { + "test": Any( + "skip-unless-schedules", "bugbug-reduced-high", split_args=split_bugbug_arg + ), + } + """Use the reduced set of tasks (and no groups) chosen by bugbug, high + confidence threshold.""" + + relevant_tests = { + "test": Any("skip-unless-schedules", "skip-unless-has-relevant-tests"), + } + """Runs task containing tests in the same directories as modified files.""" + + +class ExperimentalOverride: + """Overrides dictionaries that are stored in a container with new values. + + This can be used to modify all strategies in a collection the same way, + presumably with strategies affecting kinds of tasks tangential to the + current context. + + Args: + base (object): A container class supporting attribute access. + overrides (dict): Values to update any accessed dictionaries with. + """ + + def __init__(self, base, overrides): + self.base = base + self.overrides = overrides + + def __getattr__(self, name): + val = getattr(self.base, name).copy() + for name, strategy in self.overrides.items(): + if isinstance(strategy, str) and strategy.startswith("base:"): + strategy = val[strategy[len("base:") :]] + + val[name] = strategy + return val + + +tryselect = ExperimentalOverride( + experimental, + { + "build": Any( + "skip-unless-schedules", "bugbug-reduced", split_args=split_bugbug_arg + ), + "test-verify": "base:test", + "upload-symbols": Alias("always"), + "reprocess-symbols": Alias("always"), + }, +) diff --git a/taskcluster/gecko_taskgraph/optimize/backstop.py b/taskcluster/gecko_taskgraph/optimize/backstop.py new file mode 100644 index 0000000000..7b0c86222b --- /dev/null +++ b/taskcluster/gecko_taskgraph/optimize/backstop.py @@ -0,0 +1,47 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.optimize.base import All, OptimizationStrategy, register_strategy + +from gecko_taskgraph.util.backstop import BACKSTOP_PUSH_INTERVAL + + +@register_strategy("skip-unless-backstop") +class SkipUnlessBackstop(OptimizationStrategy): + """Always removes tasks except on backstop pushes.""" + + def should_remove_task(self, task, params, _): + return not params["backstop"] + + +class SkipUnlessPushInterval(OptimizationStrategy): + """Always removes tasks except every N pushes. + + Args: + push_interval (int): Number of pushes + """ + + def __init__(self, push_interval, remove_on_projects=None): + self.push_interval = push_interval + + @property + def description(self): + return f"skip-unless-push-interval-{self.push_interval}" + + def should_remove_task(self, task, params, _): + # On every Nth push, want to run all tasks. + return int(params["pushlog_id"]) % self.push_interval != 0 + + +# Strategy to run tasks on "expanded" pushes, currently defined as pushes that +# are half the backstop interval. The 'All' composite strategy means that the +# "backstop" strategy will prevent "expanded" from applying on backstop pushes. +register_strategy( + "skip-unless-expanded", + args=( + "skip-unless-backstop", + SkipUnlessPushInterval(BACKSTOP_PUSH_INTERVAL / 2), + ), +)(All) diff --git a/taskcluster/gecko_taskgraph/optimize/bugbug.py b/taskcluster/gecko_taskgraph/optimize/bugbug.py new file mode 100644 index 0000000000..d8603560ef --- /dev/null +++ b/taskcluster/gecko_taskgraph/optimize/bugbug.py @@ -0,0 +1,321 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from collections import defaultdict +from fnmatch import fnmatch + +from taskgraph.optimize.base import OptimizationStrategy, register_strategy, registry + +from gecko_taskgraph.util.bugbug import ( + CT_HIGH, + CT_LOW, + CT_MEDIUM, + BugbugTimeoutException, + push_schedules, +) +from gecko_taskgraph.util.hg import get_push_data + +FALLBACK = "skip-unless-has-relevant-tests" + + +def merge_bugbug_replies(data, new_data): + """Merge a bugbug reply (stored in the `new_data` argument) into another (stored + in the `data` argument). + """ + for key, value in new_data.items(): + if isinstance(value, dict): + if key not in data: + data[key] = {} + + if len(value) == 0: + continue + + dict_value = next(iter(value.values())) + if isinstance(dict_value, list): + for name, configs in value.items(): + if name not in data[key]: + data[key][name] = set() + + data[key][name].update(configs) + else: + for name, confidence in value.items(): + if name not in data[key] or data[key][name] < confidence: + data[key][name] = confidence + elif isinstance(value, list): + if key not in data: + data[key] = set() + + data[key].update(value) + + +@register_strategy("bugbug-low", args=(CT_LOW,)) +@register_strategy("bugbug-medium", args=(CT_MEDIUM,)) +@register_strategy("bugbug-high", args=(CT_HIGH,)) +@register_strategy("bugbug-tasks-medium", args=(CT_MEDIUM, True)) +@register_strategy("bugbug-tasks-high", args=(CT_HIGH, True)) +@register_strategy("bugbug-reduced", args=(CT_MEDIUM, True, True)) +@register_strategy("bugbug-reduced-fallback", args=(CT_MEDIUM, True, True, FALLBACK)) +@register_strategy("bugbug-reduced-high", args=(CT_HIGH, True, True)) +@register_strategy("bugbug-reduced-manifests", args=(CT_MEDIUM, False, True)) +@register_strategy( + "bugbug-reduced-manifests-config-selection-low", + args=(CT_LOW, False, True, None, 1, True), +) +@register_strategy( + "bugbug-reduced-manifests-config-selection", + args=(CT_MEDIUM, False, True, None, 1, True), +) +@register_strategy( + "bugbug-reduced-manifests-fallback-low", args=(CT_LOW, False, True, FALLBACK) +) +@register_strategy( + "bugbug-reduced-manifests-fallback", args=(CT_MEDIUM, False, True, FALLBACK) +) +@register_strategy( + "bugbug-reduced-manifests-fallback-last-10-pushes", + args=(0.3, False, True, FALLBACK, 10), +) +class BugBugPushSchedules(OptimizationStrategy): + """Query the 'bugbug' service to retrieve relevant tasks and manifests. + + Args: + confidence_threshold (float): The minimum confidence threshold (in + range [0, 1]) needed for a task to be scheduled. + tasks_only (bool): Whether or not to only use tasks and no groups + (default: False) + use_reduced_tasks (bool): Whether or not to use the reduced set of tasks + provided by the bugbug service (default: False). + fallback (str): The fallback strategy to use if there + was a failure in bugbug (default: None) + num_pushes (int): The number of pushes to consider for the selection + (default: 1). + select_configs (bool): Whether to select configurations for manifests + too (default: False). + """ + + def __init__( + self, + confidence_threshold, + tasks_only=False, + use_reduced_tasks=False, + fallback=None, + num_pushes=1, + select_configs=False, + ): + self.confidence_threshold = confidence_threshold + self.use_reduced_tasks = use_reduced_tasks + self.fallback = fallback + self.tasks_only = tasks_only + self.num_pushes = num_pushes + self.select_configs = select_configs + self.timedout = False + + def should_remove_task(self, task, params, importance): + project = params["project"] + + if project not in ("autoland", "try"): + return False + + current_push_id = int(params["pushlog_id"]) + + rev = params["head_rev"] + + if self.timedout: + return registry[self.fallback].should_remove_task(task, params, importance) + + data = {} + + start_push_id = current_push_id - self.num_pushes + 1 + if self.num_pushes != 1: + push_data = get_push_data( + params["head_repository"], project, start_push_id, current_push_id - 1 + ) + + for push_id in range(start_push_id, current_push_id + 1): + if push_id == current_push_id: + rev = params["head_rev"] + else: + rev = push_data[push_id]["changesets"][-1] + + try: + new_data = push_schedules(params["project"], rev) + merge_bugbug_replies(data, new_data) + except BugbugTimeoutException: + if not self.fallback: + raise + + self.timedout = True + return self.should_remove_task(task, params, importance) + + key = "reduced_tasks" if self.use_reduced_tasks else "tasks" + tasks = { + task + for task, confidence in data.get(key, {}).items() + if confidence >= self.confidence_threshold + } + + test_manifests = task.attributes.get("test_manifests") + if test_manifests is None or self.tasks_only: + if data.get("known_tasks") and task.label not in data["known_tasks"]: + return False + + if task.label not in tasks: + return True + + return False + + # If a task contains more than one group, use the max confidence. + groups = data.get("groups", {}) + confidences = [c for g, c in groups.items() if g in test_manifests] + if not confidences or max(confidences) < self.confidence_threshold: + return True + + # If the task configuration doesn't match the ones selected by bugbug for + # the manifests, optimize out. + if self.select_configs: + selected_groups = [ + g + for g, c in groups.items() + if g in test_manifests and c > self.confidence_threshold + ] + + config_groups = data.get("config_groups", defaultdict(list)) + + # Configurations returned by bugbug are in a format such as + # `test-windows10-64/opt-*-e10s`, while task labels are like + # test-windows10-64-qr/opt-mochitest-browser-chrome-e10s-6. + # In order to match the strings, we need to ignore the chunk number + # from the task label. + parts = task.label.split("-") + label_without_chunk_number = "-".join( + parts[:-1] if parts[-1].isdigit() else parts + ) + + if not any( + fnmatch(label_without_chunk_number, config) + for group in selected_groups + for config in config_groups[group] + ): + return True + + # Store group importance so future optimizers can access it. + for manifest in test_manifests: + if manifest not in groups: + continue + + confidence = groups[manifest] + if confidence >= CT_HIGH: + importance[manifest] = "high" + elif confidence >= CT_MEDIUM: + importance[manifest] = "medium" + elif confidence >= CT_LOW: + importance[manifest] = "low" + else: + importance[manifest] = "lowest" + + return False + + +@register_strategy("platform-debug") +class SkipUnlessDebug(OptimizationStrategy): + """Only run debug platforms.""" + + def should_remove_task(self, task, params, arg): + return ( + "build_type" in task.attributes and task.attributes["build_type"] != "debug" + ) + + +@register_strategy("platform-disperse") +@register_strategy("platform-disperse-no-unseen", args=(None, 0)) +@register_strategy( + "platform-disperse-only-one", + args=( + { + "high": 1, + "medium": 1, + "low": 1, + "lowest": 0, + }, + 0, + ), +) +class DisperseGroups(OptimizationStrategy): + """Disperse groups across test configs. + + Each task has an associated 'importance' dict passed in via the arg. This + is of the form `{<group>: <importance>}`. + + Where 'group' is a test group id (usually a path to a manifest), and 'importance' is + one of `{'lowest', 'low', 'medium', 'high'}`. + + Each importance value has an associated 'count' as defined in + `self.target_counts`. It guarantees that 'manifest' will run in at least + 'count' different configurations (assuming there are enough tasks + containing 'manifest'). + + On configurations that haven't been seen before, we'll increase the target + count by `self.unseen_modifier` to increase the likelihood of scheduling a + task on that configuration. + + Args: + target_counts (dict): Override DEFAULT_TARGET_COUNTS with custom counts. This + is a dict mapping the importance value ('lowest', 'low', etc) to the + minimum number of configurations manifests with this value should run + on. + + unseen_modifier (int): Override DEFAULT_UNSEEN_MODIFIER to a custom + value. This is the amount we'll increase 'target_count' by for unseen + configurations. + """ + + DEFAULT_TARGET_COUNTS = { + "high": 3, + "medium": 2, + "low": 1, + "lowest": 0, + } + DEFAULT_UNSEEN_MODIFIER = 1 + + def __init__(self, target_counts=None, unseen_modifier=DEFAULT_UNSEEN_MODIFIER): + self.target_counts = self.DEFAULT_TARGET_COUNTS.copy() + if target_counts: + self.target_counts.update(target_counts) + self.unseen_modifier = unseen_modifier + + self.count = defaultdict(int) + self.seen_configurations = set() + + def should_remove_task(self, task, params, importance): + test_manifests = task.attributes.get("test_manifests") + test_platform = task.attributes.get("test_platform") + + if not importance or not test_manifests or not test_platform: + return False + + # Build the test configuration key. + key = test_platform + if "unittest_variant" in task.attributes: + key += "-" + task.attributes["unittest_variant"] + + important_manifests = set(test_manifests) & set(importance) + for manifest in important_manifests: + target_count = self.target_counts[importance[manifest]] + + # If this configuration hasn't been seen before, increase the + # likelihood of scheduling the task. + if key not in self.seen_configurations: + target_count += self.unseen_modifier + + if self.count[manifest] < target_count: + # Update manifest counts and seen configurations. + self.seen_configurations.add(key) + for manifest in important_manifests: + self.count[manifest] += 1 + return False + + # Should remove task because all manifests have reached their + # importance count (or there were no important manifests). + return True diff --git a/taskcluster/gecko_taskgraph/optimize/schema.py b/taskcluster/gecko_taskgraph/optimize/schema.py new file mode 100644 index 0000000000..a7f878cf60 --- /dev/null +++ b/taskcluster/gecko_taskgraph/optimize/schema.py @@ -0,0 +1,60 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +import voluptuous +from mozbuild import schedules + +logger = logging.getLogger(__name__) + + +default_optimizations = ( + # always run this task (default) + None, + # always optimize this task + {"always": None}, + # optimize strategy aliases for build kind + {"build": list(schedules.ALL_COMPONENTS)}, + # search the index for the given index namespaces, and replace this task if found + # the search occurs in order, with the first match winning + {"index-search": [str]}, + # never optimize this task + {"never": None}, + # skip the task except for every Nth push + {"skip-unless-expanded": None}, + {"skip-unless-backstop": None}, + # skip this task if none of the given file patterns match + {"skip-unless-changed": [str]}, + # skip this task if unless the change files' SCHEDULES contains any of these components + {"skip-unless-schedules": list(schedules.ALL_COMPONENTS)}, + # optimize strategy aliases for the test kind + {"test": list(schedules.ALL_COMPONENTS)}, + {"test-inclusive": list(schedules.ALL_COMPONENTS)}, + # optimize strategy alias for test-verify tasks + {"test-verify": list(schedules.ALL_COMPONENTS)}, + # optimize strategy alias for upload-symbols tasks + {"upload-symbols": None}, + # optimize strategy alias for reprocess-symbols tasks + {"reprocess-symbols": None}, +) + +OptimizationSchema = voluptuous.Any(*default_optimizations) + + +def set_optimization_schema(schema_tuple): + """Sets OptimizationSchema so it can be imported by the task transform. + This function is called by projects that extend Firefox's taskgraph. + It should be called by the project's taskgraph:register function before + any transport or job runner code is imported. + + :param tuple schema_tuple: Tuple of possible optimization strategies + """ + global OptimizationSchema + if OptimizationSchema.validators == default_optimizations: + logger.info("OptimizationSchema updated.") + OptimizationSchema = voluptuous.Any(*schema_tuple) + else: + raise Exception("Can only call set_optimization_schema once.") diff --git a/taskcluster/gecko_taskgraph/optimize/strategies.py b/taskcluster/gecko_taskgraph/optimize/strategies.py new file mode 100644 index 0000000000..2e520c4750 --- /dev/null +++ b/taskcluster/gecko_taskgraph/optimize/strategies.py @@ -0,0 +1,136 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging +from datetime import datetime + +import mozpack.path as mozpath +from mozbuild.base import MozbuildObject +from mozbuild.util import memoize +from taskgraph.optimize.base import OptimizationStrategy, register_strategy +from taskgraph.util.taskcluster import find_task_id + +from gecko_taskgraph import files_changed +from gecko_taskgraph.util.taskcluster import status_task + +logger = logging.getLogger(__name__) + + +@register_strategy("index-search") +class IndexSearch(OptimizationStrategy): + + # A task with no dependencies remaining after optimization will be replaced + # if artifacts exist for the corresponding index_paths. + # Otherwise, we're in one of the following cases: + # - the task has un-optimized dependencies + # - the artifacts have expired + # - some changes altered the index_paths and new artifacts need to be + # created. + # In every of those cases, we need to run the task to create or refresh + # artifacts. + + fmt = "%Y-%m-%dT%H:%M:%S.%fZ" + + def should_replace_task(self, task, params, deadline, index_paths): + "Look for a task with one of the given index paths" + for index_path in index_paths: + try: + task_id = find_task_id(index_path) + status = status_task(task_id) + # status can be `None` if we're in `testing` mode + # (e.g. test-action-callback) + if not status or status.get("state") in ("exception", "failed"): + continue + + if deadline and datetime.strptime( + status["expires"], self.fmt + ) < datetime.strptime(deadline, self.fmt): + continue + + return task_id + except KeyError: + # 404 will end up here and go on to the next index path + pass + + return False + + +@register_strategy("skip-unless-changed") +class SkipUnlessChanged(OptimizationStrategy): + def should_remove_task(self, task, params, file_patterns): + # pushlog_id == -1 - this is the case when run from a cron.yml job + if params.get("pushlog_id") == -1: + return False + + changed = files_changed.check(params, file_patterns) + if not changed: + logger.debug( + "no files found matching a pattern in `skip-unless-changed` for " + + task.label + ) + return True + return False + + +@register_strategy("skip-unless-schedules") +class SkipUnlessSchedules(OptimizationStrategy): + @memoize + def scheduled_by_push(self, repository, revision): + changed_files = files_changed.get_changed_files(repository, revision) + + mbo = MozbuildObject.from_environment() + # the decision task has a sparse checkout, so, mozbuild_reader will use + # a MercurialRevisionFinder with revision '.', which should be the same + # as `revision`; in other circumstances, it will use a default reader + rdr = mbo.mozbuild_reader(config_mode="empty") + + components = set() + for p, m in rdr.files_info(changed_files).items(): + components |= set(m["SCHEDULES"].components) + + return components + + def should_remove_task(self, task, params, conditions): + if params.get("pushlog_id") == -1: + return False + + scheduled = self.scheduled_by_push( + params["head_repository"], params["head_rev"] + ) + conditions = set(conditions) + # if *any* of the condition components are scheduled, do not optimize + if conditions & scheduled: + return False + + return True + + +@register_strategy("skip-unless-has-relevant-tests") +class SkipUnlessHasRelevantTests(OptimizationStrategy): + """Optimizes tasks that don't run any tests that were + in child directories of a modified file. + """ + + @memoize + def get_changed_dirs(self, repo, rev): + changed = map(mozpath.dirname, files_changed.get_changed_files(repo, rev)) + # Filter out empty directories (from files modified in the root). + # Otherwise all tasks would be scheduled. + return {d for d in changed if d} + + def should_remove_task(self, task, params, _): + if not task.attributes.get("test_manifests"): + return True + + for d in self.get_changed_dirs(params["head_repository"], params["head_rev"]): + for t in task.attributes["test_manifests"]: + if t.startswith(d): + logger.debug( + "{} runs a test path ({}) contained by a modified file ({})".format( + task.label, t, d + ) + ) + return False + return True diff --git a/taskcluster/gecko_taskgraph/parameters.py b/taskcluster/gecko_taskgraph/parameters.py new file mode 100644 index 0000000000..e7e5df4605 --- /dev/null +++ b/taskcluster/gecko_taskgraph/parameters.py @@ -0,0 +1,100 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os + +from taskgraph.parameters import extend_parameters_schema +from voluptuous import Any, Required + +from gecko_taskgraph import GECKO + +logger = logging.getLogger(__name__) + + +gecko_parameters_schema = { + Required("app_version"): str, + Required("backstop"): bool, + Required("build_number"): int, + Required("enable_always_target"): bool, + Required("hg_branch"): str, + Required("message"): str, + Required("next_version"): Any(None, str), + Required("optimize_strategies"): Any(None, str), + Required("phabricator_diff"): Any(None, str), + Required("release_enable_emefree"): bool, + Required("release_enable_partner_repack"): bool, + Required("release_enable_partner_attribution"): bool, + Required("release_eta"): Any(None, str), + Required("release_history"): {str: dict}, + Required("release_partners"): Any(None, [str]), + Required("release_partner_config"): Any(None, dict), + Required("release_partner_build_number"): int, + Required("release_type"): str, + Required("release_product"): Any(None, str), + Required("required_signoffs"): [str], + Required("signoff_urls"): dict, + Required("test_manifest_loader"): str, + Required("try_mode"): Any(None, str), + Required("try_options"): Any(None, dict), + Required("try_task_config"): dict, + Required("version"): str, +} + + +def get_contents(path): + with open(path, "r") as fh: + contents = fh.readline().rstrip() + return contents + + +def get_version(product_dir="browser"): + version_path = os.path.join(GECKO, product_dir, "config", "version_display.txt") + return get_contents(version_path) + + +def get_app_version(product_dir="browser"): + app_version_path = os.path.join(GECKO, product_dir, "config", "version.txt") + return get_contents(app_version_path) + + +def get_defaults(repo_root=None): + return { + "app_version": get_app_version(), + "backstop": False, + "base_repository": "https://hg.mozilla.org/mozilla-unified", + "build_number": 1, + "enable_always_target": False, + "head_repository": "https://hg.mozilla.org/mozilla-central", + "hg_branch": "default", + "message": "", + "next_version": None, + "optimize_strategies": None, + "phabricator_diff": None, + "project": "mozilla-central", + "release_enable_emefree": False, + "release_enable_partner_repack": False, + "release_enable_partner_attribution": False, + "release_eta": "", + "release_history": {}, + "release_partners": [], + "release_partner_config": None, + "release_partner_build_number": 1, + "release_product": None, + "release_type": "nightly", + # This refers to the upstream repo rather than the local checkout, so + # should be hardcoded to 'hg' even with git-cinnabar. + "repository_type": "hg", + "required_signoffs": [], + "signoff_urls": {}, + "test_manifest_loader": "default", + "try_mode": None, + "try_options": None, + "try_task_config": {}, + "version": get_version(), + } + + +def register_parameters(): + extend_parameters_schema(gecko_parameters_schema, defaults_fn=get_defaults) diff --git a/taskcluster/gecko_taskgraph/target_tasks.py b/taskcluster/gecko_taskgraph/target_tasks.py new file mode 100644 index 0000000000..90ebd479ad --- /dev/null +++ b/taskcluster/gecko_taskgraph/target_tasks.py @@ -0,0 +1,1497 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import itertools +import os +import re +from datetime import datetime, timedelta + +from redo import retry +from taskgraph.parameters import Parameters +from taskgraph.target_tasks import _target_task, get_method +from taskgraph.util.taskcluster import find_task_id + +from gecko_taskgraph import GECKO, try_option_syntax +from gecko_taskgraph.util.attributes import ( + match_run_on_hg_branches, + match_run_on_projects, +) +from gecko_taskgraph.util.hg import find_hg_revision_push_info, get_hg_commit_message +from gecko_taskgraph.util.platforms import platform_family + +# Some tasks show up in the target task set, but are possibly special cases, +# uncommon tasks, or tasks running against limited hardware set that they +# should only be selectable with --full. +UNCOMMON_TRY_TASK_LABELS = [ + # Platforms and/or Build types + r"build-.*-gcp", # Bug 1631990 + r"mingwclang", # Bug 1631990 + r"valgrind", # Bug 1631990 + # Android tasks + r"android-geckoview-docs", + r"android-hw", + # Windows tasks + r"windows10-64-ref-hw", + r"windows10-aarch64-qr", + # Linux tasks + r"linux-", # hide all linux32 tasks by default - bug 1599197 + r"linux1804-32", # hide linux32 tests - bug 1599197 + # Test tasks + r"web-platform-tests.*backlog", # hide wpt jobs that are not implemented yet - bug 1572820 + r"-ccov", + r"-profiling-", # talos/raptor profiling jobs are run too often + # Hide shippable versions of tests we have opt versions of because the non-shippable + # versions are faster to run. This is mostly perf tests. + r"-shippable(?!.*(awsy|browsertime|marionette-headless|mochitest-devtools-chrome-fis|raptor|talos|web-platform-tests-wdspec-headless|mochitest-plain-headless))", # noqa - too long +] + + +def index_exists(index_path, reason=""): + print(f"Looking for existing index {index_path} {reason}...") + try: + task_id = find_task_id(index_path) + print(f"Index {index_path} exists: taskId {task_id}") + return True + except KeyError: + print(f"Index {index_path} doesn't exist.") + return False + + +def filter_out_shipping_phase(task, parameters): + return ( + # nightly still here because of geckodriver + not task.attributes.get("nightly") + and task.attributes.get("shipping_phase") in (None, "build") + ) + + +def filter_out_devedition(task, parameters): + return not task.attributes.get("shipping_product") == "devedition" + + +def filter_out_cron(task, parameters): + """ + Filter out tasks that run via cron. + """ + return not task.attributes.get("cron") + + +def filter_for_project(task, parameters): + """Filter tasks by project. Optionally enable nightlies.""" + run_on_projects = set(task.attributes.get("run_on_projects", [])) + return match_run_on_projects(parameters["project"], run_on_projects) + + +def filter_for_hg_branch(task, parameters): + """Filter tasks by hg branch. + If `run_on_hg_branch` is not defined, then task runs on all branches""" + run_on_hg_branches = set(task.attributes.get("run_on_hg_branches", ["all"])) + return match_run_on_hg_branches(parameters["hg_branch"], run_on_hg_branches) + + +def filter_on_platforms(task, platforms): + """Filter tasks on the given platform""" + platform = task.attributes.get("build_platform") + return platform in platforms + + +def filter_by_uncommon_try_tasks(task, optional_filters=None): + """Filters tasks that should not be commonly run on try. + + Args: + task (str): String representing the task name. + optional_filters (list, optional): + Additional filters to apply to task filtering. + + Returns: + (Boolean): True if task does not match any known filters. + False otherwise. + """ + filters = UNCOMMON_TRY_TASK_LABELS + if optional_filters: + filters = itertools.chain(filters, optional_filters) + + return not any(re.search(pattern, task) for pattern in filters) + + +def filter_by_regex(task_label, regexes, mode="include"): + """Filters tasks according to a list of pre-compiled reguar expressions. + + If mode is "include", a task label must match any regex to pass. + If it is "exclude", a task label must _not_ match any regex to pass. + """ + if not regexes: + return True + + assert mode in ["include", "exclude"] + + any_match = any(r.search(task_label) for r in regexes) + if any_match: + return mode == "include" + return mode != "include" + + +def filter_release_tasks(task, parameters): + platform = task.attributes.get("build_platform") + if platform in ( + "linux", + "linux64", + "macosx64", + "win32", + "win64", + "win64-aarch64", + ): + if task.attributes["kind"] == "l10n": + # This is on-change l10n + return True + if ( + task.attributes["build_type"] == "opt" + and task.attributes.get("unittest_suite") != "talos" + and task.attributes.get("unittest_suite") != "raptor" + ): + return False + + if task.attributes.get("shipping_phase") not in (None, "build"): + return False + + """ No debug on release, keep on ESR with 4 week cycles, release + will not be too different from central, but ESR will live for a long time. + + From June 2019 -> June 2020, we found 1 unique regression on ESR debug + and 5 unique regressions on beta/release. Keeping spidermonkey and linux + debug finds all but 1 unique regressions (windows found on try) for beta/release. + + ...but debug-only failures started showing up on ESR (esr-91, esr-102) so + desktop debug tests were added back for beta. + """ + build_type = task.attributes.get("build_type", "") + build_platform = task.attributes.get("build_platform", "") + test_platform = task.attributes.get("test_platform", "") + + if parameters["release_type"].startswith("esr") or ( + parameters["release_type"] == "beta" and "android" not in build_platform + ): + return True + + # code below here is intended to reduce release debug tasks + if task.kind == "hazard" or "toolchain" in build_platform: + # keep hazard and toolchain builds around + return True + + if build_type == "debug": + if "linux" not in build_platform: + # filter out windows/mac/android + return False + if task.kind not in ["spidermonkey"] and "-qr" in test_platform: + # filter out linux-qr tests, leave spidermonkey + return False + if "64" not in build_platform: + # filter out linux32 builds + return False + + # webrender-android-*-debug doesn't have attributes to find 'debug', using task.label. + if task.kind == "webrender" and "debug" in task.label: + return False + return True + + +def filter_out_missing_signoffs(task, parameters): + for signoff in parameters["required_signoffs"]: + if signoff not in parameters["signoff_urls"] and signoff in task.attributes.get( + "required_signoffs", [] + ): + return False + return True + + +def filter_tests_without_manifests(task, parameters): + """Remove test tasks that have an empty 'test_manifests' attribute. + + This situation can arise when the test loader (e.g bugbug) decided there + weren't any important manifests to run for the given push. We filter tasks + out here rather than in the transforms so that the full task graph is still + aware that the task exists (which is needed by the backfill action). + """ + if ( + task.kind == "test" + and "test_manifests" in task.attributes + and not task.attributes["test_manifests"] + ): + return False + return True + + +def standard_filter(task, parameters): + return all( + filter_func(task, parameters) + for filter_func in ( + filter_out_cron, + filter_for_project, + filter_for_hg_branch, + filter_tests_without_manifests, + ) + ) + + +def accept_raptor_android_build(platform): + """Helper function for selecting the correct android raptor builds.""" + if "android" not in platform: + return False + if "shippable" not in platform: + return False + if "p2" in platform and "aarch64" in platform: + return False + if "p5" in platform and "aarch64" in platform: + return False + if "g5" in platform: + return False + if "a51" in platform: + return True + + +def filter_unsupported_artifact_builds(task, parameters): + try_config = parameters.get("try_task_config", {}) + if not try_config.get("use-artifact-builds", False): + return True + + supports_artifact_builds = task.attributes.get("supports-artifact-builds", True) + return supports_artifact_builds + + +def filter_out_shippable(task): + return not task.attributes.get("shippable", False) + + +def _try_task_config(full_task_graph, parameters, graph_config): + requested_tasks = parameters["try_task_config"]["tasks"] + return list(set(requested_tasks) & full_task_graph.graph.nodes) + + +def _try_option_syntax(full_task_graph, parameters, graph_config): + """Generate a list of target tasks based on try syntax in + parameters['message'] and, for context, the full task graph.""" + options = try_option_syntax.TryOptionSyntax( + parameters, full_task_graph, graph_config + ) + target_tasks_labels = [ + t.label + for t in full_task_graph.tasks.values() + if options.task_matches(t) + and filter_by_uncommon_try_tasks(t.label) + and filter_unsupported_artifact_builds(t, parameters) + ] + + attributes = { + k: getattr(options, k) + for k in [ + "no_retry", + "tag", + ] + } + + for l in target_tasks_labels: + task = full_task_graph[l] + if "unittest_suite" in task.attributes: + task.attributes["task_duplicates"] = options.trigger_tests + + for l in target_tasks_labels: + task = full_task_graph[l] + # If the developer wants test jobs to be rebuilt N times we add that value here + if options.trigger_tests > 1 and "unittest_suite" in task.attributes: + task.attributes["task_duplicates"] = options.trigger_tests + + # If the developer wants test talos jobs to be rebuilt N times we add that value here + if ( + options.talos_trigger_tests > 1 + and task.attributes.get("unittest_suite") == "talos" + ): + task.attributes["task_duplicates"] = options.talos_trigger_tests + + # If the developer wants test raptor jobs to be rebuilt N times we add that value here + if ( + options.raptor_trigger_tests + and options.raptor_trigger_tests > 1 + and task.attributes.get("unittest_suite") == "raptor" + ): + task.attributes["task_duplicates"] = options.raptor_trigger_tests + + task.attributes.update(attributes) + + # Add notifications here as well + if options.notifications: + for task in full_task_graph: + owner = parameters.get("owner") + routes = task.task.setdefault("routes", []) + if options.notifications == "all": + routes.append(f"notify.email.{owner}.on-any") + elif options.notifications == "failure": + routes.append(f"notify.email.{owner}.on-failed") + routes.append(f"notify.email.{owner}.on-exception") + + return target_tasks_labels + + +@_target_task("try_tasks") +def target_tasks_try(full_task_graph, parameters, graph_config): + try_mode = parameters["try_mode"] + if try_mode == "try_task_config": + return _try_task_config(full_task_graph, parameters, graph_config) + if try_mode == "try_option_syntax": + return _try_option_syntax(full_task_graph, parameters, graph_config) + # With no try mode, we schedule nothing, allowing the user to add tasks + # later via treeherder. + return [] + + +@_target_task("try_select_tasks") +def target_tasks_try_select(full_task_graph, parameters, graph_config): + tasks = target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config) + return [l for l in tasks if filter_by_uncommon_try_tasks(l)] + + +@_target_task("try_select_tasks_uncommon") +def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config): + from gecko_taskgraph.decision import PER_PROJECT_PARAMETERS + + projects = ("autoland", "mozilla-central") + if parameters["project"] not in projects: + projects = (parameters["project"],) + + tasks = set() + for project in projects: + params = dict(parameters) + params["project"] = project + parameters = Parameters(**params) + + try: + target_tasks_method = PER_PROJECT_PARAMETERS[project]["target_tasks_method"] + except KeyError: + target_tasks_method = "default" + + tasks.update( + get_method(target_tasks_method)(full_task_graph, parameters, graph_config) + ) + + return sorted(tasks) + + +@_target_task("try_auto") +def target_tasks_try_auto(full_task_graph, parameters, graph_config): + """Target the tasks which have indicated they should be run on autoland + (rather than try) via the `run_on_projects` attributes. + + Should do the same thing as the `default` target tasks method. + """ + params = dict(parameters) + params["project"] = "autoland" + parameters = Parameters(**params) + + regex_filters = parameters["try_task_config"].get("tasks-regex") + include_regexes = exclude_regexes = [] + if regex_filters: + include_regexes = [re.compile(r) for r in regex_filters.get("include", [])] + exclude_regexes = [re.compile(r) for r in regex_filters.get("exclude", [])] + + return [ + l + for l, t in full_task_graph.tasks.items() + if standard_filter(t, parameters) + and filter_out_shipping_phase(t, parameters) + and filter_out_devedition(t, parameters) + and filter_by_uncommon_try_tasks(t.label) + and filter_by_regex(t.label, include_regexes, mode="include") + and filter_by_regex(t.label, exclude_regexes, mode="exclude") + and filter_unsupported_artifact_builds(t, parameters) + and filter_out_shippable(t) + ] + + +@_target_task("default") +def target_tasks_default(full_task_graph, parameters, graph_config): + """Target the tasks which have indicated they should be run on this project + via the `run_on_projects` attributes.""" + return [ + l + for l, t in full_task_graph.tasks.items() + if standard_filter(t, parameters) + and filter_out_shipping_phase(t, parameters) + and filter_out_devedition(t, parameters) + ] + + +@_target_task("autoland_tasks") +def target_tasks_autoland(full_task_graph, parameters, graph_config): + """In addition to doing the filtering by project that the 'default' + filter does, also remove any tests running against shippable builds + for non-backstop pushes.""" + filtered_for_project = target_tasks_default( + full_task_graph, parameters, graph_config + ) + + def filter(task): + if task.kind != "test": + return True + + if parameters["backstop"]: + return True + + build_type = task.attributes.get("build_type") + + if not build_type or build_type != "opt" or filter_out_shippable(task): + return True + + return False + + return [l for l in filtered_for_project if filter(full_task_graph[l])] + + +@_target_task("mozilla_central_tasks") +def target_tasks_mozilla_central(full_task_graph, parameters, graph_config): + """In addition to doing the filtering by project that the 'default' + filter does, also remove any tests running against regular (aka not shippable, + asan, etc.) opt builds.""" + filtered_for_project = target_tasks_default( + full_task_graph, parameters, graph_config + ) + + def filter(task): + if task.kind != "test": + return True + + build_platform = task.attributes.get("build_platform") + build_type = task.attributes.get("build_type") + shippable = task.attributes.get("shippable", False) + + if not build_platform or not build_type: + return True + + family = platform_family(build_platform) + # We need to know whether this test is against a "regular" opt build + # (which is to say, not shippable, asan, tsan, or any other opt build + # with other properties). There's no positive test for this, so we have to + # do it somewhat hackily. Android doesn't have variants other than shippable + # so it is pretty straightforward to check for. Other platforms have many + # variants, but none of the regular opt builds we're looking for have a "-" + # in their platform name, so this works (for now). + is_regular_opt = ( + family == "android" and not shippable + ) or "-" not in build_platform + + if build_type != "opt" or not is_regular_opt: + return True + + return False + + return [l for l in filtered_for_project if filter(full_task_graph[l])] + + +@_target_task("graphics_tasks") +def target_tasks_graphics(full_task_graph, parameters, graph_config): + """In addition to doing the filtering by project that the 'default' + filter does, also remove artifact builds because we have csets on + the graphics branch that aren't on the candidate branches of artifact + builds""" + filtered_for_project = target_tasks_default( + full_task_graph, parameters, graph_config + ) + + def filter(task): + if task.attributes["kind"] == "artifact-build": + return False + return True + + return [l for l in filtered_for_project if filter(full_task_graph[l])] + + +@_target_task("mozilla_beta_tasks") +def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a promotable beta or release build + of desktop, plus android CI. The candidates build process involves a pipeline + of builds and signing, but does not include beetmover or balrog jobs.""" + + return [ + l + for l, t in full_task_graph.tasks.items() + if filter_release_tasks(t, parameters) and standard_filter(t, parameters) + ] + + +@_target_task("mozilla_release_tasks") +def target_tasks_mozilla_release(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a promotable beta or release build + of desktop, plus android CI. The candidates build process involves a pipeline + of builds and signing, but does not include beetmover or balrog jobs.""" + + return [ + l + for l, t in full_task_graph.tasks.items() + if filter_release_tasks(t, parameters) and standard_filter(t, parameters) + ] + + +@_target_task("mozilla_esr102_tasks") +def target_tasks_mozilla_esr102(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a promotable beta or release build + of desktop, without android CI. The candidates build process involves a pipeline + of builds and signing, but does not include beetmover or balrog jobs.""" + + def filter(task): + if not filter_release_tasks(task, parameters): + return False + + if not standard_filter(task, parameters): + return False + + platform = task.attributes.get("build_platform") + + # Android is not built on esr102. + if platform and "android" in platform: + return False + + return True + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("mozilla_esr115_tasks") +def target_tasks_mozilla_esr115(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a promotable beta or release build + of desktop, without android CI. The candidates build process involves a pipeline + of builds and signing, but does not include beetmover or balrog jobs.""" + + def filter(task): + if not filter_release_tasks(task, parameters): + return False + + if not standard_filter(task, parameters): + return False + + platform = task.attributes.get("build_platform") + + # Android is not built on esr115. + if platform and "android" in platform: + return False + + return True + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("promote_desktop") +def target_tasks_promote_desktop(full_task_graph, parameters, graph_config): + """Select the superset of tasks required to promote a beta or release build + of a desktop product. This should include all non-android + mozilla_{beta,release} tasks, plus l10n, beetmover, balrog, etc.""" + + def filter(task): + # Bug 1758507 - geckoview ships in the promote phase + if not parameters["release_type"].startswith("esr") and is_geckoview( + task, parameters + ): + return True + + if task.attributes.get("shipping_product") != parameters["release_product"]: + return False + + # 'secondary' balrog/update verify/final verify tasks only run for RCs + if parameters.get("release_type") != "release-rc": + if "secondary" in task.kind: + return False + + if not filter_out_missing_signoffs(task, parameters): + return False + + if task.attributes.get("shipping_phase") == "promote": + return True + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +def is_geckoview(task, parameters): + return ( + task.attributes.get("shipping_product") == "fennec" + and task.kind in ("beetmover-geckoview", "upload-symbols") + and parameters["release_product"] == "firefox" + ) + + +@_target_task("push_desktop") +def target_tasks_push_desktop(full_task_graph, parameters, graph_config): + """Select the set of tasks required to push a build of desktop to cdns. + Previous build deps will be optimized out via action task.""" + filtered_for_candidates = target_tasks_promote_desktop( + full_task_graph, + parameters, + graph_config, + ) + + def filter(task): + if not filter_out_missing_signoffs(task, parameters): + return False + # Include promotion tasks; these will be optimized out + if task.label in filtered_for_candidates: + return True + + if ( + task.attributes.get("shipping_product") == parameters["release_product"] + and task.attributes.get("shipping_phase") == "push" + ): + return True + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("ship_desktop") +def target_tasks_ship_desktop(full_task_graph, parameters, graph_config): + """Select the set of tasks required to ship desktop. + Previous build deps will be optimized out via action task.""" + is_rc = parameters.get("release_type") == "release-rc" + if is_rc: + # ship_firefox_rc runs after `promote` rather than `push`; include + # all promote tasks. + filtered_for_candidates = target_tasks_promote_desktop( + full_task_graph, + parameters, + graph_config, + ) + else: + # ship_firefox runs after `push`; include all push tasks. + filtered_for_candidates = target_tasks_push_desktop( + full_task_graph, + parameters, + graph_config, + ) + + def filter(task): + if not filter_out_missing_signoffs(task, parameters): + return False + # Include promotion tasks; these will be optimized out + if task.label in filtered_for_candidates: + return True + + if ( + task.attributes.get("shipping_product") != parameters["release_product"] + or task.attributes.get("shipping_phase") != "ship" + ): + return False + + if "secondary" in task.kind: + return is_rc + return not is_rc + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("pine_tasks") +def target_tasks_pine(full_task_graph, parameters, graph_config): + """Bug 1339179 - no mobile automation needed on pine""" + + def filter(task): + platform = task.attributes.get("build_platform") + # disable mobile jobs + if str(platform).startswith("android"): + return False + # disable asan + if platform == "linux64-asan": + return False + # disable non-pine and tasks with a shipping phase + if standard_filter(task, parameters) or filter_out_shipping_phase( + task, parameters + ): + return True + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("kaios_tasks") +def target_tasks_kaios(full_task_graph, parameters, graph_config): + """The set of tasks to run for kaios integration""" + + def filter(task): + # We disable everything in central, and adjust downstream. + return False + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("ship_geckoview") +def target_tasks_ship_geckoview(full_task_graph, parameters, graph_config): + """Select the set of tasks required to ship geckoview nightly. The + nightly build process involves a pipeline of builds and an upload to + maven.mozilla.org.""" + index_path = ( + f"{graph_config['trust-domain']}.v2.{parameters['project']}.revision." + f"{parameters['head_rev']}.taskgraph.decision-ship-geckoview" + ) + if os.environ.get("MOZ_AUTOMATION") and retry( + index_exists, + args=(index_path,), + kwargs={ + "reason": "to avoid triggering multiple nightlies off the same revision", + }, + ): + return [] + + def filter(task): + # XXX Starting 69, we don't ship Fennec Nightly anymore. We just want geckoview to be + # uploaded + return task.attributes.get("shipping_product") == "fennec" and task.kind in ( + "beetmover-geckoview", + "upload-symbols", + ) + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("custom-car_perf_testing") +def target_tasks_custom_car_perf_testing(full_task_graph, parameters, graph_config): + """Select tasks required for running daily performance tests for custom chromium-as-release.""" + + def filter(task): + platform = task.attributes.get("test_platform") + attributes = task.attributes + if attributes.get("unittest_suite") != "raptor": + return False + + try_name = attributes.get("raptor_try_name") + + # Completely ignore all non-shippable platforms + if "shippable" not in platform: + return False + + # ignore all windows 7 perf jobs scheduled automatically + if "windows7" in platform or "windows10-32" in platform: + return False + + # Desktop selection only for CaR + if "android" not in platform: + if "browsertime" in try_name and "custom-car" in try_name: + return True + return False + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("general_perf_testing") +def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config): + """ + Select tasks required for running performance tests 3 times a week. + """ + + def filter(task): + platform = task.attributes.get("test_platform") + attributes = task.attributes + if attributes.get("unittest_suite") != "raptor": + return False + + try_name = attributes.get("raptor_try_name") + + # Completely ignore all non-shippable platforms + if "shippable" not in platform: + return False + + # ignore all windows 7 perf jobs scheduled automatically + if "windows7" in platform or "windows10-32" in platform: + return False + + # Desktop selection + if "android" not in platform: + # Select some browsertime tasks as desktop smoke-tests + if "browsertime" in try_name: + if "chrome" in try_name: + return True + if "chromium" in try_name: + return True + # chromium-as-release has it's own cron + if "custom-car" in try_name: + return False + if "-live" in try_name: + return True + if "-fis" in try_name: + return False + if "linux" in platform: + if "speedometer" in try_name: + return True + if "safari" and "benchmark" in try_name: + # Speedometer 3 is broken on Safari, see bug 1802922 + if "speedometer3" in try_name: + return False + return True + else: + # Don't run tp6 raptor tests + if "tp6" in try_name: + return False + # Android selection + elif accept_raptor_android_build(platform): + if "chrome-m" in try_name and ( + ("ebay" in try_name and "live" not in try_name) + or ( + "live" in try_name + and ("facebook" in try_name or "dailymail" in try_name) + ) + ): + return False + # Ignore all fennec tests here, we run those weekly + if "fennec" in try_name: + return False + # Only run webrender tests + if "chrome-m" not in try_name and "-qr" not in platform: + return False + # Select live site tests + if "-live" in try_name: + return True + # Select fenix resource usage tests + if "fenix" in try_name: + # Bug 1816421 disable fission perf tests + if "-fis" in try_name: + return False + if "-power" in try_name: + return True + # Select geckoview resource usage tests + if "geckoview" in try_name: + # Bug 1816421 disable fission perf tests + if "-fis" in try_name: + return False + # Run cpu+memory, and power tests + cpu_n_memory_task = "-cpu" in try_name and "-memory" in try_name + power_task = "-power" in try_name + # Ignore cpu+memory+power tests + if power_task and cpu_n_memory_task: + return False + if cpu_n_memory_task: + return False + if power_task: + return "browsertime" in try_name + # Select browsertime-specific tests + if "browsertime" in try_name: + if "speedometer" in try_name: + return True + return False + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +def make_desktop_nightly_filter(platforms): + """Returns a filter that gets all nightly tasks on the given platform.""" + + def filter(task, parameters): + return all( + [ + filter_on_platforms(task, platforms), + filter_for_project(task, parameters), + task.attributes.get("shippable", False), + # Tests and nightly only builds don't have `shipping_product` set + task.attributes.get("shipping_product") + in {None, "firefox", "thunderbird"}, + task.kind not in {"l10n"}, # no on-change l10n + ] + ) + + return filter + + +@_target_task("sp-perftests") +def target_tasks_speedometer_tests(full_task_graph, parameters, graph_config): + def filter(task): + platform = task.attributes.get("test_platform") + attributes = task.attributes + if attributes.get("unittest_suite") != "raptor": + return False + if "windows10-32" not in platform: + try_name = attributes.get("raptor_try_name") + if ( + "browsertime" in try_name + and "speedometer" in try_name + and "chrome" in try_name + ): + return True + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("nightly_linux") +def target_tasks_nightly_linux(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a nightly build of linux. The + nightly build process involves a pipeline of builds, signing, + and, eventually, uploading the tasks to balrog.""" + filter = make_desktop_nightly_filter({"linux64-shippable", "linux-shippable"}) + return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] + + +@_target_task("nightly_macosx") +def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a nightly build of macosx. The + nightly build process involves a pipeline of builds, signing, + and, eventually, uploading the tasks to balrog.""" + filter = make_desktop_nightly_filter({"macosx64-shippable"}) + return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] + + +@_target_task("nightly_win32") +def target_tasks_nightly_win32(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a nightly build of win32 and win64. + The nightly build process involves a pipeline of builds, signing, + and, eventually, uploading the tasks to balrog.""" + filter = make_desktop_nightly_filter({"win32-shippable"}) + return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] + + +@_target_task("nightly_win64") +def target_tasks_nightly_win64(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a nightly build of win32 and win64. + The nightly build process involves a pipeline of builds, signing, + and, eventually, uploading the tasks to balrog.""" + filter = make_desktop_nightly_filter({"win64-shippable"}) + return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] + + +@_target_task("nightly_win64_aarch64") +def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a nightly build of win32 and win64. + The nightly build process involves a pipeline of builds, signing, + and, eventually, uploading the tasks to balrog.""" + filter = make_desktop_nightly_filter({"win64-aarch64-shippable"}) + return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] + + +@_target_task("nightly_asan") +def target_tasks_nightly_asan(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a nightly build of asan. The + nightly build process involves a pipeline of builds, signing, + and, eventually, uploading the tasks to balrog.""" + filter = make_desktop_nightly_filter( + {"linux64-asan-reporter-shippable", "win64-asan-reporter-shippable"} + ) + return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] + + +@_target_task("daily_releases") +def target_tasks_daily_releases(full_task_graph, parameters, graph_config): + """Select the set of tasks required to identify if we should release. + If we determine that we should the task will communicate to ship-it to + schedule the release itself.""" + + def filter(task): + return task.kind in ["maybe-release"] + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("nightly_desktop") +def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a nightly build of linux, mac, + windows.""" + index_path = ( + f"{graph_config['trust-domain']}.v2.{parameters['project']}.revision." + f"{parameters['head_rev']}.taskgraph.decision-nightly-desktop" + ) + if os.environ.get("MOZ_AUTOMATION") and retry( + index_exists, + args=(index_path,), + kwargs={ + "reason": "to avoid triggering multiple nightlies off the same revision", + }, + ): + return [] + + # Tasks that aren't platform specific + release_filter = make_desktop_nightly_filter({None}) + release_tasks = [ + l for l, t in full_task_graph.tasks.items() if release_filter(t, parameters) + ] + # Avoid duplicate tasks. + return list( + set(target_tasks_nightly_win32(full_task_graph, parameters, graph_config)) + | set(target_tasks_nightly_win64(full_task_graph, parameters, graph_config)) + | set( + target_tasks_nightly_win64_aarch64( + full_task_graph, parameters, graph_config + ) + ) + | set(target_tasks_nightly_macosx(full_task_graph, parameters, graph_config)) + | set(target_tasks_nightly_linux(full_task_graph, parameters, graph_config)) + | set(target_tasks_nightly_asan(full_task_graph, parameters, graph_config)) + | set(release_tasks) + ) + + +# Run Searchfox analysis once daily. +@_target_task("searchfox_index") +def target_tasks_searchfox(full_task_graph, parameters, graph_config): + """Select tasks required for indexing Firefox for Searchfox web site each day""" + return [ + "searchfox-linux64-searchfox/debug", + "searchfox-macosx64-searchfox/debug", + "searchfox-win64-searchfox/debug", + "searchfox-android-armv7-searchfox/debug", + "source-test-file-metadata-bugzilla-components", + "source-test-file-metadata-test-info-all", + "source-test-wpt-metadata-summary", + ] + + +# Run build linux64-plain-clang-trunk/opt on mozilla-central/beta with perf tests +@_target_task("linux64_clang_trunk_perf") +def target_tasks_build_linux64_clang_trunk_perf( + full_task_graph, parameters, graph_config +): + """Select tasks required to run perf test on linux64 build with clang trunk""" + + # Only keep tasks generated from platform `linux1804-64-clang-trunk-qr/opt` + def filter(task_label): + if "linux1804-64-clang-trunk-qr/opt" in task_label: + return True + return False + + return [l for l, t in full_task_graph.tasks.items() if filter(t.label)] + + +# Run Updatebot's cron job 4 times daily. +@_target_task("updatebot_cron") +def target_tasks_updatebot_cron(full_task_graph, parameters, graph_config): + """Select tasks required to run Updatebot's cron job""" + return ["updatebot-cron"] + + +@_target_task("customv8_update") +def target_tasks_customv8_update(full_task_graph, parameters, graph_config): + """Select tasks required for building latest d8/v8 version.""" + return ["toolchain-linux64-custom-v8"] + + +@_target_task("chromium_update") +def target_tasks_chromium_update(full_task_graph, parameters, graph_config): + """Select tasks required for building latest chromium versions.""" + return [ + "fetch-linux64-chromium", + "fetch-win32-chromium", + "fetch-win64-chromium", + "fetch-mac-chromium", + "toolchain-linux64-custom-car", + "toolchain-win64-custom-car", + ] + + +@_target_task("file_update") +def target_tasks_file_update(full_task_graph, parameters, graph_config): + """Select the set of tasks required to perform nightly in-tree file updates""" + + def filter(task): + # For now any task in the repo-update kind is ok + return task.kind in ["repo-update"] + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("l10n_bump") +def target_tasks_l10n_bump(full_task_graph, parameters, graph_config): + """Select the set of tasks required to perform l10n bumping.""" + + def filter(task): + # For now any task in the repo-update kind is ok + return task.kind in ["l10n-bump"] + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("merge_automation") +def target_tasks_merge_automation(full_task_graph, parameters, graph_config): + """Select the set of tasks required to perform repository merges.""" + + def filter(task): + # For now any task in the repo-update kind is ok + return task.kind in ["merge-automation"] + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("scriptworker_canary") +def target_tasks_scriptworker_canary(full_task_graph, parameters, graph_config): + """Select the set of tasks required to run scriptworker canaries.""" + + def filter(task): + # For now any task in the repo-update kind is ok + return task.kind in ["scriptworker-canary"] + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("cron_bouncer_check") +def target_tasks_bouncer_check(full_task_graph, parameters, graph_config): + """Select the set of tasks required to perform bouncer version verification.""" + + def filter(task): + if not filter_for_project(task, parameters): + return False + # For now any task in the repo-update kind is ok + return task.kind in ["cron-bouncer-check"] + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("staging_release_builds") +def target_tasks_staging_release(full_task_graph, parameters, graph_config): + """ + Select all builds that are part of releases. + """ + + def filter(task): + if not task.attributes.get("shipping_product"): + return False + if parameters["release_type"].startswith( + "esr" + ) and "android" in task.attributes.get("build_platform", ""): + return False + if parameters["release_type"] != "beta" and "devedition" in task.attributes.get( + "build_platform", "" + ): + return False + if task.attributes.get("shipping_phase") == "build": + return True + return False + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("release_simulation") +def target_tasks_release_simulation(full_task_graph, parameters, graph_config): + """ + Select builds that would run on push on a release branch. + """ + project_by_release = { + "nightly": "mozilla-central", + "beta": "mozilla-beta", + "release": "mozilla-release", + "esr102": "mozilla-esr102", + "esr115": "mozilla-esr115", + } + target_project = project_by_release.get(parameters["release_type"]) + if target_project is None: + raise Exception("Unknown or unspecified release type in simulation run.") + + def filter_for_target_project(task): + """Filter tasks by project. Optionally enable nightlies.""" + run_on_projects = set(task.attributes.get("run_on_projects", [])) + return match_run_on_projects(target_project, run_on_projects) + + def filter_out_android_on_esr(task): + if parameters["release_type"].startswith( + "esr" + ) and "android" in task.attributes.get("build_platform", ""): + return False + return True + + return [ + l + for l, t in full_task_graph.tasks.items() + if filter_release_tasks(t, parameters) + and filter_out_cron(t, parameters) + and filter_for_target_project(t) + and filter_out_android_on_esr(t) + ] + + +@_target_task("codereview") +def target_tasks_codereview(full_task_graph, parameters, graph_config): + """Select all code review tasks needed to produce a report""" + + def filter(task): + # Ending tasks + if task.kind in ["code-review"]: + return True + + # Analyzer tasks + if task.attributes.get("code-review") is True: + return True + + return False + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("nothing") +def target_tasks_nothing(full_task_graph, parameters, graph_config): + """Select nothing, for DONTBUILD pushes""" + return [] + + +@_target_task("daily_beta_perf") +def target_tasks_daily_beta_perf(full_task_graph, parameters, graph_config): + """ + Select performance tests on the beta branch to be run daily + """ + index_path = ( + f"{graph_config['trust-domain']}.v2.{parameters['project']}.revision." + f"{parameters['head_rev']}.taskgraph.decision-daily-beta-perf" + ) + if os.environ.get("MOZ_AUTOMATION") and retry( + index_exists, + args=(index_path,), + kwargs={ + "reason": "to avoid triggering multiple daily beta perftests off of the same revision", + }, + ): + return [] + + def filter(task): + platform = task.attributes.get("test_platform") + attributes = task.attributes + try_name = attributes.get("raptor_try_name") + + if attributes.get("unittest_suite") != "raptor": + return False + + if platform and accept_raptor_android_build(platform): + # Select browsertime & geckoview specific tests + if "browsertime" and "geckoview" in try_name: + if "g5" in platform: + return False + if "power" in try_name: + return False + if "cpu" in try_name: + return False + if "profiling" in try_name: + return False + if "-live" in try_name: + return False + if "speedometer" in try_name: + return True + if "webgl" in try_name: + return True + if "tp6m" in try_name: + return True + + return False + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("weekly_release_perf") +def target_tasks_weekly_release_perf(full_task_graph, parameters, graph_config): + """ + Select performance tests on the release branch to be run weekly + """ + + def filter(task): + platform = task.attributes.get("test_platform") + attributes = task.attributes + try_name = attributes.get("raptor_try_name") + + if attributes.get("unittest_suite") != "raptor": + return False + + if platform and accept_raptor_android_build(platform): + # Select browsertime & geckoview specific tests + if "browsertime" and "geckoview" in try_name: + if "g5" in platform: + return False + if "power" in try_name: + return False + if "cpu" in try_name: + return False + if "profiling" in try_name: + return False + if "-live" in try_name: + return False + if "speedometer" in try_name: + return True + if "webgl" in try_name: + return True + if "tp6m" in try_name: + return True + if "youtube-playback" in try_name: + return True + + return False + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("raptor_tp6m") +def target_tasks_raptor_tp6m(full_task_graph, parameters, graph_config): + """ + Select tasks required for running raptor cold page-load tests on fenix and refbrow + """ + + def filter(task): + platform = task.attributes.get("build_platform") + attributes = task.attributes + + if platform and "android" not in platform: + return False + if attributes.get("unittest_suite") != "raptor": + return False + try_name = attributes.get("raptor_try_name") + if "-cold" in try_name and "shippable" in platform: + # Get browsertime amazon smoke tests + if ( + "browsertime" in try_name + and "amazon" in try_name + and "search" not in try_name + and "fenix" in try_name + ): + return True + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("backfill_all_browsertime") +def target_tasks_backfill_all_browsertime(full_task_graph, parameters, graph_config): + """ + Search for revisions that contains patches that were reviewed by perftest reviewers + and landed the day before the cron is running. Trigger backfill-all-browsertime action + task on each of them. + """ + from gecko_taskgraph.actions.util import get_decision_task_id, get_pushes + + def date_is_yesterday(date): + yesterday = datetime.today() - timedelta(days=1) + date = datetime.fromtimestamp(date) + return date.date() == yesterday.date() + + def reviewed_by_perftest(push): + try: + commit_message = get_hg_commit_message( + os.path.join(GECKO, graph_config["product-dir"]), rev=push + ) + except Exception as e: + print(e) + return False + + for line in commit_message.split("\n\n"): + if line.lower().startswith("bug ") and "r=" in line: + if "perftest-reviewers" in line.split("r=")[-1]: + print(line) + return True + return False + + pushes = get_pushes( + project=parameters["head_repository"], + end_id=int(parameters["pushlog_id"]), + depth=200, + full_response=True, + ) + for push_id in sorted([int(p) for p in pushes.keys()], reverse=True): + push_rev = pushes[str(push_id)]["changesets"][-1] + push_info = find_hg_revision_push_info( + "https://hg.mozilla.org/integration/" + parameters["project"], push_rev + ) + pushdate = int(push_info["pushdate"]) + if date_is_yesterday(pushdate) and reviewed_by_perftest(push_rev): + from gecko_taskgraph.actions.util import trigger_action + + print( + f"Revision {push_rev} was created yesterday and was reviewed by " + f"#perftest-reviewers." + ) + try: + push_decision_task_id = get_decision_task_id( + parameters["project"], push_id + ) + except Exception: + print(f"Could not find decision task for push {push_id}") + continue + try: + trigger_action( + action_name="backfill-all-browsertime", + # This lets the action know on which push we want to add a new task + decision_task_id=push_decision_task_id, + ) + except Exception as e: + print(f"Failed to trigger action for {push_rev}: {e}") + + return [] + + +@_target_task("condprof") +def target_tasks_condprof(full_task_graph, parameters, graph_config): + """ + Select tasks required for building conditioned profiles. + """ + for name, task in full_task_graph.tasks.items(): + if task.kind == "condprof": + if "a51" not in name: # bug 1765348 + yield name + + +@_target_task("system_symbols") +def target_tasks_system_symbols(full_task_graph, parameters, graph_config): + """ + Select tasks for scraping and uploading system symbols. + """ + for name, task in full_task_graph.tasks.items(): + if task.kind in [ + "system-symbols", + "system-symbols-upload", + "system-symbols-reprocess", + ]: + yield name + + +@_target_task("perftest") +def target_tasks_perftest(full_task_graph, parameters, graph_config): + """ + Select perftest tasks we want to run daily + """ + for name, task in full_task_graph.tasks.items(): + if task.kind != "perftest": + continue + if task.attributes.get("cron", False): + yield name + + +@_target_task("perftest-on-autoland") +def target_tasks_perftest_autoland(full_task_graph, parameters, graph_config): + """ + Select perftest tasks we want to run daily + """ + for name, task in full_task_graph.tasks.items(): + if task.kind != "perftest": + continue + if task.attributes.get("cron", False) and any( + test_name in name for test_name in ["view"] + ): + yield name + + +@_target_task("l10n-cross-channel") +def target_tasks_l10n_cross_channel(full_task_graph, parameters, graph_config): + """Select the set of tasks required to run l10n cross-channel.""" + + def filter(task): + return task.kind in ["l10n-cross-channel"] + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("are-we-esmified-yet") +def target_tasks_are_we_esmified_yet(full_task_graph, parameters, graph_config): + """ + select the task to track the progress of the esmification project + """ + return [ + l for l, t in full_task_graph.tasks.items() if t.kind == "are-we-esmified-yet" + ] + + +@_target_task("eslint-build") +def target_tasks_eslint_build(full_task_graph, parameters, graph_config): + """Select the task to run additional ESLint rules which require a build.""" + + for name, task in full_task_graph.tasks.items(): + if task.kind != "source-test": + continue + if "eslint-build" in name: + yield name + + +@_target_task("holly_tasks") +def target_tasks_holly(full_task_graph, parameters, graph_config): + """Bug 1814661: only run updatebot tasks on holly""" + + def filter(task): + return task.kind == "updatebot" + + return [l for l, t in full_task_graph.tasks.items() if filter(t)] diff --git a/taskcluster/gecko_taskgraph/test/__init__.py b/taskcluster/gecko_taskgraph/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/__init__.py diff --git a/taskcluster/gecko_taskgraph/test/automationrelevance.json b/taskcluster/gecko_taskgraph/test/automationrelevance.json new file mode 100644 index 0000000000..3bdfa9ed9e --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/automationrelevance.json @@ -0,0 +1,358 @@ +{ + "changesets": [ + { + "author": "James Long <longster@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1300866", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866" + } + ], + "date": [1473196655.0, 14400], + "desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins", + "extra": { + "branch": "default" + }, + "files": ["devtools/client/debugger/index.html"], + "node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "parents": ["37c9349b4e8167a61b08b7e119c21ea177b98942"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [1473261248, 0], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312890, + "reviewers": [ + { + "name": "jlast", + "revset": "reviewer(jlast)" + }, + { + "name": "bgrins", + "revset": "reviewer(bgrins)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Wes Kocher <wkocher@mozilla.com>", + "backsoutnodes": [], + "bugs": [], + "date": [1473208638.0, 25200], + "desc": "Merge m-c to fx-team, a=merge", + "extra": { + "branch": "default" + }, + "files": ["taskcluster/scripts/builder/build-l10n.sh"], + "node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "parents": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "91c2b9d5c1354ca79e5b174591dbb03b32b15bbf" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [1473261248, 0], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312891, + "reviewers": [ + { + "name": "merge", + "revset": "reviewer(merge)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Towkir Ahmed <towkir17@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1296648", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648" + } + ], + "date": [1472957580.0, 14400], + "desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim", + "extra": { + "branch": "default" + }, + "files": ["devtools/client/themes/rules.css"], + "node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "parents": ["73a6a267a50a0e1c41e689b265ad3eebe43d7ac6"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [1473261248, 0], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312892, + "reviewers": [ + { + "name": "ntim", + "revset": "reviewer(ntim)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Oriol <oriol-bugzilla@hotmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1300336", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336" + } + ], + "date": [1472921160.0, 14400], + "desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/webconsole/test/browser_webconsole_output_06.js", + "devtools/server/actors/object.js" + ], + "node": "99c542fa43a72ee863c813b5624048d1b443549b", + "parents": ["16a1a91f9269ab95dd83eb29dc5d0227665f7d94"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [1473261248, 0], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312893, + "reviewers": [ + { + "name": "fitzgen", + "revset": "reviewer(fitzgen)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Ruturaj Vartak <ruturaj@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1295010", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010" + } + ], + "date": [1472854020.0, -7200], + "desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK", + "extra": { + "amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd", + "branch": "default", + "histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba", + "rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651" + }, + "files": [ + "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js", + "devtools/client/inspector/test/head.js", + "devtools/server/actors/highlighters/eye-dropper.js" + ], + "node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "parents": ["99c542fa43a72ee863c813b5624048d1b443549b"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [1473261248, 0], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312894, + "reviewers": [ + { + "name": "pbro", + "revset": "reviewer(pbro)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Matteo Ferretti <mferretti@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1299154", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154" + } + ], + "date": [1472629906.0, -7200], + "desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n", + "extra": { + "branch": "default", + "committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200" + }, + "files": [ + "docshell/base/nsDocShell.cpp", + "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html" + ], + "node": "541c9086c0f27fba60beecc9bc94543103895c86", + "parents": ["a6b6a93eb41a05e310a11f0172f01ba9b21d3eac"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [1473261248, 0], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312895, + "reviewers": [ + { + "name": "mstange", + "revset": "reviewer(mstange)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Patrick Brosset <pbrosset@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1295010", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010" + } + ], + "date": [1473239449.0, -7200], + "desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js", + "devtools/client/inspector/test/head.js" + ], + "node": "041a925171e431bf51fb50193ab19d156088c89a", + "parents": ["541c9086c0f27fba60beecc9bc94543103895c86"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [1473261248, 0], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312896, + "reviewers": [ + { + "name": "me", + "revset": "reviewer(me)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>", + "backsoutnodes": [], + "bugs": [], + "date": [1473261233.0, -7200], + "desc": "merge fx-team to mozilla-central a=merge", + "extra": { + "branch": "default" + }, + "files": [], + "node": "a14f88a9af7a59e677478694bafd9375ac53683e", + "parents": [ + "3d0b41fdd93bd8233745eadb4e0358e385bf2cb9", + "041a925171e431bf51fb50193ab19d156088c89a" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [1473261248, 0], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312897, + "reviewers": [ + { + "name": "merge", + "revset": "reviewer(merge)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + } + ], + "visible": true +} diff --git a/taskcluster/gecko_taskgraph/test/conftest.py b/taskcluster/gecko_taskgraph/test/conftest.py new file mode 100644 index 0000000000..09231c5ab2 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/conftest.py @@ -0,0 +1,220 @@ +# Any copyright is dedicated to the public domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import os + +import pytest +from mach.logging import LoggingManager +from responses import RequestsMock +from taskgraph import generator as generator_mod +from taskgraph import target_tasks as target_tasks_mod +from taskgraph.config import GraphConfig, load_graph_config +from taskgraph.generator import Kind, TaskGraphGenerator +from taskgraph.optimize import base as optimize_mod +from taskgraph.optimize.base import OptimizationStrategy +from taskgraph.parameters import Parameters + +from gecko_taskgraph import GECKO +from gecko_taskgraph.actions import render_actions_json +from gecko_taskgraph.util.templates import merge + + +@pytest.fixture +def responses(): + with RequestsMock() as rsps: + yield rsps + + +@pytest.fixture(scope="session", autouse=True) +def patch_prefherder(request): + from _pytest.monkeypatch import MonkeyPatch + + m = MonkeyPatch() + m.setattr( + "gecko_taskgraph.util.bugbug._write_perfherder_data", + lambda lower_is_better: None, + ) + yield + m.undo() + + +@pytest.fixture(scope="session", autouse=True) +def enable_logging(): + """Ensure logs from gecko_taskgraph are displayed when a test fails.""" + lm = LoggingManager() + lm.add_terminal_logging() + + +@pytest.fixture(scope="session") +def graph_config(): + return load_graph_config(os.path.join(GECKO, "taskcluster", "ci")) + + +@pytest.fixture(scope="session") +def actions_json(graph_config): + decision_task_id = "abcdef" + return render_actions_json(Parameters(strict=False), graph_config, decision_task_id) + + +def fake_loader(kind, path, config, parameters, loaded_tasks): + for i in range(3): + dependencies = {} + if i >= 1: + dependencies["prev"] = f"{kind}-t-{i - 1}" + + task = { + "kind": kind, + "label": f"{kind}-t-{i}", + "description": f"{kind} task {i}", + "attributes": {"_tasknum": str(i)}, + "task": { + "i": i, + "metadata": {"name": f"t-{i}"}, + "deadline": "soon", + }, + "dependencies": dependencies, + } + if "job-defaults" in config: + task = merge(config["job-defaults"], task) + yield task + + +class FakeTransform: + transforms = [] + params = {} + + def __init__(self): + pass + + @classmethod + def get(self, field, default): + try: + return getattr(self, field) + except AttributeError: + return default + + +class FakeKind(Kind): + def _get_loader(self): + return fake_loader + + def load_tasks(self, parameters, loaded_tasks, write_artifacts): + FakeKind.loaded_kinds.append(self.name) + return super().load_tasks(parameters, loaded_tasks, write_artifacts) + + @staticmethod + def create(name, extra_config, graph_config): + if name == "fullfake": + config = FakeTransform() + else: + config = {"transforms": []} + if extra_config: + config.update(extra_config) + return FakeKind(name, "/fake", config, graph_config) + + +class WithFakeKind(TaskGraphGenerator): + def _load_kinds(self, graph_config, target_kind=None): + for kind_name, cfg in self.parameters["_kinds"]: + yield FakeKind.create(kind_name, cfg, graph_config) + + +def fake_load_graph_config(root_dir): + graph_config = GraphConfig( + {"trust-domain": "test-domain", "taskgraph": {}}, root_dir + ) + graph_config.__dict__["register"] = lambda: None + return graph_config + + +class FakeParameters(dict): + strict = True + + def file_url(self, path, pretty=False): + return "" + + +class FakeOptimization(OptimizationStrategy): + description = "Fake strategy for testing" + + def __init__(self, mode, *args, **kwargs): + super().__init__(*args, **kwargs) + self.mode = mode + + def should_remove_task(self, task, params, arg): + if self.mode == "always": + return True + if self.mode == "even": + return task.task["i"] % 2 == 0 + if self.mode == "odd": + return task.task["i"] % 2 != 0 + return False + + +@pytest.fixture +def maketgg(monkeypatch): + def inner(target_tasks=None, kinds=[("_fake", [])], params=None): + params = params or {} + FakeKind.loaded_kinds = loaded_kinds = [] + target_tasks = target_tasks or [] + + def target_tasks_method(full_task_graph, parameters, graph_config): + return target_tasks + + fake_registry = { + mode: FakeOptimization(mode) for mode in ("always", "never", "even", "odd") + } + + target_tasks_mod._target_task_methods["test_method"] = target_tasks_method + monkeypatch.setattr(optimize_mod, "registry", fake_registry) + + parameters = FakeParameters( + { + "_kinds": kinds, + "backstop": False, + "enable_always_target": False, + "target_tasks_method": "test_method", + "test_manifest_loader": "default", + "try_mode": None, + "try_task_config": {}, + "tasks_for": "hg-push", + "project": "mozilla-central", + } + ) + parameters.update(params) + + monkeypatch.setattr(generator_mod, "load_graph_config", fake_load_graph_config) + + tgg = WithFakeKind("/root", parameters) + tgg.loaded_kinds = loaded_kinds + return tgg + + return inner + + +@pytest.fixture +def run_transform(): + + graph_config = fake_load_graph_config("/root") + kind = FakeKind.create("fake", {}, graph_config) + + def inner(xform, tasks): + if isinstance(tasks, dict): + tasks = [tasks] + return xform(kind.config, tasks) + + return inner + + +@pytest.fixture +def run_full_config_transform(): + + graph_config = fake_load_graph_config("/root") + kind = FakeKind.create("fullfake", {}, graph_config) + + def inner(xform, tasks): + if isinstance(tasks, dict): + tasks = [tasks] + return xform(kind.config, tasks) + + return inner diff --git a/taskcluster/gecko_taskgraph/test/docs/kinds.rst b/taskcluster/gecko_taskgraph/test/docs/kinds.rst new file mode 100644 index 0000000000..fdc16db1e3 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/docs/kinds.rst @@ -0,0 +1,12 @@ +Task Kinds +========== + +Fake task kind documentation. + +newkind +---------- +Kind found in separate doc dir, + +anotherkind +----------- +Here's another. diff --git a/taskcluster/gecko_taskgraph/test/docs/parameters.rst b/taskcluster/gecko_taskgraph/test/docs/parameters.rst new file mode 100644 index 0000000000..f943f48e69 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/docs/parameters.rst @@ -0,0 +1,14 @@ +========== +Parameters +========== + +Fake parameters documentation. + +Heading +------- + +``newparameter`` + A new parameter that could be defined in a project. + +``anotherparameter`` + And here is another one. diff --git a/taskcluster/gecko_taskgraph/test/python.ini b/taskcluster/gecko_taskgraph/test/python.ini new file mode 100644 index 0000000000..2dca8fc144 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/python.ini @@ -0,0 +1,23 @@ +[DEFAULT] +subsuite = taskgraph + +[test_actions_util.py] +[test_decision.py] +[test_files_changed.py] +[test_main.py] +[test_morph.py] +[test_optimize_strategies.py] +[test_target_tasks.py] +[test_taskcluster_yml.py] +[test_transforms_job.py] +[test_transforms_test.py] +[test_try_option_syntax.py] +[test_util_attributes.py] +[test_util_backstop.py] +[test_util_bugbug.py] +[test_util_chunking.py] +[test_util_docker.py] +[test_util_partials.py] +[test_util_runnable_jobs.py] +[test_util_templates.py] +[test_util_verify.py] diff --git a/taskcluster/gecko_taskgraph/test/test_actions_util.py b/taskcluster/gecko_taskgraph/test/test_actions_util.py new file mode 100644 index 0000000000..7c38caea57 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_actions_util.py @@ -0,0 +1,179 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import unittest +from pprint import pprint +from unittest.mock import patch + +import pytest +from mozunit import MockedOpen, main +from taskgraph import create +from taskgraph.util import taskcluster + +from gecko_taskgraph import actions +from gecko_taskgraph.actions.util import combine_task_graph_files, relativize_datestamps +from gecko_taskgraph.decision import read_artifact + +TASK_DEF = { + "created": "2017-10-10T18:33:03.460Z", + # note that this is not an even number of seconds off! + "deadline": "2017-10-11T18:33:03.461Z", + "dependencies": [], + "expires": "2018-10-10T18:33:04.461Z", + "payload": { + "artifacts": { + "public": { + "expires": "2018-10-10T18:33:03.463Z", + "path": "/builds/worker/artifacts", + "type": "directory", + }, + }, + "maxRunTime": 1800, + }, +} + + +@pytest.fixture(scope="module", autouse=True) +def enable_test_mode(): + create.testing = True + taskcluster.testing = True + + +class TestRelativize(unittest.TestCase): + def test_relativize(self): + rel = relativize_datestamps(TASK_DEF) + import pprint + + pprint.pprint(rel) + assert rel["created"] == {"relative-datestamp": "0 seconds"} + assert rel["deadline"] == {"relative-datestamp": "86400 seconds"} + assert rel["expires"] == {"relative-datestamp": "31536001 seconds"} + assert rel["payload"]["artifacts"]["public"]["expires"] == { + "relative-datestamp": "31536000 seconds" + } + + +class TestCombineTaskGraphFiles(unittest.TestCase): + def test_no_suffixes(self): + with MockedOpen({}): + combine_task_graph_files([]) + self.assertRaises(Exception, open("artifacts/to-run.json")) + + @patch("gecko_taskgraph.actions.util.rename_artifact") + def test_one_suffix(self, rename_artifact): + combine_task_graph_files(["0"]) + rename_artifact.assert_any_call("task-graph-0.json", "task-graph.json") + rename_artifact.assert_any_call( + "label-to-taskid-0.json", "label-to-taskid.json" + ) + rename_artifact.assert_any_call("to-run-0.json", "to-run.json") + + def test_several_suffixes(self): + files = { + "artifacts/task-graph-0.json": json.dumps({"taska": {}}), + "artifacts/label-to-taskid-0.json": json.dumps({"taska": "TASKA"}), + "artifacts/to-run-0.json": json.dumps(["taska"]), + "artifacts/task-graph-1.json": json.dumps({"taskb": {}}), + "artifacts/label-to-taskid-1.json": json.dumps({"taskb": "TASKB"}), + "artifacts/to-run-1.json": json.dumps(["taskb"]), + } + with MockedOpen(files): + combine_task_graph_files(["0", "1"]) + self.assertEqual( + read_artifact("task-graph.json"), + { + "taska": {}, + "taskb": {}, + }, + ) + self.assertEqual( + read_artifact("label-to-taskid.json"), + { + "taska": "TASKA", + "taskb": "TASKB", + }, + ) + self.assertEqual( + sorted(read_artifact("to-run.json")), + [ + "taska", + "taskb", + ], + ) + + +def is_subset(subset, superset): + if isinstance(subset, dict): + return all( + key in superset and is_subset(val, superset[key]) + for key, val in subset.items() + ) + + if isinstance(subset, list) or isinstance(subset, set): + return all( + any(is_subset(subitem, superitem) for superitem in superset) + for subitem in subset + ) + + if isinstance(subset, str): + return subset in superset + + # assume that subset is a plain value if none of the above match + return subset == superset + + +@pytest.mark.parametrize( + "task_def,expected", + [ + pytest.param( + {"tags": {"kind": "decision-task"}}, + { + "hookPayload": { + "decision": { + "action": {"cb_name": "retrigger-decision"}, + }, + }, + }, + id="retrigger_decision", + ), + pytest.param( + {"tags": {"action": "backfill-task"}}, + { + "hookPayload": { + "decision": { + "action": {"cb_name": "retrigger-decision"}, + }, + }, + }, + id="retrigger_backfill", + ), + ], +) +def test_extract_applicable_action( + responses, monkeypatch, actions_json, task_def, expected +): + base_url = "https://taskcluster" + decision_task_id = "dddd" + task_id = "tttt" + + monkeypatch.setenv("TASK_ID", task_id) + monkeypatch.setenv("TASKCLUSTER_ROOT_URL", base_url) + monkeypatch.setenv("TASKCLUSTER_PROXY_URL", base_url) + responses.add( + responses.GET, + f"{base_url}/api/queue/v1/task/{task_id}", + status=200, + json=task_def, + ) + action = actions.util._extract_applicable_action( + actions_json, "retrigger", decision_task_id, task_id + ) + pprint(action, indent=2) + assert is_subset(expected, action) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_decision.py b/taskcluster/gecko_taskgraph/test/test_decision.py new file mode 100644 index 0000000000..81be7e0c44 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_decision.py @@ -0,0 +1,176 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +import pytest +from mozunit import MockedOpen, main +from taskgraph.util.yaml import load_yaml + +from gecko_taskgraph import decision +from gecko_taskgraph.parameters import register_parameters + +FAKE_GRAPH_CONFIG = {"product-dir": "browser", "taskgraph": {}} + + +@pytest.fixture(scope="module", autouse=True) +def register(): + register_parameters() + + +class TestDecision(unittest.TestCase): + def test_write_artifact_json(self): + data = [{"some": "data"}] + tmpdir = tempfile.mkdtemp() + try: + decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts") + decision.write_artifact("artifact.json", data) + with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.json")) as f: + self.assertEqual(json.load(f), data) + finally: + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + decision.ARTIFACTS_DIR = "artifacts" + + def test_write_artifact_yml(self): + data = [{"some": "data"}] + tmpdir = tempfile.mkdtemp() + try: + decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts") + decision.write_artifact("artifact.yml", data) + self.assertEqual(load_yaml(decision.ARTIFACTS_DIR, "artifact.yml"), data) + finally: + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + decision.ARTIFACTS_DIR = "artifacts" + + +class TestGetDecisionParameters(unittest.TestCase): + + ttc_file = os.path.join(os.getcwd(), "try_task_config.json") + + def setUp(self): + self.options = { + "base_repository": "https://hg.mozilla.org/mozilla-unified", + "head_repository": "https://hg.mozilla.org/mozilla-central", + "head_rev": "abcd", + "head_ref": "ef01", + "head_tag": "", + "message": "", + "project": "mozilla-central", + "pushlog_id": "143", + "pushdate": 1503691511, + "owner": "nobody@mozilla.com", + "repository_type": "hg", + "tasks_for": "hg-push", + "level": "3", + } + + @patch("gecko_taskgraph.decision.get_hg_revision_branch") + @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev") + def test_simple_options( + self, mock_determine_more_accurate_base_rev, mock_get_hg_revision_branch + ): + mock_get_hg_revision_branch.return_value = "default" + mock_determine_more_accurate_base_rev.return_value = "baserev" + with MockedOpen({self.ttc_file: None}): + params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) + self.assertEqual(params["pushlog_id"], "143") + self.assertEqual(params["build_date"], 1503691511) + self.assertEqual(params["hg_branch"], "default") + self.assertEqual(params["moz_build_date"], "20170825200511") + self.assertEqual(params["try_mode"], None) + self.assertEqual(params["try_options"], None) + self.assertEqual(params["try_task_config"], {}) + + @patch("gecko_taskgraph.decision.get_hg_revision_branch") + @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev") + def test_no_email_owner( + self, mock_determine_more_accurate_base_rev, mock_get_hg_revision_branch + ): + mock_get_hg_revision_branch.return_value = "default" + mock_determine_more_accurate_base_rev.return_value = "baserev" + self.options["owner"] = "ffxbld" + with MockedOpen({self.ttc_file: None}): + params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) + self.assertEqual(params["owner"], "ffxbld@noreply.mozilla.org") + + @patch("gecko_taskgraph.decision.get_hg_revision_branch") + @patch("gecko_taskgraph.decision.get_hg_commit_message") + @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev") + def test_try_options( + self, + mock_determine_more_accurate_base_rev, + mock_get_hg_commit_message, + mock_get_hg_revision_branch, + ): + mock_get_hg_commit_message.return_value = "try: -b do -t all --artifact" + mock_get_hg_revision_branch.return_value = "default" + mock_determine_more_accurate_base_rev.return_value = "baserev" + self.options["project"] = "try" + with MockedOpen({self.ttc_file: None}): + params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) + self.assertEqual(params["try_mode"], "try_option_syntax") + self.assertEqual(params["try_options"]["build_types"], "do") + self.assertEqual(params["try_options"]["unittests"], "all") + self.assertEqual( + params["try_task_config"], + { + "gecko-profile": False, + "use-artifact-builds": True, + "env": {}, + }, + ) + + @patch("gecko_taskgraph.decision.get_hg_revision_branch") + @patch("gecko_taskgraph.decision.get_hg_commit_message") + @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev") + def test_try_task_config( + self, + mock_get_hg_commit_message, + mock_get_hg_revision_branch, + mock_determine_more_accurate_base_rev, + ): + mock_get_hg_commit_message.return_value = "Fuzzy query=foo" + mock_get_hg_revision_branch.return_value = "default" + mock_determine_more_accurate_base_rev.return_value = "baserev" + ttc = {"tasks": ["a", "b"]} + self.options["project"] = "try" + with MockedOpen({self.ttc_file: json.dumps(ttc)}): + params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) + self.assertEqual(params["try_mode"], "try_task_config") + self.assertEqual(params["try_options"], None) + self.assertEqual(params["try_task_config"], ttc) + + def test_try_syntax_from_message_empty(self): + self.assertEqual(decision.try_syntax_from_message(""), "") + + def test_try_syntax_from_message_no_try_syntax(self): + self.assertEqual(decision.try_syntax_from_message("abc | def"), "") + + def test_try_syntax_from_message_initial_try_syntax(self): + self.assertEqual( + decision.try_syntax_from_message("try: -f -o -o"), "try: -f -o -o" + ) + + def test_try_syntax_from_message_initial_try_syntax_multiline(self): + self.assertEqual( + decision.try_syntax_from_message("try: -f -o -o\nabc\ndef"), "try: -f -o -o" + ) + + def test_try_syntax_from_message_embedded_try_syntax_multiline(self): + self.assertEqual( + decision.try_syntax_from_message("some stuff\ntry: -f -o -o\nabc\ndef"), + "try: -f -o -o", + ) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_files_changed.py b/taskcluster/gecko_taskgraph/test/test_files_changed.py new file mode 100644 index 0000000000..5b9a016649 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_files_changed.py @@ -0,0 +1,90 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +import unittest + +from mozunit import main + +from gecko_taskgraph import files_changed +from gecko_taskgraph.util import hg + +PARAMS = { + "head_repository": "https://hg.mozilla.org/mozilla-central", + "head_rev": "a14f88a9af7a", +} + +FILES_CHANGED = [ + "devtools/client/debugger/index.html", + "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js", + "devtools/client/inspector/test/head.js", + "devtools/client/themes/rules.css", + "devtools/client/webconsole/test/browser_webconsole_output_06.js", + "devtools/server/actors/highlighters/eye-dropper.js", + "devtools/server/actors/object.js", + "docshell/base/nsDocShell.cpp", + "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html", + "taskcluster/scripts/builder/build-l10n.sh", +] + + +class FakeResponse: + def json(self): + with open( + os.path.join(os.path.dirname(__file__), "automationrelevance.json") + ) as f: + return json.load(f) + + +class TestGetChangedFiles(unittest.TestCase): + def setUp(self): + files_changed.get_changed_files.clear() + self.old_get = hg.requests.get + + def fake_get(url, **kwargs): + return FakeResponse() + + hg.requests.get = fake_get + + def tearDown(self): + hg.requests.get = self.old_get + files_changed.get_changed_files.clear() + + def test_get_changed_files(self): + """Get_changed_files correctly gets the list of changed files in a push. + This tests against the production hg.mozilla.org so that it will detect + any changes in the format of the returned data.""" + self.assertEqual( + sorted( + files_changed.get_changed_files( + PARAMS["head_repository"], PARAMS["head_rev"] + ) + ), + FILES_CHANGED, + ) + + +class TestCheck(unittest.TestCase): + def setUp(self): + files_changed.get_changed_files[ + PARAMS["head_repository"], PARAMS["head_rev"] + ] = FILES_CHANGED + + def tearDown(self): + files_changed.get_changed_files.clear() + + def test_check_no_params(self): + self.assertTrue(files_changed.check({}, ["ignored"])) + + def test_check_no_match(self): + self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"])) + + def test_check_match(self): + self.assertTrue(files_changed.check(PARAMS, ["devtools/**"])) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_main.py b/taskcluster/gecko_taskgraph/test/test_main.py new file mode 100644 index 0000000000..bb1aa1caeb --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_main.py @@ -0,0 +1,67 @@ +# Any copyright is dedicated to the public domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import pytest +from mozunit import main as mozunit_main + +import gecko_taskgraph +from gecko_taskgraph.main import main as taskgraph_main + + +@pytest.fixture +def run_main(maketgg, monkeypatch): + def inner(args, **kwargs): + kwargs.setdefault("target_tasks", ["_fake-t-0", "_fake-t-1"]) + tgg = maketgg(**kwargs) + + def fake_get_taskgraph_generator(*args): + return tgg + + monkeypatch.setattr( + gecko_taskgraph.main, + "get_taskgraph_generator", + fake_get_taskgraph_generator, + ) + taskgraph_main(args) + return tgg + + return inner + + +@pytest.mark.parametrize( + "attr,expected", + ( + ("tasks", ["_fake-t-0", "_fake-t-1", "_fake-t-2"]), + ("full", ["_fake-t-0", "_fake-t-1", "_fake-t-2"]), + ("target", ["_fake-t-0", "_fake-t-1"]), + ("target-graph", ["_fake-t-0", "_fake-t-1"]), + ("optimized", ["_fake-t-0", "_fake-t-1"]), + ("morphed", ["_fake-t-0", "_fake-t-1"]), + ), +) +def test_show_taskgraph(run_main, capsys, attr, expected): + run_main([attr]) + out, err = capsys.readouterr() + assert out.strip() == "\n".join(expected) + assert "Dumping result" in err + + +def test_tasks_regex(run_main, capsys): + run_main(["full", "--tasks=_.*-t-1"]) + out, _ = capsys.readouterr() + assert out.strip() == "_fake-t-1" + + +def test_output_file(run_main, tmpdir): + output_file = tmpdir.join("out.txt") + assert not output_file.check() + + run_main(["full", f"--output-file={output_file.strpath}"]) + assert output_file.check() + assert output_file.read_text("utf-8").strip() == "\n".join( + ["_fake-t-0", "_fake-t-1", "_fake-t-2"] + ) + + +if __name__ == "__main__": + mozunit_main() diff --git a/taskcluster/gecko_taskgraph/test/test_morph.py b/taskcluster/gecko_taskgraph/test/test_morph.py new file mode 100644 index 0000000000..c29fb58207 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_morph.py @@ -0,0 +1,108 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import pytest +from mozunit import main +from taskgraph.graph import Graph +from taskgraph.parameters import Parameters +from taskgraph.task import Task +from taskgraph.taskgraph import TaskGraph + +from gecko_taskgraph import morph + + +@pytest.fixture +def make_taskgraph(): + def inner(tasks): + label_to_taskid = {k: k + "-tid" for k in tasks} + for label, task_id in label_to_taskid.items(): + tasks[label].task_id = task_id + graph = Graph(nodes=set(tasks), edges=set()) + taskgraph = TaskGraph(tasks, graph) + return taskgraph, label_to_taskid + + return inner + + +def test_make_index_tasks(make_taskgraph, graph_config): + task_def = { + "routes": [ + "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.es-MX", + "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.fy-NL", + "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sk", + "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sl", + "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.uk", + "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.zh-CN", + "index.gecko.v2.mozilla-central.pushdate." + "2017.04.04.20170404100210.firefox-l10n.linux64-opt.es-MX", + "index.gecko.v2.mozilla-central.pushdate." + "2017.04.04.20170404100210.firefox-l10n.linux64-opt.fy-NL", + "index.gecko.v2.mozilla-central.pushdate." + "2017.04.04.20170404100210.firefox-l10n.linux64-opt.sk", + "index.gecko.v2.mozilla-central.pushdate." + "2017.04.04.20170404100210.firefox-l10n.linux64-opt.sl", + "index.gecko.v2.mozilla-central.pushdate." + "2017.04.04.20170404100210.firefox-l10n.linux64-opt.uk", + "index.gecko.v2.mozilla-central.pushdate." + "2017.04.04.20170404100210.firefox-l10n.linux64-opt.zh-CN", + "index.gecko.v2.mozilla-central.revision." + "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.es-MX", + "index.gecko.v2.mozilla-central.revision." + "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.fy-NL", + "index.gecko.v2.mozilla-central.revision." + "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sk", + "index.gecko.v2.mozilla-central.revision." + "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sl", + "index.gecko.v2.mozilla-central.revision." + "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.uk", + "index.gecko.v2.mozilla-central.revision." + "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.zh-CN", + ], + "deadline": "soon", + "metadata": { + "description": "desc", + "owner": "owner@foo.com", + "source": "https://source", + }, + "extra": { + "index": {"rank": 1540722354}, + }, + } + task = Task(kind="test", label="a", attributes={}, task=task_def) + docker_task = Task( + kind="docker-image", label="docker-image-index-task", attributes={}, task={} + ) + taskgraph, label_to_taskid = make_taskgraph( + { + task.label: task, + docker_task.label: docker_task, + } + ) + + index_paths = [ + r.split(".", 1)[1] for r in task_def["routes"] if r.startswith("index.") + ] + index_task = morph.make_index_task( + task, + taskgraph, + label_to_taskid, + Parameters(strict=False), + graph_config, + index_paths=index_paths, + index_rank=1540722354, + purpose="index-task", + dependencies={}, + ) + + assert index_task.task["payload"]["command"][0] == "insert-indexes.js" + assert index_task.task["payload"]["env"]["TARGET_TASKID"] == "a-tid" + assert index_task.task["payload"]["env"]["INDEX_RANK"] == 1540722354 + + # check the scope summary + assert index_task.task["scopes"] == ["index:insert-task:gecko.v2.mozilla-central.*"] + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py b/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py new file mode 100644 index 0000000000..0f37332300 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py @@ -0,0 +1,551 @@ +# Any copyright is dedicated to the public domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + + +import time +from datetime import datetime +from time import mktime + +import pytest +from mozunit import main +from taskgraph.optimize.base import registry +from taskgraph.task import Task + +from gecko_taskgraph.optimize import project +from gecko_taskgraph.optimize.backstop import SkipUnlessBackstop, SkipUnlessPushInterval +from gecko_taskgraph.optimize.bugbug import ( + FALLBACK, + BugBugPushSchedules, + DisperseGroups, + SkipUnlessDebug, +) +from gecko_taskgraph.optimize.strategies import IndexSearch, SkipUnlessSchedules +from gecko_taskgraph.util.backstop import BACKSTOP_PUSH_INTERVAL +from gecko_taskgraph.util.bugbug import ( + BUGBUG_BASE_URL, + BugbugTimeoutException, + push_schedules, +) + + +@pytest.fixture(autouse=True) +def clear_push_schedules_memoize(): + push_schedules.clear() + + +@pytest.fixture +def params(): + return { + "branch": "autoland", + "head_repository": "https://hg.mozilla.org/integration/autoland", + "head_rev": "abcdef", + "project": "autoland", + "pushlog_id": 1, + "pushdate": mktime(datetime.now().timetuple()), + } + + +def generate_tasks(*tasks): + for i, task in enumerate(tasks): + task.setdefault("label", f"task-{i}-label") + task.setdefault("kind", "test") + task.setdefault("task", {}) + task.setdefault("attributes", {}) + task["attributes"].setdefault("e10s", True) + + for attr in ( + "optimization", + "dependencies", + "soft_dependencies", + ): + task.setdefault(attr, None) + + task["task"].setdefault("label", task["label"]) + yield Task.from_json(task) + + +# task sets + +default_tasks = list( + generate_tasks( + {"attributes": {"test_manifests": ["foo/test.ini", "bar/test.ini"]}}, + {"attributes": {"test_manifests": ["bar/test.ini"], "build_type": "debug"}}, + {"attributes": {"build_type": "debug"}}, + {"attributes": {"test_manifests": [], "build_type": "opt"}}, + {"attributes": {"build_type": "opt"}}, + ) +) + + +disperse_tasks = list( + generate_tasks( + { + "attributes": { + "test_manifests": ["foo/test.ini", "bar/test.ini"], + "test_platform": "linux/opt", + } + }, + { + "attributes": { + "test_manifests": ["bar/test.ini"], + "test_platform": "linux/opt", + } + }, + { + "attributes": { + "test_manifests": ["bar/test.ini"], + "test_platform": "windows/debug", + } + }, + { + "attributes": { + "test_manifests": ["bar/test.ini"], + "test_platform": "linux/opt", + "unittest_variant": "no-fission", + } + }, + { + "attributes": { + "e10s": False, + "test_manifests": ["bar/test.ini"], + "test_platform": "linux/opt", + } + }, + ) +) + + +def idfn(param): + if isinstance(param, tuple): + try: + return param[0].__name__ + except AttributeError: + return None + return None + + +@pytest.mark.parametrize( + "opt,tasks,arg,expected", + [ + # debug + pytest.param( + SkipUnlessDebug(), + default_tasks, + None, + ["task-0-label", "task-1-label", "task-2-label"], + ), + # disperse with no supplied importance + pytest.param( + DisperseGroups(), + disperse_tasks, + None, + [t.label for t in disperse_tasks], + ), + # disperse with low importance + pytest.param( + DisperseGroups(), + disperse_tasks, + {"bar/test.ini": "low"}, + ["task-0-label", "task-2-label"], + ), + # disperse with medium importance + pytest.param( + DisperseGroups(), + disperse_tasks, + {"bar/test.ini": "medium"}, + ["task-0-label", "task-1-label", "task-2-label"], + ), + # disperse with high importance + pytest.param( + DisperseGroups(), + disperse_tasks, + {"bar/test.ini": "high"}, + ["task-0-label", "task-1-label", "task-2-label", "task-3-label"], + ), + ], + ids=idfn, +) +def test_optimization_strategy_remove(params, opt, tasks, arg, expected): + labels = [t.label for t in tasks if not opt.should_remove_task(t, params, arg)] + assert sorted(labels) == sorted(expected) + + +@pytest.mark.parametrize( + "state,expires,expected", + ( + ("completed", "2021-06-06T14:53:16.937Z", False), + ("completed", "2021-06-08T14:53:16.937Z", "abc"), + ("exception", "2021-06-08T14:53:16.937Z", False), + ("failed", "2021-06-08T14:53:16.937Z", False), + ), +) +def test_index_search(responses, params, state, expires, expected): + taskid = "abc" + index_path = "foo.bar.latest" + responses.add( + responses.GET, + f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/{index_path}", + json={"taskId": taskid}, + status=200, + ) + + responses.add( + responses.GET, + f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{taskid}/status", + json={ + "status": { + "state": state, + "expires": expires, + } + }, + status=200, + ) + + opt = IndexSearch() + deadline = "2021-06-07T19:03:20.482Z" + assert opt.should_replace_task({}, params, deadline, (index_path,)) == expected + + +@pytest.mark.parametrize( + "args,data,expected", + [ + # empty + pytest.param( + (0.1,), + {}, + [], + ), + # only tasks without test manifests selected + pytest.param( + (0.1,), + {"tasks": {"task-1-label": 0.9, "task-2-label": 0.1, "task-3-label": 0.5}}, + ["task-2-label"], + ), + # tasks which are unknown to bugbug are selected + pytest.param( + (0.1,), + { + "tasks": {"task-1-label": 0.9, "task-3-label": 0.5}, + "known_tasks": ["task-1-label", "task-3-label", "task-4-label"], + }, + ["task-2-label"], + ), + # tasks containing groups selected + pytest.param( + (0.1,), + {"groups": {"foo/test.ini": 0.4}}, + ["task-0-label"], + ), + # tasks matching "tasks" or "groups" selected + pytest.param( + (0.1,), + { + "tasks": {"task-2-label": 0.2}, + "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75}, + }, + ["task-0-label", "task-1-label", "task-2-label"], + ), + # tasks matching "tasks" or "groups" selected, when they exceed the confidence threshold + pytest.param( + (0.5,), + { + "tasks": {"task-2-label": 0.2, "task-4-label": 0.5}, + "groups": {"foo/test.ini": 0.65, "bar/test.ini": 0.25}, + }, + ["task-0-label", "task-4-label"], + ), + # tasks matching "reduced_tasks" are selected, when they exceed the confidence threshold + pytest.param( + (0.7, True, True), + { + "tasks": {"task-2-label": 0.7, "task-4-label": 0.7}, + "reduced_tasks": {"task-4-label": 0.7}, + "groups": {"foo/test.ini": 0.75, "bar/test.ini": 0.25}, + }, + ["task-4-label"], + ), + # tasks matching "groups" selected, only on specific platforms. + pytest.param( + (0.1, False, False, None, 1, True), + { + "tasks": {"task-2-label": 0.2}, + "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75}, + "config_groups": { + "foo/test.ini": ["task-1-label", "task-0-label"], + "bar/test.ini": ["task-0-label"], + }, + }, + ["task-0-label", "task-2-label"], + ), + pytest.param( + (0.1, False, False, None, 1, True), + { + "tasks": {"task-2-label": 0.2}, + "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75}, + "config_groups": { + "foo/test.ini": ["task-1-label", "task-0-label"], + "bar/test.ini": ["task-1-label"], + }, + }, + ["task-0-label", "task-1-label", "task-2-label"], + ), + pytest.param( + (0.1, False, False, None, 1, True), + { + "tasks": {"task-2-label": 0.2}, + "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75}, + "config_groups": { + "foo/test.ini": ["task-1-label"], + "bar/test.ini": ["task-0-label"], + }, + }, + ["task-0-label", "task-2-label"], + ), + pytest.param( + (0.1, False, False, None, 1, True), + { + "tasks": {"task-2-label": 0.2}, + "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75}, + "config_groups": { + "foo/test.ini": ["task-1-label"], + "bar/test.ini": ["task-3-label"], + }, + }, + ["task-2-label"], + ), + ], + ids=idfn, +) +def test_bugbug_push_schedules(responses, params, args, data, expected): + query = "/push/{branch}/{head_rev}/schedules".format(**params) + url = BUGBUG_BASE_URL + query + + responses.add( + responses.GET, + url, + json=data, + status=200, + ) + + opt = BugBugPushSchedules(*args) + labels = [ + t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) + ] + assert sorted(labels) == sorted(expected) + + +def test_bugbug_multiple_pushes(responses, params): + pushes = {str(pid): {"changesets": [f"c{pid}"]} for pid in range(8, 10)} + + responses.add( + responses.GET, + "https://hg.mozilla.org/integration/autoland/json-pushes/?version=2&startID=8&endID=9", + json={"pushes": pushes}, + status=200, + ) + + responses.add( + responses.GET, + BUGBUG_BASE_URL + "/push/{}/c9/schedules".format(params["branch"]), + json={ + "tasks": {"task-2-label": 0.2, "task-4-label": 0.5}, + "groups": {"foo/test.ini": 0.2, "bar/test.ini": 0.25}, + "config_groups": {"foo/test.ini": ["linux-*"], "bar/test.ini": ["task-*"]}, + "known_tasks": ["task-4-label"], + }, + status=200, + ) + + # Tasks with a lower confidence don't override task with a higher one. + # Tasks with a higher confidence override tasks with a lower one. + # Known tasks are merged. + responses.add( + responses.GET, + BUGBUG_BASE_URL + "/push/{branch}/{head_rev}/schedules".format(**params), + json={ + "tasks": {"task-2-label": 0.2, "task-4-label": 0.2}, + "groups": {"foo/test.ini": 0.65, "bar/test.ini": 0.25}, + "config_groups": { + "foo/test.ini": ["task-*"], + "bar/test.ini": ["windows-*"], + }, + "known_tasks": ["task-1-label", "task-3-label"], + }, + status=200, + ) + + params["pushlog_id"] = 10 + + opt = BugBugPushSchedules(0.3, False, False, False, 2) + labels = [ + t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) + ] + assert sorted(labels) == sorted(["task-0-label", "task-2-label", "task-4-label"]) + + opt = BugBugPushSchedules(0.3, False, False, False, 2, True) + labels = [ + t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) + ] + assert sorted(labels) == sorted(["task-0-label", "task-2-label", "task-4-label"]) + + opt = BugBugPushSchedules(0.2, False, False, False, 2, True) + labels = [ + t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) + ] + assert sorted(labels) == sorted( + ["task-0-label", "task-1-label", "task-2-label", "task-4-label"] + ) + + +def test_bugbug_timeout(monkeypatch, responses, params): + query = "/push/{branch}/{head_rev}/schedules".format(**params) + url = BUGBUG_BASE_URL + query + responses.add( + responses.GET, + url, + json={"ready": False}, + status=202, + ) + + # Make sure the test runs fast. + monkeypatch.setattr(time, "sleep", lambda i: None) + + opt = BugBugPushSchedules(0.5) + with pytest.raises(BugbugTimeoutException): + opt.should_remove_task(default_tasks[0], params, None) + + +def test_bugbug_fallback(monkeypatch, responses, params): + query = "/push/{branch}/{head_rev}/schedules".format(**params) + url = BUGBUG_BASE_URL + query + responses.add( + responses.GET, + url, + json={"ready": False}, + status=202, + ) + + opt = BugBugPushSchedules(0.5, fallback=FALLBACK) + + # Make sure the test runs fast. + monkeypatch.setattr(time, "sleep", lambda i: None) + + def fake_should_remove_task(task, params, _): + return task.label == default_tasks[0].label + + monkeypatch.setattr( + registry[FALLBACK], "should_remove_task", fake_should_remove_task + ) + + assert opt.should_remove_task(default_tasks[0], params, None) + + # Make sure we don't hit bugbug more than once. + responses.reset() + + assert not opt.should_remove_task(default_tasks[1], params, None) + + +def test_backstop(params): + all_labels = {t.label for t in default_tasks} + opt = SkipUnlessBackstop() + + params["backstop"] = False + scheduled = { + t.label for t in default_tasks if not opt.should_remove_task(t, params, None) + } + assert scheduled == set() + + params["backstop"] = True + scheduled = { + t.label for t in default_tasks if not opt.should_remove_task(t, params, None) + } + assert scheduled == all_labels + + +def test_push_interval(params): + all_labels = {t.label for t in default_tasks} + opt = SkipUnlessPushInterval(10) # every 10th push + + # Only multiples of 10 schedule tasks. + params["pushlog_id"] = 9 + scheduled = { + t.label for t in default_tasks if not opt.should_remove_task(t, params, None) + } + assert scheduled == set() + + params["pushlog_id"] = 10 + scheduled = { + t.label for t in default_tasks if not opt.should_remove_task(t, params, None) + } + assert scheduled == all_labels + + +def test_expanded(params): + all_labels = {t.label for t in default_tasks} + opt = registry["skip-unless-expanded"] + + params["backstop"] = False + params["pushlog_id"] = BACKSTOP_PUSH_INTERVAL / 2 + scheduled = { + t.label for t in default_tasks if not opt.should_remove_task(t, params, None) + } + assert scheduled == all_labels + + params["pushlog_id"] += 1 + scheduled = { + t.label for t in default_tasks if not opt.should_remove_task(t, params, None) + } + assert scheduled == set() + + params["backstop"] = True + scheduled = { + t.label for t in default_tasks if not opt.should_remove_task(t, params, None) + } + assert scheduled == all_labels + + +def test_project_autoland_test(monkeypatch, responses, params): + """Tests the behaviour of the `project.autoland["test"]` strategy on + various types of pushes. + """ + # This is meant to test the composition of substrategies, and not the + # actual optimization implementations. So mock them out for simplicity. + monkeypatch.setattr(SkipUnlessSchedules, "should_remove_task", lambda *args: False) + monkeypatch.setattr(DisperseGroups, "should_remove_task", lambda *args: False) + + def fake_bugbug_should_remove_task(self, task, params, importance): + if self.num_pushes > 1: + return task.label == "task-4-label" + return task.label in ("task-2-label", "task-3-label", "task-4-label") + + monkeypatch.setattr( + BugBugPushSchedules, "should_remove_task", fake_bugbug_should_remove_task + ) + + opt = project.autoland["test"] + + # On backstop pushes, nothing gets optimized. + params["backstop"] = True + scheduled = { + t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) + } + assert scheduled == {t.label for t in default_tasks} + + # On expanded pushes, some things are optimized. + params["backstop"] = False + params["pushlog_id"] = 10 + scheduled = { + t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) + } + assert scheduled == {"task-0-label", "task-1-label", "task-2-label", "task-3-label"} + + # On regular pushes, more things are optimized. + params["pushlog_id"] = 11 + scheduled = { + t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) + } + assert scheduled == {"task-0-label", "task-1-label"} + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_target_tasks.py b/taskcluster/gecko_taskgraph/test/test_target_tasks.py new file mode 100644 index 0000000000..154c0f910a --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_target_tasks.py @@ -0,0 +1,371 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import contextlib +import re +import unittest + +import pytest +from mozunit import main +from taskgraph.graph import Graph +from taskgraph.task import Task +from taskgraph.taskgraph import TaskGraph + +from gecko_taskgraph import target_tasks, try_option_syntax + + +class FakeTryOptionSyntax: + def __init__(self, message, task_graph, graph_config): + self.trigger_tests = 0 + self.talos_trigger_tests = 0 + self.raptor_trigger_tests = 0 + self.notifications = None + self.env = [] + self.profile = False + self.tag = None + self.no_retry = False + + def task_matches(self, task): + return "at-at" in task.attributes + + +class TestTargetTasks(unittest.TestCase): + def default_matches_project(self, run_on_projects, project): + return self.default_matches( + attributes={ + "run_on_projects": run_on_projects, + }, + parameters={ + "project": project, + "hg_branch": "default", + }, + ) + + def default_matches_hg_branch(self, run_on_hg_branches, hg_branch): + attributes = {"run_on_projects": ["all"]} + if run_on_hg_branches is not None: + attributes["run_on_hg_branches"] = run_on_hg_branches + + return self.default_matches( + attributes=attributes, + parameters={ + "project": "mozilla-central", + "hg_branch": hg_branch, + }, + ) + + def default_matches(self, attributes, parameters): + method = target_tasks.get_method("default") + graph = TaskGraph( + tasks={ + "a": Task(kind="build", label="a", attributes=attributes, task={}), + }, + graph=Graph(nodes={"a"}, edges=set()), + ) + return "a" in method(graph, parameters, {}) + + def test_default_all(self): + """run_on_projects=[all] includes release, integration, and other projects""" + self.assertTrue(self.default_matches_project(["all"], "mozilla-central")) + self.assertTrue(self.default_matches_project(["all"], "baobab")) + + def test_default_integration(self): + """run_on_projects=[integration] includes integration projects""" + self.assertFalse( + self.default_matches_project(["integration"], "mozilla-central") + ) + self.assertFalse(self.default_matches_project(["integration"], "baobab")) + + def test_default_release(self): + """run_on_projects=[release] includes release projects""" + self.assertTrue(self.default_matches_project(["release"], "mozilla-central")) + self.assertFalse(self.default_matches_project(["release"], "baobab")) + + def test_default_nothing(self): + """run_on_projects=[] includes nothing""" + self.assertFalse(self.default_matches_project([], "mozilla-central")) + self.assertFalse(self.default_matches_project([], "baobab")) + + def test_default_hg_branch(self): + self.assertTrue(self.default_matches_hg_branch(None, "default")) + self.assertTrue(self.default_matches_hg_branch(None, "GECKOVIEW_62_RELBRANCH")) + + self.assertFalse(self.default_matches_hg_branch([], "default")) + self.assertFalse(self.default_matches_hg_branch([], "GECKOVIEW_62_RELBRANCH")) + + self.assertTrue(self.default_matches_hg_branch(["all"], "default")) + self.assertTrue( + self.default_matches_hg_branch(["all"], "GECKOVIEW_62_RELBRANCH") + ) + + self.assertTrue(self.default_matches_hg_branch(["default"], "default")) + self.assertTrue(self.default_matches_hg_branch([r"default"], "default")) + self.assertFalse( + self.default_matches_hg_branch([r"default"], "GECKOVIEW_62_RELBRANCH") + ) + + self.assertTrue( + self.default_matches_hg_branch( + ["GECKOVIEW_62_RELBRANCH"], "GECKOVIEW_62_RELBRANCH" + ) + ) + self.assertTrue( + self.default_matches_hg_branch( + [r"GECKOVIEW_\d+_RELBRANCH"], "GECKOVIEW_62_RELBRANCH" + ) + ) + self.assertTrue( + self.default_matches_hg_branch( + [r"GECKOVIEW_\d+_RELBRANCH"], "GECKOVIEW_62_RELBRANCH" + ) + ) + self.assertFalse( + self.default_matches_hg_branch([r"GECKOVIEW_\d+_RELBRANCH"], "default") + ) + + def make_task_graph(self): + tasks = { + "a": Task(kind=None, label="a", attributes={}, task={}), + "b": Task(kind=None, label="b", attributes={"at-at": "yep"}, task={}), + "c": Task( + kind=None, label="c", attributes={"run_on_projects": ["try"]}, task={} + ), + } + graph = Graph(nodes=set("abc"), edges=set()) + return TaskGraph(tasks, graph) + + @contextlib.contextmanager + def fake_TryOptionSyntax(self): + orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax + try: + try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax + yield + finally: + try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax + + def test_empty_try(self): + "try_mode = None runs nothing" + tg = self.make_task_graph() + method = target_tasks.get_method("try_tasks") + params = { + "try_mode": None, + "project": "try", + "message": "", + } + # only runs the task with run_on_projects: try + self.assertEqual(method(tg, params, {}), []) + + def test_try_option_syntax(self): + "try_mode = try_option_syntax uses TryOptionSyntax" + tg = self.make_task_graph() + method = target_tasks.get_method("try_tasks") + with self.fake_TryOptionSyntax(): + params = { + "try_mode": "try_option_syntax", + "message": "try: -p all", + } + self.assertEqual(method(tg, params, {}), ["b"]) + + def test_try_task_config(self): + "try_mode = try_task_config uses the try config" + tg = self.make_task_graph() + method = target_tasks.get_method("try_tasks") + params = { + "try_mode": "try_task_config", + "try_task_config": {"tasks": ["a"]}, + } + self.assertEqual(method(tg, params, {}), ["a"]) + + +# tests for specific filters + + +@pytest.mark.parametrize( + "name,params,expected", + ( + pytest.param( + "filter_tests_without_manifests", + { + "task": Task(kind="test", label="a", attributes={}, task={}), + "parameters": None, + }, + True, + id="filter_tests_without_manifests_not_in_attributes", + ), + pytest.param( + "filter_tests_without_manifests", + { + "task": Task( + kind="test", + label="a", + attributes={"test_manifests": ["foo"]}, + task={}, + ), + "parameters": None, + }, + True, + id="filter_tests_without_manifests_has_test_manifests", + ), + pytest.param( + "filter_tests_without_manifests", + { + "task": Task( + kind="build", + label="a", + attributes={"test_manifests": None}, + task={}, + ), + "parameters": None, + }, + True, + id="filter_tests_without_manifests_not_a_test", + ), + pytest.param( + "filter_tests_without_manifests", + { + "task": Task( + kind="test", label="a", attributes={"test_manifests": None}, task={} + ), + "parameters": None, + }, + False, + id="filter_tests_without_manifests_has_no_test_manifests", + ), + pytest.param( + "filter_by_regex", + { + "task_label": "build-linux64-debug", + "regexes": [re.compile("build")], + "mode": "include", + }, + True, + id="filter_regex_simple_include", + ), + pytest.param( + "filter_by_regex", + { + "task_label": "build-linux64-debug", + "regexes": [re.compile("linux(.+)debug")], + "mode": "include", + }, + True, + id="filter_regex_re_include", + ), + pytest.param( + "filter_by_regex", + { + "task_label": "build-linux64-debug", + "regexes": [re.compile("nothing"), re.compile("linux(.+)debug")], + "mode": "include", + }, + True, + id="filter_regex_re_include_multiple", + ), + pytest.param( + "filter_by_regex", + { + "task_label": "build-linux64-debug", + "regexes": [re.compile("build")], + "mode": "exclude", + }, + False, + id="filter_regex_simple_exclude", + ), + pytest.param( + "filter_by_regex", + { + "task_label": "build-linux64-debug", + "regexes": [re.compile("linux(.+)debug")], + "mode": "exclude", + }, + False, + id="filter_regex_re_exclude", + ), + pytest.param( + "filter_by_regex", + { + "task_label": "build-linux64-debug", + "regexes": [re.compile("linux(.+)debug"), re.compile("nothing")], + "mode": "exclude", + }, + False, + id="filter_regex_re_exclude_multiple", + ), + pytest.param( + "filter_unsupported_artifact_builds", + { + "task": Task( + kind="test", + label="a", + attributes={"supports-artifact-builds": False}, + task={}, + ), + "parameters": { + "try_task_config": { + "use-artifact-builds": False, + }, + }, + }, + True, + id="filter_unsupported_artifact_builds_no_artifact_builds", + ), + pytest.param( + "filter_unsupported_artifact_builds", + { + "task": Task( + kind="test", + label="a", + attributes={"supports-artifact-builds": False}, + task={}, + ), + "parameters": { + "try_task_config": { + "use-artifact-builds": True, + }, + }, + }, + False, + id="filter_unsupported_artifact_builds_removed", + ), + pytest.param( + "filter_unsupported_artifact_builds", + { + "task": Task( + kind="test", + label="a", + attributes={"supports-artifact-builds": True}, + task={}, + ), + "parameters": { + "try_task_config": { + "use-artifact-builds": True, + }, + }, + }, + True, + id="filter_unsupported_artifact_builds_not_removed", + ), + pytest.param( + "filter_unsupported_artifact_builds", + { + "task": Task(kind="test", label="a", attributes={}, task={}), + "parameters": { + "try_task_config": { + "use-artifact-builds": True, + }, + }, + }, + True, + id="filter_unsupported_artifact_builds_not_removed", + ), + ), +) +def test_filters(name, params, expected): + func = getattr(target_tasks, name) + assert func(**params) is expected + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_taskcluster_yml.py b/taskcluster/gecko_taskgraph/test/test_taskcluster_yml.py new file mode 100644 index 0000000000..cdf94ec3e1 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_taskcluster_yml.py @@ -0,0 +1,145 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import pprint +import unittest + +import jsone +import slugid +from mozunit import main +from taskgraph.util.time import current_json_time +from taskgraph.util.yaml import load_yaml + +from gecko_taskgraph import GECKO + + +class TestTaskclusterYml(unittest.TestCase): + @property + def taskcluster_yml(self): + return load_yaml(GECKO, ".taskcluster.yml") + + def test_push(self): + context = { + "tasks_for": "hg-push", + "push": { + "revision": "e8d2d9aff5026ef1f1777b781b47fdcbdb9d8f20", + "base_revision": "e8aebe488b2f2e567940577de25013d00e818f7c", + "owner": "dustin@mozilla.com", + "pushlog_id": 1556565286, + "pushdate": 112957, + }, + "repository": { + "url": "https://hg.mozilla.org/mozilla-central", + "project": "mozilla-central", + "level": "3", + }, + "ownTaskId": slugid.nice(), + } + rendered = jsone.render(self.taskcluster_yml, context) + pprint.pprint(rendered) + self.assertEqual( + rendered["tasks"][0]["metadata"]["name"], "Gecko Decision Task" + ) + self.assertIn("matrixBody", rendered["tasks"][0]["extra"]["notify"]) + + def test_push_non_mc(self): + context = { + "tasks_for": "hg-push", + "push": { + "revision": "e8d2d9aff5026ef1f1777b781b47fdcbdb9d8f20", + "base_revision": "e8aebe488b2f2e567940577de25013d00e818f7c", + "owner": "dustin@mozilla.com", + "pushlog_id": 1556565286, + "pushdate": 112957, + }, + "repository": { + "url": "https://hg.mozilla.org/releases/mozilla-beta", + "project": "mozilla-beta", + "level": "3", + }, + "ownTaskId": slugid.nice(), + } + rendered = jsone.render(self.taskcluster_yml, context) + pprint.pprint(rendered) + self.assertEqual( + rendered["tasks"][0]["metadata"]["name"], "Gecko Decision Task" + ) + self.assertNotIn("matrixBody", rendered["tasks"][0]["extra"]["notify"]) + + def test_cron(self): + context = { + "tasks_for": "cron", + "repository": { + "url": "https://hg.mozilla.org/mozilla-central", + "project": "mozilla-central", + "level": 3, + }, + "push": { + "revision": "e8aebe488b2f2e567940577de25013d00e818f7c", + "base_revision": "54cbb3745cdb9a8aa0a4428d405b3b2e1c7d13c2", + "pushlog_id": -1, + "pushdate": 0, + "owner": "cron", + }, + "cron": { + "task_id": "<cron task id>", + "job_name": "test", + "job_symbol": "T", + "quoted_args": "abc def", + }, + "now": current_json_time(), + "ownTaskId": slugid.nice(), + } + rendered = jsone.render(self.taskcluster_yml, context) + pprint.pprint(rendered) + self.assertEqual( + rendered["tasks"][0]["metadata"]["name"], "Decision Task for cron job test" + ) + + def test_action(self): + context = { + "tasks_for": "action", + "repository": { + "url": "https://hg.mozilla.org/mozilla-central", + "project": "mozilla-central", + "level": 3, + }, + "push": { + "revision": "e8d2d9aff5026ef1f1777b781b47fdcbdb9d8f20", + "base_revision": "e8aebe488b2f2e567940577de25013d00e818f7c", + "owner": "dustin@mozilla.com", + "pushlog_id": 1556565286, + "pushdate": 112957, + }, + "action": { + "name": "test-action", + "title": "Test Action", + "description": "Just testing", + "taskGroupId": slugid.nice(), + "symbol": "t", + "repo_scope": "assume:repo:hg.mozilla.org/try:action:generic", + "cb_name": "test_action", + }, + "input": {}, + "parameters": {}, + "now": current_json_time(), + "taskId": slugid.nice(), + "ownTaskId": slugid.nice(), + "clientId": "testing/testing/testing", + } + rendered = jsone.render(self.taskcluster_yml, context) + pprint.pprint(rendered) + self.assertEqual( + rendered["tasks"][0]["metadata"]["name"], "Action: Test Action" + ) + + def test_unknown(self): + context = {"tasks_for": "bitkeeper-push"} + rendered = jsone.render(self.taskcluster_yml, context) + pprint.pprint(rendered) + self.assertEqual(rendered["tasks"], []) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_job.py b/taskcluster/gecko_taskgraph/test/test_transforms_job.py new file mode 100644 index 0000000000..8bf85dcf62 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_transforms_job.py @@ -0,0 +1,150 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Tests for the 'job' transform subsystem. +""" + + +import os +from copy import deepcopy + +import pytest +from mozunit import main +from taskgraph.config import load_graph_config +from taskgraph.transforms.base import TransformConfig +from taskgraph.util.schema import Schema, validate_schema + +from gecko_taskgraph import GECKO +from gecko_taskgraph.test.conftest import FakeParameters +from gecko_taskgraph.transforms import job +from gecko_taskgraph.transforms.job import run_task # noqa: F401 +from gecko_taskgraph.transforms.job.common import add_cache +from gecko_taskgraph.transforms.task import payload_builders + +here = os.path.abspath(os.path.dirname(__file__)) + + +TASK_DEFAULTS = { + "description": "fake description", + "label": "fake-task-label", + "run": { + "using": "run-task", + }, +} + + +@pytest.fixture(scope="module") +def config(): + graph_config = load_graph_config(os.path.join(GECKO, "taskcluster", "ci")) + params = FakeParameters( + { + "base_repository": "http://hg.example.com", + "head_repository": "http://hg.example.com", + "head_rev": "abcdef", + "level": 1, + "project": "example", + } + ) + return TransformConfig( + "job_test", here, {}, params, {}, graph_config, write_artifacts=False + ) + + +@pytest.fixture() +def transform(monkeypatch, config): + """Run the job transforms on the specified task but return the inputs to + `configure_taskdesc_for_run` without executing it. + + This gives test functions an easy way to generate the inputs required for + many of the `run_using` subsystems. + """ + + def inner(task_input): + task = deepcopy(TASK_DEFAULTS) + task.update(task_input) + frozen_args = [] + + def _configure_taskdesc_for_run(*args): + frozen_args.extend(args) + + monkeypatch.setattr( + job, "configure_taskdesc_for_run", _configure_taskdesc_for_run + ) + + for _ in job.transforms(config, [task]): + # This forces the generator to be evaluated + pass + + return frozen_args + + return inner + + +@pytest.mark.parametrize( + "task", + [ + {"worker-type": "b-linux"}, + {"worker-type": "t-win10-64-hw"}, + ], + ids=lambda t: t["worker-type"], +) +def test_worker_caches(task, transform): + config, job, taskdesc, impl = transform(task) + add_cache(job, taskdesc, "cache1", "/cache1") + add_cache(job, taskdesc, "cache2", "/cache2", skip_untrusted=True) + + if impl not in ("docker-worker", "generic-worker"): + pytest.xfail(f"caches not implemented for '{impl}'") + + key = "caches" if impl == "docker-worker" else "mounts" + assert key in taskdesc["worker"] + assert len(taskdesc["worker"][key]) == 2 + + # Create a new schema object with just the part relevant to caches. + partial_schema = Schema(payload_builders[impl].schema.schema[key]) + validate_schema(partial_schema, taskdesc["worker"][key], "validation error") + + +@pytest.mark.parametrize( + "workerfn", [fn for fn, *_ in job.registry["run-task"].values()] +) +@pytest.mark.parametrize( + "task", + ( + { + "worker-type": "b-linux", + "run": { + "checkout": True, + "comm-checkout": False, + "command": "echo '{output}'", + "command-context": {"output": "hello", "extra": None}, + "run-as-root": False, + "sparse-profile": False, + "tooltool-downloads": False, + }, + }, + ), +) +def test_run_task_command_context(task, transform, workerfn): + config, job_, taskdesc, _ = transform(task) + job_ = deepcopy(job_) + + def assert_cmd(expected): + cmd = taskdesc["worker"]["command"] + while isinstance(cmd, list): + cmd = cmd[-1] + assert cmd == expected + + workerfn(config, job_, taskdesc) + assert_cmd("echo 'hello'") + + job_copy = job_.copy() + del job_copy["run"]["command-context"] + workerfn(config, job_copy, taskdesc) + assert_cmd("echo '{output}'") + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_test.py b/taskcluster/gecko_taskgraph/test/test_transforms_test.py new file mode 100644 index 0000000000..2b90fed3e7 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_transforms_test.py @@ -0,0 +1,328 @@ +# Any copyright is dedicated to the Public Domain. +# https://creativecommons.org/publicdomain/zero/1.0/ +""" +Tests for the 'tests.py' transforms +""" + +import hashlib +import json +from functools import partial +from pprint import pprint + +import mozunit +import pytest + +from gecko_taskgraph.transforms import test as test_transforms + + +@pytest.fixture +def make_test_task(): + """Create a test task definition with required default values.""" + + def inner(**extra): + task = { + "attributes": {}, + "build-platform": "linux64", + "mozharness": {"extra-options": []}, + "test-platform": "linux64", + "treeherder-symbol": "g(t)", + "try-name": "task", + } + task.update(extra) + return task + + return inner + + +def test_split_variants(monkeypatch, run_full_config_transform, make_test_task): + # mock out variant definitions + monkeypatch.setattr( + test_transforms.variant, + "TEST_VARIANTS", + { + "foo": { + "description": "foo variant", + "suffix": "foo", + "component": "foo bar", + "expiration": "never", + "merge": { + "mozharness": { + "extra-options": [ + "--setpref=foo=1", + ], + }, + }, + }, + "bar": { + "description": "bar variant", + "suffix": "bar", + "component": "foo bar", + "expiration": "never", + "when": { + "$eval": "task['test-platform'][:5] == 'linux'", + }, + "merge": { + "mozharness": { + "extra-options": [ + "--setpref=bar=1", + ], + }, + }, + "replace": {"tier": 2}, + }, + }, + ) + + def make_expected(variant): + """Helper to generate expected tasks.""" + return make_test_task( + **{ + "attributes": {"unittest_variant": variant}, + "description": f"{variant} variant", + "mozharness": { + "extra-options": [f"--setpref={variant}=1"], + }, + "treeherder-symbol": f"g-{variant}(t)", + "variant-suffix": f"-{variant}", + } + ) + + run_split_variants = partial( + run_full_config_transform, test_transforms.variant.split_variants + ) + + # test no variants + input_task = make_test_task( + **{ + "run-without-variant": True, + } + ) + tasks = list(run_split_variants(input_task)) + assert len(tasks) == 1 + assert tasks[0] == input_task + + # test variants are split into expected tasks + input_task = make_test_task( + **{ + "run-without-variant": True, + "variants": ["foo", "bar"], + } + ) + tasks = list(run_split_variants(input_task)) + assert len(tasks) == 3 + assert tasks[0] == make_test_task() + assert tasks[1] == make_expected("foo") + + expected = make_expected("bar") + expected["tier"] = 2 + assert tasks[2] == expected + + # test composite variants + input_task = make_test_task( + **{ + "run-without-variant": True, + "variants": ["foo+bar"], + } + ) + tasks = list(run_split_variants(input_task)) + assert len(tasks) == 2 + assert tasks[1]["attributes"]["unittest_variant"] == "foo+bar" + assert tasks[1]["mozharness"]["extra-options"] == [ + "--setpref=foo=1", + "--setpref=bar=1", + ] + assert tasks[1]["treeherder-symbol"] == "g-foo-bar(t)" + + # test 'when' filter + input_task = make_test_task( + **{ + "run-without-variant": True, + # this should cause task to be filtered out of 'bar' and 'foo+bar' variants + "test-platform": "windows", + "variants": ["foo", "bar", "foo+bar"], + } + ) + tasks = list(run_split_variants(input_task)) + assert len(tasks) == 2 + assert "unittest_variant" not in tasks[0]["attributes"] + assert tasks[1]["attributes"]["unittest_variant"] == "foo" + + # test 'run-without-variants=False' + input_task = make_test_task( + **{ + "run-without-variant": False, + "variants": ["foo"], + } + ) + tasks = list(run_split_variants(input_task)) + assert len(tasks) == 1 + assert tasks[0]["attributes"]["unittest_variant"] == "foo" + + +@pytest.mark.parametrize( + "task,expected", + ( + pytest.param( + { + "attributes": {"unittest_variant": "webrender-sw+1proc"}, + "test-platform": "linux1804-64-clang-trunk-qr/opt", + }, + { + "platform": { + "arch": "64", + "os": { + "name": "linux", + "version": "1804", + }, + }, + "build": { + "type": "opt", + "clang-trunk": True, + }, + "runtime": { + "1proc": True, + "webrender-sw": True, + }, + }, + id="linux", + ), + pytest.param( + { + "attributes": {}, + "test-platform": "linux2204-64-wayland-shippable/opt", + }, + { + "platform": { + "arch": "64", + "display": "wayland", + "os": { + "name": "linux", + "version": "2204", + }, + }, + "build": { + "type": "opt", + "shippable": True, + }, + "runtime": {}, + }, + id="linux wayland shippable", + ), + pytest.param( + { + "attributes": {}, + "test-platform": "android-hw-a51-11-0-arm7-shippable-qr/opt", + }, + { + "platform": { + "arch": "arm7", + "device": "a51", + "os": { + "name": "android", + "version": "11.0", + }, + }, + "build": { + "type": "opt", + "shippable": True, + }, + "runtime": {}, + }, + id="android", + ), + pytest.param( + { + "attributes": {}, + "test-platform": "windows10-64-2004-ref-hw-2017-ccov/debug", + }, + { + "platform": { + "arch": "64", + "machine": "ref-hw-2017", + "os": { + "build": "2004", + "name": "windows", + "version": "10", + }, + }, + "build": { + "type": "debug", + "ccov": True, + }, + "runtime": {}, + }, + id="windows", + ), + ), +) +def test_set_test_setting(run_transform, task, expected): + # add hash to 'expected' + expected["_hash"] = hashlib.sha256( + json.dumps(expected, sort_keys=True).encode("utf-8") + ).hexdigest()[:12] + + task = list(run_transform(test_transforms.other.set_test_setting, task))[0] + assert "test-setting" in task + assert task["test-setting"] == expected + + +def assert_spi_not_disabled(task): + extra_options = task["mozharness"]["extra-options"] + # The pref to enable this gets set outside of this transform, so only + # bother asserting that the pref to disable does not exist. + assert ( + "--setpref=media.peerconnection.mtransport_process=false" not in extra_options + ) + assert "--setpref=network.process.enabled=false" not in extra_options + + +def assert_spi_disabled(task): + extra_options = task["mozharness"]["extra-options"] + assert "--setpref=media.peerconnection.mtransport_process=false" in extra_options + assert "--setpref=media.peerconnection.mtransport_process=true" not in extra_options + assert "--setpref=network.process.enabled=false" in extra_options + assert "--setpref=network.process.enabled=true" not in extra_options + + +@pytest.mark.parametrize( + "task,callback", + ( + pytest.param( + {"attributes": {"unittest_variant": "socketprocess"}}, + assert_spi_not_disabled, + id="socketprocess", + ), + pytest.param( + { + "attributes": {"unittest_variant": "socketprocess_networking"}, + }, + assert_spi_not_disabled, + id="socketprocess_networking", + ), + pytest.param({}, assert_spi_disabled, id="no variant"), + pytest.param( + {"suite": "cppunit", "attributes": {"unittest_variant": "socketprocess"}}, + assert_spi_not_disabled, + id="excluded suite", + ), + pytest.param( + {"attributes": {"unittest_variant": "no-fission+socketprocess"}}, + assert_spi_not_disabled, + id="composite variant", + ), + ), +) +def test_ensure_spi_disabled_on_all_but_spi( + make_test_task, run_transform, task, callback +): + task.setdefault("suite", "mochitest-plain") + task = make_test_task(**task) + task = list( + run_transform(test_transforms.other.ensure_spi_disabled_on_all_but_spi, task) + )[0] + pprint(task) + callback(task) + + +if __name__ == "__main__": + mozunit.main() diff --git a/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py b/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py new file mode 100644 index 0000000000..a37de53378 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py @@ -0,0 +1,430 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import unittest + +from mozunit import main +from taskgraph.graph import Graph +from taskgraph.task import Task +from taskgraph.taskgraph import TaskGraph + +from gecko_taskgraph.try_option_syntax import TryOptionSyntax, parse_message + + +def unittest_task(n, tp, bt="opt"): + return ( + n, + Task( + "test", + n, + { + "unittest_try_name": n, + "test_platform": tp.split("/")[0], + "build_type": bt, + }, + {}, + ), + ) + + +def talos_task(n, tp, bt="opt"): + return ( + n, + Task( + "test", + n, + { + "talos_try_name": n, + "test_platform": tp.split("/")[0], + "build_type": bt, + }, + {}, + ), + ) + + +tasks = { + k: v + for k, v in [ + unittest_task("mochitest-browser-chrome", "linux/opt"), + unittest_task("mochitest-browser-chrome-e10s", "linux64/opt"), + unittest_task("mochitest-chrome", "linux/debug", "debug"), + unittest_task("mochitest-webgl1-core", "linux/debug", "debug"), + unittest_task("mochitest-webgl1-ext", "linux/debug", "debug"), + unittest_task("mochitest-webgl2-core", "linux/debug", "debug"), + unittest_task("mochitest-webgl2-ext", "linux/debug", "debug"), + unittest_task("mochitest-webgl2-deqp", "linux/debug", "debug"), + unittest_task("extra1", "linux", "debug/opt"), + unittest_task("extra2", "win32/opt"), + unittest_task("crashtest-e10s", "linux/other"), + unittest_task("gtest", "linux64/asan"), + talos_task("dromaeojs", "linux64/psan"), + unittest_task("extra3", "linux/opt"), + unittest_task("extra4", "linux64/debug", "debug"), + unittest_task("extra5", "linux/this"), + unittest_task("extra6", "linux/that"), + unittest_task("extra7", "linux/other"), + unittest_task("extra8", "linux64/asan"), + talos_task("extra9", "linux64/psan"), + ] +} + +RIDEALONG_BUILDS = { + "linux": ["linux-ridealong"], + "linux64": ["linux64-ridealong"], +} + +GRAPH_CONFIG = { + "try": {"ridealong-builds": RIDEALONG_BUILDS}, +} + +for r in RIDEALONG_BUILDS.values(): + tasks.update({k: v for k, v in [unittest_task(n + "-test", n) for n in r]}) + +unittest_tasks = {k: v for k, v in tasks.items() if "unittest_try_name" in v.attributes} +talos_tasks = {k: v for k, v in tasks.items() if "talos_try_name" in v.attributes} +graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set())) + + +class TestTryOptionSyntax(unittest.TestCase): + def test_unknown_args(self): + "unknown arguments are ignored" + parameters = parse_message("try: --doubledash -z extra") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + # equilvant to "try:".. + self.assertEqual(tos.build_types, []) + self.assertEqual(tos.jobs, []) + + def test_apostrophe_in_message(self): + "apostrophe does not break parsing" + parameters = parse_message("Increase spammy log's log level. try: -b do") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.build_types), ["debug", "opt"]) + + def test_b_do(self): + "-b do should produce both build_types" + parameters = parse_message("try: -b do") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.build_types), ["debug", "opt"]) + + def test_b_d(self): + "-b d should produce build_types=['debug']" + parameters = parse_message("try: -b d") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.build_types), ["debug"]) + + def test_b_o(self): + "-b o should produce build_types=['opt']" + parameters = parse_message("try: -b o") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.build_types), ["opt"]) + + def test_build_o(self): + "--build o should produce build_types=['opt']" + parameters = parse_message("try: --build o") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.build_types), ["opt"]) + + def test_b_dx(self): + "-b dx should produce build_types=['debug'], silently ignoring the x" + parameters = parse_message("try: -b dx") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.build_types), ["debug"]) + + def test_j_job(self): + "-j somejob sets jobs=['somejob']" + parameters = parse_message("try: -j somejob") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.jobs), ["somejob"]) + + def test_j_jobs(self): + "-j job1,job2 sets jobs=['job1', 'job2']" + parameters = parse_message("try: -j job1,job2") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.jobs), ["job1", "job2"]) + + def test_j_all(self): + "-j all sets jobs=None" + parameters = parse_message("try: -j all") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.jobs, None) + + def test_j_twice(self): + "-j job1 -j job2 sets jobs=job1, job2" + parameters = parse_message("try: -j job1 -j job2") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.jobs), sorted(["job1", "job2"])) + + def test_p_all(self): + "-p all sets platforms=None" + parameters = parse_message("try: -p all") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.platforms, None) + + def test_p_linux(self): + "-p linux sets platforms=['linux', 'linux-ridealong']" + parameters = parse_message("try: -p linux") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.platforms, ["linux", "linux-ridealong"]) + + def test_p_linux_win32(self): + "-p linux,win32 sets platforms=['linux', 'linux-ridealong', 'win32']" + parameters = parse_message("try: -p linux,win32") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.platforms), ["linux", "linux-ridealong", "win32"]) + + def test_p_expands_ridealongs(self): + "-p linux,linux64 includes the RIDEALONG_BUILDS" + parameters = parse_message("try: -p linux,linux64") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + platforms = set(["linux"] + RIDEALONG_BUILDS["linux"]) + platforms |= set(["linux64"] + RIDEALONG_BUILDS["linux64"]) + self.assertEqual(sorted(tos.platforms), sorted(platforms)) + + def test_u_none(self): + "-u none sets unittests=[]" + parameters = parse_message("try: -u none") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.unittests, []) + + def test_u_all(self): + "-u all sets unittests=[..whole list..]" + parameters = parse_message("try: -u all") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.unittests, [{"test": t} for t in sorted(unittest_tasks)]) + + def test_u_single(self): + "-u mochitest-webgl1-core sets unittests=[mochitest-webgl1-core]" + parameters = parse_message("try: -u mochitest-webgl1-core") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.unittests, [{"test": "mochitest-webgl1-core"}]) + + def test_u_alias(self): + "-u mochitest-gl sets unittests=[mochitest-webgl*]" + parameters = parse_message("try: -u mochitest-gl") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + tos.unittests, + [ + {"test": t} + for t in [ + "mochitest-webgl1-core", + "mochitest-webgl1-ext", + "mochitest-webgl2-core", + "mochitest-webgl2-deqp", + "mochitest-webgl2-ext", + ] + ], + ) + + def test_u_multi_alias(self): + "-u e10s sets unittests=[all e10s unittests]" + parameters = parse_message("try: -u e10s") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + tos.unittests, [{"test": t} for t in sorted(unittest_tasks) if "e10s" in t] + ) + + def test_u_commas(self): + "-u mochitest-webgl1-core,gtest sets unittests=both" + parameters = parse_message("try: -u mochitest-webgl1-core,gtest") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + tos.unittests, + [ + {"test": "gtest"}, + {"test": "mochitest-webgl1-core"}, + ], + ) + + def test_u_chunks(self): + "-u gtest-3,gtest-4 selects the third and fourth chunk of gtest" + parameters = parse_message("try: -u gtest-3,gtest-4") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + sorted(tos.unittests), + sorted( + [ + {"test": "gtest", "only_chunks": set("34")}, + ] + ), + ) + + def test_u_platform(self): + "-u gtest[linux] selects the linux platform for gtest" + parameters = parse_message("try: -u gtest[linux]") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + sorted(tos.unittests), + sorted( + [ + {"test": "gtest", "platforms": ["linux"]}, + ] + ), + ) + + def test_u_platforms(self): + "-u gtest[linux,win32] selects the linux and win32 platforms for gtest" + parameters = parse_message("try: -u gtest[linux,win32]") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + sorted(tos.unittests), + sorted( + [ + {"test": "gtest", "platforms": ["linux", "win32"]}, + ] + ), + ) + + def test_u_platforms_pretty(self): + """-u gtest[Ubuntu] selects the linux, linux64 and linux64-asan + platforms for gtest""" + parameters = parse_message("try: -u gtest[Ubuntu]") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + sorted(tos.unittests), + sorted( + [ + { + "test": "gtest", + "platforms": [ + "linux32", + "linux64", + "linux64-asan", + "linux1804-64", + "linux1804-64-asan", + ], + }, + ] + ), + ) + + def test_u_platforms_negated(self): + "-u gtest[-linux] selects all platforms but linux for gtest" + parameters = parse_message("try: -u gtest[-linux]") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + all_platforms = {x.attributes["test_platform"] for x in unittest_tasks.values()} + self.assertEqual( + sorted(tos.unittests[0]["platforms"]), + sorted(x for x in all_platforms if x != "linux"), + ) + + def test_u_platforms_negated_pretty(self): + "-u gtest[Ubuntu,-x64] selects just linux for gtest" + parameters = parse_message("try: -u gtest[Ubuntu,-x64]") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + sorted(tos.unittests), + sorted( + [ + {"test": "gtest", "platforms": ["linux32"]}, + ] + ), + ) + + def test_u_chunks_platforms(self): + "-u gtest-1[linux,win32] selects the linux and win32 platforms for chunk 1 of gtest" + parameters = parse_message("try: -u gtest-1[linux,win32]") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + tos.unittests, + [ + { + "test": "gtest", + "platforms": ["linux", "win32"], + "only_chunks": set("1"), + }, + ], + ) + + def test_t_none(self): + "-t none sets talos=[]" + parameters = parse_message("try: -t none") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.talos, []) + + def test_t_all(self): + "-t all sets talos=[..whole list..]" + parameters = parse_message("try: -t all") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.talos, [{"test": t} for t in sorted(talos_tasks)]) + + def test_t_single(self): + "-t mochitest-webgl sets talos=[mochitest-webgl]" + parameters = parse_message("try: -t mochitest-webgl") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.talos, [{"test": "mochitest-webgl"}]) + + # -t shares an implementation with -u, so it's not tested heavily + + def test_trigger_tests(self): + "--rebuild 10 sets trigger_tests" + parameters = parse_message("try: --rebuild 10") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.trigger_tests, 10) + + def test_talos_trigger_tests(self): + "--rebuild-talos 10 sets talos_trigger_tests" + parameters = parse_message("try: --rebuild-talos 10") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.talos_trigger_tests, 10) + + def test_interactive(self): + "--interactive sets interactive" + parameters = parse_message("try: --interactive") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.interactive, True) + + def test_all_email(self): + "--all-emails sets notifications" + parameters = parse_message("try: --all-emails") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.notifications, "all") + + def test_fail_email(self): + "--failure-emails sets notifications" + parameters = parse_message("try: --failure-emails") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.notifications, "failure") + + def test_no_email(self): + "no email settings don't set notifications" + parameters = parse_message("try:") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.notifications, None) + + def test_setenv(self): + "--setenv VAR=value adds a environment variables setting to env" + parameters = parse_message("try: --setenv VAR1=value1 --setenv VAR2=value2") + assert parameters["try_task_config"]["env"] == { + "VAR1": "value1", + "VAR2": "value2", + } + + def test_profile(self): + "--gecko-profile sets profile to true" + parameters = parse_message("try: --gecko-profile") + assert parameters["try_task_config"]["gecko-profile"] is True + + def test_tag(self): + "--tag TAG sets tag to TAG value" + parameters = parse_message("try: --tag tagName") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.tag, "tagName") + + def test_no_retry(self): + "--no-retry sets no_retry to true" + parameters = parse_message("try: --no-retry") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertTrue(tos.no_retry) + + def test_artifact(self): + "--artifact sets artifact to true" + parameters = parse_message("try: --artifact") + assert parameters["try_task_config"]["use-artifact-builds"] is True + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_util_attributes.py b/taskcluster/gecko_taskgraph/test/test_util_attributes.py new file mode 100644 index 0000000000..26181cb8bf --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_util_attributes.py @@ -0,0 +1,99 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import unittest + +from mozunit import main +from taskgraph.util.attributes import attrmatch + +from gecko_taskgraph.util.attributes import match_run_on_projects + + +class Attrmatch(unittest.TestCase): + def test_trivial_match(self): + """Given no conditions, anything matches""" + self.assertTrue(attrmatch({})) + + def test_missing_attribute(self): + """If a filtering attribute is not present, no match""" + self.assertFalse(attrmatch({}, someattr=10)) + + def test_literal_attribute(self): + """Literal attributes must match exactly""" + self.assertTrue(attrmatch({"att": 10}, att=10)) + self.assertFalse(attrmatch({"att": 10}, att=20)) + + def test_set_attribute(self): + """Set attributes require set membership""" + self.assertTrue(attrmatch({"att": 10}, att={9, 10})) + self.assertFalse(attrmatch({"att": 10}, att={19, 20})) + + def test_callable_attribute(self): + """Callable attributes are called and any False causes the match to fail""" + self.assertTrue(attrmatch({"att": 10}, att=lambda val: True)) + self.assertFalse(attrmatch({"att": 10}, att=lambda val: False)) + + def even(val): + return val % 2 == 0 + + self.assertTrue(attrmatch({"att": 10}, att=even)) + self.assertFalse(attrmatch({"att": 11}, att=even)) + + def test_all_matches_required(self): + """If only one attribute does not match, the result is False""" + self.assertFalse(attrmatch({"a": 1}, a=1, b=2, c=3)) + self.assertFalse(attrmatch({"a": 1, "b": 2}, a=1, b=2, c=3)) + self.assertTrue(attrmatch({"a": 1, "b": 2, "c": 3}, a=1, b=2, c=3)) + + +class MatchRunOnProjects(unittest.TestCase): + def test_empty(self): + self.assertFalse(match_run_on_projects("birch", [])) + + def test_all(self): + self.assertTrue(match_run_on_projects("birch", ["all"])) + self.assertTrue(match_run_on_projects("larch", ["all"])) + self.assertTrue(match_run_on_projects("autoland", ["all"])) + self.assertTrue(match_run_on_projects("mozilla-central", ["all"])) + self.assertTrue(match_run_on_projects("mozilla-beta", ["all"])) + self.assertTrue(match_run_on_projects("mozilla-release", ["all"])) + + def test_release(self): + self.assertFalse(match_run_on_projects("birch", ["release"])) + self.assertFalse(match_run_on_projects("larch", ["release"])) + self.assertFalse(match_run_on_projects("autoland", ["release"])) + self.assertTrue(match_run_on_projects("mozilla-central", ["release"])) + self.assertTrue(match_run_on_projects("mozilla-beta", ["release"])) + self.assertTrue(match_run_on_projects("mozilla-release", ["release"])) + + def test_integration(self): + self.assertFalse(match_run_on_projects("birch", ["integration"])) + self.assertFalse(match_run_on_projects("larch", ["integration"])) + self.assertTrue(match_run_on_projects("autoland", ["integration"])) + self.assertFalse(match_run_on_projects("mozilla-central", ["integration"])) + self.assertFalse(match_run_on_projects("mozilla-beta", ["integration"])) + self.assertFalse(match_run_on_projects("mozilla-integration", ["integration"])) + + def test_combo(self): + self.assertTrue(match_run_on_projects("birch", ["release", "birch", "maple"])) + self.assertFalse(match_run_on_projects("larch", ["release", "birch", "maple"])) + self.assertTrue(match_run_on_projects("maple", ["release", "birch", "maple"])) + self.assertFalse( + match_run_on_projects("autoland", ["release", "birch", "maple"]) + ) + self.assertTrue( + match_run_on_projects("mozilla-central", ["release", "birch", "maple"]) + ) + self.assertTrue( + match_run_on_projects("mozilla-beta", ["release", "birch", "maple"]) + ) + self.assertTrue( + match_run_on_projects("mozilla-release", ["release", "birch", "maple"]) + ) + self.assertTrue(match_run_on_projects("birch", ["birch", "trunk"])) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_util_backstop.py b/taskcluster/gecko_taskgraph/test/test_util_backstop.py new file mode 100644 index 0000000000..af9aabd5af --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_util_backstop.py @@ -0,0 +1,155 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from datetime import datetime +from textwrap import dedent +from time import mktime + +import pytest +from mozunit import main +from taskgraph.util.taskcluster import get_artifact_url, get_index_url, get_task_url + +from gecko_taskgraph.util.backstop import ( + BACKSTOP_INDEX, + BACKSTOP_PUSH_INTERVAL, + BACKSTOP_TIME_INTERVAL, + is_backstop, +) + +LAST_BACKSTOP_ID = 0 +LAST_BACKSTOP_PUSHDATE = mktime(datetime.now().timetuple()) +DEFAULT_RESPONSES = { + "index": { + "status": 200, + "json": {"taskId": LAST_BACKSTOP_ID}, + }, + "artifact": { + "status": 200, + "body": dedent( + """ + pushdate: {} + """.format( + LAST_BACKSTOP_PUSHDATE + ) + ), + }, + "status": { + "status": 200, + "json": {"status": {"state": "complete"}}, + }, +} + + +@pytest.fixture +def params(): + return { + "branch": "integration/autoland", + "head_repository": "https://hg.mozilla.org/integration/autoland", + "head_rev": "abcdef", + "project": "autoland", + "pushdate": LAST_BACKSTOP_PUSHDATE + 1, + "pushlog_id": LAST_BACKSTOP_ID + 1, + } + + +@pytest.mark.parametrize( + "response_args,extra_params,expected", + ( + pytest.param( + { + "index": {"status": 404}, + }, + {"pushlog_id": 1}, + True, + id="no previous backstop", + ), + pytest.param( + { + "index": DEFAULT_RESPONSES["index"], + "status": DEFAULT_RESPONSES["status"], + "artifact": {"status": 404}, + }, + {"pushlog_id": 1}, + False, + id="previous backstop not finished", + ), + pytest.param( + DEFAULT_RESPONSES, + { + "pushlog_id": LAST_BACKSTOP_ID + 1, + "pushdate": LAST_BACKSTOP_PUSHDATE + 1, + }, + False, + id="not a backstop", + ), + pytest.param( + {}, + { + "pushlog_id": BACKSTOP_PUSH_INTERVAL, + }, + True, + id="backstop interval", + ), + pytest.param( + DEFAULT_RESPONSES, + { + "pushdate": LAST_BACKSTOP_PUSHDATE + (BACKSTOP_TIME_INTERVAL * 60), + }, + True, + id="time elapsed", + ), + pytest.param( + {}, + { + "project": "try", + "pushlog_id": BACKSTOP_PUSH_INTERVAL, + }, + False, + id="try not a backstop", + ), + pytest.param( + {}, + { + "project": "mozilla-central", + }, + True, + id="release branches always a backstop", + ), + pytest.param( + { + "index": DEFAULT_RESPONSES["index"], + "status": { + "status": 200, + "json": {"status": {"state": "failed"}}, + }, + }, + {}, + True, + id="last backstop failed", + ), + ), +) +def test_is_backstop(responses, params, response_args, extra_params, expected): + urls = { + "index": get_index_url( + BACKSTOP_INDEX.format( + **{"trust-domain": "gecko", "project": params["project"]} + ) + ), + "artifact": get_artifact_url(LAST_BACKSTOP_ID, "public/parameters.yml"), + "status": get_task_url(LAST_BACKSTOP_ID) + "/status", + } + + for key in ("index", "status", "artifact"): + if key in response_args: + print(urls[key]) + responses.add(responses.GET, urls[key], **response_args[key]) + + params.update(extra_params) + assert is_backstop(params) is expected + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_util_bugbug.py b/taskcluster/gecko_taskgraph/test/test_util_bugbug.py new file mode 100644 index 0000000000..7e8865ddde --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_util_bugbug.py @@ -0,0 +1,57 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import mozunit + +from gecko_taskgraph.util.bugbug import BUGBUG_BASE_URL, push_schedules + + +def test_group_translation(responses): + branch = ("integration/autoland",) + rev = "abcdef" + query = f"/push/{branch}/{rev}/schedules" + url = BUGBUG_BASE_URL + query + + responses.add( + responses.GET, + url, + json={ + "groups": { + "dom/indexedDB": 1, + "testing/web-platform/tests/IndexedDB": 1, + "testing/web-platform/mozilla/tests/IndexedDB": 1, + }, + "config_groups": { + "dom/indexedDB": ["label1", "label2"], + "testing/web-platform/tests/IndexedDB": ["label3"], + "testing/web-platform/mozilla/tests/IndexedDB": ["label4"], + }, + }, + status=200, + ) + + assert len(push_schedules) == 0 + data = push_schedules(branch, rev) + print(data) + assert sorted(data["groups"]) == [ + "/IndexedDB", + "/_mozilla/IndexedDB", + "dom/indexedDB", + ] + assert data["config_groups"] == { + "dom/indexedDB": ["label1", "label2"], + "/IndexedDB": ["label3"], + "/_mozilla/IndexedDB": ["label4"], + } + assert len(push_schedules) == 1 + + # Value is memoized. + responses.reset() + push_schedules(branch, rev) + assert len(push_schedules) == 1 + + +if __name__ == "__main__": + mozunit.main() diff --git a/taskcluster/gecko_taskgraph/test/test_util_chunking.py b/taskcluster/gecko_taskgraph/test/test_util_chunking.py new file mode 100644 index 0000000000..99434af042 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_util_chunking.py @@ -0,0 +1,403 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import re +from itertools import combinations + +import pytest +from mozunit import main + +from gecko_taskgraph.util import chunking + +pytestmark = pytest.mark.slow + + +@pytest.fixture(scope="module") +def mock_manifest_runtimes(): + """Deterministically produce a list of simulated manifest runtimes. + + Args: + manifests (list): list of manifests against which simulated manifest + runtimes would be paired up to. + + Returns: + dict of manifest data paired with a float value representing runtime. + """ + + def inner(manifests): + manifests = sorted(manifests) + # Generate deterministic runtime data. + runtimes = [(i / 10) ** (i / 10) for i in range(len(manifests))] + return dict(zip(manifests, runtimes)) + + return inner + + +@pytest.fixture(scope="module") +def unchunked_manifests(): + """Produce a list of unchunked manifests to be consumed by test method. + + Args: + length (int, optional): number of path elements to keep. + cutoff (int, optional): number of generated test paths to remove + from the test set if user wants to limit the number of paths. + + Returns: + list: list of test paths. + """ + data = ["blueberry", "nashi", "peach", "watermelon"] + + def inner(suite, length=2, cutoff=0): + if "web-platform" in suite: + suffix = "" + prefix = "/" + elif "reftest" in suite: + suffix = ".list" + prefix = "" + else: + suffix = ".ini" + prefix = "" + return [prefix + "/".join(p) + suffix for p in combinations(data, length)][ + cutoff: + ] + + return inner + + +@pytest.fixture(scope="module") +def mock_task_definition(): + """Builds a mock task definition for use in testing. + + Args: + os_name (str): represents the os. + os_version (str): represents the os version + bits (int): software bits. + build_type (str): opt or debug. + build_attrs (list, optional): specify build attribute(s) + variants (list, optional): specify runtime variant(s) + + Returns: + dict: mocked task definition. + """ + + def inner(os_name, os_version, bits, build_type, build_attrs=None, variants=None): + setting = { + "platform": { + "arch": str(bits), + "os": { + "name": os_name, + "version": os_version, + }, + }, + "build": { + "type": build_type, + }, + "runtime": {}, + } + + # Optionally set build attributes and runtime variants. + if build_attrs: + if isinstance(build_attrs, str): + build_attrs = [build_attrs] + for attr in build_attrs: + setting["build"][attr] = True + + if variants: + if isinstance(variants, str): + variants = [variants] + for variant in variants: + setting["runtime"][variant] = True + return {"test-name": "foo", "test-setting": setting} + + return inner + + +@pytest.fixture(scope="module") +def mock_mozinfo(): + """Returns a mocked mozinfo object, similar to guess_mozinfo_from_task(). + + Args: + os (str): typically one of 'win, linux, mac, android'. + processor (str): processor architecture. + asan (bool, optional): addressanitizer build. + bits (int, optional): defaults to 64. + ccov (bool, optional): code coverage build. + debug (bool, optional): debug type build. + fission (bool, optional): process fission. + headless (bool, optional): headless browser testing without displays. + tsan (bool, optional): threadsanitizer build. + + Returns: + dict: Dictionary mimickign the results from guess_mozinfo_from_task. + """ + + def inner( + os, + processor, + asan=False, + bits=64, + ccov=False, + debug=False, + fission=False, + headless=False, + tsan=False, + ): + return { + "os": os, + "processor": processor, + "toolkit": "", + "asan": asan, + "bits": bits, + "ccov": ccov, + "debug": debug, + "e10s": True, + "fission": fission, + "headless": headless, + "tsan": tsan, + "appname": "firefox", + "condprof": False, + } + + return inner + + +@pytest.mark.parametrize( + "params,exception", + [ + [("win", "7", 32, "opt"), None], + [("win", "10", 64, "opt"), None], + [("linux", "1804", 64, "debug"), None], + [("macosx", "1015", 64, "debug"), None], + [("macosx", "1100", 64, "opt"), None], + [("android", "", 64, "debug"), None], + [("and", "", 64, "debug"), ValueError], + [("", "", 64, "opt"), ValueError], + [("linux", "1804", 64, "opt", ["ccov"]), None], + [("linux", "1804", 64, "opt", ["asan"]), None], + [("win", "10", 64, "opt", ["tsan"]), None], + [("mac", "1100", 64, "opt", ["ccov"]), None], + [("android", "", 64, "opt", None, ["fission"]), None], + [("win", "10", "aarch64", "opt"), None], + ], +) +def test_guess_mozinfo_from_task(params, exception, mock_task_definition): + """Tests the mozinfo guessing process.""" + # Set up a mocked task object. + task = mock_task_definition(*params) + + if exception: + with pytest.raises(exception): + result = chunking.guess_mozinfo_from_task(task) + else: + expected_toolkits = { + "android": "android", + "linux": "gtk", + "mac": "cocoa", + "win": "windows", + } + result = chunking.guess_mozinfo_from_task(task) + setting = task["test-setting"] + + assert str(result["bits"]) in setting["platform"]["arch"] + assert result["os"] in ("android", "linux", "mac", "win") + assert result["os"] in setting["platform"]["os"]["name"] + assert result["toolkit"] == expected_toolkits[result["os"]] + + # Ensure the outcome of special build variants being present match what + # guess_mozinfo_from_task method returns for these attributes. + assert ("asan" in setting["build"]) == result["asan"] + assert ("tsan" in setting["build"]) == result["tsan"] + assert ("ccov" in setting["build"]) == result["ccov"] + + # Ensure runtime variants match + assert ("fission" in setting["runtime"]) == result["fission"] + assert ("1proc" in setting["runtime"]) != result["e10s"] + + +@pytest.mark.parametrize("platform", ["unix", "windows", "android"]) +@pytest.mark.parametrize( + "suite", ["crashtest", "reftest", "web-platform-tests", "xpcshell"] +) +def test_get_runtimes(platform, suite): + """Tests that runtime information is returned for known good configurations.""" + assert chunking.get_runtimes(platform, suite) + + +@pytest.mark.parametrize( + "platform,suite,exception", + [ + ("nonexistent_platform", "nonexistent_suite", KeyError), + ("unix", "nonexistent_suite", KeyError), + ("unix", "", TypeError), + ("", "", TypeError), + ("", "nonexistent_suite", TypeError), + ], +) +def test_get_runtimes_invalid(platform, suite, exception): + """Ensure get_runtimes() method raises an exception if improper request is made.""" + with pytest.raises(exception): + chunking.get_runtimes(platform, suite) + + +@pytest.mark.parametrize( + "suite", + [ + "web-platform-tests", + "web-platform-tests-reftest", + "web-platform-tests-wdspec", + "web-platform-tests-crashtest", + ], +) +@pytest.mark.parametrize("chunks", [1, 3, 6, 20]) +def test_mock_chunk_manifests_wpt(unchunked_manifests, suite, chunks): + """Tests web-platform-tests and its subsuites chunking process.""" + # Setup. + manifests = unchunked_manifests(suite) + + # Generate the expected results, by generating list of indices that each + # manifest should go into and then appending each item to that index. + # This method is intentionally different from the way chunking.py performs + # chunking for cross-checking. + expected = [[] for _ in range(chunks)] + indexed = zip(manifests, list(range(0, chunks)) * len(manifests)) + for i in indexed: + expected[i[1]].append(i[0]) + + # Call the method under test on unchunked manifests. + chunked_manifests = chunking.chunk_manifests(suite, "unix", chunks, manifests) + + # Assertions and end test. + assert chunked_manifests + if chunks > len(manifests): + # If chunk count exceeds number of manifests, not all chunks will have + # manifests. + with pytest.raises(AssertionError): + assert all(chunked_manifests) + else: + assert all(chunked_manifests) + minimum = min(len(c) for c in chunked_manifests) + maximum = max(len(c) for c in chunked_manifests) + assert maximum - minimum <= 1 + assert expected == chunked_manifests + + +@pytest.mark.parametrize( + "suite", + [ + "mochitest-devtools-chrome", + "mochitest-browser-chrome", + "mochitest-plain", + "mochitest-chrome", + "xpcshell", + ], +) +@pytest.mark.parametrize("chunks", [1, 3, 6, 20]) +def test_mock_chunk_manifests( + mock_manifest_runtimes, unchunked_manifests, suite, chunks +): + """Tests non-WPT tests and its subsuites chunking process.""" + # Setup. + manifests = unchunked_manifests(suite) + + # Call the method under test on unchunked manifests. + chunked_manifests = chunking.chunk_manifests(suite, "unix", chunks, manifests) + + # Assertions and end test. + assert chunked_manifests + if chunks > len(manifests): + # If chunk count exceeds number of manifests, not all chunks will have + # manifests. + with pytest.raises(AssertionError): + assert all(chunked_manifests) + else: + assert all(chunked_manifests) + + +@pytest.mark.parametrize( + "suite", + [ + "web-platform-tests", + "web-platform-tests-reftest", + "xpcshell", + "mochitest-plain", + "mochitest-devtools-chrome", + "mochitest-browser-chrome", + "mochitest-chrome", + ], +) +@pytest.mark.parametrize( + "platform", + [ + ("mac", "x86_64"), + ("win", "x86_64"), + ("win", "x86"), + ("win", "aarch64"), + ("linux", "x86_64"), + ("linux", "x86"), + ], +) +def test_get_manifests(suite, platform, mock_mozinfo): + """Tests the DefaultLoader class' ability to load manifests.""" + mozinfo = mock_mozinfo(*platform) + + loader = chunking.DefaultLoader([]) + manifests = loader.get_manifests(suite, frozenset(mozinfo.items())) + + assert manifests + assert manifests["active"] + if "web-platform" in suite: + assert manifests["skipped"] == [] + else: + assert manifests["skipped"] + + items = manifests["active"] + if suite == "xpcshell": + assert all([re.search(r"xpcshell(.*)?.ini", m) for m in items]) + if "mochitest" in suite: + assert all([re.search(r"(mochitest|chrome|browser).*.ini", m) for m in items]) + if "web-platform" in suite: + assert all([m.startswith("/") and m.count("/") <= 4 for m in items]) + + +@pytest.mark.parametrize( + "suite", + [ + "mochitest-devtools-chrome", + "mochitest-browser-chrome", + "mochitest-plain", + "mochitest-chrome", + "web-platform-tests", + "web-platform-tests-reftest", + "xpcshell", + ], +) +@pytest.mark.parametrize( + "platform", + [ + ("mac", "x86_64"), + ("win", "x86_64"), + ("linux", "x86_64"), + ], +) +@pytest.mark.parametrize("chunks", [1, 3, 6, 20]) +def test_chunk_manifests(suite, platform, chunks, mock_mozinfo): + """Tests chunking with real manifests.""" + mozinfo = mock_mozinfo(*platform) + + loader = chunking.DefaultLoader([]) + manifests = loader.get_manifests(suite, frozenset(mozinfo.items())) + + chunked_manifests = chunking.chunk_manifests( + suite, platform, chunks, manifests["active"] + ) + + # Assertions and end test. + assert chunked_manifests + assert len(chunked_manifests) == chunks + assert all(chunked_manifests) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_util_docker.py b/taskcluster/gecko_taskgraph/test/test_util_docker.py new file mode 100644 index 0000000000..49c01738fe --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_util_docker.py @@ -0,0 +1,255 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import shutil +import stat +import tarfile +import tempfile +import unittest +from unittest import mock + +import taskcluster_urls as liburls +from mozunit import MockedOpen, main + +from gecko_taskgraph.util import docker + +MODE_STANDARD = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + + +@mock.patch.dict("os.environ", {"TASKCLUSTER_ROOT_URL": liburls.test_root_url()}) +class TestDocker(unittest.TestCase): + def test_generate_context_hash(self): + tmpdir = tempfile.mkdtemp() + try: + os.makedirs(os.path.join(tmpdir, "docker", "my-image")) + p = os.path.join(tmpdir, "docker", "my-image", "Dockerfile") + with open(p, "w") as f: + f.write("FROM node\nADD a-file\n") + os.chmod(p, MODE_STANDARD) + p = os.path.join(tmpdir, "docker", "my-image", "a-file") + with open(p, "w") as f: + f.write("data\n") + os.chmod(p, MODE_STANDARD) + self.assertIn( + docker.generate_context_hash( + tmpdir, + os.path.join(tmpdir, "docker/my-image"), + "my-image", + {}, + ), + ( + "680532a33c845e3b4f8ea8a7bd697da579b647f28c29f7a0a71e51e6cca33983", + "cc02f943ae87b283749369fa9c4f6a74639c27a7b9972c99de58e5d9fb3a98ae", + ), + ) + finally: + shutil.rmtree(tmpdir) + + def test_docker_image_explicit_registry(self): + files = {} + files[f"{docker.IMAGE_DIR}/myimage/REGISTRY"] = "cool-images" + files[f"{docker.IMAGE_DIR}/myimage/VERSION"] = "1.2.3" + files[f"{docker.IMAGE_DIR}/myimage/HASH"] = "sha256:434..." + with MockedOpen(files): + self.assertEqual( + docker.docker_image("myimage"), "cool-images/myimage@sha256:434..." + ) + + def test_docker_image_explicit_registry_by_tag(self): + files = {} + files[f"{docker.IMAGE_DIR}/myimage/REGISTRY"] = "myreg" + files[f"{docker.IMAGE_DIR}/myimage/VERSION"] = "1.2.3" + files[f"{docker.IMAGE_DIR}/myimage/HASH"] = "sha256:434..." + with MockedOpen(files): + self.assertEqual( + docker.docker_image("myimage", by_tag=True), "myreg/myimage:1.2.3" + ) + + def test_docker_image_default_registry(self): + files = {} + files[f"{docker.IMAGE_DIR}/REGISTRY"] = "mozilla" + files[f"{docker.IMAGE_DIR}/myimage/VERSION"] = "1.2.3" + files[f"{docker.IMAGE_DIR}/myimage/HASH"] = "sha256:434..." + with MockedOpen(files): + self.assertEqual( + docker.docker_image("myimage"), "mozilla/myimage@sha256:434..." + ) + + def test_docker_image_default_registry_by_tag(self): + files = {} + files[f"{docker.IMAGE_DIR}/REGISTRY"] = "mozilla" + files[f"{docker.IMAGE_DIR}/myimage/VERSION"] = "1.2.3" + files[f"{docker.IMAGE_DIR}/myimage/HASH"] = "sha256:434..." + with MockedOpen(files): + self.assertEqual( + docker.docker_image("myimage", by_tag=True), "mozilla/myimage:1.2.3" + ) + + def test_create_context_tar_basic(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, "test_image") + os.mkdir(d) + with open(os.path.join(d, "Dockerfile"), "a"): + pass + os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD) + + with open(os.path.join(d, "extra"), "a"): + pass + os.chmod(os.path.join(d, "extra"), MODE_STANDARD) + + tp = os.path.join(tmp, "tar") + h = docker.create_context_tar(tmp, d, tp, "my_image", {}) + self.assertIn( + h, + ( + "eae3ad00936085eb3e5958912f79fb06ee8e14a91f7157c5f38625f7ddacb9c7", + "9ff54ee091c4f346e94e809b03efae5aa49a5c1db152f9f633682cfa005f7422", + ), + ) + + # File prefix should be "my_image" + with tarfile.open(tp, "r:gz") as tf: + self.assertEqual( + tf.getnames(), + [ + "Dockerfile", + "extra", + ], + ) + finally: + shutil.rmtree(tmp) + + def test_create_context_topsrcdir_files(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, "test-image") + os.mkdir(d) + with open(os.path.join(d, "Dockerfile"), "wb") as fh: + fh.write(b"# %include extra/file0\n") + os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD) + + extra = os.path.join(tmp, "extra") + os.mkdir(extra) + with open(os.path.join(extra, "file0"), "a"): + pass + os.chmod(os.path.join(extra, "file0"), MODE_STANDARD) + + tp = os.path.join(tmp, "tar") + h = docker.create_context_tar(tmp, d, tp, "test_image", {}) + self.assertIn( + h, + ( + "49dc3827530cd344d7bcc52e1fdd4aefc632568cf442cffd3dd9633a58f271bf", + "8f8e3dd2b712003cd12bb39e5a84fc2a7c06e891cf481613a52bf3db472c4ca9", + ), + ) + + with tarfile.open(tp, "r:gz") as tf: + self.assertEqual( + tf.getnames(), + [ + "Dockerfile", + "topsrcdir/extra/file0", + ], + ) + finally: + shutil.rmtree(tmp) + + def test_create_context_absolute_path(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, "test-image") + os.mkdir(d) + + # Absolute paths in %include syntax are not allowed. + with open(os.path.join(d, "Dockerfile"), "wb") as fh: + fh.write(b"# %include /etc/shadow\n") + + with self.assertRaisesRegexp(Exception, "cannot be absolute"): + docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"), "test", {}) + finally: + shutil.rmtree(tmp) + + def test_create_context_outside_topsrcdir(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, "test-image") + os.mkdir(d) + + with open(os.path.join(d, "Dockerfile"), "wb") as fh: + fh.write(b"# %include foo/../../../etc/shadow\n") + + with self.assertRaisesRegexp(Exception, "path outside topsrcdir"): + docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"), "test", {}) + finally: + shutil.rmtree(tmp) + + def test_create_context_missing_extra(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, "test-image") + os.mkdir(d) + + with open(os.path.join(d, "Dockerfile"), "wb") as fh: + fh.write(b"# %include does/not/exist\n") + + with self.assertRaisesRegexp(Exception, "path does not exist"): + docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"), "test", {}) + finally: + shutil.rmtree(tmp) + + def test_create_context_extra_directory(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, "test-image") + os.mkdir(d) + + with open(os.path.join(d, "Dockerfile"), "wb") as fh: + fh.write(b"# %include extra\n") + fh.write(b"# %include file0\n") + os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD) + + extra = os.path.join(tmp, "extra") + os.mkdir(extra) + for i in range(3): + p = os.path.join(extra, "file%d" % i) + with open(p, "wb") as fh: + fh.write(b"file%d" % i) + os.chmod(p, MODE_STANDARD) + + with open(os.path.join(tmp, "file0"), "a"): + pass + os.chmod(os.path.join(tmp, "file0"), MODE_STANDARD) + + tp = os.path.join(tmp, "tar") + h = docker.create_context_tar(tmp, d, tp, "my_image", {}) + + self.assertIn( + h, + ( + "a392f23cd6606ae43116390a4d0113354cff1e688a41d46f48b0fb25e90baa13", + "02325bdc508c2e941959170beeb840f6bb91d0675cb8095783a7db7301d136b2", + ), + ) + + with tarfile.open(tp, "r:gz") as tf: + self.assertEqual( + tf.getnames(), + [ + "Dockerfile", + "topsrcdir/extra/file0", + "topsrcdir/extra/file1", + "topsrcdir/extra/file2", + "topsrcdir/file0", + ], + ) + finally: + shutil.rmtree(tmp) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_util_partials.py b/taskcluster/gecko_taskgraph/test/test_util_partials.py new file mode 100644 index 0000000000..3630d7b0ec --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_util_partials.py @@ -0,0 +1,128 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +from unittest import mock + +from mozunit import main + +from gecko_taskgraph.util import partials + +release_blob = { + "fileUrls": { + "release-localtest": { + "completes": { + "*": "%OS_FTP%/%LOCALE%/firefox-92.0.1.complete.mar", + } + } + }, + "platforms": { + "WINNT_x86_64-msvc": { + "locales": { + "en-US": { + "buildID": "20210922161155", + } + } + } + }, +} + + +def nightly_blob(release): + return { + "platforms": { + "WINNT_x86_64-msvc": { + "locales": { + "en-US": { + "buildID": release[-14:], + "completes": [{"fileUrl": release}], + } + } + } + } + } + + +class TestReleaseHistory(unittest.TestCase): + @mock.patch("gecko_taskgraph.util.partials.get_release_builds") + @mock.patch("gecko_taskgraph.util.partials.get_sorted_releases") + def test_populate_release_history(self, get_sorted_releases, get_release_builds): + self.assertEqual( + partials.populate_release_history( + "Firefox", "mozilla-release", partial_updates={} + ), + {}, + ) + get_sorted_releases.assert_not_called() + get_release_builds.assert_not_called() + + def patched_get_sorted_releases(product, branch): + assert branch == "mozilla-central" + return [ + "Firefox-mozilla-central-nightly-20211003201113", + "Firefox-mozilla-central-nightly-20211003100640", + "Firefox-mozilla-central-nightly-20211002213629", + "Firefox-mozilla-central-nightly-20211002095048", + "Firefox-mozilla-central-nightly-20211001214601", + "Firefox-mozilla-central-nightly-20211001093323", + ] + + def patched_get_release_builds(release, branch): + if branch == "mozilla-central": + return nightly_blob(release) + if branch == "mozilla-release": + return release_blob + + get_sorted_releases.side_effect = patched_get_sorted_releases + get_release_builds.side_effect = patched_get_release_builds + + self.assertEqual( + partials.populate_release_history( + "Firefox", + "mozilla-release", + partial_updates={"92.0.1": {"buildNumber": 1}}, + ), + { + "WINNT_x86_64-msvc": { + "en-US": { + "target-92.0.1.partial.mar": { + "buildid": "20210922161155", + "mar_url": "win64/en-US/firefox-92.0.1.complete.mar", + "previousVersion": "92.0.1", + "previousBuildNumber": 1, + "product": "Firefox", + } + } + } + }, + ) + self.assertEqual( + partials.populate_release_history("Firefox", "mozilla-central"), + { + "WINNT_x86_64-msvc": { + "en-US": { + "target.partial-1.mar": { + "buildid": "20211003201113", + "mar_url": "Firefox-mozilla-central-nightly-20211003201113", + }, + "target.partial-2.mar": { + "buildid": "20211003100640", + "mar_url": "Firefox-mozilla-central-nightly-20211003100640", + }, + "target.partial-3.mar": { + "buildid": "20211002213629", + "mar_url": "Firefox-mozilla-central-nightly-20211002213629", + }, + "target.partial-4.mar": { + "buildid": "20211002095048", + "mar_url": "Firefox-mozilla-central-nightly-20211002095048", + }, + } + } + }, + ) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_util_runnable_jobs.py b/taskcluster/gecko_taskgraph/test/test_util_runnable_jobs.py new file mode 100644 index 0000000000..993521b830 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_util_runnable_jobs.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import unittest + +from mozunit import main +from taskgraph.graph import Graph +from taskgraph.task import Task +from taskgraph.taskgraph import TaskGraph + +from gecko_taskgraph.decision import full_task_graph_to_runnable_jobs + + +class TestRunnableJobs(unittest.TestCase): + + tasks = [ + { + "kind": "build", + "label": "a", + "attributes": {}, + "task": { + "extra": {"treeherder": {"symbol": "B"}}, + }, + }, + { + "kind": "test", + "label": "b", + "attributes": {}, + "task": { + "extra": { + "treeherder": { + "collection": {"opt": True}, + "groupName": "Some group", + "groupSymbol": "GS", + "machine": {"platform": "linux64"}, + "symbol": "t", + } + }, + }, + }, + ] + + def make_taskgraph(self, tasks): + label_to_taskid = {k: k + "-tid" for k in tasks} + for label, task_id in label_to_taskid.items(): + tasks[label].task_id = task_id + graph = Graph(nodes=set(tasks), edges=set()) + taskgraph = TaskGraph(tasks, graph) + return taskgraph, label_to_taskid + + def test_taskgraph_to_runnable_jobs(self): + tg, label_to_taskid = self.make_taskgraph( + {t["label"]: Task(**t) for t in self.tasks[:]} + ) + + res = full_task_graph_to_runnable_jobs(tg.to_json()) + + self.assertEqual( + res, + { + "a": {"symbol": "B"}, + "b": { + "collection": {"opt": True}, + "groupName": "Some group", + "groupSymbol": "GS", + "symbol": "t", + "platform": "linux64", + }, + }, + ) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_util_templates.py b/taskcluster/gecko_taskgraph/test/test_util_templates.py new file mode 100644 index 0000000000..edfb13a277 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_util_templates.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import unittest + +import mozunit + +from gecko_taskgraph.util.templates import merge, merge_to + + +class MergeTest(unittest.TestCase): + def test_merge_to_dicts(self): + source = {"a": 1, "b": 2} + dest = {"b": "20", "c": 30} + expected = { + "a": 1, # source only + "b": 2, # source overrides dest + "c": 30, # dest only + } + self.assertEqual(merge_to(source, dest), expected) + self.assertEqual(dest, expected) + + def test_merge_to_lists(self): + source = {"x": [3, 4]} + dest = {"x": [1, 2]} + expected = {"x": [1, 2, 3, 4]} # dest first + self.assertEqual(merge_to(source, dest), expected) + self.assertEqual(dest, expected) + + def test_merge_diff_types(self): + source = {"x": [1, 2]} + dest = {"x": "abc"} + expected = {"x": [1, 2]} # source wins + self.assertEqual(merge_to(source, dest), expected) + self.assertEqual(dest, expected) + + def test_merge(self): + first = {"a": 1, "b": 2, "d": 11} + second = {"b": 20, "c": 30} + third = {"c": 300, "d": 400} + expected = { + "a": 1, + "b": 20, + "c": 300, + "d": 400, + } + self.assertEqual(merge(first, second, third), expected) + + # inputs haven't changed.. + self.assertEqual(first, {"a": 1, "b": 2, "d": 11}) + self.assertEqual(second, {"b": 20, "c": 30}) + self.assertEqual(third, {"c": 300, "d": 400}) + + def test_merge_by(self): + source = { + "x": "abc", + "y": {"by-foo": {"quick": "fox", "default": ["a", "b", "c"]}}, + } + dest = {"y": {"by-foo": {"purple": "rain", "default": ["x", "y", "z"]}}} + expected = { + "x": "abc", + "y": {"by-foo": {"quick": "fox", "default": ["a", "b", "c"]}}, + } # source wins + self.assertEqual(merge_to(source, dest), expected) + self.assertEqual(dest, expected) + + def test_merge_multiple_by(self): + source = {"x": {"by-foo": {"quick": "fox", "default": ["a", "b", "c"]}}} + dest = {"x": {"by-bar": {"purple": "rain", "default": ["x", "y", "z"]}}} + expected = { + "x": {"by-foo": {"quick": "fox", "default": ["a", "b", "c"]}} + } # source wins + self.assertEqual(merge_to(source, dest), expected) + + +if __name__ == "__main__": + mozunit.main() diff --git a/taskcluster/gecko_taskgraph/test/test_util_verify.py b/taskcluster/gecko_taskgraph/test/test_util_verify.py new file mode 100644 index 0000000000..e2f774a315 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_util_verify.py @@ -0,0 +1,149 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +There are some basic tests run as part of the Decision task to make sure +documentation exists for taskgraph functionality. +These functions are defined in gecko_taskgraph.generator and call +gecko_taskgraph.util.verify.verify_docs with different parameters to do the +actual checking. +""" + + +import os.path + +import pytest +from mozunit import main + +import gecko_taskgraph.util.verify +from gecko_taskgraph import GECKO +from gecko_taskgraph.util.verify import DocPaths, verify_docs + +FF_DOCS_BASE = os.path.join(GECKO, "taskcluster", "docs") +EXTRA_DOCS_BASE = os.path.abspath(os.path.join(os.path.dirname(__file__), "docs")) + + +@pytest.fixture +def mock_single_doc_path(monkeypatch): + """Set a single path containing documentation""" + mocked_documentation_paths = DocPaths() + mocked_documentation_paths.add(FF_DOCS_BASE) + monkeypatch.setattr( + gecko_taskgraph.util.verify, "documentation_paths", mocked_documentation_paths + ) + + +@pytest.fixture +def mock_two_doc_paths(monkeypatch): + """Set two paths containing documentation""" + mocked_documentation_paths = DocPaths() + mocked_documentation_paths.add(FF_DOCS_BASE) + mocked_documentation_paths.add(EXTRA_DOCS_BASE) + monkeypatch.setattr( + gecko_taskgraph.util.verify, "documentation_paths", mocked_documentation_paths + ) + + +@pytest.mark.usefixtures("mock_single_doc_path") +class PyTestSingleDocPath: + """ + Taskcluster documentation for Firefox is in a single directory. Check the tests + running at build time to make sure documentation exists, actually work themselves. + """ + + def test_heading(self): + """ + Look for a headings in filename matching identifiers. This is used when making sure + documentation exists for kinds and attributes. + """ + verify_docs( + filename="kinds.rst", + identifiers=["build", "packages", "toolchain"], + appearing_as="heading", + ) + with pytest.raises(Exception, match="missing from doc file"): + verify_docs( + filename="kinds.rst", + identifiers=["build", "packages", "badvalue"], + appearing_as="heading", + ) + + def test_inline_literal(self): + """ + Look for inline-literals in filename. Used when checking documentation for decision + task parameters and run-using functions. + """ + verify_docs( + filename="parameters.rst", + identifiers=["base_repository", "head_repository", "owner"], + appearing_as="inline-literal", + ) + with pytest.raises(Exception, match="missing from doc file"): + verify_docs( + filename="parameters.rst", + identifiers=["base_repository", "head_repository", "badvalue"], + appearing_as="inline-literal", + ) + + +@pytest.mark.usefixtures("mock_two_doc_paths") +class PyTestTwoDocPaths: + """ + Thunderbird extends Firefox's taskgraph with additional kinds. The documentation + for Thunderbird kinds are in its repository, and documentation_paths will have + two places to look for files. Run the same tests as for a single documentation + path, and cover additional possible scenarios. + """ + + def test_heading(self): + """ + Look for a headings in filename matching identifiers. This is used when + making sure documentation exists for kinds and attributes. + The first test looks for headings that are all within the first doc path, + the second test is new and has a heading found in the second path. + The final check has a identifier that will not match and should + produce an error. + """ + verify_docs( + filename="kinds.rst", + identifiers=["build", "packages", "toolchain"], + appearing_as="heading", + ) + verify_docs( + filename="kinds.rst", + identifiers=["build", "packages", "newkind"], + appearing_as="heading", + ) + with pytest.raises(Exception, match="missing from doc file"): + verify_docs( + filename="kinds.rst", + identifiers=["build", "packages", "badvalue"], + appearing_as="heading", + ) + + def test_inline_literal(self): + """ + Look for inline-literals in filename. Used when checking documentation for decision + task parameters and run-using functions. As with the heading tests, + the second check looks for an identifier in the added documentation path. + """ + verify_docs( + filename="parameters.rst", + identifiers=["base_repository", "head_repository", "owner"], + appearing_as="inline-literal", + ) + verify_docs( + filename="parameters.rst", + identifiers=["base_repository", "head_repository", "newparameter"], + appearing_as="inline-literal", + ) + with pytest.raises(Exception, match="missing from doc file"): + verify_docs( + filename="parameters.rst", + identifiers=["base_repository", "head_repository", "badvalue"], + appearing_as="inline-literal", + ) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/transforms/__init__.py b/taskcluster/gecko_taskgraph/transforms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/__init__.py diff --git a/taskcluster/gecko_taskgraph/transforms/artifact.py b/taskcluster/gecko_taskgraph/transforms/artifact.py new file mode 100644 index 0000000000..559148f7b4 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/artifact.py @@ -0,0 +1,116 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply different expiration dates to different artifacts based on a manifest file (artifacts.yml) +""" +import logging +import os +import sys + +import yaml +from taskgraph.transforms.base import TransformSequence +from yaml import YAMLError + +from gecko_taskgraph.transforms.job.common import get_expiration +from gecko_taskgraph.util.workertypes import worker_type_implementation + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +def read_artifact_manifest(manifest_path): + """Read the artifacts.yml manifest file and return it.""" + # logger.info(f"The current directory is {os.getcwd()}") + try: + with open(manifest_path, "r") as ymlf: + yml = yaml.safe_load(ymlf.read()) + return yml + except YAMLError as ye: + err = 'Failed to parse manifest "{manifest_path}". Invalid Yaml:' + err += ye + raise SystemExit(err) + except FileNotFoundError: + err = f'Failed to load manifest "{manifest_path}". File not found' + raise SystemExit(err) + except PermissionError: + err = f'Failed to load manifest "{manifest_path}". Permission Error' + raise SystemExit(err) + + +@transforms.add +def set_artifact_expiration(config, jobs): + """Set the expiration for certain artifacts based on a manifest file.""" + """--- + win: + - build_resources.json: short + + linux: + - target.crashreporter-symbols-full.tar.zst: medium + """ + transform_dir = os.path.dirname(__file__) + manifest = read_artifact_manifest(os.path.join(transform_dir, "artifacts.yml")) + + for job in jobs: + try: + platform = job["attributes"]["build_platform"] + except KeyError: + err = "Tried to get build_platfrom for job, but it does not exist. Exiting." + raise SystemExit(err) + if "worker" in job: + if "env" in job["worker"]: + if isinstance(job["worker"]["env"], dict): + job["worker"]["env"]["MOZ_ARTIFACT_PLATFORM"] = platform + else: + raise SystemExit( + f"Expected env to be a dict, but it was {type(job['worker']['env'])}" + ) + if "artifacts" in job["worker"]: + plat = platform.lower() + if "plain" in plat or "ccov" in plat or "rusttest" in plat: + art_dict = None + elif ( + plat == "toolchain-wasm32-wasi-compiler-rt-trunk" + or plat == "toolchain-linux64-x64-compiler-rt-trunk" + or plat == "toolchain-linux64-x86-compiler-rt-trunk" + or plat == "android-geckoview-docs" + ): + art_dict = None + elif plat.startswith("win"): + art_dict = manifest["win"] + elif plat.startswith("linux"): + art_dict = manifest["linux"] + elif plat.startswith("mac"): + art_dict = manifest["macos"] + elif plat.startswith("android"): + art_dict = manifest["android"] + else: + print( + 'The platform name "{plat}" didn\'t start with', + '"win", "mac", "android", or "linux".', + file=sys.stderr, + ) + art_dict = None + worker_implementation, _ = worker_type_implementation( + config.graph_config, config.params, job["worker-type"] + ) + if worker_implementation == "docker-worker": + artifact_dest = "/builds/worker/cidata/{}" + else: + artifact_dest = "cidata/{}" + + if art_dict is not None: + for art_name in art_dict.keys(): + # The 'artifacts' key of a job is a list at this stage. + # So, must append a new dict to the list + expiry_policy = art_dict[art_name] + expires = get_expiration(config, policy=expiry_policy) + new_art = { + "name": f"public/cidata/{art_name}", + "path": artifact_dest.format(art_name), + "type": "file", + "expires-after": expires, + } + job["worker"]["artifacts"].append(new_art) + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/artifacts.yml b/taskcluster/gecko_taskgraph/transforms/artifacts.yml new file mode 100644 index 0000000000..b41c657283 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/artifacts.yml @@ -0,0 +1,16 @@ +--- +win: + target.crashreporter-symbols-full.tar.zst: shortest + sccache.log: shortest + +linux: + target.crashreporter-symbols-full.tar.zst: shortest + sccache.log: shortest + +macos: + target.crashreporter-symbols-full.tar.zst: shortest + sccache.log: shortest + +android: + target.crashreporter-symbols-full.tar.zst: shortest + sccache.log: shortest diff --git a/taskcluster/gecko_taskgraph/transforms/attribution.py b/taskcluster/gecko_taskgraph/transforms/attribution.py new file mode 100644 index 0000000000..935c274c03 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/attribution.py @@ -0,0 +1,32 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def stub_installer(config, jobs): + """Not all windows builds come with a stub installer (only win32, and not + on esr), so conditionally add it here based on our dependency's + stub-installer attribute.""" + for job in jobs: + dep_name, dep_label = next(iter(job["dependencies"].items())) + dep_task = config.kind_dependencies_tasks[dep_label] + if dep_task.attributes.get("stub-installer"): + locale = job["attributes"].get("locale") + if locale: + artifact = f"{locale}/target.stub-installer.exe" + else: + artifact = "target.stub-installer.exe" + job["fetches"][dep_name].append(artifact) + job["run"]["command"] += [ + "--input", + "/builds/worker/fetches/target.stub-installer.exe", + ] + job["attributes"]["release_artifacts"].append( + "public/build/target.stub-installer.exe" + ) + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/balrog_submit.py b/taskcluster/gecko_taskgraph/transforms/balrog_submit.py new file mode 100644 index 0000000000..067600f0cb --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/balrog_submit.py @@ -0,0 +1,138 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the per-locale balrog task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from taskgraph.util.treeherder import replace_group +from voluptuous import Optional + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job + +balrog_description_schema = schema.extend( + { + # unique label to describe this balrog task, defaults to balrog-{dep.label} + Optional("label"): str, + Optional( + "update-no-wnp", + description="Whether the parallel `-No-WNP` blob should be updated as well.", + ): optionally_keyed_by("release-type", bool), + # treeherder is allowed here to override any defaults we use for beetmover. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + Optional("attributes"): task_description_schema["attributes"], + # Shipping product / phase + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + } +) + + +transforms = TransformSequence() +transforms.add_validate(balrog_description_schema) + + +@transforms.add +def handle_keyed_by(config, jobs): + """Resolve fields that can be keyed by platform, etc.""" + fields = [ + "update-no-wnp", + ] + for job in jobs: + label = job.get("dependent-task", object).__dict__.get("label", "?no-label?") + for field in fields: + resolve_keyed_by( + item=job, + field=field, + item_name=label, + **{ + "project": config.params["project"], + "release-type": config.params["release_type"], + }, + ) + yield job + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + + treeherder = job.get("treeherder", {}) + treeherder.setdefault("symbol", "c-Up(N)") + dep_th_platform = ( + dep_job.task.get("extra", {}) + .get("treeherder", {}) + .get("machine", {}) + .get("platform", "") + ) + treeherder.setdefault("platform", f"{dep_th_platform}/opt") + treeherder.setdefault( + "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1) + ) + treeherder.setdefault("kind", "build") + + attributes = copy_attributes_from_dependent_job(dep_job) + + treeherder_job_symbol = dep_job.task["extra"]["treeherder"]["symbol"] + treeherder["symbol"] = replace_group(treeherder_job_symbol, "c-Up") + + if dep_job.attributes.get("locale"): + attributes["locale"] = dep_job.attributes.get("locale") + + label = job["label"] + + description = ( + "Balrog submission for locale '{locale}' for build '" + "{build_platform}/{build_type}'".format( + locale=attributes.get("locale", "en-US"), + build_platform=attributes.get("build_platform"), + build_type=attributes.get("build_type"), + ) + ) + + upstream_artifacts = [ + { + "taskId": {"task-reference": "<beetmover>"}, + "taskType": "beetmover", + "paths": ["public/manifest.json"], + } + ] + + dependencies = {"beetmover": dep_job.label} + for kind_dep in config.kind_dependencies_tasks.values(): + if ( + kind_dep.kind == "startup-test" + and kind_dep.attributes["build_platform"] + == attributes.get("build_platform") + and kind_dep.attributes["build_type"] == attributes.get("build_type") + and kind_dep.attributes.get("shipping_product") + == job.get("shipping-product") + ): + dependencies["startup-test"] = kind_dep.label + + task = { + "label": label, + "description": description, + "worker-type": "balrog", + "worker": { + "implementation": "balrog", + "upstream-artifacts": upstream_artifacts, + "balrog-action": "v2-submit-locale", + "suffixes": ["", "-No-WNP"] if job.get("update-no-wnp") else [""], + }, + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "treeherder": treeherder, + "shipping-phase": job.get("shipping-phase", "promote"), + "shipping-product": job.get("shipping-product"), + } + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/balrog_toplevel.py b/taskcluster/gecko_taskgraph/transforms/balrog_toplevel.py new file mode 100644 index 0000000000..6b06758f69 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/balrog_toplevel.py @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +from mozilla_version.gecko import GeckoVersion +from mozrelease.balrog import generate_update_properties +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.yaml import load_yaml + +from gecko_taskgraph.util.scriptworker import get_release_config + +transforms = TransformSequence() + + +@transforms.add +def generate_update_line(config, jobs): + """Resolve fields that can be keyed by platform, etc.""" + release_config = get_release_config(config) + for job in jobs: + config_file = job.pop("whats-new-config") + update_config = load_yaml(config_file) + + product = job["shipping-product"] + if product == "devedition": + product = "firefox" + job["worker"]["update-line"] = {} + for blob_type, suffix in [("wnp", ""), ("no-wnp", "-No-WNP")]: + context = { + "release-type": config.params["release_type"], + "product": product, + "version": GeckoVersion.parse(release_config["appVersion"]), + "blob-type": blob_type, + "build-id": config.params["moz_build_date"], + } + job["worker"]["update-line"][suffix] = generate_update_properties( + context, update_config + ) + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover.py b/taskcluster/gecko_taskgraph/transforms/beetmover.py new file mode 100644 index 0000000000..93818707dd --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover.py @@ -0,0 +1,165 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import replace_group +from voluptuous import Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.scriptworker import ( + generate_beetmover_artifact_map, + generate_beetmover_upstream_artifacts, + get_beetmover_action_scope, + get_beetmover_bucket_scope, +) + +transforms = TransformSequence() + +beetmover_description_schema = schema.extend( + { + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Optional("label"): str, + # treeherder is allowed here to override any defaults we use for beetmover. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + # locale is passed only for l10n beetmoving + Optional("locale"): str, + Required("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("attributes"): task_description_schema["attributes"], + } +) + + +transforms.add_validate(beetmover_description_schema) + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = dep_job.attributes + + treeherder = job.get("treeherder", {}) + treeherder.setdefault( + "symbol", replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BM") + ) + dep_th_platform = ( + dep_job.task.get("extra", {}) + .get("treeherder", {}) + .get("machine", {}) + .get("platform", "") + ) + treeherder.setdefault("platform", f"{dep_th_platform}/opt") + treeherder.setdefault( + "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1) + ) + treeherder.setdefault("kind", "build") + label = job["label"] + description = ( + "Beetmover submission for locale '{locale}' for build '" + "{build_platform}/{build_type}'".format( + locale=attributes.get("locale", "en-US"), + build_platform=attributes.get("build_platform"), + build_type=attributes.get("build_type"), + ) + ) + + dependencies = {dep_job.kind: dep_job.label} + + # XXX release snap-repackage has a variable number of dependencies, depending on how many + # "post-beetmover-dummy" jobs there are in the graph. + if dep_job.kind != "release-snap-repackage" and len(dep_job.dependencies) > 1: + raise NotImplementedError( + "Can't beetmove a signing task with multiple dependencies" + ) + signing_dependencies = dep_job.dependencies + dependencies.update(signing_dependencies) + + attributes = copy_attributes_from_dependent_job(dep_job) + attributes.update(job.get("attributes", {})) + + if job.get("locale"): + attributes["locale"] = job["locale"] + + bucket_scope = get_beetmover_bucket_scope(config) + action_scope = get_beetmover_action_scope(config) + + task = { + "label": label, + "description": description, + "worker-type": "beetmover", + "scopes": [bucket_scope, action_scope], + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "treeherder": treeherder, + "shipping-phase": job["shipping-phase"], + } + + yield task + + +def craft_release_properties(config, job): + params = config.params + build_platform = job["attributes"]["build_platform"] + build_platform = build_platform.replace("-shippable", "") + if build_platform.endswith("-source"): + build_platform = build_platform.replace("-source", "-release") + + # XXX This should be explicitly set via build attributes or something + if "android" in job["label"] or "fennec" in job["label"]: + app_name = "Fennec" + elif config.graph_config["trust-domain"] == "comm": + app_name = "Thunderbird" + else: + # XXX Even DevEdition is called Firefox + app_name = "Firefox" + + return { + "app-name": app_name, + "app-version": params["app_version"], + "branch": params["project"], + "build-id": params["moz_build_date"], + "hash-type": "sha512", + "platform": build_platform, + } + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + valid_beetmover_job = len(job["dependencies"]) == 2 and any( + ["signing" in j for j in job["dependencies"]] + ) + # XXX release snap-repackage has a variable number of dependencies, depending on how many + # "post-beetmover-dummy" jobs there are in the graph. + if "-snap-" not in job["label"] and not valid_beetmover_job: + raise NotImplementedError("Beetmover must have two dependencies.") + + locale = job["attributes"].get("locale") + platform = job["attributes"]["build_platform"] + + worker = { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, job), + "upstream-artifacts": generate_beetmover_upstream_artifacts( + config, job, platform, locale + ), + "artifact-map": generate_beetmover_artifact_map( + config, job, platform=platform, locale=locale + ), + } + + if locale: + worker["locale"] = locale + job["worker"] = worker + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_apt.py b/taskcluster/gecko_taskgraph/transforms/beetmover_apt.py new file mode 100644 index 0000000000..8c56f1a968 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_apt.py @@ -0,0 +1,114 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from itertools import islice + +from taskgraph import MAX_DEPENDENCIES +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.platforms import architecture +from gecko_taskgraph.util.scriptworker import ( + generate_artifact_registry_gcs_sources, + get_beetmover_apt_repo_scope, + get_beetmover_repo_action_scope, +) + +transforms = TransformSequence() + + +@transforms.add +def beetmover_apt(config, tasks): + product = ( + "firefox" + if not config.params["release_type"] # try + or config.params["release_type"] == "nightly" + else config.params["release_product"] + ) + filtered_tasks = filter_beetmover_apt_tasks(tasks, product) + # There are too many beetmover-repackage dependencies for a single task + # and we hit the taskgraph dependencies limit. + # To work around this limitation, we chunk the would be task + # into tasks dependendent on, at most, half of MAX_DEPENDENCIES. + batches = batched(filtered_tasks, MAX_DEPENDENCIES // 2) + for index, batch in enumerate(batches): + dependencies = {} + gcs_sources = [] + for task in batch: + dep = task["primary-dependency"] + dependencies[dep.label] = dep.label + gcs_sources.extend(generate_artifact_registry_gcs_sources(dep)) + description = f"Batch {index + 1} of beetmover APT submissions for the {config.params['release_type']} .deb packages" + platform = "firefox-release/opt" + treeherder = { + "platform": platform, + "tier": 1, + "kind": "other", + "symbol": f"BM-apt(batch-{index + 1})", + } + apt_repo_scope = get_beetmover_apt_repo_scope(config) + repo_action_scope = get_beetmover_repo_action_scope(config) + attributes = { + "required_signoffs": ["mar-signing"], + "shippable": True, + "shipping_product": product, + } + task = { + "label": f"{config.kind}-{index + 1}-{platform}", + "description": description, + "worker-type": "beetmover", + "treeherder": treeherder, + "scopes": [apt_repo_scope, repo_action_scope], + "attributes": attributes, + "shipping-phase": "ship", + "shipping-product": product, + "dependencies": dependencies, + } + worker = { + "implementation": "beetmover-import-from-gcs-to-artifact-registry", + "product": product, + "gcs-sources": gcs_sources, + } + task["worker"] = worker + yield task + + +def batched(iterable, n): + "Batch data into tuples of length n. The last batch may be shorter." + # batched('ABCDEFG', 3) --> ABC DEF G + if n < 1: + raise ValueError("n must be at least one") + it = iter(iterable) + batch = tuple(islice(it, n)) + while batch: + yield batch + batch = tuple(islice(it, n)) + + +def filter_beetmover_apt_tasks(tasks, product): + return (task for task in tasks if filter_beetmover_apt_task(task, product)) + + +def filter_beetmover_apt_task(task, product): + # We only create beetmover-apt tasks for l10n beetmover-repackage tasks that + # beetmove langpack .deb packages. The langpack .deb packages support all + # architectures, so we generate them only on x86_64 tasks. + return ( + is_x86_64_l10n_task(task) or is_not_l10n_task(task) + ) and is_task_for_product(task, product) + + +def is_x86_64_l10n_task(task): + dep = task["primary-dependency"] + locale = dep.attributes.get("locale") + return locale and architecture(dep.attributes["build_platform"]) == "x86_64" + + +def is_not_l10n_task(task): + dep = task["primary-dependency"] + locale = dep.attributes.get("locale") + return not locale + + +def is_task_for_product(task, product): + dep = task["primary-dependency"] + return dep.attributes.get("shipping_product") == product diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_checksums.py b/taskcluster/gecko_taskgraph/transforms/beetmover_checksums.py new file mode 100644 index 0000000000..f595854a80 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_checksums.py @@ -0,0 +1,133 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the checksums signing task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import replace_group +from voluptuous import Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.beetmover import craft_release_properties +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.scriptworker import ( + generate_beetmover_artifact_map, + generate_beetmover_upstream_artifacts, + get_beetmover_action_scope, + get_beetmover_bucket_scope, +) + +beetmover_checksums_description_schema = schema.extend( + { + Required("attributes"): {str: object}, + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("locale"): str, + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + } +) + +transforms = TransformSequence() +transforms.add_validate(beetmover_checksums_description_schema) + + +@transforms.add +def make_beetmover_checksums_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = dep_job.attributes + + treeherder = job.get("treeherder", {}) + treeherder.setdefault( + "symbol", + replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BMcs"), + ) + dep_th_platform = ( + dep_job.task.get("extra", {}) + .get("treeherder", {}) + .get("machine", {}) + .get("platform", "") + ) + treeherder.setdefault("platform", f"{dep_th_platform}/opt") + treeherder.setdefault( + "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1) + ) + treeherder.setdefault("kind", "build") + + label = job["label"] + build_platform = attributes.get("build_platform") + + description = ( + "Beetmover submission of checksums for locale '{locale}' for build '" + "{build_platform}/{build_type}'".format( + locale=attributes.get("locale", "en-US"), + build_platform=build_platform, + build_type=attributes.get("build_type"), + ) + ) + + extra = {} + if "devedition" in build_platform: + extra["product"] = "devedition" + else: + extra["product"] = "firefox" + + dependencies = {dep_job.kind: dep_job.label} + + attributes = copy_attributes_from_dependent_job(dep_job) + attributes.update(job.get("attributes", {})) + + if dep_job.attributes.get("locale"): + treeherder["symbol"] = "BMcs({})".format(dep_job.attributes.get("locale")) + attributes["locale"] = dep_job.attributes.get("locale") + + bucket_scope = get_beetmover_bucket_scope(config) + action_scope = get_beetmover_action_scope(config) + + task = { + "label": label, + "description": description, + "worker-type": "beetmover", + "scopes": [bucket_scope, action_scope], + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "treeherder": treeherder, + "extra": extra, + } + + if "shipping-phase" in job: + task["shipping-phase"] = job["shipping-phase"] + + if "shipping-product" in job: + task["shipping-product"] = job["shipping-product"] + + yield task + + +@transforms.add +def make_beetmover_checksums_worker(config, jobs): + for job in jobs: + locale = job["attributes"].get("locale") + platform = job["attributes"]["build_platform"] + + worker = { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, job), + "upstream-artifacts": generate_beetmover_upstream_artifacts( + config, job, platform, locale + ), + "artifact-map": generate_beetmover_artifact_map( + config, job, platform=platform, locale=locale + ), + } + + if locale: + worker["locale"] = locale + job["worker"] = worker + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_emefree_checksums.py b/taskcluster/gecko_taskgraph/transforms/beetmover_emefree_checksums.py new file mode 100644 index 0000000000..9c2b49b6c5 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_emefree_checksums.py @@ -0,0 +1,139 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform release-beetmover-source-checksums into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from voluptuous import Optional + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.beetmover import craft_release_properties +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job + +beetmover_checksums_description_schema = schema.extend( + { + Optional("label"): str, + Optional("extra"): object, + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + } +) + + +transforms = TransformSequence() +transforms.add_validate(beetmover_checksums_description_schema) + + +@transforms.add +def make_beetmover_checksums_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = dep_job.attributes + build_platform = attributes.get("build_platform") + if not build_platform: + raise Exception("Cannot find build platform!") + repack_id = dep_job.task.get("extra", {}).get("repack_id") + if not repack_id: + raise Exception("Cannot find repack id!") + + label = dep_job.label.replace("beetmover-", "beetmover-checksums-") + description = ( + "Beetmove checksums for repack_id '{repack_id}' for build '" + "{build_platform}/{build_type}'".format( + repack_id=repack_id, + build_platform=build_platform, + build_type=attributes.get("build_type"), + ) + ) + + extra = {} + extra["partner_path"] = dep_job.task["payload"]["upstreamArtifacts"][0][ + "locale" + ] + extra["repack_id"] = repack_id + + dependencies = {dep_job.kind: dep_job.label} + for k, v in dep_job.dependencies.items(): + if k.startswith("beetmover"): + dependencies[k] = v + + attributes = copy_attributes_from_dependent_job(dep_job) + + task = { + "label": label, + "description": description, + "worker-type": "{}/{}".format( + dep_job.task["provisionerId"], + dep_job.task["workerType"], + ), + "scopes": dep_job.task["scopes"], + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "extra": extra, + } + + if "shipping-phase" in job: + task["shipping-phase"] = job["shipping-phase"] + + if "shipping-product" in job: + task["shipping-product"] = job["shipping-product"] + + yield task + + +def generate_upstream_artifacts(refs, partner_path): + # Until bug 1331141 is fixed, if you are adding any new artifacts here that + # need to be transfered to S3, please be aware you also need to follow-up + # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/. + # See example in bug 1348286 + common_paths = [ + "public/target.checksums", + ] + + upstream_artifacts = [ + { + "taskId": {"task-reference": refs["beetmover"]}, + "taskType": "signing", + "paths": common_paths, + "locale": f"beetmover-checksums/{partner_path}", + } + ] + + return upstream_artifacts + + +@transforms.add +def make_beetmover_checksums_worker(config, jobs): + for job in jobs: + valid_beetmover_job = len(job["dependencies"]) == 1 + if not valid_beetmover_job: + raise NotImplementedError("Beetmover checksums must have one dependency.") + + refs = { + "beetmover": None, + } + for dependency in job["dependencies"].keys(): + if dependency.endswith("beetmover"): + refs["beetmover"] = f"<{dependency}>" + if None in refs.values(): + raise NotImplementedError( + "Beetmover checksums must have a beetmover dependency!" + ) + + worker = { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, job), + "upstream-artifacts": generate_upstream_artifacts( + refs, + job["extra"]["partner_path"], + ), + "partner-public": True, + } + + job["worker"] = worker + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_geckoview.py b/taskcluster/gecko_taskgraph/transforms/beetmover_geckoview.py new file mode 100644 index 0000000000..6eee5954e3 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_geckoview.py @@ -0,0 +1,166 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + + +from copy import deepcopy + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from voluptuous import Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.beetmover import ( + craft_release_properties as beetmover_craft_release_properties, +) +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + release_level, +) +from gecko_taskgraph.util.declarative_artifacts import ( + get_geckoview_artifact_id, + get_geckoview_artifact_map, + get_geckoview_upstream_artifacts, +) + +beetmover_description_schema = schema.extend( + { + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Required("run-on-projects"): task_description_schema["run-on-projects"], + Required("run-on-hg-branches"): task_description_schema["run-on-hg-branches"], + Optional("bucket-scope"): optionally_keyed_by("release-level", str), + Optional("shipping-phase"): optionally_keyed_by( + "project", task_description_schema["shipping-phase"] + ), + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("attributes"): task_description_schema["attributes"], + } +) + +transforms = TransformSequence() +transforms.add_validate(beetmover_description_schema) + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "run-on-hg-branches", + item_name=job["label"], + project=config.params["project"], + ) + resolve_keyed_by( + job, + "shipping-phase", + item_name=job["label"], + project=config.params["project"], + ) + resolve_keyed_by( + job, + "bucket-scope", + item_name=job["label"], + **{"release-level": release_level(config.params["project"])}, + ) + yield job + + +@transforms.add +def split_maven_packages(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = copy_attributes_from_dependent_job(dep_job) + for package in attributes["maven_packages"]: + package_job = deepcopy(job) + package_job["maven-package"] = package + yield package_job + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = copy_attributes_from_dependent_job(dep_job) + attributes.update(job.get("attributes", {})) + + treeherder = job.get("treeherder", {}) + dep_th_platform = ( + dep_job.task.get("extra", {}) + .get("treeherder", {}) + .get("machine", {}) + .get("platform", "") + ) + treeherder.setdefault("platform", f"{dep_th_platform}/opt") + treeherder.setdefault("tier", 2) + treeherder.setdefault("kind", "build") + package = job["maven-package"] + treeherder.setdefault("symbol", f"BM-{package}") + label = job["label"] + description = ( + "Beetmover submission for geckoview" + "{build_platform}/{build_type}'".format( + build_platform=attributes.get("build_platform"), + build_type=attributes.get("build_type"), + ) + ) + + dependencies = deepcopy(dep_job.dependencies) + dependencies[dep_job.kind] = dep_job.label + + if job.get("locale"): + attributes["locale"] = job["locale"] + + attributes["run_on_hg_branches"] = job["run-on-hg-branches"] + + task = { + "label": f"{package}-{label}", + "description": description, + "worker-type": "beetmover", + "scopes": [ + job["bucket-scope"], + "project:releng:beetmover:action:push-to-maven", + ], + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": job["run-on-projects"], + "treeherder": treeherder, + "shipping-phase": job["shipping-phase"], + "maven-package": package, + } + + yield task + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + job["worker"] = { + "artifact-map": get_geckoview_artifact_map(config, job), + "implementation": "beetmover-maven", + "release-properties": craft_release_properties(config, job), + "upstream-artifacts": get_geckoview_upstream_artifacts( + config, job, job["maven-package"] + ), + } + del job["maven-package"] + + yield job + + +def craft_release_properties(config, job): + release_properties = beetmover_craft_release_properties(config, job) + + release_properties["artifact-id"] = get_geckoview_artifact_id( + config, + job["attributes"]["build_platform"], + job["maven-package"], + job["attributes"].get("update-channel"), + ) + release_properties["app-name"] = "geckoview" + + return release_properties diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_langpack_checksums.py b/taskcluster/gecko_taskgraph/transforms/beetmover_langpack_checksums.py new file mode 100644 index 0000000000..9318ed29d4 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_langpack_checksums.py @@ -0,0 +1,128 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform release-beetmover-langpack-checksums into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import inherit_treeherder_from_dep +from voluptuous import Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.beetmover import craft_release_properties +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.scriptworker import ( + generate_beetmover_artifact_map, + generate_beetmover_upstream_artifacts, + get_beetmover_action_scope, + get_beetmover_bucket_scope, +) + +beetmover_checksums_description_schema = schema.extend( + { + Required("attributes"): {str: object}, + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("locale"): str, + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + } +) + +transforms = TransformSequence() +transforms.add_validate(beetmover_checksums_description_schema) + + +@transforms.add +def make_beetmover_checksums_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = dep_job.attributes + + treeherder = inherit_treeherder_from_dep(job, dep_job) + treeherder.setdefault( + "symbol", "BMcslang(N{})".format(attributes.get("l10n_chunk", "")) + ) + + label = job["label"] + build_platform = attributes.get("build_platform") + + description = "Beetmover submission of checksums for langpack files" + + extra = {} + if "devedition" in build_platform: + extra["product"] = "devedition" + else: + extra["product"] = "firefox" + + dependencies = {dep_job.kind: dep_job.label} + for k, v in dep_job.dependencies.items(): + if k.startswith("beetmover"): + dependencies[k] = v + + attributes = copy_attributes_from_dependent_job(dep_job) + if "chunk_locales" in dep_job.attributes: + attributes["chunk_locales"] = dep_job.attributes["chunk_locales"] + attributes.update(job.get("attributes", {})) + + bucket_scope = get_beetmover_bucket_scope(config) + action_scope = get_beetmover_action_scope(config) + + task = { + "label": label, + "description": description, + "worker-type": "beetmover", + "scopes": [bucket_scope, action_scope], + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "treeherder": treeherder, + "extra": extra, + } + + if "shipping-phase" in job: + task["shipping-phase"] = job["shipping-phase"] + + if "shipping-product" in job: + task["shipping-product"] = job["shipping-product"] + + yield task + + +@transforms.add +def make_beetmover_checksums_worker(config, jobs): + for job in jobs: + valid_beetmover_job = len(job["dependencies"]) == 1 + if not valid_beetmover_job: + raise NotImplementedError("Beetmover checksums must have one dependency.") + + locales = job["attributes"].get("chunk_locales") + platform = job["attributes"]["build_platform"] + + refs = { + "beetmover": None, + } + for dependency in job["dependencies"].keys(): + if dependency.startswith("release-beetmover"): + refs["beetmover"] = f"<{dependency}>" + if None in refs.values(): + raise NotImplementedError( + "Beetmover checksums must have a beetmover dependency!" + ) + + worker = { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, job), + "upstream-artifacts": generate_beetmover_upstream_artifacts( + config, job, platform, locales + ), + "artifact-map": generate_beetmover_artifact_map( + config, job, platform=platform, locale=locales + ), + } + + job["worker"] = worker + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_push_to_release.py b/taskcluster/gecko_taskgraph/transforms/beetmover_push_to_release.py new file mode 100644 index 0000000000..b6307d93cf --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_push_to_release.py @@ -0,0 +1,93 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover-push-to-release task into a task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema, taskref_or_string +from voluptuous import Optional, Required + +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.scriptworker import ( + add_scope_prefix, + get_beetmover_bucket_scope, +) + +beetmover_push_to_release_description_schema = Schema( + { + Required("name"): str, + Required("product"): str, + Required("treeherder-platform"): str, + Optional("attributes"): {str: object}, + Optional("job-from"): task_description_schema["job-from"], + Optional("run"): {str: object}, + Optional("run-on-projects"): task_description_schema["run-on-projects"], + Optional("dependencies"): {str: taskref_or_string}, + Optional("index"): {str: str}, + Optional("routes"): [str], + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + Optional("extra"): task_description_schema["extra"], + Optional("worker"): { + Optional("max-run-time"): int, + }, + } +) + + +transforms = TransformSequence() +transforms.add_validate(beetmover_push_to_release_description_schema) + + +@transforms.add +def make_beetmover_push_to_release_description(config, jobs): + for job in jobs: + treeherder = job.get("treeherder", {}) + treeherder.setdefault("symbol", "Rel(BM-C)") + treeherder.setdefault("tier", 1) + treeherder.setdefault("kind", "build") + treeherder.setdefault("platform", job["treeherder-platform"]) + + label = job["name"] + description = "Beetmover push to release for '{product}'".format( + product=job["product"] + ) + + bucket_scope = get_beetmover_bucket_scope(config) + action_scope = add_scope_prefix(config, "beetmover:action:push-to-releases") + + task = { + "label": label, + "description": description, + "worker-type": "beetmover", + "scopes": [bucket_scope, action_scope], + "product": job["product"], + "dependencies": job["dependencies"], + "attributes": job.get("attributes", {}), + "run-on-projects": job.get("run-on-projects"), + "treeherder": treeherder, + "shipping-phase": job.get("shipping-phase", "push"), + "shipping-product": job.get("shipping-product"), + "routes": job.get("routes", []), + "extra": job.get("extra", {}), + "worker": job.get("worker", {}), + } + + yield task + + +@transforms.add +def make_beetmover_push_to_release_worker(config, jobs): + for job in jobs: + worker = { + "implementation": "beetmover-push-to-release", + "product": job["product"], + } + if job.get("worker", {}).get("max-run-time"): + worker["max-run-time"] = job["worker"]["max-run-time"] + job["worker"] = worker + del job["product"] + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_repackage.py b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage.py new file mode 100644 index 0000000000..a1ce911b9f --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage.py @@ -0,0 +1,327 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.taskcluster import get_artifact_prefix +from taskgraph.util.treeherder import inherit_treeherder_from_dep, replace_group +from voluptuous import Optional, Required + +from gecko_taskgraph.loader.multi_dep import schema +from gecko_taskgraph.transforms.beetmover import craft_release_properties +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.partials import ( + get_balrog_platform_name, + get_partials_artifacts_from_params, + get_partials_info_from_params, +) +from gecko_taskgraph.util.scriptworker import ( + generate_beetmover_artifact_map, + generate_beetmover_partials_artifact_map, + generate_beetmover_upstream_artifacts, + get_beetmover_action_scope, + get_beetmover_bucket_scope, +) + +logger = logging.getLogger(__name__) + + +beetmover_description_schema = schema.extend( + { + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Required("label"): str, + # treeherder is allowed here to override any defaults we use for beetmover. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + Optional("attributes"): task_description_schema["attributes"], + # locale is passed only for l10n beetmoving + Optional("locale"): str, + Required("shipping-phase"): task_description_schema["shipping-phase"], + # Optional until we fix asan (run_on_projects?) + Optional("shipping-product"): task_description_schema["shipping-product"], + } +) + +transforms = TransformSequence() +transforms.add_validate(beetmover_description_schema) + + +def get_task_by_suffix(tasks, suffix): + """ + Given tasks<dict>, returns the key to the task with provided suffix<str> + Raises exception if more than one task is found + + Args: + tasks (Dict): Map of labels to tasks + suffix (str): Suffix for the desired task + + Returns + str: The key to the desired task + """ + labels = [] + for label in tasks.keys(): + if label.endswith(suffix): + labels.append(label) + if len(labels) > 1: + raise Exception( + f"There should only be a single task with suffix: {suffix} - found {len(labels)}" + ) + return labels[0] + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = dep_job.attributes + + treeherder = inherit_treeherder_from_dep(job, dep_job) + upstream_symbol = dep_job.task["extra"]["treeherder"]["symbol"] + if "build" in job["dependent-tasks"]: + upstream_symbol = job["dependent-tasks"]["build"].task["extra"][ + "treeherder" + ]["symbol"] + treeherder.setdefault("symbol", replace_group(upstream_symbol, "BMR")) + label = job["label"] + description = ( + "Beetmover submission for locale '{locale}' for build '" + "{build_platform}/{build_type}'".format( + locale=attributes.get("locale", "en-US"), + build_platform=attributes.get("build_platform"), + build_type=attributes.get("build_type"), + ) + ) + + upstream_deps = job["dependent-tasks"] + + signing_name = "build-signing" + build_name = "build" + repackage_name = "repackage" + repackage_signing_name = "repackage-signing" + msi_signing_name = "repackage-signing-msi" + msix_signing_name = "repackage-signing-shippable-l10n-msix" + mar_signing_name = "mar-signing" + attribution_name = "attribution" + repackage_deb_name = "repackage-deb" + if job.get("locale"): + signing_name = "shippable-l10n-signing" + build_name = "shippable-l10n" + repackage_name = "repackage-l10n" + repackage_signing_name = "repackage-signing-l10n" + mar_signing_name = "mar-signing-l10n" + attribution_name = "attribution-l10n" + repackage_deb_name = "repackage-deb-l10n" + + # The upstream "signing" task for macosx is either *-mac-signing or *-mac-notarization + if attributes.get("build_platform", "").startswith("macosx"): + # We use the signing task on level 1 and notarization on level 3 + if int(config.params.get("level", 0)) < 3: + signing_name = get_task_by_suffix(upstream_deps, "-mac-signing") + else: + signing_name = get_task_by_suffix(upstream_deps, "-mac-notarization") + if not signing_name: + raise Exception("Could not find upstream kind for mac signing.") + + dependencies = { + "build": upstream_deps[build_name], + "repackage": upstream_deps[repackage_name], + "signing": upstream_deps[signing_name], + "mar-signing": upstream_deps[mar_signing_name], + } + if "partials-signing" in upstream_deps: + dependencies["partials-signing"] = upstream_deps["partials-signing"] + if msi_signing_name in upstream_deps: + dependencies[msi_signing_name] = upstream_deps[msi_signing_name] + if msix_signing_name in upstream_deps: + dependencies[msix_signing_name] = upstream_deps[msix_signing_name] + if repackage_signing_name in upstream_deps: + dependencies["repackage-signing"] = upstream_deps[repackage_signing_name] + if attribution_name in upstream_deps: + dependencies[attribution_name] = upstream_deps[attribution_name] + if repackage_deb_name in upstream_deps: + dependencies[repackage_deb_name] = upstream_deps[repackage_deb_name] + + attributes = copy_attributes_from_dependent_job(dep_job) + attributes.update(job.get("attributes", {})) + if job.get("locale"): + attributes["locale"] = job["locale"] + + bucket_scope = get_beetmover_bucket_scope(config) + action_scope = get_beetmover_action_scope(config) + + task = { + "label": label, + "description": description, + "worker-type": "beetmover", + "scopes": [bucket_scope, action_scope], + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "treeherder": treeherder, + "shipping-phase": job["shipping-phase"], + "shipping-product": job.get("shipping-product"), + } + + yield task + + +def generate_partials_upstream_artifacts(job, artifacts, platform, locale=None): + artifact_prefix = get_artifact_prefix(job) + if locale and locale != "en-US": + artifact_prefix = f"{artifact_prefix}/{locale}" + + upstream_artifacts = [ + { + "taskId": {"task-reference": "<partials-signing>"}, + "taskType": "signing", + "paths": [f"{artifact_prefix}/{path}" for path, _ in artifacts], + "locale": locale or "en-US", + } + ] + + return upstream_artifacts + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + locale = job["attributes"].get("locale") + platform = job["attributes"]["build_platform"] + + worker = { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, job), + "upstream-artifacts": generate_beetmover_upstream_artifacts( + config, job, platform, locale + ), + "artifact-map": generate_beetmover_artifact_map( + config, job, platform=platform, locale=locale + ), + } + + if locale: + worker["locale"] = locale + job["worker"] = worker + + yield job + + +@transforms.add +def strip_unwanted_langpacks_from_worker(config, jobs): + """Strips out langpacks where we didn't sign them. + + This explicitly deletes langpacks from upstream artifacts and from artifact-maps. + Due to limitations in declarative artifacts, doing this was our easiest way right now. + """ + ALWAYS_OK_PLATFORMS = {"linux64-shippable", "linux64-devedition"} + OSX_OK_PLATFORMS = {"macosx64-shippable", "macosx64-devedition"} + for job in jobs: + platform = job["attributes"].get("build_platform") + if platform in ALWAYS_OK_PLATFORMS: + # No need to strip anything + yield job + continue + + for map in job["worker"].get("artifact-map", [])[:]: + if not any([path.endswith("target.langpack.xpi") for path in map["paths"]]): + continue + if map["locale"] == "ja-JP-mac": + # This locale should only exist on mac + assert platform in OSX_OK_PLATFORMS + continue + # map[paths] is being modified while iterating, so we need to resolve the + # ".keys()" iterator up front by throwing it into a list. + for path in list(map["paths"].keys()): + if path.endswith("target.langpack.xpi"): + del map["paths"][path] + if map["paths"] == {}: + job["worker"]["artifact-map"].remove(map) + + for artifact in job["worker"].get("upstream-artifacts", []): + if not any( + [path.endswith("target.langpack.xpi") for path in artifact["paths"]] + ): + continue + if artifact["locale"] == "ja-JP-mac": + # This locale should only exist on mac + assert platform in OSX_OK_PLATFORMS + continue + artifact["paths"] = [ + path + for path in artifact["paths"] + if not path.endswith("target.langpack.xpi") + ] + if artifact["paths"] == []: + job["worker"]["upstream-artifacts"].remove(artifact) + + yield job + + +@transforms.add +def make_partials_artifacts(config, jobs): + for job in jobs: + locale = job["attributes"].get("locale") + if not locale: + locale = "en-US" + + platform = job["attributes"]["build_platform"] + + if "partials-signing" not in job["dependencies"]: + yield job + continue + + balrog_platform = get_balrog_platform_name(platform) + artifacts = get_partials_artifacts_from_params( + config.params.get("release_history"), balrog_platform, locale + ) + + upstream_artifacts = generate_partials_upstream_artifacts( + job, artifacts, balrog_platform, locale + ) + + job["worker"]["upstream-artifacts"].extend(upstream_artifacts) + + extra = list() + + partials_info = get_partials_info_from_params( + config.params.get("release_history"), balrog_platform, locale + ) + + job["worker"]["artifact-map"].extend( + generate_beetmover_partials_artifact_map( + config, job, partials_info, platform=platform, locale=locale + ) + ) + + for artifact in partials_info: + artifact_extra = { + "locale": locale, + "artifact_name": artifact, + "buildid": partials_info[artifact]["buildid"], + "platform": balrog_platform, + } + for rel_attr in ("previousBuildNumber", "previousVersion"): + if partials_info[artifact].get(rel_attr): + artifact_extra[rel_attr] = partials_info[artifact][rel_attr] + extra.append(artifact_extra) + + job.setdefault("extra", {}) + job["extra"]["partials"] = extra + + yield job + + +@transforms.add +def convert_deps(config, jobs): + for job in jobs: + job["dependencies"] = { + name: dep_job.label for name, dep_job in job["dependencies"].items() + } + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_l10n.py b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_l10n.py new file mode 100644 index 0000000000..da2a41ccb3 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_l10n.py @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the signing task into an actual task description. +""" + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import join_symbol + +transforms = TransformSequence() + + +@transforms.add +def make_beetmover_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + + locale = dep_job.attributes.get("locale") + if not locale: + yield job + continue + + group = "BMR" + + # add the locale code + symbol = locale + + treeherder = { + "symbol": join_symbol(group, symbol), + } + + beet_description = { + "label": job["label"], + "primary-dependency": dep_job, + "dependent-tasks": job["dependent-tasks"], + "attributes": job["attributes"], + "treeherder": treeherder, + "locale": locale, + "shipping-phase": job["shipping-phase"], + "shipping-product": job["shipping-product"], + } + yield beet_description diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py new file mode 100644 index 0000000000..40dc370f33 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py @@ -0,0 +1,326 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +import logging +from copy import deepcopy + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from taskgraph.util.taskcluster import get_artifact_prefix +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.beetmover import craft_release_properties +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + release_level, +) +from gecko_taskgraph.util.partners import get_ftp_platform, get_partner_config_by_kind +from gecko_taskgraph.util.scriptworker import ( + add_scope_prefix, + get_beetmover_bucket_scope, +) + +logger = logging.getLogger(__name__) + + +beetmover_description_schema = schema.extend( + { + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Optional("label"): str, + Required("partner-bucket-scope"): optionally_keyed_by("release-level", str), + Required("partner-public-path"): Any(None, str), + Required("partner-private-path"): Any(None, str), + Optional("extra"): object, + Required("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("priority"): task_description_schema["priority"], + } +) + +transforms = TransformSequence() +transforms.add_validate(beetmover_description_schema) + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "partner-bucket-scope", + item_name=job["label"], + **{"release-level": release_level(config.params["project"])}, + ) + yield job + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + repack_id = dep_job.task.get("extra", {}).get("repack_id") + if not repack_id: + raise Exception("Cannot find repack id!") + + attributes = dep_job.attributes + build_platform = attributes.get("build_platform") + if not build_platform: + raise Exception("Cannot find build platform!") + + label = dep_job.label.replace("repackage-signing-l10n", "beetmover-") + label = dep_job.label.replace("repackage-signing-", "beetmover-") + label = label.replace("repackage-", "beetmover-") + label = label.replace("chunking-dummy-", "beetmover-") + description = ( + "Beetmover submission for repack_id '{repack_id}' for build '" + "{build_platform}/{build_type}'".format( + repack_id=repack_id, + build_platform=build_platform, + build_type=attributes.get("build_type"), + ) + ) + + dependencies = {} + + base_label = "release-partner-repack" + if "eme" in config.kind: + base_label = "release-eme-free-repack" + dependencies["build"] = f"{base_label}-{build_platform}" + if "macosx" in build_platform or "win" in build_platform: + dependencies["repackage"] = "{}-repackage-{}-{}".format( + base_label, build_platform, repack_id.replace("/", "-") + ) + dependencies["repackage-signing"] = "{}-repackage-signing-{}-{}".format( + base_label, build_platform, repack_id.replace("/", "-") + ) + + attributes = copy_attributes_from_dependent_job(dep_job) + + task = { + "label": label, + "description": description, + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "shipping-phase": job["shipping-phase"], + "shipping-product": job.get("shipping-product"), + "partner-private-path": job["partner-private-path"], + "partner-public-path": job["partner-public-path"], + "partner-bucket-scope": job["partner-bucket-scope"], + "extra": { + "repack_id": repack_id, + }, + } + # we may have reduced the priority for partner jobs, otherwise task.py will set it + if job.get("priority"): + task["priority"] = job["priority"] + + yield task + + +def populate_scopes_and_worker_type(config, job, bucket_scope, partner_public=False): + action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner") + + task = deepcopy(job) + task["scopes"] = [bucket_scope, action_scope] + task["worker-type"] = "beetmover" + task["partner_public"] = partner_public + if partner_public: + task["label"] = "{}-public".format(task["label"]) + return task + + +@transforms.add +def split_public_and_private(config, jobs): + public_bucket_scope = get_beetmover_bucket_scope(config) + partner_config = get_partner_config_by_kind(config, config.kind) + + for job in jobs: + partner_bucket_scope = add_scope_prefix(config, job["partner-bucket-scope"]) + partner, subpartner, _ = job["extra"]["repack_id"].split("/") + + if partner_config[partner][subpartner].get("upload_to_candidates"): + # public + yield populate_scopes_and_worker_type( + config, job, public_bucket_scope, partner_public=True + ) + else: + # private + yield populate_scopes_and_worker_type( + config, job, partner_bucket_scope, partner_public=False + ) + + +def generate_upstream_artifacts( + job, + build_task_ref, + repackage_task_ref, + repackage_signing_task_ref, + platform, + repack_id, + partner_path, + repack_stub_installer=False, +): + + upstream_artifacts = [] + artifact_prefix = get_artifact_prefix(job) + + if "linux" in platform: + upstream_artifacts.append( + { + "taskId": {"task-reference": build_task_ref}, + "taskType": "build", + "paths": [f"{artifact_prefix}/{repack_id}/target.tar.bz2"], + "locale": partner_path, + } + ) + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.tar.bz2.asc"], + "locale": partner_path, + } + ) + elif "macosx" in platform: + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.dmg"], + "locale": partner_path, + } + ) + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.dmg.asc"], + "locale": partner_path, + } + ) + elif "win" in platform: + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.installer.exe"], + "locale": partner_path, + } + ) + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.installer.exe.asc"], + "locale": partner_path, + } + ) + if platform.startswith("win32") and repack_stub_installer: + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [ + "{}/{}/target.stub-installer.exe".format( + artifact_prefix, repack_id + ) + ], + "locale": partner_path, + } + ) + upstream_artifacts.append( + { + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [ + "{}/{}/target.stub-installer.exe.asc".format( + artifact_prefix, repack_id + ) + ], + "locale": partner_path, + } + ) + + if not upstream_artifacts: + raise Exception("Couldn't find any upstream artifacts.") + + return upstream_artifacts + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + platform = job["attributes"]["build_platform"] + repack_id = job["extra"]["repack_id"] + partner, subpartner, locale = job["extra"]["repack_id"].split("/") + partner_config = get_partner_config_by_kind(config, config.kind) + repack_stub_installer = partner_config[partner][subpartner].get( + "repack_stub_installer" + ) + build_task = None + repackage_task = None + repackage_signing_task = None + + for dependency in job["dependencies"].keys(): + if "repackage-signing" in dependency: + repackage_signing_task = dependency + elif "repackage" in dependency: + repackage_task = dependency + else: + build_task = "build" + + build_task_ref = "<" + str(build_task) + ">" + repackage_task_ref = "<" + str(repackage_task) + ">" + repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">" + + # generate the partner path; we'll send this to beetmover as the "locale" + ftp_platform = get_ftp_platform(platform) + repl_dict = { + "build_number": config.params["build_number"], + "locale": locale, + "partner": partner, + "platform": ftp_platform, + "release_partner_build_number": config.params[ + "release_partner_build_number" + ], + "subpartner": subpartner, + "version": config.params["version"], + } + partner_public = job["partner_public"] + if partner_public: + partner_path_key = "partner-public-path" + else: + partner_path_key = "partner-private-path" + # Kinds can set these to None + if not job[partner_path_key]: + continue + partner_path = job[partner_path_key].format(**repl_dict) + del job["partner_public"] + del job["partner-private-path"] + del job["partner-public-path"] + del job["partner-bucket-scope"] + + worker = { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, job), + "upstream-artifacts": generate_upstream_artifacts( + job, + build_task_ref, + repackage_task_ref, + repackage_signing_task_ref, + platform, + repack_id, + partner_path, + repack_stub_installer, + ), + "partner-public": partner_public, + } + job["worker"] = worker + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_snap.py b/taskcluster/gecko_taskgraph/transforms/beetmover_snap.py new file mode 100644 index 0000000000..40f5132cc1 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_snap.py @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the snap beetmover kind into an actual task description. +""" + + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def leave_snap_repackage_dependencies_only(config, jobs): + for job in jobs: + # XXX: We delete the build dependency because, unlike the other beetmover + # tasks, source doesn't depend on any build task at all. This hack should + # go away when we rewrite beetmover transforms to allow more flexibility in deps + + job["dependencies"] = { + key: value + for key, value in job["dependencies"].items() + if key == "release-snap-repackage" + } + + job["worker"]["upstream-artifacts"] = [ + upstream_artifact + for upstream_artifact in job["worker"]["upstream-artifacts"] + if upstream_artifact["taskId"]["task-reference"] + == "<release-snap-repackage>" + ] + + yield job + + +@transforms.add +def set_custom_treeherder_job_name(config, jobs): + for job in jobs: + job.get("treeherder", {})["symbol"] = "Snap(BM)" + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_source.py b/taskcluster/gecko_taskgraph/transforms/beetmover_source.py new file mode 100644 index 0000000000..573f684a98 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_source.py @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover-source task to also append `build` as dependency +""" + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def remove_build_dependency_in_beetmover_source(config, jobs): + for job in jobs: + # XXX: We delete the build dependency because, unlike the other beetmover + # tasks, source doesn't depend on any build task at all. This hack should + # go away when we rewrite beetmover transforms to allow more flexibility in deps + # Essentially, we should use multi_dep for beetmover. + for depname in job["dependencies"]: + if "signing" not in depname: + del job["dependencies"][depname] + break + else: + raise Exception("Can't find build dep in beetmover source!") + + all_upstream_artifacts = job["worker"]["upstream-artifacts"] + upstream_artifacts_without_build = [ + upstream_artifact + for upstream_artifact in all_upstream_artifacts + if upstream_artifact["taskId"]["task-reference"] != f"<{depname}>" + ] + job["worker"]["upstream-artifacts"] = upstream_artifacts_without_build + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_source_checksums.py b/taskcluster/gecko_taskgraph/transforms/beetmover_source_checksums.py new file mode 100644 index 0000000000..bcaa889903 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_source_checksums.py @@ -0,0 +1,137 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform release-beetmover-source-checksums into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from voluptuous import Optional + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.beetmover import craft_release_properties +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.scriptworker import ( + generate_beetmover_artifact_map, + generate_beetmover_upstream_artifacts, + get_beetmover_action_scope, + get_beetmover_bucket_scope, +) + +beetmover_checksums_description_schema = schema.extend( + { + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("locale"): str, + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("attributes"): task_description_schema["attributes"], + } +) + +transforms = TransformSequence() +transforms.add_validate(beetmover_checksums_description_schema) + + +@transforms.add +def make_beetmover_checksums_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = dep_job.attributes + + treeherder = job.get("treeherder", {}) + treeherder.setdefault("symbol", "BMcss(N)") + dep_th_platform = ( + dep_job.task.get("extra", {}) + .get("treeherder", {}) + .get("machine", {}) + .get("platform", "") + ) + treeherder.setdefault("platform", f"{dep_th_platform}/opt") + treeherder.setdefault("tier", 1) + treeherder.setdefault("kind", "build") + + label = job["label"] + build_platform = attributes.get("build_platform") + + description = "Beetmover submission of checksums for source file" + + extra = {} + if "devedition" in build_platform: + extra["product"] = "devedition" + else: + extra["product"] = "firefox" + + dependencies = {dep_job.kind: dep_job.label} + for k, v in dep_job.dependencies.items(): + if k.startswith("beetmover"): + dependencies[k] = v + + attributes = copy_attributes_from_dependent_job(dep_job) + attributes.update(job.get("attributes", {})) + + bucket_scope = get_beetmover_bucket_scope(config) + action_scope = get_beetmover_action_scope(config) + + task = { + "label": label, + "description": description, + "worker-type": "beetmover", + "scopes": [bucket_scope, action_scope], + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "treeherder": treeherder, + "extra": extra, + } + + if "shipping-phase" in job: + task["shipping-phase"] = job["shipping-phase"] + + if "shipping-product" in job: + task["shipping-product"] = job["shipping-product"] + + yield task + + +@transforms.add +def make_beetmover_checksums_worker(config, jobs): + for job in jobs: + valid_beetmover_job = len(job["dependencies"]) == 2 + if not valid_beetmover_job: + raise NotImplementedError("Beetmover checksums must have two dependencies.") + + locale = job["attributes"].get("locale") + platform = job["attributes"]["build_platform"] + + refs = { + "beetmover": None, + "signing": None, + } + for dependency in job["dependencies"].keys(): + if dependency.startswith("beetmover"): + refs["beetmover"] = f"<{dependency}>" + else: + refs["signing"] = f"<{dependency}>" + if None in refs.values(): + raise NotImplementedError( + "Beetmover checksums must have a beetmover and signing dependency!" + ) + + worker = { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, job), + "upstream-artifacts": generate_beetmover_upstream_artifacts( + config, job, platform, locale + ), + "artifact-map": generate_beetmover_artifact_map( + config, job, platform=platform + ), + } + + if locale: + worker["locale"] = locale + job["worker"] = worker + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/bootstrap.py b/taskcluster/gecko_taskgraph/transforms/bootstrap.py new file mode 100644 index 0000000000..e4537cab01 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/bootstrap.py @@ -0,0 +1,132 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema +from voluptuous import Any, Optional, Required + +transforms = TransformSequence() + +bootstrap_schema = Schema( + { + # Name of the bootstrap task. + Required("name"): str, + # Name of the docker image. Ideally, we'd also have tasks for mac and windows, + # but we unfortunately don't have workers barebones enough for such testing + # to be satisfactory. + Required("image"): Any(str, {"in-tree": str}), + # Initialization commands. + Required("pre-commands"): [str], + # relative path (from config.path) to the file task was defined in + Optional("job-from"): str, + } +) + + +transforms.add_validate(bootstrap_schema) + + +@transforms.add +def bootstrap_tasks(config, tasks): + for task in tasks: + name = task.pop("name") + image = task.pop("image") + pre_commands = task.pop("pre-commands") + + head_repo = config.params["head_repository"] + head_rev = config.params["head_rev"] + + # Get all the non macos/windows local toolchains (the only ones bootstrap can use), + # and use them as dependencies for the tasks we create, so that they don't start + # before any potential toolchain task that would be triggered on the same push + # (which would lead to bootstrap failing). + dependencies = { + name: name + for name, task in config.kind_dependencies_tasks.items() + if task.attributes.get("local-toolchain") + and not name.startswith(("toolchain-macos", "toolchain-win")) + } + # We don't test the artifacts variants, or js, because they are essentially subsets. + # Mobile and browser are different enough to warrant testing them separately. + for app in ("browser", "mobile_android"): + commands = pre_commands + [ + # MOZ_AUTOMATION changes the behavior, and we want something closer to user + # machines. + "unset MOZ_AUTOMATION", + f"curl -O {head_repo}/raw-file/{head_rev}/python/mozboot/bin/bootstrap.py", + f"python3 bootstrap.py --no-interactive --application-choice {app}", + "cd mozilla-unified", + # After bootstrap, configure should go through without its own auto-bootstrap. + "./mach configure --disable-bootstrap", + # Then a build should go through too. + "./mach build", + ] + + os_specific = [] + if app == "mobile_android": + os_specific += ["android*"] + for os, filename in ( + ("debian", "debian.py"), + ("ubuntu", "debian.py"), + ("fedora", "centosfedora.py"), + ("rockylinux", "centosfedora.py"), + ("opensuse", "opensuse.py"), + ("gentoo", "gentoo.py"), + ("archlinux", "archlinux.py"), + ("voidlinux", "void.py"), + ): + if name.startswith(os): + os_specific.append(filename) + break + else: + raise Exception(f"Missing OS specific bootstrap file for {name}") + + taskdesc = { + "label": f"{config.kind}-{name}-{app}", + "description": f"Bootstrap {app} build on {name}", + "always-target": True, + "scopes": [], + "treeherder": { + "symbol": f"Boot({name})", + "platform": { + "browser": "linux64/opt", + "mobile_android": "android-5-0-armv7/opt", + }[app], + "kind": "other", + "tier": 2, + }, + "run-on-projects": ["trunk"], + "worker-type": "b-linux-gcp", + "worker": { + "implementation": "docker-worker", + "docker-image": image, + "os": "linux", + "env": { + "GECKO_HEAD_REPOSITORY": head_repo, + "GECKO_HEAD_REV": head_rev, + "MACH_NO_TERMINAL_FOOTER": "1", + "MOZ_SCM_LEVEL": config.params["level"], + }, + "command": ["sh", "-c", "-x", "-e", " && ".join(commands)], + "max-run-time": 7200, + }, + "dependencies": dependencies, + "optimization": { + "skip-unless-changed": [ + "python/mozboot/bin/bootstrap.py", + "python/mozboot/mozboot/base.py", + "python/mozboot/mozboot/bootstrap.py", + "python/mozboot/mozboot/linux_common.py", + "python/mozboot/mozboot/mach_commands.py", + "python/mozboot/mozboot/mozconfig.py", + "python/mozboot/mozboot/rust.py", + "python/mozboot/mozboot/sccache.py", + "python/mozboot/mozboot/util.py", + ] + + [f"python/mozboot/mozboot/{f}" for f in os_specific] + }, + } + + yield taskdesc diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_aliases.py b/taskcluster/gecko_taskgraph/transforms/bouncer_aliases.py new file mode 100644 index 0000000000..38f1aa136a --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/bouncer_aliases.py @@ -0,0 +1,108 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Add from parameters.yml into bouncer submission tasks. +""" + + +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.transforms.bouncer_submission import craft_bouncer_product_name +from gecko_taskgraph.transforms.bouncer_submission_partners import ( + craft_partner_bouncer_product_name, +) +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.partners import get_partners_to_be_published +from gecko_taskgraph.util.scriptworker import get_release_config + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "worker-type", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])}, + ) + resolve_keyed_by( + job, + "scopes", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])}, + ) + resolve_keyed_by( + job, + "bouncer-products-per-alias", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + if "partner-bouncer-products-per-alias" in job: + resolve_keyed_by( + job, + "partner-bouncer-products-per-alias", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + + job["worker"]["entries"] = craft_bouncer_entries(config, job) + + del job["bouncer-products-per-alias"] + if "partner-bouncer-products-per-alias" in job: + del job["partner-bouncer-products-per-alias"] + + if job["worker"]["entries"]: + yield job + else: + logger.warn( + 'No bouncer entries defined in bouncer submission task for "{}". \ +Job deleted.'.format( + job["name"] + ) + ) + + +def craft_bouncer_entries(config, job): + release_config = get_release_config(config) + + product = job["shipping-product"] + current_version = release_config["version"] + bouncer_products_per_alias = job["bouncer-products-per-alias"] + + entries = { + bouncer_alias: craft_bouncer_product_name( + product, + bouncer_product, + current_version, + ) + for bouncer_alias, bouncer_product in bouncer_products_per_alias.items() + } + + partner_bouncer_products_per_alias = job.get("partner-bouncer-products-per-alias") + if partner_bouncer_products_per_alias: + partners = get_partners_to_be_published(config) + for partner, sub_config_name, _ in partners: + entries.update( + { + bouncer_alias.replace( + "PARTNER", f"{partner}-{sub_config_name}" + ): craft_partner_bouncer_product_name( + product, + bouncer_product, + current_version, + partner, + sub_config_name, + ) + for bouncer_alias, bouncer_product in partner_bouncer_products_per_alias.items() # NOQA: E501 + } + ) + + return entries diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_check.py b/taskcluster/gecko_taskgraph/transforms/bouncer_check.py new file mode 100644 index 0000000000..9261ef9463 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/bouncer_check.py @@ -0,0 +1,111 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import logging +from pipes import quote as shell_quote + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import get_release_config + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def add_command(config, jobs): + for job in jobs: + command = [ + "python", + "testing/mozharness/scripts/release/bouncer_check.py", + ] + job["run"].update( + { + "using": "mach", + "mach": command, + } + ) + yield job + + +@transforms.add +def add_previous_versions(config, jobs): + release_config = get_release_config(config) + if not release_config.get("partial_versions"): + for job in jobs: + yield job + else: + extra_params = [] + for partial in release_config["partial_versions"].split(","): + extra_params.append( + "--previous-version={}".format(partial.split("build")[0].strip()) + ) + + for job in jobs: + job["run"]["mach"].extend(extra_params) + yield job + + +@transforms.add +def handle_keyed_by(config, jobs): + """Resolve fields that can be keyed by project, etc.""" + fields = [ + "run.config", + "run.product-field", + "run.extra-config", + ] + + release_config = get_release_config(config) + version = release_config["version"] + + for job in jobs: + for field in fields: + resolve_keyed_by( + item=job, + field=field, + item_name=job["name"], + **{ + "project": config.params["project"], + "release-level": release_level(config.params["project"]), + "release-type": config.params["release_type"], + }, + ) + + for cfg in job["run"]["config"]: + job["run"]["mach"].extend(["--config", cfg]) + + if config.kind == "cron-bouncer-check": + job["run"]["mach"].extend( + [ + "--product-field={}".format(job["run"]["product-field"]), + "--products-url={}".format(job["run"]["products-url"]), + ] + ) + del job["run"]["product-field"] + del job["run"]["products-url"] + elif config.kind == "release-bouncer-check": + job["run"]["mach"].append(f"--version={version}") + + del job["run"]["config"] + + if "extra-config" in job["run"]: + env = job["worker"].setdefault("env", {}) + env["EXTRA_MOZHARNESS_CONFIG"] = json.dumps( + job["run"]["extra-config"], sort_keys=True + ) + del job["run"]["extra-config"] + + yield job + + +@transforms.add +def command_to_string(config, jobs): + """Convert command to string to make it work properly with run-task""" + for job in jobs: + job["run"]["mach"] = " ".join(map(shell_quote, job["run"]["mach"])) + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_locations.py b/taskcluster/gecko_taskgraph/transforms/bouncer_locations.py new file mode 100644 index 0000000000..e755b73c27 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/bouncer_locations.py @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +logger = logging.getLogger(__name__) + + +transforms = TransformSequence() + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + resolve_keyed_by( + job, "worker-type", item_name=job["name"], project=config.params["project"] + ) + resolve_keyed_by( + job, "scopes", item_name=job["name"], project=config.params["project"] + ) + resolve_keyed_by( + job, + "bouncer-products", + item_name=job["name"], + project=config.params["project"], + ) + + job["worker"]["bouncer-products"] = job["bouncer-products"] + + del job["bouncer-products"] + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py new file mode 100644 index 0000000000..d6320a9312 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py @@ -0,0 +1,335 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Add from parameters.yml into bouncer submission tasks. +""" + + +import copy +import logging + +import attr +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.transforms.l10n import parse_locales_file +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import get_release_config + +logger = logging.getLogger(__name__) + + +FTP_PLATFORMS_PER_BOUNCER_PLATFORM = { + "linux": "linux-i686", + "linux64": "linux-x86_64", + "osx": "mac", + "win": "win32", + "win64": "win64", + "win64-aarch64": "win64-aarch64", +} + +# :lang is interpolated by bouncer at runtime +CANDIDATES_PATH_TEMPLATE = "/{ftp_product}/candidates/{version}-candidates/build{build_number}/\ +{update_folder}{ftp_platform}/:lang/{file}" +RELEASES_PATH_TEMPLATE = "/{ftp_product}/releases/{version}/\ +{update_folder}{ftp_platform}/:lang/{file}" + + +CONFIG_PER_BOUNCER_PRODUCT = { + "complete-mar": { + "name_postfix": "-Complete", + "path_template": RELEASES_PATH_TEMPLATE, + "file_names": { + "default": "{product}-{version}.complete.mar", + }, + }, + "complete-mar-candidates": { + "name_postfix": "build{build_number}-Complete", + "path_template": CANDIDATES_PATH_TEMPLATE, + "file_names": { + "default": "{product}-{version}.complete.mar", + }, + }, + "installer": { + "path_template": RELEASES_PATH_TEMPLATE, + "file_names": { + "linux": "{product}-{version}.tar.bz2", + "linux64": "{product}-{version}.tar.bz2", + "osx": "{pretty_product}%20{version}.dmg", + "win": "{pretty_product}%20Setup%20{version}.exe", + "win64": "{pretty_product}%20Setup%20{version}.exe", + "win64-aarch64": "{pretty_product}%20Setup%20{version}.exe", + }, + }, + "partial-mar": { + "name_postfix": "-Partial-{previous_version}", + "path_template": RELEASES_PATH_TEMPLATE, + "file_names": { + "default": "{product}-{previous_version}-{version}.partial.mar", + }, + }, + "partial-mar-candidates": { + "name_postfix": "build{build_number}-Partial-{previous_version}build{previous_build}", + "path_template": CANDIDATES_PATH_TEMPLATE, + "file_names": { + "default": "{product}-{previous_version}-{version}.partial.mar", + }, + }, + "stub-installer": { + "name_postfix": "-stub", + # We currently have a sole win32 stub installer that is to be used + # in all windows platforms to toggle between full installers + "path_template": RELEASES_PATH_TEMPLATE.replace("{ftp_platform}", "win32"), + "file_names": { + "win": "{pretty_product}%20Installer.exe", + "win64": "{pretty_product}%20Installer.exe", + "win64-aarch64": "{pretty_product}%20Installer.exe", + }, + }, + "msi": { + "name_postfix": "-msi-SSL", + "path_template": RELEASES_PATH_TEMPLATE, + "file_names": { + "win": "{pretty_product}%20Setup%20{version}.msi", + "win64": "{pretty_product}%20Setup%20{version}.msi", + }, + }, + "msix": { + "name_postfix": "-msix-SSL", + "path_template": RELEASES_PATH_TEMPLATE.replace(":lang", "multi"), + "file_names": { + "win": "{pretty_product}%20Setup%20{version}.msix", + "win64": "{pretty_product}%20Setup%20{version}.msix", + }, + }, + "pkg": { + "name_postfix": "-pkg-SSL", + "path_template": RELEASES_PATH_TEMPLATE, + "file_names": { + "osx": "{pretty_product}%20{version}.pkg", + }, + }, + "langpack": { + "name_postfix": "-langpack-SSL", + "path_template": RELEASES_PATH_TEMPLATE.replace(":lang", "xpi"), + "file_names": {"default": ":lang.xpi"}, + }, +} +CONFIG_PER_BOUNCER_PRODUCT["installer-ssl"] = copy.deepcopy( + CONFIG_PER_BOUNCER_PRODUCT["installer"] +) +CONFIG_PER_BOUNCER_PRODUCT["installer-ssl"]["name_postfix"] = "-SSL" + +transforms = TransformSequence() + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "worker-type", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])} + ) + resolve_keyed_by( + job, + "scopes", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])} + ) + resolve_keyed_by( + job, + "bouncer-products", + item_name=job["name"], + **{"release-type": config.params["release_type"]} + ) + + # No need to filter out ja-JP-mac, we need to upload both; but we do + # need to filter out the platforms they come with + all_locales = sorted( + locale + for locale in parse_locales_file(job["locales-file"]).keys() + if locale not in ("linux", "win32", "osx") + ) + + job["worker"]["locales"] = all_locales + job["worker"]["entries"] = craft_bouncer_entries(config, job) + + del job["locales-file"] + del job["bouncer-platforms"] + del job["bouncer-products"] + + if job["worker"]["entries"]: + yield job + else: + logger.warn( + 'No bouncer entries defined in bouncer submission task for "{}". \ +Job deleted.'.format( + job["name"] + ) + ) + + +def craft_bouncer_entries(config, job): + release_config = get_release_config(config) + + product = job["shipping-product"] + bouncer_platforms = job["bouncer-platforms"] + + current_version = release_config["version"] + current_build_number = release_config["build_number"] + + bouncer_products = job["bouncer-products"] + previous_versions_string = release_config.get("partial_versions", None) + if previous_versions_string: + previous_versions = previous_versions_string.split(", ") + else: + logger.warn( + 'No partials defined! Bouncer submission task won\'t send any \ +partial-related entry for "{}"'.format( + job["name"] + ) + ) + bouncer_products = [ + bouncer_product + for bouncer_product in bouncer_products + if "partial" not in bouncer_product + ] + previous_versions = [None] + + project = config.params["project"] + + return { + craft_bouncer_product_name( + product, + bouncer_product, + current_version, + current_build_number, + previous_version, + ): { + "options": { + "add_locales": False if "msix" in bouncer_product else True, + "ssl_only": craft_ssl_only(bouncer_product, project), + }, + "paths_per_bouncer_platform": craft_paths_per_bouncer_platform( + product, + bouncer_product, + bouncer_platforms, + current_version, + current_build_number, + previous_version, + ), + } + for bouncer_product in bouncer_products + for previous_version in previous_versions + } + + +def craft_paths_per_bouncer_platform( + product, + bouncer_product, + bouncer_platforms, + current_version, + current_build_number, + previous_version=None, +): + paths_per_bouncer_platform = {} + for bouncer_platform in bouncer_platforms: + file_names_per_platform = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product][ + "file_names" + ] + file_name_template = file_names_per_platform.get( + bouncer_platform, file_names_per_platform.get("default", None) + ) + if not file_name_template: + # Some bouncer product like stub-installer are only meant to be on Windows. + # Thus no default value is defined there + continue + + file_name_product = _craft_filename_product(product) + file_name = file_name_template.format( + product=file_name_product, + pretty_product=file_name_product.capitalize(), + version=current_version, + previous_version=split_build_data(previous_version)[0], + ) + + path_template = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]["path_template"] + file_relative_location = path_template.format( + ftp_product=_craft_ftp_product(product), + version=current_version, + build_number=current_build_number, + update_folder="update/" if "-mar" in bouncer_product else "", + ftp_platform=FTP_PLATFORMS_PER_BOUNCER_PLATFORM[bouncer_platform], + file=file_name, + ) + + paths_per_bouncer_platform[bouncer_platform] = file_relative_location + + return paths_per_bouncer_platform + + +def _craft_ftp_product(product): + return product.lower() + + +def _craft_filename_product(product): + return "firefox" if product == "devedition" else product + + +@attr.s +class InvalidSubstitution: + error = attr.ib(type=str) + + def __str__(self): + raise Exception("Partial is being processed, but no previous version defined.") + + +def craft_bouncer_product_name( + product, + bouncer_product, + current_version, + current_build_number=None, + previous_version=None, +): + if previous_version is None: + previous_version = previous_build = InvalidSubstitution( + "Partial is being processed, but no previous version defined." + ) + else: + previous_version, previous_build = split_build_data(previous_version) + postfix = ( + CONFIG_PER_BOUNCER_PRODUCT[bouncer_product] + .get("name_postfix", "") + .format( + build_number=current_build_number, + previous_version=previous_version, + previous_build=previous_build, + ) + ) + + return "{product}-{version}{postfix}".format( + product=product.capitalize(), version=current_version, postfix=postfix + ) + + +def craft_ssl_only(bouncer_product, project): + # XXX ESR is the only channel where we force serve the installer over SSL + if "-esr" in project and bouncer_product == "installer": + return True + + return bouncer_product not in ( + "complete-mar", + "complete-mar-candidates", + "installer", + "partial-mar", + "partial-mar-candidates", + ) + + +def split_build_data(version): + if version and "build" in version: + return version.split("build") + return version, InvalidSubstitution("k") diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_submission_partners.py b/taskcluster/gecko_taskgraph/transforms/bouncer_submission_partners.py new file mode 100644 index 0000000000..0f298cb120 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/bouncer_submission_partners.py @@ -0,0 +1,193 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Add from parameters.yml into bouncer submission tasks. +""" + + +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.transforms.bouncer_submission import ( + CONFIG_PER_BOUNCER_PRODUCT as CONFIG_PER_BOUNCER_PRODUCT_VANILLA, +) +from gecko_taskgraph.transforms.bouncer_submission import ( + FTP_PLATFORMS_PER_BOUNCER_PLATFORM, + _craft_filename_product, + _craft_ftp_product, +) +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.partners import ( + check_if_partners_enabled, + get_partners_to_be_published, +) +from gecko_taskgraph.util.scriptworker import get_release_config + +logger = logging.getLogger(__name__) + + +PARTNER_PLATFORMS_TO_BOUNCER = { + "linux-shippable": "linux", + "linux64-shippable": "linux64", + "macosx64-shippable": "osx", + "win32-shippable": "win", + "win64-shippable": "win64", + "win64-aarch64-shippable": "win64-aarch64", +} + +# :lang is interpolated by bouncer at runtime +RELEASES_PARTNERS_PATH_TEMPLATE = "/{ftp_product}/releases/partners/{partner}/{sub_config}/\ +{version}/{ftp_platform}/:lang/{file}" + +CONFIG_PER_BOUNCER_PRODUCT = { + "installer": { + "name_postfix": "-{partner}-{sub_config}", + "path_template": RELEASES_PARTNERS_PATH_TEMPLATE, + "file_names": CONFIG_PER_BOUNCER_PRODUCT_VANILLA["installer"]["file_names"], + }, + "stub-installer": { + "name_postfix": "-{partner}-{sub_config}-stub", + # We currently have a sole win32 stub installer that is to be used + # in all windows platforms to toggle between full installers + "path_template": RELEASES_PARTNERS_PATH_TEMPLATE.replace( + "{ftp_platform}", "win32" + ), + "file_names": CONFIG_PER_BOUNCER_PRODUCT_VANILLA["stub-installer"][ + "file_names" + ], + }, +} + +transforms = TransformSequence() +transforms.add(check_if_partners_enabled) + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "worker-type", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])} + ) + resolve_keyed_by( + job, + "scopes", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])} + ) + resolve_keyed_by( + job, + "bouncer-products", + item_name=job["name"], + **{"release-type": config.params["release_type"]} + ) + + # the schema requires at least one locale but this will not be used + job["worker"]["locales"] = ["fake"] + job["worker"]["entries"] = craft_bouncer_entries(config, job) + + del job["locales-file"] + del job["bouncer-platforms"] + del job["bouncer-products"] + + if job["worker"]["entries"]: + yield job + + +def craft_bouncer_entries(config, job): + release_config = get_release_config(config) + + product = job["shipping-product"] + current_version = release_config["version"] + bouncer_products = job["bouncer-products"] + + partners = get_partners_to_be_published(config) + entries = {} + for partner, sub_config_name, platforms in partners: + platforms = [PARTNER_PLATFORMS_TO_BOUNCER[p] for p in platforms] + entries.update( + { + craft_partner_bouncer_product_name( + product, bouncer_product, current_version, partner, sub_config_name + ): { + "options": { + "add_locales": False, # partners may use different sets of locales + "ssl_only": craft_ssl_only(bouncer_product), + }, + "paths_per_bouncer_platform": craft_paths_per_bouncer_platform( + product, + bouncer_product, + platforms, + current_version, + partner, + sub_config_name, + ), + } + for bouncer_product in bouncer_products + } + ) + return entries + + +def craft_paths_per_bouncer_platform( + product, bouncer_product, bouncer_platforms, current_version, partner, sub_config +): + paths_per_bouncer_platform = {} + for bouncer_platform in bouncer_platforms: + file_names_per_platform = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product][ + "file_names" + ] + file_name_template = file_names_per_platform.get( + bouncer_platform, file_names_per_platform.get("default", None) + ) + if not file_name_template: + # Some bouncer product like stub-installer are only meant to be on Windows. + # Thus no default value is defined there + continue + + file_name_product = _craft_filename_product(product) + file_name = file_name_template.format( + product=file_name_product, + pretty_product=file_name_product.capitalize(), + version=current_version, + ) + + path_template = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]["path_template"] + file_relative_location = path_template.format( + ftp_product=_craft_ftp_product(product), + version=current_version, + ftp_platform=FTP_PLATFORMS_PER_BOUNCER_PLATFORM[bouncer_platform], + partner=partner, + sub_config=sub_config, + file=file_name, + ) + + paths_per_bouncer_platform[bouncer_platform] = file_relative_location + + return paths_per_bouncer_platform + + +def craft_partner_bouncer_product_name( + product, bouncer_product, current_version, partner, sub_config +): + postfix = ( + CONFIG_PER_BOUNCER_PRODUCT[bouncer_product] + .get("name_postfix", "") + .format( + partner=partner, + sub_config=sub_config, + ) + ) + + return "{product}-{version}{postfix}".format( + product=product.capitalize(), version=current_version, postfix=postfix + ) + + +def craft_ssl_only(bouncer_product): + return bouncer_product == "stub-installer" diff --git a/taskcluster/gecko_taskgraph/transforms/build.py b/taskcluster/gecko_taskgraph/transforms/build.py new file mode 100644 index 0000000000..94a4d71b0b --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/build.py @@ -0,0 +1,238 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the build +kind. +""" +import logging + +from mozbuild.artifact_builds import JOB_CHOICES as ARTIFACT_JOBS +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by +from taskgraph.util.treeherder import add_suffix + +from gecko_taskgraph.util.attributes import RELEASE_PROJECTS, is_try, release_level +from gecko_taskgraph.util.workertypes import worker_type_implementation + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def set_defaults(config, jobs): + """Set defaults, including those that differ per worker implementation""" + for job in jobs: + job["treeherder"].setdefault("kind", "build") + job["treeherder"].setdefault("tier", 1) + _, worker_os = worker_type_implementation( + config.graph_config, config.params, job["worker-type"] + ) + worker = job.setdefault("worker", {}) + worker.setdefault("env", {}) + worker["chain-of-trust"] = True + if worker_os == "linux": + worker.setdefault("docker-image", {"in-tree": "debian11-amd64-build"}) + + yield job + + +@transforms.add +def stub_installer(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "stub-installer", + item_name=job["name"], + project=config.params["project"], + **{ + "release-type": config.params["release_type"], + }, + ) + job.setdefault("attributes", {}) + if job.get("stub-installer"): + job["attributes"]["stub-installer"] = job["stub-installer"] + job["worker"]["env"].update({"USE_STUB_INSTALLER": "1"}) + if "stub-installer" in job: + del job["stub-installer"] + yield job + + +@transforms.add +def resolve_shipping_product(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "shipping-product", + item_name=job["name"], + **{ + "release-type": config.params["release_type"], + }, + ) + yield job + + +@transforms.add +def update_channel(config, jobs): + keys = [ + "run.update-channel", + "run.mar-channel-id", + "run.accepted-mar-channel-ids", + ] + for job in jobs: + job["worker"].setdefault("env", {}) + for key in keys: + resolve_keyed_by( + job, + key, + item_name=job["name"], + **{ + "project": config.params["project"], + "release-type": config.params["release_type"], + }, + ) + update_channel = job["run"].pop("update-channel", None) + if update_channel: + job["run"].setdefault("extra-config", {})["update_channel"] = update_channel + job["attributes"]["update-channel"] = update_channel + mar_channel_id = job["run"].pop("mar-channel-id", None) + if mar_channel_id: + job["attributes"]["mar-channel-id"] = mar_channel_id + job["worker"]["env"]["MAR_CHANNEL_ID"] = mar_channel_id + accepted_mar_channel_ids = job["run"].pop("accepted-mar-channel-ids", None) + if accepted_mar_channel_ids: + job["attributes"]["accepted-mar-channel-ids"] = accepted_mar_channel_ids + job["worker"]["env"]["ACCEPTED_MAR_CHANNEL_IDS"] = accepted_mar_channel_ids + + yield job + + +@transforms.add +def mozconfig(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "run.mozconfig-variant", + item_name=job["name"], + **{ + "release-type": config.params["release_type"], + }, + ) + mozconfig_variant = job["run"].pop("mozconfig-variant", None) + if mozconfig_variant: + job["run"].setdefault("extra-config", {})[ + "mozconfig_variant" + ] = mozconfig_variant + yield job + + +@transforms.add +def use_artifact(config, jobs): + if is_try(config.params): + use_artifact = config.params["try_task_config"].get( + "use-artifact-builds", False + ) + else: + use_artifact = False + for job in jobs: + if ( + config.kind == "build" + and use_artifact + and job.get("index", {}).get("job-name") in ARTIFACT_JOBS + # If tests aren't packaged, then we are not able to rebuild all the packages + and job["worker"]["env"].get("MOZ_AUTOMATION_PACKAGE_TESTS") == "1" + ): + job["treeherder"]["symbol"] = add_suffix(job["treeherder"]["symbol"], "a") + job["worker"]["env"]["USE_ARTIFACT"] = "1" + job["attributes"]["artifact-build"] = True + yield job + + +@transforms.add +def use_profile_data(config, jobs): + for job in jobs: + use_pgo = job.pop("use-pgo", False) + disable_pgo = config.params["try_task_config"].get("disable-pgo", False) + artifact_build = job["attributes"].get("artifact-build") + if not use_pgo or disable_pgo or artifact_build: + yield job + continue + + # If use_pgo is True, the task uses the generate-profile task of the + # same name. Otherwise a task can specify a specific generate-profile + # task to use in the use_pgo field. + if use_pgo is True: + name = job["name"] + else: + name = use_pgo + dependencies = f"generate-profile-{name}" + job.setdefault("dependencies", {})["generate-profile"] = dependencies + job.setdefault("fetches", {})["generate-profile"] = ["profdata.tar.xz"] + job["worker"]["env"].update({"TASKCLUSTER_PGO_PROFILE_USE": "1"}) + + _, worker_os = worker_type_implementation( + config.graph_config, config.params, job["worker-type"] + ) + if worker_os == "linux": + # LTO linkage needs more open files than the default from run-task. + job["worker"]["env"].update({"MOZ_LIMIT_NOFILE": "8192"}) + + if job.get("use-sccache"): + raise Exception( + "use-sccache is incompatible with use-pgo in {}".format(job["name"]) + ) + + yield job + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "use-sccache", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])}, + ) + yield job + + +@transforms.add +def enable_full_crashsymbols(config, jobs): + """Enable full crashsymbols on jobs with + 'enable-full-crashsymbols' set to True and on release branches, or + on try""" + branches = RELEASE_PROJECTS | { + "toolchains", + "try", + } + for job in jobs: + enable_full_crashsymbols = job["attributes"].get("enable-full-crashsymbols") + if enable_full_crashsymbols and config.params["project"] in branches: + logger.debug("Enabling full symbol generation for %s", job["name"]) + job["worker"]["env"]["MOZ_ENABLE_FULL_SYMBOLS"] = "1" + else: + logger.debug("Disabling full symbol generation for %s", job["name"]) + job["attributes"].pop("enable-full-crashsymbols", None) + yield job + + +@transforms.add +def set_expiry(config, jobs): + for job in jobs: + attributes = job["attributes"] + if ( + "shippable" in attributes + and attributes["shippable"] + and config.kind + in { + "build", + } + ): + expiration_policy = "long" + else: + expiration_policy = "medium" + + job["expiration-policy"] = expiration_policy + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/build_attrs.py b/taskcluster/gecko_taskgraph/transforms/build_attrs.py new file mode 100644 index 0000000000..9cda71718a --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/build_attrs.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.platforms import platform_family + +transforms = TransformSequence() + + +@transforms.add +def set_build_attributes(config, jobs): + """ + Set the build_platform and build_type attributes based on the job name. + Although not all jobs using this transform are actual "builds", the try + option syntax treats them as such, and this arranges the attributes + appropriately for that purpose. + """ + for job in jobs: + build_platform, build_type = job["name"].split("/") + + # pgo builds are represented as a different platform, type opt + if build_type == "pgo": + build_platform = build_platform + "-pgo" + build_type = "opt" + + attributes = job.setdefault("attributes", {}) + attributes.update( + { + "build_platform": build_platform, + "build_type": build_type, + } + ) + + yield job + + +@transforms.add +def set_schedules_optimization(config, jobs): + """Set the `skip-unless-affected` optimization based on the build platform.""" + for job in jobs: + # don't add skip-unless-schedules if there's already a when defined + if "when" in job: + yield job + continue + + build_platform = job["attributes"]["build_platform"] + job.setdefault("optimization", {"build": [platform_family(build_platform)]}) + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/build_fat_aar.py b/taskcluster/gecko_taskgraph/transforms/build_fat_aar.py new file mode 100644 index 0000000000..61df2111d2 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/build_fat_aar.py @@ -0,0 +1,78 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import copy + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.taskcluster import get_artifact_prefix + +from gecko_taskgraph.util.declarative_artifacts import get_geckoview_upstream_artifacts + +transforms = TransformSequence() + + +MOZ_ANDROID_FAT_AAR_ENV_MAP = { + "android-arm-shippable": "MOZ_ANDROID_FAT_AAR_ARMEABI_V7A", + "android-arm-shippable-lite": "MOZ_ANDROID_FAT_AAR_ARMEABI_V7A", + "android-aarch64-shippable": "MOZ_ANDROID_FAT_AAR_ARM64_V8A", + "android-aarch64-shippable-lite": "MOZ_ANDROID_FAT_AAR_ARM64_V8A", + "android-x86-shippable": "MOZ_ANDROID_FAT_AAR_X86", + "android-x86-shippable-lite": "MOZ_ANDROID_FAT_AAR_X86", + "android-x86_64-shippable": "MOZ_ANDROID_FAT_AAR_X86_64", + "android-x86_64-shippable-lite": "MOZ_ANDROID_FAT_AAR_X86_64", + "android-arm-opt": "MOZ_ANDROID_FAT_AAR_ARMEABI_V7A", + "android-aarch64-opt": "MOZ_ANDROID_FAT_AAR_ARM64_V8A", + "android-x86-opt": "MOZ_ANDROID_FAT_AAR_X86", + "android-x86_64-opt": "MOZ_ANDROID_FAT_AAR_X86_64", +} + + +@transforms.add +def set_fetches_and_locations(config, jobs): + """Set defaults, including those that differ per worker implementation""" + for job in jobs: + dependencies = copy.deepcopy(job["dependencies"]) + + for platform, label in dependencies.items(): + job["dependencies"] = {"build": label} + + aar_location = _get_aar_location(config, job, platform) + prefix = get_artifact_prefix(job) + if not prefix.endswith("/"): + prefix = prefix + "/" + if aar_location.startswith(prefix): + aar_location = aar_location[len(prefix) :] + + job.setdefault("fetches", {}).setdefault(platform, []).append( + { + "artifact": aar_location, + "extract": False, + } + ) + + aar_file_name = aar_location.split("/")[-1] + env_var = MOZ_ANDROID_FAT_AAR_ENV_MAP[platform] + job["worker"]["env"][env_var] = aar_file_name + + job["dependencies"] = dependencies + + yield job + + +def _get_aar_location(config, job, platform): + artifacts_locations = [] + + for package in job["attributes"]["maven_packages"]: + artifacts_locations += get_geckoview_upstream_artifacts( + config, job, package, platform=platform + ) + + aar_locations = [ + path for path in artifacts_locations[0]["paths"] if path.endswith(".aar") + ] + if len(aar_locations) != 1: + raise ValueError(f"Only a single AAR must be given. Got: {aar_locations}") + + return aar_locations[0] diff --git a/taskcluster/gecko_taskgraph/transforms/build_lints.py b/taskcluster/gecko_taskgraph/transforms/build_lints.py new file mode 100644 index 0000000000..d1bd276059 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/build_lints.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the build +kind. +""" + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def check_mozharness_perfherder_options(config, jobs): + """Verify that multiple jobs don't use the same perfherder bucket. + + Build jobs record perfherder metrics by default. Perfherder metrics go + to a bucket derived by the platform by default. The name can further be + customized by the presence of "extra options" either defined in + mozharness sub-configs or in an environment variable. + + This linter tries to verify that no 2 jobs will send Perfherder metrics + to the same bucket by looking for jobs not defining extra options when + their platform or mozharness config are otherwise similar. + """ + + SEEN_CONFIGS = {} + + for job in jobs: + if job["run"]["using"] != "mozharness": + yield job + continue + + worker = job.get("worker", {}) + + platform = job["treeherder"]["platform"] + primary_config = job["run"]["config"][0] + options = worker.get("env", {}).get("PERFHERDER_EXTRA_OPTIONS") + shippable = job.get("attributes", {}).get("shippable", False) + + # This isn't strictly necessary. But the Perfherder code looking at the + # values we care about is only active on builds. So it doesn't make + # sense to run this linter elsewhere. + assert primary_config.startswith("builds/") + + key = (platform, primary_config, shippable, options) + + if key in SEEN_CONFIGS: + raise Exception( + "Non-unique Perfherder data collection for jobs %s-%s and %s: " + "set PERFHERDER_EXTRA_OPTIONS in worker environment variables " + "or use different mozconfigs" + % (config.kind, job["name"], SEEN_CONFIGS[key]) + ) + + SEEN_CONFIGS[key] = "{}-{}".format(config.kind, job["name"]) + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/build_signing.py b/taskcluster/gecko_taskgraph/transforms/build_signing.py new file mode 100644 index 0000000000..36f39f0b56 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/build_signing.py @@ -0,0 +1,71 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the signing task into an actual task description. +""" + + +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.signed_artifacts import ( + generate_specifications_of_artifacts_to_sign, +) + +transforms = TransformSequence() + + +@transforms.add +def add_signed_routes(config, jobs): + """Add routes corresponding to the routes of the build task + this corresponds to, with .signed inserted, for all gecko.v2 routes""" + + for job in jobs: + dep_job = job["primary-dependency"] + enable_signing_routes = job.pop("enable-signing-routes", True) + + job["routes"] = [] + if dep_job.attributes.get("shippable") and enable_signing_routes: + for dep_route in dep_job.task.get("routes", []): + if not dep_route.startswith("index.gecko.v2"): + continue + branch = dep_route.split(".")[3] + rest = ".".join(dep_route.split(".")[4:]) + job["routes"].append(f"index.gecko.v2.{branch}.signed.{rest}") + + yield job + + +@transforms.add +def define_upstream_artifacts(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + upstream_artifact_task = job.pop("upstream-artifact-task", dep_job) + + job["attributes"] = copy_attributes_from_dependent_job(dep_job) + + artifacts_specifications = generate_specifications_of_artifacts_to_sign( + config, + job, + keep_locale_template=False, + kind=config.kind, + dep_kind=upstream_artifact_task.kind, + ) + + task_ref = f"<{upstream_artifact_task.kind}>" + task_type = "build" + if "notarization" in upstream_artifact_task.kind: + task_type = "scriptworker" + + job["upstream-artifacts"] = [ + { + "taskId": {"task-reference": task_ref}, + "taskType": task_type, + "paths": spec["artifacts"], + "formats": spec["formats"], + } + for spec in artifacts_specifications + ] + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/cached_tasks.py b/taskcluster/gecko_taskgraph/transforms/cached_tasks.py new file mode 100644 index 0000000000..bb7e6e6778 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/cached_tasks.py @@ -0,0 +1,101 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from collections import deque + +import taskgraph +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.cached_tasks import add_optimization + +transforms = TransformSequence() + + +def order_tasks(config, tasks): + """Iterate image tasks in an order where parent tasks come first.""" + kind_prefix = config.kind + "-" + + pending = deque(tasks) + task_labels = {task["label"] for task in pending} + emitted = set() + while True: + try: + task = pending.popleft() + except IndexError: + break + parents = { + task + for task in task.get("dependencies", {}).values() + if task.startswith(kind_prefix) + } + if parents and not emitted.issuperset(parents & task_labels): + pending.append(task) + continue + emitted.add(task["label"]) + yield task + + +def format_task_digest(cached_task): + return "/".join( + [ + cached_task["type"], + cached_task["name"], + cached_task["digest"], + ] + ) + + +@transforms.add +def cache_task(config, tasks): + if taskgraph.fast: + for task in tasks: + yield task + return + + digests = {} + for task in config.kind_dependencies_tasks.values(): + if ( + "cached_task" in task.attributes + and task.attributes["cached_task"] is not False + ): + digests[task.label] = format_task_digest(task.attributes["cached_task"]) + + for task in order_tasks(config, tasks): + cache = task.pop("cache", None) + if cache is None: + yield task + continue + + dependency_digests = [] + for p in task.get("dependencies", {}).values(): + if p in digests: + dependency_digests.append(digests[p]) + elif config.params["project"] == "toolchains": + # The toolchains repository uses non-cached toolchain artifacts. Allow + # tasks to use them. + cache = None + break + else: + raise Exception( + "Cached task {} has uncached parent task: {}".format( + task["label"], p + ) + ) + + if cache is None: + yield task + continue + + digest_data = cache["digest-data"] + sorted(dependency_digests) + add_optimization( + config, + task, + cache_type=cache["type"], + cache_name=cache["name"], + digest_data=digest_data, + ) + digests[task["label"]] = format_task_digest(task["attributes"]["cached_task"]) + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/chunk_partners.py b/taskcluster/gecko_taskgraph/transforms/chunk_partners.py new file mode 100644 index 0000000000..ed74cc6232 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/chunk_partners.py @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Chunk the partner repack tasks by subpartner and locale +""" + + +import copy + +from mozbuild.chunkify import chunkify +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.partners import ( + apply_partner_priority, + get_repack_ids_by_platform, +) + +transforms = TransformSequence() +transforms.add(apply_partner_priority) + + +@transforms.add +def chunk_partners(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + build_platform = dep_job.attributes["build_platform"] + repack_id = dep_job.task.get("extra", {}).get("repack_id") + repack_ids = dep_job.task.get("extra", {}).get("repack_ids") + copy_repack_ids = job.pop("copy-repack-ids", False) + + if copy_repack_ids: + assert repack_ids, "dep_job {} doesn't have repack_ids!".format( + dep_job.label + ) + job.setdefault("extra", {})["repack_ids"] = repack_ids + yield job + # first downstream of the repack task, no chunking or fanout has been done yet + elif not any([repack_id, repack_ids]): + platform_repack_ids = get_repack_ids_by_platform(config, build_platform) + # we chunk mac signing + if config.kind in ( + "release-partner-repack-signing", + "release-eme-free-repack-signing", + "release-eme-free-repack-mac-signing", + "release-partner-repack-mac-signing", + ): + repacks_per_chunk = job.get("repacks-per-chunk") + chunks, remainder = divmod(len(platform_repack_ids), repacks_per_chunk) + if remainder: + chunks = int(chunks + 1) + for this_chunk in range(1, chunks + 1): + chunk = chunkify(platform_repack_ids, this_chunk, chunks) + partner_job = copy.deepcopy(job) + partner_job.setdefault("extra", {}).setdefault("repack_ids", chunk) + partner_job["extra"]["repack_suffix"] = str(this_chunk) + yield partner_job + # linux and windows we fan out immediately to one task per partner-sub_partner-locale + else: + for repack_id in platform_repack_ids: + partner_job = copy.deepcopy(job) # don't overwrite dict values here + partner_job.setdefault("extra", {}) + partner_job["extra"]["repack_id"] = repack_id + yield partner_job + # fan out chunked mac signing for repackage + elif repack_ids: + for repack_id in repack_ids: + partner_job = copy.deepcopy(job) + partner_job.setdefault("extra", {}).setdefault("repack_id", repack_id) + yield partner_job + # otherwise we've fully fanned out already, continue by passing repack_id on + else: + partner_job = copy.deepcopy(job) + partner_job.setdefault("extra", {}).setdefault("repack_id", repack_id) + yield partner_job diff --git a/taskcluster/gecko_taskgraph/transforms/code_review.py b/taskcluster/gecko_taskgraph/transforms/code_review.py new file mode 100644 index 0000000000..d644e17d0e --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/code_review.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Add soft dependencies and configuration to code-review tasks. +""" + + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def add_dependencies(config, jobs): + for job in jobs: + job.setdefault("soft-dependencies", []) + job["soft-dependencies"] += [ + dep_task.label + for dep_task in config.kind_dependencies_tasks.values() + if dep_task.attributes.get("code-review") is True + ] + yield job + + +@transforms.add +def add_phabricator_config(config, jobs): + for job in jobs: + diff = config.params.get("phabricator_diff") + if diff is not None: + code_review = job.setdefault("extra", {}).setdefault("code-review", {}) + code_review["phabricator-diff"] = diff + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/condprof.py b/taskcluster/gecko_taskgraph/transforms/condprof.py new file mode 100644 index 0000000000..34cf7e7dd3 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/condprof.py @@ -0,0 +1,85 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +This transform constructs tasks generate conditioned profiles from +the condprof/kind.yml file +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema +from voluptuous import Optional + +from gecko_taskgraph.transforms.job import job_description_schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.copy_task import copy_task + +diff_description_schema = Schema( + { + # default is settled, but add 'full' to get both + Optional("scenarios"): [str], + Optional("description"): task_description_schema["description"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("fetches"): job_description_schema["fetches"], + Optional("index"): task_description_schema["index"], + Optional("job-from"): str, + Optional("name"): str, + Optional("run"): job_description_schema["run"], + Optional("run-on-projects"): task_description_schema["run-on-projects"], + Optional("scopes"): task_description_schema["scopes"], + Optional("treeherder"): task_description_schema["treeherder"], + Optional("worker"): job_description_schema["worker"], + Optional("worker-type"): task_description_schema["worker-type"], + } +) + +transforms = TransformSequence() +transforms.add_validate(diff_description_schema) + + +@transforms.add +def generate_scenarios(config, tasks): + for task in tasks: + cmds = task["run"]["command"] + symbol = task["treeherder"]["symbol"].split(")")[0] + index = task["index"] + jobname = index["job-name"] + label = task["name"] + run_as_root = task["run"].get("run-as-root", False) + + for scenario in set(task["scenarios"]): + extra_args = "" + if scenario == "settled": + extra_args = " --force-new " + + tcmd = cmds.replace("${EXTRA_ARGS}", extra_args) + tcmd = tcmd.replace("${SCENARIO}", scenario) + + index["job-name"] = "%s-%s" % (jobname, scenario) + + taskdesc = { + "name": "%s-%s" % (label, scenario), + "description": task["description"], + "treeherder": { + "symbol": "%s-%s)" % (symbol, scenario), + "platform": task["treeherder"]["platform"], + "kind": task["treeherder"]["kind"], + "tier": task["treeherder"]["tier"], + }, + "worker-type": copy_task(task["worker-type"]), + "worker": copy_task(task["worker"]), + "index": copy_task(index), + "run": { + "using": "run-task", + "cwd": task["run"]["cwd"], + "checkout": task["run"]["checkout"], + "tooltool-downloads": copy_task(task["run"]["tooltool-downloads"]), + "command": tcmd, + "run-as-root": run_as_root, + }, + "run-on-projects": copy_task(task["run-on-projects"]), + "scopes": copy_task(task["scopes"]), + "dependencies": copy_task(task["dependencies"]), + "fetches": copy_task(task["fetches"]), + } + yield taskdesc diff --git a/taskcluster/gecko_taskgraph/transforms/copy_attributes_from_dependent_task.py b/taskcluster/gecko_taskgraph/transforms/copy_attributes_from_dependent_task.py new file mode 100644 index 0000000000..281dd5938f --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/copy_attributes_from_dependent_task.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the repackage task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job + +transforms = TransformSequence() + + +@transforms.add +def copy_attributes(config, jobs): + for job in jobs: + job.setdefault("attributes", {}) + job["attributes"].update( + copy_attributes_from_dependent_job(job["primary-dependency"]) + ) + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/cross_channel.py b/taskcluster/gecko_taskgraph/transforms/cross_channel.py new file mode 100644 index 0000000000..bf6d3a3a4f --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/cross_channel.py @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Build a command to run `mach l10n-cross-channel`. +""" + + +from pipes import quote as shell_quote + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + for item in ["ssh-key-secret", "run.actions"]: + resolve_keyed_by(job, item, item, **{"level": str(config.params["level"])}) + yield job + + +@transforms.add +def build_command(config, jobs): + for job in jobs: + command = [ + "l10n-cross-channel", + "-o", + "/builds/worker/artifacts/outgoing.diff", + "--attempts", + "5", + ] + ssh_key_secret = job.pop("ssh-key-secret") + if ssh_key_secret: + command.extend(["--ssh-secret", ssh_key_secret]) + job.setdefault("scopes", []).append(f"secrets:get:{ssh_key_secret}") + + command.extend(job["run"].pop("actions", [])) + job.setdefault("run", {}).update( + {"using": "mach", "mach": " ".join(map(shell_quote, command))} + ) + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/diffoscope.py b/taskcluster/gecko_taskgraph/transforms/diffoscope.py new file mode 100644 index 0000000000..b74dc5bb8f --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/diffoscope.py @@ -0,0 +1,172 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +This transform construct tasks to perform diffs between builds, as +defined in kind.yml +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema +from taskgraph.util.taskcluster import get_artifact_path +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.transforms.task import task_description_schema + +index_or_string = Any( + str, + {Required("index-search"): str}, +) + +diff_description_schema = Schema( + { + # Name of the diff task. + Required("name"): str, + # Treeherder tier. + Required("tier"): int, + # Treeherder symbol. + Required("symbol"): str, + # relative path (from config.path) to the file the task was defined in. + Optional("job-from"): str, + # Original and new builds to compare. + Required("original"): index_or_string, + Required("new"): index_or_string, + # Arguments to pass to diffoscope, used for job-defaults in + # taskcluster/ci/diffoscope/kind.yml + Optional("args"): str, + # Extra arguments to pass to diffoscope, that can be set per job. + Optional("extra-args"): str, + # Fail the task when differences are detected. + Optional("fail-on-diff"): bool, + # What artifact to check the differences of. Defaults to target.tar.bz2 + # for Linux, target.dmg for Mac, target.zip for Windows, target.apk for + # Android. + Optional("artifact"): str, + # Whether to unpack first. Diffoscope can normally work without unpacking, + # but when one needs to --exclude some contents, that doesn't work out well + # if said content is packed (e.g. in omni.ja). + Optional("unpack"): bool, + # Commands to run before performing the diff. + Optional("pre-diff-commands"): [str], + # Only run the task on a set of projects/branches. + Optional("run-on-projects"): task_description_schema["run-on-projects"], + Optional("optimization"): task_description_schema["optimization"], + } +) + +transforms = TransformSequence() +transforms.add_validate(diff_description_schema) + + +@transforms.add +def fill_template(config, tasks): + dummy_tasks = {} + + for task in tasks: + name = task["name"] + + deps = {} + urls = {} + previous_artifact = None + artifact = task.get("artifact") + for k in ("original", "new"): + value = task[k] + if isinstance(value, str): + deps[k] = value + dep_name = k + os_hint = value + else: + index = value["index-search"] + if index not in dummy_tasks: + dummy_tasks[index] = { + "label": "index-search-" + index, + "description": index, + "worker-type": "invalid/always-optimized", + "run": { + "using": "always-optimized", + }, + "optimization": { + "index-search": [index], + }, + } + yield dummy_tasks[index] + deps[index] = "index-search-" + index + dep_name = index + os_hint = index.split(".")[-1] + if artifact: + pass + elif "linux" in os_hint: + artifact = "target.tar.bz2" + elif "macosx" in os_hint: + artifact = "target.dmg" + elif "android" in os_hint: + artifact = "target.apk" + elif "win" in os_hint: + artifact = "target.zip" + else: + raise Exception(f"Cannot figure out the OS for {value!r}") + if previous_artifact is not None and previous_artifact != artifact: + raise Exception("Cannot compare builds from different OSes") + urls[k] = { + "artifact-reference": "<{}/{}>".format( + dep_name, get_artifact_path(task, artifact) + ), + } + previous_artifact = artifact + + taskdesc = { + "label": "diff-" + name, + "description": name, + "treeherder": { + "symbol": task["symbol"], + "platform": "diff/opt", + "kind": "other", + "tier": task["tier"], + }, + "worker-type": "b-linux-gcp", + "worker": { + "docker-image": {"in-tree": "diffoscope"}, + "artifacts": [ + { + "type": "file", + "path": f"/builds/worker/{f}", + "name": f"public/{f}", + } + for f in ( + "diff.html", + "diff.txt", + ) + ], + "env": { + "ORIG_URL": urls["original"], + "NEW_URL": urls["new"], + "DIFFOSCOPE_ARGS": " ".join( + task[k] for k in ("args", "extra-args") if k in task + ), + "PRE_DIFF": "; ".join(task.get("pre-diff-commands", [])), + }, + "max-run-time": 1800, + }, + "run": { + "using": "run-task", + "checkout": task.get("unpack", False), + "command": "/builds/worker/bin/get_and_diffoscope{}{}".format( + " --unpack" if task.get("unpack") else "", + " --fail" if task.get("fail-on-diff") else "", + ), + }, + "dependencies": deps, + "optimization": task.get("optimization"), + } + if "run-on-projects" in task: + taskdesc["run-on-projects"] = task["run-on-projects"] + + if artifact.endswith(".dmg"): + taskdesc.setdefault("fetches", {}).setdefault("toolchain", []).extend( + [ + "linux64-cctools-port", + "linux64-libdmg", + ] + ) + + yield taskdesc diff --git a/taskcluster/gecko_taskgraph/transforms/docker_image.py b/taskcluster/gecko_taskgraph/transforms/docker_image.py new file mode 100644 index 0000000000..0ccf83bf38 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/docker_image.py @@ -0,0 +1,209 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import logging +import os +import re + +import mozpack.path as mozpath +import taskgraph +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema +from voluptuous import Optional, Required + +from gecko_taskgraph.util.docker import ( + create_context_tar, + generate_context_hash, + image_path, +) + +from .. import GECKO +from .task import task_description_schema + +logger = logging.getLogger(__name__) + +CONTEXTS_DIR = "docker-contexts" + +DIGEST_RE = re.compile("^[0-9a-f]{64}$") + +IMAGE_BUILDER_IMAGE = ( + "mozillareleases/image_builder:5.0.0" + "@sha256:" + "e510a9a9b80385f71c112d61b2f2053da625aff2b6d430411ac42e424c58953f" +) + +transforms = TransformSequence() + +docker_image_schema = Schema( + { + # Name of the docker image. + Required("name"): str, + # Name of the parent docker image. + Optional("parent"): str, + # Treeherder symbol. + Required("symbol"): str, + # relative path (from config.path) to the file the docker image was defined + # in. + Optional("job-from"): str, + # Arguments to use for the Dockerfile. + Optional("args"): {str: str}, + # Name of the docker image definition under taskcluster/docker, when + # different from the docker image name. + Optional("definition"): str, + # List of package tasks this docker image depends on. + Optional("packages"): [str], + Optional( + "index", + description="information for indexing this build so its artifacts can be discovered", + ): task_description_schema["index"], + Optional( + "cache", + description="Whether this image should be cached based on inputs.", + ): bool, + } +) + + +transforms.add_validate(docker_image_schema) + + +@transforms.add +def fill_template(config, tasks): + if not taskgraph.fast and config.write_artifacts: + if not os.path.isdir(CONTEXTS_DIR): + os.makedirs(CONTEXTS_DIR) + + for task in tasks: + image_name = task.pop("name") + job_symbol = task.pop("symbol") + args = task.pop("args", {}) + packages = task.pop("packages", []) + parent = task.pop("parent", None) + + for p in packages: + if f"packages-{p}" not in config.kind_dependencies_tasks: + raise Exception( + "Missing package job for {}-{}: {}".format( + config.kind, image_name, p + ) + ) + + if not taskgraph.fast: + context_path = mozpath.relpath(image_path(image_name), GECKO) + if config.write_artifacts: + context_file = os.path.join(CONTEXTS_DIR, f"{image_name}.tar.gz") + logger.info(f"Writing {context_file} for docker image {image_name}") + context_hash = create_context_tar( + GECKO, context_path, context_file, image_name, args + ) + else: + context_hash = generate_context_hash( + GECKO, context_path, image_name, args + ) + else: + if config.write_artifacts: + raise Exception("Can't write artifacts if `taskgraph.fast` is set.") + context_hash = "0" * 40 + digest_data = [context_hash] + digest_data += [json.dumps(args, sort_keys=True)] + + description = "Build the docker image {} for use by dependent tasks".format( + image_name + ) + + args["DOCKER_IMAGE_PACKAGES"] = " ".join(f"<{p}>" for p in packages) + + # Adjust the zstandard compression level based on the execution level. + # We use faster compression for level 1 because we care more about + # end-to-end times. We use slower/better compression for other levels + # because images are read more often and it is worth the trade-off to + # burn more CPU once to reduce image size. + zstd_level = "3" if int(config.params["level"]) == 1 else "10" + + # include some information that is useful in reconstructing this task + # from JSON + taskdesc = { + "label": f"{config.kind}-{image_name}", + "description": description, + "attributes": { + "image_name": image_name, + "artifact_prefix": "public", + }, + "expiration-policy": "long", + "scopes": [], + "treeherder": { + "symbol": job_symbol, + "platform": "taskcluster-images/opt", + "kind": "other", + "tier": 1, + }, + "run-on-projects": [], + "worker-type": "images-gcp", + "worker": { + "implementation": "docker-worker", + "os": "linux", + "artifacts": [ + { + "type": "file", + "path": "/workspace/image.tar.zst", + "name": "public/image.tar.zst", + } + ], + "env": { + "CONTEXT_TASK_ID": {"task-reference": "<decision>"}, + "CONTEXT_PATH": "public/docker-contexts/{}.tar.gz".format( + image_name + ), + "HASH": context_hash, + "PROJECT": config.params["project"], + "IMAGE_NAME": image_name, + "DOCKER_IMAGE_ZSTD_LEVEL": zstd_level, + "DOCKER_BUILD_ARGS": {"task-reference": json.dumps(args)}, + "GECKO_BASE_REPOSITORY": config.params["base_repository"], + "GECKO_HEAD_REPOSITORY": config.params["head_repository"], + "GECKO_HEAD_REV": config.params["head_rev"], + }, + "chain-of-trust": True, + "max-run-time": 7200, + # FIXME: We aren't currently propagating the exit code + }, + } + # Retry for 'funsize-update-generator' if exit status code is -1 + if image_name in ["funsize-update-generator"]: + taskdesc["worker"]["retry-exit-status"] = [-1] + + worker = taskdesc["worker"] + + if image_name == "image_builder": + worker["docker-image"] = IMAGE_BUILDER_IMAGE + digest_data.append(f"image-builder-image:{IMAGE_BUILDER_IMAGE}") + else: + worker["docker-image"] = {"in-tree": "image_builder"} + deps = taskdesc.setdefault("dependencies", {}) + deps["docker-image"] = f"{config.kind}-image_builder" + + if packages: + deps = taskdesc.setdefault("dependencies", {}) + for p in sorted(packages): + deps[p] = f"packages-{p}" + + if parent: + deps = taskdesc.setdefault("dependencies", {}) + deps["parent"] = f"{config.kind}-{parent}" + worker["env"]["PARENT_TASK_ID"] = { + "task-reference": "<parent>", + } + if "index" in task: + taskdesc["index"] = task["index"] + + if task.get("cache", True) and not taskgraph.fast: + taskdesc["cache"] = { + "type": "docker-images.v2", + "name": image_name, + "digest-data": digest_data, + } + + yield taskdesc diff --git a/taskcluster/gecko_taskgraph/transforms/fetch.py b/taskcluster/gecko_taskgraph/transforms/fetch.py new file mode 100644 index 0000000000..b51362a905 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/fetch.py @@ -0,0 +1,387 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Support for running tasks that download remote content and re-export +# it as task artifacts. + + +import os +import re + +import attr +import taskgraph +from mozbuild.shellutil import quote as shell_quote +from mozpack import path as mozpath +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema, validate_schema +from taskgraph.util.treeherder import join_symbol +from voluptuous import Any, Extra, Optional, Required + +import gecko_taskgraph + +from ..util.cached_tasks import add_optimization + +CACHE_TYPE = "content.v1" + +FETCH_SCHEMA = Schema( + { + # Name of the task. + Required("name"): str, + # Relative path (from config.path) to the file the task was defined + # in. + Optional("job-from"): str, + # Description of the task. + Required("description"): str, + Optional( + "fetch-alias", + description="An alias that can be used instead of the real fetch job name in " + "fetch stanzas for jobs.", + ): str, + Optional( + "artifact-prefix", + description="The prefix of the taskcluster artifact being uploaded. " + "Defaults to `public/`; if it starts with something other than " + "`public/` the artifact will require scopes to access.", + ): str, + Optional("attributes"): {str: object}, + Required("fetch"): { + Required("type"): str, + Extra: object, + }, + } +) + + +# define a collection of payload builders, depending on the worker implementation +fetch_builders = {} + + +@attr.s(frozen=True) +class FetchBuilder: + schema = attr.ib(type=Schema) + builder = attr.ib() + + +def fetch_builder(name, schema): + schema = Schema({Required("type"): name}).extend(schema) + + def wrap(func): + fetch_builders[name] = FetchBuilder(schema, func) + return func + + return wrap + + +transforms = TransformSequence() +transforms.add_validate(FETCH_SCHEMA) + + +@transforms.add +def process_fetch_job(config, jobs): + # Converts fetch-url entries to the job schema. + for job in jobs: + typ = job["fetch"]["type"] + name = job["name"] + fetch = job.pop("fetch") + + if typ not in fetch_builders: + raise Exception(f"Unknown fetch type {typ} in fetch {name}") + validate_schema(fetch_builders[typ].schema, fetch, f"In task.fetch {name!r}:") + + job.update(configure_fetch(config, typ, name, fetch)) + + yield job + + +def configure_fetch(config, typ, name, fetch): + if typ not in fetch_builders: + raise Exception(f"No fetch type {typ} in fetch {name}") + validate_schema(fetch_builders[typ].schema, fetch, f"In task.fetch {name!r}:") + + return fetch_builders[typ].builder(config, name, fetch) + + +@transforms.add +def make_task(config, jobs): + # Fetch tasks are idempotent and immutable. Have them live for + # essentially forever. + if config.params["level"] == "3": + expires = "1000 years" + else: + expires = "28 days" + + for job in jobs: + name = job["name"] + artifact_prefix = job.get("artifact-prefix", "public") + env = job.get("env", {}) + env.update({"UPLOAD_DIR": "/builds/worker/artifacts"}) + attributes = job.get("attributes", {}) + attributes["fetch-artifact"] = mozpath.join( + artifact_prefix, job["artifact_name"] + ) + alias = job.get("fetch-alias") + if alias: + attributes["fetch-alias"] = alias + + task_expires = "28 days" if attributes.get("cached_task") is False else expires + artifact_expires = ( + "2 days" if attributes.get("cached_task") is False else expires + ) + + task = { + "attributes": attributes, + "name": name, + "description": job["description"], + "expires-after": task_expires, + "label": "fetch-%s" % name, + "run-on-projects": [], + "treeherder": { + "symbol": join_symbol("Fetch", name), + "kind": "build", + "platform": "fetch/opt", + "tier": 1, + }, + "run": { + "using": "run-task", + "checkout": False, + "command": job["command"], + }, + "worker-type": "b-linux-gcp", + "worker": { + "chain-of-trust": True, + "docker-image": {"in-tree": "fetch"}, + "env": env, + "max-run-time": 900, + "artifacts": [ + { + "type": "directory", + "name": artifact_prefix, + "path": "/builds/worker/artifacts", + "expires-after": artifact_expires, + } + ], + }, + } + + if job.get("secret", None): + task["scopes"] = ["secrets:get:" + job.get("secret")] + task["worker"]["taskcluster-proxy"] = True + + if not taskgraph.fast: + cache_name = task["label"].replace(f"{config.kind}-", "", 1) + + # This adds the level to the index path automatically. + add_optimization( + config, + task, + cache_type=CACHE_TYPE, + cache_name=cache_name, + digest_data=job["digest_data"], + ) + yield task + + +@fetch_builder( + "static-url", + schema={ + # The URL to download. + Required("url"): str, + # The SHA-256 of the downloaded content. + Required("sha256"): str, + # Size of the downloaded entity, in bytes. + Required("size"): int, + # GPG signature verification. + Optional("gpg-signature"): { + # URL where GPG signature document can be obtained. Can contain the + # value ``{url}``, which will be substituted with the value from + # ``url``. + Required("sig-url"): str, + # Path to file containing GPG public key(s) used to validate + # download. + Required("key-path"): str, + }, + # The name to give to the generated artifact. Defaults to the file + # portion of the URL. Using a different extension converts the + # archive to the given type. Only conversion to .tar.zst is + # supported. + Optional("artifact-name"): str, + # Strip the given number of path components at the beginning of + # each file entry in the archive. + # Requires an artifact-name ending with .tar.zst. + Optional("strip-components"): int, + # Add the given prefix to each file entry in the archive. + # Requires an artifact-name ending with .tar.zst. + Optional("add-prefix"): str, + # IMPORTANT: when adding anything that changes the behavior of the task, + # it is important to update the digest data used to compute cache hits. + }, +) +def create_fetch_url_task(config, name, fetch): + artifact_name = fetch.get("artifact-name") + if not artifact_name: + artifact_name = fetch["url"].split("/")[-1] + + command = [ + "/builds/worker/bin/fetch-content", + "static-url", + ] + + # Arguments that matter to the cache digest + args = [ + "--sha256", + fetch["sha256"], + "--size", + "%d" % fetch["size"], + ] + + if fetch.get("strip-components"): + args.extend(["--strip-components", "%d" % fetch["strip-components"]]) + + if fetch.get("add-prefix"): + args.extend(["--add-prefix", fetch["add-prefix"]]) + + command.extend(args) + + env = {} + + if "gpg-signature" in fetch: + sig_url = fetch["gpg-signature"]["sig-url"].format(url=fetch["url"]) + key_path = os.path.join( + gecko_taskgraph.GECKO, fetch["gpg-signature"]["key-path"] + ) + + with open(key_path, "r") as fh: + gpg_key = fh.read() + + env["FETCH_GPG_KEY"] = gpg_key + command.extend( + [ + "--gpg-sig-url", + sig_url, + "--gpg-key-env", + "FETCH_GPG_KEY", + ] + ) + + command.extend( + [ + fetch["url"], + "/builds/worker/artifacts/%s" % artifact_name, + ] + ) + + return { + "command": command, + "artifact_name": artifact_name, + "env": env, + # We don't include the GPG signature in the digest because it isn't + # materially important for caching: GPG signatures are supplemental + # trust checking beyond what the shasum already provides. + "digest_data": args + [artifact_name], + } + + +@fetch_builder( + "git", + schema={ + Required("repo"): str, + Required(Any("revision", "branch")): str, + Optional("include-dot-git"): bool, + Optional("artifact-name"): str, + Optional("path-prefix"): str, + # ssh-key is a taskcluster secret path (e.g. project/civet/github-deploy-key) + # In the secret dictionary, the key should be specified as + # "ssh_privkey": "-----BEGIN OPENSSH PRIVATE KEY-----\nkfksnb3jc..." + # n.b. The OpenSSH private key file format requires a newline at the end of the file. + Optional("ssh-key"): str, + }, +) +def create_git_fetch_task(config, name, fetch): + path_prefix = fetch.get("path-prefix") + if not path_prefix: + path_prefix = fetch["repo"].rstrip("/").rsplit("/", 1)[-1] + artifact_name = fetch.get("artifact-name") + if not artifact_name: + artifact_name = f"{path_prefix}.tar.zst" + + if "revision" in fetch and "branch" in fetch: + raise Exception("revision and branch cannot be used in the same context") + + revision_or_branch = None + + if "revision" in fetch: + revision_or_branch = fetch["revision"] + if not re.match(r"[0-9a-fA-F]{40}", fetch["revision"]): + raise Exception(f'Revision is not a sha1 in fetch task "{name}"') + else: + # we are sure we are dealing with a branch + revision_or_branch = fetch["branch"] + + args = [ + "/builds/worker/bin/fetch-content", + "git-checkout-archive", + "--path-prefix", + path_prefix, + fetch["repo"], + revision_or_branch, + "/builds/worker/artifacts/%s" % artifact_name, + ] + + ssh_key = fetch.get("ssh-key") + if ssh_key: + args.append("--ssh-key-secret") + args.append(ssh_key) + + digest_data = [revision_or_branch, path_prefix, artifact_name] + if fetch.get("include-dot-git", False): + args.append("--include-dot-git") + digest_data.append(".git") + + return { + "command": args, + "artifact_name": artifact_name, + "digest_data": digest_data, + "secret": ssh_key, + } + + +@fetch_builder( + "chromium-fetch", + schema={ + Required("script"): str, + # Platform type for chromium build + Required("platform"): str, + # Chromium revision to obtain + Optional("revision"): str, + # The name to give to the generated artifact. + Required("artifact-name"): str, + }, +) +def create_chromium_fetch_task(config, name, fetch): + artifact_name = fetch.get("artifact-name") + + workdir = "/builds/worker" + + platform = fetch.get("platform") + revision = fetch.get("revision") + + args = "--platform " + shell_quote(platform) + if revision: + args += " --revision " + shell_quote(revision) + + cmd = [ + "bash", + "-c", + "cd {} && " "/usr/bin/python3 {} {}".format(workdir, fetch["script"], args), + ] + + return { + "command": cmd, + "artifact_name": artifact_name, + "digest_data": [ + f"revision={revision}", + f"platform={platform}", + f"artifact_name={artifact_name}", + ], + } diff --git a/taskcluster/gecko_taskgraph/transforms/final_verify.py b/taskcluster/gecko_taskgraph/transforms/final_verify.py new file mode 100644 index 0000000000..aa8be35a0d --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/final_verify.py @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def add_command(config, tasks): + for task in tasks: + if not task["worker"].get("env"): + task["worker"]["env"] = {} + + final_verify_configs = [] + for upstream in sorted(task.get("dependencies", {}).keys()): + if "update-verify-config" in upstream: + final_verify_configs.append( + f"<{upstream}/public/build/update-verify.cfg>", + ) + task["run"] = { + "using": "run-task", + "cwd": "{checkout}", + "command": { + "artifact-reference": "tools/update-verify/release/final-verification.sh " + + " ".join(final_verify_configs), + }, + "sparse-profile": "update-verify", + } + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/fxrecord.py b/taskcluster/gecko_taskgraph/transforms/fxrecord.py new file mode 100644 index 0000000000..0d8a23bfb4 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/fxrecord.py @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def fxrecord(config, jobs): + for job in jobs: + dep_job = job.pop("primary-dependency", None) + + if dep_job is not None: + job["dependencies"] = {dep_job.label: dep_job.label} + job["treeherder"]["platform"] = dep_job.task["extra"]["treeherder-platform"] + job["worker"].setdefault("env", {})["FXRECORD_TASK_ID"] = { + "task-reference": f"<{dep_job.label}>" + } + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py b/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py new file mode 100644 index 0000000000..1a996c1fc4 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py @@ -0,0 +1,68 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the repackage signing task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from voluptuous import Optional + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.scriptworker import add_scope_prefix + +repackage_signing_description_schema = schema.extend( + { + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("worker"): task_description_schema["worker"], + Optional("worker-type"): task_description_schema["worker-type"], + } +) + +transforms = TransformSequence() +transforms.add_validate(repackage_signing_description_schema) + + +@transforms.add +def geckodriver_mac_notarization(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + + attributes = copy_attributes_from_dependent_job(dep_job) + treeherder = job.get("treeherder", {}) + dep_treeherder = dep_job.task.get("extra", {}).get("treeherder", {}) + treeherder.setdefault( + "platform", dep_job.task.get("extra", {}).get("treeherder-platform") + ) + treeherder.setdefault("tier", dep_treeherder.get("tier", 1)) + treeherder.setdefault("kind", "build") + + dependencies = {dep_job.kind: dep_job.label} + + description = "Mac notarization - Geckodriver for build '{}'".format( + attributes.get("build_platform"), + ) + + build_platform = dep_job.attributes.get("build_platform") + + scopes = [add_scope_prefix(config, "signing:cert:release-apple-notarization")] + + platform = build_platform.rsplit("-", 1)[0] + + task = { + "label": job["label"], + "description": description, + "worker-type": job["worker-type"], + "worker": job["worker"], + "scopes": scopes, + "dependencies": dependencies, + "attributes": attributes, + "treeherder": treeherder, + "run-on-projects": ["mozilla-central"], + "index": {"product": "geckodriver", "job-name": f"{platform}-notarized"}, + } + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py b/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py new file mode 100644 index 0000000000..05bc1319de --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the repackage signing task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from voluptuous import Optional + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.scriptworker import get_signing_cert_scope_per_platform + +repackage_signing_description_schema = schema.extend( + { + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + } +) + +transforms = TransformSequence() +transforms.add_validate(repackage_signing_description_schema) + + +@transforms.add +def make_signing_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + + attributes = copy_attributes_from_dependent_job(dep_job) + attributes["repackage_type"] = "repackage-signing" + + treeherder = job.get("treeherder", {}) + dep_treeherder = dep_job.task.get("extra", {}).get("treeherder", {}) + treeherder.setdefault( + "symbol", "{}(gd-s)".format(dep_treeherder["groupSymbol"]) + ) + treeherder.setdefault( + "platform", dep_job.task.get("extra", {}).get("treeherder-platform") + ) + treeherder.setdefault("tier", dep_treeherder.get("tier", 1)) + treeherder.setdefault("kind", "build") + + dependencies = {dep_job.kind: dep_job.label} + signing_dependencies = dep_job.dependencies + dependencies.update( + {k: v for k, v in signing_dependencies.items() if k != "docker-image"} + ) + + description = "Signing Geckodriver for build '{}'".format( + attributes.get("build_platform"), + ) + + build_platform = dep_job.attributes.get("build_platform") + is_shippable = dep_job.attributes.get("shippable") + signing_cert_scope = get_signing_cert_scope_per_platform( + build_platform, is_shippable, config + ) + + upstream_artifacts = _craft_upstream_artifacts( + dep_job, dep_job.kind, build_platform + ) + + scopes = [signing_cert_scope] + + platform = build_platform.rsplit("-", 1)[0] + + task = { + "label": job["label"], + "description": description, + "worker-type": "linux-signing", + "worker": { + "implementation": "scriptworker-signing", + "upstream-artifacts": upstream_artifacts, + }, + "scopes": scopes, + "dependencies": dependencies, + "attributes": attributes, + "treeherder": treeherder, + "run-on-projects": ["mozilla-central"], + "index": {"product": "geckodriver", "job-name": platform}, + } + + if build_platform.startswith("macosx"): + worker_type = task["worker-type"] + worker_type_alias_map = { + "linux-depsigning": "mac-depsigning", + "linux-signing": "mac-signing", + } + + assert worker_type in worker_type_alias_map, ( + "Make sure to adjust the below worker_type_alias logic for " + "mac if you change the signing workerType aliases!" + " ({} not found in mapping)".format(worker_type) + ) + worker_type = worker_type_alias_map[worker_type] + + task["worker-type"] = worker_type_alias_map[task["worker-type"]] + task["worker"]["mac-behavior"] = "mac_geckodriver" + + yield task + + +def _craft_upstream_artifacts(dep_job, dependency_kind, build_platform): + if build_platform.startswith("win"): + signing_format = "autograph_authenticode_sha2" + elif build_platform.startswith("linux"): + signing_format = "autograph_gpg" + elif build_platform.startswith("macosx"): + signing_format = "mac_geckodriver" + else: + raise ValueError(f'Unsupported build platform "{build_platform}"') + + return [ + { + "taskId": {"task-reference": f"<{dependency_kind}>"}, + "taskType": "build", + "paths": [dep_job.attributes["toolchain-artifact"]], + "formats": [signing_format], + } + ] diff --git a/taskcluster/gecko_taskgraph/transforms/github_sync.py b/taskcluster/gecko_taskgraph/transforms/github_sync.py new file mode 100644 index 0000000000..6f48f794ce --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/github_sync.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def sync_github(config, tasks): + """Do transforms specific to github-sync tasks.""" + for task in tasks: + # Add the secret to the scopes, only in m-c. + # Doing this on any other tree will result in decision task failure + # because m-c is the only one allowed to have that scope. + secret = task["secret"] + if config.params["project"] == "mozilla-central": + task.setdefault("scopes", []) + task["scopes"].append("secrets:get:" + secret) + task["worker"].setdefault("env", {})["GITHUB_SECRET"] = secret + del task["secret"] + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/job/__init__.py b/taskcluster/gecko_taskgraph/transforms/job/__init__.py new file mode 100644 index 0000000000..9b6924b605 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/job/__init__.py @@ -0,0 +1,504 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Convert a job description into a task description. + +Jobs descriptions are similar to task descriptions, but they specify how to run +the job at a higher level, using a "run" field that can be interpreted by +run-using handlers in `taskcluster/gecko_taskgraph/transforms/job`. +""" + + +import json +import logging + +import mozpack.path as mozpath +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.python_path import import_sibling_modules +from taskgraph.util.schema import Schema, validate_schema +from taskgraph.util.taskcluster import get_artifact_prefix +from voluptuous import Any, Exclusive, Extra, Optional, Required + +from gecko_taskgraph.transforms.cached_tasks import order_tasks +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.copy_task import copy_task +from gecko_taskgraph.util.workertypes import worker_type_implementation + +logger = logging.getLogger(__name__) + +# Schema for a build description +job_description_schema = Schema( + { + # The name of the job and the job's label. At least one must be specified, + # and the label will be generated from the name if necessary, by prepending + # the kind. + Optional("name"): str, + Optional("label"): str, + # the following fields are passed directly through to the task description, + # possibly modified by the run implementation. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details. + Required("description"): task_description_schema["description"], + Optional("attributes"): task_description_schema["attributes"], + Optional("job-from"): task_description_schema["job-from"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("if-dependencies"): task_description_schema["if-dependencies"], + Optional("soft-dependencies"): task_description_schema["soft-dependencies"], + Optional("if-dependencies"): task_description_schema["if-dependencies"], + Optional("requires"): task_description_schema["requires"], + Optional("expires-after"): task_description_schema["expires-after"], + Optional("expiration-policy"): task_description_schema["expiration-policy"], + Optional("routes"): task_description_schema["routes"], + Optional("scopes"): task_description_schema["scopes"], + Optional("tags"): task_description_schema["tags"], + Optional("extra"): task_description_schema["extra"], + Optional("treeherder"): task_description_schema["treeherder"], + Optional("index"): task_description_schema["index"], + Optional("run-on-projects"): task_description_schema["run-on-projects"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("always-target"): task_description_schema["always-target"], + Exclusive("optimization", "optimization"): task_description_schema[ + "optimization" + ], + Optional("use-sccache"): task_description_schema["use-sccache"], + Optional("use-system-python"): bool, + Optional("priority"): task_description_schema["priority"], + # The "when" section contains descriptions of the circumstances under which + # this task should be included in the task graph. This will be converted + # into an optimization, so it cannot be specified in a job description that + # also gives 'optimization'. + Exclusive("when", "optimization"): Any( + None, + { + # This task only needs to be run if a file matching one of the given + # patterns has changed in the push. The patterns use the mozpack + # match function (python/mozbuild/mozpack/path.py). + Optional("files-changed"): [str], + }, + ), + # A list of artifacts to install from 'fetch' tasks. + Optional("fetches"): { + str: [ + str, + { + Required("artifact"): str, + Optional("dest"): str, + Optional("extract"): bool, + Optional("verify-hash"): bool, + }, + ], + }, + # A description of how to run this job. + "run": { + # The key to a job implementation in a peer module to this one + "using": str, + # Base work directory used to set up the task. + Optional("workdir"): str, + # Any remaining content is verified against that job implementation's + # own schema. + Extra: object, + }, + Required("worker-type"): task_description_schema["worker-type"], + # This object will be passed through to the task description, with additions + # provided by the job's run-using function + Optional("worker"): dict, + } +) + +transforms = TransformSequence() +transforms.add_validate(job_description_schema) + + +@transforms.add +def rewrite_when_to_optimization(config, jobs): + for job in jobs: + when = job.pop("when", {}) + if not when: + yield job + continue + + files_changed = when.get("files-changed") + + # implicitly add task config directory. + files_changed.append(f"{config.path}/**") + + # "only when files changed" implies "skip if files have not changed" + job["optimization"] = {"skip-unless-changed": files_changed} + + assert "when" not in job + yield job + + +@transforms.add +def set_implementation(config, jobs): + for job in jobs: + impl, os = worker_type_implementation( + config.graph_config, config.params, job["worker-type"] + ) + if os: + job.setdefault("tags", {})["os"] = os + if impl: + job.setdefault("tags", {})["worker-implementation"] = impl + worker = job.setdefault("worker", {}) + assert "implementation" not in worker + worker["implementation"] = impl + if os: + worker["os"] = os + yield job + + +@transforms.add +def set_label(config, jobs): + for job in jobs: + if "label" not in job: + if "name" not in job: + raise Exception("job has neither a name nor a label") + job["label"] = "{}-{}".format(config.kind, job["name"]) + if job.get("name"): + del job["name"] + yield job + + +@transforms.add +def add_resource_monitor(config, jobs): + for job in jobs: + if job.get("attributes", {}).get("resource-monitor"): + worker_implementation, worker_os = worker_type_implementation( + config.graph_config, config.params, job["worker-type"] + ) + # Normalise worker os so that linux-bitbar and similar use linux tools. + worker_os = worker_os.split("-")[0] + # We don't currently support an Arm worker, due to gopsutil's indirect + # dependencies (go-ole) + if "aarch64" in job["worker-type"]: + yield job + continue + elif "win7" in job["worker-type"]: + arch = "32" + else: + arch = "64" + job.setdefault("fetches", {}) + job["fetches"].setdefault("toolchain", []) + job["fetches"]["toolchain"].append(f"{worker_os}{arch}-resource-monitor") + + if worker_implementation == "docker-worker": + artifact_source = "/builds/worker/monitoring/resource-monitor.json" + else: + artifact_source = "monitoring/resource-monitor.json" + job["worker"].setdefault("artifacts", []) + job["worker"]["artifacts"].append( + { + "name": "public/monitoring/resource-monitor.json", + "type": "file", + "path": artifact_source, + } + ) + # Set env for output file + job["worker"].setdefault("env", {}) + job["worker"]["env"]["RESOURCE_MONITOR_OUTPUT"] = artifact_source + + yield job + + +@transforms.add +def make_task_description(config, jobs): + """Given a build description, create a task description""" + # import plugin modules first, before iterating over jobs + import_sibling_modules(exceptions=("common.py",)) + + for job in jobs: + # only docker-worker uses a fixed absolute path to find directories + if job["worker"]["implementation"] == "docker-worker": + job["run"].setdefault("workdir", "/builds/worker") + + taskdesc = copy_task(job) + + # fill in some empty defaults to make run implementations easier + taskdesc.setdefault("attributes", {}) + taskdesc.setdefault("dependencies", {}) + taskdesc.setdefault("if-dependencies", []) + taskdesc.setdefault("soft-dependencies", []) + taskdesc.setdefault("routes", []) + taskdesc.setdefault("scopes", []) + taskdesc.setdefault("extra", {}) + + # give the function for job.run.using on this worker implementation a + # chance to set up the task description. + configure_taskdesc_for_run( + config, job, taskdesc, job["worker"]["implementation"] + ) + del taskdesc["run"] + + # yield only the task description, discarding the job description + yield taskdesc + + +def get_attribute(dict, key, attributes, attribute_name): + """Get `attribute_name` from the given `attributes` dict, and if there + is a corresponding value, set `key` in `dict` to that value.""" + value = attributes.get(attribute_name) + if value: + dict[key] = value + + +@transforms.add +def use_system_python(config, jobs): + for job in jobs: + if job.pop("use-system-python", True): + yield job + else: + fetches = job.setdefault("fetches", {}) + toolchain = fetches.setdefault("toolchain", []) + + moz_python_home = mozpath.join("fetches", "python") + if "win" in job["worker"]["os"]: + platform = "win64" + elif "linux" in job["worker"]["os"]: + platform = "linux64" + elif "macosx" in job["worker"]["os"]: + platform = "macosx64" + else: + raise ValueError("unexpected worker.os value {}".format(platform)) + + toolchain.append("{}-python".format(platform)) + + worker = job.setdefault("worker", {}) + env = worker.setdefault("env", {}) + env["MOZ_PYTHON_HOME"] = moz_python_home + + yield job + + +@transforms.add +def use_fetches(config, jobs): + artifact_names = {} + extra_env = {} + aliases = {} + tasks = [] + + if config.kind in ("toolchain", "fetch"): + jobs = list(jobs) + tasks.extend((config.kind, j) for j in jobs) + + tasks.extend( + (task.kind, task.__dict__) + for task in config.kind_dependencies_tasks.values() + if task.kind in ("fetch", "toolchain") + ) + for (kind, task) in tasks: + get_attribute( + artifact_names, task["label"], task["attributes"], f"{kind}-artifact" + ) + get_attribute(extra_env, task["label"], task["attributes"], f"{kind}-env") + value = task["attributes"].get(f"{kind}-alias") + if not value: + value = [] + elif isinstance(value, str): + value = [value] + for alias in value: + fully_qualified = f"{kind}-{alias}" + label = task["label"] + if fully_qualified == label: + raise Exception(f"The alias {alias} of task {label} points to itself!") + aliases[fully_qualified] = label + + artifact_prefixes = {} + for job in order_tasks(config, jobs): + artifact_prefixes[job["label"]] = get_artifact_prefix(job) + + fetches = job.pop("fetches", None) + if not fetches: + yield job + continue + + job_fetches = [] + name = job.get("name") or job.get("label").replace(f"{config.kind}-", "") + dependencies = job.setdefault("dependencies", {}) + worker = job.setdefault("worker", {}) + env = worker.setdefault("env", {}) + prefix = get_artifact_prefix(job) + has_sccache = False + for kind, artifacts in fetches.items(): + if kind in ("fetch", "toolchain"): + for fetch_name in artifacts: + label = f"{kind}-{fetch_name}" + label = aliases.get(label, label) + if label not in artifact_names: + raise Exception( + "Missing fetch job for {kind}-{name}: {fetch}".format( + kind=config.kind, name=name, fetch=fetch_name + ) + ) + if label in extra_env: + env.update(extra_env[label]) + + path = artifact_names[label] + + dependencies[label] = label + job_fetches.append( + { + "artifact": path, + "task": f"<{label}>", + "extract": True, + } + ) + + if kind == "toolchain" and fetch_name.endswith("-sccache"): + has_sccache = True + else: + if kind not in dependencies: + raise Exception( + "{name} can't fetch {kind} artifacts because " + "it has no {kind} dependencies!".format(name=name, kind=kind) + ) + dep_label = dependencies[kind] + if dep_label in artifact_prefixes: + prefix = artifact_prefixes[dep_label] + else: + if dep_label not in config.kind_dependencies_tasks: + raise Exception( + "{name} can't fetch {kind} artifacts because " + "there are no tasks with label {label} in kind dependencies!".format( + name=name, + kind=kind, + label=dependencies[kind], + ) + ) + + prefix = get_artifact_prefix( + config.kind_dependencies_tasks[dep_label] + ) + + for artifact in artifacts: + if isinstance(artifact, str): + path = artifact + dest = None + extract = True + verify_hash = False + else: + path = artifact["artifact"] + dest = artifact.get("dest") + extract = artifact.get("extract", True) + verify_hash = artifact.get("verify-hash", False) + + fetch = { + "artifact": f"{prefix}/{path}" + if not path.startswith("/") + else path[1:], + "task": f"<{kind}>", + "extract": extract, + } + if dest is not None: + fetch["dest"] = dest + if verify_hash: + fetch["verify-hash"] = verify_hash + job_fetches.append(fetch) + + if job.get("use-sccache") and not has_sccache: + raise Exception("Must provide an sccache toolchain if using sccache.") + + job_artifact_prefixes = { + mozpath.dirname(fetch["artifact"]) + for fetch in job_fetches + if not fetch["artifact"].startswith("public/") + } + if job_artifact_prefixes: + # Use taskcluster-proxy and request appropriate scope. For example, add + # 'scopes: [queue:get-artifact:path/to/*]' for 'path/to/artifact.tar.xz'. + worker["taskcluster-proxy"] = True + for prefix in sorted(job_artifact_prefixes): + scope = f"queue:get-artifact:{prefix}/*" + if scope not in job.setdefault("scopes", []): + job["scopes"].append(scope) + + artifacts = {} + for f in job_fetches: + _, __, artifact = f["artifact"].rpartition("/") + if "dest" in f: + artifact = f"{f['dest']}/{artifact}" + task = f["task"][1:-1] + if artifact in artifacts: + raise Exception( + f"Task {name} depends on {artifacts[artifact]} and {task} " + f"that both provide {artifact}" + ) + artifacts[artifact] = task + + env["MOZ_FETCHES"] = { + "task-reference": json.dumps( + sorted(job_fetches, key=lambda x: sorted(x.items())), sort_keys=True + ) + } + # The path is normalized to an absolute path in run-task + env.setdefault("MOZ_FETCHES_DIR", "fetches") + + yield job + + +# A registry of all functions decorated with run_job_using +registry = {} + + +def run_job_using(worker_implementation, run_using, schema=None, defaults={}): + """Register the decorated function as able to set up a task description for + jobs with the given worker implementation and `run.using` property. If + `schema` is given, the job's run field will be verified to match it. + + The decorated function should have the signature `using_foo(config, job, taskdesc)` + and should modify the task description in-place. The skeleton of + the task description is already set up, but without a payload.""" + + def wrap(func): + for_run_using = registry.setdefault(run_using, {}) + if worker_implementation in for_run_using: + raise Exception( + "run_job_using({!r}, {!r}) already exists: {!r}".format( + run_using, worker_implementation, for_run_using[run_using] + ) + ) + for_run_using[worker_implementation] = (func, schema, defaults) + return func + + return wrap + + +@run_job_using( + "always-optimized", "always-optimized", Schema({"using": "always-optimized"}) +) +def always_optimized(config, job, taskdesc): + pass + + +def configure_taskdesc_for_run(config, job, taskdesc, worker_implementation): + """ + Run the appropriate function for this job against the given task + description. + + This will raise an appropriate error if no function exists, or if the job's + run is not valid according to the schema. + """ + run_using = job["run"]["using"] + if run_using not in registry: + raise Exception(f"no functions for run.using {run_using!r}") + + if worker_implementation not in registry[run_using]: + raise Exception( + "no functions for run.using {!r} on {!r}".format( + run_using, worker_implementation + ) + ) + + func, schema, defaults = registry[run_using][worker_implementation] + for k, v in defaults.items(): + job["run"].setdefault(k, v) + + if schema: + validate_schema( + schema, + job["run"], + "In job.run using {!r}/{!r} for job {!r}:".format( + job["run"]["using"], worker_implementation, job["label"] + ), + ) + func(config, job, taskdesc) diff --git a/taskcluster/gecko_taskgraph/transforms/job/common.py b/taskcluster/gecko_taskgraph/transforms/job/common.py new file mode 100644 index 0000000000..0c6289a6db --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/job/common.py @@ -0,0 +1,269 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Common support for various job types. These functions are all named after the +worker implementation they operate on, and take the same three parameters, for +consistency. +""" + + +from taskgraph.util.keyed_by import evaluate_keyed_by +from taskgraph.util.taskcluster import get_artifact_prefix + +SECRET_SCOPE = "secrets:get:project/releng/{trust_domain}/{kind}/level-{level}/{secret}" + + +def add_cache(job, taskdesc, name, mount_point, skip_untrusted=False): + """Adds a cache based on the worker's implementation. + + Args: + job (dict): Task's job description. + taskdesc (dict): Target task description to modify. + name (str): Name of the cache. + mount_point (path): Path on the host to mount the cache. + skip_untrusted (bool): Whether cache is used in untrusted environments + (default: False). Only applies to docker-worker. + """ + if not job["run"].get("use-caches", True): + return + + worker = job["worker"] + + if worker["implementation"] == "docker-worker": + taskdesc["worker"].setdefault("caches", []).append( + { + "type": "persistent", + "name": name, + "mount-point": mount_point, + "skip-untrusted": skip_untrusted, + } + ) + + elif worker["implementation"] == "generic-worker": + taskdesc["worker"].setdefault("mounts", []).append( + { + "cache-name": name, + "directory": mount_point, + } + ) + + else: + # Caches not implemented + pass + + +def add_artifacts(config, job, taskdesc, path): + taskdesc["worker"].setdefault("artifacts", []).append( + { + "name": get_artifact_prefix(taskdesc), + "path": path, + "type": "directory", + } + ) + + +def docker_worker_add_artifacts(config, job, taskdesc): + """Adds an artifact directory to the task""" + path = "{workdir}/artifacts/".format(**job["run"]) + taskdesc["worker"].setdefault("env", {})["UPLOAD_DIR"] = path + add_artifacts(config, job, taskdesc, path) + + +def generic_worker_add_artifacts(config, job, taskdesc): + """Adds an artifact directory to the task""" + # The path is the location on disk; it doesn't necessarily + # mean the artifacts will be public or private; that is set via the name + # attribute in add_artifacts. + path = get_artifact_prefix(taskdesc) + taskdesc["worker"].setdefault("env", {})["UPLOAD_DIR"] = path + add_artifacts(config, job, taskdesc, path=path) + + +def support_vcs_checkout(config, job, taskdesc, sparse=False): + """Update a job/task with parameters to enable a VCS checkout. + + This can only be used with ``run-task`` tasks, as the cache name is + reserved for ``run-task`` tasks. + """ + worker = job["worker"] + is_mac = worker["os"] == "macosx" + is_win = worker["os"] == "windows" + is_linux = worker["os"] == "linux" or "linux-bitbar" + is_docker = worker["implementation"] == "docker-worker" + assert is_mac or is_win or is_linux + + if is_win: + checkoutdir = "./build" + geckodir = f"{checkoutdir}/src" + hgstore = "y:/hg-shared" + elif is_docker: + checkoutdir = "{workdir}/checkouts".format(**job["run"]) + geckodir = f"{checkoutdir}/gecko" + hgstore = f"{checkoutdir}/hg-store" + else: + checkoutdir = "./checkouts" + geckodir = f"{checkoutdir}/gecko" + hgstore = f"{checkoutdir}/hg-shared" + + cache_name = "checkouts" + + # Sparse checkouts need their own cache because they can interfere + # with clients that aren't sparse aware. + if sparse: + cache_name += "-sparse" + + # Workers using Mercurial >= 5.8 will enable revlog-compression-zstd, which + # workers using older versions can't understand, so they can't share cache. + # At the moment, only docker workers use the newer version. + if is_docker: + cache_name += "-hg58" + + add_cache(job, taskdesc, cache_name, checkoutdir) + + taskdesc["worker"].setdefault("env", {}).update( + { + "GECKO_BASE_REPOSITORY": config.params["base_repository"], + "GECKO_HEAD_REPOSITORY": config.params["head_repository"], + "GECKO_HEAD_REV": config.params["head_rev"], + "HG_STORE_PATH": hgstore, + } + ) + taskdesc["worker"]["env"].setdefault("GECKO_PATH", geckodir) + + if "comm_base_repository" in config.params: + taskdesc["worker"]["env"].update( + { + "COMM_BASE_REPOSITORY": config.params["comm_base_repository"], + "COMM_HEAD_REPOSITORY": config.params["comm_head_repository"], + "COMM_HEAD_REV": config.params["comm_head_rev"], + } + ) + elif job["run"].get("comm-checkout", False): + raise Exception( + "Can't checkout from comm-* repository if not given a repository." + ) + + # Give task access to hgfingerprint secret so it can pin the certificate + # for hg.mozilla.org. + taskdesc["scopes"].append("secrets:get:project/taskcluster/gecko/hgfingerprint") + taskdesc["scopes"].append("secrets:get:project/taskcluster/gecko/hgmointernal") + + # only some worker platforms have taskcluster-proxy enabled + if job["worker"]["implementation"] in ("docker-worker",): + taskdesc["worker"]["taskcluster-proxy"] = True + + +def generic_worker_hg_commands( + base_repo, head_repo, head_rev, path, sparse_profile=None +): + """Obtain commands needed to obtain a Mercurial checkout on generic-worker. + + Returns two command strings. One performs the checkout. Another logs. + """ + args = [ + r'"c:\Program Files\Mercurial\hg.exe"', + "robustcheckout", + "--sharebase", + r"y:\hg-shared", + "--purge", + "--upstream", + base_repo, + "--revision", + head_rev, + ] + + if sparse_profile: + args.extend(["--config", "extensions.sparse="]) + args.extend(["--sparseprofile", sparse_profile]) + + args.extend( + [ + head_repo, + path, + ] + ) + + logging_args = [ + b":: TinderboxPrint:<a href={source_repo}/rev/{revision} " + b"title='Built from {repo_name} revision {revision}'>{revision}</a>" + b"\n".format( + revision=head_rev, source_repo=head_repo, repo_name=head_repo.split("/")[-1] + ), + ] + + return [" ".join(args), " ".join(logging_args)] + + +def setup_secrets(config, job, taskdesc): + """Set up access to secrets via taskcluster-proxy. The value of + run['secrets'] should be a boolean or a list of secret names that + can be accessed.""" + if not job["run"].get("secrets"): + return + + taskdesc["worker"]["taskcluster-proxy"] = True + secrets = job["run"]["secrets"] + if secrets is True: + secrets = ["*"] + for secret in secrets: + taskdesc["scopes"].append( + SECRET_SCOPE.format( + trust_domain=config.graph_config["trust-domain"], + kind=job["treeherder"]["kind"], + level=config.params["level"], + secret=secret, + ) + ) + + +def add_tooltool(config, job, taskdesc, internal=False): + """Give the task access to tooltool. + + Enables the tooltool cache. Adds releng proxy. Configures scopes. + + By default, only public tooltool access will be granted. Access to internal + tooltool can be enabled via ``internal=True``. + + This can only be used with ``run-task`` tasks, as the cache name is + reserved for use with ``run-task``. + """ + + if job["worker"]["implementation"] in ("docker-worker",): + add_cache( + job, + taskdesc, + "tooltool-cache", + "{workdir}/tooltool-cache".format(**job["run"]), + ) + + taskdesc["worker"].setdefault("env", {}).update( + { + "TOOLTOOL_CACHE": "{workdir}/tooltool-cache".format(**job["run"]), + } + ) + elif not internal: + return + + taskdesc["worker"]["taskcluster-proxy"] = True + taskdesc["scopes"].extend( + [ + "project:releng:services/tooltool/api/download/public", + ] + ) + + if internal: + taskdesc["scopes"].extend( + [ + "project:releng:services/tooltool/api/download/internal", + ] + ) + + +def get_expiration(config, policy="default"): + expires = evaluate_keyed_by( + config.graph_config["expiration-policy"], + "artifact expiration", + {"project": config.params["project"]}, + )[policy] + return expires diff --git a/taskcluster/gecko_taskgraph/transforms/job/distro_package.py b/taskcluster/gecko_taskgraph/transforms/job/distro_package.py new file mode 100644 index 0000000000..44236b1abc --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/job/distro_package.py @@ -0,0 +1,238 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running spidermonkey jobs via dedicated scripts +""" + + +import os +import re + +import taskgraph +from taskgraph.util.schema import Schema +from taskgraph.util.taskcluster import get_root_url +from voluptuous import Any, Optional, Required + +from gecko_taskgraph import GECKO +from gecko_taskgraph.transforms.job import run_job_using +from gecko_taskgraph.transforms.job.common import add_artifacts +from gecko_taskgraph.util.hash import hash_path + +DSC_PACKAGE_RE = re.compile(".*(?=_)") +SOURCE_PACKAGE_RE = re.compile(r".*(?=[-_]\d)") + +source_definition = { + Required("url"): str, + Required("sha256"): str, +} + +common_schema = Schema( + { + # URL/SHA256 of a source file to build, which can either be a source + # control (.dsc), or a tarball. + Required(Any("dsc", "tarball")): source_definition, + # Package name. Normally derived from the source control or tarball file + # name. Use in case the name doesn't match DSC_PACKAGE_RE or + # SOURCE_PACKAGE_RE. + Optional("name"): str, + # Patch to apply to the extracted source. + Optional("patch"): str, + # Command to run before dpkg-buildpackage. + Optional("pre-build-command"): str, + # Architecture to build the package for. + Optional("arch"): str, + # List of package tasks to get build dependencies from. + Optional("packages"): [str], + # What resolver to use to install build dependencies. The default + # (apt-get) is good in most cases, but in subtle cases involving + # a *-backports archive, its solver might not be able to find a + # solution that satisfies the build dependencies. + Optional("resolver"): Any("apt-get", "aptitude"), + # Base work directory used to set up the task. + Required("workdir"): str, + } +) + +debian_schema = common_schema.extend( + { + Required("using"): "debian-package", + # Debian distribution + Required("dist"): str, + } +) + +ubuntu_schema = common_schema.extend( + { + Required("using"): "ubuntu-package", + # Ubuntu distribution + Required("dist"): str, + } +) + + +def common_package(config, job, taskdesc, distro, version): + run = job["run"] + + name = taskdesc["label"].replace(f"{config.kind}-", "", 1) + + arch = run.get("arch", "amd64") + + worker = taskdesc["worker"] + worker.setdefault("artifacts", []) + + image = "%s%d" % (distro, version) + if arch != "amd64": + image += "-" + arch + image += "-packages" + worker["docker-image"] = {"in-tree": image} + + add_artifacts(config, job, taskdesc, path="/tmp/artifacts") + + env = worker.setdefault("env", {}) + env["DEBFULLNAME"] = "Mozilla build team" + env["DEBEMAIL"] = "dev-builds@lists.mozilla.org" + + if "dsc" in run: + src = run["dsc"] + unpack = "dpkg-source -x {src_file} {package}" + package_re = DSC_PACKAGE_RE + elif "tarball" in run: + src = run["tarball"] + unpack = ( + "mkdir {package} && " + "tar -C {package} -axf {src_file} --strip-components=1" + ) + package_re = SOURCE_PACKAGE_RE + else: + raise RuntimeError("Unreachable") + src_url = src["url"] + src_file = os.path.basename(src_url) + src_sha256 = src["sha256"] + package = run.get("name") + if not package: + package = package_re.match(src_file).group(0) + unpack = unpack.format(src_file=src_file, package=package) + + resolver = run.get("resolver", "apt-get") + if resolver == "apt-get": + resolver = "apt-get -yyq --no-install-recommends" + elif resolver == "aptitude": + resolver = ( + "aptitude -y --without-recommends -o " + "Aptitude::ProblemResolver::Hints::KeepBuildDeps=" + '"reject {}-build-deps :UNINST"' + ).format(package) + else: + raise RuntimeError("Unreachable") + + adjust = "" + if "patch" in run: + # We don't use robustcheckout or run-task to get a checkout. So for + # this one file we'd need from a checkout, download it. + env["PATCH_URL"] = config.params.file_url( + "build/debian-packages/{patch}".format(patch=run["patch"]), + ) + adjust += "curl -sL $PATCH_URL | patch -p1 && " + if "pre-build-command" in run: + adjust += run["pre-build-command"] + " && " + if "tarball" in run: + adjust += "mv ../{src_file} ../{package}_{ver}.orig.tar.gz && ".format( + src_file=src_file, + package=package, + ver="$(dpkg-parsechangelog | awk '$1==\"Version:\"{print $2}' | cut -f 1 -d -)", + ) + if "patch" not in run and "pre-build-command" not in run: + adjust += ( + 'debchange -l ".{prefix}moz" --distribution "{dist}"' + ' "Mozilla backport for {dist}." < /dev/null && ' + ).format( + prefix=name.split("-", 1)[0], + dist=run["dist"], + ) + + worker["command"] = [ + "sh", + "-x", + "-c", + # Add sources for packages coming from other package tasks. + "/usr/local/sbin/setup_packages.sh {root_url} $PACKAGES && " + "apt-get update && " + # Upgrade packages that might have new versions in package tasks. + "apt-get dist-upgrade && " "cd /tmp && " + # Get, validate and extract the package source. + "(dget -d -u {src_url} || exit 100) && " + 'echo "{src_sha256} {src_file}" | sha256sum -c && ' + "{unpack} && " + "cd {package} && " + # Optionally apply patch and/or pre-build command. + "{adjust}" + # Install the necessary build dependencies. + "(cd ..; mk-build-deps -i -r {package}/debian/control -t '{resolver}' || exit 100) && " + # Build the package + 'DEB_BUILD_OPTIONS="parallel=$(nproc) nocheck" dpkg-buildpackage -sa && ' + # Copy the artifacts + "mkdir -p {artifacts}/apt && " + "dcmd cp ../{package}_*.changes {artifacts}/apt/ && " + "cd {artifacts} && " + # Make the artifacts directory usable as an APT repository. + "apt-ftparchive sources apt | gzip -c9 > apt/Sources.gz && " + "apt-ftparchive packages apt | gzip -c9 > apt/Packages.gz".format( + root_url=get_root_url(False), + package=package, + src_url=src_url, + src_file=src_file, + src_sha256=src_sha256, + unpack=unpack, + adjust=adjust, + artifacts="/tmp/artifacts", + resolver=resolver, + ), + ] + + if run.get("packages"): + env = worker.setdefault("env", {}) + env["PACKAGES"] = { + "task-reference": " ".join(f"<{p}>" for p in run["packages"]) + } + deps = taskdesc.setdefault("dependencies", {}) + for p in run["packages"]: + deps[p] = f"packages-{p}" + + # Use the command generated above as the base for the index hash. + # We rely on it not varying depending on the head_repository or head_rev. + digest_data = list(worker["command"]) + if "patch" in run: + digest_data.append( + hash_path(os.path.join(GECKO, "build", "debian-packages", run["patch"])) + ) + + if not taskgraph.fast: + taskdesc["cache"] = { + "type": "packages.v1", + "name": name, + "digest-data": digest_data, + } + + +@run_job_using("docker-worker", "debian-package", schema=debian_schema) +def docker_worker_debian_package(config, job, taskdesc): + run = job["run"] + version = { + "wheezy": 7, + "jessie": 8, + "stretch": 9, + "buster": 10, + "bullseye": 11, + }[run["dist"]] + common_package(config, job, taskdesc, "debian", version) + + +@run_job_using("docker-worker", "ubuntu-package", schema=ubuntu_schema) +def docker_worker_ubuntu_package(config, job, taskdesc): + run = job["run"] + version = { + "bionic": 1804, + "focal": 2004, + }[run["dist"]] + common_package(config, job, taskdesc, "ubuntu", version) diff --git a/taskcluster/gecko_taskgraph/transforms/job/hazard.py b/taskcluster/gecko_taskgraph/transforms/job/hazard.py new file mode 100644 index 0000000000..af0e8616e0 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/job/hazard.py @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running hazard jobs via dedicated scripts +""" + + +from taskgraph.util.schema import Schema +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using +from gecko_taskgraph.transforms.job.common import ( + add_tooltool, + docker_worker_add_artifacts, + setup_secrets, +) + +haz_run_schema = Schema( + { + Required("using"): "hazard", + # The command to run within the task image (passed through to the worker) + Required("command"): str, + # The mozconfig to use; default in the script is used if omitted + Optional("mozconfig"): str, + # The set of secret names to which the task has access; these are prefixed + # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting + # this will enable any worker features required and set the task's scopes + # appropriately. `true` here means ['*'], all secrets. Not supported on + # Windows + Optional("secrets"): Any(bool, [str]), + # Base work directory used to set up the task. + Optional("workdir"): str, + } +) + + +@run_job_using("docker-worker", "hazard", schema=haz_run_schema) +def docker_worker_hazard(config, job, taskdesc): + run = job["run"] + + worker = taskdesc["worker"] = job["worker"] + worker.setdefault("artifacts", []) + + docker_worker_add_artifacts(config, job, taskdesc) + worker.setdefault("required-volumes", []).append( + "{workdir}/workspace".format(**run) + ) + add_tooltool(config, job, taskdesc) + setup_secrets(config, job, taskdesc) + + env = worker["env"] + env.update( + { + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + } + ) + + # script parameters + if run.get("mozconfig"): + env["MOZCONFIG"] = run.pop("mozconfig") + + run["using"] = "run-task" + run["cwd"] = run["workdir"] + configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) diff --git a/taskcluster/gecko_taskgraph/transforms/job/mach.py b/taskcluster/gecko_taskgraph/transforms/job/mach.py new file mode 100644 index 0000000000..a418b44794 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/job/mach.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running mach tasks (via run-task) +""" + +from taskgraph.util.schema import Schema, taskref_or_string +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using + +mach_schema = Schema( + { + Required("using"): "mach", + # The mach command (omitting `./mach`) to run + Required("mach"): taskref_or_string, + # The version of Python to run with. Either an absolute path to the binary + # on the worker, a version identifier (e.g python2.7 or 3.8). There is no + # validation performed to ensure the specified binaries actually exist. + Optional("python-version"): Any(str, int, float), + # The sparse checkout profile to use. Value is the filename relative to the + # directory where sparse profiles are defined (build/sparse-profiles/). + Optional("sparse-profile"): Any(str, None), + # if true, perform a checkout of a comm-central based branch inside the + # gecko checkout + Required("comm-checkout"): bool, + # Base work directory used to set up the task. + Optional("workdir"): str, + # Context to substitute into the command using format string + # substitution (e.g {value}). This is useful if certain aspects of the + # command need to be generated in transforms. + Optional("command-context"): dict, + } +) + + +defaults = { + "comm-checkout": False, +} + + +@run_job_using("docker-worker", "mach", schema=mach_schema, defaults=defaults) +@run_job_using("generic-worker", "mach", schema=mach_schema, defaults=defaults) +def configure_mach(config, job, taskdesc): + run = job["run"] + worker = job["worker"] + + additional_prefix = [] + if worker["os"] == "macosx": + additional_prefix = ["LC_ALL=en_US.UTF-8", "LANG=en_US.UTF-8"] + + python = run.get("python-version") + if python: + del run["python-version"] + + if worker["os"] == "macosx" and python == 3: + python = "/usr/local/bin/python3" + + python = str(python) + try: + float(python) + python = "python" + python + except ValueError: + pass + + additional_prefix.append(python) + + command_prefix = " ".join(additional_prefix + ["./mach "]) + + mach = run["mach"] + if isinstance(mach, dict): + ref, pattern = next(iter(mach.items())) + command = {ref: command_prefix + pattern} + else: + command = command_prefix + mach + + # defer to the run_task implementation + run["command"] = command + run["cwd"] = "{checkout}" + run["using"] = "run-task" + del run["mach"] + configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"]) diff --git a/taskcluster/gecko_taskgraph/transforms/job/mozharness.py b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py new file mode 100644 index 0000000000..3dbcc6e015 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py @@ -0,0 +1,366 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" + +Support for running jobs via mozharness. Ideally, most stuff gets run this +way, and certainly anything using mozharness should use this approach. + +""" + +import json +from textwrap import dedent + +from mozpack import path as mozpath +from taskgraph.util.schema import Schema +from voluptuous import Any, Optional, Required +from voluptuous.validators import Match + +from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using +from gecko_taskgraph.transforms.job.common import ( + docker_worker_add_artifacts, + generic_worker_add_artifacts, + get_expiration, + setup_secrets, +) +from gecko_taskgraph.transforms.task import get_branch_repo, get_branch_rev +from gecko_taskgraph.util.attributes import is_try + +mozharness_run_schema = Schema( + { + Required("using"): "mozharness", + # the mozharness script used to run this task, relative to the testing/ + # directory and using forward slashes even on Windows + Required("script"): str, + # Additional paths to look for mozharness configs in. These should be + # relative to the base of the source checkout + Optional("config-paths"): [str], + # the config files required for the task, relative to + # testing/mozharness/configs or one of the paths specified in + # `config-paths` and using forward slashes even on Windows + Required("config"): [str], + # any additional actions to pass to the mozharness command + Optional("actions"): [ + Match("^[a-z0-9-]+$", "actions must be `-` seperated alphanumeric strings") + ], + # any additional options (without leading --) to be passed to mozharness + Optional("options"): [ + Match( + "^[a-z0-9-]+(=[^ ]+)?$", + "options must be `-` seperated alphanumeric strings (with optional argument)", + ) + ], + # --custom-build-variant-cfg value + Optional("custom-build-variant-cfg"): str, + # Extra configuration options to pass to mozharness. + Optional("extra-config"): dict, + # If not false, tooltool downloads will be enabled via relengAPIProxy + # for either just public files, or all files. Not supported on Windows + Required("tooltool-downloads"): Any( + False, + "public", + "internal", + ), + # The set of secret names to which the task has access; these are prefixed + # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting + # this will enable any worker features required and set the task's scopes + # appropriately. `true` here means ['*'], all secrets. Not supported on + # Windows + Required("secrets"): Any(bool, [str]), + # If true, taskcluster proxy will be enabled; note that it may also be enabled + # automatically e.g., for secrets support. Not supported on Windows. + Required("taskcluster-proxy"): bool, + # If false, indicate that builds should skip producing artifacts. Not + # supported on Windows. + Required("keep-artifacts"): bool, + # If specified, use the in-tree job script specified. + Optional("job-script"): str, + Required("requires-signed-builds"): bool, + # Whether or not to use caches. + Optional("use-caches"): bool, + # If false, don't set MOZ_SIMPLE_PACKAGE_NAME + # Only disableable on windows + Required("use-simple-package"): bool, + # If false don't pass --branch mozharness script + # Only disableable on windows + Required("use-magic-mh-args"): bool, + # if true, perform a checkout of a comm-central based branch inside the + # gecko checkout + Required("comm-checkout"): bool, + # Base work directory used to set up the task. + Optional("workdir"): str, + Optional("run-as-root"): bool, + } +) + + +mozharness_defaults = { + "tooltool-downloads": False, + "secrets": False, + "taskcluster-proxy": False, + "keep-artifacts": True, + "requires-signed-builds": False, + "use-simple-package": True, + "use-magic-mh-args": True, + "comm-checkout": False, + "run-as-root": False, +} + + +@run_job_using( + "docker-worker", + "mozharness", + schema=mozharness_run_schema, + defaults=mozharness_defaults, +) +def mozharness_on_docker_worker_setup(config, job, taskdesc): + run = job["run"] + + worker = taskdesc["worker"] = job["worker"] + + if not run.pop("use-simple-package", None): + raise NotImplementedError( + "Simple packaging cannot be disabled via" + "'use-simple-package' on docker-workers" + ) + if not run.pop("use-magic-mh-args", None): + raise NotImplementedError( + "Cannot disabled mh magic arg passing via" + "'use-magic-mh-args' on docker-workers" + ) + + # Running via mozharness assumes an image that contains build.sh: + # by default, debian11-amd64-build, but it could be another image (like + # android-build). + worker.setdefault("docker-image", {"in-tree": "debian11-amd64-build"}) + + worker.setdefault("artifacts", []).append( + { + "name": "public/logs", + "path": "{workdir}/logs/".format(**run), + "type": "directory", + "expires-after": get_expiration(config, "medium"), + } + ) + worker["taskcluster-proxy"] = run.pop("taskcluster-proxy", None) + docker_worker_add_artifacts(config, job, taskdesc) + + env = worker.setdefault("env", {}) + env.update( + { + "WORKSPACE": "{workdir}/workspace".format(**run), + "MOZHARNESS_CONFIG": " ".join(run.pop("config")), + "MOZHARNESS_SCRIPT": run.pop("script"), + "MH_BRANCH": config.params["project"], + "MOZ_SOURCE_CHANGESET": get_branch_rev(config), + "MOZ_SOURCE_REPO": get_branch_repo(config), + "MH_BUILD_POOL": "taskcluster", + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + "PYTHONUNBUFFERED": "1", + } + ) + + worker.setdefault("required-volumes", []).append(env["WORKSPACE"]) + + if "actions" in run: + env["MOZHARNESS_ACTIONS"] = " ".join(run.pop("actions")) + + if "options" in run: + env["MOZHARNESS_OPTIONS"] = " ".join(run.pop("options")) + + if "config-paths" in run: + env["MOZHARNESS_CONFIG_PATHS"] = " ".join(run.pop("config-paths")) + + if "custom-build-variant-cfg" in run: + env["MH_CUSTOM_BUILD_VARIANT_CFG"] = run.pop("custom-build-variant-cfg") + + extra_config = run.pop("extra-config", {}) + extra_config["objdir"] = "obj-build" + env["EXTRA_MOZHARNESS_CONFIG"] = json.dumps(extra_config, sort_keys=True) + + if "job-script" in run: + env["JOB_SCRIPT"] = run["job-script"] + + if is_try(config.params): + env["TRY_COMMIT_MSG"] = config.params["message"] + + # if we're not keeping artifacts, set some env variables to empty values + # that will cause the build process to skip copying the results to the + # artifacts directory. This will have no effect for operations that are + # not builds. + if not run.pop("keep-artifacts"): + env["DIST_TARGET_UPLOADS"] = "" + env["DIST_UPLOADS"] = "" + + # Retry if mozharness returns TBPL_RETRY + worker["retry-exit-status"] = [4] + + setup_secrets(config, job, taskdesc) + + run["using"] = "run-task" + run["command"] = mozpath.join( + "${GECKO_PATH}", + run.pop("job-script", "taskcluster/scripts/builder/build-linux.sh"), + ) + run.pop("secrets") + run.pop("requires-signed-builds") + + configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) + + +@run_job_using( + "generic-worker", + "mozharness", + schema=mozharness_run_schema, + defaults=mozharness_defaults, +) +def mozharness_on_generic_worker(config, job, taskdesc): + assert job["worker"]["os"] in ( + "windows", + "macosx", + ), "only supports windows and macOS right now: {}".format(job["label"]) + + run = job["run"] + + # fail if invalid run options are included + invalid = [] + if not run.pop("keep-artifacts", True): + invalid.append("keep-artifacts") + if invalid: + raise Exception( + "Jobs run using mozharness on Windows do not support properties " + + ", ".join(invalid) + ) + + worker = taskdesc["worker"] = job["worker"] + + worker["taskcluster-proxy"] = run.pop("taskcluster-proxy", None) + + setup_secrets(config, job, taskdesc) + + taskdesc["worker"].setdefault("artifacts", []).append( + { + "name": "public/logs", + "path": "logs", + "type": "directory", + "expires-after": get_expiration(config, "medium"), + } + ) + + if not worker.get("skip-artifacts", False): + generic_worker_add_artifacts(config, job, taskdesc) + + env = worker.setdefault("env", {}) + env.update( + { + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + "MH_BRANCH": config.params["project"], + "MOZ_SOURCE_CHANGESET": get_branch_rev(config), + "MOZ_SOURCE_REPO": get_branch_repo(config), + } + ) + if run.pop("use-simple-package"): + env.update({"MOZ_SIMPLE_PACKAGE_NAME": "target"}) + + extra_config = run.pop("extra-config", {}) + extra_config["objdir"] = "obj-build" + env["EXTRA_MOZHARNESS_CONFIG"] = json.dumps(extra_config, sort_keys=True) + + # The windows generic worker uses batch files to pass environment variables + # to commands. Setting a variable to empty in a batch file unsets, so if + # there is no `TRY_COMMIT_MESSAGE`, pass a space instead, so that + # mozharness doesn't try to find the commit message on its own. + if is_try(config.params): + env["TRY_COMMIT_MSG"] = config.params["message"] or "no commit message" + + if not job["attributes"]["build_platform"].startswith(("win", "macosx")): + raise Exception( + "Task generation for mozharness build jobs currently only supported on " + "Windows and macOS" + ) + + mh_command = [] + if job["worker"]["os"] == "windows": + system_python_dir = "c:/mozilla-build/python3/" + gecko_path = "%GECKO_PATH%" + else: + system_python_dir = "" + gecko_path = "$GECKO_PATH" + + if run.get("use-system-python", True): + python_bindir = system_python_dir + else: + # $MOZ_PYTHON_HOME is going to be substituted in run-task, when we + # know the actual MOZ_PYTHON_HOME value. + is_windows = job["worker"]["os"] == "windows" + if is_windows: + python_bindir = "%MOZ_PYTHON_HOME%/" + else: + python_bindir = "${MOZ_PYTHON_HOME}/bin/" + + mh_command = ["{}python3".format(python_bindir)] + + mh_command += [ + f"{gecko_path}/mach", + "python", + "{}/testing/{}".format(gecko_path, run.pop("script")), + ] + + for path in run.pop("config-paths", []): + mh_command.append(f"--extra-config-path {gecko_path}/{path}") + + for cfg in run.pop("config"): + mh_command.extend(("--config", cfg)) + if run.pop("use-magic-mh-args"): + mh_command.extend(("--branch", config.params["project"])) + if job["worker"]["os"] == "windows": + mh_command.extend(("--work-dir", r"%cd:Z:=z:%\workspace")) + for action in run.pop("actions", []): + mh_command.append("--" + action) + + for option in run.pop("options", []): + mh_command.append("--" + option) + if run.get("custom-build-variant-cfg"): + mh_command.append("--custom-build-variant") + mh_command.append(run.pop("custom-build-variant-cfg")) + + if job["worker"]["os"] == "macosx": + # Ideally, we'd use shellutil.quote, but that would single-quote + # $GECKO_PATH, which would defeat having the variable in the command + # in the first place, as it wouldn't be expanded. + # In practice, arguments are expected not to contain characters that + # would require quoting. + mh_command = " ".join(mh_command) + + run["using"] = "run-task" + run["command"] = mh_command + run.pop("secrets") + run.pop("requires-signed-builds") + run.pop("job-script", None) + configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) + + # Everything past this point is Windows-specific. + if job["worker"]["os"] == "macosx": + return + + if taskdesc.get("use-sccache"): + worker["command"] = ( + [ + # Make the comment part of the first command, as it will help users to + # understand what is going on, and why these steps are implemented. + dedent( + """\ + :: sccache currently uses the full compiler commandline as input to the + :: cache hash key, so create a symlink to the task dir and build from + :: the symlink dir to get consistent paths. + if exist z:\\build rmdir z:\\build""" + ), + r"mklink /d z:\build %cd%", + # Grant delete permission on the link to everyone. + r"icacls z:\build /grant *S-1-1-0:D /L", + r"cd /d z:\build", + ] + + worker["command"] + ) diff --git a/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py b/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py new file mode 100644 index 0000000000..eb4aea609f --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py @@ -0,0 +1,477 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +import re + +from taskgraph.util.schema import Schema +from taskgraph.util.taskcluster import get_artifact_path, get_artifact_url +from voluptuous import Extra, Optional, Required + +from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using +from gecko_taskgraph.transforms.job.common import get_expiration, support_vcs_checkout +from gecko_taskgraph.transforms.test import normpath, test_description_schema +from gecko_taskgraph.util.attributes import is_try + +VARIANTS = [ + "shippable", + "shippable-qr", + "shippable-lite", + "shippable-lite-qr", + "devedition", + "pgo", + "asan", + "stylo", + "qr", + "ccov", +] + + +def get_variant(test_platform): + for v in VARIANTS: + if f"-{v}/" in test_platform: + return v + return "" + + +mozharness_test_run_schema = Schema( + { + Required("using"): "mozharness-test", + Required("test"): { + Required("test-platform"): str, + Required("mozharness"): test_description_schema["mozharness"], + Required("docker-image"): test_description_schema["docker-image"], + Required("loopback-video"): test_description_schema["loopback-video"], + Required("loopback-audio"): test_description_schema["loopback-audio"], + Required("max-run-time"): test_description_schema["max-run-time"], + Optional("retry-exit-status"): test_description_schema["retry-exit-status"], + Extra: object, + }, + # Base work directory used to set up the task. + Optional("workdir"): str, + } +) + + +def test_packages_url(taskdesc): + """Account for different platforms that name their test packages differently""" + artifact_url = get_artifact_url( + "<build>", get_artifact_path(taskdesc, "target.test_packages.json") + ) + # for android shippable we need to add 'en-US' to the artifact url + test = taskdesc["run"]["test"] + if "android" in test["test-platform"] and ( + get_variant(test["test-platform"]) + in ("shippable", "shippable-qr", "shippable-lite", "shippable-lite-qr") + ): + head, tail = os.path.split(artifact_url) + artifact_url = os.path.join(head, "en-US", tail) + return artifact_url + + +def installer_url(taskdesc): + test = taskdesc["run"]["test"] + mozharness = test["mozharness"] + + if "installer-url" in mozharness: + installer_url = mozharness["installer-url"] + else: + upstream_task = ( + "<build-signing>" if mozharness["requires-signed-builds"] else "<build>" + ) + installer_url = get_artifact_url( + upstream_task, mozharness["build-artifact-name"] + ) + + return installer_url + + +@run_job_using("docker-worker", "mozharness-test", schema=mozharness_test_run_schema) +def mozharness_test_on_docker(config, job, taskdesc): + run = job["run"] + test = taskdesc["run"]["test"] + mozharness = test["mozharness"] + worker = taskdesc["worker"] = job["worker"] + + # apply some defaults + worker["docker-image"] = test["docker-image"] + worker["allow-ptrace"] = True # required for all tests, for crashreporter + worker["loopback-video"] = test["loopback-video"] + worker["loopback-audio"] = test["loopback-audio"] + worker["max-run-time"] = test["max-run-time"] + worker["retry-exit-status"] = test["retry-exit-status"] + if "android-em-7.0-x86" in test["test-platform"]: + worker["privileged"] = True + + artifacts = [ + # (artifact name prefix, in-image path) + ("public/logs", "{workdir}/workspace/logs/".format(**run)), + ("public/test", "{workdir}/artifacts/".format(**run)), + ( + "public/test_info", + "{workdir}/workspace/build/blobber_upload_dir/".format(**run), + ), + ] + + installer = installer_url(taskdesc) + + mozharness_url = get_artifact_url( + "<build>", get_artifact_path(taskdesc, "mozharness.zip") + ) + + worker.setdefault("artifacts", []) + worker["artifacts"].extend( + [ + { + "name": prefix, + "path": os.path.join("{workdir}/workspace".format(**run), path), + "type": "directory", + "expires-after": get_expiration(config, "default"), + } + for (prefix, path) in artifacts + ] + ) + + env = worker.setdefault("env", {}) + env.update( + { + "MOZHARNESS_CONFIG": " ".join(mozharness["config"]), + "MOZHARNESS_SCRIPT": mozharness["script"], + "MOZILLA_BUILD_URL": {"task-reference": installer}, + "NEED_PULSEAUDIO": "true", + "NEED_WINDOW_MANAGER": "true", + "ENABLE_E10S": str(bool(test.get("e10s"))).lower(), + "WORKING_DIR": "/builds/worker", + } + ) + + env["PYTHON"] = "python3" + + # Legacy linux64 tests rely on compiz. + if test.get("docker-image", {}).get("in-tree") == "desktop1604-test": + env.update({"NEED_COMPIZ": "true"}) + + # Bug 1602701/1601828 - use compiz on ubuntu1804 due to GTK asynchiness + # when manipulating windows. + if test.get("docker-image", {}).get("in-tree") == "ubuntu1804-test": + if "wdspec" in job["run"]["test"]["suite"] or ( + "marionette" in job["run"]["test"]["suite"] + and "headless" not in job["label"] + ): + env.update({"NEED_COMPIZ": "true"}) + + # Set MOZ_ENABLE_WAYLAND env variables to enable Wayland backend. + if "wayland" in job["label"]: + env["MOZ_ENABLE_WAYLAND"] = "1" + + if mozharness.get("mochitest-flavor"): + env["MOCHITEST_FLAVOR"] = mozharness["mochitest-flavor"] + + if mozharness["set-moz-node-path"]: + env["MOZ_NODE_PATH"] = "/usr/local/bin/node" + + if "actions" in mozharness: + env["MOZHARNESS_ACTIONS"] = " ".join(mozharness["actions"]) + + if is_try(config.params): + env["TRY_COMMIT_MSG"] = config.params["message"] + + # handle some of the mozharness-specific options + if test["reboot"]: + raise Exception( + "reboot: {} not supported on generic-worker".format(test["reboot"]) + ) + + # Support vcs checkouts regardless of whether the task runs from + # source or not in case it is needed on an interactive loaner. + support_vcs_checkout(config, job, taskdesc) + + # If we have a source checkout, run mozharness from it instead of + # downloading a zip file with the same content. + if test["checkout"]: + env["MOZHARNESS_PATH"] = "{workdir}/checkouts/gecko/testing/mozharness".format( + **run + ) + else: + env["MOZHARNESS_URL"] = {"task-reference": mozharness_url} + + extra_config = { + "installer_url": installer, + "test_packages_url": test_packages_url(taskdesc), + } + env["EXTRA_MOZHARNESS_CONFIG"] = { + "task-reference": json.dumps(extra_config, sort_keys=True) + } + + # Bug 1634554 - pass in decision task artifact URL to mozharness for WPT. + # Bug 1645974 - test-verify-wpt and test-coverage-wpt need artifact URL. + if "web-platform-tests" in test["suite"] or re.match( + "test-(coverage|verify)-wpt", test["suite"] + ): + env["TESTS_BY_MANIFEST_URL"] = { + "artifact-reference": "<decision/public/tests-by-manifest.json.gz>" + } + + command = [ + "{workdir}/bin/test-linux.sh".format(**run), + ] + command.extend(mozharness.get("extra-options", [])) + + if test.get("test-manifests"): + env["MOZHARNESS_TEST_PATHS"] = json.dumps( + {test["suite"]: test["test-manifests"]}, sort_keys=True + ) + + # TODO: remove the need for run['chunked'] + elif mozharness.get("chunked") or test["chunks"] > 1: + command.append("--total-chunk={}".format(test["chunks"])) + command.append("--this-chunk={}".format(test["this-chunk"])) + + if "download-symbols" in mozharness: + download_symbols = mozharness["download-symbols"] + download_symbols = {True: "true", False: "false"}.get( + download_symbols, download_symbols + ) + command.append("--download-symbols=" + download_symbols) + + job["run"] = { + "workdir": run["workdir"], + "tooltool-downloads": mozharness["tooltool-downloads"], + "checkout": test["checkout"], + "command": command, + "using": "run-task", + } + configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) + + +@run_job_using("generic-worker", "mozharness-test", schema=mozharness_test_run_schema) +def mozharness_test_on_generic_worker(config, job, taskdesc): + test = taskdesc["run"]["test"] + mozharness = test["mozharness"] + worker = taskdesc["worker"] = job["worker"] + + bitbar_script = "test-linux.sh" + + is_macosx = worker["os"] == "macosx" + is_windows = worker["os"] == "windows" + is_linux = worker["os"] == "linux" or worker["os"] == "linux-bitbar" + is_bitbar = worker["os"] == "linux-bitbar" + assert is_macosx or is_windows or is_linux + + artifacts = [ + { + "name": "public/logs", + "path": "logs", + "type": "directory", + "expires-after": get_expiration(config, "default"), + } + ] + + # jittest doesn't have blob_upload_dir + if test["test-name"] != "jittest": + artifacts.append( + { + "name": "public/test_info", + "path": "build/blobber_upload_dir", + "type": "directory", + "expires-after": get_expiration(config, "default"), + } + ) + + if is_bitbar: + artifacts = [ + { + "name": "public/test/", + "path": "artifacts/public", + "type": "directory", + "expires-after": get_expiration(config, "default"), + }, + { + "name": "public/logs/", + "path": "workspace/logs", + "type": "directory", + "expires-after": get_expiration(config, "default"), + }, + { + "name": "public/test_info/", + "path": "workspace/build/blobber_upload_dir", + "type": "directory", + "expires-after": get_expiration(config, "default"), + }, + ] + + installer = installer_url(taskdesc) + + worker["os-groups"] = test["os-groups"] + + # run-as-administrator is a feature for workers with UAC enabled and as such should not be + # included in tasks on workers that have UAC disabled. Currently UAC is only enabled on + # gecko Windows 10 workers, however this may be subject to change. Worker type + # environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig + # See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control + # for more information about UAC. + if test.get("run-as-administrator", False): + if job["worker-type"].startswith("win10-64") or job["worker-type"].startswith( + "win11-64" + ): + worker["run-as-administrator"] = True + else: + raise Exception( + "run-as-administrator not supported on {}".format(job["worker-type"]) + ) + + if test["reboot"]: + raise Exception( + "reboot: {} not supported on generic-worker".format(test["reboot"]) + ) + + worker["max-run-time"] = test["max-run-time"] + worker["retry-exit-status"] = test["retry-exit-status"] + worker.setdefault("artifacts", []) + worker["artifacts"].extend(artifacts) + + env = worker.setdefault("env", {}) + env["GECKO_HEAD_REPOSITORY"] = config.params["head_repository"] + env["GECKO_HEAD_REV"] = config.params["head_rev"] + + # this list will get cleaned up / reduced / removed in bug 1354088 + if is_macosx: + env.update( + { + "LC_ALL": "en_US.UTF-8", + "LANG": "en_US.UTF-8", + "MOZ_NODE_PATH": "/usr/local/bin/node", + "PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", + "SHELL": "/bin/bash", + } + ) + elif is_bitbar: + env.update( + { + "LANG": "en_US.UTF-8", + "MOZHARNESS_CONFIG": " ".join(mozharness["config"]), + "MOZHARNESS_SCRIPT": mozharness["script"], + "MOZHARNESS_URL": { + "artifact-reference": "<build/public/build/mozharness.zip>" + }, + "MOZILLA_BUILD_URL": {"task-reference": installer}, + "MOZ_NO_REMOTE": "1", + "NEED_XVFB": "false", + "XPCOM_DEBUG_BREAK": "warn", + "NO_FAIL_ON_TEST_ERRORS": "1", + "MOZ_HIDE_RESULTS_TABLE": "1", + "MOZ_NODE_PATH": "/usr/local/bin/node", + "TASKCLUSTER_WORKER_TYPE": job["worker-type"], + } + ) + + extra_config = { + "installer_url": installer, + "test_packages_url": test_packages_url(taskdesc), + } + env["EXTRA_MOZHARNESS_CONFIG"] = { + "task-reference": json.dumps(extra_config, sort_keys=True) + } + + # Bug 1634554 - pass in decision task artifact URL to mozharness for WPT. + # Bug 1645974 - test-verify-wpt and test-coverage-wpt need artifact URL. + if "web-platform-tests" in test["suite"] or re.match( + "test-(coverage|verify)-wpt", test["suite"] + ): + env["TESTS_BY_MANIFEST_URL"] = { + "artifact-reference": "<decision/public/tests-by-manifest.json.gz>" + } + + if is_windows: + py_binary = "c:\\mozilla-build\\{python}\\{python}.exe".format(python="python3") + mh_command = [ + py_binary, + "-u", + "mozharness\\scripts\\" + normpath(mozharness["script"]), + ] + elif is_bitbar: + py_binary = "python3" + mh_command = ["bash", f"./{bitbar_script}"] + elif is_macosx: + py_binary = "/usr/local/bin/{}".format("python3") + mh_command = [ + py_binary, + "-u", + "mozharness/scripts/" + mozharness["script"], + ] + else: + # is_linux + py_binary = "/usr/bin/{}".format("python3") + mh_command = [ + # Using /usr/bin/python2.7 rather than python2.7 because + # /usr/local/bin/python2.7 is broken on the mac workers. + # See bug #1547903. + py_binary, + "-u", + "mozharness/scripts/" + mozharness["script"], + ] + + env["PYTHON"] = py_binary + + for mh_config in mozharness["config"]: + cfg_path = "mozharness/configs/" + mh_config + if is_windows: + cfg_path = normpath(cfg_path) + mh_command.extend(["--cfg", cfg_path]) + mh_command.extend(mozharness.get("extra-options", [])) + if mozharness.get("download-symbols"): + if isinstance(mozharness["download-symbols"], str): + mh_command.extend(["--download-symbols", mozharness["download-symbols"]]) + else: + mh_command.extend(["--download-symbols", "true"]) + if mozharness.get("include-blob-upload-branch"): + mh_command.append("--blob-upload-branch=" + config.params["project"]) + + if test.get("test-manifests"): + env["MOZHARNESS_TEST_PATHS"] = json.dumps( + {test["suite"]: test["test-manifests"]}, sort_keys=True + ) + + # TODO: remove the need for run['chunked'] + elif mozharness.get("chunked") or test["chunks"] > 1: + mh_command.append("--total-chunk={}".format(test["chunks"])) + mh_command.append("--this-chunk={}".format(test["this-chunk"])) + + if is_try(config.params): + env["TRY_COMMIT_MSG"] = config.params["message"] + + worker["mounts"] = [ + { + "directory": "mozharness", + "content": { + "artifact": get_artifact_path(taskdesc, "mozharness.zip"), + "task-id": {"task-reference": "<build>"}, + }, + "format": "zip", + } + ] + if is_bitbar: + a_url = config.params.file_url( + f"taskcluster/scripts/tester/{bitbar_script}", + ) + worker["mounts"] = [ + { + "file": bitbar_script, + "content": { + "url": a_url, + }, + } + ] + + job["run"] = { + "tooltool-downloads": mozharness["tooltool-downloads"], + "checkout": test["checkout"], + "command": mh_command, + "using": "run-task", + } + if is_bitbar: + job["run"]["run-as-root"] = True + configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) diff --git a/taskcluster/gecko_taskgraph/transforms/job/python_test.py b/taskcluster/gecko_taskgraph/transforms/job/python_test.py new file mode 100644 index 0000000000..b572061217 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/job/python_test.py @@ -0,0 +1,47 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running mach python-test tasks (via run-task) +""" + + +from taskgraph.util.schema import Schema +from voluptuous import Optional, Required + +from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using + +python_test_schema = Schema( + { + Required("using"): "python-test", + # Python version to use + Required("python-version"): int, + # The subsuite to run + Required("subsuite"): str, + # Base work directory used to set up the task. + Optional("workdir"): str, + } +) + + +defaults = { + "python-version": 3, + "subsuite": "default", +} + + +@run_job_using( + "docker-worker", "python-test", schema=python_test_schema, defaults=defaults +) +@run_job_using( + "generic-worker", "python-test", schema=python_test_schema, defaults=defaults +) +def configure_python_test(config, job, taskdesc): + run = job["run"] + worker = job["worker"] + + # defer to the mach implementation + run["mach"] = ("python-test --subsuite {subsuite} --run-slow").format(**run) + run["using"] = "mach" + del run["subsuite"] + configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) diff --git a/taskcluster/gecko_taskgraph/transforms/job/run_task.py b/taskcluster/gecko_taskgraph/transforms/job/run_task.py new file mode 100644 index 0000000000..201c0b825a --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/job/run_task.py @@ -0,0 +1,308 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running jobs that are invoked via the `run-task` script. +""" + + +import os + +from mozbuild.util import memoize +from mozpack import path +from taskgraph.util.schema import Schema +from taskgraph.util.yaml import load_yaml +from voluptuous import Any, Extra, Optional, Required + +from gecko_taskgraph import GECKO +from gecko_taskgraph.transforms.job import run_job_using +from gecko_taskgraph.transforms.job.common import add_tooltool, support_vcs_checkout +from gecko_taskgraph.transforms.task import taskref_or_string + +run_task_schema = Schema( + { + Required("using"): "run-task", + # if true, add a cache at ~worker/.cache, which is where things like pip + # tend to hide their caches. This cache is never added for level-1 jobs. + # TODO Once bug 1526028 is fixed, this and 'use-caches' should be merged. + Required("cache-dotcache"): bool, + # Whether or not to use caches. + Optional("use-caches"): bool, + # if true (the default), perform a checkout of gecko on the worker + Required("checkout"): bool, + Optional( + "cwd", + description="Path to run command in. If a checkout is present, the path " + "to the checkout will be interpolated with the key `checkout`", + ): str, + # The sparse checkout profile to use. Value is the filename relative to + # "sparse-profile-prefix" which defaults to "build/sparse-profiles/". + Required("sparse-profile"): Any(str, None), + # The relative path to the sparse profile. + Optional("sparse-profile-prefix"): str, + # if true, perform a checkout of a comm-central based branch inside the + # gecko checkout + Required("comm-checkout"): bool, + # The command arguments to pass to the `run-task` script, after the + # checkout arguments. If a list, it will be passed directly; otherwise + # it will be included in a single argument to `bash -cx`. + Required("command"): Any([taskref_or_string], taskref_or_string), + # Context to substitute into the command using format string + # substitution (e.g {value}). This is useful if certain aspects of the + # command need to be generated in transforms. + Optional("command-context"): { + # If present, loads a set of context variables from an unnested yaml + # file. If a value is present in both the provided file and directly + # in command-context, the latter will take priority. + Optional("from-file"): str, + Extra: object, + }, + # Base work directory used to set up the task. + Optional("workdir"): str, + # If not false, tooltool downloads will be enabled via relengAPIProxy + # for either just public files, or all files. Only supported on + # docker-worker. + Required("tooltool-downloads"): Any( + False, + "public", + "internal", + ), + # Whether to run as root. (defaults to False) + Optional("run-as-root"): bool, + } +) + + +def common_setup(config, job, taskdesc, command): + run = job["run"] + if run["checkout"]: + support_vcs_checkout(config, job, taskdesc, sparse=bool(run["sparse-profile"])) + command.append( + "--gecko-checkout={}".format(taskdesc["worker"]["env"]["GECKO_PATH"]) + ) + + if run["sparse-profile"]: + sparse_profile_prefix = run.pop( + "sparse-profile-prefix", "build/sparse-profiles" + ) + sparse_profile_path = path.join(sparse_profile_prefix, run["sparse-profile"]) + command.append(f"--gecko-sparse-profile={sparse_profile_path}") + + taskdesc["worker"].setdefault("env", {})["MOZ_SCM_LEVEL"] = config.params["level"] + + +worker_defaults = { + "cache-dotcache": False, + "checkout": True, + "comm-checkout": False, + "sparse-profile": None, + "tooltool-downloads": False, + "run-as-root": False, +} + + +load_yaml = memoize(load_yaml) + + +def script_url(config, script): + if "MOZ_AUTOMATION" in os.environ and "TASK_ID" not in os.environ: + raise Exception("TASK_ID must be defined to use run-task on generic-worker") + task_id = os.environ.get("TASK_ID", "<TASK_ID>") + tc_url = "http://firefox-ci-tc.services.mozilla.com" + return f"{tc_url}/api/queue/v1/task/{task_id}/artifacts/public/{script}" + + +def substitute_command_context(command_context, command): + from_file = command_context.pop("from-file", None) + full_context = {} + if from_file: + full_context = load_yaml(os.path.join(GECKO, from_file)) + else: + full_context = {} + + full_context.update(command_context) + + if isinstance(command, list): + for i in range(len(command)): + command[i] = command[i].format(**full_context) + else: + command = command.format(**full_context) + + return command + + +@run_job_using( + "docker-worker", "run-task", schema=run_task_schema, defaults=worker_defaults +) +def docker_worker_run_task(config, job, taskdesc): + run = job["run"] + worker = taskdesc["worker"] = job["worker"] + command = ["/builds/worker/bin/run-task"] + common_setup(config, job, taskdesc, command) + + if run["tooltool-downloads"]: + internal = run["tooltool-downloads"] == "internal" + add_tooltool(config, job, taskdesc, internal=internal) + + if run.get("cache-dotcache"): + worker["caches"].append( + { + "type": "persistent", + "name": "{project}-dotcache".format(**config.params), + "mount-point": "{workdir}/.cache".format(**run), + "skip-untrusted": True, + } + ) + + if run.get("command-context"): + run_command = substitute_command_context( + run.get("command-context"), run["command"] + ) + else: + run_command = run["command"] + + run_cwd = run.get("cwd") + if run_cwd and run["checkout"]: + run_cwd = path.normpath( + run_cwd.format(checkout=taskdesc["worker"]["env"]["GECKO_PATH"]) + ) + elif run_cwd and "{checkout}" in run_cwd: + raise Exception( + "Found `{{checkout}}` interpolation in `cwd` for task {name} " + "but the task doesn't have a checkout: {cwd}".format( + cwd=run_cwd, name=job.get("name", job.get("label")) + ) + ) + + # dict is for the case of `{'task-reference': text_type}`. + if isinstance(run_command, (str, dict)): + run_command = ["bash", "-cx", run_command] + if run["comm-checkout"]: + command.append( + "--comm-checkout={}/comm".format(taskdesc["worker"]["env"]["GECKO_PATH"]) + ) + if run["run-as-root"]: + command.extend(("--user", "root", "--group", "root")) + if run_cwd: + command.extend(("--task-cwd", run_cwd)) + command.append("--") + command.extend(run_command) + worker["command"] = command + + +@run_job_using( + "generic-worker", "run-task", schema=run_task_schema, defaults=worker_defaults +) +def generic_worker_run_task(config, job, taskdesc): + run = job["run"] + worker = taskdesc["worker"] = job["worker"] + is_win = worker["os"] == "windows" + is_mac = worker["os"] == "macosx" + is_bitbar = worker["os"] == "linux-bitbar" + + if run["tooltool-downloads"]: + internal = run["tooltool-downloads"] == "internal" + add_tooltool(config, job, taskdesc, internal=internal) + + if is_win: + command = ["C:/mozilla-build/python3/python3.exe", "run-task"] + elif is_mac: + command = ["/usr/local/bin/python3", "run-task"] + else: + command = ["./run-task"] + + common_setup(config, job, taskdesc, command) + + worker.setdefault("mounts", []) + if run.get("cache-dotcache"): + worker["mounts"].append( + { + "cache-name": "{project}-dotcache".format(**config.params), + "directory": "{workdir}/.cache".format(**run), + } + ) + worker["mounts"].append( + { + "content": { + "url": script_url(config, "run-task"), + }, + "file": "./run-task", + } + ) + if job.get("fetches", {}): + worker["mounts"].append( + { + "content": { + "url": script_url(config, "fetch-content"), + }, + "file": "./fetch-content", + } + ) + + run_command = run["command"] + run_cwd = run.get("cwd") + if run_cwd and run["checkout"]: + run_cwd = path.normpath( + run_cwd.format(checkout=taskdesc["worker"]["env"]["GECKO_PATH"]) + ) + elif run_cwd and "{checkout}" in run_cwd: + raise Exception( + "Found `{{checkout}}` interpolation in `cwd` for task {name} " + "but the task doesn't have a checkout: {cwd}".format( + cwd=run_cwd, name=job.get("name", job.get("label")) + ) + ) + + # dict is for the case of `{'task-reference': text_type}`. + if isinstance(run_command, (str, dict)): + if is_win: + if isinstance(run_command, dict): + for k in run_command.keys(): + run_command[k] = f'"{run_command[k]}"' + else: + run_command = f'"{run_command}"' + run_command = ["bash", "-cx", run_command] + + if run.get("command-context"): + run_command = substitute_command_context( + run.get("command-context"), run_command + ) + + if run["comm-checkout"]: + command.append( + "--comm-checkout={}/comm".format(taskdesc["worker"]["env"]["GECKO_PATH"]) + ) + + if run["run-as-root"]: + command.extend(("--user", "root", "--group", "root")) + if run_cwd: + command.extend(("--task-cwd", run_cwd)) + command.append("--") + if is_bitbar: + # Use the bitbar wrapper script which sets up the device and adb + # environment variables + command.append("/builds/taskcluster/script.py") + command.extend(run_command) + + if is_win: + taskref = False + for c in command: + if isinstance(c, dict): + taskref = True + + if taskref: + cmd = [] + for c in command: + if isinstance(c, dict): + for v in c.values(): + cmd.append(v) + else: + cmd.append(c) + worker["command"] = [{"artifact-reference": " ".join(cmd)}] + else: + worker["command"] = [" ".join(command)] + else: + worker["command"] = [ + ["chmod", "+x", "run-task"], + command, + ] diff --git a/taskcluster/gecko_taskgraph/transforms/job/spidermonkey.py b/taskcluster/gecko_taskgraph/transforms/job/spidermonkey.py new file mode 100644 index 0000000000..91c7e93bd6 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/job/spidermonkey.py @@ -0,0 +1,109 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running spidermonkey jobs via dedicated scripts +""" + + +from taskgraph.util.schema import Schema +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using +from gecko_taskgraph.transforms.job.common import ( + docker_worker_add_artifacts, + generic_worker_add_artifacts, +) + +sm_run_schema = Schema( + { + Required("using"): Any( + "spidermonkey", + "spidermonkey-package", + ), + # SPIDERMONKEY_VARIANT and SPIDERMONKEY_PLATFORM + Required("spidermonkey-variant"): str, + Optional("spidermonkey-platform"): str, + # Base work directory used to set up the task. + Optional("workdir"): str, + Required("tooltool-downloads"): Any( + False, + "public", + "internal", + ), + } +) + + +@run_job_using("docker-worker", "spidermonkey", schema=sm_run_schema) +@run_job_using("docker-worker", "spidermonkey-package", schema=sm_run_schema) +def docker_worker_spidermonkey(config, job, taskdesc): + run = job["run"] + + worker = taskdesc["worker"] = job["worker"] + worker.setdefault("artifacts", []) + + docker_worker_add_artifacts(config, job, taskdesc) + + env = worker.setdefault("env", {}) + env.update( + { + "MOZHARNESS_DISABLE": "true", + "SPIDERMONKEY_VARIANT": run.pop("spidermonkey-variant"), + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + } + ) + if "spidermonkey-platform" in run: + env["SPIDERMONKEY_PLATFORM"] = run.pop("spidermonkey-platform") + + script = "build-sm.sh" + if run["using"] == "spidermonkey-package": + script = "build-sm-package.sh" + + run["using"] = "run-task" + run["cwd"] = run["workdir"] + run["command"] = [f"./checkouts/gecko/taskcluster/scripts/builder/{script}"] + + configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) + + +@run_job_using("generic-worker", "spidermonkey", schema=sm_run_schema) +def generic_worker_spidermonkey(config, job, taskdesc): + assert job["worker"]["os"] == "windows", "only supports windows right now" + + run = job["run"] + + worker = taskdesc["worker"] = job["worker"] + + generic_worker_add_artifacts(config, job, taskdesc) + + env = worker.setdefault("env", {}) + env.update( + { + "MOZHARNESS_DISABLE": "true", + "SPIDERMONKEY_VARIANT": run.pop("spidermonkey-variant"), + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + "SCCACHE_DISABLE": "1", + "WORK": ".", # Override the defaults in build scripts + "GECKO_PATH": "./src", # with values suiteable for windows generic worker + "UPLOAD_DIR": "./public/build", + } + ) + if "spidermonkey-platform" in run: + env["SPIDERMONKEY_PLATFORM"] = run.pop("spidermonkey-platform") + + script = "build-sm.sh" + if run["using"] == "spidermonkey-package": + script = "build-sm-package.sh" + # Don't allow untested configurations yet + raise Exception("spidermonkey-package is not a supported configuration") + + run["using"] = "run-task" + run["command"] = [ + "c:\\mozilla-build\\msys2\\usr\\bin\\bash.exe " # string concat + '"./src/taskcluster/scripts/builder/%s"' % script + ] + + configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) diff --git a/taskcluster/gecko_taskgraph/transforms/job/toolchain.py b/taskcluster/gecko_taskgraph/transforms/job/toolchain.py new file mode 100644 index 0000000000..fb030019fc --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/job/toolchain.py @@ -0,0 +1,257 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running toolchain-building jobs via dedicated scripts +""" + + +import os + +import taskgraph +from mozbuild.shellutil import quote as shell_quote +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from voluptuous import Any, Optional, Required + +from gecko_taskgraph import GECKO +from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using +from gecko_taskgraph.transforms.job.common import ( + docker_worker_add_artifacts, + generic_worker_add_artifacts, +) +from gecko_taskgraph.util.attributes import RELEASE_PROJECTS +from gecko_taskgraph.util.hash import hash_paths + +CACHE_TYPE = "toolchains.v3" + +toolchain_run_schema = Schema( + { + Required("using"): "toolchain-script", + # The script (in taskcluster/scripts/misc) to run. + # Python scripts are invoked with `mach python` so vendored libraries + # are available. + Required("script"): str, + # Arguments to pass to the script. + Optional("arguments"): [str], + # If not false, tooltool downloads will be enabled via relengAPIProxy + # for either just public files, or all files. Not supported on Windows + Required("tooltool-downloads"): Any( + False, + "public", + "internal", + ), + # Sparse profile to give to checkout using `run-task`. If given, + # Defaults to "toolchain-build". The value is relative to + # "sparse-profile-prefix", optionally defined below is the path, + # defaulting to "build/sparse-profiles". + # i.e. `build/sparse-profiles/toolchain-build`. + # If `None`, instructs `run-task` to not use a sparse profile at all. + Required("sparse-profile"): Any(str, None), + # The relative path to the sparse profile. + Optional("sparse-profile-prefix"): str, + # Paths/patterns pointing to files that influence the outcome of a + # toolchain build. + Optional("resources"): [str], + # Path to the artifact produced by the toolchain job + Required("toolchain-artifact"): str, + Optional( + "toolchain-alias", + description="An alias that can be used instead of the real toolchain job name in " + "fetch stanzas for jobs.", + ): optionally_keyed_by("project", Any(None, str, [str])), + Optional( + "toolchain-env", + description="Additional env variables to add to the worker when using this toolchain", + ): {str: object}, + # Base work directory used to set up the task. + Optional("workdir"): str, + } +) + + +def get_digest_data(config, run, taskdesc): + files = list(run.pop("resources", [])) + # The script + files.append("taskcluster/scripts/misc/{}".format(run["script"])) + # Tooltool manifest if any is defined: + tooltool_manifest = taskdesc["worker"]["env"].get("TOOLTOOL_MANIFEST") + if tooltool_manifest: + files.append(tooltool_manifest) + + # Accumulate dependency hashes for index generation. + data = [hash_paths(GECKO, files)] + + data.append(taskdesc["attributes"]["toolchain-artifact"]) + + # If the task uses an in-tree docker image, we want it to influence + # the index path as well. Ideally, the content of the docker image itself + # should have an influence, but at the moment, we can't get that + # information here. So use the docker image name as a proxy. Not a lot of + # changes to docker images actually have an impact on the resulting + # toolchain artifact, so we'll just rely on such important changes to be + # accompanied with a docker image name change. + image = taskdesc["worker"].get("docker-image", {}).get("in-tree") + if image: + data.append(image) + + # Likewise script arguments should influence the index. + args = run.get("arguments") + if args: + data.extend(args) + + if taskdesc["attributes"].get("rebuild-on-release"): + # Add whether this is a release branch or not + data.append(str(config.params["project"] in RELEASE_PROJECTS)) + return data + + +def common_toolchain(config, job, taskdesc, is_docker): + run = job["run"] + + worker = taskdesc["worker"] = job["worker"] + worker["chain-of-trust"] = True + + if is_docker: + # If the task doesn't have a docker-image, set a default + worker.setdefault("docker-image", {"in-tree": "deb11-toolchain-build"}) + + if job["worker"]["os"] == "windows": + # There were no caches on generic-worker before bug 1519472, and they cause + # all sorts of problems with Windows toolchain tasks, disable them until + # tasks are ready. + run["use-caches"] = False + + env = worker.setdefault("env", {}) + env.update( + { + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + "TOOLCHAIN_ARTIFACT": run["toolchain-artifact"], + } + ) + + if is_docker: + # Toolchain checkouts don't live under {workdir}/checkouts + workspace = "{workdir}/workspace/build".format(**run) + env["GECKO_PATH"] = f"{workspace}/src" + + attributes = taskdesc.setdefault("attributes", {}) + attributes["toolchain-artifact"] = run.pop("toolchain-artifact") + toolchain_artifact = attributes["toolchain-artifact"] + if not toolchain_artifact.startswith("public/build/"): + if "artifact_prefix" in attributes: + raise Exception( + "Toolchain {} has an artifact_prefix attribute. That is not" + " allowed on toolchain tasks.".format(taskdesc["label"]) + ) + attributes["artifact_prefix"] = os.path.dirname(toolchain_artifact) + + resolve_keyed_by( + run, + "toolchain-alias", + item_name=taskdesc["label"], + project=config.params["project"], + ) + alias = run.pop("toolchain-alias", None) + if alias: + attributes["toolchain-alias"] = alias + if "toolchain-env" in run: + attributes["toolchain-env"] = run.pop("toolchain-env") + + # Allow the job to specify where artifacts come from, but add + # public/build if it's not there already. + artifacts = worker.setdefault("artifacts", []) + if not artifacts: + if is_docker: + docker_worker_add_artifacts(config, job, taskdesc) + else: + generic_worker_add_artifacts(config, job, taskdesc) + + digest_data = get_digest_data(config, run, taskdesc) + + if job.get("attributes", {}).get("cached_task") is not False and not taskgraph.fast: + name = taskdesc["label"].replace(f"{config.kind}-", "", 1) + taskdesc["cache"] = { + "type": CACHE_TYPE, + "name": name, + "digest-data": digest_data, + } + + # Toolchains that are used for local development need to be built on a + # level-3 branch to be installable via `mach bootstrap`. + local_toolchain = taskdesc["attributes"].get("local-toolchain") + if local_toolchain: + if taskdesc.get("run-on-projects"): + raise Exception( + "Toolchain {} used for local developement must not have" + " run-on-projects set".format(taskdesc["label"]) + ) + taskdesc["run-on-projects"] = ["integration", "release"] + + script = run.pop("script") + arguments = run.pop("arguments", []) + if local_toolchain and not attributes["toolchain-artifact"].startswith("public/"): + # Local toolchains with private artifacts are expected to have a script that + # fill a directory given as a final command line argument. That script, and the + # arguments provided, are used by the build system bootstrap code, and for the + # corresponding CI tasks, the command is wrapped with a script that creates an + # artifact based on that filled directory. + # We prefer automatic wrapping rather than manual wrapping in the yaml because + # it makes the index independent of the wrapper script, which is irrelevant. + # Also, an attribute is added for the bootstrap code to be able to easily parse + # the command. + attributes["toolchain-command"] = { + "script": script, + "arguments": list(arguments), + } + arguments.insert(0, script) + script = "private_local_toolchain.sh" + + run["using"] = "run-task" + if is_docker: + gecko_path = "workspace/build/src" + elif job["worker"]["os"] == "windows": + gecko_path = "%GECKO_PATH%" + else: + gecko_path = "$GECKO_PATH" + + if is_docker: + run["cwd"] = run["workdir"] + run["command"] = [ + "{}/taskcluster/scripts/misc/{}".format(gecko_path, script) + ] + arguments + if not is_docker: + # Don't quote the first item in the command because it purposely contains + # an environment variable that is not meant to be quoted. + if len(run["command"]) > 1: + run["command"] = run["command"][0] + " " + shell_quote(*run["command"][1:]) + else: + run["command"] = run["command"][0] + + configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) + + +toolchain_defaults = { + "tooltool-downloads": False, + "sparse-profile": "toolchain-build", +} + + +@run_job_using( + "docker-worker", + "toolchain-script", + schema=toolchain_run_schema, + defaults=toolchain_defaults, +) +def docker_worker_toolchain(config, job, taskdesc): + common_toolchain(config, job, taskdesc, is_docker=True) + + +@run_job_using( + "generic-worker", + "toolchain-script", + schema=toolchain_run_schema, + defaults=toolchain_defaults, +) +def generic_worker_toolchain(config, job, taskdesc): + common_toolchain(config, job, taskdesc, is_docker=False) diff --git a/taskcluster/gecko_taskgraph/transforms/l10n.py b/taskcluster/gecko_taskgraph/transforms/l10n.py new file mode 100644 index 0000000000..e36c70246b --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/l10n.py @@ -0,0 +1,416 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Do transforms specific to l10n kind +""" + + +import json + +from mozbuild.chunkify import chunkify +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import ( + optionally_keyed_by, + resolve_keyed_by, + taskref_or_string, +) +from taskgraph.util.taskcluster import get_artifact_prefix +from taskgraph.util.treeherder import add_suffix +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.loader.multi_dep import schema +from gecko_taskgraph.transforms.job import job_description_schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + task_name, +) +from gecko_taskgraph.util.copy_task import copy_task + + +def _by_platform(arg): + return optionally_keyed_by("build-platform", arg) + + +l10n_description_schema = schema.extend( + { + # Name for this job, inferred from the dependent job before validation + Required("name"): str, + # build-platform, inferred from dependent job before validation + Required("build-platform"): str, + # max run time of the task + Required("run-time"): _by_platform(int), + # Locales not to repack for + Required("ignore-locales"): _by_platform([str]), + # All l10n jobs use mozharness + Required("mozharness"): { + # Script to invoke for mozharness + Required("script"): _by_platform(str), + # Config files passed to the mozharness script + Required("config"): _by_platform([str]), + # Additional paths to look for mozharness configs in. These should be + # relative to the base of the source checkout + Optional("config-paths"): [str], + # Options to pass to the mozharness script + Optional("options"): _by_platform([str]), + # Action commands to provide to mozharness script + Required("actions"): _by_platform([str]), + # if true, perform a checkout of a comm-central based branch inside the + # gecko checkout + Optional("comm-checkout"): bool, + }, + # Items for the taskcluster index + Optional("index"): { + # Product to identify as in the taskcluster index + Required("product"): _by_platform(str), + # Job name to identify as in the taskcluster index + Required("job-name"): _by_platform(str), + # Type of index + Optional("type"): _by_platform(str), + }, + # Description of the localized task + Required("description"): _by_platform(str), + Optional("run-on-projects"): job_description_schema["run-on-projects"], + # worker-type to utilize + Required("worker-type"): _by_platform(str), + # File which contains the used locales + Required("locales-file"): _by_platform(str), + # Tooltool visibility required for task. + Required("tooltool"): _by_platform(Any("internal", "public")), + # Docker image required for task. We accept only in-tree images + # -- generally desktop-build or android-build -- for now. + Optional("docker-image"): _by_platform( + # an in-tree generated docker image (from `taskcluster/docker/<name>`) + {"in-tree": str}, + ), + Optional("fetches"): { + str: _by_platform([str]), + }, + # The set of secret names to which the task has access; these are prefixed + # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting + # this will enable any worker features required and set the task's scopes + # appropriately. `true` here means ['*'], all secrets. Not supported on + # Windows + Optional("secrets"): _by_platform(Any(bool, [str])), + # Information for treeherder + Required("treeherder"): { + # Platform to display the task on in treeherder + Required("platform"): _by_platform(str), + # Symbol to use + Required("symbol"): str, + # Tier this task is + Required("tier"): _by_platform(int), + }, + # Extra environment values to pass to the worker + Optional("env"): _by_platform({str: taskref_or_string}), + # Max number locales per chunk + Optional("locales-per-chunk"): _by_platform(int), + # Task deps to chain this task with, added in transforms from primary-dependency + # if this is a shippable-style build + Optional("dependencies"): {str: str}, + # Run the task when the listed files change (if present). + Optional("when"): {"files-changed": [str]}, + # passed through directly to the job description + Optional("attributes"): job_description_schema["attributes"], + Optional("extra"): job_description_schema["extra"], + # Shipping product and phase + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + } +) + +transforms = TransformSequence() + + +def parse_locales_file(locales_file, platform=None): + """Parse the passed locales file for a list of locales.""" + locales = [] + + with open(locales_file, mode="r") as f: + if locales_file.endswith("json"): + all_locales = json.load(f) + # XXX Only single locales are fetched + locales = { + locale: data["revision"] + for locale, data in all_locales.items() + if platform is None or platform in data["platforms"] + } + else: + all_locales = f.read().split() + # 'default' is the hg revision at the top of hg repo, in this context + locales = {locale: "default" for locale in all_locales} + return locales + + +def _remove_locales(locales, to_remove=None): + # ja-JP-mac is a mac-only locale, but there are no mac builds being repacked, + # so just omit it unconditionally + return { + locale: revision + for locale, revision in locales.items() + if locale not in to_remove + } + + +@transforms.add +def setup_name(config, jobs): + for job in jobs: + dep = job["primary-dependency"] + # Set the name to the same as the dep task, without kind name. + # Label will get set automatically with this kinds name. + job["name"] = job.get("name", task_name(dep)) + yield job + + +@transforms.add +def copy_in_useful_magic(config, jobs): + for job in jobs: + dep = job["primary-dependency"] + attributes = copy_attributes_from_dependent_job(dep) + attributes.update(job.get("attributes", {})) + # build-platform is needed on `job` for by-build-platform + job["build-platform"] = attributes.get("build_platform") + job["attributes"] = attributes + yield job + + +transforms.add_validate(l10n_description_schema) + + +@transforms.add +def setup_shippable_dependency(config, jobs): + """Sets up a task dependency to the signing job this relates to""" + for job in jobs: + job["dependencies"] = {"build": job["dependent-tasks"]["build"].label} + if job["attributes"]["build_platform"].startswith("win") or job["attributes"][ + "build_platform" + ].startswith("linux"): + job["dependencies"].update( + { + "build-signing": job["dependent-tasks"]["build-signing"].label, + } + ) + if job["attributes"]["build_platform"].startswith("macosx"): + job["dependencies"].update( + {"repackage": job["dependent-tasks"]["repackage"].label} + ) + yield job + + +@transforms.add +def handle_keyed_by(config, jobs): + """Resolve fields that can be keyed by platform, etc.""" + fields = [ + "locales-file", + "locales-per-chunk", + "worker-type", + "description", + "run-time", + "docker-image", + "secrets", + "fetches.toolchain", + "fetches.fetch", + "tooltool", + "env", + "ignore-locales", + "mozharness.config", + "mozharness.options", + "mozharness.actions", + "mozharness.script", + "treeherder.tier", + "treeherder.platform", + "index.type", + "index.product", + "index.job-name", + "when.files-changed", + ] + for job in jobs: + job = copy_task(job) # don't overwrite dict values here + for field in fields: + resolve_keyed_by(item=job, field=field, item_name=job["name"]) + yield job + + +@transforms.add +def handle_artifact_prefix(config, jobs): + """Resolve ``artifact_prefix`` in env vars""" + for job in jobs: + artifact_prefix = get_artifact_prefix(job) + for k1, v1 in job.get("env", {}).items(): + if isinstance(v1, str): + job["env"][k1] = v1.format(artifact_prefix=artifact_prefix) + elif isinstance(v1, dict): + for k2, v2 in v1.items(): + job["env"][k1][k2] = v2.format(artifact_prefix=artifact_prefix) + yield job + + +@transforms.add +def all_locales_attribute(config, jobs): + for job in jobs: + locales_platform = job["attributes"]["build_platform"].replace("-shippable", "") + locales_platform = locales_platform.replace("-pgo", "") + locales_with_changesets = parse_locales_file( + job["locales-file"], platform=locales_platform + ) + locales_with_changesets = _remove_locales( + locales_with_changesets, to_remove=job["ignore-locales"] + ) + + locales = sorted(locales_with_changesets.keys()) + attributes = job.setdefault("attributes", {}) + attributes["all_locales"] = locales + attributes["all_locales_with_changesets"] = locales_with_changesets + if job.get("shipping-product"): + attributes["shipping_product"] = job["shipping-product"] + yield job + + +@transforms.add +def chunk_locales(config, jobs): + """Utilizes chunking for l10n stuff""" + for job in jobs: + locales_per_chunk = job.get("locales-per-chunk") + locales_with_changesets = job["attributes"]["all_locales_with_changesets"] + if locales_per_chunk: + chunks, remainder = divmod(len(locales_with_changesets), locales_per_chunk) + if remainder: + chunks = int(chunks + 1) + for this_chunk in range(1, chunks + 1): + chunked = copy_task(job) + chunked["name"] = chunked["name"].replace("/", f"-{this_chunk}/", 1) + chunked["mozharness"]["options"] = chunked["mozharness"].get( + "options", [] + ) + # chunkify doesn't work with dicts + locales_with_changesets_as_list = sorted( + locales_with_changesets.items() + ) + chunked_locales = chunkify( + locales_with_changesets_as_list, this_chunk, chunks + ) + chunked["mozharness"]["options"].extend( + [ + f"locale={locale}:{changeset}" + for locale, changeset in chunked_locales + ] + ) + chunked["attributes"]["l10n_chunk"] = str(this_chunk) + # strip revision + chunked["attributes"]["chunk_locales"] = [ + locale for locale, _ in chunked_locales + ] + + # add the chunk number to the TH symbol + chunked["treeherder"]["symbol"] = add_suffix( + chunked["treeherder"]["symbol"], this_chunk + ) + yield chunked + else: + job["mozharness"]["options"] = job["mozharness"].get("options", []) + job["mozharness"]["options"].extend( + [ + f"locale={locale}:{changeset}" + for locale, changeset in sorted(locales_with_changesets.items()) + ] + ) + yield job + + +transforms.add_validate(l10n_description_schema) + + +@transforms.add +def stub_installer(config, jobs): + for job in jobs: + job.setdefault("attributes", {}) + job.setdefault("env", {}) + if job["attributes"].get("stub-installer"): + job["env"].update({"USE_STUB_INSTALLER": "1"}) + yield job + + +@transforms.add +def set_extra_config(config, jobs): + for job in jobs: + job["mozharness"].setdefault("extra-config", {})["branch"] = config.params[ + "project" + ] + if "update-channel" in job["attributes"]: + job["mozharness"]["extra-config"]["update_channel"] = job["attributes"][ + "update-channel" + ] + yield job + + +@transforms.add +def make_job_description(config, jobs): + for job in jobs: + job["mozharness"].update( + { + "using": "mozharness", + "job-script": "taskcluster/scripts/builder/build-l10n.sh", + "secrets": job.get("secrets", False), + } + ) + job_description = { + "name": job["name"], + "worker-type": job["worker-type"], + "description": job["description"], + "run": job["mozharness"], + "attributes": job["attributes"], + "treeherder": { + "kind": "build", + "tier": job["treeherder"]["tier"], + "symbol": job["treeherder"]["symbol"], + "platform": job["treeherder"]["platform"], + }, + "run-on-projects": job.get("run-on-projects") + if job.get("run-on-projects") + else [], + } + if job.get("extra"): + job_description["extra"] = job["extra"] + + job_description["run"]["tooltool-downloads"] = job["tooltool"] + + job_description["worker"] = { + "max-run-time": job["run-time"], + "chain-of-trust": True, + } + if job["worker-type"] == "b-win2012": + job_description["worker"]["os"] = "windows" + job_description["run"]["use-simple-package"] = False + job_description["run"]["use-magic-mh-args"] = False + + if job.get("docker-image"): + job_description["worker"]["docker-image"] = job["docker-image"] + + if job.get("fetches"): + job_description["fetches"] = job["fetches"] + + if job.get("index"): + job_description["index"] = { + "product": job["index"]["product"], + "job-name": job["index"]["job-name"], + "type": job["index"].get("type", "generic"), + } + + if job.get("dependencies"): + job_description["dependencies"] = job["dependencies"] + if job.get("env"): + job_description["worker"]["env"] = job["env"] + if job.get("when", {}).get("files-changed"): + job_description.setdefault("when", {}) + job_description["when"]["files-changed"] = [job["locales-file"]] + job[ + "when" + ]["files-changed"] + + if "shipping-phase" in job: + job_description["shipping-phase"] = job["shipping-phase"] + + if "shipping-product" in job: + job_description["shipping-product"] = job["shipping-product"] + + yield job_description diff --git a/taskcluster/gecko_taskgraph/transforms/mac_dummy.py b/taskcluster/gecko_taskgraph/transforms/mac_dummy.py new file mode 100644 index 0000000000..f134ee2765 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/mac_dummy.py @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Add dependencies to dummy macosx64 tasks. +""" + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def add_dependencies(config, jobs): + for job in jobs: + dependencies = {} + + platform = job.get("attributes", {}).get("build_platform") + if not platform: + continue + arm = platform.replace("macosx64", "macosx64-aarch64") + intel = platform.replace("macosx64", "macosx64-x64") + for dep_task in config.kind_dependencies_tasks.values(): + # Weed out unwanted tasks. + if dep_task.attributes.get("build_platform"): + if dep_task.attributes["build_platform"] not in (platform, arm, intel): + continue + # Add matching tasks to deps + dependencies[dep_task.label] = dep_task.label + # Pick one task to copy run-on-projects from + if ( + dep_task.kind == "build" + and dep_task.attributes["build_platform"] == platform + ): + job["run-on-projects"] = dep_task.attributes.get("run_on_projects") + + job.setdefault("dependencies", {}).update(dependencies) + job["if-dependencies"] = list(dependencies) + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/mac_notarization.py b/taskcluster/gecko_taskgraph/transforms/mac_notarization.py new file mode 100644 index 0000000000..5591022e1b --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/mac_notarization.py @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform mac notarization tasks +""" + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def only_level_3_notarization(config, jobs): + """Filter out any notarization jobs that are not level 3""" + for job in jobs: + if "notarization" in config.kind and int(config.params["level"]) != 3: + continue + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/mar_signing.py b/taskcluster/gecko_taskgraph/transforms/mar_signing.py new file mode 100644 index 0000000000..d923721f45 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/mar_signing.py @@ -0,0 +1,140 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the {partials,mar}-signing task into an actual task description. +""" + +import logging +import os + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.taskcluster import get_artifact_prefix +from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol + +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + sorted_unique_list, +) +from gecko_taskgraph.util.partials import get_partials_artifacts_from_params +from gecko_taskgraph.util.scriptworker import get_signing_cert_scope_per_platform + +logger = logging.getLogger(__name__) + +SIGNING_FORMATS = { + "mar-signing-autograph-stage": { + "target.complete.mar": ["autograph_stage_mar384"], + }, + "default": { + "target.complete.mar": ["autograph_hash_only_mar384"], + }, +} + +transforms = TransformSequence() + + +def generate_partials_artifacts(job, release_history, platform, locale=None): + artifact_prefix = get_artifact_prefix(job) + if locale: + artifact_prefix = f"{artifact_prefix}/{locale}" + else: + locale = "en-US" + + artifacts = get_partials_artifacts_from_params(release_history, platform, locale) + + upstream_artifacts = [ + { + "taskId": {"task-reference": "<partials>"}, + "taskType": "partials", + "paths": [f"{artifact_prefix}/{path}" for path, version in artifacts], + "formats": ["autograph_hash_only_mar384"], + } + ] + + return upstream_artifacts + + +def generate_complete_artifacts(job, kind): + upstream_artifacts = [] + if kind not in SIGNING_FORMATS: + kind = "default" + for artifact in job.attributes["release_artifacts"]: + basename = os.path.basename(artifact) + if basename in SIGNING_FORMATS[kind]: + upstream_artifacts.append( + { + "taskId": {"task-reference": f"<{job.kind}>"}, + "taskType": "build", + "paths": [artifact], + "formats": SIGNING_FORMATS[kind][basename], + } + ) + + return upstream_artifacts + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + locale = dep_job.attributes.get("locale") + + treeherder = inherit_treeherder_from_dep(job, dep_job) + treeherder.setdefault( + "symbol", join_symbol(job.get("treeherder-group", "ms"), locale or "N") + ) + + label = job.get("label", f"{config.kind}-{dep_job.label}") + + dependencies = {dep_job.kind: dep_job.label} + signing_dependencies = dep_job.dependencies + # This is so we get the build task etc in our dependencies to + # have better beetmover support. + dependencies.update(signing_dependencies) + + attributes = copy_attributes_from_dependent_job(dep_job) + attributes["required_signoffs"] = sorted_unique_list( + attributes.get("required_signoffs", []), job.pop("required_signoffs") + ) + attributes["shipping_phase"] = job["shipping-phase"] + if locale: + attributes["locale"] = locale + + build_platform = attributes.get("build_platform") + if config.kind == "partials-signing": + upstream_artifacts = generate_partials_artifacts( + dep_job, config.params["release_history"], build_platform, locale + ) + else: + upstream_artifacts = generate_complete_artifacts(dep_job, config.kind) + + is_shippable = job.get( + "shippable", dep_job.attributes.get("shippable") # First check current job + ) # Then dep job for 'shippable' + signing_cert_scope = get_signing_cert_scope_per_platform( + build_platform, is_shippable, config + ) + + scopes = [signing_cert_scope] + + task = { + "label": label, + "description": "{} {}".format( + dep_job.description, job["description-suffix"] + ), + "worker-type": job.get("worker-type", "linux-signing"), + "worker": { + "implementation": "scriptworker-signing", + "upstream-artifacts": upstream_artifacts, + "max-run-time": 3600, + }, + "dependencies": dependencies, + "attributes": attributes, + "scopes": scopes, + "run-on-projects": job.get( + "run-on-projects", dep_job.attributes.get("run_on_projects") + ), + "treeherder": treeherder, + } + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/maybe_release.py b/taskcluster/gecko_taskgraph/transforms/maybe_release.py new file mode 100644 index 0000000000..08a066001a --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/maybe_release.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.util.attributes import release_level + +transforms = TransformSequence() + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + for key in ["worker-type", "scopes"]: + resolve_keyed_by( + job, + key, + item_name=job["name"], + **{"release-level": release_level(config.params["project"])} + ) + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/merge_automation.py b/taskcluster/gecko_taskgraph/transforms/merge_automation.py new file mode 100644 index 0000000000..ca5f3b6bde --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/merge_automation.py @@ -0,0 +1,81 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the update generation task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def handle_keyed_by(config, tasks): + """Resolve fields that can be keyed by platform, etc.""" + if "merge_config" not in config.params: + return + merge_config = config.params["merge_config"] + fields = [ + "worker.push", + "worker-type", + "worker.l10n-bump-info", + "worker.source-repo", + ] + for task in tasks: + for field in fields: + resolve_keyed_by( + task, + field, + item_name=task["name"], + **{ + "project": config.params["project"], + "release-type": config.params["release_type"], + "behavior": merge_config["behavior"], + } + ) + yield task + + +@transforms.add +def update_labels(config, tasks): + for task in tasks: + merge_config = config.params["merge_config"] + task["label"] = "merge-{}".format(merge_config["behavior"]) + treeherder = task.get("treeherder", {}) + treeherder["symbol"] = "Rel({})".format(merge_config["behavior"]) + task["treeherder"] = treeherder + yield task + + +@transforms.add +def add_payload_config(config, tasks): + for task in tasks: + if "merge_config" not in config.params: + break + merge_config = config.params["merge_config"] + worker = task["worker"] + worker["merge-info"] = config.graph_config["merge-automation"]["behaviors"][ + merge_config["behavior"] + ] + + if "l10n-bump-info" in worker and worker["l10n-bump-info"] is None: + del worker["l10n-bump-info"] + + # Override defaults, useful for testing. + for field in [ + "from-repo", + "from-branch", + "to-repo", + "to-branch", + "fetch-version-from", + ]: + if merge_config.get(field): + worker["merge-info"][field] = merge_config[field] + + worker["force-dry-run"] = merge_config["force-dry-run"] + worker["ssh-user"] = merge_config.get("ssh-user-alias", "merge_user") + if merge_config.get("push"): + worker["push"] = merge_config["push"] + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/name_sanity.py b/taskcluster/gecko_taskgraph/transforms/name_sanity.py new file mode 100644 index 0000000000..856f4f82be --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/name_sanity.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Generate labels for tasks without names, consistently. +Uses attributes from `primary-dependency`. +""" + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def make_label(config, jobs): + """Generate a sane label for a new task constructed from a dependency + Using attributes from the dependent job and the current task kind""" + for job in jobs: + dep_job = job["primary-dependency"] + attr = dep_job.attributes.get + + if attr("locale", job.get("locale")): + template = "{kind}-{locale}-{build_platform}/{build_type}" + elif attr("l10n_chunk"): + template = "{kind}-{build_platform}-{l10n_chunk}/{build_type}" + elif config.kind.startswith("release-eme-free") or config.kind.startswith( + "release-partner-repack" + ): + suffix = job.get("extra", {}).get("repack_suffix", None) or job.get( + "extra", {} + ).get("repack_id", None) + template = "{kind}-{build_platform}" + if suffix: + template += "-{}".format(suffix.replace("/", "-")) + else: + template = "{kind}-{build_platform}/{build_type}" + job["label"] = template.format( + kind=config.kind, + build_platform=attr("build_platform"), + build_type=attr("build_type"), + locale=attr("locale", job.get("locale", "")), # Locale can be absent + l10n_chunk=attr("l10n_chunk", ""), # Can be empty + ) + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/openh264.py b/taskcluster/gecko_taskgraph/transforms/openh264.py new file mode 100644 index 0000000000..f41215d20b --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/openh264.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +This transform is used to help populate mozharness options for openh264 jobs +""" + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def set_mh_options(config, jobs): + """ + This transform sets the 'openh264_rev' attribute. + """ + for job in jobs: + repo = job.pop("repo") + rev = job.pop("revision") + attributes = job.setdefault("attributes", {}) + attributes["openh264_rev"] = rev + run = job.setdefault("run", {}) + options = run.setdefault("options", []) + options.extend([f"repo={repo}", f"rev={rev}"]) + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/openh264_signing.py b/taskcluster/gecko_taskgraph/transforms/openh264_signing.py new file mode 100644 index 0000000000..dfffc7cc17 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/openh264_signing.py @@ -0,0 +1,109 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the repackage signing task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import inherit_treeherder_from_dep +from voluptuous import Optional + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.scriptworker import get_signing_cert_scope_per_platform + +transforms = TransformSequence() + +signing_description_schema = schema.extend( + { + Optional("label"): str, + Optional("extra"): object, + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + } +) + +transforms.add_validate(signing_description_schema) + + +@transforms.add +def make_signing_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = dep_job.attributes + build_platform = dep_job.attributes.get("build_platform") + is_nightly = True # cert_scope_per_platform uses this to choose the right cert + + description = ( + "Signing of OpenH264 Binaries for '" + "{build_platform}/{build_type}'".format( + build_platform=attributes.get("build_platform"), + build_type=attributes.get("build_type"), + ) + ) + + # we have a genuine repackage job as our parent + dependencies = {"openh264": dep_job.label} + + my_attributes = copy_attributes_from_dependent_job(dep_job) + + signing_cert_scope = get_signing_cert_scope_per_platform( + build_platform, is_nightly, config + ) + + scopes = [signing_cert_scope] + worker_type = "linux-signing" + worker = { + "implementation": "scriptworker-signing", + "max-run-time": 3600, + } + rev = attributes["openh264_rev"] + upstream_artifact = { + "taskId": {"task-reference": "<openh264>"}, + "taskType": "build", + } + + if "win" in build_platform: + # job['primary-dependency'].task['payload']['command'] + upstream_artifact["formats"] = ["autograph_authenticode_sha2"] + elif "mac" in build_platform: + upstream_artifact["formats"] = ["mac_single_file"] + upstream_artifact["singleFileGlobs"] = ["libgmpopenh264.dylib"] + worker_type = "mac-signing" + worker["mac-behavior"] = "mac_notarize_single_file" + else: + upstream_artifact["formats"] = ["autograph_gpg"] + + upstream_artifact["paths"] = [ + f"private/openh264/openh264-{build_platform}-{rev}.zip", + ] + worker["upstream-artifacts"] = [upstream_artifact] + + treeherder = inherit_treeherder_from_dep(job, dep_job) + treeherder.setdefault( + "symbol", + _generate_treeherder_symbol( + dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol") + ), + ) + + task = { + "label": job["label"], + "description": description, + "worker-type": worker_type, + "worker": worker, + "scopes": scopes, + "dependencies": dependencies, + "attributes": my_attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "treeherder": treeherder, + } + + yield task + + +def _generate_treeherder_symbol(build_symbol): + symbol = build_symbol + "s" + return symbol diff --git a/taskcluster/gecko_taskgraph/transforms/partials.py b/taskcluster/gecko_taskgraph/transforms/partials.py new file mode 100644 index 0000000000..267739cfc3 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/partials.py @@ -0,0 +1,172 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the partials task into an actual task description. +""" + +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.taskcluster import get_artifact_prefix +from taskgraph.util.treeherder import inherit_treeherder_from_dep + +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + release_level, +) +from gecko_taskgraph.util.partials import get_builds +from gecko_taskgraph.util.platforms import architecture + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +def _generate_task_output_files(job, filenames, locale=None): + locale_output_path = f"{locale}/" if locale else "" + artifact_prefix = get_artifact_prefix(job) + + data = list() + for filename in filenames: + data.append( + { + "type": "file", + "path": f"/home/worker/artifacts/{filename}", + "name": f"{artifact_prefix}/{locale_output_path}{filename}", + } + ) + data.append( + { + "type": "file", + "path": "/home/worker/artifacts/manifest.json", + "name": f"{artifact_prefix}/{locale_output_path}manifest.json", + } + ) + return data + + +def identify_desired_signing_keys(project, product): + if project in ["mozilla-central", "comm-central", "oak"]: + return "nightly" + if project == "mozilla-beta": + if product == "devedition": + return "nightly" + return "release" + if ( + project in ["mozilla-release", "comm-beta"] + or project.startswith("mozilla-esr") + or project.startswith("comm-esr") + ): + return "release" + return "dep1" + + +@transforms.add +def make_task_description(config, jobs): + # If no balrog release history, then don't generate partials + if not config.params.get("release_history"): + return + for job in jobs: + dep_job = job["primary-dependency"] + + treeherder = inherit_treeherder_from_dep(job, dep_job) + treeherder.setdefault("symbol", "p(N)") + + label = job.get("label", f"partials-{dep_job.label}") + + dependencies = {dep_job.kind: dep_job.label} + + attributes = copy_attributes_from_dependent_job(dep_job) + locale = dep_job.attributes.get("locale") + if locale: + attributes["locale"] = locale + treeherder["symbol"] = f"p({locale})" + attributes["shipping_phase"] = job["shipping-phase"] + + build_locale = locale or "en-US" + + build_platform = attributes["build_platform"] + builds = get_builds( + config.params["release_history"], build_platform, build_locale + ) + + # If the list is empty there's no available history for this platform + # and locale combination, so we can't build any partials. + if not builds: + continue + + extra = {"funsize": {"partials": list()}} + update_number = 1 + + locale_suffix = "" + if locale: + locale_suffix = f"{locale}/" + artifact_path = "<{}/{}/{}target.complete.mar>".format( + dep_job.kind, + get_artifact_prefix(dep_job), + locale_suffix, + ) + for build in sorted(builds): + partial_info = { + "locale": build_locale, + "from_mar": builds[build]["mar_url"], + "to_mar": {"artifact-reference": artifact_path}, + "branch": config.params["project"], + "update_number": update_number, + "dest_mar": build, + } + if "product" in builds[build]: + partial_info["product"] = builds[build]["product"] + if "previousVersion" in builds[build]: + partial_info["previousVersion"] = builds[build]["previousVersion"] + if "previousBuildNumber" in builds[build]: + partial_info["previousBuildNumber"] = builds[build][ + "previousBuildNumber" + ] + extra["funsize"]["partials"].append(partial_info) + update_number += 1 + + level = config.params["level"] + + worker = { + "artifacts": _generate_task_output_files(dep_job, builds.keys(), locale), + "implementation": "docker-worker", + "docker-image": {"in-tree": "funsize-update-generator"}, + "os": "linux", + "max-run-time": 3600 if "asan" in dep_job.label else 1800, + "chain-of-trust": True, + "taskcluster-proxy": True, + "env": { + "SIGNING_CERT": identify_desired_signing_keys( + config.params["project"], config.params["release_product"] + ), + "EXTRA_PARAMS": f"--arch={architecture(build_platform)}", + "MAR_CHANNEL_ID": attributes["mar-channel-id"], + }, + } + if release_level(config.params["project"]) == "staging": + worker["env"]["FUNSIZE_ALLOW_STAGING_PREFIXES"] = "true" + + task = { + "label": label, + "description": f"{dep_job.description} Partials", + "worker-type": "b-linux-gcp", + "dependencies": dependencies, + "scopes": [], + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "treeherder": treeherder, + "extra": extra, + "worker": worker, + } + + # We only want caching on linux/windows due to bug 1436977 + if int(level) == 3 and any( + [build_platform.startswith(prefix) for prefix in ["linux", "win"]] + ): + task["scopes"].append( + "auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/" + ) + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/partner_attribution.py b/taskcluster/gecko_taskgraph/transforms/partner_attribution.py new file mode 100644 index 0000000000..0bd5e0d141 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/partner_attribution.py @@ -0,0 +1,129 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the partner attribution task into an actual task description. +""" + + +import json +import logging +from collections import defaultdict + +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.partners import ( + apply_partner_priority, + check_if_partners_enabled, + generate_attribution_code, + get_partner_config_by_kind, +) + +log = logging.getLogger(__name__) + +transforms = TransformSequence() +transforms.add(check_if_partners_enabled) +transforms.add(apply_partner_priority) + + +@transforms.add +def add_command_arguments(config, tasks): + enabled_partners = config.params.get("release_partners") + dependencies = {} + fetches = defaultdict(set) + attributions = [] + release_artifacts = [] + attribution_config = get_partner_config_by_kind(config, config.kind) + + for partner_config in attribution_config.get("configs", []): + # we might only be interested in a subset of all partners, eg for a respin + if enabled_partners and partner_config["campaign"] not in enabled_partners: + continue + attribution_code = generate_attribution_code( + attribution_config["defaults"], partner_config + ) + for platform in partner_config["platforms"]: + stage_platform = platform.replace("-shippable", "") + for locale in partner_config["locales"]: + # find the upstream, throw away locales we don't have, somehow. Skip ? + if locale == "en-US": + upstream_label = "repackage-signing-{platform}/opt".format( + platform=platform + ) + upstream_artifact = "target.installer.exe" + else: + upstream_label = ( + "repackage-signing-l10n-{locale}-{platform}/opt".format( + locale=locale, platform=platform + ) + ) + upstream_artifact = "{locale}/target.installer.exe".format( + locale=locale + ) + if upstream_label not in config.kind_dependencies_tasks: + raise Exception(f"Can't find upstream task for {platform} {locale}") + upstream = config.kind_dependencies_tasks[upstream_label] + + # set the dependencies to just what we need rather than all of l10n + dependencies.update({upstream.label: upstream.label}) + + fetches[upstream_label].add((upstream_artifact, stage_platform, locale)) + + artifact_part = "{platform}/{locale}/target.installer.exe".format( + platform=stage_platform, locale=locale + ) + artifact = ( + "releng/partner/{partner}/{sub_partner}/{artifact_part}".format( + partner=partner_config["campaign"], + sub_partner=partner_config["content"], + artifact_part=artifact_part, + ) + ) + # config for script + # TODO - generalise input & output ?? + # add releng/partner prefix via get_artifact_prefix..() + attributions.append( + { + "input": f"/builds/worker/fetches/{artifact_part}", + "output": f"/builds/worker/artifacts/{artifact}", + "attribution": attribution_code, + } + ) + release_artifacts.append(artifact) + + # bail-out early if we don't have any attributions to do + if not attributions: + return + + for task in tasks: + worker = task.get("worker", {}) + worker["chain-of-trust"] = True + + task.setdefault("dependencies", {}).update(dependencies) + task.setdefault("fetches", {}) + for upstream_label, upstream_artifacts in fetches.items(): + task["fetches"][upstream_label] = [ + { + "artifact": upstream_artifact, + "dest": "{platform}/{locale}".format( + platform=platform, locale=locale + ), + "extract": False, + "verify-hash": True, + } + for upstream_artifact, platform, locale in upstream_artifacts + ] + worker.setdefault("env", {})["ATTRIBUTION_CONFIG"] = json.dumps( + attributions, sort_keys=True + ) + worker["artifacts"] = [ + { + "name": "releng/partner", + "path": "/builds/worker/artifacts/releng/partner", + "type": "directory", + } + ] + task.setdefault("attributes", {})["release_artifacts"] = release_artifacts + task["label"] = config.kind + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/partner_attribution_beetmover.py b/taskcluster/gecko_taskgraph/transforms/partner_attribution_beetmover.py new file mode 100644 index 0000000000..b2c435dd81 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/partner_attribution_beetmover.py @@ -0,0 +1,202 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +from collections import defaultdict +from copy import deepcopy + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from taskgraph.util.taskcluster import get_artifact_prefix +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.beetmover import craft_release_properties +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + release_level, +) +from gecko_taskgraph.util.partners import ( + apply_partner_priority, + get_partner_config_by_kind, +) +from gecko_taskgraph.util.scriptworker import ( + add_scope_prefix, + get_beetmover_bucket_scope, +) + +beetmover_description_schema = schema.extend( + { + # depname is used in taskref's to identify the taskID of the unsigned things + Required("depname", default="build"): str, + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Optional("label"): str, + Required("partner-bucket-scope"): optionally_keyed_by("release-level", str), + Required("partner-public-path"): Any(None, str), + Required("partner-private-path"): Any(None, str), + Optional("extra"): object, + Required("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("priority"): task_description_schema["priority"], + } +) + +transforms = TransformSequence() +transforms.add_validate(beetmover_description_schema) +transforms.add(apply_partner_priority) + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "partner-bucket-scope", + item_name=job["label"], + **{"release-level": release_level(config.params["project"])}, + ) + yield job + + +@transforms.add +def split_public_and_private(config, jobs): + # we need to separate private vs public destinations because beetmover supports one + # in a single task. Only use a single task for each type though. + partner_config = get_partner_config_by_kind(config, config.kind) + for job in jobs: + upstream_artifacts = job["primary-dependency"].attributes.get( + "release_artifacts" + ) + attribution_task_ref = "<{}>".format(job["primary-dependency"].label) + prefix = get_artifact_prefix(job["primary-dependency"]) + artifacts = defaultdict(list) + for artifact in upstream_artifacts: + partner, sub_partner, platform, locale, _ = artifact.replace( + prefix + "/", "" + ).split("/", 4) + destination = "private" + this_config = [ + p + for p in partner_config["configs"] + if (p["campaign"] == partner and p["content"] == sub_partner) + ] + if this_config[0].get("upload_to_candidates"): + destination = "public" + artifacts[destination].append( + (artifact, partner, sub_partner, platform, locale) + ) + + action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner") + public_bucket_scope = get_beetmover_bucket_scope(config) + partner_bucket_scope = add_scope_prefix(config, job["partner-bucket-scope"]) + repl_dict = { + "build_number": config.params["build_number"], + "release_partner_build_number": config.params[ + "release_partner_build_number" + ], + "version": config.params["version"], + "partner": "{partner}", # we'll replace these later, per artifact + "subpartner": "{subpartner}", + "platform": "{platform}", + "locale": "{locale}", + } + for destination, destination_artifacts in artifacts.items(): + this_job = deepcopy(job) + + if destination == "public": + this_job["scopes"] = [public_bucket_scope, action_scope] + this_job["partner_public"] = True + else: + this_job["scopes"] = [partner_bucket_scope, action_scope] + this_job["partner_public"] = False + + partner_path_key = f"partner-{destination}-path" + partner_path = this_job[partner_path_key].format(**repl_dict) + this_job.setdefault("worker", {})[ + "upstream-artifacts" + ] = generate_upstream_artifacts( + attribution_task_ref, destination_artifacts, partner_path + ) + + yield this_job + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + + attributes = dep_job.attributes + build_platform = attributes.get("build_platform") + if not build_platform: + raise Exception("Cannot find build platform!") + + label = config.kind + description = "Beetmover for partner attribution" + if job["partner_public"]: + label = f"{label}-public" + description = f"{description} public" + else: + label = f"{label}-private" + description = f"{description} private" + attributes = copy_attributes_from_dependent_job(dep_job) + + task = { + "label": label, + "description": description, + "dependencies": {dep_job.kind: dep_job.label}, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "shipping-phase": job["shipping-phase"], + "shipping-product": job.get("shipping-product"), + "partner_public": job["partner_public"], + "worker": job["worker"], + "scopes": job["scopes"], + } + # we may have reduced the priority for partner jobs, otherwise task.py will set it + if job.get("priority"): + task["priority"] = job["priority"] + + yield task + + +def generate_upstream_artifacts(attribution_task, artifacts, partner_path): + upstream_artifacts = [] + for artifact, partner, subpartner, platform, locale in artifacts: + upstream_artifacts.append( + { + "taskId": {"task-reference": attribution_task}, + "taskType": "repackage", + "paths": [artifact], + "locale": partner_path.format( + partner=partner, + subpartner=subpartner, + platform=platform, + locale=locale, + ), + } + ) + + if not upstream_artifacts: + raise Exception("Couldn't find any upstream artifacts.") + + return upstream_artifacts + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + job["worker-type"] = "beetmover" + worker = { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, job), + "partner-public": job["partner_public"], + } + job["worker"].update(worker) + del job["partner_public"] + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/partner_repack.py b/taskcluster/gecko_taskgraph/transforms/partner_repack.py new file mode 100644 index 0000000000..d164c10a59 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/partner_repack.py @@ -0,0 +1,136 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the partner repack task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.partners import ( + apply_partner_priority, + check_if_partners_enabled, + get_partner_config_by_kind, + get_partner_url_config, + get_repack_ids_by_platform, +) +from gecko_taskgraph.util.scriptworker import get_release_config + +transforms = TransformSequence() +transforms.add(check_if_partners_enabled) +transforms.add(apply_partner_priority) + + +@transforms.add +def skip_unnecessary_platforms(config, tasks): + for task in tasks: + if config.kind == "release-partner-repack": + platform = task["attributes"]["build_platform"] + repack_ids = get_repack_ids_by_platform(config, platform) + if not repack_ids: + continue + yield task + + +@transforms.add +def remove_mac_dependency(config, tasks): + """Remove mac dependency depending on current level + to accomodate for mac notarization not running on level 1 + """ + level = int(config.params.get("level", 0)) + for task in tasks: + if "macosx" not in task["attributes"]["build_platform"]: + yield task + continue + skipped_kind = "mac-signing" if level == 3 else "mac-notarization" + for dep_label in list(task["dependencies"].keys()): + if skipped_kind in dep_label: + del task["dependencies"][dep_label] + yield task + + +@transforms.add +def populate_repack_manifests_url(config, tasks): + for task in tasks: + partner_url_config = get_partner_url_config(config.params, config.graph_config) + + for k in partner_url_config: + if config.kind.startswith(k): + task["worker"].setdefault("env", {})[ + "REPACK_MANIFESTS_URL" + ] = partner_url_config[k] + break + else: + raise Exception("Can't find partner REPACK_MANIFESTS_URL") + + for property in ("limit-locales",): + property = f"extra.{property}" + resolve_keyed_by( + task, + property, + property, + **{"release-level": release_level(config.params["project"])}, + ) + + if task["worker"]["env"]["REPACK_MANIFESTS_URL"].startswith("git@"): + task.setdefault("scopes", []).append( + "secrets:get:project/releng/gecko/build/level-{level}/partner-github-ssh".format( + **config.params + ) + ) + + yield task + + +@transforms.add +def make_label(config, tasks): + for task in tasks: + task["label"] = "{}-{}".format(config.kind, task["name"]) + yield task + + +@transforms.add +def add_command_arguments(config, tasks): + release_config = get_release_config(config) + + # staging releases - pass reduced set of locales to the repacking script + all_locales = set() + partner_config = get_partner_config_by_kind(config, config.kind) + for partner in partner_config.values(): + for sub_partner in partner.values(): + all_locales.update(sub_partner.get("locales", [])) + + for task in tasks: + # add the MOZHARNESS_OPTIONS, eg version=61.0, build-number=1, platform=win64 + if not task["attributes"]["build_platform"].endswith("-shippable"): + raise Exception( + "Unexpected partner repack platform: {}".format( + task["attributes"]["build_platform"], + ), + ) + platform = task["attributes"]["build_platform"].partition("-shippable")[0] + task["run"]["options"] = [ + "version={}".format(release_config["version"]), + "build-number={}".format(release_config["build_number"]), + f"platform={platform}", + ] + if task["extra"]["limit-locales"]: + for locale in all_locales: + task["run"]["options"].append(f"limit-locale={locale}") + if "partner" in config.kind and config.params["release_partners"]: + for partner in config.params["release_partners"]: + task["run"]["options"].append(f"partner={partner}") + + # The upstream taskIds are stored a special environment variable, because we want to use + # task-reference's to resolve dependencies, but the string handling of MOZHARNESS_OPTIONS + # blocks that. It's space-separated string of ids in the end. + task["worker"]["env"]["UPSTREAM_TASKIDS"] = { + "task-reference": " ".join([f"<{dep}>" for dep in task["dependencies"]]) + } + + # Forward the release type for bouncer product construction + task["worker"]["env"]["RELEASE_TYPE"] = config.params["release_type"] + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/partner_signing.py b/taskcluster/gecko_taskgraph/transforms/partner_signing.py new file mode 100644 index 0000000000..c9b6b7a8cc --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/partner_signing.py @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the signing task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.partners import get_partner_config_by_kind +from gecko_taskgraph.util.signed_artifacts import ( + generate_specifications_of_artifacts_to_sign, +) + +transforms = TransformSequence() + + +@transforms.add +def set_mac_label(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + if "mac-notarization" in config.kind: + default_label = dep_job.label.replace("mac-signing", "mac-notarization") + job.setdefault("label", default_label) + assert job["label"] != dep_job.label, "Unable to determine label for {}".format( + config.kind + ) + yield job + + +@transforms.add +def define_upstream_artifacts(config, jobs): + partner_configs = get_partner_config_by_kind(config, config.kind) + if not partner_configs: + return + + for job in jobs: + dep_job = job["primary-dependency"] + job["depname"] = dep_job.label + job["attributes"] = copy_attributes_from_dependent_job(dep_job) + + repack_ids = job["extra"]["repack_ids"] + artifacts_specifications = generate_specifications_of_artifacts_to_sign( + config, + job, + keep_locale_template=True, + kind=config.kind, + ) + task_type = "build" + if "notarization" in job["depname"] or "mac-signing" in job["depname"]: + task_type = "scriptworker" + job["upstream-artifacts"] = [ + { + "taskId": {"task-reference": f"<{dep_job.kind}>"}, + "taskType": task_type, + "paths": [ + path_template.format(locale=repack_id) + for path_template in spec["artifacts"] + for repack_id in repack_ids + ], + "formats": spec["formats"], + } + for spec in artifacts_specifications + ] + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/per_platform_dummy.py b/taskcluster/gecko_taskgraph/transforms/per_platform_dummy.py new file mode 100644 index 0000000000..5237f4f281 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/per_platform_dummy.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the repackage task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job + +transforms = TransformSequence() + + +@transforms.add +def one_task_per_product_and_platform(config, jobs): + unique_products_and_platforms = set() + for job in jobs: + dep_task = job["primary-dependency"] + if "primary-dependency" in job: + del job["primary-dependency"] + product = dep_task.attributes.get("shipping_product") + platform = dep_task.attributes.get("build_platform") + if (product, platform) not in unique_products_and_platforms: + attr_denylist = ("l10n_chunk", "locale", "artifact_map", "artifact_prefix") + attributes = copy_attributes_from_dependent_job( + dep_task, denylist=attr_denylist + ) + attributes.update(job.get("attributes", {})) + job["attributes"] = attributes + job["name"] = f"{product}-{platform}" + yield job + unique_products_and_platforms.add((product, platform)) diff --git a/taskcluster/gecko_taskgraph/transforms/perftest.py b/taskcluster/gecko_taskgraph/transforms/perftest.py new file mode 100644 index 0000000000..5c579b48b5 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/perftest.py @@ -0,0 +1,351 @@ +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +This transform passes options from `mach perftest` to the corresponding task. +""" + + +import json +from copy import deepcopy +from datetime import date, timedelta + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from taskgraph.util.treeherder import join_symbol, split_symbol +from voluptuous import Any, Extra, Optional + +transforms = TransformSequence() + + +perftest_description_schema = Schema( + { + # The test names and the symbols to use for them: [test-symbol, test-path] + Optional("perftest"): [[str]], + # Metrics to gather for the test. These will be merged + # with options specified through perftest-perfherder-global + Optional("perftest-metrics"): optionally_keyed_by( + "perftest", + Any( + [str], + {str: Any(None, {str: Any(None, str, [str])})}, + ), + ), + # Perfherder data options that will be applied to + # all metrics gathered. + Optional("perftest-perfherder-global"): optionally_keyed_by( + "perftest", {str: Any(None, str, [str])} + ), + # Extra options to add to the test's command + Optional("perftest-extra-options"): optionally_keyed_by("perftest", [str]), + # Variants of the test to make based on extra browsertime + # arguments. Expecting: + # [variant-suffix, options-to-use] + # If variant-suffix is `null` then the options will be added + # to the existing task. Otherwise, a new variant is created + # with the given suffix and with its options replaced. + Optional("perftest-btime-variants"): optionally_keyed_by( + "perftest", [[Any(None, str)]] + ), + # These options will be parsed in the next schemas + Extra: object, + } +) + + +transforms.add_validate(perftest_description_schema) + + +@transforms.add +def split_tests(config, jobs): + for job in jobs: + if job.get("perftest") is None: + yield job + continue + + for test_symbol, test_name in job.pop("perftest"): + job_new = deepcopy(job) + + job_new["perftest"] = test_symbol + job_new["name"] += "-" + test_symbol + job_new["treeherder"]["symbol"] = job["treeherder"]["symbol"].format( + symbol=test_symbol + ) + job_new["run"]["command"] = job["run"]["command"].replace( + "{perftest_testname}", test_name + ) + + yield job_new + + +@transforms.add +def handle_keyed_by_perftest(config, jobs): + fields = ["perftest-metrics", "perftest-extra-options", "perftest-btime-variants"] + for job in jobs: + if job.get("perftest") is None: + yield job + continue + + for field in fields: + resolve_keyed_by(job, field, item_name=job["name"]) + + job.pop("perftest") + yield job + + +@transforms.add +def parse_perftest_metrics(config, jobs): + """Parse the metrics into a dictionary immediately. + + This way we can modify the extraOptions field (and others) entry through the + transforms that come later. The metrics aren't formatted until the end of the + transforms. + """ + for job in jobs: + if job.get("perftest-metrics") is None: + yield job + continue + perftest_metrics = job.pop("perftest-metrics") + + # If perftest metrics is a string, split it up first + if isinstance(perftest_metrics, list): + new_metrics_info = [{"name": metric} for metric in perftest_metrics] + else: + new_metrics_info = [] + for metric, options in perftest_metrics.items(): + entry = {"name": metric} + entry.update(options) + new_metrics_info.append(entry) + + job["perftest-metrics"] = new_metrics_info + yield job + + +@transforms.add +def split_perftest_variants(config, jobs): + for job in jobs: + if job.get("variants") is None: + yield job + continue + + for variant in job.pop("variants"): + job_new = deepcopy(job) + + group, symbol = split_symbol(job_new["treeherder"]["symbol"]) + group += "-" + variant + job_new["treeherder"]["symbol"] = join_symbol(group, symbol) + job_new["name"] += "-" + variant + job_new.setdefault("perftest-perfherder-global", {}).setdefault( + "extraOptions", [] + ).append(variant) + job_new[variant] = True + + yield job_new + + yield job + + +@transforms.add +def split_btime_variants(config, jobs): + for job in jobs: + if job.get("perftest-btime-variants") is None: + yield job + continue + + variants = job.pop("perftest-btime-variants") + if not variants: + yield job + continue + + yield_existing = False + for suffix, options in variants: + if suffix is None: + # Append options to the existing job + job.setdefault("perftest-btime-variants", []).append(options) + yield_existing = True + else: + job_new = deepcopy(job) + group, symbol = split_symbol(job_new["treeherder"]["symbol"]) + symbol += "-" + suffix + job_new["treeherder"]["symbol"] = join_symbol(group, symbol) + job_new["name"] += "-" + suffix + job_new.setdefault("perftest-perfherder-global", {}).setdefault( + "extraOptions", [] + ).append(suffix) + # Replace the existing options with the new ones + job_new["perftest-btime-variants"] = [options] + yield job_new + + # The existing job has been modified so we should also return it + if yield_existing: + yield job + + +@transforms.add +def setup_http3_tests(config, jobs): + for job in jobs: + if job.get("http3") is None or not job.pop("http3"): + yield job + continue + job.setdefault("perftest-btime-variants", []).append( + "firefox.preference=network.http.http3.enable:true" + ) + yield job + + +@transforms.add +def setup_perftest_metrics(config, jobs): + for job in jobs: + if job.get("perftest-metrics") is None: + yield job + continue + perftest_metrics = job.pop("perftest-metrics") + + # Options to apply to each metric + global_options = job.pop("perftest-perfherder-global", {}) + for metric_info in perftest_metrics: + for opt, val in global_options.items(): + if isinstance(val, list) and opt in metric_info: + metric_info[opt].extend(val) + elif not (isinstance(val, list) and len(val) == 0): + metric_info[opt] = val + + quote_escape = '\\"' + if "win" in job.get("platform", ""): + # Escaping is a bit different on windows platforms + quote_escape = '\\\\\\"' + + job["run"]["command"] = job["run"]["command"].replace( + "{perftest_metrics}", + " ".join( + [ + ",".join( + [ + ":".join( + [ + option, + str(value) + .replace(" ", "") + .replace("'", quote_escape), + ] + ) + for option, value in metric_info.items() + ] + ) + for metric_info in perftest_metrics + ] + ), + ) + + yield job + + +@transforms.add +def setup_perftest_browsertime_variants(config, jobs): + for job in jobs: + if job.get("perftest-btime-variants") is None: + yield job + continue + + job["run"]["command"] += " --browsertime-extra-options %s" % ",".join( + [opt.strip() for opt in job.pop("perftest-btime-variants")] + ) + + yield job + + +@transforms.add +def setup_perftest_extra_options(config, jobs): + for job in jobs: + if job.get("perftest-extra-options") is None: + yield job + continue + job["run"]["command"] += " " + " ".join(job.pop("perftest-extra-options")) + yield job + + +@transforms.add +def pass_perftest_options(config, jobs): + for job in jobs: + env = job.setdefault("worker", {}).setdefault("env", {}) + env["PERFTEST_OPTIONS"] = json.dumps( + config.params["try_task_config"].get("perftest-options") + ) + yield job + + +@transforms.add +def setup_perftest_test_date(config, jobs): + for job in jobs: + if ( + job.get("attributes", {}).get("batch", False) + and "--test-date" not in job["run"]["command"] + ): + yesterday = (date.today() - timedelta(1)).strftime("%Y.%m.%d") + job["run"]["command"] += " --test-date %s" % yesterday + yield job + + +@transforms.add +def setup_regression_detector(config, jobs): + for job in jobs: + if "change-detector" in job.get("name"): + + tasks_to_analyze = [] + for task in config.params["try_task_config"].get("tasks", []): + # Explicitly skip these tasks since they're + # part of the mozperftest tasks + if "side-by-side" in task: + continue + if "change-detector" in task: + continue + + # Select these tasks + if "browsertime" in task: + tasks_to_analyze.append(task) + elif "talos" in task: + tasks_to_analyze.append(task) + elif "awsy" in task: + tasks_to_analyze.append(task) + elif "perftest" in task: + tasks_to_analyze.append(task) + + if len(tasks_to_analyze) == 0: + yield job + continue + + # Make the change detector task depend on the tasks to analyze. + # This prevents the task from running until all data is available + # within the current push. + job["soft-dependencies"] = tasks_to_analyze + job["requires"] = "all-completed" + + new_project = config.params["project"] + if ( + "try" in config.params["project"] + or config.params["try_mode"] == "try_select" + ): + new_project = "try" + + base_project = None + if ( + config.params.get("try_task_config", {}) + .get("env", {}) + .get("PERF_BASE_REVISION", None) + is not None + ): + task_names = " --task-name ".join(tasks_to_analyze) + base_revision = config.params["try_task_config"]["env"][ + "PERF_BASE_REVISION" + ] + base_project = new_project + + # Add all the required information to the task + job["run"]["command"] = job["run"]["command"].format( + task_name=task_names, + base_revision=base_revision, + base_branch=base_project, + new_branch=new_project, + new_revision=config.params["head_rev"], + ) + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/python_update.py b/taskcluster/gecko_taskgraph/transforms/python_update.py new file mode 100644 index 0000000000..f4f135b585 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/python_update.py @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the repo-update task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + env = task["worker"].setdefault("env", {}) + env["BRANCH"] = config.params["project"] + for envvar in env: + resolve_keyed_by(env, envvar, envvar, **config.params) + + for envvar in list(env.keys()): + if not env.get(envvar): + del env[envvar] + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/release.py b/taskcluster/gecko_taskgraph/transforms/release.py new file mode 100644 index 0000000000..1158252fe7 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release.py @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Transforms for release tasks +""" + + +def run_on_releases(config, jobs): + """ + Filter out jobs with `run-on-releases` set, and that don't match the + `release_type` paramater. + """ + for job in jobs: + release_type = config.params["release_type"] + run_on_release_types = job.pop("run-on-releases", None) + + if run_on_release_types is None or release_type in run_on_release_types: + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/release_beetmover_signed_addons.py b/taskcluster/gecko_taskgraph/transforms/release_beetmover_signed_addons.py new file mode 100644 index 0000000000..fb8eb91f44 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_beetmover_signed_addons.py @@ -0,0 +1,243 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +import copy +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from taskgraph.util.treeherder import inherit_treeherder_from_dep +from voluptuous import Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.beetmover import craft_release_properties +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + release_level, +) +from gecko_taskgraph.util.scriptworker import ( + generate_beetmover_artifact_map, + generate_beetmover_upstream_artifacts, + get_beetmover_action_scope, + get_beetmover_bucket_scope, +) + +logger = logging.getLogger(__name__) + + +transforms = TransformSequence() + + +beetmover_description_schema = schema.extend( + { + # attributes is used for enabling artifact-map by declarative artifacts + Required("attributes"): {str: object}, + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Optional("label"): str, + # treeherder is allowed here to override any defaults we use for beetmover. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + Required("description"): str, + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("run-on-projects"): [], + # locale is passed only for l10n beetmoving + Optional("locale"): str, + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + } +) + + +transforms.add_validate(beetmover_description_schema) + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + for field in ("worker-type", "attributes.artifact_map"): + resolve_keyed_by( + job, + field, + item_name=job["label"], + **{ + "release-level": release_level(config.params["project"]), + "release-type": config.params["release_type"], + "project": config.params["project"], + }, + ) + yield job + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = dep_job.attributes + + treeherder = inherit_treeherder_from_dep(job, dep_job) + treeherder.setdefault( + "symbol", "langpack(BM{})".format(attributes.get("l10n_chunk", "")) + ) + + job["attributes"].update(copy_attributes_from_dependent_job(dep_job)) + job["attributes"]["chunk_locales"] = dep_job.attributes.get( + "chunk_locales", ["en-US"] + ) + + job["description"] = job["description"].format( + locales="/".join(job["attributes"]["chunk_locales"]), + platform=job["attributes"]["build_platform"], + ) + + job["scopes"] = [ + get_beetmover_bucket_scope(config), + get_beetmover_action_scope(config), + ] + + job["dependencies"] = {"langpack-copy": dep_job.label} + + job["run-on-projects"] = job.get( + "run_on_projects", dep_job.attributes["run_on_projects"] + ) + job["treeherder"] = treeherder + job["shipping-phase"] = job.get( + "shipping-phase", dep_job.attributes["shipping_phase"] + ) + job["shipping-product"] = dep_job.attributes["shipping_product"] + + yield job + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + platform = job["attributes"]["build_platform"] + locale = job["attributes"]["chunk_locales"] + + job["worker"] = { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, job), + "upstream-artifacts": generate_beetmover_upstream_artifacts( + config, + job, + platform, + locale, + ), + "artifact-map": generate_beetmover_artifact_map( + config, job, platform=platform, locale=locale + ), + } + + yield job + + +@transforms.add +def strip_unused_data(config, jobs): + for job in jobs: + del job["primary-dependency"] + + yield job + + +@transforms.add +def yield_all_platform_jobs(config, jobs): + # Even though langpacks are now platform independent, we keep beetmoving them at old + # platform-specific locations. That's why this transform exist + # The linux64 and mac specific ja-JP-mac are beetmoved along with the signing beetmover + # So while the dependent jobs are linux here, we only yield jobs for other platforms + for job in jobs: + platforms = ("linux", "macosx64", "win32", "win64") + if "devedition" in job["attributes"]["build_platform"]: + platforms = (f"{plat}-devedition" for plat in platforms) + for platform in platforms: + platform_job = copy.deepcopy(job) + if "ja" in platform_job["attributes"]["chunk_locales"] and platform in ( + "macosx64", + "macosx64-devedition", + ): + platform_job = _strip_ja_data_from_linux_job(platform_job) + + platform_job = _change_platform_data(config, platform_job, platform) + + yield platform_job + + +def _strip_ja_data_from_linux_job(platform_job): + # Let's take "ja" out the description. This locale is in a substring like "aa/bb/cc/dd", where + # "ja" could be any of "aa", "bb", "cc", "dd" + platform_job["description"] = platform_job["description"].replace("ja/", "") + platform_job["description"] = platform_job["description"].replace("/ja", "") + + platform_job["worker"]["upstream-artifacts"] = [ + artifact + for artifact in platform_job["worker"]["upstream-artifacts"] + if artifact["locale"] != "ja" + ] + + return platform_job + + +def _change_platform_in_artifact_map_paths(paths, orig_platform, new_platform): + amended_paths = {} + for artifact, artifact_info in paths.items(): + amended_artifact_info = { + "checksums_path": artifact_info["checksums_path"].replace( + orig_platform, new_platform + ), + "destinations": [ + d.replace(orig_platform, new_platform) + for d in artifact_info["destinations"] + ], + } + amended_paths[artifact] = amended_artifact_info + + return amended_paths + + +def _change_platform_data(config, platform_job, platform): + orig_platform = "linux64" + if "devedition" in platform: + orig_platform = "linux64-devedition" + platform_job["attributes"]["build_platform"] = platform + platform_job["label"] = platform_job["label"].replace(orig_platform, platform) + platform_job["description"] = platform_job["description"].replace( + orig_platform, platform + ) + platform_job["treeherder"]["platform"] = platform_job["treeherder"][ + "platform" + ].replace(orig_platform, platform) + platform_job["worker"]["release-properties"]["platform"] = platform + + # amend artifactMap entries as well + platform_mapping = { + "linux64": "linux-x86_64", + "linux": "linux-i686", + "macosx64": "mac", + "win32": "win32", + "win64": "win64", + "linux64-devedition": "linux-x86_64", + "linux-devedition": "linux-i686", + "macosx64-devedition": "mac", + "win32-devedition": "win32", + "win64-devedition": "win64", + } + orig_platform = platform_mapping.get(orig_platform, orig_platform) + platform = platform_mapping.get(platform, platform) + platform_job["worker"]["artifact-map"] = [ + { + "locale": entry["locale"], + "taskId": entry["taskId"], + "paths": _change_platform_in_artifact_map_paths( + entry["paths"], orig_platform, platform + ), + } + for entry in platform_job["worker"]["artifact-map"] + ] + + return platform_job diff --git a/taskcluster/gecko_taskgraph/transforms/release_deps.py b/taskcluster/gecko_taskgraph/transforms/release_deps.py new file mode 100644 index 0000000000..e44af576eb --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_deps.py @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Add dependencies to release tasks. +""" + +from taskgraph.transforms.base import TransformSequence + +PHASES = ["build", "promote", "push", "ship"] + +transforms = TransformSequence() + + +@transforms.add +def add_dependencies(config, jobs): + for job in jobs: + dependencies = {} + # Add any kind_dependencies_tasks with matching product as dependencies + product = job.get("shipping-product") + phase = job.get("shipping-phase") + if product is None: + continue + + required_signoffs = set( + job.setdefault("attributes", {}).get("required_signoffs", []) + ) + for dep_task in config.kind_dependencies_tasks.values(): + # Weed out unwanted tasks. + # XXX we have run-on-projects which specifies the on-push behavior; + # we need another attribute that specifies release promotion, + # possibly which action(s) each task belongs in. + + # We can only depend on tasks in the current or previous phases + dep_phase = dep_task.attributes.get("shipping_phase") + if dep_phase and PHASES.index(dep_phase) > PHASES.index(phase): + continue + + if dep_task.attributes.get("build_platform") and job.get( + "attributes", {} + ).get("build_platform"): + if ( + dep_task.attributes["build_platform"] + != job["attributes"]["build_platform"] + ): + continue + # Add matching product tasks to deps + if ( + dep_task.task.get("shipping-product") == product + or dep_task.attributes.get("shipping_product") == product + ): + dependencies[dep_task.label] = dep_task.label + required_signoffs.update( + dep_task.attributes.get("required_signoffs", []) + ) + + job.setdefault("dependencies", {}).update(dependencies) + if required_signoffs: + job["attributes"]["required_signoffs"] = sorted(required_signoffs) + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/release_flatpak_push.py b/taskcluster/gecko_taskgraph/transforms/release_flatpak_push.py new file mode 100644 index 0000000000..8a336502e6 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_flatpak_push.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the release-flatpak-push kind into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from voluptuous import Optional, Required + +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import add_scope_prefix + +push_flatpak_description_schema = Schema( + { + Required("name"): str, + Required("job-from"): task_description_schema["job-from"], + Required("dependencies"): task_description_schema["dependencies"], + Required("description"): task_description_schema["description"], + Required("treeherder"): task_description_schema["treeherder"], + Required("run-on-projects"): task_description_schema["run-on-projects"], + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("worker"): object, + Optional("scopes"): [str], + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + Optional("extra"): task_description_schema["extra"], + Optional("attributes"): task_description_schema["attributes"], + } +) + +transforms = TransformSequence() +transforms.add_validate(push_flatpak_description_schema) + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + if len(job["dependencies"]) != 1: + raise Exception("Exactly 1 dependency is required") + + job["worker"]["upstream-artifacts"] = generate_upstream_artifacts( + job["dependencies"] + ) + + resolve_keyed_by( + job, + "worker.channel", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + resolve_keyed_by( + job, + "worker-type", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])}, + ) + if release_level(config.params["project"]) == "production": + job.setdefault("scopes", []).append( + add_scope_prefix( + config, + "flathub:firefox:{}".format(job["worker"]["channel"]), + ) + ) + + yield job + + +def generate_upstream_artifacts(dependencies): + return [ + { + "taskId": {"task-reference": f"<{task_kind}>"}, + "taskType": "build", + "paths": ["public/build/target.flatpak.tar.xz"], + } + for task_kind in dependencies.keys() + ] diff --git a/taskcluster/gecko_taskgraph/transforms/release_flatpak_repackage.py b/taskcluster/gecko_taskgraph/transforms/release_flatpak_repackage.py new file mode 100644 index 0000000000..7af1134c3a --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_flatpak_repackage.py @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import get_release_config + +transforms = TransformSequence() + + +@transforms.add +def format(config, tasks): + """Apply format substitution to worker.env and worker.command.""" + + format_params = { + "release_config": get_release_config(config), + "config_params": config.params, + } + + for task in tasks: + format_params["task"] = task + + command = task.get("worker", {}).get("command", []) + task["worker"]["command"] = [x.format(**format_params) for x in command] + + env = task.get("worker", {}).get("env", {}) + for k in env.keys(): + resolve_keyed_by( + env, + k, + "flatpak envs", + **{ + "release-level": release_level(config.params["project"]), + "project": config.params["project"], + } + ) + task["worker"]["env"][k] = env[k].format(**format_params) + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/release_generate_checksums.py b/taskcluster/gecko_taskgraph/transforms/release_generate_checksums.py new file mode 100644 index 0000000000..0024b88726 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_generate_checksums.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the checksums task into an actual task description. +""" + +import copy +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import get_release_config + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def handle_keyed_by(config, jobs): + """Resolve fields that can be keyed by project, etc.""" + fields = [ + "run.config", + "run.extra-config", + ] + for job in jobs: + job = copy.deepcopy(job) + for field in fields: + resolve_keyed_by( + item=job, + field=field, + item_name=job["name"], + **{"release-level": release_level(config.params["project"])} + ) + yield job + + +@transforms.add +def interpolate(config, jobs): + release_config = get_release_config(config) + for job in jobs: + mh_options = list(job["run"]["options"]) + job["run"]["options"] = [ + option.format( + version=release_config["version"], + build_number=release_config["build_number"], + ) + for option in mh_options + ] + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/release_generate_checksums_beetmover.py b/taskcluster/gecko_taskgraph/transforms/release_generate_checksums_beetmover.py new file mode 100644 index 0000000000..c91e807b27 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_generate_checksums_beetmover.py @@ -0,0 +1,118 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the `release-generate-checksums-beetmover` task to also append `build` as dependency +""" + +from taskgraph.transforms.base import TransformSequence +from voluptuous import Optional + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.beetmover import craft_release_properties +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.scriptworker import ( + generate_beetmover_artifact_map, + generate_beetmover_upstream_artifacts, + get_beetmover_action_scope, + get_beetmover_bucket_scope, +) + +transforms = TransformSequence() + + +release_generate_checksums_beetmover_schema = schema.extend( + { + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Optional("label"): str, + # treeherder is allowed here to override any defaults we use for beetmover. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("attributes"): task_description_schema["attributes"], + } +) + +transforms = TransformSequence() +transforms.add_validate(release_generate_checksums_beetmover_schema) + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = copy_attributes_from_dependent_job(dep_job) + attributes.update(job.get("attributes", {})) + + treeherder = job.get("treeherder", {}) + treeherder.setdefault("symbol", "BM-SGenChcks") + dep_th_platform = ( + dep_job.task.get("extra", {}) + .get("treeherder", {}) + .get("machine", {}) + .get("platform", "") + ) + treeherder.setdefault("platform", f"{dep_th_platform}/opt") + treeherder.setdefault("tier", 1) + treeherder.setdefault("kind", "build") + + job_template = f"{dep_job.label}" + label = job_template.replace("signing", "beetmover") + + description = "Transfer *SUMS and *SUMMARY checksums file to S3." + + # first dependency is the signing task for the *SUMS files + dependencies = {dep_job.kind: dep_job.label} + + if len(dep_job.dependencies) > 1: + raise NotImplementedError( + "Can't beetmove a signing task with multiple dependencies" + ) + # update the dependencies with the dependencies of the signing task + dependencies.update(dep_job.dependencies) + + bucket_scope = get_beetmover_bucket_scope(config) + action_scope = get_beetmover_action_scope(config) + + task = { + "label": label, + "description": description, + "worker-type": "beetmover", + "scopes": [bucket_scope, action_scope], + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "treeherder": treeherder, + "shipping-phase": "promote", + } + + yield task + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + valid_beetmover_job = len(job["dependencies"]) == 2 and any( + ["signing" in j for j in job["dependencies"]] + ) + if not valid_beetmover_job: + raise NotImplementedError("Beetmover must have two dependencies.") + + platform = job["attributes"]["build_platform"] + worker = { + "implementation": "beetmover", + "release-properties": craft_release_properties(config, job), + "upstream-artifacts": generate_beetmover_upstream_artifacts( + config, job, platform=None, locale=None + ), + "artifact-map": generate_beetmover_artifact_map( + config, job, platform=platform + ), + } + + job["worker"] = worker + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/release_generate_checksums_signing.py b/taskcluster/gecko_taskgraph/transforms/release_generate_checksums_signing.py new file mode 100644 index 0000000000..6dfee1d33b --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_generate_checksums_signing.py @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the release-generate-checksums-signing task into task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.taskcluster import get_artifact_path +from voluptuous import Optional + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.scriptworker import get_signing_cert_scope + +release_generate_checksums_signing_schema = schema.extend( + { + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + } +) + +transforms = TransformSequence() +transforms.add_validate(release_generate_checksums_signing_schema) + + +@transforms.add +def make_release_generate_checksums_signing_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = copy_attributes_from_dependent_job(dep_job) + + treeherder = job.get("treeherder", {}) + treeherder.setdefault("symbol", "SGenChcks") + dep_th_platform = ( + dep_job.task.get("extra", {}) + .get("treeherder", {}) + .get("machine", {}) + .get("platform", "") + ) + treeherder.setdefault("platform", f"{dep_th_platform}/opt") + treeherder.setdefault("tier", 1) + treeherder.setdefault("kind", "build") + + job_template = "{}-{}".format(dep_job.label, "signing") + label = job.get("label", job_template) + description = "Signing of the overall release-related checksums" + + dependencies = {dep_job.kind: dep_job.label} + + upstream_artifacts = [ + { + "taskId": {"task-reference": f"<{str(dep_job.kind)}>"}, + "taskType": "build", + "paths": [ + get_artifact_path(dep_job, "SHA256SUMS"), + get_artifact_path(dep_job, "SHA512SUMS"), + ], + "formats": ["autograph_gpg"], + } + ] + + signing_cert_scope = get_signing_cert_scope(config) + + task = { + "label": label, + "description": description, + "worker-type": "linux-signing", + "worker": { + "implementation": "scriptworker-signing", + "upstream-artifacts": upstream_artifacts, + "max-run-time": 3600, + }, + "scopes": [ + signing_cert_scope, + ], + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "treeherder": treeherder, + } + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/release_mark_as_shipped.py b/taskcluster/gecko_taskgraph/transforms/release_mark_as_shipped.py new file mode 100644 index 0000000000..f2ce148320 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_mark_as_shipped.py @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import get_release_config + +transforms = TransformSequence() + + +@transforms.add +def make_task_description(config, jobs): + release_config = get_release_config(config) + for job in jobs: + resolve_keyed_by( + job, + "worker-type", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])} + ) + resolve_keyed_by( + job, + "scopes", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])} + ) + + job["worker"][ + "release-name" + ] = "{product}-{version}-build{build_number}".format( + product=job["shipping-product"].capitalize(), + version=release_config["version"], + build_number=release_config["build_number"], + ) + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/release_msix_push.py b/taskcluster/gecko_taskgraph/transforms/release_msix_push.py new file mode 100644 index 0000000000..817341f92c --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_msix_push.py @@ -0,0 +1,88 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the release-msix-push kind into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from voluptuous import Optional, Required + +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import add_scope_prefix + +push_msix_description_schema = Schema( + { + Required("name"): str, + Required("job-from"): task_description_schema["job-from"], + Required("dependencies"): task_description_schema["dependencies"], + Required("description"): task_description_schema["description"], + Required("treeherder"): task_description_schema["treeherder"], + Required("run-on-projects"): task_description_schema["run-on-projects"], + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("worker"): object, + Optional("scopes"): [str], + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + Optional("extra"): task_description_schema["extra"], + Optional("attributes"): task_description_schema["attributes"], + } +) + +transforms = TransformSequence() +transforms.add_validate(push_msix_description_schema) + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + + job["worker"]["upstream-artifacts"] = generate_upstream_artifacts( + job["dependencies"] + ) + + resolve_keyed_by( + job, + "worker.channel", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + resolve_keyed_by( + job, + "worker.publish-mode", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + resolve_keyed_by( + job, + "worker-type", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])}, + ) + if release_level(config.params["project"]) == "production": + job.setdefault("scopes", []).append( + add_scope_prefix( + config, + "microsoftstore:{}".format(job["worker"]["channel"]), + ) + ) + + # Override shipping-phase for release: push to the Store early to + # allow time for certification. + if job["worker"]["publish-mode"] == "Manual": + job["shipping-phase"] = "promote" + + yield job + + +def generate_upstream_artifacts(dependencies): + return [ + { + "taskId": {"task-reference": f"<{task_kind}>"}, + "taskType": "build", + "paths": ["public/build/target.store.msix"], + } + for task_kind in dependencies.keys() + ] diff --git a/taskcluster/gecko_taskgraph/transforms/release_notifications.py b/taskcluster/gecko_taskgraph/transforms/release_notifications.py new file mode 100644 index 0000000000..86109ec5ed --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_notifications.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Add notifications via taskcluster-notify for release tasks +""" +from string import Formatter + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.util.scriptworker import get_release_config + +transforms = TransformSequence() + + +class TitleCaseFormatter(Formatter): + """Support title formatter for strings""" + + def convert_field(self, value, conversion): + if conversion == "t": + return str(value).title() + super().convert_field(value, conversion) + return value + + +titleformatter = TitleCaseFormatter() + + +@transforms.add +def add_notifications(config, jobs): + release_config = get_release_config(config) + + for job in jobs: + label = "{}-{}".format(config.kind, job["name"]) + + notifications = job.pop("notifications", None) + if notifications: + resolve_keyed_by( + notifications, "emails", label, project=config.params["project"] + ) + emails = notifications["emails"] + format_kwargs = dict( + task=job, + config=config.__dict__, + release_config=release_config, + ) + subject = titleformatter.format(notifications["subject"], **format_kwargs) + message = titleformatter.format(notifications["message"], **format_kwargs) + emails = [email.format(**format_kwargs) for email in emails] + + # By default, we only send mail on success to avoid messages like 'blah is in the + # candidates dir' when cancelling graphs, dummy job failure, etc + status_types = notifications.get("status-types", ["on-completed"]) + for s in status_types: + job.setdefault("routes", []).extend( + [f"notify.email.{email}.{s}" for email in emails] + ) + + # Customize the email subject to include release name and build number + job.setdefault("extra", {}).update( + { + "notify": { + "email": { + "subject": subject, + } + } + } + ) + if message: + job["extra"]["notify"]["email"]["content"] = message + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py b/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py new file mode 100644 index 0000000000..17e4d37fb3 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py @@ -0,0 +1,180 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the release-sign-and-push task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from taskgraph.util.treeherder import inherit_treeherder_from_dep +from voluptuous import Any, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + release_level, +) + +transforms = TransformSequence() + +langpack_sign_push_description_schema = schema.extend( + { + Required("label"): str, + Required("description"): str, + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("worker"): { + Required("channel"): optionally_keyed_by( + "project", "platform", Any("listed", "unlisted") + ), + Required("upstream-artifacts"): None, # Processed here below + }, + Required("run-on-projects"): [], + Required("scopes"): optionally_keyed_by("release-level", [str]), + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + } +) + + +@transforms.add +def set_label(config, jobs): + for job in jobs: + label = "push-langpacks-{}".format(job["primary-dependency"].label) + job["label"] = label + + yield job + + +transforms.add_validate(langpack_sign_push_description_schema) + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "worker-type", + item_name=job["label"], + **{"release-level": release_level(config.params["project"])}, + ) + resolve_keyed_by( + job, + "scopes", + item_name=job["label"], + **{"release-level": release_level(config.params["project"])}, + ) + resolve_keyed_by( + job, + "worker.channel", + item_name=job["label"], + project=config.params["project"], + platform=job["primary-dependency"].attributes["build_platform"], + ) + + yield job + + +@transforms.add +def copy_attributes(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + job["attributes"] = copy_attributes_from_dependent_job(dep_job) + job["attributes"]["chunk_locales"] = dep_job.attributes.get( + "chunk_locales", ["en-US"] + ) + + yield job + + +@transforms.add +def filter_out_macos_jobs_but_mac_only_locales(config, jobs): + for job in jobs: + build_platform = job["primary-dependency"].attributes.get("build_platform") + + if build_platform in ("linux64-devedition", "linux64-shippable"): + yield job + elif ( + build_platform in ("macosx64-devedition", "macosx64-shippable") + and "ja-JP-mac" in job["attributes"]["chunk_locales"] + ): + # Other locales of the same job shouldn't be processed + job["attributes"]["chunk_locales"] = ["ja-JP-mac"] + job["label"] = job["label"].replace( + # Guard against a chunk 10 or chunk 1 (latter on try) weird munging + "-{}/".format(job["attributes"]["l10n_chunk"]), + "-ja-JP-mac/", + ) + yield job + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + + treeherder = inherit_treeherder_from_dep(job, dep_job) + treeherder.setdefault( + "symbol", "langpack(SnP{})".format(job["attributes"].get("l10n_chunk", "")) + ) + + job["description"] = job["description"].format( + locales="/".join(job["attributes"]["chunk_locales"]), + ) + + job["dependencies"] = {dep_job.kind: dep_job.label} + job["treeherder"] = treeherder + + yield job + + +def generate_upstream_artifacts(upstream_task_ref, locales): + return [ + { + "taskId": {"task-reference": upstream_task_ref}, + "taskType": "build", + "paths": [ + "public/build{locale}/target.langpack.xpi".format( + locale="" if locale == "en-US" else "/" + locale + ) + for locale in locales + ], + } + ] + + +@transforms.add +def make_task_worker(config, jobs): + for job in jobs: + upstream_task_ref = get_upstream_task_ref( + job, expected_kinds=("build", "shippable-l10n") + ) + + job["worker"]["implementation"] = "push-addons" + job["worker"]["upstream-artifacts"] = generate_upstream_artifacts( + upstream_task_ref, job["attributes"]["chunk_locales"] + ) + + yield job + + +def get_upstream_task_ref(job, expected_kinds): + upstream_tasks = [ + job_kind + for job_kind in job["dependencies"].keys() + if job_kind in expected_kinds + ] + + if len(upstream_tasks) > 1: + raise Exception("Only one dependency expected") + + return f"<{upstream_tasks[0]}>" + + +@transforms.add +def strip_unused_data(config, jobs): + for job in jobs: + del job["primary-dependency"] + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/release_snap_repackage.py b/taskcluster/gecko_taskgraph/transforms/release_snap_repackage.py new file mode 100644 index 0000000000..659a203971 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_snap_repackage.py @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import get_release_config + +transforms = TransformSequence() + + +@transforms.add +def format(config, tasks): + """Apply format substitution to worker.env and worker.command.""" + + format_params = { + "release_config": get_release_config(config), + "config_params": config.params, + } + + for task in tasks: + format_params["task"] = task + + command = task.get("worker", {}).get("command", []) + task["worker"]["command"] = [x.format(**format_params) for x in command] + + env = task.get("worker", {}).get("env", {}) + for k in env.keys(): + resolve_keyed_by( + env, + k, + "snap envs", + **{"release-level": release_level(config.params["project"])} + ) + task["worker"]["env"][k] = env[k].format(**format_params) + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/release_started.py b/taskcluster/gecko_taskgraph/transforms/release_started.py new file mode 100644 index 0000000000..0f54c8e098 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_started.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Add notifications via taskcluster-notify for release tasks +""" +from pipes import quote as shell_quote + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def add_notifications(config, jobs): + for job in jobs: + label = "{}-{}".format(config.kind, job["name"]) + + resolve_keyed_by(job, "emails", label, project=config.params["project"]) + emails = [email.format(config=config.__dict__) for email in job.pop("emails")] + + command = [ + "release", + "send-buglist-email", + "--version", + config.params["version"], + "--product", + job["shipping-product"], + "--revision", + config.params["head_rev"], + "--build-number", + str(config.params["build_number"]), + "--repo", + config.params["head_repository"], + ] + for address in emails: + command += ["--address", address] + command += [ + # We wrap this in `{'task-reference': ...}` below + "--task-group-id", + "<decision>", + ] + + job["scopes"] = [f"notify:email:{address}" for address in emails] + job["run"] = { + "using": "mach", + "sparse-profile": "mach", + "mach": {"task-reference": " ".join(map(shell_quote, command))}, + } + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/release_version_bump.py b/taskcluster/gecko_taskgraph/transforms/release_version_bump.py new file mode 100644 index 0000000000..a0f3f69d05 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/release_version_bump.py @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the update generation task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def handle_keyed_by(config, tasks): + """Resolve fields that can be keyed by platform, etc.""" + default_fields = [ + "worker.push", + "worker.bump-files", + "worker-type", + ] + for task in tasks: + fields = default_fields[:] + for additional_field in ( + "l10n-bump-info", + "source-repo", + "dontbuild", + "ignore-closed-tree", + ): + if additional_field in task["worker"]: + fields.append(f"worker.{additional_field}") + for field in fields: + resolve_keyed_by( + task, + field, + item_name=task["name"], + **{ + "project": config.params["project"], + "release-type": config.params["release_type"], + }, + ) + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/repackage.py b/taskcluster/gecko_taskgraph/transforms/repackage.py new file mode 100644 index 0000000000..2fe849c32d --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/repackage.py @@ -0,0 +1,684 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the repackage task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from taskgraph.util.taskcluster import get_artifact_prefix +from voluptuous import Extra, Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.job import job_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.copy_task import copy_task +from gecko_taskgraph.util.platforms import architecture, archive_format +from gecko_taskgraph.util.workertypes import worker_type_implementation + +packaging_description_schema = schema.extend( + { + # unique label to describe this repackaging task + Optional("label"): str, + Optional("worker-type"): str, + Optional("worker"): object, + # treeherder is allowed here to override any defaults we use for repackaging. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): job_description_schema["treeherder"], + # If a l10n task, the corresponding locale + Optional("locale"): str, + # Routes specific to this task, if defined + Optional("routes"): [str], + # passed through directly to the job description + Optional("extra"): job_description_schema["extra"], + # passed through to job description + Optional("fetches"): job_description_schema["fetches"], + Optional("run-on-projects"): job_description_schema["run-on-projects"], + # Shipping product and phase + Optional("shipping-product"): job_description_schema["shipping-product"], + Optional("shipping-phase"): job_description_schema["shipping-phase"], + Required("package-formats"): optionally_keyed_by( + "build-platform", "release-type", [str] + ), + Optional("msix"): { + Optional("channel"): optionally_keyed_by( + "package-format", + "level", + "build-platform", + "release-type", + "shipping-product", + str, + ), + Optional("identity-name"): optionally_keyed_by( + "package-format", + "level", + "build-platform", + "release-type", + "shipping-product", + str, + ), + Optional("publisher"): optionally_keyed_by( + "package-format", + "level", + "build-platform", + "release-type", + "shipping-product", + str, + ), + Optional("publisher-display-name"): optionally_keyed_by( + "package-format", + "level", + "build-platform", + "release-type", + "shipping-product", + str, + ), + Optional("vendor"): str, + }, + # All l10n jobs use mozharness + Required("mozharness"): { + Extra: object, + # Config files passed to the mozharness script + Required("config"): optionally_keyed_by("build-platform", [str]), + # Additional paths to look for mozharness configs in. These should be + # relative to the base of the source checkout + Optional("config-paths"): [str], + # if true, perform a checkout of a comm-central based branch inside the + # gecko checkout + Optional("comm-checkout"): bool, + Optional("run-as-root"): bool, + Optional("use-caches"): bool, + }, + } +) + +# The configuration passed to the mozharness repackage script. This defines the +# arguments passed to `mach repackage` +# - `args` is interpolated by mozharness (`{package-name}`, `{installer-tag}`, +# `{stub-installer-tag}`, `{sfx-stub}`, `{wsx-stub}`, `{fetch-dir}`), with values +# from mozharness. +# - `inputs` are passed as long-options, with the filename prefixed by +# `MOZ_FETCH_DIR`. The filename is interpolated by taskgraph +# (`{archive_format}`). +# - `output` is passed to `--output`, with the filename prefixed by the output +# directory. +PACKAGE_FORMATS = { + "mar": { + "args": [ + "mar", + "--arch", + "{architecture}", + "--mar-channel-id", + "{mar-channel-id}", + ], + "inputs": { + "input": "target{archive_format}", + "mar": "mar-tools/mar", + }, + "output": "target.complete.mar", + }, + "msi": { + "args": [ + "msi", + "--wsx", + "{wsx-stub}", + "--version", + "{version_display}", + "--locale", + "{_locale}", + "--arch", + "{architecture}", + "--candle", + "{fetch-dir}/candle.exe", + "--light", + "{fetch-dir}/light.exe", + ], + "inputs": { + "setupexe": "target.installer.exe", + }, + "output": "target.installer.msi", + }, + "msix": { + "args": [ + "msix", + "--channel", + "{msix-channel}", + "--publisher", + "{msix-publisher}", + "--publisher-display-name", + "{msix-publisher-display-name}", + "--identity-name", + "{msix-identity-name}", + "--vendor", + "{msix-vendor}", + "--arch", + "{architecture}", + # For langpacks. Ignored if directory does not exist. + "--distribution-dir", + "{fetch-dir}/distribution", + "--verbose", + "--makeappx", + "{fetch-dir}/msix-packaging/makemsix", + ], + "inputs": { + "input": "target{archive_format}", + }, + "output": "target.installer.msix", + }, + "msix-store": { + "args": [ + "msix", + "--channel", + "{msix-channel}", + "--publisher", + "{msix-publisher}", + "--publisher-display-name", + "{msix-publisher-display-name}", + "--identity-name", + "{msix-identity-name}", + "--vendor", + "{msix-vendor}", + "--arch", + "{architecture}", + # For langpacks. Ignored if directory does not exist. + "--distribution-dir", + "{fetch-dir}/distribution", + "--verbose", + "--makeappx", + "{fetch-dir}/msix-packaging/makemsix", + ], + "inputs": { + "input": "target{archive_format}", + }, + "output": "target.store.msix", + }, + "dmg": { + "args": ["dmg"], + "inputs": { + "input": "target{archive_format}", + }, + "output": "target.dmg", + }, + "pkg": { + "args": ["pkg"], + "inputs": { + "input": "target{archive_format}", + }, + "output": "target.pkg", + }, + "installer": { + "args": [ + "installer", + "--package-name", + "{package-name}", + "--tag", + "{installer-tag}", + "--sfx-stub", + "{sfx-stub}", + ], + "inputs": { + "package": "target{archive_format}", + "setupexe": "setup.exe", + }, + "output": "target.installer.exe", + }, + "installer-stub": { + "args": [ + "installer", + "--tag", + "{stub-installer-tag}", + "--sfx-stub", + "{sfx-stub}", + ], + "inputs": { + "setupexe": "setup-stub.exe", + }, + "output": "target.stub-installer.exe", + }, + "deb": { + "args": [ + "deb", + "--arch", + "{architecture}", + "--templates", + "browser/installer/linux/app/debian", + "--version", + "{version_display}", + "--build-number", + "{build_number}", + "--release-product", + "{release_product}", + "--release-type", + "{release_type}", + ], + "inputs": { + "input": "target{archive_format}", + }, + "output": "target.deb", + }, + "deb-l10n": { + "args": [ + "deb-l10n", + "--version", + "{version_display}", + "--build-number", + "{build_number}", + "--templates", + "browser/installer/linux/langpack/debian", + ], + "inputs": { + "input-xpi-file": "target.langpack.xpi", + "input-tar-file": "target{archive_format}", + }, + "output": "target.langpack.deb", + }, +} +MOZHARNESS_EXPANSIONS = [ + "package-name", + "installer-tag", + "fetch-dir", + "stub-installer-tag", + "sfx-stub", + "wsx-stub", +] + +transforms = TransformSequence() +transforms.add_validate(packaging_description_schema) + + +@transforms.add +def copy_in_useful_magic(config, jobs): + """Copy attributes from upstream task to be used for keyed configuration.""" + for job in jobs: + dep = job["primary-dependency"] + job["build-platform"] = dep.attributes.get("build_platform") + job["shipping-product"] = dep.attributes.get("shipping_product") + yield job + + +@transforms.add +def handle_keyed_by(config, jobs): + """Resolve fields that can be keyed by platform, etc, but not `msix.*` fields + that can be keyed by `package-format`. Such fields are handled specially below. + """ + fields = [ + "mozharness.config", + "package-formats", + "worker.max-run-time", + ] + for job in jobs: + job = copy_task(job) # don't overwrite dict values here + for field in fields: + resolve_keyed_by( + item=job, + field=field, + item_name="?", + **{ + "release-type": config.params["release_type"], + "level": config.params["level"], + }, + ) + yield job + + +@transforms.add +def make_repackage_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + + label = job.get("label", dep_job.label.replace("signing-", "repackage-")) + job["label"] = label + + yield job + + +@transforms.add +def make_job_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + dependencies = {dep_job.kind: dep_job.label} + + attributes = copy_attributes_from_dependent_job(dep_job) + attributes["repackage_type"] = "repackage" + + locale = attributes.get("locale", job.get("locale")) + if locale: + attributes["locale"] = locale + + description = ( + "Repackaging for locale '{locale}' for build '" + "{build_platform}/{build_type}'".format( + locale=attributes.get("locale", "en-US"), + build_platform=attributes.get("build_platform"), + build_type=attributes.get("build_type"), + ) + ) + + treeherder = job.get("treeherder", {}) + treeherder.setdefault("symbol", "Rpk") + dep_th_platform = dep_job.task.get("extra", {}).get("treeherder-platform") + treeherder.setdefault("platform", dep_th_platform) + treeherder.setdefault("tier", 1) + treeherder.setdefault("kind", "build") + + # Search dependencies before adding langpack dependencies. + signing_task = None + repackage_signing_task = None + for dependency in dependencies.keys(): + if "repackage-signing" in dependency: + repackage_signing_task = dependency + elif "signing" in dependency or "notarization" in dependency: + signing_task = dependency + + if config.kind == "repackage-msi": + treeherder["symbol"] = "MSI({})".format(locale or "N") + + elif config.kind == "repackage-msix": + assert not locale + + # Like "MSIXs(Bs)". + treeherder["symbol"] = "MSIX({})".format( + dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol", "B") + ) + + elif config.kind == "repackage-shippable-l10n-msix": + assert not locale + + if attributes.get("l10n_chunk") or attributes.get("chunk_locales"): + # We don't want to produce MSIXes for single-locale repack builds. + continue + + description = ( + "Repackaging with multiple locales for build '" + "{build_platform}/{build_type}'".format( + build_platform=attributes.get("build_platform"), + build_type=attributes.get("build_type"), + ) + ) + + # Like "MSIXs(Bs-multi)". + treeherder["symbol"] = "MSIX({}-multi)".format( + dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol", "B") + ) + + fetches = job.setdefault("fetches", {}) + + # The keys are unique, like `shippable-l10n-signing-linux64-shippable-1/opt`, so we + # can't ask for the tasks directly, we must filter for them. + for t in config.kind_dependencies_tasks.values(): + if t.kind != "shippable-l10n-signing": + continue + if t.attributes["build_platform"] != "linux64-shippable": + continue + if t.attributes["build_type"] != "opt": + continue + + dependencies.update({t.label: t.label}) + + fetches.update( + { + t.label: [ + { + "artifact": f"{loc}/target.langpack.xpi", + "extract": False, + # Otherwise we can't disambiguate locales! + "dest": f"distribution/extensions/{loc}", + } + for loc in t.attributes["chunk_locales"] + ] + } + ) + + elif config.kind == "repackage-deb": + attributes["repackage_type"] = "repackage-deb" + description = ( + "Repackaging the '{build_platform}/{build_type}' " + "{version} build into a '.deb' package" + ).format( + build_platform=attributes.get("build_platform"), + build_type=attributes.get("build_type"), + version=config.params["version"], + ) + + _fetch_subst_locale = "en-US" + if locale: + _fetch_subst_locale = locale + + worker_type = job["worker-type"] + build_platform = attributes["build_platform"] + + use_stub = attributes.get("stub-installer") + + repackage_config = [] + package_formats = job.get("package-formats") + if use_stub and not repackage_signing_task and "msix" not in package_formats: + # if repackage_signing_task doesn't exists, generate the stub installer + package_formats += ["installer-stub"] + for format in package_formats: + command = copy_task(PACKAGE_FORMATS[format]) + substs = { + "archive_format": archive_format(build_platform), + "_locale": _fetch_subst_locale, + "architecture": architecture(build_platform), + "version_display": config.params["version"], + "mar-channel-id": attributes["mar-channel-id"], + "build_number": config.params["build_number"], + "release_product": config.params["release_product"], + "release_type": config.params["release_type"], + } + # Allow us to replace `args` as well, but specifying things expanded in mozharness + # without breaking .format and without allowing unknown through. + substs.update({name: f"{{{name}}}" for name in MOZHARNESS_EXPANSIONS}) + + # We need to resolve `msix.*` values keyed by `package-format` for each format, not + # just once, so we update a temporary copy just for extracting these values. + temp_job = copy_task(job) + for msix_key in ( + "channel", + "identity-name", + "publisher", + "publisher-display-name", + "vendor", + ): + resolve_keyed_by( + item=temp_job, + field=f"msix.{msix_key}", + item_name="?", + **{ + "package-format": format, + "release-type": config.params["release_type"], + "level": config.params["level"], + }, + ) + + # Turn `msix.channel` into `msix-channel`, etc. + value = temp_job.get("msix", {}).get(msix_key) + if value: + substs.update( + {f"msix-{msix_key}": value}, + ) + + command["inputs"] = { + name: filename.format(**substs) + for name, filename in command["inputs"].items() + } + command["args"] = [arg.format(**substs) for arg in command["args"]] + if "installer" in format and "aarch64" not in build_platform: + command["args"].append("--use-upx") + + repackage_config.append(command) + + run = job.get("mozharness", {}) + run.update( + { + "using": "mozharness", + "script": "mozharness/scripts/repackage.py", + "job-script": "taskcluster/scripts/builder/repackage.sh", + "actions": ["setup", "repackage"], + "extra-config": { + "repackage_config": repackage_config, + }, + "run-as-root": run.get("run-as-root", False), + "use-caches": run.get("use-caches", True), + } + ) + + worker = job.get("worker", {}) + worker.update( + { + "chain-of-trust": True, + # Don't add generic artifact directory. + "skip-artifacts": True, + } + ) + worker.setdefault("max-run-time", 3600) + + if locale: + # Make sure we specify the locale-specific upload dir + worker.setdefault("env", {})["LOCALE"] = locale + + worker["artifacts"] = _generate_task_output_files( + dep_job, + worker_type_implementation(config.graph_config, config.params, worker_type), + repackage_config=repackage_config, + locale=locale, + ) + attributes["release_artifacts"] = [ + artifact["name"] for artifact in worker["artifacts"] + ] + + task = { + "label": job["label"], + "description": description, + "worker-type": worker_type, + "dependencies": dependencies, + "if-dependencies": [dep_job.kind], + "attributes": attributes, + "run-on-projects": job.get( + "run-on-projects", dep_job.attributes.get("run_on_projects") + ), + "optimization": dep_job.optimization, + "treeherder": treeherder, + "routes": job.get("routes", []), + "extra": job.get("extra", {}), + "worker": worker, + "run": run, + "fetches": _generate_download_config( + config, + dep_job, + build_platform, + signing_task, + repackage_signing_task, + locale=locale, + existing_fetch=job.get("fetches"), + ), + } + + if build_platform.startswith("macosx"): + task.setdefault("fetches", {}).setdefault("toolchain", []).extend( + [ + "linux64-libdmg", + "linux64-hfsplus", + "linux64-node", + "linux64-xar", + "linux64-mkbom", + ] + ) + + if "shipping-phase" in job: + task["shipping-phase"] = job["shipping-phase"] + + yield task + + +def _generate_download_config( + config, + task, + build_platform, + signing_task, + repackage_signing_task, + locale=None, + existing_fetch=None, +): + locale_path = f"{locale}/" if locale else "" + fetch = {} + if existing_fetch: + fetch.update(existing_fetch) + + if repackage_signing_task and build_platform.startswith("win"): + fetch.update( + { + repackage_signing_task: [f"{locale_path}target.installer.exe"], + } + ) + elif build_platform.startswith("linux") or build_platform.startswith("macosx"): + signing_fetch = [ + { + "artifact": "{}target{}".format( + locale_path, archive_format(build_platform) + ), + "extract": False, + }, + ] + if config.kind == "repackage-deb-l10n": + signing_fetch.append( + { + "artifact": f"{locale_path}target.langpack.xpi", + "extract": False, + } + ) + fetch.update({signing_task: signing_fetch}) + elif build_platform.startswith("win"): + fetch.update( + { + signing_task: [ + { + "artifact": f"{locale_path}target.zip", + "extract": False, + }, + f"{locale_path}setup.exe", + ], + } + ) + + use_stub = task.attributes.get("stub-installer") + if use_stub: + fetch[signing_task].append(f"{locale_path}setup-stub.exe") + + if fetch: + return fetch + + raise NotImplementedError(f'Unsupported build_platform: "{build_platform}"') + + +def _generate_task_output_files( + task, worker_implementation, repackage_config, locale=None +): + locale_output_path = f"{locale}/" if locale else "" + artifact_prefix = get_artifact_prefix(task) + + if worker_implementation == ("docker-worker", "linux"): + local_prefix = "/builds/worker/workspace/" + elif worker_implementation == ("generic-worker", "windows"): + local_prefix = "workspace/" + else: + raise NotImplementedError( + f'Unsupported worker implementation: "{worker_implementation}"' + ) + + output_files = [] + for config in repackage_config: + output_files.append( + { + "type": "file", + "path": "{}outputs/{}{}".format( + local_prefix, locale_output_path, config["output"] + ), + "name": "{}/{}{}".format( + artifact_prefix, locale_output_path, config["output"] + ), + } + ) + return output_files diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_l10n.py b/taskcluster/gecko_taskgraph/transforms/repackage_l10n.py new file mode 100644 index 0000000000..e0f46e6fdc --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/repackage_l10n.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the repackage task into an actual task description. +""" + + +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.copy_task import copy_task + +transforms = TransformSequence() + + +@transforms.add +def split_locales(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + for locale in dep_job.attributes.get("chunk_locales", []): + locale_job = copy_task(job) # don't overwrite dict values here + treeherder = locale_job.setdefault("treeherder", {}) + treeherder_group = locale_job.pop("treeherder-group") + treeherder["symbol"] = f"{treeherder_group}({locale})" + locale_job["locale"] = locale + yield locale_job diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_partner.py b/taskcluster/gecko_taskgraph/transforms/repackage_partner.py new file mode 100644 index 0000000000..582a86dfad --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/repackage_partner.py @@ -0,0 +1,302 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the repackage task into an actual task description. +""" + + +import copy + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from taskgraph.util.taskcluster import get_artifact_prefix +from voluptuous import Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.repackage import ( + PACKAGE_FORMATS as PACKAGE_FORMATS_VANILLA, +) +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.partners import get_partner_config_by_kind +from gecko_taskgraph.util.platforms import archive_format, executable_extension +from gecko_taskgraph.util.workertypes import worker_type_implementation + + +def _by_platform(arg): + return optionally_keyed_by("build-platform", arg) + + +# When repacking the stub installer we need to pass a zip file and package name to the +# repackage task. This is not needed for vanilla stub but analogous to the full installer. +PACKAGE_FORMATS = copy.deepcopy(PACKAGE_FORMATS_VANILLA) +PACKAGE_FORMATS["installer-stub"]["inputs"]["package"] = "target-stub{archive_format}" +PACKAGE_FORMATS["installer-stub"]["args"].extend(["--package-name", "{package-name}"]) + +packaging_description_schema = schema.extend( + { + # unique label to describe this repackaging task + Optional("label"): str, + # Routes specific to this task, if defined + Optional("routes"): [str], + # passed through directly to the job description + Optional("extra"): task_description_schema["extra"], + # Shipping product and phase + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Required("package-formats"): _by_platform([str]), + # All l10n jobs use mozharness + Required("mozharness"): { + # Config files passed to the mozharness script + Required("config"): _by_platform([str]), + # Additional paths to look for mozharness configs in. These should be + # relative to the base of the source checkout + Optional("config-paths"): [str], + # if true, perform a checkout of a comm-central based branch inside the + # gecko checkout + Optional("comm-checkout"): bool, + }, + # Override the default priority for the project + Optional("priority"): task_description_schema["priority"], + } +) + +transforms = TransformSequence() +transforms.add_validate(packaging_description_schema) + + +@transforms.add +def copy_in_useful_magic(config, jobs): + """Copy attributes from upstream task to be used for keyed configuration.""" + for job in jobs: + dep = job["primary-dependency"] + job["build-platform"] = dep.attributes.get("build_platform") + yield job + + +@transforms.add +def handle_keyed_by(config, jobs): + """Resolve fields that can be keyed by platform, etc.""" + fields = [ + "mozharness.config", + "package-formats", + ] + for job in jobs: + job = copy.deepcopy(job) # don't overwrite dict values here + for field in fields: + resolve_keyed_by(item=job, field=field, item_name="?") + yield job + + +@transforms.add +def make_repackage_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + + label = job.get("label", dep_job.label.replace("signing-", "repackage-")) + job["label"] = label + + yield job + + +@transforms.add +def make_job_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = copy_attributes_from_dependent_job(dep_job) + build_platform = attributes["build_platform"] + + if job["build-platform"].startswith("win"): + if dep_job.kind.endswith("signing"): + continue + if job["build-platform"].startswith("macosx"): + if dep_job.kind.endswith("repack"): + continue + dependencies = {dep_job.attributes.get("kind"): dep_job.label} + dependencies.update(dep_job.dependencies) + + signing_task = None + for dependency in dependencies.keys(): + if build_platform.startswith("macosx") and dependency.endswith("signing"): + signing_task = dependency + elif build_platform.startswith("win") and dependency.endswith("repack"): + signing_task = dependency + + attributes["repackage_type"] = "repackage" + + repack_id = job["extra"]["repack_id"] + + partner_config = get_partner_config_by_kind(config, config.kind) + partner, subpartner, _ = repack_id.split("/") + repack_stub_installer = partner_config[partner][subpartner].get( + "repack_stub_installer" + ) + if build_platform.startswith("win32") and repack_stub_installer: + job["package-formats"].append("installer-stub") + + repackage_config = [] + for format in job.get("package-formats"): + command = copy.deepcopy(PACKAGE_FORMATS[format]) + substs = { + "archive_format": archive_format(build_platform), + "executable_extension": executable_extension(build_platform), + } + command["inputs"] = { + name: filename.format(**substs) + for name, filename in command["inputs"].items() + } + repackage_config.append(command) + + run = job.get("mozharness", {}) + run.update( + { + "using": "mozharness", + "script": "mozharness/scripts/repackage.py", + "job-script": "taskcluster/scripts/builder/repackage.sh", + "actions": ["setup", "repackage"], + "extra-config": { + "repackage_config": repackage_config, + }, + } + ) + + worker = { + "chain-of-trust": True, + "max-run-time": 3600, + "taskcluster-proxy": True if get_artifact_prefix(dep_job) else False, + "env": { + "REPACK_ID": repack_id, + }, + # Don't add generic artifact directory. + "skip-artifacts": True, + } + + worker_type = "b-linux-gcp" + worker["docker-image"] = {"in-tree": "debian11-amd64-build"} + + worker["artifacts"] = _generate_task_output_files( + dep_job, + worker_type_implementation(config.graph_config, config.params, worker_type), + repackage_config, + partner=repack_id, + ) + + description = ( + "Repackaging for repack_id '{repack_id}' for build '" + "{build_platform}/{build_type}'".format( + repack_id=job["extra"]["repack_id"], + build_platform=attributes.get("build_platform"), + build_type=attributes.get("build_type"), + ) + ) + + task = { + "label": job["label"], + "description": description, + "worker-type": worker_type, + "dependencies": dependencies, + "attributes": attributes, + "scopes": ["queue:get-artifact:releng/partner/*"], + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "routes": job.get("routes", []), + "extra": job.get("extra", {}), + "worker": worker, + "run": run, + "fetches": _generate_download_config( + dep_job, + build_platform, + signing_task, + partner=repack_id, + project=config.params["project"], + repack_stub_installer=repack_stub_installer, + ), + } + + # we may have reduced the priority for partner jobs, otherwise task.py will set it + if job.get("priority"): + task["priority"] = job["priority"] + if build_platform.startswith("macosx"): + task.setdefault("fetches", {}).setdefault("toolchain", []).extend( + [ + "linux64-libdmg", + "linux64-hfsplus", + "linux64-node", + ] + ) + yield task + + +def _generate_download_config( + task, + build_platform, + signing_task, + partner=None, + project=None, + repack_stub_installer=False, +): + locale_path = f"{partner}/" if partner else "" + + if build_platform.startswith("macosx"): + return { + signing_task: [ + { + "artifact": f"{locale_path}target.tar.gz", + "extract": False, + }, + ], + } + if build_platform.startswith("win"): + download_config = [ + { + "artifact": f"{locale_path}target.zip", + "extract": False, + }, + f"{locale_path}setup.exe", + ] + if build_platform.startswith("win32") and repack_stub_installer: + download_config.extend( + [ + { + "artifact": f"{locale_path}target-stub.zip", + "extract": False, + }, + f"{locale_path}setup-stub.exe", + ] + ) + return {signing_task: download_config} + + raise NotImplementedError(f'Unsupported build_platform: "{build_platform}"') + + +def _generate_task_output_files(task, worker_implementation, repackage_config, partner): + """We carefully generate an explicit list here, but there's an artifacts directory + too, courtesy of generic_worker_add_artifacts() (windows) or docker_worker_add_artifacts(). + Any errors here are likely masked by that. + """ + partner_output_path = f"{partner}/" + artifact_prefix = get_artifact_prefix(task) + + if worker_implementation == ("docker-worker", "linux"): + local_prefix = "/builds/worker/workspace/" + elif worker_implementation == ("generic-worker", "windows"): + local_prefix = "workspace/" + else: + raise NotImplementedError( + f'Unsupported worker implementation: "{worker_implementation}"' + ) + + output_files = [] + for config in repackage_config: + output_files.append( + { + "type": "file", + "path": "{}outputs/{}{}".format( + local_prefix, partner_output_path, config["output"] + ), + "name": "{}/{}{}".format( + artifact_prefix, partner_output_path, config["output"] + ), + } + ) + return output_files diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_routes.py b/taskcluster/gecko_taskgraph/transforms/repackage_routes.py new file mode 100644 index 0000000000..2973ee35bd --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/repackage_routes.py @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Add indexes to repackage kinds +""" + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def add_indexes(config, jobs): + for job in jobs: + repackage_type = job["attributes"].get("repackage_type") + if repackage_type and job["attributes"]["build_type"] != "debug": + build_platform = job["attributes"]["build_platform"] + job_name = f"{build_platform}-{repackage_type}" + product = job.get("index", {}).get("product", "firefox") + index_type = "generic" + if job["attributes"].get("shippable") and job["attributes"].get("locale"): + index_type = "shippable-l10n" + if job["attributes"].get("shippable"): + index_type = "shippable" + if job["attributes"].get("locale"): + index_type = "l10n" + job["index"] = { + "job-name": job_name, + "product": product, + "type": index_type, + } + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_set_upstream_mac_kind.py b/taskcluster/gecko_taskgraph/transforms/repackage_set_upstream_mac_kind.py new file mode 100644 index 0000000000..14c865eea5 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/repackage_set_upstream_mac_kind.py @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform mac notarization tasks +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def repackage_set_upstream_mac_kind(config, tasks): + """ + Notarization only runs on level 3 + If level < 3 then repackage the mac-signing task artifact + Exception for debug builds, which will use signed build on level 3 + """ + for task in tasks: + if "macosx64" not in task["primary-dependency"].attributes["build_platform"]: + task.pop("upstream-mac-kind") + yield task + continue + resolve_keyed_by( + task, + "upstream-mac-kind", + item_name=config.kind, + **{ + "build-type": task["primary-dependency"].attributes["build_type"], + "project": config.params.get("project"), + } + ) + upstream_mac_kind = task.pop("upstream-mac-kind") + + if task["primary-dependency"].kind != upstream_mac_kind: + continue + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_signing.py b/taskcluster/gecko_taskgraph/transforms/repackage_signing.py new file mode 100644 index 0000000000..f98f8f0814 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/repackage_signing.py @@ -0,0 +1,137 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the repackage signing task into an actual task description. +""" + +import os + +from taskgraph.transforms.base import TransformSequence +from voluptuous import Optional + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.scriptworker import get_signing_cert_scope_per_platform + +repackage_signing_description_schema = schema.extend( + { + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + } +) + +SIGNING_FORMATS = { + "target.installer.exe": ["autograph_authenticode_sha2_stub"], + "target.stub-installer.exe": ["autograph_authenticode_sha2_stub"], + "target.installer.msi": ["autograph_authenticode_sha2"], + "target.installer.msix": ["autograph_authenticode_sha2"], +} + +transforms = TransformSequence() +transforms.add_validate(repackage_signing_description_schema) + + +@transforms.add +def make_repackage_signing_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = copy_attributes_from_dependent_job(dep_job) + locale = attributes.get("locale", dep_job.attributes.get("locale")) + attributes["repackage_type"] = "repackage-signing" + + treeherder = job.get("treeherder", {}) + treeherder.setdefault("symbol", "rs(B)") + dep_th_platform = dep_job.task.get("extra", {}).get("treeherder-platform") + treeherder.setdefault("platform", dep_th_platform) + treeherder.setdefault( + "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1) + ) + treeherder.setdefault("kind", "build") + + if locale: + treeherder["symbol"] = f"rs({locale})" + + if config.kind == "repackage-signing-msi": + treeherder["symbol"] = "MSIs({})".format(locale or "N") + + elif config.kind in ( + "repackage-signing-msix", + "repackage-signing-shippable-l10n-msix", + ): + # Like "MSIXs(Bs-multi)". + treeherder["symbol"] = "MSIXs({})".format( + dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol", "B") + ) + + label = job["label"] + + dep_kind = dep_job.kind + if "l10n" in dep_kind: + dep_kind = "repackage" + + dependencies = {dep_kind: dep_job.label} + + signing_dependencies = dep_job.dependencies + # This is so we get the build task etc in our dependencies to have better beetmover + # support. But for multi-locale MSIX packages, we don't want the signing task to directly + # depend on the langpack tasks. + dependencies.update( + { + k: v + for k, v in signing_dependencies.items() + if k != "docker-image" + and not k.startswith("shippable-l10n-signing-linux64") + } + ) + + description = ( + "Signing of repackaged artifacts for locale '{locale}' for build '" + "{build_platform}/{build_type}'".format( + locale=attributes.get("locale", "en-US"), + build_platform=attributes.get("build_platform"), + build_type=attributes.get("build_type"), + ) + ) + + build_platform = dep_job.attributes.get("build_platform") + is_shippable = dep_job.attributes.get("shippable") + signing_cert_scope = get_signing_cert_scope_per_platform( + build_platform, is_shippable, config + ) + scopes = [signing_cert_scope] + + upstream_artifacts = [] + for artifact in sorted(dep_job.attributes.get("release_artifacts")): + basename = os.path.basename(artifact) + if basename in SIGNING_FORMATS: + upstream_artifacts.append( + { + "taskId": {"task-reference": f"<{dep_kind}>"}, + "taskType": "repackage", + "paths": [artifact], + "formats": SIGNING_FORMATS[os.path.basename(artifact)], + } + ) + + task = { + "label": label, + "description": description, + "worker-type": "linux-signing" if is_shippable else "linux-depsigning", + "worker": { + "implementation": "scriptworker-signing", + "upstream-artifacts": upstream_artifacts, + "max-run-time": 3600, + }, + "scopes": scopes, + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "optimization": dep_job.optimization, + "treeherder": treeherder, + } + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py b/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py new file mode 100644 index 0000000000..eaf71f92a2 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py @@ -0,0 +1,145 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the repackage signing task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.taskcluster import get_artifact_path +from voluptuous import Optional + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.partners import get_partner_config_by_kind +from gecko_taskgraph.util.scriptworker import get_signing_cert_scope_per_platform + +transforms = TransformSequence() + +repackage_signing_description_schema = schema.extend( + { + Optional("label"): str, + Optional("extra"): object, + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("priority"): task_description_schema["priority"], + } +) + +transforms.add_validate(repackage_signing_description_schema) + + +@transforms.add +def make_repackage_signing_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + repack_id = dep_job.task["extra"]["repack_id"] + attributes = dep_job.attributes + build_platform = dep_job.attributes.get("build_platform") + is_shippable = dep_job.attributes.get("shippable") + + # Mac & windows + label = dep_job.label.replace("repackage-", "repackage-signing-") + # Linux + label = label.replace("chunking-dummy-", "repackage-signing-") + description = "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( # NOQA: E501 + repack_id=repack_id, + build_platform=attributes.get("build_platform"), + build_type=attributes.get("build_type"), + ) + + if "linux" in build_platform: + # we want the repack job, via the dependencies for the the chunking-dummy dep_job + for dep in dep_job.dependencies.values(): + if dep.startswith("release-partner-repack"): + dependencies = {"repack": dep} + break + else: + # we have a genuine repackage job as our parent + dependencies = {"repackage": dep_job.label} + + attributes = copy_attributes_from_dependent_job(dep_job) + attributes["repackage_type"] = "repackage-signing" + + signing_cert_scope = get_signing_cert_scope_per_platform( + build_platform, is_shippable, config + ) + scopes = [signing_cert_scope] + + if "win" in build_platform: + upstream_artifacts = [ + { + "taskId": {"task-reference": "<repackage>"}, + "taskType": "repackage", + "paths": [ + get_artifact_path(dep_job, f"{repack_id}/target.installer.exe"), + ], + "formats": ["autograph_authenticode_sha2", "autograph_gpg"], + } + ] + + partner_config = get_partner_config_by_kind(config, config.kind) + partner, subpartner, _ = repack_id.split("/") + repack_stub_installer = partner_config[partner][subpartner].get( + "repack_stub_installer" + ) + if build_platform.startswith("win32") and repack_stub_installer: + upstream_artifacts.append( + { + "taskId": {"task-reference": "<repackage>"}, + "taskType": "repackage", + "paths": [ + get_artifact_path( + dep_job, + f"{repack_id}/target.stub-installer.exe", + ), + ], + "formats": ["autograph_authenticode_sha2", "autograph_gpg"], + } + ) + elif "mac" in build_platform: + upstream_artifacts = [ + { + "taskId": {"task-reference": "<repackage>"}, + "taskType": "repackage", + "paths": [ + get_artifact_path(dep_job, f"{repack_id}/target.dmg"), + ], + "formats": ["autograph_gpg"], + } + ] + elif "linux" in build_platform: + upstream_artifacts = [ + { + "taskId": {"task-reference": "<repack>"}, + "taskType": "repackage", + "paths": [ + get_artifact_path(dep_job, f"{repack_id}/target.tar.bz2"), + ], + "formats": ["autograph_gpg"], + } + ] + + task = { + "label": label, + "description": description, + "worker-type": "linux-signing", + "worker": { + "implementation": "scriptworker-signing", + "upstream-artifacts": upstream_artifacts, + "max-run-time": 3600, + }, + "scopes": scopes, + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "extra": { + "repack_id": repack_id, + }, + } + # we may have reduced the priority for partner jobs, otherwise task.py will set it + if job.get("priority"): + task["priority"] = job["priority"] + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/repo_update.py b/taskcluster/gecko_taskgraph/transforms/repo_update.py new file mode 100644 index 0000000000..f4f135b585 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/repo_update.py @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the repo-update task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + env = task["worker"].setdefault("env", {}) + env["BRANCH"] = config.params["project"] + for envvar in env: + resolve_keyed_by(env, envvar, envvar, **config.params) + + for envvar in list(env.keys()): + if not env.get(envvar): + del env[envvar] + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py b/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py new file mode 100644 index 0000000000..5ad359bd5a --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py @@ -0,0 +1,67 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the reprocess-symbols task description template, +taskcluster/ci/reprocess-symbols/job-template.yml into an actual task description. +""" + + +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol + +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def fill_template(config, tasks): + for task in tasks: + assert len(task["dependent-tasks"]) == 2 + + build_dep = task["primary-dependency"] + upload_dep = None + for dep_idx in task["dependent-tasks"]: + dep = task["dependent-tasks"][dep_idx] + if dep_idx != build_dep: + upload_dep = dep + + task.pop("dependent-tasks", None) + + # Fill out the dynamic fields in the task description + task["label"] = build_dep.label + "-reprocess-symbols" + task["dependencies"] = {"build": build_dep.label, "upload": upload_dep.label} + task["worker"]["env"]["GECKO_HEAD_REPOSITORY"] = config.params[ + "head_repository" + ] + task["worker"]["env"]["GECKO_HEAD_REV"] = config.params["head_rev"] + task["worker"]["env"]["CRASHSTATS_SECRET"] = task["worker"]["env"][ + "CRASHSTATS_SECRET" + ].format(level=config.params["level"]) + + attributes = copy_attributes_from_dependent_job(build_dep) + attributes.update(task.get("attributes", {})) + task["attributes"] = attributes + + treeherder = inherit_treeherder_from_dep(task, build_dep) + th = build_dep.task.get("extra")["treeherder"] + th_symbol = th.get("symbol") + th_groupsymbol = th.get("groupSymbol", "?") + + # Disambiguate the treeherder symbol. + sym = "Rep" + (th_symbol[1:] if th_symbol.startswith("B") else th_symbol) + treeherder.setdefault("symbol", join_symbol(th_groupsymbol, sym)) + task["treeherder"] = treeherder + + task["run-on-projects"] = build_dep.attributes.get("run_on_projects") + task["optimization"] = {"reprocess-symbols": None} + task["if-dependencies"] = ["build"] + + del task["primary-dependency"] + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/reverse_chunk_deps.py b/taskcluster/gecko_taskgraph/transforms/reverse_chunk_deps.py new file mode 100644 index 0000000000..ac2f282799 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/reverse_chunk_deps.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Adjust dependencies to not exceed MAX_DEPENDENCIES +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import add_suffix + +import gecko_taskgraph.transforms.release_deps as release_deps +from gecko_taskgraph import MAX_DEPENDENCIES +from gecko_taskgraph.util.copy_task import copy_task + +transforms = TransformSequence() + + +def yield_job(orig_job, deps, count): + job = copy_task(orig_job) + job["dependencies"] = deps + job["name"] = "{}-{}".format(orig_job["name"], count) + if "treeherder" in job: + job["treeherder"]["symbol"] = add_suffix( + job["treeherder"]["symbol"], f"-{count}" + ) + + return job + + +@transforms.add +def add_dependencies(config, jobs): + for job in release_deps.add_dependencies(config, jobs): + count = 1 + deps = {} + + # sort for deterministic chunking + for dep_label in sorted(job["dependencies"].keys()): + deps[dep_label] = dep_label + if len(deps) == MAX_DEPENDENCIES: + yield yield_job(job, deps, count) + deps = {} + count += 1 + if deps: + yield yield_job(job, deps, count) + count += 1 diff --git a/taskcluster/gecko_taskgraph/transforms/run_pgo_profile.py b/taskcluster/gecko_taskgraph/transforms/run_pgo_profile.py new file mode 100644 index 0000000000..2585bdd712 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/run_pgo_profile.py @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the pgo jobs. +""" + +import logging + +from taskgraph.transforms.base import TransformSequence + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def run_profile_data(config, jobs): + for job in jobs: + build_platform = job["attributes"].get("build_platform") + instr = "instrumented-build-{}".format(job["name"]) + if "android" in build_platform: + artifact = "geckoview-test_runner.apk" + elif "macosx64" in build_platform: + artifact = "target.dmg" + elif "win" in build_platform: + artifact = "target.zip" + else: + artifact = "target.tar.bz2" + job.setdefault("fetches", {})[instr] = [ + artifact, + "target.crashreporter-symbols.zip", + ] + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/scriptworker.py b/taskcluster/gecko_taskgraph/transforms/scriptworker.py new file mode 100644 index 0000000000..5d382702af --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/scriptworker.py @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Transforms for adding appropriate scopes to scriptworker tasks. +""" + + +from gecko_taskgraph.util.scriptworker import get_balrog_server_scope + + +def add_balrog_scopes(config, jobs): + for job in jobs: + server_scope = get_balrog_server_scope(config) + job["scopes"] = [server_scope] + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/scriptworker_canary.py b/taskcluster/gecko_taskgraph/transforms/scriptworker_canary.py new file mode 100644 index 0000000000..43735f3dce --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/scriptworker_canary.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Build a command to run `mach release push-scriptworker-canaries`. +""" + + +from pipes import quote as shell_quote + +from mozrelease.scriptworker_canary import TASK_TYPES +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def build_command(config, jobs): + scriptworkers = config.params["try_task_config"].get( + "scriptworker-canary-workers", [] + ) + # Filter the list of workers to those we have configured a set of canary + # tasks for. + scriptworkers = [ + scriptworker for scriptworker in scriptworkers if scriptworker in TASK_TYPES + ] + + if not scriptworkers: + return + + for job in jobs: + + command = ["release", "push-scriptworker-canary"] + for scriptworker in scriptworkers: + command.extend(["--scriptworker", scriptworker]) + for address in job.pop("addresses"): + command.extend(["--address", address]) + if "ssh-key-secret" in job: + ssh_key_secret = job.pop("ssh-key-secret") + command.extend(["--ssh-key-secret", ssh_key_secret]) + job.setdefault("scopes", []).append(f"secrets:get:{ssh_key_secret}") + + job.setdefault("run", {}).update( + {"using": "mach", "mach": " ".join(map(shell_quote, command))} + ) + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/sentry.py b/taskcluster/gecko_taskgraph/transforms/sentry.py new file mode 100644 index 0000000000..2e43a15518 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/sentry.py @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def sentry(config, tasks): + """Do transforms specific to github-sync tasks.""" + if config.params["project"] not in ["mozilla-central", "try"]: + return + for task in tasks: + scopes = [ + scope.format(level=config.params["level"]) for scope in task["scopes"] + ] + task["scopes"] = scopes + + env = { + key: value.format( + level=config.params["level"], + head_repository=config.params["head_repository"], + head_rev=config.params["head_rev"], + ) + for (key, value) in task["worker"]["env"].items() + } + task["worker"]["env"] = env + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/shippable_l10n_signing.py b/taskcluster/gecko_taskgraph/transforms/shippable_l10n_signing.py new file mode 100644 index 0000000000..324efa92dd --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/shippable_l10n_signing.py @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the signing task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import join_symbol + +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.signed_artifacts import ( + generate_specifications_of_artifacts_to_sign, +) + +transforms = TransformSequence() + + +@transforms.add +def make_signing_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + job["depname"] = dep_job.label + + # add the chunk number to the TH symbol + symbol = job.get("treeherder", {}).get("symbol", "Bs") + symbol = "{}{}".format(symbol, dep_job.attributes.get("l10n_chunk")) + group = "L10n" + + job["treeherder"] = { + "symbol": join_symbol(group, symbol), + } + + yield job + + +@transforms.add +def define_upstream_artifacts(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + upstream_artifact_task = job.pop("upstream-artifact-task", dep_job) + + job["attributes"] = copy_attributes_from_dependent_job(dep_job) + if dep_job.attributes.get("chunk_locales"): + # Used for l10n attribute passthrough + job["attributes"]["chunk_locales"] = dep_job.attributes.get("chunk_locales") + + locale_specifications = generate_specifications_of_artifacts_to_sign( + config, + job, + keep_locale_template=True, + dep_kind=upstream_artifact_task.kind, + ) + + upstream_artifacts = [] + for spec in locale_specifications: + upstream_task_type = "l10n" + if upstream_artifact_task.kind.endswith( + ("-mac-notarization", "-mac-signing") + ): + # Upstream is mac signing or notarization + upstream_task_type = "scriptworker" + upstream_artifacts.append( + { + "taskId": {"task-reference": f"<{upstream_artifact_task.kind}>"}, + "taskType": upstream_task_type, + # Set paths based on artifacts in the specs (above) one per + # locale present in the chunk this is signing stuff for. + # Pass paths through set and sorted() so we get a list back + # and we remove any duplicates (e.g. hardcoded ja-JP-mac langpack) + "paths": sorted( + { + path_template.format(locale=locale) + for locale in upstream_artifact_task.attributes.get( + "chunk_locales", [] + ) + for path_template in spec["artifacts"] + } + ), + "formats": spec["formats"], + } + ) + + job["upstream-artifacts"] = upstream_artifacts + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/signing.py b/taskcluster/gecko_taskgraph/transforms/signing.py new file mode 100644 index 0000000000..9a5873fc81 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/signing.py @@ -0,0 +1,266 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the signing task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.keyed_by import evaluate_keyed_by +from taskgraph.util.schema import taskref_or_string +from voluptuous import Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + release_level, +) +from gecko_taskgraph.util.scriptworker import ( + add_scope_prefix, + get_signing_cert_scope_per_platform, +) + +transforms = TransformSequence() + +signing_description_schema = schema.extend( + { + # Artifacts from dep task to sign - Sync with taskgraph/transforms/task.py + # because this is passed directly into the signingscript worker + Required("upstream-artifacts"): [ + { + # taskId of the task with the artifact + Required("taskId"): taskref_or_string, + # type of signing task (for CoT) + Required("taskType"): str, + # Paths to the artifacts to sign + Required("paths"): [str], + # Signing formats to use on each of the paths + Required("formats"): [str], + } + ], + # depname is used in taskref's to identify the taskID of the unsigned things + Required("depname"): str, + # attributes for this task + Optional("attributes"): {str: object}, + # unique label to describe this signing task, defaults to {dep.label}-signing + Optional("label"): str, + # treeherder is allowed here to override any defaults we use for signing. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + # Routes specific to this task, if defined + Optional("routes"): [str], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("dependent-tasks"): {str: object}, + # Optional control for how long a task may run (aka maxRunTime) + Optional("max-run-time"): int, + Optional("extra"): {str: object}, + # Max number of partner repacks per chunk + Optional("repacks-per-chunk"): int, + # Override the default priority for the project + Optional("priority"): task_description_schema["priority"], + } +) + + +@transforms.add +def set_defaults(config, jobs): + for job in jobs: + if not job.get("depname"): + dep_job = job["primary-dependency"] + job["depname"] = dep_job.kind + yield job + + +transforms.add_validate(signing_description_schema) + + +@transforms.add +def add_entitlements_link(config, jobs): + for job in jobs: + entitlements_path = evaluate_keyed_by( + config.graph_config["mac-notarization"]["mac-entitlements"], + "mac entitlements", + { + "platform": job["primary-dependency"].attributes.get("build_platform"), + "release-level": release_level(config.params["project"]), + }, + ) + if entitlements_path: + job["entitlements-url"] = config.params.file_url( + entitlements_path, + ) + yield job + + +@transforms.add +def add_requirements_link(config, jobs): + for job in jobs: + requirements_path = evaluate_keyed_by( + config.graph_config["mac-notarization"]["mac-requirements"], + "mac requirements", + { + "platform": job["primary-dependency"].attributes.get("build_platform"), + }, + ) + if requirements_path: + job["requirements-plist-url"] = config.params.file_url( + requirements_path, + ) + yield job + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = dep_job.attributes + + signing_format_scopes = [] + formats = set() + for artifacts in job["upstream-artifacts"]: + for f in artifacts["formats"]: + formats.add(f) # Add each format only once + + is_shippable = dep_job.attributes.get("shippable", False) + build_platform = dep_job.attributes.get("build_platform") + treeherder = None + if "partner" not in config.kind and "eme-free" not in config.kind: + treeherder = job.get("treeherder", {}) + + dep_th_platform = ( + dep_job.task.get("extra", {}) + .get("treeherder", {}) + .get("machine", {}) + .get("platform", "") + ) + build_type = dep_job.attributes.get("build_type") + treeherder.setdefault( + "platform", + _generate_treeherder_platform( + dep_th_platform, build_platform, build_type + ), + ) + + # ccov builds are tier 2, so they cannot have tier 1 tasks + # depending on them. + treeherder.setdefault( + "tier", + dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1), + ) + treeherder.setdefault( + "symbol", + _generate_treeherder_symbol( + dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol") + ), + ) + treeherder.setdefault("kind", "build") + + label = job["label"] + description = ( + "Initial Signing for locale '{locale}' for build '" + "{build_platform}/{build_type}'".format( + locale=attributes.get("locale", "en-US"), + build_platform=build_platform, + build_type=attributes.get("build_type"), + ) + ) + + attributes = ( + job["attributes"] + if job.get("attributes") + else copy_attributes_from_dependent_job(dep_job) + ) + attributes["signed"] = True + + if "linux" in build_platform: + attributes["release_artifacts"] = ["public/build/KEY"] + + if dep_job.attributes.get("chunk_locales"): + # Used for l10n attribute passthrough + attributes["chunk_locales"] = dep_job.attributes.get("chunk_locales") + + signing_cert_scope = get_signing_cert_scope_per_platform( + build_platform, is_shippable, config + ) + worker_type_alias = "linux-signing" if is_shippable else "linux-depsigning" + task = { + "label": label, + "description": description, + "worker": { + "implementation": "scriptworker-signing", + "upstream-artifacts": job["upstream-artifacts"], + "max-run-time": job.get("max-run-time", 3600), + }, + "scopes": [signing_cert_scope] + signing_format_scopes, + "dependencies": _generate_dependencies(job), + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "optimization": dep_job.optimization, + "routes": job.get("routes", []), + "shipping-product": job.get("shipping-product"), + "shipping-phase": job.get("shipping-phase"), + } + if dep_job.kind in task["dependencies"]: + task["if-dependencies"] = [dep_job.kind] + + # build-mac-{signing,notarization} uses signingscript instead of iscript + if "macosx" in build_platform and config.kind.endswith("-mac-notarization"): + task["worker"]["mac-behavior"] = "apple_notarization" + task["scopes"] = [ + add_scope_prefix(config, "signing:cert:release-apple-notarization") + ] + elif "macosx" in build_platform: + # iscript overrides + task["worker"]["mac-behavior"] = "mac_sign_and_pkg" + + worker_type_alias_map = { + "linux-depsigning": "mac-depsigning", + "linux-signing": "mac-signing", + } + assert worker_type_alias in worker_type_alias_map, ( + "Make sure to adjust the below worker_type_alias logic for " + "mac if you change the signing workerType aliases!" + " ({} not found in mapping)".format(worker_type_alias) + ) + worker_type_alias = worker_type_alias_map[worker_type_alias] + for attr in ("entitlements-url", "requirements-plist-url"): + if job.get(attr): + task["worker"][attr] = job[attr] + + task["worker-type"] = worker_type_alias + if treeherder: + task["treeherder"] = treeherder + if job.get("extra"): + task["extra"] = job["extra"] + # we may have reduced the priority for partner jobs, otherwise task.py will set it + if job.get("priority"): + task["priority"] = job["priority"] + + yield task + + +def _generate_dependencies(job): + if isinstance(job.get("dependent-tasks"), dict): + deps = {} + for k, v in job["dependent-tasks"].items(): + deps[k] = v.label + return deps + return {job["depname"]: job["primary-dependency"].label} + + +def _generate_treeherder_platform(dep_th_platform, build_platform, build_type): + if "-pgo" in build_platform: + actual_build_type = "pgo" + elif "-ccov" in build_platform: + actual_build_type = "ccov" + else: + actual_build_type = build_type + return f"{dep_th_platform}/{actual_build_type}" + + +def _generate_treeherder_symbol(build_symbol): + symbol = build_symbol + "s" + return symbol diff --git a/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py b/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py new file mode 100644 index 0000000000..60c5f30ed3 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py @@ -0,0 +1,83 @@ +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the checksums signing task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from voluptuous import Optional + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job +from gecko_taskgraph.util.scriptworker import get_signing_cert_scope + +checksums_signing_description_schema = schema.extend( + { + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + } +) + +transforms = TransformSequence() +transforms.add_validate(checksums_signing_description_schema) + + +@transforms.add +def make_checksums_signing_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + attributes = dep_job.attributes + + treeherder = job.get("treeherder", {}) + treeherder.setdefault("symbol", "css(N)") + dep_th_platform = ( + dep_job.task.get("extra", {}) + .get("treeherder", {}) + .get("machine", {}) + .get("platform", "") + ) + treeherder.setdefault("platform", f"{dep_th_platform}/opt") + treeherder.setdefault("tier", 1) + treeherder.setdefault("kind", "build") + + label = job["label"] + description = "Signing of release-source checksums file" + dependencies = {"beetmover": dep_job.label} + + attributes = copy_attributes_from_dependent_job(dep_job) + + upstream_artifacts = [ + { + "taskId": {"task-reference": "<beetmover>"}, + "taskType": "beetmover", + "paths": [ + "public/target-source.checksums", + ], + "formats": ["autograph_gpg"], + } + ] + + signing_cert_scope = get_signing_cert_scope(config) + + task = { + "label": label, + "description": description, + "worker-type": "linux-signing", + "worker": { + "implementation": "scriptworker-signing", + "upstream-artifacts": upstream_artifacts, + "max-run-time": 3600, + }, + "scopes": [ + signing_cert_scope, + ], + "dependencies": dependencies, + "attributes": attributes, + "run-on-projects": dep_job.attributes.get("run_on_projects"), + "treeherder": treeherder, + } + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/source_test.py b/taskcluster/gecko_taskgraph/transforms/source_test.py new file mode 100644 index 0000000000..b480fcad02 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/source_test.py @@ -0,0 +1,270 @@ +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Source-test jobs can run on multiple platforms. These transforms allow jobs +with either `platform` or a list of `platforms`, and set the appropriate +treeherder configuration and attributes for that platform. +""" + + +import copy +import os + +import taskgraph +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.attributes import keymatch +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from taskgraph.util.treeherder import join_symbol, split_symbol +from voluptuous import Any, Extra, Optional, Required + +from gecko_taskgraph.transforms.job import job_description_schema +from gecko_taskgraph.util.hg import get_json_automationrelevance + +source_test_description_schema = Schema( + { + # most fields are passed directly through as job fields, and are not + # repeated here + Extra: object, + # The platform on which this task runs. This will be used to set up attributes + # (for try selection) and treeherder metadata (for display). If given as a list, + # the job will be "split" into multiple tasks, one with each platform. + Required("platform"): Any(str, [str]), + # Build labels required for the task. If this key is provided it must + # contain a build label for the task platform. + # The task will then depend on a build task, and the installer url will be + # saved to the GECKO_INSTALLER_URL environment variable. + Optional("require-build"): optionally_keyed_by("project", {str: str}), + # These fields can be keyed by "platform", and are otherwise identical to + # job descriptions. + Required("worker-type"): optionally_keyed_by( + "platform", job_description_schema["worker-type"] + ), + Required("worker"): optionally_keyed_by( + "platform", job_description_schema["worker"] + ), + Optional("python-version"): [int], + Optional("dependencies"): { + k: optionally_keyed_by("platform", v) + for k, v in job_description_schema["dependencies"].items() + }, + # A list of artifacts to install from 'fetch' tasks. + Optional("fetches"): { + str: optionally_keyed_by( + "platform", job_description_schema["fetches"][str] + ), + }, + } +) + +transforms = TransformSequence() + +transforms.add_validate(source_test_description_schema) + + +@transforms.add +def set_job_name(config, jobs): + for job in jobs: + if "job-from" in job and job["job-from"] != "kind.yml": + from_name = os.path.splitext(job["job-from"])[0] + job["name"] = "{}-{}".format(from_name, job["name"]) + yield job + + +@transforms.add +def expand_platforms(config, jobs): + for job in jobs: + if isinstance(job["platform"], str): + yield job + continue + + for platform in job["platform"]: + pjob = copy.deepcopy(job) + pjob["platform"] = platform + + if "name" in pjob: + pjob["name"] = "{}-{}".format(pjob["name"], platform) + else: + pjob["label"] = "{}-{}".format(pjob["label"], platform) + yield pjob + + +@transforms.add +def split_python(config, jobs): + for job in jobs: + key = "python-version" + versions = job.pop(key, []) + if not versions: + yield job + continue + for version in versions: + group = f"py{version}" + pyjob = copy.deepcopy(job) + if "name" in pyjob: + pyjob["name"] += f"-{group}" + else: + pyjob["label"] += f"-{group}" + symbol = split_symbol(pyjob["treeherder"]["symbol"])[1] + pyjob["treeherder"]["symbol"] = join_symbol(group, symbol) + pyjob["run"][key] = version + yield pyjob + + +@transforms.add +def split_jsshell(config, jobs): + all_shells = {"sm": "Spidermonkey", "v8": "Google V8"} + + for job in jobs: + if not job["name"].startswith("jsshell"): + yield job + continue + + test = job.pop("test") + for shell in job.get("shell", all_shells.keys()): + assert shell in all_shells + + new_job = copy.deepcopy(job) + new_job["name"] = "{}-{}".format(new_job["name"], shell) + new_job["description"] = "{} on {}".format( + new_job["description"], all_shells[shell] + ) + new_job["shell"] = shell + + group = f"js-bench-{shell}" + symbol = split_symbol(new_job["treeherder"]["symbol"])[1] + new_job["treeherder"]["symbol"] = join_symbol(group, symbol) + + run = new_job["run"] + run["mach"] = run["mach"].format( + shell=shell, SHELL=shell.upper(), test=test + ) + yield new_job + + +def add_build_dependency(config, job): + """ + Add build dependency to the job and installer_url to env. + """ + key = job["platform"] + build_labels = job.pop("require-build", {}) + matches = keymatch(build_labels, key) + if not matches: + raise Exception( + "No build platform found. " + "Define 'require-build' for {} in the task config.".format(key) + ) + + if len(matches) > 1: + raise Exception(f"More than one build platform found for '{key}'.") + + label = matches[0] + deps = job.setdefault("dependencies", {}) + deps.update({"build": label}) + + +@transforms.add +def handle_platform(config, jobs): + """ + Handle the 'platform' property, setting up treeherder context as well as + try-related attributes. + """ + fields = [ + "always-target", + "fetches.toolchain", + "require-build", + "worker-type", + "worker", + ] + + for job in jobs: + platform = job["platform"] + + for field in fields: + resolve_keyed_by( + job, field, item_name=job["name"], project=config.params["project"] + ) + for field in job.get("dependencies", {}): + resolve_keyed_by( + job, + f"dependencies.{field}", + item_name=job["name"], + project=config.params["project"], + ) + + if "treeherder" in job: + job["treeherder"].setdefault("platform", platform) + + if "require-build" in job: + add_build_dependency(config, job) + + del job["platform"] + yield job + + +@transforms.add +def handle_shell(config, jobs): + """ + Handle the 'shell' property. + """ + fields = [ + "run-on-projects", + "worker.env", + ] + + for job in jobs: + if not job.get("shell"): + yield job + continue + + for field in fields: + resolve_keyed_by(job, field, item_name=job["name"]) + + del job["shell"] + yield job + + +@transforms.add +def set_code_review_env(config, jobs): + """ + Add a CODE_REVIEW environment variable when running in code-review bot mode + """ + is_code_review = config.params["target_tasks_method"] == "codereview" + + for job in jobs: + attrs = job.get("attributes", {}) + if is_code_review and attrs.get("code-review") is True: + env = job["worker"].setdefault("env", {}) + env["CODE_REVIEW"] = "1" + + yield job + + +@transforms.add +def set_base_revision_in_tgdiff(config, jobs): + # Don't attempt to download 'json-automation' locally as the revision may + # not exist in the repository. + if not os.environ.get("MOZ_AUTOMATION") or taskgraph.fast: + yield from jobs + return + + data = get_json_automationrelevance( + config.params["head_repository"], config.params["head_rev"] + ) + for job in jobs: + if job["name"] != "taskgraph-diff": + yield job + continue + + job["run"]["command-context"] = { + "base_rev": data["changesets"][0]["parents"][0] + } + yield job + + +@transforms.add +def set_worker_exit_code(config, jobs): + for job in jobs: + worker = job["worker"] + worker.setdefault("retry-exit-status", []) + if 137 not in worker["retry-exit-status"]: + worker["retry-exit-status"].append(137) + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/spidermonkey.py b/taskcluster/gecko_taskgraph/transforms/spidermonkey.py new file mode 100644 index 0000000000..8e652f1668 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/spidermonkey.py @@ -0,0 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import copy + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def handle_keyed_by(config, jobs): + """Resolve fields that can be keyed by platform, etc.""" + fields = ["fetches.toolchain"] + for job in jobs: + job = copy.deepcopy(job) # don't overwrite dict values here + for field in fields: + resolve_keyed_by(item=job, field=field, item_name=job["name"]) + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/split_by_locale.py b/taskcluster/gecko_taskgraph/transforms/split_by_locale.py new file mode 100644 index 0000000000..ae68ab5051 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/split_by_locale.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +This transform splits the jobs it receives into per-locale tasks. Locales are +provided by the `locales-file`. +""" + +from copy import deepcopy +from pprint import pprint + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema +from voluptuous import Extra, Optional, Required + +from gecko_taskgraph.transforms.l10n import parse_locales_file + +transforms = TransformSequence() + +split_by_locale_schema = Schema( + { + # The file to pull locale information from. This should be a json file + # such as browser/locales/l10n-changesets.json. + Required("locales-file"): str, + # The platform name in the form used by the locales files. Defaults to + # attributes.build_platform if not provided. + Optional("locale-file-platform"): str, + # A list of properties elsewhere in the job that need to have the locale + # name substituted into them. The referenced properties may be strings + # or lists. In the case of the latter, all list values will have + # substitutions performed. + Optional("properties-with-locale"): [str], + Extra: object, + } +) + + +transforms.add_validate(split_by_locale_schema) + + +@transforms.add +def add_command(config, jobs): + for job in jobs: + locales_file = job.pop("locales-file") + properties_with_locale = job.pop("properties-with-locale") + build_platform = job.pop( + "locale-file-platform", job["attributes"]["build_platform"] + ) + + for locale in parse_locales_file(locales_file, build_platform): + locale_job = deepcopy(job) + locale_job["attributes"]["locale"] = locale + for prop in properties_with_locale: + container, subfield = locale_job, prop + while "." in subfield: + f, subfield = subfield.split(".", 1) + if f not in container: + raise Exception( + f"Unable to find property {prop} to perform locale substitution on. Job is:\n{pprint(job)}" + ) + container = container[f] + if not isinstance(container, dict): + raise Exception( + f"{container} is not a dict, cannot perform locale substitution. Job is:\n{pprint(job)}" + ) + + if isinstance(container[subfield], str): + container[subfield] = container[subfield].format(locale=locale) + elif isinstance(container[subfield], list): + for i in range(len(container[subfield])): + container[subfield][i] = container[subfield][i].format( + locale=locale + ) + else: + raise Exception( + f"Don't know how to subtitute locale for value of type: {type(container[subfield])}; value is: {container[subfield]}" + ) + + yield locale_job diff --git a/taskcluster/gecko_taskgraph/transforms/startup_test.py b/taskcluster/gecko_taskgraph/transforms/startup_test.py new file mode 100644 index 0000000000..2660ef6e93 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/startup_test.py @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def add_command(config, jobs): + for job in jobs: + extra_config = job.pop("extra-config") + upstream_kind = extra_config["upstream_kind"] + upstream_artifact = extra_config["upstream_artifact"] + binary = extra_config["binary"] + package_to_test = "<{}/public/build/{}>".format( + upstream_kind, upstream_artifact + ) + + if job["attributes"]["build_platform"].startswith("linux"): + job["run"]["command"] = { + "artifact-reference": ". $HOME/scripts/xvfb.sh && start_xvfb '1600x1200x24' 0 && " + + "python3 ./mach python testing/mozharness/scripts/does_it_crash.py " + + "--run-for 30 --thing-url " + + package_to_test + + " --thing-to-run " + + binary + } + else: + job["run"]["mach"] = { + "artifact-reference": "python testing/mozharness/scripts/does_it_crash.py " + + "--run-for 30 --thing-url " + + package_to_test + + " --thing-to-run " + + binary + } + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/strip_dependent_task.py b/taskcluster/gecko_taskgraph/transforms/strip_dependent_task.py new file mode 100644 index 0000000000..4e1ec8783a --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/strip_dependent_task.py @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +FIXME +""" + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def strip_dependent_task(config, jobs): + for job in jobs: + del job["primary-dependency"] + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/task.py b/taskcluster/gecko_taskgraph/transforms/task.py new file mode 100644 index 0000000000..e823e96b21 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/task.py @@ -0,0 +1,2266 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +These transformations take a task description and turn it into a TaskCluster +task definition (along with attributes, label, etc.). The input to these +transformations is generic to any kind of task, but abstracts away some of the +complexities of worker implementations, scopes, and treeherder annotations. +""" + + +import datetime +import hashlib +import os +import re +import time + +import attr +from mozbuild.util import memoize +from taskcluster.utils import fromNow +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.keyed_by import evaluate_keyed_by +from taskgraph.util.schema import ( + Schema, + optionally_keyed_by, + resolve_keyed_by, + taskref_or_string, + validate_schema, +) +from taskgraph.util.treeherder import split_symbol +from voluptuous import All, Any, Extra, Match, NotIn, Optional, Required + +from gecko_taskgraph import GECKO, MAX_DEPENDENCIES +from gecko_taskgraph.optimize.schema import OptimizationSchema +from gecko_taskgraph.transforms.job.common import get_expiration +from gecko_taskgraph.util import docker as dockerutil +from gecko_taskgraph.util.attributes import TRUNK_PROJECTS, is_try, release_level +from gecko_taskgraph.util.copy_task import copy_task +from gecko_taskgraph.util.hash import hash_path +from gecko_taskgraph.util.partners import get_partners_to_be_published +from gecko_taskgraph.util.scriptworker import BALROG_ACTIONS, get_release_config +from gecko_taskgraph.util.signed_artifacts import get_signed_artifacts +from gecko_taskgraph.util.workertypes import get_worker_type, worker_type_implementation + +RUN_TASK = os.path.join(GECKO, "taskcluster", "scripts", "run-task") + +SCCACHE_GCS_PROJECT = "sccache-3" + + +@memoize +def _run_task_suffix(): + """String to append to cache names under control of run-task.""" + return hash_path(RUN_TASK)[0:20] + + +def _compute_geckoview_version(app_version, moz_build_date): + """Geckoview version string that matches geckoview gradle configuration""" + # Must be synchronized with /mobile/android/geckoview/build.gradle computeVersionCode(...) + version_without_milestone = re.sub(r"a[0-9]", "", app_version, 1) + parts = version_without_milestone.split(".") + return f"{parts[0]}.{parts[1]}.{moz_build_date}" + + +# A task description is a general description of a TaskCluster task +task_description_schema = Schema( + { + # the label for this task + Required("label"): str, + # description of the task (for metadata) + Required("description"): str, + # attributes for this task + Optional("attributes"): {str: object}, + # relative path (from config.path) to the file task was defined in + Optional("job-from"): str, + # dependencies of this task, keyed by name; these are passed through + # verbatim and subject to the interpretation of the Task's get_dependencies + # method. + Optional("dependencies"): { + All( + str, + NotIn( + ["self", "decision"], + "Can't use 'self` or 'decision' as depdency names.", + ), + ): object, + }, + # Soft dependencies of this task, as a list of tasks labels + Optional("soft-dependencies"): [str], + # Dependencies that must be scheduled in order for this task to run. + Optional("if-dependencies"): [str], + Optional("requires"): Any("all-completed", "all-resolved"), + # expiration and deadline times, relative to task creation, with units + # (e.g., "14 days"). Defaults are set based on the project. + Optional("expires-after"): str, + Optional("deadline-after"): str, + Optional("expiration-policy"): str, + # custom routes for this task; the default treeherder routes will be added + # automatically + Optional("routes"): [str], + # custom scopes for this task; any scopes required for the worker will be + # added automatically. The following parameters will be substituted in each + # scope: + # {level} -- the scm level of this push + # {project} -- the project of this push + Optional("scopes"): [str], + # Tags + Optional("tags"): {str: str}, + # custom "task.extra" content + Optional("extra"): {str: object}, + # treeherder-related information; see + # https://firefox-ci-tc.services.mozilla.com/schemas/taskcluster-treeherder/v1/task-treeherder-config.json + # If not specified, no treeherder extra information or routes will be + # added to the task + Optional("treeherder"): { + # either a bare symbol, or "grp(sym)". + "symbol": str, + # the job kind + "kind": Any("build", "test", "other"), + # tier for this task + "tier": int, + # task platform, in the form platform/collection, used to set + # treeherder.machine.platform and treeherder.collection or + # treeherder.labels + "platform": Match("^[A-Za-z0-9_-]{1,50}/[A-Za-z0-9_-]{1,50}$"), + }, + # information for indexing this build so its artifacts can be discovered; + # if omitted, the build will not be indexed. + Optional("index"): { + # the name of the product this build produces + "product": str, + # the names to use for this job in the TaskCluster index + "job-name": str, + # Type of gecko v2 index to use + "type": Any( + "generic", + "l10n", + "shippable", + "shippable-l10n", + "android-shippable", + "android-shippable-with-multi-l10n", + "shippable-with-multi-l10n", + ), + # The rank that the task will receive in the TaskCluster + # index. A newly completed task supercedes the currently + # indexed task iff it has a higher rank. If unspecified, + # 'by-tier' behavior will be used. + "rank": Any( + # Rank is equal the timestamp of the build_date for tier-1 + # tasks, and zero for non-tier-1. This sorts tier-{2,3} + # builds below tier-1 in the index. + "by-tier", + # Rank is given as an integer constant (e.g. zero to make + # sure a task is last in the index). + int, + # Rank is equal to the timestamp of the build_date. This + # option can be used to override the 'by-tier' behavior + # for non-tier-1 tasks. + "build_date", + ), + }, + # The `run_on_projects` attribute, defaulting to "all". This dictates the + # projects on which this task should be included in the target task set. + # See the attributes documentation for details. + Optional("run-on-projects"): optionally_keyed_by("build-platform", [str]), + # Like `run_on_projects`, `run-on-hg-branches` defaults to "all". + Optional("run-on-hg-branches"): optionally_keyed_by("project", [str]), + # The `shipping_phase` attribute, defaulting to None. This specifies the + # release promotion phase that this task belongs to. + Required("shipping-phase"): Any( + None, + "build", + "promote", + "push", + "ship", + ), + # The `shipping_product` attribute, defaulting to None. This specifies the + # release promotion product that this task belongs to. + Required("shipping-product"): Any(None, str), + # The `always-target` attribute will cause the task to be included in the + # target_task_graph regardless of filtering. Tasks included in this manner + # will be candidates for optimization even when `optimize_target_tasks` is + # False, unless the task was also explicitly chosen by the target_tasks + # method. + Required("always-target"): bool, + # Optimization to perform on this task during the optimization phase. + # Optimizations are defined in taskcluster/gecko_taskgraph/optimize.py. + Required("optimization"): OptimizationSchema, + # the provisioner-id/worker-type for the task. The following parameters will + # be substituted in this string: + # {level} -- the scm level of this push + "worker-type": str, + # Whether the job should use sccache compiler caching. + Required("use-sccache"): bool, + # information specific to the worker implementation that will run this task + Optional("worker"): { + Required("implementation"): str, + Extra: object, + }, + # Override the default priority for the project + Optional("priority"): str, + } +) + +TC_TREEHERDER_SCHEMA_URL = ( + "https://github.com/taskcluster/taskcluster-treeherder/" + "blob/master/schemas/task-treeherder-config.yml" +) + + +UNKNOWN_GROUP_NAME = ( + "Treeherder group {} (from {}) has no name; " "add it to taskcluster/ci/config.yml" +) + +V2_ROUTE_TEMPLATES = [ + "index.{trust-domain}.v2.{project}.latest.{product}.{job-name}", + "index.{trust-domain}.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}", + "index.{trust-domain}.v2.{project}.pushdate.{build_date}.latest.{product}.{job-name}", + "index.{trust-domain}.v2.{project}.pushlog-id.{pushlog_id}.{product}.{job-name}", + "index.{trust-domain}.v2.{project}.revision.{branch_rev}.{product}.{job-name}", +] + +# {central, inbound, autoland} write to a "trunk" index prefix. This facilitates +# walking of tasks with similar configurations. +V2_TRUNK_ROUTE_TEMPLATES = [ + "index.{trust-domain}.v2.trunk.revision.{branch_rev}.{product}.{job-name}", +] + +V2_SHIPPABLE_TEMPLATES = [ + "index.{trust-domain}.v2.{project}.shippable.latest.{product}.{job-name}", + "index.{trust-domain}.v2.{project}.shippable.{build_date}.revision.{branch_rev}.{product}.{job-name}", # noqa - too long + "index.{trust-domain}.v2.{project}.shippable.{build_date}.latest.{product}.{job-name}", + "index.{trust-domain}.v2.{project}.shippable.revision.{branch_rev}.{product}.{job-name}", +] + +V2_SHIPPABLE_L10N_TEMPLATES = [ + "index.{trust-domain}.v2.{project}.shippable.latest.{product}-l10n.{job-name}.{locale}", + "index.{trust-domain}.v2.{project}.shippable.{build_date}.revision.{branch_rev}.{product}-l10n.{job-name}.{locale}", # noqa - too long + "index.{trust-domain}.v2.{project}.shippable.{build_date}.latest.{product}-l10n.{job-name}.{locale}", # noqa - too long + "index.{trust-domain}.v2.{project}.shippable.revision.{branch_rev}.{product}-l10n.{job-name}.{locale}", # noqa - too long +] + +V2_L10N_TEMPLATES = [ + "index.{trust-domain}.v2.{project}.revision.{branch_rev}.{product}-l10n.{job-name}.{locale}", + "index.{trust-domain}.v2.{project}.pushdate.{build_date_long}.{product}-l10n.{job-name}.{locale}", # noqa - too long + "index.{trust-domain}.v2.{project}.pushlog-id.{pushlog_id}.{product}-l10n.{job-name}.{locale}", + "index.{trust-domain}.v2.{project}.latest.{product}-l10n.{job-name}.{locale}", +] + +# This index is specifically for builds that include geckoview releases, +# so we can hard-code the project to "geckoview" +V2_GECKOVIEW_RELEASE = "index.{trust-domain}.v2.{project}.geckoview-version.{geckoview-version}.{product}.{job-name}" # noqa - too long + +# the roots of the treeherder routes +TREEHERDER_ROUTE_ROOT = "tc-treeherder" + + +def get_branch_rev(config): + return config.params[ + "{}head_rev".format(config.graph_config["project-repo-param-prefix"]) + ] + + +def get_branch_repo(config): + return config.params[ + "{}head_repository".format( + config.graph_config["project-repo-param-prefix"], + ) + ] + + +@memoize +def get_default_priority(graph_config, project): + return evaluate_keyed_by( + graph_config["task-priority"], "Graph Config", {"project": project} + ) + + +# define a collection of payload builders, depending on the worker implementation +payload_builders = {} + + +@attr.s(frozen=True) +class PayloadBuilder: + schema = attr.ib(type=Schema) + builder = attr.ib() + + +def payload_builder(name, schema): + schema = Schema({Required("implementation"): name, Optional("os"): str}).extend( + schema + ) + + def wrap(func): + payload_builders[name] = PayloadBuilder(schema, func) + return func + + return wrap + + +# define a collection of index builders, depending on the type implementation +index_builders = {} + + +def index_builder(name): + def wrap(func): + index_builders[name] = func + return func + + return wrap + + +UNSUPPORTED_INDEX_PRODUCT_ERROR = """\ +The gecko-v2 product {product} is not in the list of configured products in +`taskcluster/ci/config.yml'. +""" + + +def verify_index(config, index): + product = index["product"] + if product not in config.graph_config["index"]["products"]: + raise Exception(UNSUPPORTED_INDEX_PRODUCT_ERROR.format(product=product)) + + +@payload_builder( + "docker-worker", + schema={ + Required("os"): "linux", + # For tasks that will run in docker-worker, this is the + # name of the docker image or in-tree docker image to run the task in. If + # in-tree, then a dependency will be created automatically. This is + # generally `desktop-test`, or an image that acts an awful lot like it. + Required("docker-image"): Any( + # a raw Docker image path (repo/image:tag) + str, + # an in-tree generated docker image (from `taskcluster/docker/<name>`) + {"in-tree": str}, + # an indexed docker image + {"indexed": str}, + ), + # worker features that should be enabled + Required("chain-of-trust"): bool, + Required("taskcluster-proxy"): bool, + Required("allow-ptrace"): bool, + Required("loopback-video"): bool, + Required("loopback-audio"): bool, + Required("docker-in-docker"): bool, # (aka 'dind') + Required("privileged"): bool, + # Paths to Docker volumes. + # + # For in-tree Docker images, volumes can be parsed from Dockerfile. + # This only works for the Dockerfile itself: if a volume is defined in + # a base image, it will need to be declared here. Out-of-tree Docker + # images will also require explicit volume annotation. + # + # Caches are often mounted to the same path as Docker volumes. In this + # case, they take precedence over a Docker volume. But a volume still + # needs to be declared for the path. + Optional("volumes"): [str], + Optional( + "required-volumes", + description=( + "Paths that are required to be volumes for performance reasons. " + "For in-tree images, these paths will be checked to verify that they " + "are defined as volumes." + ), + ): [str], + # caches to set up for the task + Optional("caches"): [ + { + # only one type is supported by any of the workers right now + "type": "persistent", + # name of the cache, allowing re-use by subsequent tasks naming the + # same cache + "name": str, + # location in the task image where the cache will be mounted + "mount-point": str, + # Whether the cache is not used in untrusted environments + # (like the Try repo). + Optional("skip-untrusted"): bool, + } + ], + # artifacts to extract from the task image after completion + Optional("artifacts"): [ + { + # type of artifact -- simple file, or recursive directory + "type": Any("file", "directory"), + # task image path from which to read artifact + "path": str, + # name of the produced artifact (root of the names for + # type=directory) + "name": str, + "expires-after": str, + } + ], + # environment variables + Required("env"): {str: taskref_or_string}, + # the command to run; if not given, docker-worker will default to the + # command in the docker image + Optional("command"): [taskref_or_string], + # the maximum time to run, in seconds + Required("max-run-time"): int, + # the exit status code(s) that indicates the task should be retried + Optional("retry-exit-status"): [int], + # the exit status code(s) that indicates the caches used by the task + # should be purged + Optional("purge-caches-exit-status"): [int], + # Whether any artifacts are assigned to this worker + Optional("skip-artifacts"): bool, + }, +) +def build_docker_worker_payload(config, task, task_def): + worker = task["worker"] + level = int(config.params["level"]) + + image = worker["docker-image"] + if isinstance(image, dict): + if "in-tree" in image: + name = image["in-tree"] + docker_image_task = "docker-image-" + image["in-tree"] + task.setdefault("dependencies", {})["docker-image"] = docker_image_task + + image = { + "path": "public/image.tar.zst", + "taskId": {"task-reference": "<docker-image>"}, + "type": "task-image", + } + + # Find VOLUME in Dockerfile. + volumes = dockerutil.parse_volumes(name) + for v in sorted(volumes): + if v in worker["volumes"]: + raise Exception( + "volume %s already defined; " + "if it is defined in a Dockerfile, " + "it does not need to be specified in the " + "worker definition" % v + ) + + worker["volumes"].append(v) + + elif "indexed" in image: + image = { + "path": "public/image.tar.zst", + "namespace": image["indexed"], + "type": "indexed-image", + } + else: + raise Exception("unknown docker image type") + + features = {} + + if worker.get("taskcluster-proxy"): + features["taskclusterProxy"] = True + + if worker.get("allow-ptrace"): + features["allowPtrace"] = True + task_def["scopes"].append("docker-worker:feature:allowPtrace") + + if worker.get("chain-of-trust"): + features["chainOfTrust"] = True + + if worker.get("docker-in-docker"): + features["dind"] = True + + # Never enable sccache on the toolchains repo, as there is no benefit from it + # because each push uses a different compiler. + if task.get("use-sccache") and config.params["project"] != "toolchains": + features["taskclusterProxy"] = True + task_def["scopes"].append( + "assume:project:taskcluster:{trust_domain}:level-{level}-sccache-buckets".format( + trust_domain=config.graph_config["trust-domain"], + level=config.params["level"], + ) + ) + worker["env"]["USE_SCCACHE"] = "1" + worker["env"]["SCCACHE_GCS_PROJECT"] = SCCACHE_GCS_PROJECT + # Disable sccache idle shutdown. + worker["env"]["SCCACHE_IDLE_TIMEOUT"] = "0" + else: + worker["env"]["SCCACHE_DISABLE"] = "1" + + capabilities = {} + + for lo in "audio", "video": + if worker.get("loopback-" + lo): + capitalized = "loopback" + lo.capitalize() + devices = capabilities.setdefault("devices", {}) + devices[capitalized] = True + task_def["scopes"].append("docker-worker:capability:device:" + capitalized) + + if worker.get("privileged"): + capabilities["privileged"] = True + task_def["scopes"].append("docker-worker:capability:privileged") + + task_def["payload"] = payload = { + "image": image, + "env": worker["env"], + } + if "command" in worker: + payload["command"] = worker["command"] + + if "max-run-time" in worker: + payload["maxRunTime"] = worker["max-run-time"] + + run_task = payload.get("command", [""])[0].endswith("run-task") + + # run-task exits EXIT_PURGE_CACHES if there is a problem with caches. + # Automatically retry the tasks and purge caches if we see this exit + # code. + # TODO move this closer to code adding run-task once bug 1469697 is + # addressed. + if run_task: + worker.setdefault("retry-exit-status", []).append(72) + worker.setdefault("purge-caches-exit-status", []).append(72) + + payload["onExitStatus"] = {} + if "retry-exit-status" in worker: + payload["onExitStatus"]["retry"] = worker["retry-exit-status"] + if "purge-caches-exit-status" in worker: + payload["onExitStatus"]["purgeCaches"] = worker["purge-caches-exit-status"] + + if "artifacts" in worker: + artifacts = {} + expires_policy = get_expiration( + config, task.get("expiration-policy", "default") + ) + now = datetime.datetime.utcnow() + task_exp = task_def["expires"]["relative-datestamp"] + task_exp_from_now = fromNow(task_exp) + for artifact in worker["artifacts"]: + art_exp = artifact.get("expires-after", expires_policy) + expires = art_exp if fromNow(art_exp, now) < task_exp_from_now else task_exp + artifacts[artifact["name"]] = { + "path": artifact["path"], + "type": artifact["type"], + "expires": {"relative-datestamp": expires}, + } + payload["artifacts"] = artifacts + + if isinstance(worker.get("docker-image"), str): + out_of_tree_image = worker["docker-image"] + else: + out_of_tree_image = None + image = worker.get("docker-image", {}).get("in-tree") + + if "caches" in worker: + caches = {} + + # run-task knows how to validate caches. + # + # To help ensure new run-task features and bug fixes don't interfere + # with existing caches, we seed the hash of run-task into cache names. + # So, any time run-task changes, we should get a fresh set of caches. + # This means run-task can make changes to cache interaction at any time + # without regards for backwards or future compatibility. + # + # But this mechanism only works for in-tree Docker images that are built + # with the current run-task! For out-of-tree Docker images, we have no + # way of knowing their content of run-task. So, in addition to varying + # cache names by the contents of run-task, we also take the Docker image + # name into consideration. This means that different Docker images will + # never share the same cache. This is a bit unfortunate. But it is the + # safest thing to do. Fortunately, most images are defined in-tree. + # + # For out-of-tree Docker images, we don't strictly need to incorporate + # the run-task content into the cache name. However, doing so preserves + # the mechanism whereby changing run-task results in new caches + # everywhere. + + # As an additional mechanism to force the use of different caches, the + # string literal in the variable below can be changed. This is + # preferred to changing run-task because it doesn't require images + # to be rebuilt. + cache_version = "v3" + + if run_task: + suffix = f"{cache_version}-{_run_task_suffix()}" + + if out_of_tree_image: + name_hash = hashlib.sha256( + out_of_tree_image.encode("utf-8") + ).hexdigest() + suffix += name_hash[0:12] + + else: + suffix = cache_version + + skip_untrusted = is_try(config.params) or level == 1 + + for cache in worker["caches"]: + # Some caches aren't enabled in environments where we can't + # guarantee certain behavior. Filter those out. + if cache.get("skip-untrusted") and skip_untrusted: + continue + + name = "{trust_domain}-level-{level}-{name}-{suffix}".format( + trust_domain=config.graph_config["trust-domain"], + level=config.params["level"], + name=cache["name"], + suffix=suffix, + ) + + caches[name] = cache["mount-point"] + task_def["scopes"].append("docker-worker:cache:%s" % name) + + # Assertion: only run-task is interested in this. + if run_task: + payload["env"]["TASKCLUSTER_CACHES"] = ";".join(sorted(caches.values())) + + payload["cache"] = caches + + # And send down volumes information to run-task as well. + if run_task and worker.get("volumes"): + payload["env"]["TASKCLUSTER_VOLUMES"] = ";".join(sorted(worker["volumes"])) + + if payload.get("cache") and skip_untrusted: + payload["env"]["TASKCLUSTER_UNTRUSTED_CACHES"] = "1" + + if features: + payload["features"] = features + if capabilities: + payload["capabilities"] = capabilities + + check_caches_are_volumes(task) + check_required_volumes(task) + + +@payload_builder( + "generic-worker", + schema={ + Required("os"): Any("windows", "macosx", "linux", "linux-bitbar"), + # see http://schemas.taskcluster.net/generic-worker/v1/payload.json + # and https://docs.taskcluster.net/reference/workers/generic-worker/payload + # command is a list of commands to run, sequentially + # on Windows, each command is a string, on OS X and Linux, each command is + # a string array + Required("command"): Any( + [taskref_or_string], [[taskref_or_string]] # Windows # Linux / OS X + ), + # artifacts to extract from the task image after completion; note that artifacts + # for the generic worker cannot have names + Optional("artifacts"): [ + { + # type of artifact -- simple file, or recursive directory + "type": Any("file", "directory"), + # filesystem path from which to read artifact + "path": str, + # if not specified, path is used for artifact name + Optional("name"): str, + "expires-after": str, + } + ], + # Directories and/or files to be mounted. + # The actual allowed combinations are stricter than the model below, + # but this provides a simple starting point. + # See https://docs.taskcluster.net/reference/workers/generic-worker/payload + Optional("mounts"): [ + { + # A unique name for the cache volume, implies writable cache directory + # (otherwise mount is a read-only file or directory). + Optional("cache-name"): str, + # Optional content for pre-loading cache, or mandatory content for + # read-only file or directory. Pre-loaded content can come from either + # a task artifact or from a URL. + Optional("content"): { + # *** Either (artifact and task-id) or url must be specified. *** + # Artifact name that contains the content. + Optional("artifact"): str, + # Task ID that has the artifact that contains the content. + Optional("task-id"): taskref_or_string, + # URL that supplies the content in response to an unauthenticated + # GET request. + Optional("url"): str, + }, + # *** Either file or directory must be specified. *** + # If mounting a cache or read-only directory, the filesystem location of + # the directory should be specified as a relative path to the task + # directory here. + Optional("directory"): str, + # If mounting a file, specify the relative path within the task + # directory to mount the file (the file will be read only). + Optional("file"): str, + # Required if and only if `content` is specified and mounting a + # directory (not a file). This should be the archive format of the + # content (either pre-loaded cache or read-only directory). + Optional("format"): Any("rar", "tar.bz2", "tar.gz", "zip"), + } + ], + # environment variables + Required("env"): {str: taskref_or_string}, + # the maximum time to run, in seconds + Required("max-run-time"): int, + # os user groups for test task workers + Optional("os-groups"): [str], + # feature for test task to run as administarotr + Optional("run-as-administrator"): bool, + # optional features + Required("chain-of-trust"): bool, + Optional("taskcluster-proxy"): bool, + # the exit status code(s) that indicates the task should be retried + Optional("retry-exit-status"): [int], + # Wether any artifacts are assigned to this worker + Optional("skip-artifacts"): bool, + }, +) +def build_generic_worker_payload(config, task, task_def): + worker = task["worker"] + features = {} + + task_def["payload"] = { + "command": worker["command"], + "maxRunTime": worker["max-run-time"], + } + + if worker["os"] == "windows": + task_def["payload"]["onExitStatus"] = { + "retry": [ + # These codes (on windows) indicate a process interruption, + # rather than a task run failure. See bug 1544403. + 1073807364, # process force-killed due to system shutdown + 3221225786, # sigint (any interrupt) + ] + } + if "retry-exit-status" in worker: + task_def["payload"].setdefault("onExitStatus", {}).setdefault( + "retry", [] + ).extend(worker["retry-exit-status"]) + if worker["os"] == "linux-bitbar": + task_def["payload"].setdefault("onExitStatus", {}).setdefault("retry", []) + # exit code 4 is used to indicate an intermittent android device error + if 4 not in task_def["payload"]["onExitStatus"]["retry"]: + task_def["payload"]["onExitStatus"]["retry"].extend([4]) + + env = worker.get("env", {}) + + # Never enable sccache on the toolchains repo, as there is no benefit from it + # because each push uses a different compiler. + if task.get("use-sccache") and config.params["project"] != "toolchains": + features["taskclusterProxy"] = True + task_def["scopes"].append( + "assume:project:taskcluster:{trust_domain}:level-{level}-sccache-buckets".format( + trust_domain=config.graph_config["trust-domain"], + level=config.params["level"], + ) + ) + env["USE_SCCACHE"] = "1" + worker["env"]["SCCACHE_GCS_PROJECT"] = SCCACHE_GCS_PROJECT + # Disable sccache idle shutdown. + env["SCCACHE_IDLE_TIMEOUT"] = "0" + else: + env["SCCACHE_DISABLE"] = "1" + + if env: + task_def["payload"]["env"] = env + + artifacts = [] + + expires_policy = get_expiration(config, task.get("expiration-policy", "default")) + now = datetime.datetime.utcnow() + task_exp = task_def["expires"]["relative-datestamp"] + task_exp_from_now = fromNow(task_exp) + for artifact in worker.get("artifacts", []): + art_exp = artifact.get("expires-after", expires_policy) + task_exp = task_def["expires"]["relative-datestamp"] + expires = art_exp if fromNow(art_exp, now) < task_exp_from_now else task_exp + a = { + "path": artifact["path"], + "type": artifact["type"], + "expires": {"relative-datestamp": expires}, + } + if "name" in artifact: + a["name"] = artifact["name"] + artifacts.append(a) + + if artifacts: + task_def["payload"]["artifacts"] = artifacts + + # Need to copy over mounts, but rename keys to respect naming convention + # * 'cache-name' -> 'cacheName' + # * 'task-id' -> 'taskId' + # All other key names are already suitable, and don't need renaming. + mounts = copy_task(worker.get("mounts", [])) + for mount in mounts: + if "cache-name" in mount: + mount["cacheName"] = "{trust_domain}-level-{level}-{name}".format( + trust_domain=config.graph_config["trust-domain"], + level=config.params["level"], + name=mount.pop("cache-name"), + ) + task_def["scopes"].append( + "generic-worker:cache:{}".format(mount["cacheName"]) + ) + if "content" in mount: + if "task-id" in mount["content"]: + mount["content"]["taskId"] = mount["content"].pop("task-id") + if "artifact" in mount["content"]: + if not mount["content"]["artifact"].startswith("public/"): + task_def["scopes"].append( + "queue:get-artifact:{}".format(mount["content"]["artifact"]) + ) + + if mounts: + task_def["payload"]["mounts"] = mounts + + if worker.get("os-groups"): + task_def["payload"]["osGroups"] = worker["os-groups"] + task_def["scopes"].extend( + [ + "generic-worker:os-group:{}/{}".format(task["worker-type"], group) + for group in worker["os-groups"] + ] + ) + + if worker.get("chain-of-trust"): + features["chainOfTrust"] = True + + if worker.get("taskcluster-proxy"): + features["taskclusterProxy"] = True + + if worker.get("run-as-administrator", False): + features["runAsAdministrator"] = True + task_def["scopes"].append( + "generic-worker:run-as-administrator:{}".format(task["worker-type"]), + ) + + if features: + task_def["payload"]["features"] = features + + +@payload_builder( + "scriptworker-signing", + schema={ + # the maximum time to run, in seconds + Required("max-run-time"): int, + # list of artifact URLs for the artifacts that should be signed + Required("upstream-artifacts"): [ + { + # taskId of the task with the artifact + Required("taskId"): taskref_or_string, + # type of signing task (for CoT) + Required("taskType"): str, + # Paths to the artifacts to sign + Required("paths"): [str], + # Signing formats to use on each of the paths + Required("formats"): [str], + Optional("singleFileGlobs"): [str], + } + ], + # behavior for mac iscript + Optional("mac-behavior"): Any( + "apple_notarization", + "mac_sign_and_pkg", + "mac_geckodriver", + "mac_notarize_geckodriver", + "mac_single_file", + "mac_notarize_single_file", + ), + Optional("entitlements-url"): str, + Optional("requirements-plist-url"): str, + }, +) +def build_scriptworker_signing_payload(config, task, task_def): + worker = task["worker"] + + task_def["payload"] = { + "maxRunTime": worker["max-run-time"], + "upstreamArtifacts": worker["upstream-artifacts"], + } + if worker.get("mac-behavior"): + task_def["payload"]["behavior"] = worker["mac-behavior"] + for attribute in ("entitlements-url", "requirements-plist-url"): + if worker.get(attribute): + task_def["payload"][attribute] = worker[attribute] + + artifacts = set(task.setdefault("attributes", {}).get("release_artifacts", [])) + for upstream_artifact in worker["upstream-artifacts"]: + for path in upstream_artifact["paths"]: + artifacts.update( + get_signed_artifacts( + input=path, + formats=upstream_artifact["formats"], + behavior=worker.get("mac-behavior"), + ) + ) + task["attributes"]["release_artifacts"] = sorted(list(artifacts)) + + +@payload_builder( + "beetmover", + schema={ + # the maximum time to run, in seconds + Required("max-run-time"): int, + # locale key, if this is a locale beetmover job + Optional("locale"): str, + Optional("partner-public"): bool, + Required("release-properties"): { + "app-name": str, + "app-version": str, + "branch": str, + "build-id": str, + "hash-type": str, + "platform": str, + }, + # list of artifact URLs for the artifacts that should be beetmoved + Required("upstream-artifacts"): [ + { + # taskId of the task with the artifact + Required("taskId"): taskref_or_string, + # type of signing task (for CoT) + Required("taskType"): str, + # Paths to the artifacts to sign + Required("paths"): [str], + # locale is used to map upload path and allow for duplicate simple names + Required("locale"): str, + } + ], + Optional("artifact-map"): object, + }, +) +def build_beetmover_payload(config, task, task_def): + worker = task["worker"] + release_config = get_release_config(config) + release_properties = worker["release-properties"] + + task_def["payload"] = { + "maxRunTime": worker["max-run-time"], + "releaseProperties": { + "appName": release_properties["app-name"], + "appVersion": release_properties["app-version"], + "branch": release_properties["branch"], + "buildid": release_properties["build-id"], + "hashType": release_properties["hash-type"], + "platform": release_properties["platform"], + }, + "upload_date": config.params["build_date"], + "upstreamArtifacts": worker["upstream-artifacts"], + } + if worker.get("locale"): + task_def["payload"]["locale"] = worker["locale"] + if worker.get("artifact-map"): + task_def["payload"]["artifactMap"] = worker["artifact-map"] + if worker.get("partner-public"): + task_def["payload"]["is_partner_repack_public"] = worker["partner-public"] + if release_config: + task_def["payload"].update(release_config) + + +@payload_builder( + "beetmover-push-to-release", + schema={ + # the maximum time to run, in seconds + Required("max-run-time"): int, + Required("product"): str, + }, +) +def build_beetmover_push_to_release_payload(config, task, task_def): + worker = task["worker"] + release_config = get_release_config(config) + partners = [f"{p}/{s}" for p, s, _ in get_partners_to_be_published(config)] + + task_def["payload"] = { + "maxRunTime": worker["max-run-time"], + "product": worker["product"], + "version": release_config["version"], + "build_number": release_config["build_number"], + "partners": partners, + } + + +@payload_builder( + "beetmover-import-from-gcs-to-artifact-registry", + schema={ + Required("max-run-time"): int, + Required("gcs-sources"): [str], + Required("product"): str, + }, +) +def build_import_from_gcs_to_artifact_registry_payload(config, task, task_def): + task_def["payload"] = { + "product": task["worker"]["product"], + "gcs_sources": task["worker"]["gcs-sources"], + } + + +@payload_builder( + "beetmover-maven", + schema={ + Required("max-run-time"): int, + Required("release-properties"): { + "app-name": str, + "app-version": str, + "branch": str, + "build-id": str, + "artifact-id": str, + "hash-type": str, + "platform": str, + }, + Required("upstream-artifacts"): [ + { + Required("taskId"): taskref_or_string, + Required("taskType"): str, + Required("paths"): [str], + Optional("zipExtract"): bool, + } + ], + Optional("artifact-map"): object, + }, +) +def build_beetmover_maven_payload(config, task, task_def): + build_beetmover_payload(config, task, task_def) + + task_def["payload"]["artifact_id"] = task["worker"]["release-properties"][ + "artifact-id" + ] + if task["worker"].get("artifact-map"): + task_def["payload"]["artifactMap"] = task["worker"]["artifact-map"] + + task_def["payload"]["version"] = _compute_geckoview_version( + task["worker"]["release-properties"]["app-version"], + task["worker"]["release-properties"]["build-id"], + ) + + del task_def["payload"]["releaseProperties"]["hashType"] + del task_def["payload"]["releaseProperties"]["platform"] + + +@payload_builder( + "balrog", + schema={ + Required("balrog-action"): Any(*BALROG_ACTIONS), + Optional("product"): str, + Optional("platforms"): [str], + Optional("release-eta"): str, + Optional("channel-names"): optionally_keyed_by("release-type", [str]), + Optional("require-mirrors"): bool, + Optional("publish-rules"): optionally_keyed_by( + "release-type", "release-level", [int] + ), + Optional("rules-to-update"): optionally_keyed_by( + "release-type", "release-level", [str] + ), + Optional("archive-domain"): optionally_keyed_by("release-level", str), + Optional("download-domain"): optionally_keyed_by("release-level", str), + Optional("blob-suffix"): str, + Optional("complete-mar-filename-pattern"): str, + Optional("complete-mar-bouncer-product-pattern"): str, + Optional("update-line"): object, + Optional("suffixes"): [str], + Optional("background-rate"): optionally_keyed_by( + "release-type", "beta-number", Any(int, None) + ), + Optional("force-fallback-mapping-update"): optionally_keyed_by( + "release-type", "beta-number", bool + ), + Optional("pin-channels"): optionally_keyed_by( + "release-type", "release-level", [str] + ), + # list of artifact URLs for the artifacts that should be beetmoved + Optional("upstream-artifacts"): [ + { + # taskId of the task with the artifact + Required("taskId"): taskref_or_string, + # type of signing task (for CoT) + Required("taskType"): str, + # Paths to the artifacts to sign + Required("paths"): [str], + } + ], + }, +) +def build_balrog_payload(config, task, task_def): + worker = task["worker"] + release_config = get_release_config(config) + beta_number = None + if "b" in release_config["version"]: + beta_number = release_config["version"].split("b")[-1] + + task_def["payload"] = { + "behavior": worker["balrog-action"], + } + + if ( + worker["balrog-action"] == "submit-locale" + or worker["balrog-action"] == "v2-submit-locale" + ): + task_def["payload"].update( + { + "upstreamArtifacts": worker["upstream-artifacts"], + "suffixes": worker["suffixes"], + } + ) + else: + for prop in ( + "archive-domain", + "channel-names", + "download-domain", + "publish-rules", + "rules-to-update", + "background-rate", + "force-fallback-mapping-update", + "pin-channels", + ): + if prop in worker: + resolve_keyed_by( + worker, + prop, + task["description"], + **{ + "release-type": config.params["release_type"], + "release-level": release_level(config.params["project"]), + "beta-number": beta_number, + }, + ) + task_def["payload"].update( + { + "build_number": release_config["build_number"], + "product": worker["product"], + "version": release_config["version"], + } + ) + for prop in ( + "blob-suffix", + "complete-mar-filename-pattern", + "complete-mar-bouncer-product-pattern", + "pin-channels", + ): + if prop in worker: + task_def["payload"][prop.replace("-", "_")] = worker[prop] + if ( + worker["balrog-action"] == "submit-toplevel" + or worker["balrog-action"] == "v2-submit-toplevel" + ): + task_def["payload"].update( + { + "app_version": release_config["appVersion"], + "archive_domain": worker["archive-domain"], + "channel_names": worker["channel-names"], + "download_domain": worker["download-domain"], + "partial_versions": release_config.get("partial_versions", ""), + "platforms": worker["platforms"], + "rules_to_update": worker["rules-to-update"], + "require_mirrors": worker["require-mirrors"], + "update_line": worker["update-line"], + } + ) + else: # schedule / ship + task_def["payload"].update( + { + "publish_rules": worker["publish-rules"], + "release_eta": worker.get( + "release-eta", config.params.get("release_eta") + ) + or "", + } + ) + if worker.get("force-fallback-mapping-update"): + task_def["payload"]["force_fallback_mapping_update"] = worker[ + "force-fallback-mapping-update" + ] + if worker.get("background-rate"): + task_def["payload"]["background_rate"] = worker["background-rate"] + + +@payload_builder( + "bouncer-aliases", + schema={ + Required("entries"): object, + }, +) +def build_bouncer_aliases_payload(config, task, task_def): + worker = task["worker"] + + task_def["payload"] = {"aliases_entries": worker["entries"]} + + +@payload_builder( + "bouncer-locations", + schema={ + Required("implementation"): "bouncer-locations", + Required("bouncer-products"): [str], + }, +) +def build_bouncer_locations_payload(config, task, task_def): + worker = task["worker"] + release_config = get_release_config(config) + + task_def["payload"] = { + "bouncer_products": worker["bouncer-products"], + "version": release_config["version"], + "product": task["shipping-product"], + } + + +@payload_builder( + "bouncer-submission", + schema={ + Required("locales"): [str], + Required("entries"): object, + }, +) +def build_bouncer_submission_payload(config, task, task_def): + worker = task["worker"] + + task_def["payload"] = { + "locales": worker["locales"], + "submission_entries": worker["entries"], + } + + +@payload_builder( + "push-flatpak", + schema={ + Required("channel"): str, + Required("upstream-artifacts"): [ + { + Required("taskId"): taskref_or_string, + Required("taskType"): str, + Required("paths"): [str], + } + ], + }, +) +def build_push_flatpak_payload(config, task, task_def): + worker = task["worker"] + + task_def["payload"] = { + "channel": worker["channel"], + "upstreamArtifacts": worker["upstream-artifacts"], + } + + +@payload_builder( + "push-msix", + schema={ + Required("channel"): str, + Optional("publish-mode"): str, + Required("upstream-artifacts"): [ + { + Required("taskId"): taskref_or_string, + Required("taskType"): str, + Required("paths"): [str], + } + ], + }, +) +def build_push_msix_payload(config, task, task_def): + worker = task["worker"] + + task_def["payload"] = { + "channel": worker["channel"], + "upstreamArtifacts": worker["upstream-artifacts"], + } + if worker.get("publish-mode"): + task_def["payload"]["publishMode"] = worker["publish-mode"] + + +@payload_builder( + "shipit-shipped", + schema={ + Required("release-name"): str, + }, +) +def build_ship_it_shipped_payload(config, task, task_def): + worker = task["worker"] + + task_def["payload"] = {"release_name": worker["release-name"]} + + +@payload_builder( + "shipit-maybe-release", + schema={ + Required("phase"): str, + }, +) +def build_ship_it_maybe_release_payload(config, task, task_def): + # expect branch name, including path + branch = config.params["head_repository"][len("https://hg.mozilla.org/") :] + # 'version' is e.g. '71.0b13' (app_version doesn't have beta number) + version = config.params["version"] + + task_def["payload"] = { + "product": task["shipping-product"], + "branch": branch, + "phase": task["worker"]["phase"], + "version": version, + "cron_revision": config.params["head_rev"], + } + + +@payload_builder( + "push-addons", + schema={ + Required("channel"): Any("listed", "unlisted"), + Required("upstream-artifacts"): [ + { + Required("taskId"): taskref_or_string, + Required("taskType"): str, + Required("paths"): [str], + } + ], + }, +) +def build_push_addons_payload(config, task, task_def): + worker = task["worker"] + + task_def["payload"] = { + "channel": worker["channel"], + "upstreamArtifacts": worker["upstream-artifacts"], + } + + +@payload_builder( + "treescript", + schema={ + Required("tags"): [Any("buildN", "release", None)], + Required("bump"): bool, + Optional("bump-files"): [str], + Optional("repo-param-prefix"): str, + Optional("dontbuild"): bool, + Optional("ignore-closed-tree"): bool, + Optional("force-dry-run"): bool, + Optional("push"): bool, + Optional("source-repo"): str, + Optional("ssh-user"): str, + Optional("l10n-bump-info"): { + Required("name"): str, + Required("path"): str, + Required("version-path"): str, + Optional("l10n-repo-url"): str, + Optional("ignore-config"): object, + Required("platform-configs"): [ + { + Required("platforms"): [str], + Required("path"): str, + Optional("format"): str, + } + ], + }, + Optional("merge-info"): object, + }, +) +def build_treescript_payload(config, task, task_def): + worker = task["worker"] + release_config = get_release_config(config) + + task_def["payload"] = {"actions": []} + actions = task_def["payload"]["actions"] + if worker["tags"]: + tag_names = [] + product = task["shipping-product"].upper() + version = release_config["version"].replace(".", "_") + buildnum = release_config["build_number"] + if "buildN" in worker["tags"]: + tag_names.extend( + [ + f"{product}_{version}_BUILD{buildnum}", + ] + ) + if "release" in worker["tags"]: + tag_names.extend([f"{product}_{version}_RELEASE"]) + tag_info = { + "tags": tag_names, + "revision": config.params[ + "{}head_rev".format(worker.get("repo-param-prefix", "")) + ], + } + task_def["payload"]["tag_info"] = tag_info + actions.append("tag") + + if worker["bump"]: + if not worker["bump-files"]: + raise Exception("Version Bump requested without bump-files") + + bump_info = {} + bump_info["next_version"] = release_config["next_version"] + bump_info["files"] = worker["bump-files"] + task_def["payload"]["version_bump_info"] = bump_info + actions.append("version_bump") + + if worker.get("l10n-bump-info"): + l10n_bump_info = {} + for k, v in worker["l10n-bump-info"].items(): + l10n_bump_info[k.replace("-", "_")] = worker["l10n-bump-info"][k] + task_def["payload"]["l10n_bump_info"] = [l10n_bump_info] + actions.append("l10n_bump") + + if worker.get("merge-info"): + merge_info = { + merge_param_name.replace("-", "_"): merge_param_value + for merge_param_name, merge_param_value in worker["merge-info"].items() + if merge_param_name != "version-files" + } + merge_info["version_files"] = [ + { + file_param_name.replace("-", "_"): file_param_value + for file_param_name, file_param_value in file_entry.items() + } + for file_entry in worker["merge-info"]["version-files"] + ] + task_def["payload"]["merge_info"] = merge_info + actions.append("merge_day") + + if worker["push"]: + actions.append("push") + + if worker.get("force-dry-run"): + task_def["payload"]["dry_run"] = True + + if worker.get("dontbuild"): + task_def["payload"]["dontbuild"] = True + + if worker.get("ignore-closed-tree") is not None: + task_def["payload"]["ignore_closed_tree"] = worker["ignore-closed-tree"] + + if worker.get("source-repo"): + task_def["payload"]["source_repo"] = worker["source-repo"] + + if worker.get("ssh-user"): + task_def["payload"]["ssh_user"] = worker["ssh-user"] + + +@payload_builder( + "invalid", + schema={ + # an invalid task is one which should never actually be created; this is used in + # release automation on branches where the task just doesn't make sense + Extra: object, + }, +) +def build_invalid_payload(config, task, task_def): + task_def["payload"] = "invalid task - should never be created" + + +@payload_builder( + "always-optimized", + schema={ + Extra: object, + }, +) +@payload_builder("succeed", schema={}) +def build_dummy_payload(config, task, task_def): + task_def["payload"] = {} + + +transforms = TransformSequence() + + +@transforms.add +def set_implementation(config, tasks): + """ + Set the worker implementation based on the worker-type alias. + """ + for task in tasks: + if "implementation" in task["worker"]: + yield task + continue + + impl, os = worker_type_implementation( + config.graph_config, config.params, task["worker-type"] + ) + + tags = task.setdefault("tags", {}) + tags["worker-implementation"] = impl + if os: + tags["os"] = os + + worker = task.setdefault("worker", {}) + worker["implementation"] = impl + if os: + worker["os"] = os + + yield task + + +@transforms.add +def set_defaults(config, tasks): + for task in tasks: + task.setdefault("shipping-phase", None) + task.setdefault("shipping-product", None) + task.setdefault("always-target", False) + task.setdefault("optimization", None) + task.setdefault("use-sccache", False) + + worker = task["worker"] + if worker["implementation"] in ("docker-worker",): + worker.setdefault("chain-of-trust", False) + worker.setdefault("taskcluster-proxy", False) + worker.setdefault("allow-ptrace", True) + worker.setdefault("loopback-video", False) + worker.setdefault("loopback-audio", False) + worker.setdefault("docker-in-docker", False) + worker.setdefault("privileged", False) + worker.setdefault("volumes", []) + worker.setdefault("env", {}) + if "caches" in worker: + for c in worker["caches"]: + c.setdefault("skip-untrusted", False) + elif worker["implementation"] == "generic-worker": + worker.setdefault("env", {}) + worker.setdefault("os-groups", []) + if worker["os-groups"] and worker["os"] != "windows": + raise Exception( + "os-groups feature of generic-worker is only supported on " + "Windows, not on {}".format(worker["os"]) + ) + worker.setdefault("chain-of-trust", False) + elif worker["implementation"] in ( + "scriptworker-signing", + "beetmover", + "beetmover-push-to-release", + "beetmover-maven", + "beetmover-import-from-gcs-to-artifact-registry", + ): + worker.setdefault("max-run-time", 600) + elif worker["implementation"] == "push-apk": + worker.setdefault("commit", False) + + yield task + + +@transforms.add +def setup_raptor(config, tasks): + """Add options that are specific to raptor jobs (identified by suite=raptor). + + This variant uses a separate set of transforms for manipulating the tests at the + task-level. Currently only used for setting the taskcluster proxy setting and + the scopes required for perftest secrets. + """ + from gecko_taskgraph.transforms.test.raptor import ( + task_transforms as raptor_transforms, + ) + + for task in tasks: + if task.get("extra", {}).get("suite", "") != "raptor": + yield task + continue + + yield from raptor_transforms(config, [task]) + + +@transforms.add +def task_name_from_label(config, tasks): + for task in tasks: + taskname = task.pop("name", None) + if "label" not in task: + if taskname is None: + raise Exception("task has neither a name nor a label") + task["label"] = "{}-{}".format(config.kind, taskname) + yield task + + +UNSUPPORTED_SHIPPING_PRODUCT_ERROR = """\ +The shipping product {product} is not in the list of configured products in +`taskcluster/ci/config.yml'. +""" + + +def validate_shipping_product(config, product): + if product not in config.graph_config["release-promotion"]["products"]: + raise Exception(UNSUPPORTED_SHIPPING_PRODUCT_ERROR.format(product=product)) + + +@transforms.add +def validate(config, tasks): + for task in tasks: + validate_schema( + task_description_schema, + task, + "In task {!r}:".format(task.get("label", "?no-label?")), + ) + validate_schema( + payload_builders[task["worker"]["implementation"]].schema, + task["worker"], + "In task.run {!r}:".format(task.get("label", "?no-label?")), + ) + if task["shipping-product"] is not None: + validate_shipping_product(config, task["shipping-product"]) + yield task + + +@index_builder("generic") +def add_generic_index_routes(config, task): + index = task.get("index") + routes = task.setdefault("routes", []) + + verify_index(config, index) + + subs = config.params.copy() + subs["job-name"] = index["job-name"] + subs["build_date_long"] = time.strftime( + "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"]) + ) + subs["build_date"] = time.strftime( + "%Y.%m.%d", time.gmtime(config.params["build_date"]) + ) + subs["product"] = index["product"] + subs["trust-domain"] = config.graph_config["trust-domain"] + subs["branch_rev"] = get_branch_rev(config) + + project = config.params.get("project") + + for tpl in V2_ROUTE_TEMPLATES: + routes.append(tpl.format(**subs)) + + # Additionally alias all tasks for "trunk" repos into a common + # namespace. + if project and project in TRUNK_PROJECTS: + for tpl in V2_TRUNK_ROUTE_TEMPLATES: + routes.append(tpl.format(**subs)) + + return task + + +@index_builder("shippable") +def add_shippable_index_routes(config, task): + index = task.get("index") + routes = task.setdefault("routes", []) + + verify_index(config, index) + + subs = config.params.copy() + subs["job-name"] = index["job-name"] + subs["build_date_long"] = time.strftime( + "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"]) + ) + subs["build_date"] = time.strftime( + "%Y.%m.%d", time.gmtime(config.params["build_date"]) + ) + subs["product"] = index["product"] + subs["trust-domain"] = config.graph_config["trust-domain"] + subs["branch_rev"] = get_branch_rev(config) + + for tpl in V2_SHIPPABLE_TEMPLATES: + routes.append(tpl.format(**subs)) + + # Also add routes for en-US + task = add_shippable_l10n_index_routes(config, task, force_locale="en-US") + + return task + + +@index_builder("shippable-with-multi-l10n") +def add_shippable_multi_index_routes(config, task): + task = add_shippable_index_routes(config, task) + task = add_l10n_index_routes(config, task, force_locale="multi") + return task + + +@index_builder("l10n") +def add_l10n_index_routes(config, task, force_locale=None): + index = task.get("index") + routes = task.setdefault("routes", []) + + verify_index(config, index) + + subs = config.params.copy() + subs["job-name"] = index["job-name"] + subs["build_date_long"] = time.strftime( + "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"]) + ) + subs["product"] = index["product"] + subs["trust-domain"] = config.graph_config["trust-domain"] + subs["branch_rev"] = get_branch_rev(config) + + locales = task["attributes"].get( + "chunk_locales", task["attributes"].get("all_locales") + ) + # Some tasks has only one locale set + if task["attributes"].get("locale"): + locales = [task["attributes"]["locale"]] + + if force_locale: + # Used for en-US and multi-locale + locales = [force_locale] + + if not locales: + raise Exception("Error: Unable to use l10n index for tasks without locales") + + # If there are too many locales, we can't write a route for all of them + # See Bug 1323792 + if len(locales) > 18: # 18 * 3 = 54, max routes = 64 + return task + + for locale in locales: + for tpl in V2_L10N_TEMPLATES: + routes.append(tpl.format(locale=locale, **subs)) + + return task + + +@index_builder("shippable-l10n") +def add_shippable_l10n_index_routes(config, task, force_locale=None): + index = task.get("index") + routes = task.setdefault("routes", []) + + verify_index(config, index) + + subs = config.params.copy() + subs["job-name"] = index["job-name"] + subs["build_date_long"] = time.strftime( + "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"]) + ) + subs["product"] = index["product"] + subs["trust-domain"] = config.graph_config["trust-domain"] + subs["branch_rev"] = get_branch_rev(config) + + locales = task["attributes"].get( + "chunk_locales", task["attributes"].get("all_locales") + ) + # Some tasks has only one locale set + if task["attributes"].get("locale"): + locales = [task["attributes"]["locale"]] + + if force_locale: + # Used for en-US and multi-locale + locales = [force_locale] + + if not locales: + raise Exception("Error: Unable to use l10n index for tasks without locales") + + # If there are too many locales, we can't write a route for all of them + # See Bug 1323792 + if len(locales) > 18: # 18 * 3 = 54, max routes = 64 + return task + + for locale in locales: + for tpl in V2_SHIPPABLE_L10N_TEMPLATES: + routes.append(tpl.format(locale=locale, **subs)) + + return task + + +def add_geckoview_index_routes(config, task): + index = task.get("index") + routes = task.setdefault("routes", []) + geckoview_version = _compute_geckoview_version( + config.params["app_version"], config.params["moz_build_date"] + ) + + subs = { + "geckoview-version": geckoview_version, + "job-name": index["job-name"], + "product": index["product"], + "project": config.params["project"], + "trust-domain": config.graph_config["trust-domain"], + } + routes.append(V2_GECKOVIEW_RELEASE.format(**subs)) + + return task + + +@index_builder("android-shippable") +def add_android_shippable_index_routes(config, task): + task = add_shippable_index_routes(config, task) + task = add_geckoview_index_routes(config, task) + + return task + + +@index_builder("android-shippable-with-multi-l10n") +def add_android_shippable_multi_index_routes(config, task): + task = add_shippable_multi_index_routes(config, task) + task = add_geckoview_index_routes(config, task) + + return task + + +@transforms.add +def add_index_routes(config, tasks): + for task in tasks: + index = task.get("index", {}) + + # The default behavior is to rank tasks according to their tier + extra_index = task.setdefault("extra", {}).setdefault("index", {}) + rank = index.get("rank", "by-tier") + + if rank == "by-tier": + # rank is zero for non-tier-1 tasks and based on pushid for others; + # this sorts tier-{2,3} builds below tier-1 in the index + tier = task.get("treeherder", {}).get("tier", 3) + extra_index["rank"] = 0 if tier > 1 else int(config.params["build_date"]) + elif rank == "build_date": + extra_index["rank"] = int(config.params["build_date"]) + else: + extra_index["rank"] = rank + + if not index: + yield task + continue + + index_type = index.get("type", "generic") + task = index_builders[index_type](config, task) + + del task["index"] + yield task + + +@transforms.add +def try_task_config_env(config, tasks): + """Set environment variables in the task.""" + env = config.params["try_task_config"].get("env") + if not env: + yield from tasks + return + + # Find all implementations that have an 'env' key. + implementations = { + name + for name, builder in payload_builders.items() + if "env" in builder.schema.schema + } + for task in tasks: + if task["worker"]["implementation"] in implementations: + task["worker"]["env"].update(env) + yield task + + +@transforms.add +def try_task_config_chemspill_prio(config, tasks): + """Increase the priority from lowest and very-low -> low, but leave others unchanged.""" + chemspill_prio = config.params["try_task_config"].get("chemspill-prio") + if not chemspill_prio: + yield from tasks + return + + for task in tasks: + if task["priority"] in ("lowest", "very-low"): + task["priority"] = "low" + yield task + + +@transforms.add +def try_task_config_routes(config, tasks): + """Set routes in the task.""" + routes = config.params["try_task_config"].get("routes") + for task in tasks: + if routes: + task_routes = task.setdefault("routes", []) + task_routes.extend(routes) + yield task + + +@transforms.add +def set_task_and_artifact_expiry(config, jobs): + """Set the default expiry for tasks and their artifacts. + + These values are read from ci/config.yml + """ + now = datetime.datetime.utcnow() + for job in jobs: + expires = get_expiration(config, job.get("expiration-policy", "default")) + job_expiry = job.setdefault("expires-after", expires) + job_expiry_from_now = fromNow(job_expiry) + + for artifact in job["worker"].get("artifacts", ()): + artifact_expiry = artifact.setdefault("expires-after", expires) + + # By using > instead of >=, there's a chance of mismatch + # where the artifact expires sooner than the task. + # There is no chance, however, of mismatch where artifacts + # expire _after_ the task. + # Currently this leads to some build tasks having logs + # that expire in 1 year while the task expires in 3 years. + if fromNow(artifact_expiry, now) > job_expiry_from_now: + artifact["expires-after"] = job_expiry + + yield job + + +@transforms.add +def build_task(config, tasks): + for task in tasks: + level = str(config.params["level"]) + + task_worker_type = task["worker-type"] + worker_overrides = config.params["try_task_config"].get("worker-overrides", {}) + if task_worker_type in worker_overrides: + worker_pool = worker_overrides[task_worker_type] + provisioner_id, worker_type = worker_pool.split("/", 1) + else: + provisioner_id, worker_type = get_worker_type( + config.graph_config, + config.params, + task_worker_type, + ) + task["worker-type"] = "/".join([provisioner_id, worker_type]) + project = config.params["project"] + + routes = task.get("routes", []) + scopes = [ + s.format(level=level, project=project) for s in task.get("scopes", []) + ] + + # set up extra + extra = task.get("extra", {}) + extra["parent"] = {"task-reference": "<decision>"} + task_th = task.get("treeherder") + if task_th: + extra.setdefault("treeherder-platform", task_th["platform"]) + treeherder = extra.setdefault("treeherder", {}) + + machine_platform, collection = task_th["platform"].split("/", 1) + treeherder["machine"] = {"platform": machine_platform} + treeherder["collection"] = {collection: True} + + group_names = config.graph_config["treeherder"]["group-names"] + groupSymbol, symbol = split_symbol(task_th["symbol"]) + if groupSymbol != "?": + treeherder["groupSymbol"] = groupSymbol + if groupSymbol not in group_names: + path = os.path.join(config.path, task.get("job-from", "")) + raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol, path)) + treeherder["groupName"] = group_names[groupSymbol] + treeherder["symbol"] = symbol + if len(symbol) > 25 or len(groupSymbol) > 25: + raise RuntimeError( + "Treeherder group and symbol names must not be longer than " + "25 characters: {} (see {})".format( + task_th["symbol"], + TC_TREEHERDER_SCHEMA_URL, + ) + ) + treeherder["jobKind"] = task_th["kind"] + treeherder["tier"] = task_th["tier"] + + branch_rev = get_branch_rev(config) + + routes.append( + "{}.v2.{}.{}".format( + TREEHERDER_ROUTE_ROOT, + config.params["project"], + branch_rev, + ) + ) + + if "deadline-after" not in task: + task["deadline-after"] = "1 day" + + if "priority" not in task: + task["priority"] = get_default_priority( + config.graph_config, config.params["project"] + ) + + tags = task.get("tags", {}) + attributes = task.get("attributes", {}) + + tags.update( + { + "createdForUser": config.params["owner"], + "kind": config.kind, + "label": task["label"], + "retrigger": "true" if attributes.get("retrigger", False) else "false", + } + ) + + task_def = { + "provisionerId": provisioner_id, + "workerType": worker_type, + "routes": routes, + "created": {"relative-datestamp": "0 seconds"}, + "deadline": {"relative-datestamp": task["deadline-after"]}, + "expires": {"relative-datestamp": task["expires-after"]}, + "scopes": scopes, + "metadata": { + "description": task["description"], + "name": task["label"], + "owner": config.params["owner"], + "source": config.params.file_url(config.path, pretty=True), + }, + "extra": extra, + "tags": tags, + "priority": task["priority"], + } + + if task.get("requires", None): + task_def["requires"] = task["requires"] + + if task_th: + # link back to treeherder in description + th_job_link = ( + "https://treeherder.mozilla.org/#/jobs?repo={}&revision={}&selectedTaskRun=<self>" + ).format(config.params["project"], branch_rev) + task_def["metadata"]["description"] = { + "task-reference": "{description} ([Treeherder job]({th_job_link}))".format( + description=task_def["metadata"]["description"], + th_job_link=th_job_link, + ) + } + + # add the payload and adjust anything else as required (e.g., scopes) + payload_builders[task["worker"]["implementation"]].builder( + config, task, task_def + ) + + # Resolve run-on-projects + build_platform = attributes.get("build_platform") + resolve_keyed_by( + task, + "run-on-projects", + item_name=task["label"], + **{"build-platform": build_platform}, + ) + attributes["run_on_projects"] = task.get("run-on-projects", ["all"]) + attributes["always_target"] = task["always-target"] + # This logic is here since downstream tasks don't always match their + # upstream dependency's shipping_phase. + # A text_type task['shipping-phase'] takes precedence, then + # an existing attributes['shipping_phase'], then fall back to None. + if task.get("shipping-phase") is not None: + attributes["shipping_phase"] = task["shipping-phase"] + else: + attributes.setdefault("shipping_phase", None) + # shipping_product will always match the upstream task's + # shipping_product, so a pre-set existing attributes['shipping_product'] + # takes precedence over task['shipping-product']. However, make sure + # we don't have conflicting values. + if task.get("shipping-product") and attributes.get("shipping_product") not in ( + None, + task["shipping-product"], + ): + raise Exception( + "{} shipping_product {} doesn't match task shipping-product {}!".format( + task["label"], + attributes["shipping_product"], + task["shipping-product"], + ) + ) + attributes.setdefault("shipping_product", task["shipping-product"]) + + # Set MOZ_AUTOMATION on all jobs. + if task["worker"]["implementation"] in ( + "generic-worker", + "docker-worker", + ): + payload = task_def.get("payload") + if payload: + env = payload.setdefault("env", {}) + env["MOZ_AUTOMATION"] = "1" + + dependencies = task.get("dependencies", {}) + if_dependencies = task.get("if-dependencies", []) + if if_dependencies: + for i, dep in enumerate(if_dependencies): + if dep in dependencies: + if_dependencies[i] = dependencies[dep] + continue + + raise Exception( + "{label} specifies '{dep}' in if-dependencies, " + "but {dep} is not a dependency!".format( + label=task["label"], dep=dep + ) + ) + + yield { + "label": task["label"], + "description": task["description"], + "task": task_def, + "dependencies": dependencies, + "if-dependencies": if_dependencies, + "soft-dependencies": task.get("soft-dependencies", []), + "attributes": attributes, + "optimization": task.get("optimization", None), + } + + +@transforms.add +def chain_of_trust(config, tasks): + for task in tasks: + if task["task"].get("payload", {}).get("features", {}).get("chainOfTrust"): + image = task.get("dependencies", {}).get("docker-image") + if image: + cot = ( + task["task"].setdefault("extra", {}).setdefault("chainOfTrust", {}) + ) + cot.setdefault("inputs", {})["docker-image"] = { + "task-reference": "<docker-image>" + } + yield task + + +@transforms.add +def check_task_identifiers(config, tasks): + """Ensures that all tasks have well defined identifiers: + ``^[a-zA-Z0-9_-]{1,38}$`` + """ + e = re.compile("^[a-zA-Z0-9_-]{1,38}$") + for task in tasks: + for attrib in ("workerType", "provisionerId"): + if not e.match(task["task"][attrib]): + raise Exception( + "task {}.{} is not a valid identifier: {}".format( + task["label"], attrib, task["task"][attrib] + ) + ) + yield task + + +@transforms.add +def check_task_dependencies(config, tasks): + """Ensures that tasks don't have more than 100 dependencies.""" + for task in tasks: + if len(task["dependencies"]) > MAX_DEPENDENCIES: + raise Exception( + "task {}/{} has too many dependencies ({} > {})".format( + config.kind, + task["label"], + len(task["dependencies"]), + MAX_DEPENDENCIES, + ) + ) + yield task + + +def check_caches_are_volumes(task): + """Ensures that all cache paths are defined as volumes. + + Caches and volumes are the only filesystem locations whose content + isn't defined by the Docker image itself. Some caches are optional + depending on the job environment. We want paths that are potentially + caches to have as similar behavior regardless of whether a cache is + used. To help enforce this, we require that all paths used as caches + to be declared as Docker volumes. This check won't catch all offenders. + But it is better than nothing. + """ + volumes = {s for s in task["worker"]["volumes"]} + paths = {c["mount-point"] for c in task["worker"].get("caches", [])} + missing = paths - volumes + + if not missing: + return + + raise Exception( + "task %s (image %s) has caches that are not declared as " + "Docker volumes: %s " + "(have you added them as VOLUMEs in the Dockerfile?)" + % (task["label"], task["worker"]["docker-image"], ", ".join(sorted(missing))) + ) + + +def check_required_volumes(task): + """ + Ensures that all paths that are required to be volumes are defined as volumes. + + Performance of writing to files in poor in directories not marked as + volumes, in docker. Ensure that paths that are often written to are marked + as volumes. + """ + volumes = set(task["worker"]["volumes"]) + paths = set(task["worker"].get("required-volumes", [])) + missing = paths - volumes + + if not missing: + return + + raise Exception( + "task %s (image %s) has paths that should be volumes for peformance " + "that are not declared as Docker volumes: %s " + "(have you added them as VOLUMEs in the Dockerfile?)" + % (task["label"], task["worker"]["docker-image"], ", ".join(sorted(missing))) + ) + + +@transforms.add +def check_run_task_caches(config, tasks): + """Audit for caches requiring run-task. + + run-task manages caches in certain ways. If a cache managed by run-task + is used by a non run-task task, it could cause problems. So we audit for + that and make sure certain cache names are exclusive to run-task. + + IF YOU ARE TEMPTED TO MAKE EXCLUSIONS TO THIS POLICY, YOU ARE LIKELY + CONTRIBUTING TECHNICAL DEBT AND WILL HAVE TO SOLVE MANY OF THE PROBLEMS + THAT RUN-TASK ALREADY SOLVES. THINK LONG AND HARD BEFORE DOING THAT. + """ + re_reserved_caches = re.compile( + """^ + (checkouts|tooltool-cache) + """, + re.VERBOSE, + ) + + re_sparse_checkout_cache = re.compile("^checkouts-sparse") + + cache_prefix = "{trust_domain}-level-{level}-".format( + trust_domain=config.graph_config["trust-domain"], + level=config.params["level"], + ) + + suffix = _run_task_suffix() + + for task in tasks: + payload = task["task"].get("payload", {}) + command = payload.get("command") or [""] + + main_command = command[0] if isinstance(command[0], str) else "" + run_task = main_command.endswith("run-task") + + require_sparse_cache = False + have_sparse_cache = False + + if run_task: + for arg in command[1:]: + if not isinstance(arg, str): + continue + + if arg == "--": + break + + if arg.startswith("--gecko-sparse-profile"): + if "=" not in arg: + raise Exception( + "{} is specifying `--gecko-sparse-profile` to run-task " + "as two arguments. Unable to determine if the sparse " + "profile exists.".format(task["label"]) + ) + _, sparse_profile = arg.split("=", 1) + if not os.path.exists(os.path.join(GECKO, sparse_profile)): + raise Exception( + "{} is using non-existant sparse profile {}.".format( + task["label"], sparse_profile + ) + ) + require_sparse_cache = True + break + + for cache in payload.get("cache", {}): + if not cache.startswith(cache_prefix): + raise Exception( + "{} is using a cache ({}) which is not appropriate " + "for its trust-domain and level. It should start with {}.".format( + task["label"], cache, cache_prefix + ) + ) + + cache = cache[len(cache_prefix) :] + + if re_sparse_checkout_cache.match(cache): + have_sparse_cache = True + + if not re_reserved_caches.match(cache): + continue + + if not run_task: + raise Exception( + "%s is using a cache (%s) reserved for run-task " + "change the task to use run-task or use a different " + "cache name" % (task["label"], cache) + ) + + if not cache.endswith(suffix): + raise Exception( + "%s is using a cache (%s) reserved for run-task " + "but the cache name is not dependent on the contents " + "of run-task; change the cache name to conform to the " + "naming requirements" % (task["label"], cache) + ) + + if require_sparse_cache and not have_sparse_cache: + raise Exception( + "%s is using a sparse checkout but not using " + "a sparse checkout cache; change the checkout " + "cache name so it is sparse aware" % task["label"] + ) + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/test/__init__.py b/taskcluster/gecko_taskgraph/transforms/test/__init__.py new file mode 100644 index 0000000000..ac17554baa --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/test/__init__.py @@ -0,0 +1,538 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +These transforms construct a task description to run the given test, based on a +test description. The implementation here is shared among all test kinds, but +contains specific support for how we run tests in Gecko (via mozharness, +invoked in particular ways). + +This is a good place to translate a test-description option such as +`single-core: true` to the implementation of that option in a task description +(worker options, mozharness commandline, environment variables, etc.) + +The test description should be fully formed by the time it reaches these +transforms, and these transforms should not embody any specific knowledge about +what should run where. this is the wrong place for special-casing platforms, +for example - use `all_tests.py` instead. +""" + + +import logging +from importlib import import_module + +from mozbuild.schedules import INCLUSIVE_COMPONENTS +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from voluptuous import Any, Exclusive, Optional, Required + +from gecko_taskgraph.optimize.schema import OptimizationSchema +from gecko_taskgraph.transforms.test.other import get_mobile_project +from gecko_taskgraph.util.chunking import manifest_loaders + +logger = logging.getLogger(__name__) +transforms = TransformSequence() + + +# Schema for a test description +# +# *****WARNING***** +# +# This is a great place for baffling cruft to accumulate, and that makes +# everyone move more slowly. Be considerate of your fellow hackers! +# See the warnings in taskcluster/docs/how-tos.rst +# +# *****WARNING***** +test_description_schema = Schema( + { + # description of the suite, for the task metadata + Required("description"): str, + # test suite category and name + Optional("suite"): Any( + optionally_keyed_by("variant", str), + { + Optional("category"): str, + Optional("name"): optionally_keyed_by("variant", str), + }, + ), + # base work directory used to set up the task. + Optional("workdir"): optionally_keyed_by("test-platform", Any(str, "default")), + # the name by which this test suite is addressed in try syntax; defaults to + # the test-name. This will translate to the `unittest_try_name` or + # `talos_try_name` attribute. + Optional("try-name"): str, + # additional tags to mark up this type of test + Optional("tags"): {str: object}, + # the symbol, or group(symbol), under which this task should appear in + # treeherder. + Required("treeherder-symbol"): str, + # the value to place in task.extra.treeherder.machine.platform; ideally + # this is the same as build-platform, and that is the default, but in + # practice it's not always a match. + Optional("treeherder-machine-platform"): str, + # attributes to appear in the resulting task (later transforms will add the + # common attributes) + Optional("attributes"): {str: object}, + # relative path (from config.path) to the file task was defined in + Optional("job-from"): str, + # The `run_on_projects` attribute, defaulting to "all". This dictates the + # projects on which this task should be included in the target task set. + # See the attributes documentation for details. + # + # Note that the special case 'built-projects', the default, uses the parent + # build task's run-on-projects, meaning that tests run only on platforms + # that are built. + Optional("run-on-projects"): optionally_keyed_by( + "app", + "subtest", + "test-platform", + "test-name", + "variant", + Any([str], "built-projects"), + ), + # When set only run on projects where the build would already be running. + # This ensures tasks where this is True won't be the cause of the build + # running on a project it otherwise wouldn't have. + Optional("built-projects-only"): bool, + # the sheriffing tier for this task (default: set based on test platform) + Optional("tier"): optionally_keyed_by( + "test-platform", "variant", "app", "subtest", Any(int, "default") + ), + # number of chunks to create for this task. This can be keyed by test + # platform by passing a dictionary in the `by-test-platform` key. If the + # test platform is not found, the key 'default' will be tried. + Required("chunks"): optionally_keyed_by( + "test-platform", "variant", Any(int, "dynamic") + ), + # Custom 'test_manifest_loader' to use, overriding the one configured in the + # parameters. When 'null', no test chunking will be performed. Can also + # be used to disable "manifest scheduling". + Optional("test-manifest-loader"): Any(None, *list(manifest_loaders)), + # the time (with unit) after which this task is deleted; default depends on + # the branch (see below) + Optional("expires-after"): str, + # The different configurations that should be run against this task, defined + # in the TEST_VARIANTS object in the variant.py transforms. + Optional("variants"): [str], + # Whether to run this task without any variants applied. + Required("run-without-variant"): optionally_keyed_by("test-platform", bool), + # The EC2 instance size to run these tests on. + Required("instance-size"): optionally_keyed_by( + "test-platform", Any("default", "large", "xlarge") + ), + # type of virtualization or hardware required by test. + Required("virtualization"): optionally_keyed_by( + "test-platform", Any("virtual", "virtual-with-gpu", "hardware") + ), + # Whether the task requires loopback audio or video (whatever that may mean + # on the platform) + Required("loopback-audio"): bool, + Required("loopback-video"): bool, + # Whether the test can run using a software GL implementation on Linux + # using the GL compositor. May not be used with "legacy" sized instances + # due to poor LLVMPipe performance (bug 1296086). Defaults to true for + # unit tests on linux platforms and false otherwise + Optional("allow-software-gl-layers"): bool, + # For tasks that will run in docker-worker, this is the + # name of the docker image or in-tree docker image to run the task in. If + # in-tree, then a dependency will be created automatically. This is + # generally `desktop-test`, or an image that acts an awful lot like it. + Required("docker-image"): optionally_keyed_by( + "test-platform", + Any( + # a raw Docker image path (repo/image:tag) + str, + # an in-tree generated docker image (from `taskcluster/docker/<name>`) + {"in-tree": str}, + # an indexed docker image + {"indexed": str}, + ), + ), + # seconds of runtime after which the task will be killed. Like 'chunks', + # this can be keyed by test platform, but also variant. + Required("max-run-time"): optionally_keyed_by( + "test-platform", "subtest", "variant", "app", int + ), + # the exit status code that indicates the task should be retried + Optional("retry-exit-status"): [int], + # Whether to perform a gecko checkout. + Required("checkout"): bool, + # Wheter to perform a machine reboot after test is done + Optional("reboot"): Any(False, "always", "on-exception", "on-failure"), + # What to run + Required("mozharness"): { + # the mozharness script used to run this task + Required("script"): optionally_keyed_by("test-platform", str), + # the config files required for the task + Required("config"): optionally_keyed_by("test-platform", [str]), + # mochitest flavor for mochitest runs + Optional("mochitest-flavor"): str, + # any additional actions to pass to the mozharness command + Optional("actions"): [str], + # additional command-line options for mozharness, beyond those + # automatically added + Required("extra-options"): optionally_keyed_by("test-platform", [str]), + # the artifact name (including path) to test on the build task; this is + # generally set in a per-kind transformation + Optional("build-artifact-name"): str, + Optional("installer-url"): str, + # If not false, tooltool downloads will be enabled via relengAPIProxy + # for either just public files, or all files. Not supported on Windows + Required("tooltool-downloads"): Any( + False, + "public", + "internal", + ), + # Add --blob-upload-branch=<project> mozharness parameter + Optional("include-blob-upload-branch"): bool, + # The setting for --download-symbols (if omitted, the option will not + # be passed to mozharness) + Optional("download-symbols"): Any(True, "ondemand"), + # If set, then MOZ_NODE_PATH=/usr/local/bin/node is included in the + # environment. This is more than just a helpful path setting -- it + # causes xpcshell tests to start additional servers, and runs + # additional tests. + Required("set-moz-node-path"): bool, + # If true, include chunking information in the command even if the number + # of chunks is 1 + Required("chunked"): optionally_keyed_by("test-platform", bool), + Required("requires-signed-builds"): optionally_keyed_by( + "test-platform", "variant", bool + ), + }, + # The set of test manifests to run. + Optional("test-manifests"): Any( + [str], + {"active": [str], "skipped": [str]}, + ), + # The current chunk (if chunking is enabled). + Optional("this-chunk"): int, + # os user groups for test task workers; required scopes, will be + # added automatically + Optional("os-groups"): optionally_keyed_by("test-platform", [str]), + Optional("run-as-administrator"): optionally_keyed_by("test-platform", bool), + # -- values supplied by the task-generation infrastructure + # the platform of the build this task is testing + Required("build-platform"): str, + # the label of the build task generating the materials to test + Required("build-label"): str, + # the label of the signing task generating the materials to test. + # Signed builds are used in xpcshell tests on Windows, for instance. + Optional("build-signing-label"): optionally_keyed_by("variant", str), + # the build's attributes + Required("build-attributes"): {str: object}, + # the platform on which the tests will run + Required("test-platform"): str, + # limit the test-platforms (as defined in test-platforms.yml) + # that the test will run on + Optional("limit-platforms"): optionally_keyed_by("app", "subtest", [str]), + # the name of the test (the key in tests.yml) + Required("test-name"): str, + # the product name, defaults to firefox + Optional("product"): str, + # conditional files to determine when these tests should be run + Exclusive("when", "optimization"): { + Optional("files-changed"): [str], + }, + # Optimization to perform on this task during the optimization phase. + # Optimizations are defined in taskcluster/gecko_taskgraph/optimize.py. + Exclusive("optimization", "optimization"): OptimizationSchema, + # The SCHEDULES component for this task; this defaults to the suite + # (not including the flavor) but can be overridden here. + Exclusive("schedules-component", "optimization"): Any( + str, + [str], + ), + Optional("worker-type"): optionally_keyed_by( + "test-platform", + Any(str, None), + ), + Optional( + "require-signed-extensions", + description="Whether the build being tested requires extensions be signed.", + ): optionally_keyed_by("release-type", "test-platform", bool), + # The target name, specifying the build artifact to be tested. + # If None or not specified, a transform sets the target based on OS: + # target.dmg (Mac), target.apk (Android), target.tar.bz2 (Linux), + # or target.zip (Windows). + Optional("target"): optionally_keyed_by( + "app", + "test-platform", + "variant", + Any( + str, + None, + {Required("index"): str, Required("name"): str}, + ), + ), + # A list of artifacts to install from 'fetch' tasks. Validation deferred + # to 'job' transforms. + Optional("fetches"): object, + # Raptor / browsertime specific keys, defer validation to 'raptor.py' + # transform. + Optional("raptor"): object, + # Raptor / browsertime specific keys that need to be here since 'raptor' schema + # is evluated *before* test_description_schema + Optional("app"): str, + Optional("subtest"): str, + # Define if a given task supports artifact builds or not, see bug 1695325. + Optional("supports-artifact-builds"): bool, + } +) + + +@transforms.add +def handle_keyed_by_mozharness(config, tasks): + """Resolve a mozharness field if it is keyed by something""" + fields = [ + "mozharness", + "mozharness.chunked", + "mozharness.config", + "mozharness.extra-options", + "mozharness.script", + ] + for task in tasks: + for field in fields: + resolve_keyed_by( + task, + field, + item_name=task["test-name"], + enforce_single_match=False, + ) + yield task + + +@transforms.add +def set_defaults(config, tasks): + for task in tasks: + build_platform = task["build-platform"] + if build_platform.startswith("android"): + # all Android test tasks download internal objects from tooltool + task["mozharness"]["tooltool-downloads"] = "internal" + task["mozharness"]["actions"] = ["get-secrets"] + + # loopback-video is always true for Android, but false for other + # platform phyla + task["loopback-video"] = True + task["mozharness"]["set-moz-node-path"] = True + + # software-gl-layers is only meaningful on linux unittests, where it defaults to True + if task["test-platform"].startswith("linux") and task["suite"] not in [ + "talos", + "raptor", + ]: + task.setdefault("allow-software-gl-layers", True) + else: + task["allow-software-gl-layers"] = False + + task.setdefault("try-name", task["test-name"]) + task.setdefault("os-groups", []) + task.setdefault("run-as-administrator", False) + task.setdefault("chunks", 1) + task.setdefault("run-on-projects", "built-projects") + task.setdefault("built-projects-only", False) + task.setdefault("instance-size", "default") + task.setdefault("max-run-time", 3600) + task.setdefault("reboot", False) + task.setdefault("virtualization", "virtual") + task.setdefault("loopback-audio", False) + task.setdefault("loopback-video", False) + task.setdefault("limit-platforms", []) + task.setdefault("docker-image", {"in-tree": "ubuntu1804-test"}) + task.setdefault("checkout", False) + task.setdefault("require-signed-extensions", False) + task.setdefault("run-without-variant", True) + task.setdefault("variants", []) + task.setdefault("supports-artifact-builds", True) + + task["mozharness"].setdefault("extra-options", []) + task["mozharness"].setdefault("requires-signed-builds", False) + task["mozharness"].setdefault("tooltool-downloads", "public") + task["mozharness"].setdefault("set-moz-node-path", False) + task["mozharness"].setdefault("chunked", False) + yield task + + +transforms.add_validate(test_description_schema) + + +@transforms.add +def run_variant_transforms(config, tasks): + """Variant transforms are run as soon as possible to allow other transforms + to key by variant.""" + for task in tasks: + xforms = TransformSequence() + mod = import_module("gecko_taskgraph.transforms.test.variant") + xforms.add(mod.transforms) + + yield from xforms(config, [task]) + + +@transforms.add +def resolve_keys(config, tasks): + keys = ("require-signed-extensions", "run-without-variant", "suite", "suite.name") + for task in tasks: + for key in keys: + resolve_keyed_by( + task, + key, + item_name=task["test-name"], + enforce_single_match=False, + **{ + "release-type": config.params["release_type"], + "variant": task["attributes"].get("unittest_variant"), + }, + ) + yield task + + +@transforms.add +def run_remaining_transforms(config, tasks): + """Runs other transform files next to this module.""" + # List of modules to load transforms from in order. + transform_modules = ( + ("raptor", lambda t: t["suite"] == "raptor"), + ("other", None), + ("worker", None), + # These transforms should always run last as there is never any + # difference in configuration from one chunk to another (other than + # chunk number). + ("chunk", None), + ) + + for task in tasks: + xforms = TransformSequence() + for name, filterfn in transform_modules: + if filterfn and not filterfn(task): + continue + + mod = import_module(f"gecko_taskgraph.transforms.test.{name}") + xforms.add(mod.transforms) + + yield from xforms(config, [task]) + + +@transforms.add +def make_job_description(config, tasks): + """Convert *test* descriptions to *job* descriptions (input to + gecko_taskgraph.transforms.job)""" + + for task in tasks: + attributes = task.get("attributes", {}) + + mobile = get_mobile_project(task) + if mobile and (mobile not in task["test-name"]): + label = "{}-{}-{}-{}".format( + config.kind, task["test-platform"], mobile, task["test-name"] + ) + else: + label = "{}-{}-{}".format( + config.kind, task["test-platform"], task["test-name"] + ) + + try_name = task["try-name"] + if attributes.get("unittest_variant"): + suffix = task.pop("variant-suffix") + label += suffix + try_name += suffix + + if task["chunks"] > 1: + label += "-{}".format(task["this-chunk"]) + + build_label = task["build-label"] + + if task["suite"] == "talos": + attr_try_name = "talos_try_name" + elif task["suite"] == "raptor": + attr_try_name = "raptor_try_name" + else: + attr_try_name = "unittest_try_name" + + attr_build_platform, attr_build_type = task["build-platform"].split("/", 1) + attributes.update( + { + "build_platform": attr_build_platform, + "build_type": attr_build_type, + "test_platform": task["test-platform"], + "test_chunk": str(task["this-chunk"]), + "supports-artifact-builds": task["supports-artifact-builds"], + attr_try_name: try_name, + } + ) + + if "test-manifests" in task: + attributes["test_manifests"] = task["test-manifests"] + + jobdesc = {} + name = "{}-{}".format(task["test-platform"], task["test-name"]) + jobdesc["name"] = name + jobdesc["label"] = label + jobdesc["description"] = task["description"] + jobdesc["attributes"] = attributes + jobdesc["dependencies"] = {"build": build_label} + jobdesc["job-from"] = task["job-from"] + + if task.get("fetches"): + jobdesc["fetches"] = task["fetches"] + + if task["mozharness"]["requires-signed-builds"] is True: + jobdesc["dependencies"]["build-signing"] = task["build-signing-label"] + + if "expires-after" in task: + jobdesc["expires-after"] = task["expires-after"] + + jobdesc["routes"] = [] + jobdesc["run-on-projects"] = sorted(task["run-on-projects"]) + jobdesc["scopes"] = [] + jobdesc["tags"] = task.get("tags", {}) + jobdesc["extra"] = { + "chunks": { + "current": task["this-chunk"], + "total": task["chunks"], + }, + "suite": attributes["unittest_suite"], + "test-setting": task.pop("test-setting"), + } + jobdesc["treeherder"] = { + "symbol": task["treeherder-symbol"], + "kind": "test", + "tier": task["tier"], + "platform": task.get("treeherder-machine-platform", task["build-platform"]), + } + + schedules = task.get("schedules-component", []) + if task.get("when"): + # This may still be used by comm-central. + jobdesc["when"] = task["when"] + elif "optimization" in task: + jobdesc["optimization"] = task["optimization"] + elif set(schedules) & set(INCLUSIVE_COMPONENTS): + jobdesc["optimization"] = {"test-inclusive": schedules} + else: + jobdesc["optimization"] = {"test": schedules} + + run = jobdesc["run"] = {} + run["using"] = "mozharness-test" + run["test"] = task + + if "workdir" in task: + run["workdir"] = task.pop("workdir") + + jobdesc["worker-type"] = task.pop("worker-type") + + if "worker" in task: + jobdesc["worker"] = task.pop("worker") + + if task.get("fetches"): + jobdesc["fetches"] = task.pop("fetches") + + yield jobdesc + + +def normpath(path): + return path.replace("/", "\\") + + +def get_firefox_version(): + with open("browser/config/version.txt") as f: + return f.readline().strip() diff --git a/taskcluster/gecko_taskgraph/transforms/test/chunk.py b/taskcluster/gecko_taskgraph/transforms/test/chunk.py new file mode 100644 index 0000000000..f6442e3755 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/test/chunk.py @@ -0,0 +1,262 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json + +import taskgraph +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.attributes import keymatch +from taskgraph.util.treeherder import join_symbol, split_symbol + +from gecko_taskgraph.util.attributes import is_try +from gecko_taskgraph.util.chunking import ( + DefaultLoader, + chunk_manifests, + get_manifest_loader, + get_runtimes, + guess_mozinfo_from_task, +) +from gecko_taskgraph.util.copy_task import copy_task +from gecko_taskgraph.util.perfile import perfile_number_of_chunks + +DYNAMIC_CHUNK_DURATION = 20 * 60 # seconds +"""The approximate time each test chunk should take to run.""" + + +DYNAMIC_CHUNK_MULTIPLIER = { + # Desktop xpcshell tests run in parallel. Reduce the total runtime to + # compensate. + "^(?!android).*-xpcshell.*": 0.2, +} +"""A multiplication factor to tweak the total duration per platform / suite.""" + + +transforms = TransformSequence() + + +@transforms.add +def set_test_verify_chunks(config, tasks): + """Set the number of chunks we use for test-verify.""" + for task in tasks: + if any(task["suite"].startswith(s) for s in ("test-verify", "test-coverage")): + env = config.params.get("try_task_config", {}) or {} + env = env.get("templates", {}).get("env", {}) + task["chunks"] = perfile_number_of_chunks( + is_try(config.params), + env.get("MOZHARNESS_TEST_PATHS", ""), + config.params.get("head_repository", ""), + config.params.get("head_rev", ""), + task["test-name"], + ) + + # limit the number of chunks we run for test-verify mode because + # test-verify is comprehensive and takes a lot of time, if we have + # >30 tests changed, this is probably an import of external tests, + # or a patch renaming/moving files in bulk + maximum_number_verify_chunks = 3 + if task["chunks"] > maximum_number_verify_chunks: + task["chunks"] = maximum_number_verify_chunks + + yield task + + +@transforms.add +def set_test_manifests(config, tasks): + """Determine the set of test manifests that should run in this task.""" + + for task in tasks: + # When a task explicitly requests no 'test_manifest_loader', test + # resolving will happen at test runtime rather than in the taskgraph. + if "test-manifest-loader" in task and task["test-manifest-loader"] is None: + yield task + continue + + # Set 'tests_grouped' to "1", so we can differentiate between suites that are + # chunked at the test runtime and those that are chunked in the taskgraph. + task.setdefault("tags", {})["tests_grouped"] = "1" + + if taskgraph.fast: + # We want to avoid evaluating manifests when taskgraph.fast is set. But + # manifests are required for dynamic chunking. Just set the number of + # chunks to one in this case. + if task["chunks"] == "dynamic": + task["chunks"] = 1 + yield task + continue + + manifests = task.get("test-manifests") + if manifests: + if isinstance(manifests, list): + task["test-manifests"] = {"active": manifests, "skipped": []} + yield task + continue + + mozinfo = guess_mozinfo_from_task( + task, config.params.get("head_repository", "") + ) + + loader_name = task.pop( + "test-manifest-loader", config.params["test_manifest_loader"] + ) + loader = get_manifest_loader(loader_name, config.params) + + task["test-manifests"] = loader.get_manifests( + task["suite"], + frozenset(mozinfo.items()), + ) + + # When scheduling with test paths, we often find manifests scheduled but all tests + # are skipped on a given config. This will remove the task from the task set if + # no manifests have active tests for the given task/config + mh_test_paths = {} + if "MOZHARNESS_TEST_PATHS" in config.params.get("try_task_config", {}).get( + "env", {} + ): + mh_test_paths = json.loads( + config.params["try_task_config"]["env"]["MOZHARNESS_TEST_PATHS"] + ) + + if task["attributes"]["unittest_suite"] in mh_test_paths.keys(): + input_paths = mh_test_paths[task["attributes"]["unittest_suite"]] + remaining_manifests = [] + + # if we have web-platform tests incoming, just yield task + for m in input_paths: + if m.startswith("testing/web-platform/tests/"): + if not isinstance(loader, DefaultLoader): + task["chunks"] = "dynamic" + yield task + break + + # input paths can exist in other directories (i.e. [../../dir/test.js]) + # we need to look for all [active] manifests that include tests in the path + for m in input_paths: + if [tm for tm in task["test-manifests"]["active"] if tm.startswith(m)]: + remaining_manifests.append(m) + + # look in the 'other' manifests + for m in input_paths: + man = m + for tm in task["test-manifests"]["other_dirs"]: + matched_dirs = [ + dp + for dp in task["test-manifests"]["other_dirs"].get(tm) + if dp.startswith(man) + ] + if matched_dirs: + if tm not in task["test-manifests"]["active"]: + continue + if m not in remaining_manifests: + remaining_manifests.append(m) + + if remaining_manifests == []: + continue + + # The default loader loads all manifests. If we use a non-default + # loader, we'll only run some subset of manifests and the hardcoded + # chunk numbers will no longer be valid. Dynamic chunking should yield + # better results. + if not isinstance(loader, DefaultLoader): + task["chunks"] = "dynamic" + + yield task + + +@transforms.add +def resolve_dynamic_chunks(config, tasks): + """Determine how many chunks are needed to handle the given set of manifests.""" + + for task in tasks: + if task["chunks"] != "dynamic": + yield task + continue + + if not task.get("test-manifests"): + raise Exception( + "{} must define 'test-manifests' to use dynamic chunking!".format( + task["test-name"] + ) + ) + + runtimes = { + m: r + for m, r in get_runtimes(task["test-platform"], task["suite"]).items() + if m in task["test-manifests"]["active"] + } + + # Truncate runtimes that are above the desired chunk duration. They + # will be assigned to a chunk on their own and the excess duration + # shouldn't cause additional chunks to be needed. + times = [min(DYNAMIC_CHUNK_DURATION, r) for r in runtimes.values()] + avg = round(sum(times) / len(times), 2) if times else 0 + total = sum(times) + + # If there are manifests missing from the runtimes data, fill them in + # with the average of all present manifests. + missing = [m for m in task["test-manifests"]["active"] if m not in runtimes] + total += avg * len(missing) + + # Apply any chunk multipliers if found. + key = "{}-{}".format(task["test-platform"], task["test-name"]) + matches = keymatch(DYNAMIC_CHUNK_MULTIPLIER, key) + if len(matches) > 1: + raise Exception( + "Multiple matching values for {} found while " + "determining dynamic chunk multiplier!".format(key) + ) + elif matches: + total = total * matches[0] + + chunks = int(round(total / DYNAMIC_CHUNK_DURATION)) + + # Make sure we never exceed the number of manifests, nor have a chunk + # length of 0. + task["chunks"] = min(chunks, len(task["test-manifests"]["active"])) or 1 + yield task + + +@transforms.add +def split_chunks(config, tasks): + """Based on the 'chunks' key, split tests up into chunks by duplicating + them and assigning 'this-chunk' appropriately and updating the treeherder + symbol. + """ + + for task in tasks: + # If test-manifests are set, chunk them ahead of time to avoid running + # the algorithm more than once. + chunked_manifests = None + if "test-manifests" in task: + manifests = task["test-manifests"] + chunked_manifests = chunk_manifests( + task["suite"], + task["test-platform"], + task["chunks"], + manifests["active"], + ) + + # Add all skipped manifests to the first chunk of backstop pushes + # so they still show up in the logs. They won't impact runtime much + # and this way tools like ActiveData are still aware that they + # exist. + if config.params["backstop"] and manifests["active"]: + chunked_manifests[0].extend(manifests["skipped"]) + + for i in range(task["chunks"]): + this_chunk = i + 1 + + # copy the test and update with the chunk number + chunked = copy_task(task) + chunked["this-chunk"] = this_chunk + + if chunked_manifests is not None: + chunked["test-manifests"] = sorted(chunked_manifests[i]) + + group, symbol = split_symbol(chunked["treeherder-symbol"]) + if task["chunks"] > 1 or not symbol: + # add the chunk number to the TH symbol + symbol += str(this_chunk) + chunked["treeherder-symbol"] = join_symbol(group, symbol) + + yield chunked diff --git a/taskcluster/gecko_taskgraph/transforms/test/other.py b/taskcluster/gecko_taskgraph/transforms/test/other.py new file mode 100644 index 0000000000..1c08d290e4 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/test/other.py @@ -0,0 +1,1081 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import hashlib +import json +import re + +from mozbuild.schedules import INCLUSIVE_COMPONENTS +from mozbuild.util import ReadOnlyDict +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.attributes import keymatch +from taskgraph.util.keyed_by import evaluate_keyed_by +from taskgraph.util.schema import Schema, resolve_keyed_by +from taskgraph.util.taskcluster import get_artifact_path, get_index_url +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.transforms.test.variant import TEST_VARIANTS +from gecko_taskgraph.util.platforms import platform_family + +transforms = TransformSequence() + + +@transforms.add +def limit_platforms(config, tasks): + for task in tasks: + if not task["limit-platforms"]: + yield task + continue + + limited_platforms = {key: key for key in task["limit-platforms"]} + if keymatch(limited_platforms, task["test-platform"]): + yield task + + +@transforms.add +def handle_suite_category(config, tasks): + for task in tasks: + task.setdefault("suite", {}) + + if isinstance(task["suite"], str): + task["suite"] = {"name": task["suite"]} + + suite = task["suite"].setdefault("name", task["test-name"]) + category = task["suite"].setdefault("category", suite) + + task.setdefault("attributes", {}) + task["attributes"]["unittest_suite"] = suite + task["attributes"]["unittest_category"] = category + + script = task["mozharness"]["script"] + category_arg = None + if suite.startswith("test-verify") or suite.startswith("test-coverage"): + pass + elif script in ("android_emulator_unittest.py", "android_hardware_unittest.py"): + category_arg = "--test-suite" + elif script == "desktop_unittest.py": + category_arg = f"--{category}-suite" + + if category_arg: + task["mozharness"].setdefault("extra-options", []) + extra = task["mozharness"]["extra-options"] + if not any(arg.startswith(category_arg) for arg in extra): + extra.append(f"{category_arg}={suite}") + + # From here on out we only use the suite name. + task["suite"] = suite + yield task + + +@transforms.add +def setup_talos(config, tasks): + """Add options that are specific to talos jobs (identified by suite=talos)""" + for task in tasks: + if task["suite"] != "talos": + yield task + continue + + extra_options = task.setdefault("mozharness", {}).setdefault( + "extra-options", [] + ) + extra_options.append("--use-talos-json") + + # win7 needs to test skip + if task["build-platform"].startswith("win32"): + extra_options.append("--add-option") + extra_options.append("--setpref,gfx.direct2d.disabled=true") + + if config.params.get("project", None): + extra_options.append("--project=%s" % config.params["project"]) + + yield task + + +@transforms.add +def setup_browsertime_flag(config, tasks): + """Optionally add `--browsertime` flag to Raptor pageload tests.""" + + browsertime_flag = config.params["try_task_config"].get("browsertime", False) + + for task in tasks: + if not browsertime_flag or task["suite"] != "raptor": + yield task + continue + + if task["treeherder-symbol"].startswith("Rap"): + # The Rap group is subdivided as Rap{-fenix,-refbrow(...), + # so `taskgraph.util.treeherder.replace_group` isn't appropriate. + task["treeherder-symbol"] = task["treeherder-symbol"].replace( + "Rap", "Btime", 1 + ) + + extra_options = task.setdefault("mozharness", {}).setdefault( + "extra-options", [] + ) + extra_options.append("--browsertime") + + yield task + + +@transforms.add +def handle_artifact_prefix(config, tasks): + """Handle translating `artifact_prefix` appropriately""" + for task in tasks: + if task["build-attributes"].get("artifact_prefix"): + task.setdefault("attributes", {}).setdefault( + "artifact_prefix", task["build-attributes"]["artifact_prefix"] + ) + yield task + + +@transforms.add +def set_treeherder_machine_platform(config, tasks): + """Set the appropriate task.extra.treeherder.machine.platform""" + translation = { + # Linux64 build platform for asan is specified differently to + # treeherder. + "macosx1100-64/opt": "osx-1100/opt", + "macosx1100-64-shippable/opt": "osx-1100-shippable/opt", + "win64-asan/opt": "windows10-64/asan", + "win64-aarch64/opt": "windows10-aarch64/opt", + } + for task in tasks: + # For most desktop platforms, the above table is not used for "regular" + # builds, so we'll always pick the test platform here. + # On macOS though, the regular builds are in the table. This causes a + # conflict in `verify_task_graph_symbol` once you add a new test + # platform based on regular macOS builds, such as for QR. + # Since it's unclear if the regular macOS builds can be removed from + # the table, workaround the issue for QR. + if "android" in task["test-platform"] and "pgo/opt" in task["test-platform"]: + platform_new = task["test-platform"].replace("-pgo/opt", "/pgo") + task["treeherder-machine-platform"] = platform_new + elif "android-em-7.0-x86_64-qr" in task["test-platform"]: + task["treeherder-machine-platform"] = task["test-platform"].replace( + ".", "-" + ) + elif "android-em-7.0-x86_64-shippable-qr" in task["test-platform"]: + task["treeherder-machine-platform"] = task["test-platform"].replace( + ".", "-" + ) + elif "android-em-7.0-x86_64-lite-qr" in task["test-platform"]: + task["treeherder-machine-platform"] = task["test-platform"].replace( + ".", "-" + ) + elif "android-em-7.0-x86_64-shippable-lite-qr" in task["test-platform"]: + task["treeherder-machine-platform"] = task["test-platform"].replace( + ".", "-" + ) + elif "-qr" in task["test-platform"]: + task["treeherder-machine-platform"] = task["test-platform"] + elif "android-hw" in task["test-platform"]: + task["treeherder-machine-platform"] = task["test-platform"] + elif "android-em-7.0-x86_64" in task["test-platform"]: + task["treeherder-machine-platform"] = task["test-platform"].replace( + ".", "-" + ) + elif "android-em-7.0-x86" in task["test-platform"]: + task["treeherder-machine-platform"] = task["test-platform"].replace( + ".", "-" + ) + # Bug 1602863 - must separately define linux64/asan and linux1804-64/asan + # otherwise causes an exception during taskgraph generation about + # duplicate treeherder platform/symbol. + elif "linux64-asan/opt" in task["test-platform"]: + task["treeherder-machine-platform"] = "linux64/asan" + elif "linux1804-asan/opt" in task["test-platform"]: + task["treeherder-machine-platform"] = "linux1804-64/asan" + else: + task["treeherder-machine-platform"] = translation.get( + task["build-platform"], task["test-platform"] + ) + yield task + + +@transforms.add +def set_download_symbols(config, tasks): + """In general, we download symbols immediately for debug builds, but only + on demand for everything else. ASAN builds shouldn't download + symbols since they don't product symbol zips see bug 1283879""" + for task in tasks: + if task["test-platform"].split("/")[-1] == "debug": + task["mozharness"]["download-symbols"] = True + elif "asan" in task["build-platform"] or "tsan" in task["build-platform"]: + if "download-symbols" in task["mozharness"]: + del task["mozharness"]["download-symbols"] + else: + task["mozharness"]["download-symbols"] = "ondemand" + yield task + + +@transforms.add +def handle_keyed_by(config, tasks): + """Resolve fields that can be keyed by platform, etc.""" + fields = [ + "instance-size", + "docker-image", + "max-run-time", + "chunks", + "suite", + "run-on-projects", + "os-groups", + "run-as-administrator", + "workdir", + "worker-type", + "virtualization", + "fetches.fetch", + "fetches.toolchain", + "target", + "webrender-run-on-projects", + "mozharness.requires-signed-builds", + "build-signing-label", + ] + for task in tasks: + for field in fields: + resolve_keyed_by( + task, + field, + item_name=task["test-name"], + enforce_single_match=False, + project=config.params["project"], + variant=task["attributes"].get("unittest_variant"), + ) + yield task + + +@transforms.add +def set_target(config, tasks): + for task in tasks: + build_platform = task["build-platform"] + target = None + if "target" in task: + target = task["target"] + if not target: + if build_platform.startswith("macosx"): + target = "target.dmg" + elif build_platform.startswith("android"): + target = "target.apk" + elif build_platform.startswith("win"): + target = "target.zip" + else: + target = "target.tar.bz2" + + if isinstance(target, dict): + # TODO Remove hardcoded mobile artifact prefix + index_url = get_index_url(target["index"]) + installer_url = "{}/artifacts/public/{}".format(index_url, target["name"]) + task["mozharness"]["installer-url"] = installer_url + else: + task["mozharness"]["build-artifact-name"] = get_artifact_path(task, target) + + yield task + + +@transforms.add +def setup_browsertime(config, tasks): + """Configure browsertime dependencies for Raptor pageload tests that have + `--browsertime` extra option.""" + + for task in tasks: + # We need to make non-trivial changes to various fetches, and our + # `by-test-platform` may not be "compatible" with existing + # `by-test-platform` filters. Therefore we do everything after + # `handle_keyed_by` so that existing fields have been resolved down to + # simple lists. But we use the `by-test-platform` machinery to express + # filters so that when the time comes to move browsertime into YAML + # files, the transition is straight-forward. + extra_options = task.get("mozharness", {}).get("extra-options", []) + + if task["suite"] != "raptor" or "--webext" in extra_options: + yield task + continue + + ts = { + "by-test-platform": { + "android.*": ["browsertime", "linux64-geckodriver", "linux64-node-16"], + "linux.*": ["browsertime", "linux64-geckodriver", "linux64-node-16"], + "macosx.*": ["browsertime", "macosx64-geckodriver", "macosx64-node-16"], + "windows.*aarch64.*": [ + "browsertime", + "win32-geckodriver", + "win32-node-16", + ], + "windows.*-32.*": ["browsertime", "win32-geckodriver", "win32-node-16"], + "windows.*-64.*": ["browsertime", "win64-geckodriver", "win64-node-16"], + }, + } + + task.setdefault("fetches", {}).setdefault("toolchain", []).extend( + evaluate_keyed_by(ts, "fetches.toolchain", task) + ) + + fs = { + "by-test-platform": { + "android.*": ["linux64-ffmpeg-4.4.1"], + "linux.*": ["linux64-ffmpeg-4.4.1"], + "macosx.*": ["mac64-ffmpeg-4.4.1"], + "windows.*aarch64.*": ["win64-ffmpeg-4.4.1"], + "windows.*-32.*": ["win64-ffmpeg-4.4.1"], + "windows.*-64.*": ["win64-ffmpeg-4.4.1"], + }, + } + + cd_fetches = { + "android.*": [ + "linux64-chromedriver-109", + "linux64-chromedriver-110", + "linux64-chromedriver-111", + "linux64-chromedriver-112", + "linux64-chromedriver-113", + "linux64-chromedriver-114", + ], + "linux.*": [ + "linux64-chromedriver-112", + "linux64-chromedriver-113", + "linux64-chromedriver-114", + ], + "macosx.*": [ + "mac64-chromedriver-109", + "mac64-chromedriver-110", + "mac64-chromedriver-111", + "mac64-chromedriver-112", + "mac64-chromedriver-113", + "mac64-chromedriver-114", + ], + "windows.*aarch64.*": [ + "win32-chromedriver-112", + "win32-chromedriver-113", + "win32-chromedriver-114", + ], + "windows.*-32.*": [ + "win32-chromedriver-112", + "win32-chromedriver-113", + "win32-chromedriver-114", + ], + "windows.*-64.*": [ + "win32-chromedriver-112", + "win32-chromedriver-113", + "win32-chromedriver-114", + ], + } + + chromium_fetches = { + "linux.*": ["linux64-chromium"], + "macosx.*": ["mac-chromium"], + "windows.*aarch64.*": ["win32-chromium"], + "windows.*-32.*": ["win32-chromium"], + "windows.*-64.*": ["win64-chromium"], + } + + cd_extracted_name = { + "windows": "{}chromedriver.exe", + "mac": "{}chromedriver", + "default": "{}chromedriver", + } + + if "--app=chrome" in extra_options or "--app=chrome-m" in extra_options: + # Only add the chromedriver fetches when chrome is running + for platform in cd_fetches: + fs["by-test-platform"][platform].extend(cd_fetches[platform]) + if "--app=chromium" in extra_options or "--app=custom-car" in extra_options: + for platform in chromium_fetches: + fs["by-test-platform"][platform].extend(chromium_fetches[platform]) + + # The chromedrivers for chromium are repackaged into the archives + # that we get the chromium binary from so we always have a compatible + # version. + cd_extracted_name = { + "windows": "chrome-win/chromedriver.exe", + "mac": "chrome-mac/chromedriver", + "default": "chrome-linux/chromedriver", + } + + # Disable the Raptor install step + if "--app=chrome-m" in extra_options: + extra_options.append("--noinstall") + + task.setdefault("fetches", {}).setdefault("fetch", []).extend( + evaluate_keyed_by(fs, "fetches.fetch", task) + ) + + extra_options.extend( + ( + "--browsertime-browsertimejs", + "$MOZ_FETCHES_DIR/browsertime/node_modules/browsertime/bin/browsertime.js", + ) + ) # noqa: E501 + + eos = { + "by-test-platform": { + "windows.*": [ + "--browsertime-node", + "$MOZ_FETCHES_DIR/node/node.exe", + "--browsertime-geckodriver", + "$MOZ_FETCHES_DIR/geckodriver.exe", + "--browsertime-chromedriver", + "$MOZ_FETCHES_DIR/" + cd_extracted_name["windows"], + "--browsertime-ffmpeg", + "$MOZ_FETCHES_DIR/ffmpeg-4.4.1-full_build/bin/ffmpeg.exe", + ], + "macosx.*": [ + "--browsertime-node", + "$MOZ_FETCHES_DIR/node/bin/node", + "--browsertime-geckodriver", + "$MOZ_FETCHES_DIR/geckodriver", + "--browsertime-chromedriver", + "$MOZ_FETCHES_DIR/" + cd_extracted_name["mac"], + "--browsertime-ffmpeg", + "$MOZ_FETCHES_DIR/ffmpeg-macos/ffmpeg", + ], + "default": [ + "--browsertime-node", + "$MOZ_FETCHES_DIR/node/bin/node", + "--browsertime-geckodriver", + "$MOZ_FETCHES_DIR/geckodriver", + "--browsertime-chromedriver", + "$MOZ_FETCHES_DIR/" + cd_extracted_name["default"], + "--browsertime-ffmpeg", + "$MOZ_FETCHES_DIR/ffmpeg-4.4.1-i686-static/ffmpeg", + ], + } + } + + extra_options.extend(evaluate_keyed_by(eos, "mozharness.extra-options", task)) + + yield task + + +def get_mobile_project(task): + """Returns the mobile project of the specified task or None.""" + + if not task["build-platform"].startswith("android"): + return + + mobile_projects = ("fenix", "geckoview", "refbrow", "chrome-m") + + for name in mobile_projects: + if name in task["test-name"]: + return name + + target = None + if "target" in task: + resolve_keyed_by( + task, "target", item_name=task["test-name"], enforce_single_match=False + ) + target = task["target"] + if target: + if isinstance(target, dict): + target = target["name"] + + for name in mobile_projects: + if name in target: + return name + + return None + + +@transforms.add +def disable_wpt_timeouts_on_autoland(config, tasks): + """do not run web-platform-tests that are expected TIMEOUT on autoland""" + for task in tasks: + if ( + "web-platform-tests" in task["test-name"] + and config.params["project"] == "autoland" + ): + task["mozharness"].setdefault("extra-options", []).append("--skip-timeout") + yield task + + +@transforms.add +def enable_code_coverage(config, tasks): + """Enable code coverage for the ccov build-platforms""" + for task in tasks: + if "ccov" in task["build-platform"]: + # Do not run tests on fuzzing builds + if "fuzzing" in task["build-platform"]: + task["run-on-projects"] = [] + continue + + # Skip this transform for android code coverage builds. + if "android" in task["build-platform"]: + task.setdefault("fetches", {}).setdefault("toolchain", []).append( + "linux64-grcov" + ) + task["mozharness"].setdefault("extra-options", []).append( + "--java-code-coverage" + ) + yield task + continue + task["mozharness"].setdefault("extra-options", []).append("--code-coverage") + task["instance-size"] = "xlarge" + + # Temporarily disable Mac tests on mozilla-central + if "mac" in task["build-platform"]: + task["run-on-projects"] = [] + + # Ensure we always run on the projects defined by the build, unless the test + # is try only or shouldn't run at all. + if task["run-on-projects"] not in [[]]: + task["run-on-projects"] = "built-projects" + + # Ensure we don't optimize test suites out. + # We always want to run all test suites for coverage purposes. + task.pop("schedules-component", None) + task.pop("when", None) + task["optimization"] = None + + # Add a toolchain and a fetch task for the grcov binary. + if any(p in task["build-platform"] for p in ("linux", "osx", "win")): + task.setdefault("fetches", {}) + task["fetches"].setdefault("fetch", []) + task["fetches"].setdefault("toolchain", []) + task["fetches"].setdefault("build", []) + + if "linux" in task["build-platform"]: + task["fetches"]["toolchain"].append("linux64-grcov") + elif "osx" in task["build-platform"]: + task["fetches"]["toolchain"].append("macosx64-grcov") + elif "win" in task["build-platform"]: + task["fetches"]["toolchain"].append("win64-grcov") + + task["fetches"]["build"].append({"artifact": "target.mozinfo.json"}) + + if "talos" in task["test-name"]: + task["max-run-time"] = 7200 + if "linux" in task["build-platform"]: + task["docker-image"] = {"in-tree": "ubuntu1804-test"} + task["mozharness"]["extra-options"].append("--add-option") + task["mozharness"]["extra-options"].append("--cycles,1") + task["mozharness"]["extra-options"].append("--add-option") + task["mozharness"]["extra-options"].append("--tppagecycles,1") + task["mozharness"]["extra-options"].append("--add-option") + task["mozharness"]["extra-options"].append("--no-upload-results") + task["mozharness"]["extra-options"].append("--add-option") + task["mozharness"]["extra-options"].append("--tptimeout,15000") + if "raptor" in task["test-name"]: + task["max-run-time"] = 1800 + yield task + + +@transforms.add +def handle_run_on_projects(config, tasks): + """Handle translating `built-projects` appropriately""" + for task in tasks: + if task["run-on-projects"] == "built-projects": + task["run-on-projects"] = task["build-attributes"].get( + "run_on_projects", ["all"] + ) + + if task.pop("built-projects-only", False): + built_projects = set( + task["build-attributes"].get("run_on_projects", {"all"}) + ) + run_on_projects = set(task.get("run-on-projects", set())) + + # If 'all' exists in run-on-projects, then the intersection of both + # is built-projects. Similarly if 'all' exists in built-projects, + # the intersection is run-on-projects (so do nothing). When neither + # contains 'all', take the actual set intersection. + if "all" in run_on_projects: + task["run-on-projects"] = sorted(built_projects) + elif "all" not in built_projects: + task["run-on-projects"] = sorted(run_on_projects & built_projects) + yield task + + +@transforms.add +def handle_tier(config, tasks): + """Set the tier based on policy for all test descriptions that do not + specify a tier otherwise.""" + for task in tasks: + if "tier" in task: + resolve_keyed_by( + task, + "tier", + item_name=task["test-name"], + variant=task["attributes"].get("unittest_variant"), + enforce_single_match=False, + ) + + # only override if not set for the test + if "tier" not in task or task["tier"] == "default": + if task["test-platform"] in [ + "linux64/opt", + "linux64/debug", + "linux64-shippable/opt", + "linux64-devedition/opt", + "linux64-asan/opt", + "linux64-qr/opt", + "linux64-qr/debug", + "linux64-shippable-qr/opt", + "linux1804-64/opt", + "linux1804-64/debug", + "linux1804-64-shippable/opt", + "linux1804-64-devedition/opt", + "linux1804-64-qr/opt", + "linux1804-64-qr/debug", + "linux1804-64-shippable-qr/opt", + "linux1804-64-asan-qr/opt", + "linux1804-64-tsan-qr/opt", + "windows7-32-qr/debug", + "windows7-32-qr/opt", + "windows7-32-devedition-qr/opt", + "windows7-32-shippable-qr/opt", + "windows10-32-qr/debug", + "windows10-32-qr/opt", + "windows10-32-shippable-qr/opt", + "windows10-32-2004-qr/debug", + "windows10-32-2004-qr/opt", + "windows10-32-2004-shippable-qr/opt", + "windows10-aarch64-qr/opt", + "windows10-64/debug", + "windows10-64/opt", + "windows10-64-shippable/opt", + "windows10-64-devedition/opt", + "windows10-64-qr/opt", + "windows10-64-qr/debug", + "windows10-64-shippable-qr/opt", + "windows10-64-devedition-qr/opt", + "windows10-64-asan-qr/opt", + "windows10-64-2004-qr/opt", + "windows10-64-2004-qr/debug", + "windows10-64-2004-shippable-qr/opt", + "windows10-64-2004-devedition-qr/opt", + "windows10-64-2004-asan-qr/opt", + "windows11-32-2009-qr/debug", + "windows11-32-2009-qr/opt", + "windows11-32-2009-shippable-qr/opt", + "windows11-64-2009-qr/opt", + "windows11-64-2009-qr/debug", + "windows11-64-2009-shippable-qr/opt", + "windows11-64-2009-devedition-qr/opt", + "windows11-64-2009-asan-qr/opt", + "macosx1015-64/opt", + "macosx1015-64/debug", + "macosx1015-64-shippable/opt", + "macosx1015-64-devedition/opt", + "macosx1015-64-devedition-qr/opt", + "macosx1015-64-qr/opt", + "macosx1015-64-shippable-qr/opt", + "macosx1015-64-qr/debug", + "macosx1100-64-shippable-qr/opt", + "macosx1100-64-qr/debug", + "android-em-7.0-x86_64-shippable/opt", + "android-em-7.0-x86_64-shippable-lite/opt", + "android-em-7.0-x86_64/debug", + "android-em-7.0-x86_64/debug-isolated-process", + "android-em-7.0-x86_64/opt", + "android-em-7.0-x86_64-lite/opt", + "android-em-7.0-x86-shippable/opt", + "android-em-7.0-x86-shippable-lite/opt", + "android-em-7.0-x86_64-shippable-qr/opt", + "android-em-7.0-x86_64-qr/debug", + "android-em-7.0-x86_64-qr/debug-isolated-process", + "android-em-7.0-x86_64-qr/opt", + "android-em-7.0-x86_64-shippable-lite-qr/opt", + "android-em-7.0-x86_64-lite-qr/debug", + "android-em-7.0-x86_64-lite-qr/opt", + ]: + task["tier"] = 1 + else: + task["tier"] = 2 + + yield task + + +@transforms.add +def apply_raptor_tier_optimization(config, tasks): + for task in tasks: + if task["suite"] != "raptor": + yield task + continue + + if "regression-tests" in task["test-name"]: + # Don't optimize the regression tests + yield task + continue + + if not task["test-platform"].startswith("android-hw"): + task["optimization"] = {"skip-unless-expanded": None} + if task["tier"] > 1: + task["optimization"] = {"skip-unless-backstop": None} + + if task["attributes"].get("unittest_variant"): + task["tier"] = max(task["tier"], 2) + yield task + + +@transforms.add +def disable_try_only_platforms(config, tasks): + """Turns off platforms that should only run on try.""" + try_only_platforms = () + for task in tasks: + if any(re.match(k + "$", task["test-platform"]) for k in try_only_platforms): + task["run-on-projects"] = [] + yield task + + +@transforms.add +def ensure_spi_disabled_on_all_but_spi(config, tasks): + for task in tasks: + variant = task["attributes"].get("unittest_variant", "") + has_no_setpref = ("gtest", "cppunit", "jittest", "junit", "raptor") + + if ( + all(s not in task["suite"] for s in has_no_setpref) + and "socketprocess" not in variant + ): + task["mozharness"]["extra-options"].append( + "--setpref=media.peerconnection.mtransport_process=false" + ) + task["mozharness"]["extra-options"].append( + "--setpref=network.process.enabled=false" + ) + + yield task + + +test_setting_description_schema = Schema( + { + Required("_hash"): str, + "platform": { + Required("arch"): Any("32", "64", "aarch64", "arm7", "x86_64"), + Required("os"): { + Required("name"): Any("android", "linux", "macosx", "windows"), + Required("version"): str, + Optional("build"): str, + }, + Optional("device"): str, + Optional("display"): "wayland", + Optional("machine"): Any("ref-hw-2017"), + }, + "build": { + Required("type"): Any("opt", "debug", "debug-isolated-process"), + Any( + "asan", + "ccov", + "clang-trunk", + "devedition", + "domstreams", + "lite", + "mingwclang", + "nightlyasrelease", + "shippable", + "tsan", + ): bool, + }, + "runtime": {Any(*list(TEST_VARIANTS.keys()) + ["1proc"]): bool}, + }, + check=False, +) +"""Schema test settings must conform to. Validated by +:py:func:`~test.test_mozilla_central.test_test_setting`""" + + +@transforms.add +def set_test_setting(config, tasks): + """A test ``setting`` is the set of configuration that uniquely + distinguishes a test task from other tasks that run the same suite + (ignoring chunks). + + There are three different types of information that make up a setting: + + 1. Platform - Information describing the underlying platform tests run on, + e.g, OS, CPU architecture, etc. + + 2. Build - Information describing the build being tested, e.g build type, + ccov, asan/tsan, etc. + + 3. Runtime - Information describing which runtime parameters are enabled, + e.g, prefs, environment variables, etc. + + This transform adds a ``test-setting`` object to the ``extra`` portion of + all test tasks, of the form: + + .. code-block:: + + { + "platform": { ... }, + "build": { ... }, + "runtime": { ... } + } + + This information could be derived from the label, but consuming this + object is less brittle. + """ + # Some attributes have a dash in them which complicates parsing. Ensure we + # don't split them up. + # TODO Rename these so they don't have a dash. + dash_attrs = [ + "clang-trunk", + "ref-hw-2017", + ] + dash_token = "%D%" + platform_re = re.compile(r"(\D+)(\d*)") + + for task in tasks: + setting = { + "platform": { + "os": {}, + }, + "build": {}, + "runtime": {}, + } + + # parse platform and build information out of 'test-platform' + platform, build_type = task["test-platform"].split("/", 1) + + # ensure dashed attributes don't get split up + for attr in dash_attrs: + if attr in platform: + platform = platform.replace(attr, attr.replace("-", dash_token)) + + parts = platform.split("-") + + # restore dashes now that split is finished + for i, part in enumerate(parts): + if dash_token in part: + parts[i] = part.replace(dash_token, "-") + + match = platform_re.match(parts.pop(0)) + assert match + os_name, os_version = match.groups() + + device = machine = os_build = display = None + if os_name == "android": + device = parts.pop(0) + if device == "hw": + device = parts.pop(0) + else: + device = "emulator" + + os_version = parts.pop(0) + if parts[0].isdigit(): + os_version = f"{os_version}.{parts.pop(0)}" + + if parts[0] == "android": + parts.pop(0) + + arch = parts.pop(0) + + else: + arch = parts.pop(0) + if parts[0].isdigit(): + os_build = parts.pop(0) + + if parts[0] == "ref-hw-2017": + machine = parts.pop(0) + + if parts[0] == "wayland": + display = parts.pop(0) + + # It's not always possible to glean the exact architecture used from + # the task, so sometimes this will just be set to "32" or "64". + setting["platform"]["arch"] = arch + setting["platform"]["os"] = { + "name": os_name, + "version": os_version, + } + + if os_build: + setting["platform"]["os"]["build"] = os_build + + if device: + setting["platform"]["device"] = device + + if machine: + setting["platform"]["machine"] = machine + + if display: + setting["platform"]["display"] = display + + # parse remaining parts as build attributes + setting["build"]["type"] = build_type + while parts: + attr = parts.pop(0) + if attr == "qr": + # all tasks are webrender now, no need to store it + continue + + setting["build"][attr] = True + + unittest_variant = task["attributes"].get("unittest_variant") + if unittest_variant: + for variant in unittest_variant.split("+"): + setting["runtime"][variant] = True + + # add a hash of the setting object for easy comparisons + setting["_hash"] = hashlib.sha256( + json.dumps(setting, sort_keys=True).encode("utf-8") + ).hexdigest()[:12] + + task["test-setting"] = ReadOnlyDict(**setting) + yield task + + +@transforms.add +def allow_software_gl_layers(config, tasks): + """ + Handle the "allow-software-gl-layers" property for platforms where it + applies. + """ + for task in tasks: + if task.get("allow-software-gl-layers"): + # This should be set always once bug 1296086 is resolved. + task["mozharness"].setdefault("extra-options", []).append( + "--allow-software-gl-layers" + ) + + yield task + + +@transforms.add +def enable_webrender(config, tasks): + """ + Handle the "webrender" property by passing a flag to mozharness if it is + enabled. + """ + for task in tasks: + # TODO: this was all conditionally in enable_webrender- do we still need this? + extra_options = task["mozharness"].setdefault("extra-options", []) + # We only want to 'setpref' on tests that have a profile + if not task["attributes"]["unittest_category"] in [ + "cppunittest", + "geckoview-junit", + "gtest", + "jittest", + "raptor", + ]: + extra_options.append("--setpref=layers.d3d11.enable-blacklist=false") + + yield task + + +@transforms.add +def set_schedules_for_webrender_android(config, tasks): + """android-hw has limited resources, we need webrender on phones""" + for task in tasks: + if task["suite"] in ["crashtest", "reftest"] and task[ + "test-platform" + ].startswith("android-hw"): + task["schedules-component"] = "android-hw-gfx" + yield task + + +@transforms.add +def set_retry_exit_status(config, tasks): + """Set the retry exit status to TBPL_RETRY, the value returned by mozharness + scripts to indicate a transient failure that should be retried.""" + for task in tasks: + # add in 137 as it is an error with GCP workers + task["retry-exit-status"] = [4, 137] + yield task + + +@transforms.add +def set_profile(config, tasks): + """Set profiling mode for tests.""" + ttconfig = config.params["try_task_config"] + profile = ttconfig.get("gecko-profile", False) + settings = ( + "gecko-profile-interval", + "gecko-profile-entries", + "gecko-profile-threads", + "gecko-profile-features", + ) + + for task in tasks: + if profile and task["suite"] in ["talos", "raptor"]: + extras = task["mozharness"]["extra-options"] + extras.append("--gecko-profile") + for setting in settings: + value = ttconfig.get(setting) + if value is not None: + # These values can contain spaces (eg the "DOM Worker" + # thread) and the command is constructed in different, + # incompatible ways on different platforms. + + if task["test-platform"].startswith("win"): + # Double quotes for Windows (single won't work). + extras.append("--" + setting + '="' + str(value) + '"') + else: + # Other platforms keep things as separate values, + # rather than joining with spaces. + extras.append("--" + setting + "=" + str(value)) + + yield task + + +@transforms.add +def set_tag(config, tasks): + """Set test for a specific tag.""" + tag = None + if config.params["try_mode"] == "try_option_syntax": + tag = config.params["try_options"]["tag"] + for task in tasks: + if tag: + task["mozharness"]["extra-options"].extend(["--tag", tag]) + yield task + + +@transforms.add +def set_test_type(config, tasks): + types = ["mochitest", "reftest", "talos", "raptor", "geckoview-junit", "gtest"] + for task in tasks: + for test_type in types: + if test_type in task["suite"] and "web-platform" not in task["suite"]: + task.setdefault("tags", {})["test-type"] = test_type + yield task + + +@transforms.add +def set_schedules_components(config, tasks): + for task in tasks: + if "optimization" in task or "when" in task: + yield task + continue + + category = task["attributes"]["unittest_category"] + schedules = task.get("schedules-component", category) + if isinstance(schedules, str): + schedules = [schedules] + + schedules = set(schedules) + if schedules & set(INCLUSIVE_COMPONENTS): + # if this is an "inclusive" test, then all files which might + # cause it to run are annotated with SCHEDULES in moz.build, + # so do not include the platform or any other components here + task["schedules-component"] = sorted(schedules) + yield task + continue + + schedules.add(category) + schedules.add(platform_family(task["build-platform"])) + + task["schedules-component"] = sorted(schedules) + yield task + + +@transforms.add +def enable_parallel_marking_in_tsan_tests(config, tasks): + """Enable parallel marking in TSAN tests""" + skip_list = ["cppunittest", "gtest"] + for task in tasks: + if "-tsan-" in task["test-platform"]: + if task["suite"] not in skip_list: + extra_options = task["mozharness"].setdefault("extra-options", []) + extra_options.append( + "--setpref=javascript.options.mem.gc_parallel_marking=true" + ) + + yield task + + +@transforms.add +def apply_windows7_optimization(config, tasks): + for task in tasks: + if task["test-platform"].startswith("windows7"): + task["optimization"] = {"skip-unless-backstop": None} + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/test/raptor.py b/taskcluster/gecko_taskgraph/transforms/test/raptor.py new file mode 100644 index 0000000000..3eac5dd9ef --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/test/raptor.py @@ -0,0 +1,317 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from taskgraph.util.treeherder import join_symbol, split_symbol +from voluptuous import Extra, Optional, Required + +from gecko_taskgraph.transforms.test import test_description_schema +from gecko_taskgraph.util.copy_task import copy_task + +transforms = TransformSequence() +task_transforms = TransformSequence() + +raptor_description_schema = Schema( + { + # Raptor specific configs. + Optional("raptor"): { + Optional("activity"): optionally_keyed_by("app", str), + Optional("apps"): optionally_keyed_by("test-platform", "subtest", [str]), + Optional("binary-path"): optionally_keyed_by("app", str), + Optional("run-visual-metrics"): optionally_keyed_by("app", bool), + Optional("subtests"): optionally_keyed_by("app", "test-platform", list), + Optional("test"): str, + Optional("test-url-param"): optionally_keyed_by( + "subtest", "test-platform", str + ), + }, + # Configs defined in the 'test_description_schema'. + Optional("max-run-time"): optionally_keyed_by( + "app", "subtest", "test-platform", test_description_schema["max-run-time"] + ), + Optional("run-on-projects"): optionally_keyed_by( + "app", + "test-name", + "raptor.test", + "subtest", + "variant", + test_description_schema["run-on-projects"], + ), + Optional("variants"): test_description_schema["variants"], + Optional("target"): optionally_keyed_by( + "app", test_description_schema["target"] + ), + Optional("tier"): optionally_keyed_by( + "app", "raptor.test", "subtest", "variant", test_description_schema["tier"] + ), + Required("test-name"): test_description_schema["test-name"], + Required("test-platform"): test_description_schema["test-platform"], + Required("require-signed-extensions"): test_description_schema[ + "require-signed-extensions" + ], + Required("treeherder-symbol"): test_description_schema["treeherder-symbol"], + # Any unrecognized keys will be validated against the test_description_schema. + Extra: object, + } +) + +transforms.add_validate(raptor_description_schema) + + +@transforms.add +def set_defaults(config, tests): + for test in tests: + test.setdefault("raptor", {}).setdefault("run-visual-metrics", False) + yield test + + +@transforms.add +def split_apps(config, tests): + app_symbols = { + "chrome": "ChR", + "chrome-m": "ChR", + "chromium": "Cr", + "fenix": "fenix", + "refbrow": "refbrow", + "safari": "Saf", + "custom-car": "CaR", + } + + for test in tests: + apps = test["raptor"].pop("apps", None) + if not apps: + yield test + continue + + for app in apps: + # Ignore variants for non-Firefox or non-mobile applications. + if app not in ["firefox", "geckoview", "fenix", "chrome-m"] and test[ + "attributes" + ].get("unittest_variant"): + continue + + atest = copy_task(test) + suffix = f"-{app}" + atest["app"] = app + atest["description"] += f" on {app.capitalize()}" + + name = atest["test-name"] + suffix + atest["test-name"] = name + atest["try-name"] = name + + if app in app_symbols: + group, symbol = split_symbol(atest["treeherder-symbol"]) + group += f"-{app_symbols[app]}" + atest["treeherder-symbol"] = join_symbol(group, symbol) + + yield atest + + +@transforms.add +def handle_keyed_by_prereqs(config, tests): + """ + Only resolve keys for prerequisite fields here since the + these keyed-by options might have keyed-by fields + as well. + """ + for test in tests: + resolve_keyed_by(test, "raptor.subtests", item_name=test["test-name"]) + yield test + + +@transforms.add +def split_raptor_subtests(config, tests): + for test in tests: + # For tests that have 'subtests' listed, we want to create a separate + # test job for every subtest (i.e. split out each page-load URL into its own job) + subtests = test["raptor"].pop("subtests", None) + if not subtests: + yield test + continue + + for chunk_number, subtest in enumerate(subtests): + + # Create new test job + chunked = copy_task(test) + chunked["chunk-number"] = 1 + chunk_number + chunked["subtest"] = subtest + chunked["subtest-symbol"] = subtest + if isinstance(chunked["subtest"], list): + chunked["subtest"] = subtest[0] + chunked["subtest-symbol"] = subtest[1] + chunked = resolve_keyed_by( + chunked, "tier", chunked["subtest"], defer=["variant"] + ) + yield chunked + + +@transforms.add +def handle_keyed_by(config, tests): + fields = [ + "raptor.test-url-param", + "raptor.run-visual-metrics", + "raptor.activity", + "raptor.binary-path", + "limit-platforms", + "fetches.fetch", + "max-run-time", + "run-on-projects", + "target", + "tier", + ] + for test in tests: + for field in fields: + resolve_keyed_by( + test, field, item_name=test["test-name"], defer=["variant"] + ) + yield test + + +@transforms.add +def split_page_load_by_url(config, tests): + for test in tests: + # `chunk-number` and 'subtest' only exists when the task had a + # definition for `subtests` + chunk_number = test.pop("chunk-number", None) + subtest = test.get( + "subtest" + ) # don't pop as some tasks need this value after splitting variants + subtest_symbol = test.pop("subtest-symbol", None) + + if not chunk_number or not subtest: + yield test + continue + + if len(subtest_symbol) > 10 and "ytp" not in subtest_symbol: + raise Exception( + "Treeherder symbol %s is larger than 10 char! Please use a different symbol." + % subtest_symbol + ) + + if test["test-name"].startswith("browsertime-"): + test["raptor"]["test"] = subtest + + # Remove youtube-playback in the test name to avoid duplication + test["test-name"] = test["test-name"].replace("youtube-playback-", "") + else: + # Use full test name if running on webextension + test["raptor"]["test"] = "raptor-tp6-" + subtest + "-{}".format(test["app"]) + + # Only run the subtest/single URL + test["test-name"] += f"-{subtest}" + test["try-name"] += f"-{subtest}" + + # Set treeherder symbol and description + group, _ = split_symbol(test["treeherder-symbol"]) + test["treeherder-symbol"] = join_symbol(group, subtest_symbol) + test["description"] += f" on {subtest}" + + yield test + + +@transforms.add +def modify_extra_options(config, tests): + for test in tests: + test_name = test.get("test-name", None) + + if "first-install" in test_name: + # First-install tests should never use conditioned profiles + extra_options = test.setdefault("mozharness", {}).setdefault( + "extra-options", [] + ) + + for i, opt in enumerate(extra_options): + if "conditioned-profile" in opt: + if i: + extra_options.pop(i) + break + + if "-widevine" in test_name: + extra_options = test.setdefault("mozharness", {}).setdefault( + "extra-options", [] + ) + for i, opt in enumerate(extra_options): + if "--conditioned-profile=settled" in opt: + if i: + extra_options[i] += "-youtube" + break + + if "unity-webgl" in test_name: + # Disable the extra-profiler-run for unity-webgl tests. + extra_options = test.setdefault("mozharness", {}).setdefault( + "extra-options", [] + ) + for i, opt in enumerate(extra_options): + if "extra-profiler-run" in opt: + if i: + extra_options.pop(i) + break + + yield test + + +@transforms.add +def add_extra_options(config, tests): + for test in tests: + mozharness = test.setdefault("mozharness", {}) + if test.get("app", "") == "chrome-m": + mozharness["tooltool-downloads"] = "internal" + + extra_options = mozharness.setdefault("extra-options", []) + + # Adding device name if we're on android + test_platform = test["test-platform"] + if test_platform.startswith("android-hw-g5"): + extra_options.append("--device-name=g5") + elif test_platform.startswith("android-hw-a51"): + extra_options.append("--device-name=a51") + elif test_platform.startswith("android-hw-p5"): + extra_options.append("--device-name=p5_aarch64") + + if test["raptor"].pop("run-visual-metrics", False): + extra_options.append("--browsertime-video") + extra_options.append("--browsertime-visualmetrics") + test["attributes"]["run-visual-metrics"] = True + + if "app" in test: + extra_options.append( + "--app={}".format(test["app"]) + ) # don't pop as some tasks need this value after splitting variants + + if "activity" in test["raptor"]: + extra_options.append("--activity={}".format(test["raptor"].pop("activity"))) + + if "binary-path" in test["raptor"]: + extra_options.append( + "--binary-path={}".format(test["raptor"].pop("binary-path")) + ) + + if "test" in test["raptor"]: + extra_options.append("--test={}".format(test["raptor"].pop("test"))) + + if test["require-signed-extensions"]: + extra_options.append("--is-release-build") + + if "test-url-param" in test["raptor"]: + param = test["raptor"].pop("test-url-param") + if not param == []: + extra_options.append( + "--test-url-params={}".format(param.replace(" ", "")) + ) + + extra_options.append("--project={}".format(config.params.get("project"))) + + yield test + + +@task_transforms.add +def add_scopes_and_proxy(config, tasks): + for task in tasks: + task.setdefault("worker", {})["taskcluster-proxy"] = True + task.setdefault("scopes", []).append( + "secrets:get:project/perftest/gecko/level-{level}/perftest-login" + ) + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/test/variant.py b/taskcluster/gecko_taskgraph/transforms/test/variant.py new file mode 100644 index 0000000000..e2fd9764e1 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/test/variant.py @@ -0,0 +1,128 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import datetime + +import jsone +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema, validate_schema +from taskgraph.util.treeherder import join_symbol, split_symbol +from taskgraph.util.yaml import load_yaml +from voluptuous import Any, Optional, Required + +import gecko_taskgraph +from gecko_taskgraph.util.copy_task import copy_task +from gecko_taskgraph.util.templates import merge + +transforms = TransformSequence() + +TEST_VARIANTS = load_yaml( + gecko_taskgraph.GECKO, "taskcluster", "ci", "test", "variants.yml" +) +"""List of available test variants defined.""" + + +variant_description_schema = Schema( + { + str: { + Required("description"): str, + Required("suffix"): str, + Required("component"): str, + Required("expiration"): str, + Optional("when"): {Any("$eval", "$if"): str}, + Optional("replace"): {str: object}, + Optional("merge"): {str: object}, + } + } +) +"""variant description schema""" + + +@transforms.add +def split_variants(config, tasks): + """Splits test definitions into multiple tasks based on the `variants` key. + + If `variants` are defined, the original task will be yielded along with a + copy of the original task for each variant defined in the list. The copies + will have the 'unittest_variant' attribute set. + """ + validate_schema(variant_description_schema, TEST_VARIANTS, "In variants.yml:") + + def find_expired_variants(variants): + expired = [] + + # do not expire on esr/beta/release + if config.params.get("release_type", "") in [ + "release", + "beta", + ]: + return [] + + if "esr" in config.params.get("release_type", ""): + return [] + + today = datetime.datetime.today() + for variant in variants: + + expiration = variants[variant]["expiration"] + if len(expiration.split("-")) == 1: + continue + expires_at = datetime.datetime.strptime(expiration, "%Y-%m-%d") + if expires_at < today: + expired.append(variant) + return expired + + def remove_expired(variants, expired): + remaining_variants = [] + for name in variants: + parts = [p for p in name.split("+") if p not in expired] + if len(parts) == 0: + continue + + remaining_variants.append(name) + return remaining_variants + + def apply_variant(variant, task): + task["description"] = variant["description"].format(**task) + + suffix = f"-{variant['suffix']}" + group, symbol = split_symbol(task["treeherder-symbol"]) + if group != "?": + group += suffix + else: + symbol += suffix + task["treeherder-symbol"] = join_symbol(group, symbol) + + # This will be used to set the label and try-name in 'make_job_description'. + task.setdefault("variant-suffix", "") + task["variant-suffix"] += suffix + + # Replace and/or merge the configuration. + task.update(variant.get("replace", {})) + return merge(task, variant.get("merge", {})) + + expired_variants = find_expired_variants(TEST_VARIANTS) + for task in tasks: + variants = task.pop("variants", []) + variants = remove_expired(variants, expired_variants) + + if task.pop("run-without-variant"): + yield copy_task(task) + + for name in variants: + # Apply composite variants (joined by '+') in order. + parts = name.split("+") + taskv = copy_task(task) + for part in parts: + variant = TEST_VARIANTS[part] + + # If any variant in a composite fails this check we skip it. + if "when" in variant: + context = {"task": task} + if not jsone.render(variant["when"], context): + break + + taskv = apply_variant(variant, taskv) + else: + taskv["attributes"]["unittest_variant"] = name + yield taskv diff --git a/taskcluster/gecko_taskgraph/transforms/test/worker.py b/taskcluster/gecko_taskgraph/transforms/test/worker.py new file mode 100644 index 0000000000..0d8d72162d --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/test/worker.py @@ -0,0 +1,201 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +# default worker types keyed by instance-size +LINUX_WORKER_TYPES = { + "large": "t-linux-large", + "xlarge": "t-linux-xlarge", + "default": "t-linux-large", +} + +# windows worker types keyed by test-platform and virtualization +WINDOWS_WORKER_TYPES = { + "windows7-32-qr": { + "virtual": "t-win7-32", + "virtual-with-gpu": "t-win7-32-gpu", + "hardware": "t-win10-64-1803-hw", + }, + "windows7-32-shippable-qr": { + "virtual": "t-win7-32", + "virtual-with-gpu": "t-win7-32-gpu", + "hardware": "t-win10-64-1803-hw", + }, + "windows7-32-devedition-qr": { # build only, tests have no value + "virtual": "t-win7-32", + "virtual-with-gpu": "t-win7-32-gpu", + "hardware": "t-win10-64-1803-hw", + }, + "windows10-64": { # source-test + "virtual": "t-win10-64", + "virtual-with-gpu": "t-win10-64-gpu-s", + "hardware": "t-win10-64-1803-hw", + }, + "windows10-64-shippable-qr": { + "virtual": "t-win10-64", + "virtual-with-gpu": "t-win10-64-gpu-s", + "hardware": "t-win10-64-1803-hw", + }, + "windows10-64-ref-hw-2017": { + "virtual": "t-win10-64", + "virtual-with-gpu": "t-win10-64-gpu-s", + "hardware": "t-win10-64-ref-hw", + }, + "windows10-64-2009-qr": { + "virtual": "win10-64-2009", + "virtual-with-gpu": "win10-64-2009-gpu", + }, + "windows10-64-2009-shippable-qr": { + "virtual": "win10-64-2009", + "virtual-with-gpu": "win10-64-2009-gpu", + }, + "windows11-32-2009-mingwclang-qr": { + "virtual": "win11-64-2009", + "virtual-with-gpu": "win11-64-2009-gpu", + }, + "windows11-32-2009-qr": { + "virtual": "win11-64-2009", + "virtual-with-gpu": "win11-64-2009-gpu", + }, + "windows11-32-2009-shippable-qr": { + "virtual": "win11-64-2009", + "virtual-with-gpu": "win11-64-2009-gpu", + }, + "windows11-64-2009": { + "virtual": "win11-64-2009", + "virtual-with-gpu": "win11-64-2009-gpu", + }, + "windows11-64-2009-ccov": { + "virtual": "win11-64-2009-ssd", + "virtual-with-gpu": "win11-64-2009-ssd-gpu", + }, + "windows11-64-2009-ccov-qr": { + "virtual": "win11-64-2009-ssd", + "virtual-with-gpu": "win11-64-2009-ssd-gpu", + }, + "windows11-64-2009-devedition": { + "virtual": "win11-64-2009", + "virtual-with-gpu": "win11-64-2009-gpu", + }, + "windows11-64-2009-shippable": { + "virtual": "win11-64-2009", + "virtual-with-gpu": "win11-64-2009-gpu", + }, + "windows11-64-2009-qr": { + "virtual": "win11-64-2009", + "virtual-with-gpu": "win11-64-2009-gpu", + }, + "windows11-64-2009-shippable-qr": { + "virtual": "win11-64-2009", + "virtual-with-gpu": "win11-64-2009-gpu", + }, + "windows11-64-2009-devedition-qr": { + "virtual": "win11-64-2009", + "virtual-with-gpu": "win11-64-2009-gpu", + }, + "windows11-64-2009-asan-qr": { + "virtual": "win11-64-2009", + "virtual-with-gpu": "win11-64-2009-gpu", + }, + "windows11-64-2009-mingwclang-qr": { + "virtual": "win11-64-2009", + "virtual-with-gpu": "win11-64-2009-gpu", + }, +} + +# os x worker types keyed by test-platform +MACOSX_WORKER_TYPES = { + "macosx1015-64-power": "t-osx-1015-power", + "macosx1015-64": "t-osx-1015-r8", + "macosx1100-64": "t-osx-1100-m1", +} + +transforms = TransformSequence() + + +@transforms.add +def set_worker_type(config, tasks): + """Set the worker type based on the test platform.""" + for task in tasks: + # during the taskcluster migration, this is a bit tortured, but it + # will get simpler eventually! + test_platform = task["test-platform"] + if task.get("worker-type", "default") != "default": + # This test already has its worker type defined, so just use that (yields below) + # Unless the value is set to "default", in that case ignore it. + pass + elif test_platform.startswith("macosx1015-64"): + if "--power-test" in task["mozharness"]["extra-options"]: + task["worker-type"] = MACOSX_WORKER_TYPES["macosx1015-64-power"] + else: + task["worker-type"] = MACOSX_WORKER_TYPES["macosx1015-64"] + elif test_platform.startswith("macosx1100-64"): + task["worker-type"] = MACOSX_WORKER_TYPES["macosx1100-64"] + elif test_platform.startswith("win"): + # figure out what platform the job needs to run on + if task["virtualization"] == "hardware": + # some jobs like talos and reftest run on real h/w - those are all win10 + if test_platform.startswith("windows10-64-ref-hw-2017"): + win_worker_type_platform = WINDOWS_WORKER_TYPES[ + "windows10-64-ref-hw-2017" + ] + else: + win_worker_type_platform = WINDOWS_WORKER_TYPES["windows10-64"] + else: + # the other jobs run on a vm which may or may not be a win10 vm + win_worker_type_platform = WINDOWS_WORKER_TYPES[ + test_platform.split("/")[0] + ] + if task[ + "virtualization" + ] == "virtual-with-gpu" and test_platform.startswith("windows1"): + # add in `--requires-gpu` to the mozharness options + task["mozharness"]["extra-options"].append("--requires-gpu") + + # now we have the right platform set the worker type accordingly + task["worker-type"] = win_worker_type_platform[task["virtualization"]] + elif test_platform.startswith("android-hw-g5"): + if task["suite"] != "raptor": + task["worker-type"] = "t-bitbar-gw-unit-g5" + else: + task["worker-type"] = "t-bitbar-gw-perf-g5" + elif test_platform.startswith("android-hw-p5"): + if task["suite"] != "raptor": + task["worker-type"] = "t-bitbar-gw-unit-p5" + else: + task["worker-type"] = "t-bitbar-gw-perf-p5" + elif test_platform.startswith("android-hw-a51"): + if task["suite"] != "raptor": + task["worker-type"] = "t-bitbar-gw-unit-a51" + else: + task["worker-type"] = "t-bitbar-gw-perf-a51" + elif test_platform.startswith("android-em-7.0-x86"): + task["worker-type"] = "t-linux-kvm" + elif test_platform.startswith("linux") or test_platform.startswith("android"): + if "wayland" in test_platform: + task["worker-type"] = "t-linux-wayland" + elif task.get("suite", "") in ["talos", "raptor"] and not task[ + "build-platform" + ].startswith("linux64-ccov"): + task["worker-type"] = "t-linux-talos-1804" + else: + task["worker-type"] = LINUX_WORKER_TYPES[task["instance-size"]] + else: + raise Exception(f"unknown test_platform {test_platform}") + + yield task + + +@transforms.add +def set_wayland_env(config, tasks): + for task in tasks: + if task["worker-type"] != "t-linux-wayland": + yield task + continue + + env = task.setdefault("worker", {}).setdefault("env", {}) + env["MOZ_ENABLE_WAYLAND"] = "1" + env["WAYLAND_DISPLAY"] = "wayland-0" + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/try_job.py b/taskcluster/gecko_taskgraph/transforms/try_job.py new file mode 100644 index 0000000000..4b3281f5c5 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/try_job.py @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def set_job_try_name(config, jobs): + """ + For a task which is governed by `-j` in try syntax, set the `job_try_name` + attribute based on the job name. + """ + for job in jobs: + job.setdefault("attributes", {}).setdefault("job_try_name", job["name"]) + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/update_verify.py b/taskcluster/gecko_taskgraph/transforms/update_verify.py new file mode 100644 index 0000000000..19c932c746 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/update_verify.py @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + + +from copy import deepcopy + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import add_suffix, inherit_treeherder_from_dep + +from gecko_taskgraph.util.attributes import task_name + +transforms = TransformSequence() + + +@transforms.add +def add_command(config, tasks): + config_tasks = {} + for dep in config.kind_dependencies_tasks.values(): + if ( + "update-verify-config" in dep.kind + or "update-verify-next-config" in dep.kind + ): + config_tasks[task_name(dep)] = dep + + for task in tasks: + config_task = config_tasks[task["name"]] + total_chunks = task["extra"]["chunks"] + task["worker"].setdefault("env", {})["CHANNEL"] = config_task.task["extra"][ + "channel" + ] + task.setdefault("fetches", {})[config_task.label] = [ + "update-verify.cfg", + ] + task["treeherder"] = inherit_treeherder_from_dep(task, config_task) + + for this_chunk in range(1, total_chunks + 1): + chunked = deepcopy(task) + chunked["treeherder"]["symbol"] = add_suffix( + chunked["treeherder"]["symbol"], this_chunk + ) + chunked["label"] = "release-update-verify-{}-{}/{}".format( + chunked["name"], this_chunk, total_chunks + ) + if not chunked["worker"].get("env"): + chunked["worker"]["env"] = {} + chunked["run"] = { + "using": "run-task", + "cwd": "{checkout}", + "command": "tools/update-verify/scripts/chunked-verify.sh " + f"--total-chunks={total_chunks} --this-chunk={this_chunk}", + "sparse-profile": "update-verify", + } + + yield chunked diff --git a/taskcluster/gecko_taskgraph/transforms/update_verify_config.py b/taskcluster/gecko_taskgraph/transforms/update_verify_config.py new file mode 100644 index 0000000000..1de808f82b --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/update_verify_config.py @@ -0,0 +1,148 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +from urllib.parse import urlsplit + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.transforms.task import get_branch_repo, get_branch_rev +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import get_release_config + +transforms = TransformSequence() + + +# The beta regexes do not match point releases. +# In the rare event that we do ship a point +# release to beta, we need to either: +# 1) update these regexes to match that specific version +# 2) pass a second include version that matches that specific version +INCLUDE_VERSION_REGEXES = { + "beta": r"'^(\d+\.\d+(b\d+)?)$'", + "nonbeta": r"'^\d+\.\d+(\.\d+)?$'", + # Same as nonbeta, except for the esr suffix + "esr": r"'^\d+\.\d+(\.\d+)?esr$'", + # Previous esr versions, for update testing before we update users to esr102 + "esr115-next": r"'^(52|60|68|78|91|102)+\.\d+(\.\d+)?esr$'", +} + +MAR_CHANNEL_ID_OVERRIDE_REGEXES = { + "beta": r"'^\d+\.\d+(\.\d+)?$$,firefox-mozilla-beta,firefox-mozilla-release'", +} + + +def ensure_wrapped_singlequote(regexes): + """Ensure that a regex (from INCLUDE_VERSION_REGEXES or MAR_CHANNEL_ID_OVERRIDE_REGEXES) + is wrapper in single quotes. + """ + for name, regex in regexes.items(): + if regex[0] != "'" or regex[-1] != "'": + raise Exception( + "Regex {} is invalid: not wrapped with single quotes.\n{}".format( + name, regex + ) + ) + + +ensure_wrapped_singlequote(INCLUDE_VERSION_REGEXES) +ensure_wrapped_singlequote(MAR_CHANNEL_ID_OVERRIDE_REGEXES) + + +@transforms.add +def add_command(config, tasks): + keyed_by_args = [ + "channel", + "archive-prefix", + "previous-archive-prefix", + "aus-server", + "override-certs", + "include-version", + "mar-channel-id-override", + "last-watershed", + ] + optional_args = [ + "updater-platform", + ] + + release_config = get_release_config(config) + + for task in tasks: + task["description"] = "generate update verify config for {}".format( + task["attributes"]["build_platform"] + ) + + command = [ + "python", + "testing/mozharness/scripts/release/update-verify-config-creator.py", + "--product", + task["extra"]["product"], + "--stage-product", + task["shipping-product"], + "--app-name", + task["extra"]["app-name"], + "--branch-prefix", + task["extra"]["branch-prefix"], + "--platform", + task["extra"]["platform"], + "--to-version", + release_config["version"], + "--to-app-version", + release_config["appVersion"], + "--to-build-number", + str(release_config["build_number"]), + "--to-buildid", + config.params["moz_build_date"], + "--to-revision", + get_branch_rev(config), + "--output-file", + "update-verify.cfg", + ] + + repo_path = urlsplit(get_branch_repo(config)).path.lstrip("/") + command.extend(["--repo-path", repo_path]) + + if release_config.get("partial_versions"): + for partial in release_config["partial_versions"].split(","): + command.extend(["--partial-version", partial.split("build")[0]]) + + for arg in optional_args: + if task["extra"].get(arg): + command.append(f"--{arg}") + command.append(task["extra"][arg]) + + for arg in keyed_by_args: + thing = f"extra.{arg}" + resolve_keyed_by( + task, + thing, + item_name=task["name"], + platform=task["attributes"]["build_platform"], + **{ + "release-type": config.params["release_type"], + "release-level": release_level(config.params["project"]), + }, + ) + # ignore things that resolved to null + if not task["extra"].get(arg): + continue + if arg == "include-version": + task["extra"][arg] = INCLUDE_VERSION_REGEXES[task["extra"][arg]] + if arg == "mar-channel-id-override": + task["extra"][arg] = MAR_CHANNEL_ID_OVERRIDE_REGEXES[task["extra"][arg]] + + command.append(f"--{arg}") + command.append(task["extra"][arg]) + + task["run"].update( + { + "using": "mach", + "mach": " ".join(command), + } + ) + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py b/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py new file mode 100644 index 0000000000..b862645eed --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the upload-generated-files task description template, +taskcluster/ci/upload-generated-sources/kind.yml, into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def add_task_info(config, jobs): + for job in jobs: + dep_task = job["primary-dependency"] + del job["primary-dependency"] + + # Add a dependency on the build task. + job["dependencies"] = {"build": dep_task.label} + # Label the job to match the build task it's uploading from. + job["label"] = dep_task.label.replace("build-", "upload-generated-sources-") + # Copy over some bits of metdata from the build task. + dep_th = dep_task.task["extra"]["treeherder"] + job.setdefault("attributes", {}) + job["attributes"]["build_platform"] = dep_task.attributes.get("build_platform") + if dep_task.attributes.get("shippable"): + job["attributes"]["shippable"] = True + plat = "{}/{}".format( + dep_th["machine"]["platform"], dep_task.attributes.get("build_type") + ) + job["treeherder"]["platform"] = plat + job["treeherder"]["tier"] = dep_th["tier"] + if dep_th["symbol"] != "N": + job["treeherder"]["symbol"] = "Ugs{}".format(dep_th["symbol"]) + job["run-on-projects"] = dep_task.attributes.get("run_on_projects") + job["optimization"] = dep_task.optimization + + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/upload_symbols.py b/taskcluster/gecko_taskgraph/transforms/upload_symbols.py new file mode 100644 index 0000000000..f6d40e9a45 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/upload_symbols.py @@ -0,0 +1,95 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the upload-symbols task description template, +taskcluster/ci/upload-symbols/job-template.yml into an actual task description. +""" + + +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol + +from gecko_taskgraph.util.attributes import ( + RELEASE_PROJECTS, + copy_attributes_from_dependent_job, +) + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def check_nightlies(config, tasks): + """Ensure that we upload symbols for all shippable builds, so that crash-stats can + resolve any reports sent to it. Try may enable full symbols but not upload them. + + Putting this check here (instead of the transforms for the build kind) lets us + leverage the any not-for-build-platforms set in the update-symbols kind.""" + for task in tasks: + dep = task["primary-dependency"] + if ( + config.params["project"] in RELEASE_PROJECTS + and dep.attributes.get("shippable") + and not dep.attributes.get("enable-full-crashsymbols") + and not dep.attributes.get("skip-upload-crashsymbols") + ): + raise Exception( + "Shippable job %s should have enable-full-crashsymbols attribute " + "set to true to enable symbol upload to crash-stats" % dep.label + ) + yield task + + +@transforms.add +def fill_template(config, tasks): + for task in tasks: + dep = task["primary-dependency"] + task.pop("dependent-tasks", None) + + # Fill out the dynamic fields in the task description + task["label"] = dep.label + "-upload-symbols" + + # Skip tasks where we don't have the full crashsymbols enabled + if not dep.attributes.get("enable-full-crashsymbols") or dep.attributes.get( + "skip-upload-crashsymbols" + ): + logger.debug("Skipping upload symbols task for %s", task["label"]) + continue + + task["dependencies"] = {"build": dep.label} + task["worker"]["env"]["GECKO_HEAD_REPOSITORY"] = config.params[ + "head_repository" + ] + task["worker"]["env"]["GECKO_HEAD_REV"] = config.params["head_rev"] + task["worker"]["env"]["SYMBOL_SECRET"] = task["worker"]["env"][ + "SYMBOL_SECRET" + ].format(level=config.params["level"]) + + attributes = copy_attributes_from_dependent_job(dep) + attributes.update(task.get("attributes", {})) + task["attributes"] = attributes + + treeherder = inherit_treeherder_from_dep(task, dep) + th = dep.task.get("extra")["treeherder"] + th_symbol = th.get("symbol") + th_groupsymbol = th.get("groupSymbol", "?") + + # Disambiguate the treeherder symbol. + sym = "Sym" + (th_symbol[1:] if th_symbol.startswith("B") else th_symbol) + treeherder.setdefault("symbol", join_symbol(th_groupsymbol, sym)) + task["treeherder"] = treeherder + + # We only want to run these tasks if the build is run. + # XXX Better to run this on promote phase instead? + task["run-on-projects"] = dep.attributes.get("run_on_projects") + task["optimization"] = {"upload-symbols": None} + task["if-dependencies"] = ["build"] + + # clear out the stuff that's not part of a task description + del task["primary-dependency"] + + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/upstream_artifact_task.py b/taskcluster/gecko_taskgraph/transforms/upstream_artifact_task.py new file mode 100644 index 0000000000..62f94a8238 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/upstream_artifact_task.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Find upstream artifact task. +""" + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def find_upstream_artifact_task(config, jobs): + for job in jobs: + dep_job = None + if job.get("dependent-tasks"): + dep_labels = [l for l in job["dependent-tasks"].keys()] + for label in dep_labels: + if label.endswith("-mac-signing"): + assert ( + dep_job is None + ), "Can't determine whether " "{} or {} is dep_job!".format( + dep_job.label, label + ) + dep_job = job["dependent-tasks"][label] + if dep_job is not None: + job["upstream-artifact-task"] = dep_job + yield job diff --git a/taskcluster/gecko_taskgraph/try_option_syntax.py b/taskcluster/gecko_taskgraph/try_option_syntax.py new file mode 100644 index 0000000000..3e054d63f9 --- /dev/null +++ b/taskcluster/gecko_taskgraph/try_option_syntax.py @@ -0,0 +1,755 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import argparse +import copy +import logging +import re +import shlex +from collections import defaultdict + +logger = logging.getLogger(__name__) + +# The build type aliases are very cryptic and only used in try flags these are +# mappings from the single char alias to a longer more recognizable form. +BUILD_TYPE_ALIASES = {"o": "opt", "d": "debug"} + +# consider anything in this whitelist of kinds to be governed by -b/-p +BUILD_KINDS = { + "build", + "artifact-build", + "hazard", + "l10n", + "valgrind", + "spidermonkey", +} + + +# mapping from shortcut name (usable with -u) to a boolean function identifying +# matching test names +def alias_prefix(prefix): + return lambda name: name.startswith(prefix) + + +def alias_contains(infix): + return lambda name: infix in name + + +def alias_matches(pattern): + pattern = re.compile(pattern) + return lambda name: pattern.match(name) + + +UNITTEST_ALIASES = { + # Aliases specify shorthands that can be used in try syntax. The shorthand + # is the dictionary key, with the value representing a pattern for matching + # unittest_try_names. + # + # Note that alias expansion is performed in the absence of any chunk + # prefixes. For example, the first example above would replace "foo-7" + # with "foobar-7". Note that a few aliases allowed chunks to be specified + # without a leading `-`, for example 'mochitest-dt1'. That's no longer + # supported. + "cppunit": alias_prefix("cppunit"), + "crashtest": alias_prefix("crashtest"), + "crashtest-e10s": alias_prefix("crashtest-e10s"), + "e10s": alias_contains("e10s"), + "firefox-ui-functional": alias_prefix("firefox-ui-functional"), + "gaia-js-integration": alias_contains("gaia-js-integration"), + "gtest": alias_prefix("gtest"), + "jittest": alias_prefix("jittest"), + "jittests": alias_prefix("jittest"), + "jsreftest": alias_prefix("jsreftest"), + "jsreftest-e10s": alias_prefix("jsreftest-e10s"), + "marionette": alias_prefix("marionette"), + "mochitest": alias_prefix("mochitest"), + "mochitests": alias_prefix("mochitest"), + "mochitest-e10s": alias_prefix("mochitest-e10s"), + "mochitests-e10s": alias_prefix("mochitest-e10s"), + "mochitest-debug": alias_prefix("mochitest-debug-"), + "mochitest-a11y": alias_contains("mochitest-a11y"), + "mochitest-bc": alias_prefix("mochitest-browser-chrome"), + "mochitest-e10s-bc": alias_prefix("mochitest-browser-chrome-e10s"), + "mochitest-browser-chrome": alias_prefix("mochitest-browser-chrome"), + "mochitest-e10s-browser-chrome": alias_prefix("mochitest-browser-chrome-e10s"), + "mochitest-chrome": alias_contains("mochitest-chrome"), + "mochitest-dt": alias_prefix("mochitest-devtools-chrome"), + "mochitest-e10s-dt": alias_prefix("mochitest-devtools-chrome-e10s"), + "mochitest-gl": alias_prefix("mochitest-webgl"), + "mochitest-gl-e10s": alias_prefix("mochitest-webgl-e10s"), + "mochitest-gpu": alias_prefix("mochitest-gpu"), + "mochitest-gpu-e10s": alias_prefix("mochitest-gpu-e10s"), + "mochitest-media": alias_prefix("mochitest-media"), + "mochitest-media-e10s": alias_prefix("mochitest-media-e10s"), + "mochitest-vg": alias_prefix("mochitest-valgrind"), + "reftest": alias_matches(r"^(plain-)?reftest.*$"), + "reftest-no-accel": alias_matches(r"^(plain-)?reftest-no-accel.*$"), + "reftests": alias_matches(r"^(plain-)?reftest.*$"), + "reftests-e10s": alias_matches(r"^(plain-)?reftest-e10s.*$"), + "robocop": alias_prefix("robocop"), + "web-platform-test": alias_prefix("web-platform-tests"), + "web-platform-tests": alias_prefix("web-platform-tests"), + "web-platform-tests-e10s": alias_prefix("web-platform-tests-e10s"), + "web-platform-tests-crashtests": alias_prefix("web-platform-tests-crashtest"), + "web-platform-tests-print-reftest": alias_prefix( + "web-platform-tests-print-reftest" + ), + "web-platform-tests-reftests": alias_prefix("web-platform-tests-reftest"), + "web-platform-tests-reftests-e10s": alias_prefix("web-platform-tests-reftest-e10s"), + "web-platform-tests-wdspec": alias_prefix("web-platform-tests-wdspec"), + "web-platform-tests-wdspec-e10s": alias_prefix("web-platform-tests-wdspec-e10s"), + "xpcshell": alias_prefix("xpcshell"), +} + +# unittest platforms can be specified by substring of the "pretty name", which +# is basically the old Buildbot builder name. This dict has {pretty name, +# [test_platforms]} translations, This includes only the most commonly-used +# substrings. It is OK to add new test platforms to various shorthands here; +# if you add a new Linux64 test platform for instance, people will expect that +# their previous methods of requesting "all linux64 tests" will include this +# new platform, and they shouldn't have to explicitly spell out the new platform +# every time for such cases. +# +# Note that the test platforms here are only the prefix up to the `/`. +UNITTEST_PLATFORM_PRETTY_NAMES = { + "Ubuntu": [ + "linux32", + "linux64", + "linux64-asan", + "linux1804-64", + "linux1804-64-asan", + ], + "x64": ["linux64", "linux64-asan", "linux1804-64", "linux1804-64-asan"], + "Android 7.0 Moto G5 32bit": ["android-hw-g5-7.0-arm7"], + "Android 7.0 Samsung A51 32bit": ["android-hw-a51-11.0-arm7"], + "Android 7.0 Samsung A51 64bit": ["android-hw-a51-11.0-aarch64"], + "Android 8.0 Google Pixel 2 32bit": ["android-hw-p2-8.0-arm7"], + "Android 8.0 Google Pixel 2 64bit": ["android-hw-p2-8.0-android-aarch64"], + "Android 13.0 Google Pixel 5 32bit": ["android-hw-p5-13.0-arm7"], + "Android 13.0 Google Pixel 5 64bit": ["android-hw-p5-13.0-android-aarch64"], + "Windows 7": ["windows7-32"], + "Windows 7 VM": ["windows7-32-vm"], + "Windows 10": ["windows10-64"], +} + +TEST_CHUNK_SUFFIX = re.compile("(.*)-([0-9]+)$") + + +def escape_whitespace_in_brackets(input_str): + """ + In tests you may restrict them by platform [] inside of the brackets + whitespace may occur this is typically invalid shell syntax so we escape it + with backslash sequences . + """ + result = "" + in_brackets = False + for char in input_str: + if char == "[": + in_brackets = True + result += char + continue + + if char == "]": + in_brackets = False + result += char + continue + + if char == " " and in_brackets: + result += r"\ " + continue + + result += char + + return result + + +def split_try_msg(message): + try: + try_idx = message.index("try:") + except ValueError: + return [] + message = message[try_idx:].split("\n")[0] + # shlex used to ensure we split correctly when giving values to argparse. + return shlex.split(escape_whitespace_in_brackets(message)) + + +def parse_message(message): + parts = split_try_msg(message) + + # Argument parser based on try flag flags + parser = argparse.ArgumentParser() + parser.add_argument("-b", "--build", dest="build_types") + parser.add_argument( + "-p", "--platform", nargs="?", dest="platforms", const="all", default="all" + ) + parser.add_argument( + "-u", "--unittests", nargs="?", dest="unittests", const="all", default="all" + ) + parser.add_argument( + "-t", "--talos", nargs="?", dest="talos", const="all", default="none" + ) + parser.add_argument( + "-r", "--raptor", nargs="?", dest="raptor", const="all", default="none" + ) + parser.add_argument( + "-i", "--interactive", dest="interactive", action="store_true", default=False + ) + parser.add_argument( + "-e", "--all-emails", dest="notifications", action="store_const", const="all" + ) + parser.add_argument( + "-f", + "--failure-emails", + dest="notifications", + action="store_const", + const="failure", + ) + parser.add_argument("-j", "--job", dest="jobs", action="append") + parser.add_argument( + "--rebuild-talos", + dest="talos_trigger_tests", + action="store", + type=int, + default=1, + ) + parser.add_argument( + "--rebuild-raptor", + dest="raptor_trigger_tests", + action="store", + type=int, + default=1, + ) + parser.add_argument("--setenv", dest="env", action="append") + parser.add_argument("--gecko-profile", dest="profile", action="store_true") + parser.add_argument("--tag", dest="tag", action="store", default=None) + parser.add_argument("--no-retry", dest="no_retry", action="store_true") + parser.add_argument( + "--include-nightly", dest="include_nightly", action="store_true" + ) + parser.add_argument("--artifact", dest="artifact", action="store_true") + + # While we are transitioning from BB to TC, we want to push jobs to tc-worker + # machines but not overload machines with every try push. Therefore, we add + # this temporary option to be able to push jobs to tc-worker. + parser.add_argument( + "-w", + "--taskcluster-worker", + dest="taskcluster_worker", + action="store_true", + default=False, + ) + + # In order to run test jobs multiple times + parser.add_argument("--rebuild", dest="trigger_tests", type=int, default=1) + args, _ = parser.parse_known_args(parts) + + try_options = vars(args) + try_task_config = { + "use-artifact-builds": try_options.pop("artifact"), + "gecko-profile": try_options.pop("profile"), + "env": dict(arg.split("=") for arg in try_options.pop("env") or []), + } + return { + "try_options": try_options, + "try_task_config": try_task_config, + } + + +class TryOptionSyntax: + def __init__(self, parameters, full_task_graph, graph_config): + """ + Apply the try options in parameters. + + The resulting object has attributes: + + - build_types: a list containing zero or more of 'opt' and 'debug' + - platforms: a list of selected platform names, or None for all + - unittests: a list of tests, of the form given below, or None for all + - jobs: a list of requested job names, or None for all + - trigger_tests: the number of times tests should be triggered (--rebuild) + - interactive: true if --interactive + - notifications: either None if no notifications or one of 'all' or 'failure' + - talos_trigger_tests: the number of time talos tests should be triggered (--rebuild-talos) + - tag: restrict tests to the specified tag + - no_retry: do not retry failed jobs + + The unittests and talos lists contain dictionaries of the form: + + { + 'test': '<suite name>', + 'platforms': [..platform names..], # to limit to only certain platforms + 'only_chunks': set([..chunk numbers..]), # to limit only to certain chunks + } + """ + self.full_task_graph = full_task_graph + self.graph_config = graph_config + self.jobs = [] + self.build_types = [] + self.platforms = [] + self.unittests = [] + self.talos = [] + self.raptor = [] + self.trigger_tests = 0 + self.interactive = False + self.notifications = None + self.talos_trigger_tests = 0 + self.raptor_trigger_tests = 0 + self.tag = None + self.no_retry = False + + options = parameters["try_options"] + if not options: + return None + self.jobs = self.parse_jobs(options["jobs"]) + self.build_types = self.parse_build_types( + options["build_types"], full_task_graph + ) + self.platforms = self.parse_platforms(options, full_task_graph) + self.unittests = self.parse_test_option( + "unittest_try_name", options["unittests"], full_task_graph + ) + self.talos = self.parse_test_option( + "talos_try_name", options["talos"], full_task_graph + ) + self.raptor = self.parse_test_option( + "raptor_try_name", options["raptor"], full_task_graph + ) + self.trigger_tests = options["trigger_tests"] + self.interactive = options["interactive"] + self.notifications = options["notifications"] + self.talos_trigger_tests = options["talos_trigger_tests"] + self.raptor_trigger_tests = options["raptor_trigger_tests"] + self.tag = options["tag"] + self.no_retry = options["no_retry"] + self.include_nightly = options["include_nightly"] + + self.test_tiers = self.generate_test_tiers(full_task_graph) + + def generate_test_tiers(self, full_task_graph): + retval = defaultdict(set) + for t in full_task_graph.tasks.values(): + if t.attributes.get("kind") == "test": + try: + tier = t.task["extra"]["treeherder"]["tier"] + name = t.attributes.get("unittest_try_name") + retval[name].add(tier) + except KeyError: + pass + + return retval + + def parse_jobs(self, jobs_arg): + if not jobs_arg or jobs_arg == ["none"]: + return [] # default is `-j none` + if jobs_arg == ["all"]: + return None + expanded = [] + for job in jobs_arg: + expanded.extend(j.strip() for j in job.split(",")) + return expanded + + def parse_build_types(self, build_types_arg, full_task_graph): + if build_types_arg is None: + build_types_arg = [] + + build_types = [ + _f + for _f in ( + BUILD_TYPE_ALIASES.get(build_type) for build_type in build_types_arg + ) + if _f + ] + + all_types = { + t.attributes["build_type"] + for t in full_task_graph.tasks.values() + if "build_type" in t.attributes + } + bad_types = set(build_types) - all_types + if bad_types: + raise Exception( + "Unknown build type(s) [%s] specified for try" % ",".join(bad_types) + ) + + return build_types + + def parse_platforms(self, options, full_task_graph): + platform_arg = options["platforms"] + if platform_arg == "all": + return None + + RIDEALONG_BUILDS = self.graph_config["try"]["ridealong-builds"] + results = [] + for build in platform_arg.split(","): + if build in ("macosx64",): + # Regular opt builds are faster than shippable ones, but we don't run + # tests against them. + # We want to choose them (and only them) if no tests were requested. + if ( + options["unittests"] == "none" + and options["talos"] == "none" + and options["raptor"] == "none" + ): + results.append("macosx64") + logger.info("adding macosx64 for try syntax using macosx64.") + # Otherwise, use _just_ the shippable builds. + else: + results.append("macosx64-shippable") + logger.info( + "adding macosx64-shippable for try syntax using macosx64." + ) + else: + results.append(build) + if build in RIDEALONG_BUILDS: + results.extend(RIDEALONG_BUILDS[build]) + logger.info( + "platform %s triggers ridealong builds %s" + % (build, ", ".join(RIDEALONG_BUILDS[build])) + ) + + test_platforms = { + t.attributes["test_platform"] + for t in full_task_graph.tasks.values() + if "test_platform" in t.attributes + } + build_platforms = { + t.attributes["build_platform"] + for t in full_task_graph.tasks.values() + if "build_platform" in t.attributes + } + all_platforms = test_platforms | build_platforms + bad_platforms = set(results) - all_platforms + if bad_platforms: + raise Exception( + "Unknown platform(s) [%s] specified for try" % ",".join(bad_platforms) + ) + + return results + + def parse_test_option(self, attr_name, test_arg, full_task_graph): + """ + + Parse a unittest (-u) or talos (-t) option, in the context of a full + task graph containing available `unittest_try_name` or `talos_try_name` + attributes. There are three cases: + + - test_arg is == 'none' (meaning an empty list) + - test_arg is == 'all' (meaning use the list of jobs for that job type) + - test_arg is comma string which needs to be parsed + """ + + # Empty job list case... + if test_arg is None or test_arg == "none": + return [] + + all_platforms = { + t.attributes["test_platform"].split("/")[0] + for t in full_task_graph.tasks.values() + if "test_platform" in t.attributes + } + + tests = self.parse_test_opts(test_arg, all_platforms) + + if not tests: + return [] + + all_tests = { + t.attributes[attr_name] + for t in full_task_graph.tasks.values() + if attr_name in t.attributes + } + + # Special case where tests is 'all' and must be expanded + if tests[0]["test"] == "all": + results = [] + all_entry = tests[0] + for test in all_tests: + entry = {"test": test} + # If there are platform restrictions copy them across the list. + if "platforms" in all_entry: + entry["platforms"] = list(all_entry["platforms"]) + results.append(entry) + return self.parse_test_chunks(all_tests, results) + return self.parse_test_chunks(all_tests, tests) + + def parse_test_opts(self, input_str, all_platforms): + """ + Parse `testspec,testspec,..`, where each testspec is a test name + optionally followed by a list of test platforms or negated platforms in + `[]`. + + No brackets indicates that tests should run on all platforms for which + builds are available. If testspecs are provided, then each is treated, + from left to right, as an instruction to include or (if negated) + exclude a set of test platforms. A single spec may expand to multiple + test platforms via UNITTEST_PLATFORM_PRETTY_NAMES. If the first test + spec is negated, processing begins with the full set of available test + platforms; otherwise, processing begins with an empty set of test + platforms. + """ + + # Final results which we will return. + tests = [] + + cur_test = {} + token = "" + in_platforms = False + + def normalize_platforms(): + if "platforms" not in cur_test: + return + # if the first spec is a negation, start with all platforms + if cur_test["platforms"][0][0] == "-": + platforms = all_platforms.copy() + else: + platforms = [] + for platform in cur_test["platforms"]: + if platform[0] == "-": + platforms = [p for p in platforms if p != platform[1:]] + else: + platforms.append(platform) + cur_test["platforms"] = platforms + + def add_test(value): + normalize_platforms() + cur_test["test"] = value.strip() + tests.insert(0, cur_test) + + def add_platform(value): + platform = value.strip() + if platform[0] == "-": + negated = True + platform = platform[1:] + else: + negated = False + platforms = UNITTEST_PLATFORM_PRETTY_NAMES.get(platform, [platform]) + if negated: + platforms = ["-" + p for p in platforms] + cur_test["platforms"] = platforms + cur_test.get("platforms", []) + + # This might be somewhat confusing but we parse the string _backwards_ so + # there is no ambiguity over what state we are in. + for char in reversed(input_str): + + # , indicates exiting a state + if char == ",": + + # Exit a particular platform. + if in_platforms: + add_platform(token) + + # Exit a particular test. + else: + add_test(token) + cur_test = {} + + # Token must always be reset after we exit a state + token = "" + elif char == "[": + # Exiting platform state entering test state. + add_platform(token) + token = "" + in_platforms = False + elif char == "]": + # Entering platform state. + in_platforms = True + else: + # Accumulator. + token = char + token + + # Handle any left over tokens. + if token: + add_test(token) + + return tests + + def handle_alias(self, test, all_tests): + """ + Expand a test if its name refers to an alias, returning a list of test + dictionaries cloned from the first (to maintain any metadata). + """ + if test["test"] not in UNITTEST_ALIASES: + return [test] + + alias = UNITTEST_ALIASES[test["test"]] + + def mktest(name): + newtest = copy.deepcopy(test) + newtest["test"] = name + return newtest + + def exprmatch(alias): + return [t for t in all_tests if alias(t)] + + return [mktest(t) for t in exprmatch(alias)] + + def parse_test_chunks(self, all_tests, tests): + """ + Test flags may include parameters to narrow down the number of chunks in a + given push. We don't model 1 chunk = 1 job in taskcluster so we must check + each test flag to see if it is actually specifying a chunk. + """ + results = [] + seen_chunks = {} + for test in tests: + matches = TEST_CHUNK_SUFFIX.match(test["test"]) + if matches: + name = matches.group(1) + chunk = matches.group(2) + if name in seen_chunks: + seen_chunks[name].add(chunk) + else: + seen_chunks[name] = {chunk} + test["test"] = name + test["only_chunks"] = seen_chunks[name] + results.append(test) + else: + results.extend(self.handle_alias(test, all_tests)) + + # uniquify the results over the test names + results = sorted( + {test["test"]: test for test in results}.values(), + key=lambda test: test["test"], + ) + return results + + def find_all_attribute_suffixes(self, graph, prefix): + rv = set() + for t in graph.tasks.values(): + for a in t.attributes: + if a.startswith(prefix): + rv.add(a[len(prefix) :]) + return sorted(rv) + + def task_matches(self, task): + attr = task.attributes.get + + def check_run_on_projects(): + return {"all"} & set(attr("run_on_projects", [])) + + def match_test(try_spec, attr_name): + run_by_default = True + if attr("build_type") not in self.build_types: + return False + + if ( + self.platforms is not None + and attr("build_platform") not in self.platforms + ): + return False + if not check_run_on_projects(): + run_by_default = False + + if try_spec is None: + return run_by_default + + # TODO: optimize this search a bit + for test in try_spec: + if attr(attr_name) == test["test"]: + break + else: + return False + + if "only_chunks" in test and attr("test_chunk") not in test["only_chunks"]: + return False + + tier = task.task["extra"]["treeherder"]["tier"] + if "platforms" in test: + if "all" in test["platforms"]: + return True + platform = attr("test_platform", "").split("/")[0] + # Platforms can be forced by syntax like "-u xpcshell[Windows 8]" + return platform in test["platforms"] + if tier != 1: + # Run Tier 2/3 tests if their build task is Tier 2/3 OR if there is + # no tier 1 test of that name. + build_task = self.full_task_graph.tasks[task.dependencies["build"]] + build_task_tier = build_task.task["extra"]["treeherder"]["tier"] + + name = attr("unittest_try_name") + test_tiers = self.test_tiers.get(name) + + if tier <= build_task_tier: + logger.debug( + "not skipping tier {} test {} because build task {} " + "is tier {}".format( + tier, task.label, build_task.label, build_task_tier + ) + ) + return True + if 1 not in test_tiers: + logger.debug( + "not skipping tier {} test {} without explicit inclusion; " + "it is configured to run on tiers {}".format( + tier, task.label, test_tiers + ) + ) + return True + logger.debug( + "skipping tier {} test {} because build task {} is " + "tier {} and there is a higher-tier test of the same name".format( + tier, task.label, build_task.label, build_task_tier + ) + ) + return False + if run_by_default: + return check_run_on_projects() + return False + + if attr("job_try_name"): + # Beware the subtle distinction between [] and None for self.jobs and self.platforms. + # They will be [] if there was no try syntax, and None if try syntax was detected but + # they remained unspecified. + if self.jobs is not None: + return attr("job_try_name") in self.jobs + + # User specified `-j all` + if ( + self.platforms is not None + and attr("build_platform") not in self.platforms + ): + return False # honor -p for jobs governed by a platform + # "all" means "everything with `try` in run_on_projects" + return check_run_on_projects() + if attr("kind") == "test": + return ( + match_test(self.unittests, "unittest_try_name") + or match_test(self.talos, "talos_try_name") + or match_test(self.raptor, "raptor_try_name") + ) + if attr("kind") in BUILD_KINDS: + if attr("build_type") not in self.build_types: + return False + if self.platforms is None: + # for "-p all", look for try in the 'run_on_projects' attribute + return check_run_on_projects() + if attr("build_platform") not in self.platforms: + return False + return True + return False + + def __str__(self): + def none_for_all(list): + if list is None: + return "<all>" + return ", ".join(str(e) for e in list) + + return "\n".join( + [ + "build_types: " + ", ".join(self.build_types), + "platforms: " + none_for_all(self.platforms), + "unittests: " + none_for_all(self.unittests), + "talos: " + none_for_all(self.talos), + "raptor" + none_for_all(self.raptor), + "jobs: " + none_for_all(self.jobs), + "trigger_tests: " + str(self.trigger_tests), + "interactive: " + str(self.interactive), + "notifications: " + str(self.notifications), + "talos_trigger_tests: " + str(self.talos_trigger_tests), + "raptor_trigger_tests: " + str(self.raptor_trigger_tests), + "tag: " + str(self.tag), + "no_retry: " + str(self.no_retry), + ] + ) diff --git a/taskcluster/gecko_taskgraph/util/__init__.py b/taskcluster/gecko_taskgraph/util/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/__init__.py diff --git a/taskcluster/gecko_taskgraph/util/attributes.py b/taskcluster/gecko_taskgraph/util/attributes.py new file mode 100644 index 0000000000..0ed0be5a8d --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/attributes.py @@ -0,0 +1,143 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import re + +INTEGRATION_PROJECTS = { + "autoland", +} + +TRUNK_PROJECTS = INTEGRATION_PROJECTS | {"mozilla-central", "comm-central"} + +RELEASE_PROJECTS = { + "mozilla-central", + "mozilla-beta", + "mozilla-release", + "mozilla-esr102", + "mozilla-esr115", + "comm-central", + "comm-beta", + "comm-esr102", + "comm-esr115", +} + +RELEASE_PROMOTION_PROJECTS = { + "jamun", + "maple", + "try", + "try-comm-central", +} | RELEASE_PROJECTS + +TEMPORARY_PROJECTS = set( + { + # When using a "Disposable Project Branch" you can specify your branch here. e.g.: + "oak", + } +) + +TRY_PROJECTS = { + "try", + "try-comm-central", +} + +ALL_PROJECTS = RELEASE_PROMOTION_PROJECTS | TRUNK_PROJECTS | TEMPORARY_PROJECTS + +RUN_ON_PROJECT_ALIASES = { + # key is alias, value is lambda to test it against + "all": lambda project: True, + "integration": lambda project: ( + project in INTEGRATION_PROJECTS or project == "toolchains" + ), + "release": lambda project: (project in RELEASE_PROJECTS or project == "toolchains"), + "trunk": lambda project: (project in TRUNK_PROJECTS or project == "toolchains"), + "trunk-only": lambda project: project in TRUNK_PROJECTS, + "autoland": lambda project: project in ("autoland", "toolchains"), + "autoland-only": lambda project: project == "autoland", + "mozilla-central": lambda project: project in ("mozilla-central", "toolchains"), + "mozilla-central-only": lambda project: project == "mozilla-central", +} + +_COPYABLE_ATTRIBUTES = ( + "accepted-mar-channel-ids", + "artifact_map", + "artifact_prefix", + "build_platform", + "build_type", + "l10n_chunk", + "locale", + "mar-channel-id", + "maven_packages", + "nightly", + "required_signoffs", + "shippable", + "shipping_phase", + "shipping_product", + "signed", + "stub-installer", + "update-channel", +) + + +def match_run_on_projects(project, run_on_projects): + """Determine whether the given project is included in the `run-on-projects` + parameter, applying expansions for things like "integration" mentioned in + the attribute documentation.""" + aliases = RUN_ON_PROJECT_ALIASES.keys() + run_aliases = set(aliases) & set(run_on_projects) + if run_aliases: + if any(RUN_ON_PROJECT_ALIASES[alias](project) for alias in run_aliases): + return True + + return project in run_on_projects + + +def match_run_on_hg_branches(hg_branch, run_on_hg_branches): + """Determine whether the given project is included in the `run-on-hg-branches` + parameter. Allows 'all'.""" + if "all" in run_on_hg_branches: + return True + + for expected_hg_branch_pattern in run_on_hg_branches: + if re.match(expected_hg_branch_pattern, hg_branch): + return True + + return False + + +def copy_attributes_from_dependent_job(dep_job, denylist=()): + return { + attr: dep_job.attributes[attr] + for attr in _COPYABLE_ATTRIBUTES + if attr in dep_job.attributes and attr not in denylist + } + + +def sorted_unique_list(*args): + """Join one or more lists, and return a sorted list of unique members""" + combined = set().union(*args) + return sorted(combined) + + +def release_level(project): + """ + Whether this is a staging release or not. + + :return str: One of "production" or "staging". + """ + return "production" if project in RELEASE_PROJECTS else "staging" + + +def is_try(params): + """ + Determine whether this graph is being built on a try project or for + `mach try fuzzy`. + """ + return "try" in params["project"] or params["try_mode"] == "try_select" + + +def task_name(task): + if task.label.startswith(task.kind + "-"): + return task.label[len(task.kind) + 1 :] + raise AttributeError(f"Task {task.label} does not have a name.") diff --git a/taskcluster/gecko_taskgraph/util/backstop.py b/taskcluster/gecko_taskgraph/util/backstop.py new file mode 100644 index 0000000000..26c9a4fb91 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/backstop.py @@ -0,0 +1,84 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from requests import HTTPError +from taskgraph.util.taskcluster import find_task_id, get_artifact + +from gecko_taskgraph.util.attributes import INTEGRATION_PROJECTS, TRY_PROJECTS +from gecko_taskgraph.util.taskcluster import state_task + +BACKSTOP_PUSH_INTERVAL = 20 +BACKSTOP_TIME_INTERVAL = 60 * 4 # minutes +BACKSTOP_INDEX = "{trust-domain}.v2.{project}.latest.taskgraph.backstop" + + +def is_backstop( + params, + push_interval=BACKSTOP_PUSH_INTERVAL, + time_interval=BACKSTOP_TIME_INTERVAL, + trust_domain="gecko", + integration_projects=INTEGRATION_PROJECTS, +): + """Determines whether the given parameters represent a backstop push. + + Args: + push_interval (int): Number of pushes + time_interval (int): Minutes between forced schedules. + Use 0 to disable. + trust_domain (str): "gecko" for Firefox, "comm" for Thunderbird + integration_projects (set): project that uses backstop optimization + Returns: + bool: True if this is a backstop, otherwise False. + """ + # In case this is being faked on try. + if params.get("backstop", False): + return True + + project = params["project"] + pushid = int(params["pushlog_id"]) + pushdate = int(params["pushdate"]) + + if project in TRY_PROJECTS: + return False + if project not in integration_projects: + return True + + # On every Nth push, want to run all tasks. + if pushid % push_interval == 0: + return True + + if time_interval <= 0: + return False + + # We also want to ensure we run all tasks at least once per N minutes. + subs = {"trust-domain": trust_domain, "project": project} + index = BACKSTOP_INDEX.format(**subs) + + try: + last_backstop_id = find_task_id(index) + except KeyError: + # Index wasn't found, implying there hasn't been a backstop push yet. + return True + + if state_task(last_backstop_id) in ("failed", "exception"): + # If the last backstop failed its decision task, make this a backstop. + return True + + try: + last_pushdate = get_artifact(last_backstop_id, "public/parameters.yml")[ + "pushdate" + ] + except HTTPError as e: + # If the last backstop decision task exists in the index, but + # parameters.yml isn't available yet, it means the decision task is + # still running. If that's the case, we can be pretty sure the time + # component will not cause a backstop, so just return False. + if e.response.status_code == 404: + return False + raise + + if (pushdate - last_pushdate) / 60 >= time_interval: + return True + return False diff --git a/taskcluster/gecko_taskgraph/util/bugbug.py b/taskcluster/gecko_taskgraph/util/bugbug.py new file mode 100644 index 0000000000..50e02d69c6 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/bugbug.py @@ -0,0 +1,125 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +import sys +import time + +import requests +from mozbuild.util import memoize +from taskgraph import create +from taskgraph.util.taskcluster import requests_retry_session + +try: + # TODO(py3): use time.monotonic() + from time import monotonic +except ImportError: + from time import time as monotonic + +BUGBUG_BASE_URL = "https://bugbug.herokuapp.com" +RETRY_TIMEOUT = 9 * 60 # seconds +RETRY_INTERVAL = 10 # seconds + +# Preset confidence thresholds. +CT_LOW = 0.7 +CT_MEDIUM = 0.8 +CT_HIGH = 0.9 + +GROUP_TRANSLATIONS = { + "testing/web-platform/tests": "", + "testing/web-platform/mozilla/tests": "/_mozilla", +} + + +def translate_group(group): + for prefix, value in GROUP_TRANSLATIONS.items(): + if group.startswith(prefix): + return group.replace(prefix, value) + + return group + + +class BugbugTimeoutException(Exception): + pass + + +@memoize +def get_session(): + s = requests.Session() + s.headers.update({"X-API-KEY": "gecko-taskgraph"}) + return requests_retry_session(retries=5, session=s) + + +def _write_perfherder_data(lower_is_better): + if os.environ.get("MOZ_AUTOMATION", "0") == "1": + perfherder_data = { + "framework": {"name": "build_metrics"}, + "suites": [ + { + "name": suite, + "value": value, + "lowerIsBetter": True, + "shouldAlert": False, + "subtests": [], + } + for suite, value in lower_is_better.items() + ], + } + print(f"PERFHERDER_DATA: {json.dumps(perfherder_data)}", file=sys.stderr) + + +@memoize +def push_schedules(branch, rev): + # Noop if we're in test-action-callback + if create.testing: + return + + url = BUGBUG_BASE_URL + "/push/{branch}/{rev}/schedules".format( + branch=branch, rev=rev + ) + start = monotonic() + session = get_session() + + # On try there is no fallback and pulling is slower, so we allow bugbug more + # time to compute the results. + # See https://github.com/mozilla/bugbug/issues/1673. + timeout = RETRY_TIMEOUT + if branch == "try": + timeout += int(timeout / 3) + + attempts = timeout / RETRY_INTERVAL + i = 0 + while i < attempts: + r = session.get(url) + r.raise_for_status() + + if r.status_code != 202: + break + + time.sleep(RETRY_INTERVAL) + i += 1 + end = monotonic() + + _write_perfherder_data( + lower_is_better={ + "bugbug_push_schedules_time": end - start, + "bugbug_push_schedules_retries": i, + } + ) + + data = r.json() + if r.status_code == 202: + raise BugbugTimeoutException(f"Timed out waiting for result from '{url}'") + + if "groups" in data: + data["groups"] = {translate_group(k): v for k, v in data["groups"].items()} + + if "config_groups" in data: + data["config_groups"] = { + translate_group(k): v for k, v in data["config_groups"].items() + } + + return data diff --git a/taskcluster/gecko_taskgraph/util/cached_tasks.py b/taskcluster/gecko_taskgraph/util/cached_tasks.py new file mode 100644 index 0000000000..fff9bb9844 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/cached_tasks.py @@ -0,0 +1,82 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import hashlib +import time + +TARGET_CACHE_INDEX = "{trust_domain}.cache.level-{level}.{type}.{name}.hash.{digest}" +EXTRA_CACHE_INDEXES = [ + "{trust_domain}.cache.level-{level}.{type}.{name}.latest", + "{trust_domain}.cache.level-{level}.{type}.{name}.pushdate.{build_date_long}", +] + + +def add_optimization( + config, taskdesc, cache_type, cache_name, digest=None, digest_data=None +): + """ + Allow the results of this task to be cached. This adds index routes to the + task so it can be looked up for future runs, and optimization hints so that + cached artifacts can be found. Exactly one of `digest` and `digest_data` + must be passed. + + :param TransformConfig config: The configuration for the kind being transformed. + :param dict taskdesc: The description of the current task. + :param str cache_type: The type of task result being cached. + :param str cache_name: The name of the object being cached. + :param digest: A unique string indentifying this version of the artifacts + being generated. Typically this will be the hash of inputs to the task. + :type digest: bytes or None + :param digest_data: A list of bytes representing the inputs of this task. + They will be concatenated and hashed to create the digest for this + task. + :type digest_data: list of bytes or None + """ + cached_task = taskdesc.get("attributes", {}).get("cached_task") + if cached_task is False: + return + + if (digest is None) == (digest_data is None): + raise Exception("Must pass exactly one of `digest` and `digest_data`.") + if digest is None: + digest = hashlib.sha256("\n".join(digest_data).encode("utf-8")).hexdigest() + + subs = { + "trust_domain": config.graph_config["trust-domain"], + "type": cache_type, + "name": cache_name, + "digest": digest, + } + + # We'll try to find a cached version of the toolchain at levels above + # and including the current level, starting at the highest level. + index_routes = [] + for level in reversed(range(int(config.params["level"]), 4)): + subs["level"] = level + index_routes.append(TARGET_CACHE_INDEX.format(**subs)) + taskdesc["optimization"] = {"index-search": index_routes} + + # ... and cache at the lowest level. + taskdesc.setdefault("routes", []).append( + f"index.{TARGET_CACHE_INDEX.format(**subs)}" + ) + + # ... and add some extra routes for humans + subs["build_date_long"] = time.strftime( + "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"]) + ) + taskdesc["routes"].extend( + [f"index.{route.format(**subs)}" for route in EXTRA_CACHE_INDEXES] + ) + + taskdesc["attributes"]["cached_task"] = { + "type": cache_type, + "name": cache_name, + "digest": digest, + } + + # Allow future pushes to find this task before it completes + # Implementation in morphs + taskdesc["attributes"]["eager_indexes"] = [TARGET_CACHE_INDEX.format(**subs)] diff --git a/taskcluster/gecko_taskgraph/util/chunking.py b/taskcluster/gecko_taskgraph/util/chunking.py new file mode 100644 index 0000000000..2c6c429a2c --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/chunking.py @@ -0,0 +1,315 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +"""Utility functions to handle test chunking.""" + +import json +import logging +import os +from abc import ABCMeta, abstractmethod + +from manifestparser import TestManifest +from manifestparser.filters import chunk_by_runtime, tags +from mozbuild.util import memoize +from moztest.resolve import TEST_SUITES, TestManifestLoader, TestResolver + +from gecko_taskgraph import GECKO +from gecko_taskgraph.util.bugbug import CT_LOW, BugbugTimeoutException, push_schedules + +logger = logging.getLogger(__name__) +here = os.path.abspath(os.path.dirname(__file__)) +resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader) + + +def guess_mozinfo_from_task(task, repo=""): + """Attempt to build a mozinfo dict from a task definition. + + This won't be perfect and many values used in the manifests will be missing. But + it should cover most of the major ones and be "good enough" for chunking in the + taskgraph. + + Args: + task (dict): A task definition. + + Returns: + A dict that can be used as a mozinfo replacement. + """ + setting = task["test-setting"] + arch = setting["platform"]["arch"] + p_os = setting["platform"]["os"] + + info = { + "asan": setting["build"].get("asan", False), + "bits": 32 if "32" in arch else 64, + "ccov": setting["build"].get("ccov", False), + "debug": setting["build"]["type"] in ("debug", "debug-isolated-process"), + "e10s": not setting["runtime"].get("1proc", False), + "no-fission": "no-fission" in setting["runtime"].keys(), + "fission": any( + "1proc" not in key or "no-fission" not in key + for key in setting["runtime"].keys() + ), + "headless": "-headless" in task["test-name"], + "condprof": "conditioned_profile" in setting["runtime"].keys(), + "tsan": setting["build"].get("tsan", False), + "xorigin": any("xorigin" in key for key in setting["runtime"].keys()), + "socketprocess_networking": "socketprocess_networking" + in setting["runtime"].keys(), + "nightly_build": repo in ["mozilla-central", "autoland", "try", ""], # trunk + "http3": "http3" in setting["runtime"].keys(), + } + for platform in ("android", "linux", "mac", "win"): + if p_os["name"].startswith(platform): + info["os"] = platform + break + else: + raise ValueError("{} is not a known platform!".format(p_os["name"])) + + # crashreporter is disabled for asan / tsan builds + if info["asan"] or info["tsan"]: + info["crashreporter"] = False + else: + info["crashreporter"] = True + + info["appname"] = "fennec" if info["os"] == "android" else "firefox" + + # guess processor + if arch == "aarch64": + info["processor"] = "aarch64" + elif info["os"] == "android" and "arm" in arch: + info["processor"] = "arm" + elif info["bits"] == 32: + info["processor"] = "x86" + else: + info["processor"] = "x86_64" + + # guess toolkit + if info["os"] == "android": + info["toolkit"] = "android" + elif info["os"] == "win": + info["toolkit"] = "windows" + elif info["os"] == "mac": + info["toolkit"] = "cocoa" + else: + info["toolkit"] = "gtk" + + # guess os_version + os_versions = { + ("linux", "1804"): "18.04", + ("macosx", "1015"): "10.15", + ("macosx", "1100"): "11.00", + ("windows", "7"): "6.1", + ("windows", "10"): "10.0", + } + for (name, old_ver), new_ver in os_versions.items(): + if p_os["name"] == name and p_os["version"] == old_ver: + info["os_version"] = new_ver + break + + return info + + +@memoize +def get_runtimes(platform, suite_name): + if not suite_name or not platform: + raise TypeError("suite_name and platform cannot be empty.") + + base = os.path.join(GECKO, "testing", "runtimes", "manifest-runtimes-{}.json") + for key in ("android", "windows"): + if key in platform: + path = base.format(key) + break + else: + path = base.format("unix") + + if not os.path.exists(path): + raise OSError(f"manifest runtime file at {path} not found.") + + with open(path) as fh: + return json.load(fh)[suite_name] + + +def chunk_manifests(suite, platform, chunks, manifests): + """Run the chunking algorithm. + + Args: + platform (str): Platform used to find runtime info. + chunks (int): Number of chunks to split manifests into. + manifests(list): Manifests to chunk. + + Returns: + A list of length `chunks` where each item contains a list of manifests + that run in that chunk. + """ + manifests = set(manifests) + + if "web-platform-tests" not in suite: + runtimes = { + k: v for k, v in get_runtimes(platform, suite).items() if k in manifests + } + return [ + c[1] + for c in chunk_by_runtime(None, chunks, runtimes).get_chunked_manifests( + manifests + ) + ] + + # Keep track of test paths for each chunk, and the runtime information. + chunked_manifests = [[] for _ in range(chunks)] + + # Spread out the test manifests evenly across all chunks. + for index, key in enumerate(sorted(manifests)): + chunked_manifests[index % chunks].append(key) + + # One last sort by the number of manifests. Chunk size should be more or less + # equal in size. + chunked_manifests.sort(key=lambda x: len(x)) + + # Return just the chunked test paths. + return chunked_manifests + + +class BaseManifestLoader(metaclass=ABCMeta): + def __init__(self, params): + self.params = params + + @abstractmethod + def get_manifests(self, flavor, subsuite, mozinfo): + """Compute which manifests should run for the given flavor, subsuite and mozinfo. + + This function returns skipped manifests separately so that more balanced + chunks can be achieved by only considering "active" manifests in the + chunking algorithm. + + Args: + flavor (str): The suite to run. Values are defined by the 'build_flavor' key + in `moztest.resolve.TEST_SUITES`. + subsuite (str): The subsuite to run or 'undefined' to denote no subsuite. + mozinfo (frozenset): Set of data in the form of (<key>, <value>) used + for filtering. + + Returns: + A tuple of two manifest lists. The first is the set of active manifests (will + run at least one test. The second is a list of skipped manifests (all tests are + skipped). + """ + + +class DefaultLoader(BaseManifestLoader): + """Load manifests using metadata from the TestResolver.""" + + @memoize + def get_tests(self, suite): + suite_definition = TEST_SUITES[suite] + return list( + resolver.resolve_tests( + flavor=suite_definition["build_flavor"], + subsuite=suite_definition.get("kwargs", {}).get( + "subsuite", "undefined" + ), + ) + ) + + @memoize + def get_manifests(self, suite, mozinfo): + mozinfo = dict(mozinfo) + # Compute all tests for the given suite/subsuite. + tests = self.get_tests(suite) + + # TODO: the only exception here is we schedule webgpu as that is a --tag + if "web-platform-tests" in suite: + manifests = set() + for t in tests: + manifests.add(t["manifest"]) + return { + "active": list(manifests), + "skipped": [], + "other_dirs": dict.fromkeys(manifests, ""), + } + + manifests = {chunk_by_runtime.get_manifest(t) for t in tests} + + filters = None + if mozinfo["condprof"]: + filters = [tags(["condprof"])] + + # Compute the active tests. + m = TestManifest() + m.tests = tests + tests = m.active_tests(disabled=False, exists=False, filters=filters, **mozinfo) + active = {} + # map manifests and 'other' directories included + for t in tests: + mp = chunk_by_runtime.get_manifest(t) + active.setdefault(mp, []) + + if not mp.startswith(t["dir_relpath"]): + active[mp].append(t["dir_relpath"]) + + skipped = manifests - set(active.keys()) + other = {} + for m in active: + if len(active[m]) > 0: + other[m] = list(set(active[m])) + return { + "active": list(active.keys()), + "skipped": list(skipped), + "other_dirs": other, + } + + +class BugbugLoader(DefaultLoader): + """Load manifests using metadata from the TestResolver, and then + filter them based on a query to bugbug.""" + + CONFIDENCE_THRESHOLD = CT_LOW + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.timedout = False + + @memoize + def get_manifests(self, suite, mozinfo): + manifests = super().get_manifests(suite, mozinfo) + + # Don't prune any manifests if we're on a backstop push or there was a timeout. + if self.params["backstop"] or self.timedout: + return manifests + + try: + data = push_schedules(self.params["project"], self.params["head_rev"]) + except BugbugTimeoutException: + logger.warning("Timed out waiting for bugbug, loading all test manifests.") + self.timedout = True + return self.get_manifests(suite, mozinfo) + + bugbug_manifests = { + m + for m, c in data.get("groups", {}).items() + if c >= self.CONFIDENCE_THRESHOLD + } + + manifests["active"] = list(set(manifests["active"]) & bugbug_manifests) + manifests["skipped"] = list(set(manifests["skipped"]) & bugbug_manifests) + return manifests + + +manifest_loaders = { + "bugbug": BugbugLoader, + "default": DefaultLoader, +} + +_loader_cache = {} + + +def get_manifest_loader(name, params): + # Ensure we never create more than one instance of the same loader type for + # performance reasons. + if name in _loader_cache: + return _loader_cache[name] + + loader = manifest_loaders[name](dict(params)) + _loader_cache[name] = loader + return loader diff --git a/taskcluster/gecko_taskgraph/util/copy_task.py b/taskcluster/gecko_taskgraph/util/copy_task.py new file mode 100644 index 0000000000..0aaf43361e --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/copy_task.py @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozbuild.util import ReadOnlyDict +from taskgraph.task import Task + +immutable_types = {int, float, bool, str, type(None), ReadOnlyDict} + + +def copy_task(obj): + """ + Perform a deep copy of a task that has a tree-like structure. + + Unlike copy.deepcopy, this does *not* support copying graph-like structure, + but it does it more efficiently than deepcopy. + """ + ty = type(obj) + if ty in immutable_types: + return obj + if ty is dict: + return {k: copy_task(v) for k, v in obj.items()} + if ty is list: + return [copy_task(elt) for elt in obj] + if ty is Task: + task = Task( + kind=copy_task(obj.kind), + label=copy_task(obj.label), + attributes=copy_task(obj.attributes), + task=copy_task(obj.task), + description=copy_task(obj.description), + optimization=copy_task(obj.optimization), + dependencies=copy_task(obj.dependencies), + soft_dependencies=copy_task(obj.soft_dependencies), + if_dependencies=copy_task(obj.if_dependencies), + ) + if obj.task_id: + task.task_id = obj.task_id + return task + raise NotImplementedError(f"copying '{ty}' from '{obj}'") diff --git a/taskcluster/gecko_taskgraph/util/declarative_artifacts.py b/taskcluster/gecko_taskgraph/util/declarative_artifacts.py new file mode 100644 index 0000000000..545472dec3 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/declarative_artifacts.py @@ -0,0 +1,92 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import re + +from gecko_taskgraph.util.scriptworker import ( + generate_beetmover_artifact_map, + generate_beetmover_upstream_artifacts, +) + +_ARTIFACT_ID_PER_PLATFORM = { + "android-aarch64-opt": "{package}-default-arm64-v8a", + "android-arm-opt": "{package}-default-armeabi-v7a", + "android-x86-opt": "{package}-default-x86", + "android-x86_64-opt": "{package}-default-x86_64", + "android-geckoview-fat-aar-opt": "{package}-default", + "android-aarch64-shippable": "{package}{update_channel}-omni-arm64-v8a", + "android-aarch64-shippable-lite": "{package}{update_channel}-arm64-v8a", + "android-arm-shippable": "{package}{update_channel}-omni-armeabi-v7a", + "android-arm-shippable-lite": "{package}{update_channel}-armeabi-v7a", + "android-x86-shippable": "{package}{update_channel}-omni-x86", + "android-x86-shippable-lite": "{package}{update_channel}-x86", + "android-x86_64-shippable": "{package}{update_channel}-omni-x86_64", + "android-x86_64-shippable-lite": "{package}{update_channel}-x86_64", + "android-geckoview-fat-aar-shippable": "{package}{update_channel}-omni", + "android-geckoview-fat-aar-shippable-lite": "{package}{update_channel}", +} + + +def get_geckoview_artifact_map(config, job): + return generate_beetmover_artifact_map( + config, + job, + **get_geckoview_template_vars( + config, + job["attributes"]["build_platform"], + job["maven-package"], + job["attributes"].get("update-channel"), + ), + ) + + +def get_geckoview_upstream_artifacts(config, job, package, platform=""): + if not platform: + platform = job["attributes"]["build_platform"] + upstream_artifacts = generate_beetmover_upstream_artifacts( + config, + job, + platform="", + **get_geckoview_template_vars( + config, platform, package, job["attributes"].get("update-channel") + ), + ) + return [ + {key: value for key, value in upstream_artifact.items() if key != "locale"} + for upstream_artifact in upstream_artifacts + ] + + +def get_geckoview_template_vars(config, platform, package, update_channel): + version_groups = re.match(r"(\d+).(\d+).*", config.params["version"]) + if version_groups: + major_version, minor_version = version_groups.groups() + + return { + "artifact_id": get_geckoview_artifact_id( + config, + platform, + package, + update_channel, + ), + "build_date": config.params["moz_build_date"], + "major_version": major_version, + "minor_version": minor_version, + } + + +def get_geckoview_artifact_id(config, platform, package, update_channel=None): + if update_channel == "release": + update_channel = "" + elif update_channel is not None: + update_channel = f"-{update_channel}" + else: + # For shippable builds, mozharness defaults to using + # "nightly-{project}" for the update channel. For other builds, the + # update channel is not set, but the value is not substituted. + update_channel = "-nightly-{}".format(config.params["project"]) + return _ARTIFACT_ID_PER_PLATFORM[platform].format( + update_channel=update_channel, package=package + ) diff --git a/taskcluster/gecko_taskgraph/util/docker.py b/taskcluster/gecko_taskgraph/util/docker.py new file mode 100644 index 0000000000..e8de7d1fdb --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/docker.py @@ -0,0 +1,333 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import hashlib +import json +import os +import re +import sys +from collections.abc import Mapping +from urllib.parse import quote, urlencode, urlunparse + +import requests +import requests_unixsocket +from mozbuild.util import memoize +from mozpack.archive import create_tar_gz_from_files +from mozpack.files import GeneratedFile +from taskgraph.util.yaml import load_yaml + +from .. import GECKO + +IMAGE_DIR = os.path.join(GECKO, "taskcluster", "docker") + + +def docker_url(path, **kwargs): + docker_socket = os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock") + return urlunparse( + ("http+unix", quote(docker_socket, safe=""), path, "", urlencode(kwargs), "") + ) + + +def post_to_docker(tar, api_path, **kwargs): + """POSTs a tar file to a given docker API path. + + The tar argument can be anything that can be passed to requests.post() + as data (e.g. iterator or file object). + The extra keyword arguments are passed as arguments to the docker API. + """ + # requests-unixsocket doesn't honor requests timeouts + # See https://github.com/msabramo/requests-unixsocket/issues/44 + # We have some large docker images that trigger the default timeout, + # so we increase the requests-unixsocket timeout here. + session = requests.Session() + session.mount( + requests_unixsocket.DEFAULT_SCHEME, + requests_unixsocket.UnixAdapter(timeout=120), + ) + req = session.post( + docker_url(api_path, **kwargs), + data=tar, + stream=True, + headers={"Content-Type": "application/x-tar"}, + ) + if req.status_code != 200: + message = req.json().get("message") + if not message: + message = f"docker API returned HTTP code {req.status_code}" + raise Exception(message) + status_line = {} + + buf = b"" + for content in req.iter_content(chunk_size=None): + if not content: + continue + # Sometimes, a chunk of content is not a complete json, so we cumulate + # with leftovers from previous iterations. + buf += content + try: + data = json.loads(buf) + except Exception: + continue + buf = b"" + # data is sometimes an empty dict. + if not data: + continue + # Mimick how docker itself presents the output. This code was tested + # with API version 1.18 and 1.26. + if "status" in data: + if "id" in data: + if sys.stderr.isatty(): + total_lines = len(status_line) + line = status_line.setdefault(data["id"], total_lines) + n = total_lines - line + if n > 0: + # Move the cursor up n lines. + sys.stderr.write(f"\033[{n}A") + # Clear line and move the cursor to the beginning of it. + sys.stderr.write("\033[2K\r") + sys.stderr.write( + "{}: {} {}\n".format( + data["id"], data["status"], data.get("progress", "") + ) + ) + if n > 1: + # Move the cursor down n - 1 lines, which, considering + # the carriage return on the last write, gets us back + # where we started. + sys.stderr.write(f"\033[{n - 1}B") + else: + status = status_line.get(data["id"]) + # Only print status changes. + if status != data["status"]: + sys.stderr.write("{}: {}\n".format(data["id"], data["status"])) + status_line[data["id"]] = data["status"] + else: + status_line = {} + sys.stderr.write("{}\n".format(data["status"])) + elif "stream" in data: + sys.stderr.write(data["stream"]) + elif "aux" in data: + sys.stderr.write(repr(data["aux"])) + elif "error" in data: + sys.stderr.write("{}\n".format(data["error"])) + # Sadly, docker doesn't give more than a plain string for errors, + # so the best we can do to propagate the error code from the command + # that failed is to parse the error message... + errcode = 1 + m = re.search(r"returned a non-zero code: (\d+)", data["error"]) + if m: + errcode = int(m.group(1)) + sys.exit(errcode) + else: + raise NotImplementedError(repr(data)) + sys.stderr.flush() + + +def docker_image(name, by_tag=False): + """ + Resolve in-tree prebuilt docker image to ``<registry>/<repository>@sha256:<digest>``, + or ``<registry>/<repository>:<tag>`` if `by_tag` is `True`. + """ + try: + with open(os.path.join(IMAGE_DIR, name, "REGISTRY")) as f: + registry = f.read().strip() + except OSError: + with open(os.path.join(IMAGE_DIR, "REGISTRY")) as f: + registry = f.read().strip() + + if not by_tag: + hashfile = os.path.join(IMAGE_DIR, name, "HASH") + try: + with open(hashfile) as f: + return f"{registry}/{name}@{f.read().strip()}" + except OSError: + raise Exception(f"Failed to read HASH file {hashfile}") + + try: + with open(os.path.join(IMAGE_DIR, name, "VERSION")) as f: + tag = f.read().strip() + except OSError: + tag = "latest" + return f"{registry}/{name}:{tag}" + + +class VoidWriter: + """A file object with write capabilities that does nothing with the written + data.""" + + def write(self, buf): + pass + + +def generate_context_hash(topsrcdir, image_path, image_name, args): + """Generates a sha256 hash for context directory used to build an image.""" + + return stream_context_tar( + topsrcdir, image_path, VoidWriter(), image_name, args=args + ) + + +class HashingWriter: + """A file object with write capabilities that hashes the written data at + the same time it passes down to a real file object.""" + + def __init__(self, writer): + self._hash = hashlib.sha256() + self._writer = writer + + def write(self, buf): + self._hash.update(buf) + self._writer.write(buf) + + def hexdigest(self): + return self._hash.hexdigest() + + +def create_context_tar(topsrcdir, context_dir, out_path, image_name, args): + """Create a context tarball. + + A directory ``context_dir`` containing a Dockerfile will be assembled into + a gzipped tar file at ``out_path``. + + We also scan the source Dockerfile for special syntax that influences + context generation. + + If a line in the Dockerfile has the form ``# %include <path>``, + the relative path specified on that line will be matched against + files in the source repository and added to the context under the + path ``topsrcdir/``. If an entry is a directory, we add all files + under that directory. + + Returns the SHA-256 hex digest of the created archive. + """ + with open(out_path, "wb") as fh: + return stream_context_tar( + topsrcdir, + context_dir, + fh, + image_name=image_name, + args=args, + ) + + +def stream_context_tar(topsrcdir, context_dir, out_file, image_name, args): + """Like create_context_tar, but streams the tar file to the `out_file` file + object.""" + archive_files = {} + content = [] + + context_dir = os.path.join(topsrcdir, context_dir) + + for root, dirs, files in os.walk(context_dir): + for f in files: + source_path = os.path.join(root, f) + archive_path = source_path[len(context_dir) + 1 :] + archive_files[archive_path] = source_path + + # Parse Dockerfile for special syntax of extra files to include. + with open(os.path.join(context_dir, "Dockerfile"), "r") as fh: + for line in fh: + content.append(line) + + if not line.startswith("# %include"): + continue + + p = line[len("# %include ") :].strip() + if os.path.isabs(p): + raise Exception("extra include path cannot be absolute: %s" % p) + + fs_path = os.path.normpath(os.path.join(topsrcdir, p)) + # Check for filesystem traversal exploits. + if not fs_path.startswith(topsrcdir): + raise Exception("extra include path outside topsrcdir: %s" % p) + + if not os.path.exists(fs_path): + raise Exception("extra include path does not exist: %s" % p) + + if os.path.isdir(fs_path): + for root, dirs, files in os.walk(fs_path): + for f in files: + source_path = os.path.join(root, f) + rel = source_path[len(fs_path) + 1 :] + archive_path = os.path.join("topsrcdir", p, rel) + archive_files[archive_path] = source_path + else: + archive_path = os.path.join("topsrcdir", p) + archive_files[archive_path] = fs_path + + archive_files["Dockerfile"] = GeneratedFile("".join(content).encode("utf-8")) + + writer = HashingWriter(out_file) + create_tar_gz_from_files(writer, archive_files, f"{image_name}.tar") + return writer.hexdigest() + + +class ImagePathsMap(Mapping): + """ImagePathsMap contains the mapping of Docker image names to their + context location in the filesystem. The register function allows Thunderbird + to define additional images under comm/taskcluster. + """ + + def __init__(self, config_path, image_dir=IMAGE_DIR): + config = load_yaml(GECKO, config_path) + self.__update_image_paths(config["jobs"], image_dir) + + def __getitem__(self, key): + return self.__dict__[key] + + def __iter__(self): + return iter(self.__dict__) + + def __len__(self): + return len(self.__dict__) + + def __update_image_paths(self, jobs, image_dir): + self.__dict__.update( + { + k: os.path.join(image_dir, v.get("definition", k)) + for k, v in jobs.items() + } + ) + + def register(self, jobs_config_path, image_dir): + """Register additional image_paths. In this case, there is no 'jobs' + key in the loaded YAML as this file is loaded via jobs-from in kind.yml.""" + jobs = load_yaml(GECKO, jobs_config_path) + self.__update_image_paths(jobs, image_dir) + + +image_paths = ImagePathsMap("taskcluster/ci/docker-image/kind.yml") + + +def image_path(name): + if name in image_paths: + return image_paths[name] + return os.path.join(IMAGE_DIR, name) + + +@memoize +def parse_volumes(image): + """Parse VOLUME entries from a Dockerfile for an image.""" + volumes = set() + + path = image_path(image) + + with open(os.path.join(path, "Dockerfile"), "rb") as fh: + for line in fh: + line = line.strip() + # We assume VOLUME definitions don't use ARGS. + if not line.startswith(b"VOLUME "): + continue + + v = line.split(None, 1)[1] + if v.startswith(b"["): + raise ValueError( + "cannot parse array syntax for VOLUME; " + "convert to multiple entries" + ) + + volumes |= {v.decode("utf-8") for v in v.split()} + + return volumes diff --git a/taskcluster/gecko_taskgraph/util/hash.py b/taskcluster/gecko_taskgraph/util/hash.py new file mode 100644 index 0000000000..485c9a7c48 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/hash.py @@ -0,0 +1,68 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import hashlib + +import mozpack.path as mozpath +from mozbuild.util import memoize +from mozversioncontrol import get_repository_object + + +@memoize +def hash_path(path): + """Hash a single file. + + Returns the SHA-256 hash in hex form. + """ + with open(path, mode="rb") as fh: + return hashlib.sha256(fh.read()).hexdigest() + + +@memoize +def get_file_finder(base_path): + from pathlib import Path + + repo = get_repository_object(base_path) + if repo: + files = repo.get_tracked_files_finder(base_path) + if files: + return files + else: + return None + else: + return get_repository_object(Path(base_path)).get_tracked_files_finder( + base_path + ) + + +def hash_paths(base_path, patterns): + """ + Give a list of path patterns, return a digest of the contents of all + the corresponding files, similarly to git tree objects or mercurial + manifests. + + Each file is hashed. The list of all hashes and file paths is then + itself hashed to produce the result. + """ + finder = get_file_finder(base_path) + h = hashlib.sha256() + files = {} + if finder: + for pattern in patterns: + found = list(finder.find(pattern)) + if found: + files.update(found) + else: + raise Exception("%s did not match anything" % pattern) + for path in sorted(files.keys()): + if path.endswith((".pyc", ".pyd", ".pyo")): + continue + h.update( + "{} {}\n".format( + hash_path(mozpath.abspath(mozpath.join(base_path, path))), + mozpath.normsep(path), + ).encode("utf-8") + ) + + return h.hexdigest() diff --git a/taskcluster/gecko_taskgraph/util/hg.py b/taskcluster/gecko_taskgraph/util/hg.py new file mode 100644 index 0000000000..18a92fbd0d --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/hg.py @@ -0,0 +1,139 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging +import subprocess + +import requests +from mozbuild.util import memoize +from redo import retry + +logger = logging.getLogger(__name__) + +PUSHLOG_CHANGESET_TMPL = ( + "{repository}/json-pushes?version=2&changeset={revision}&tipsonly=1" +) +PUSHLOG_PUSHES_TMPL = ( + "{repository}/json-pushes/?version=2&startID={push_id_start}&endID={push_id_end}" +) + + +def _query_pushlog(url): + response = retry( + requests.get, + attempts=5, + sleeptime=10, + args=(url,), + kwargs={"timeout": 60, "headers": {"User-Agent": "TaskCluster"}}, + ) + + return response.json()["pushes"] + + +def find_hg_revision_push_info(repository, revision): + """Given the parameters for this action and a revision, find the + pushlog_id of the revision.""" + url = PUSHLOG_CHANGESET_TMPL.format(repository=repository, revision=revision) + + pushes = _query_pushlog(url) + + if len(pushes) != 1: + raise RuntimeError( + "Found {} pushlog_ids, expected 1, for {} revision {}: {}".format( + len(pushes), repository, revision, pushes + ) + ) + + pushid = list(pushes.keys())[0] + return { + "pushdate": pushes[pushid]["date"], + "pushid": pushid, + "user": pushes[pushid]["user"], + } + + +@memoize +def get_push_data(repository, project, push_id_start, push_id_end): + url = PUSHLOG_PUSHES_TMPL.format( + repository=repository, + push_id_start=push_id_start - 1, + push_id_end=push_id_end, + ) + + try: + pushes = _query_pushlog(url) + + return { + push_id: pushes[str(push_id)] + for push_id in range(push_id_start, push_id_end + 1) + } + + # In the event of request times out, requests will raise a TimeoutError. + except requests.exceptions.Timeout: + logger.warning("json-pushes timeout") + + # In the event of a network problem (e.g. DNS failure, refused connection, etc), + # requests will raise a ConnectionError. + except requests.exceptions.ConnectionError: + logger.warning("json-pushes connection error") + + # In the event of the rare invalid HTTP response(e.g 404, 401), + # requests will raise an HTTPError exception + except requests.exceptions.HTTPError: + logger.warning("Bad Http response") + + # When we get invalid JSON (i.e. 500 error), it results in a ValueError (bug 1313426) + except ValueError as error: + logger.warning(f"Invalid JSON, possible server error: {error}") + + # We just print the error out as a debug message if we failed to catch the exception above + except requests.exceptions.RequestException as error: + logger.warning(error) + + return None + + +@memoize +def get_json_automationrelevance(repository, revision): + url = "{}/json-automationrelevance/{}".format(repository.rstrip("/"), revision) + logger.debug("Querying version control for metadata: %s", url) + + def get_automationrelevance(): + response = requests.get(url, timeout=30) + return response.json() + + return retry(get_automationrelevance, attempts=10, sleeptime=10) + + +def get_hg_revision_branch(root, revision): + """Given the parameters for a revision, find the hg_branch (aka + relbranch) of the revision.""" + return subprocess.check_output( + [ + "hg", + "identify", + "-T", + "{branch}", + "--rev", + revision, + ], + cwd=root, + universal_newlines=True, + ) + + +# For these functions, we assume that run-task has correctly checked out the +# revision indicated by GECKO_HEAD_REF, so all that remains is to see what the +# current revision is. Mercurial refers to that as `.`. +def get_hg_commit_message(root, rev="."): + return subprocess.check_output( + ["hg", "log", "-r", rev, "-T", "{desc}"], cwd=root, universal_newlines=True + ) + + +def calculate_head_rev(root): + return subprocess.check_output( + ["hg", "log", "-r", ".", "-T", "{node}"], cwd=root, universal_newlines=True + ) diff --git a/taskcluster/gecko_taskgraph/util/partials.py b/taskcluster/gecko_taskgraph/util/partials.py new file mode 100644 index 0000000000..1a3affcc42 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/partials.py @@ -0,0 +1,297 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging + +import redo +import requests + +from gecko_taskgraph.util.scriptworker import ( + BALROG_SCOPE_ALIAS_TO_PROJECT, + BALROG_SERVER_SCOPES, +) + +logger = logging.getLogger(__name__) + +PLATFORM_RENAMES = { + "windows2012-32": "win32", + "windows2012-64": "win64", + "windows2012-aarch64": "win64-aarch64", + "osx-cross": "macosx64", + "osx": "macosx64", +} + +BALROG_PLATFORM_MAP = { + "linux": ["Linux_x86-gcc3"], + "linux32": ["Linux_x86-gcc3"], + "linux64": ["Linux_x86_64-gcc3"], + "linux64-asan-reporter": ["Linux_x86_64-gcc3-asan"], + "macosx64": [ + "Darwin_x86_64-gcc3-u-i386-x86_64", + "Darwin_x86-gcc3-u-i386-x86_64", + "Darwin_aarch64-gcc3", + "Darwin_x86-gcc3", + "Darwin_x86_64-gcc3", + ], + "win32": ["WINNT_x86-msvc", "WINNT_x86-msvc-x86", "WINNT_x86-msvc-x64"], + "win64": ["WINNT_x86_64-msvc", "WINNT_x86_64-msvc-x64"], + "win64-asan-reporter": ["WINNT_x86_64-msvc-x64-asan"], + "win64-aarch64": [ + "WINNT_aarch64-msvc-aarch64", + ], +} + +FTP_PLATFORM_MAP = { + "Darwin_x86-gcc3": "mac", + "Darwin_x86-gcc3-u-i386-x86_64": "mac", + "Darwin_x86_64-gcc3": "mac", + "Darwin_x86_64-gcc3-u-i386-x86_64": "mac", + "Darwin_aarch64-gcc3": "mac", + "Linux_x86-gcc3": "linux-i686", + "Linux_x86_64-gcc3": "linux-x86_64", + "Linux_x86_64-gcc3-asan": "linux-x86_64-asan-reporter", + "WINNT_x86_64-msvc-x64-asan": "win64-asan-reporter", + "WINNT_x86-msvc": "win32", + "WINNT_x86-msvc-x64": "win32", + "WINNT_x86-msvc-x86": "win32", + "WINNT_x86_64-msvc": "win64", + "WINNT_x86_64-msvc-x64": "win64", + "WINNT_aarch64-msvc-aarch64": "win64-aarch64", +} + + +def get_balrog_platform_name(platform): + """Convert build platform names into balrog platform names. + + Remove known values instead to catch aarch64 and other platforms + that may be added. + """ + removals = ["-devedition", "-shippable"] + for remove in removals: + platform = platform.replace(remove, "") + return PLATFORM_RENAMES.get(platform, platform) + + +def _sanitize_platform(platform): + platform = get_balrog_platform_name(platform) + if platform not in BALROG_PLATFORM_MAP: + return platform + return BALROG_PLATFORM_MAP[platform][0] + + +def get_builds(release_history, platform, locale): + """Examine cached balrog release history and return the list of + builds we need to generate diffs from""" + platform = _sanitize_platform(platform) + return release_history.get(platform, {}).get(locale, {}) + + +def get_partials_artifacts_from_params(release_history, platform, locale): + platform = _sanitize_platform(platform) + return [ + (artifact, details.get("previousVersion", None)) + for artifact, details in release_history.get(platform, {}) + .get(locale, {}) + .items() + ] + + +def get_partials_info_from_params(release_history, platform, locale): + platform = _sanitize_platform(platform) + + artifact_map = {} + for k in release_history.get(platform, {}).get(locale, {}): + details = release_history[platform][locale][k] + attributes = ("buildid", "previousBuildNumber", "previousVersion") + artifact_map[k] = { + attr: details[attr] for attr in attributes if attr in details + } + return artifact_map + + +def _retry_on_http_errors(url, verify, params, errors): + if params: + params_str = "&".join("=".join([k, str(v)]) for k, v in params.items()) + else: + params_str = "" + logger.info("Connecting to %s?%s", url, params_str) + for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10): + try: + req = requests.get(url, verify=verify, params=params, timeout=10) + req.raise_for_status() + return req + except requests.HTTPError as e: + if e.response.status_code in errors: + logger.exception( + "Got HTTP %s trying to reach %s", e.response.status_code, url + ) + else: + raise + else: + raise Exception(f"Cannot connect to {url}!") + + +def get_sorted_releases(product, branch): + """Returns a list of release names from Balrog. + :param product: product name, AKA appName + :param branch: branch name, e.g. mozilla-central + :return: a sorted list of release names, most recent first. + """ + url = f"{_get_balrog_api_root(branch)}/releases" + params = { + "product": product, + # Adding -nightly-2 (2 stands for the beginning of build ID + # based on date) should filter out release and latest blobs. + # This should be changed to -nightly-3 in 3000 ;) + "name_prefix": f"{product}-{branch}-nightly-2", + "names_only": True, + } + req = _retry_on_http_errors(url=url, verify=True, params=params, errors=[500]) + releases = req.json()["names"] + releases = sorted(releases, reverse=True) + return releases + + +def get_release_builds(release, branch): + url = f"{_get_balrog_api_root(branch)}/releases/{release}" + req = _retry_on_http_errors(url=url, verify=True, params=None, errors=[500]) + return req.json() + + +def _get_balrog_api_root(branch): + # Query into the scopes scriptworker uses to make sure we check against the same balrog server + # That our jobs would use. + scope = None + for alias, projects in BALROG_SCOPE_ALIAS_TO_PROJECT: + if branch in projects and alias in BALROG_SERVER_SCOPES: + scope = BALROG_SERVER_SCOPES[alias] + break + else: + scope = BALROG_SERVER_SCOPES["default"] + + if scope == "balrog:server:dep": + return "https://stage.balrog.nonprod.cloudops.mozgcp.net/api/v1" + return "https://aus5.mozilla.org/api/v1" + + +def find_localtest(fileUrls): + for channel in fileUrls: + if "-localtest" in channel: + return channel + + +def populate_release_history( + product, branch, maxbuilds=4, maxsearch=10, partial_updates=None +): + # Assuming we are using release branches when we know the list of previous + # releases in advance + if partial_updates is not None: + return _populate_release_history( + product, branch, partial_updates=partial_updates + ) + return _populate_nightly_history( + product, branch, maxbuilds=maxbuilds, maxsearch=maxsearch + ) + + +def _populate_nightly_history(product, branch, maxbuilds=4, maxsearch=10): + """Find relevant releases in Balrog + Not all releases have all platforms and locales, due + to Taskcluster migration. + + Args: + product (str): capitalized product name, AKA appName, e.g. Firefox + branch (str): branch name (mozilla-central) + maxbuilds (int): Maximum number of historical releases to populate + maxsearch(int): Traverse at most this many releases, to avoid + working through the entire history. + Returns: + json object based on data from balrog api + + results = { + 'platform1': { + 'locale1': { + 'buildid1': mar_url, + 'buildid2': mar_url, + 'buildid3': mar_url, + }, + 'locale2': { + 'target.partial-1.mar': {'buildid1': 'mar_url'}, + } + }, + 'platform2': { + } + } + """ + last_releases = get_sorted_releases(product, branch) + + partial_mar_tmpl = "target.partial-{}.mar" + + builds = dict() + for release in last_releases[:maxsearch]: + # maxbuilds in all categories, don't make any more queries + full = len(builds) > 0 and all( + len(builds[platform][locale]) >= maxbuilds + for platform in builds + for locale in builds[platform] + ) + if full: + break + history = get_release_builds(release, branch) + + for platform in history["platforms"]: + if "alias" in history["platforms"][platform]: + continue + if platform not in builds: + builds[platform] = dict() + for locale in history["platforms"][platform]["locales"]: + if locale not in builds[platform]: + builds[platform][locale] = dict() + if len(builds[platform][locale]) >= maxbuilds: + continue + buildid = history["platforms"][platform]["locales"][locale]["buildID"] + url = history["platforms"][platform]["locales"][locale]["completes"][0][ + "fileUrl" + ] + nextkey = len(builds[platform][locale]) + 1 + builds[platform][locale][partial_mar_tmpl.format(nextkey)] = { + "buildid": buildid, + "mar_url": url, + } + return builds + + +def _populate_release_history(product, branch, partial_updates): + builds = dict() + for version, release in partial_updates.items(): + prev_release_blob = "{product}-{version}-build{build_number}".format( + product=product, version=version, build_number=release["buildNumber"] + ) + partial_mar_key = f"target-{version}.partial.mar" + history = get_release_builds(prev_release_blob, branch) + # use one of the localtest channels to avoid relying on bouncer + localtest = find_localtest(history["fileUrls"]) + url_pattern = history["fileUrls"][localtest]["completes"]["*"] + + for platform in history["platforms"]: + if "alias" in history["platforms"][platform]: + continue + if platform not in builds: + builds[platform] = dict() + for locale in history["platforms"][platform]["locales"]: + if locale not in builds[platform]: + builds[platform][locale] = dict() + buildid = history["platforms"][platform]["locales"][locale]["buildID"] + url = url_pattern.replace( + "%OS_FTP%", FTP_PLATFORM_MAP[platform] + ).replace("%LOCALE%", locale) + builds[platform][locale][partial_mar_key] = { + "buildid": buildid, + "mar_url": url, + "previousVersion": version, + "previousBuildNumber": release["buildNumber"], + "product": product, + } + return builds diff --git a/taskcluster/gecko_taskgraph/util/partners.py b/taskcluster/gecko_taskgraph/util/partners.py new file mode 100644 index 0000000000..2546e1ae88 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/partners.py @@ -0,0 +1,555 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import logging +import os +import xml.etree.ElementTree as ET +from urllib.parse import urlencode + +import requests +import yaml +from redo import retry +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.copy_task import copy_task + +# Suppress chatty requests logging +logging.getLogger("requests").setLevel(logging.WARNING) + +log = logging.getLogger(__name__) + +GITHUB_API_ENDPOINT = "https://api.github.com/graphql" + +""" +LOGIN_QUERY, MANIFEST_QUERY, and REPACK_CFG_QUERY are all written to the Github v4 API, +which users GraphQL. See https://developer.github.com/v4/ +""" + +LOGIN_QUERY = """query { + viewer { + login + name + } +} +""" + +# Returns the contents of default.xml from a manifest repository +MANIFEST_QUERY = """query { + repository(owner:"%(owner)s", name:"%(repo)s") { + object(expression: "master:%(file)s") { + ... on Blob { + text + } + } + } +} +""" +# Example response: +# { +# "data": { +# "repository": { +# "object": { +# "text": "<?xml version=\"1.0\" ?>\n<manifest>\n " + +# "<remote fetch=\"git@github.com:mozilla-partners/\" name=\"mozilla-partners\"/>\n " + +# "<remote fetch=\"git@github.com:mozilla/\" name=\"mozilla\"/>\n\n " + +# "<project name=\"repack-scripts\" path=\"scripts\" remote=\"mozilla-partners\" " + +# "revision=\"master\"/>\n <project name=\"build-tools\" path=\"scripts/tools\" " + +# "remote=\"mozilla\" revision=\"master\"/>\n <project name=\"mozilla-EME-free\" " + +# "path=\"partners/mozilla-EME-free\" remote=\"mozilla-partners\" " + +# "revision=\"master\"/>\n</manifest>\n" +# } +# } +# } +# } + +# Returns the contents of desktop/*/repack.cfg for a partner repository +REPACK_CFG_QUERY = """query{ + repository(owner:"%(owner)s", name:"%(repo)s") { + object(expression: "%(revision)s:desktop/"){ + ... on Tree { + entries { + name + object { + ... on Tree { + entries { + name + object { + ... on Blob { + text + } + } + } + } + } + } + } + } + } +} +""" +# Example response: +# { +# "data": { +# "repository": { +# "object": { +# "entries": [ +# { +# "name": "mozilla-EME-free", +# "object": { +# "entries": [ +# { +# "name": "distribution", +# "object": {} +# }, +# { +# "name": "repack.cfg", +# "object": { +# "text": "aus=\"mozilla-EMEfree\"\ndist_id=\"mozilla-EMEfree\"\n" + +# "dist_version=\"1.0\"\nlinux-i686=true\nlinux-x86_64=true\n" + +# " locales=\"ach af de en-US\"\nmac=true\nwin32=true\nwin64=true\n" + +# "output_dir=\"%(platform)s-EME-free/%(locale)s\"\n\n" + +# "# Upload params\nbucket=\"net-mozaws-prod-delivery-firefox\"\n" + +# "upload_to_candidates=true\n" +# } +# } +# ] +# } +# } +# ] +# } +# } +# } +# } + +# Map platforms in repack.cfg into their equivalents in taskcluster +TC_PLATFORM_PER_FTP = { + "linux-i686": "linux-shippable", + "linux-x86_64": "linux64-shippable", + "mac": "macosx64-shippable", + "win32": "win32-shippable", + "win64": "win64-shippable", + "win64-aarch64": "win64-aarch64-shippable", +} + +TASKCLUSTER_PROXY_SECRET_ROOT = "http://taskcluster/secrets/v1/secret" + +LOCALES_FILE = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), + "browser", + "locales", + "l10n-changesets.json", +) + +# cache data at the module level +partner_configs = {} + + +def get_token(params): + """We use a Personal Access Token from Github to lookup partner config. No extra scopes are + needed on the token to read public repositories, but need the 'repo' scope to see private + repositories. This is not fine grained and also grants r/w access, but is revoked at the repo + level. + """ + + # Allow for local taskgraph debugging + if os.environ.get("GITHUB_API_TOKEN"): + return os.environ["GITHUB_API_TOKEN"] + + # The 'usual' method - via taskClusterProxy for decision tasks + url = "{secret_root}/project/releng/gecko/build/level-{level}/partner-github-api".format( + secret_root=TASKCLUSTER_PROXY_SECRET_ROOT, **params + ) + try: + resp = retry( + requests.get, + attempts=2, + sleeptime=10, + args=(url,), + kwargs={"timeout": 60, "headers": ""}, + ) + j = resp.json() + return j["secret"]["key"] + except (requests.ConnectionError, ValueError, KeyError): + raise RuntimeError("Could not get Github API token to lookup partner data") + + +def query_api(query, token): + """Make a query with a Github auth header, returning the json""" + headers = {"Authorization": "bearer %s" % token} + r = requests.post(GITHUB_API_ENDPOINT, json={"query": query}, headers=headers) + r.raise_for_status() + + j = r.json() + if "errors" in j: + raise RuntimeError("Github query error - %s", j["errors"]) + return j + + +def check_login(token): + log.debug("Checking we have a valid login") + query_api(LOGIN_QUERY, token) + + +def get_repo_params(repo): + """Parse the organisation and repo name from an https or git url for a repo""" + if repo.startswith("https"): + # eg https://github.com/mozilla-partners/mozilla-EME-free + return repo.rsplit("/", 2)[-2:] + if repo.startswith("git@"): + # eg git@github.com:mozilla-partners/mailru.git + repo = repo.replace(".git", "") + return repo.split(":")[-1].split("/") + + +def get_partners(manifestRepo, token): + """Given the url to a manifest repository, retrieve the default.xml and parse it into a + list of partner repos. + """ + log.debug("Querying for manifest default.xml in %s", manifestRepo) + owner, repo = get_repo_params(manifestRepo) + query = MANIFEST_QUERY % {"owner": owner, "repo": repo, "file": "default.xml"} + raw_manifest = query_api(query, token) + log.debug("Raw manifest: %s", raw_manifest) + if not raw_manifest["data"]["repository"]: + raise RuntimeError( + "Couldn't load partner manifest at %s, insufficient permissions ?" + % manifestRepo + ) + e = ET.fromstring(raw_manifest["data"]["repository"]["object"]["text"]) + + remotes = {} + partners = {} + for child in e: + if child.tag == "remote": + name = child.attrib["name"] + url = child.attrib["fetch"] + remotes[name] = url + log.debug("Added remote %s at %s", name, url) + elif child.tag == "project": + # we don't need to check any code repos + if "scripts" in child.attrib["path"]: + continue + owner, _ = get_repo_params(remotes[child.attrib["remote"]] + "_") + partner_url = { + "owner": owner, + "repo": child.attrib["name"], + "revision": child.attrib["revision"], + } + partners[child.attrib["name"]] = partner_url + log.debug( + "Added partner %s at revision %s" + % (partner_url["repo"], partner_url["revision"]) + ) + return partners + + +def parse_config(data): + """Parse a single repack.cfg file into a python dictionary. + data is contents of the file, in "foo=bar\nbaz=buzz" style. We do some translation on + locales and platforms data, otherwise passthrough + """ + ALLOWED_KEYS = ( + "locales", + "platforms", + "upload_to_candidates", + "repack_stub_installer", + "publish_to_releases", + ) + config = {"platforms": []} + for l in data.splitlines(): + if "=" in l: + l = str(l) + key, value = l.split("=", 1) + value = value.strip("'\"").rstrip("'\"") + if key in TC_PLATFORM_PER_FTP.keys(): + if value.lower() == "true": + config["platforms"].append(TC_PLATFORM_PER_FTP[key]) + continue + if key not in ALLOWED_KEYS: + continue + if key == "locales": + # a list please + value = value.split(" ") + config[key] = value + return config + + +def get_repack_configs(repackRepo, token): + """For a partner repository, retrieve all the repack.cfg files and parse them into a dict""" + log.debug("Querying for configs in %s", repackRepo) + query = REPACK_CFG_QUERY % repackRepo + raw_configs = query_api(query, token) + raw_configs = raw_configs["data"]["repository"]["object"]["entries"] + + configs = {} + for sub_config in raw_configs: + name = sub_config["name"] + for file in sub_config["object"].get("entries", []): + if file["name"] != "repack.cfg": + continue + configs[name] = parse_config(file["object"]["text"]) + return configs + + +def get_attribution_config(manifestRepo, token): + log.debug("Querying for manifest attribution_config.yml in %s", manifestRepo) + owner, repo = get_repo_params(manifestRepo) + query = MANIFEST_QUERY % { + "owner": owner, + "repo": repo, + "file": "attribution_config.yml", + } + raw_manifest = query_api(query, token) + if not raw_manifest["data"]["repository"]: + raise RuntimeError( + "Couldn't load partner manifest at %s, insufficient permissions ?" + % manifestRepo + ) + # no file has been set up, gracefully continue + if raw_manifest["data"]["repository"]["object"] is None: + log.debug("No attribution_config.yml file found") + return {} + + return yaml.safe_load(raw_manifest["data"]["repository"]["object"]["text"]) + + +def get_partner_config_by_url(manifest_url, kind, token, partner_subset=None): + """Retrieve partner data starting from the manifest url, which points to a repository + containing a default.xml that is intended to be drive the Google tool 'repo'. It + descends into each partner repo to lookup and parse the repack.cfg file(s). + + If partner_subset is a list of sub_config names only return data for those. + + Supports caching data by kind to avoid repeated requests, relying on the related kinds for + partner repacking, signing, repackage, repackage signing all having the same kind prefix. + """ + if not manifest_url: + raise RuntimeError(f"Manifest url for {kind} not defined") + if kind not in partner_configs: + log.info("Looking up data for %s from %s", kind, manifest_url) + check_login(token) + if kind == "release-partner-attribution": + partner_configs[kind] = get_attribution_config(manifest_url, token) + else: + partners = get_partners(manifest_url, token) + + partner_configs[kind] = {} + for partner, partner_url in partners.items(): + if partner_subset and partner not in partner_subset: + continue + partner_configs[kind][partner] = get_repack_configs(partner_url, token) + + return partner_configs[kind] + + +def check_if_partners_enabled(config, tasks): + if ( + ( + config.params["release_enable_partner_repack"] + and config.kind.startswith("release-partner-repack") + ) + or ( + config.params["release_enable_partner_attribution"] + and config.kind.startswith("release-partner-attribution") + ) + or ( + config.params["release_enable_emefree"] + and config.kind.startswith("release-eme-free-") + ) + ): + yield from tasks + + +def get_partner_config_by_kind(config, kind): + """Retrieve partner data starting from the manifest url, which points to a repository + containing a default.xml that is intended to be drive the Google tool 'repo'. It + descends into each partner repo to lookup and parse the repack.cfg file(s). + + Supports caching data by kind to avoid repeated requests, relying on the related kinds for + partner repacking, signing, repackage, repackage signing all having the same kind prefix. + """ + partner_subset = config.params["release_partners"] + partner_configs = config.params["release_partner_config"] or {} + + # TODO eme-free should be a partner; we shouldn't care about per-kind + for k in partner_configs: + if kind.startswith(k): + kind_config = partner_configs[k] + break + else: + return {} + # if we're only interested in a subset of partners we remove the rest + if partner_subset: + if kind.startswith("release-partner-repack"): + # TODO - should be fatal to have an unknown partner in partner_subset + for partner in [p for p in kind_config.keys() if p not in partner_subset]: + del kind_config[partner] + elif kind.startswith("release-partner-attribution") and isinstance( + kind_config, dict + ): + all_configs = copy_task(kind_config.get("configs", [])) + kind_config["configs"] = [] + for this_config in all_configs: + if this_config["campaign"] in partner_subset: + kind_config["configs"].append(this_config) + return kind_config + + +def _fix_subpartner_locales(orig_config, all_locales): + subpartner_config = copy_task(orig_config) + # Get an ordered list of subpartner locales that is a subset of all_locales + subpartner_config["locales"] = sorted( + list(set(orig_config["locales"]) & set(all_locales)) + ) + return subpartner_config + + +def fix_partner_config(orig_config): + pc = {} + with open(LOCALES_FILE) as fh: + all_locales = list(json.load(fh).keys()) + # l10n-changesets.json doesn't include en-US, but the repack list does + if "en-US" not in all_locales: + all_locales.append("en-US") + for kind, kind_config in orig_config.items(): + if kind == "release-partner-attribution": + pc[kind] = {} + if kind_config: + pc[kind] = {"defaults": kind_config["defaults"]} + for config in kind_config["configs"]: + # Make sure our locale list is a subset of all_locales + pc[kind].setdefault("configs", []).append( + _fix_subpartner_locales(config, all_locales) + ) + else: + for partner, partner_config in kind_config.items(): + for subpartner, subpartner_config in partner_config.items(): + # get rid of empty subpartner configs + if not subpartner_config: + continue + # Make sure our locale list is a subset of all_locales + pc.setdefault(kind, {}).setdefault(partner, {})[ + subpartner + ] = _fix_subpartner_locales(subpartner_config, all_locales) + return pc + + +# seems likely this exists elsewhere already +def get_ftp_platform(platform): + if platform.startswith("win32"): + return "win32" + if platform.startswith("win64-aarch64"): + return "win64-aarch64" + if platform.startswith("win64"): + return "win64" + if platform.startswith("macosx"): + return "mac" + if platform.startswith("linux-"): + return "linux-i686" + if platform.startswith("linux64"): + return "linux-x86_64" + raise ValueError(f"Unimplemented platform {platform}") + + +# Ugh +def locales_per_build_platform(build_platform, locales): + if build_platform.startswith("mac"): + exclude = ["ja"] + else: + exclude = ["ja-JP-mac"] + return [locale for locale in locales if locale not in exclude] + + +def get_partner_url_config(parameters, graph_config): + partner_url_config = copy_task(graph_config["partner-urls"]) + substitutions = { + "release-product": parameters["release_product"], + "release-level": release_level(parameters["project"]), + "release-type": parameters["release_type"], + } + resolve_keyed_by( + partner_url_config, + "release-eme-free-repack", + "eme-free manifest_url", + **substitutions, + ) + resolve_keyed_by( + partner_url_config, + "release-partner-repack", + "partner manifest url", + **substitutions, + ) + resolve_keyed_by( + partner_url_config, + "release-partner-attribution", + "partner attribution url", + **substitutions, + ) + return partner_url_config + + +def get_repack_ids_by_platform(config, build_platform): + partner_config = get_partner_config_by_kind(config, config.kind) + combinations = [] + for partner, subconfigs in partner_config.items(): + for sub_config_name, sub_config in subconfigs.items(): + if build_platform not in sub_config.get("platforms", []): + continue + locales = locales_per_build_platform( + build_platform, sub_config.get("locales", []) + ) + for locale in locales: + combinations.append(f"{partner}/{sub_config_name}/{locale}") + return sorted(combinations) + + +def get_partners_to_be_published(config): + # hardcoded kind because release-bouncer-aliases doesn't match otherwise + partner_config = get_partner_config_by_kind(config, "release-partner-repack") + partners = [] + for partner, subconfigs in partner_config.items(): + for sub_config_name, sub_config in subconfigs.items(): + if sub_config.get("publish_to_releases"): + partners.append((partner, sub_config_name, sub_config["platforms"])) + return partners + + +def apply_partner_priority(config, jobs): + priority = None + # Reduce the priority of the partner repack jobs because they don't block QE. Meanwhile + # leave EME-free jobs alone because they do, and they'll get the branch priority like the rest + # of the release. Only bother with this in production, not on staging releases on try. + # medium is the same as mozilla-central, see taskcluster/ci/config.yml. ie higher than + # integration branches because we don't want to wait a lot for the graph to be done, but + # for multiple releases the partner tasks always wait for non-partner. + if ( + config.kind.startswith( + ("release-partner-repack", "release-partner-attribution") + ) + and release_level(config.params["project"]) == "production" + ): + priority = "medium" + for job in jobs: + if priority: + job["priority"] = priority + yield job + + +def generate_attribution_code(defaults, partner): + params = { + "medium": defaults["medium"], + "source": defaults["source"], + "campaign": partner["campaign"], + "content": partner["content"], + } + if partner.get("variation"): + params["variation"] = partner["variation"] + if partner.get("experiment"): + params["experiment"] = partner["experiment"] + + code = urlencode(params) + return code diff --git a/taskcluster/gecko_taskgraph/util/perfile.py b/taskcluster/gecko_taskgraph/util/perfile.py new file mode 100644 index 0000000000..4e82d87dad --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/perfile.py @@ -0,0 +1,104 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import itertools +import json +import logging +import math + +import taskgraph +from mozbuild.util import memoize +from mozpack.path import match as mozpackmatch + +from gecko_taskgraph import files_changed + +from .. import GECKO + +logger = logging.getLogger(__name__) + + +@memoize +def perfile_number_of_chunks(is_try, try_task_config, head_repository, head_rev, type): + if taskgraph.fast and not is_try: + # When iterating on taskgraph changes, the exact number of chunks that + # test-verify runs usually isn't important, so skip it when going fast. + return 3 + tests_per_chunk = 10.0 + if type.startswith("test-coverage"): + tests_per_chunk = 30.0 + + if type.startswith("test-verify-wpt") or type.startswith("test-coverage-wpt"): + file_patterns = [ + "testing/web-platform/tests/**", + "testing/web-platform/mozilla/tests/**", + ] + elif type.startswith("test-verify-gpu") or type.startswith("test-coverage-gpu"): + file_patterns = [ + "**/*webgl*/**/test_*", + "**/dom/canvas/**/test_*", + "**/gfx/tests/**/test_*", + "**/devtools/canvasdebugger/**/browser_*", + "**/reftest*/**", + ] + elif type.startswith("test-verify") or type.startswith("test-coverage"): + file_patterns = [ + "**/test_*", + "**/browser_*", + "**/crashtest*/**", + "js/src/tests/test/**", + "js/src/tests/non262/**", + "js/src/tests/test262/**", + ] + else: + # Returning 0 means no tests to run, this captures non test-verify tasks + return 1 + + changed_files = set() + if try_task_config: + suite_to_paths = json.loads(try_task_config) + specified_files = itertools.chain.from_iterable(suite_to_paths.values()) + changed_files.update(specified_files) + + if is_try: + changed_files.update(files_changed.get_locally_changed_files(GECKO)) + else: + changed_files.update(files_changed.get_changed_files(head_repository, head_rev)) + + test_count = 0 + for pattern in file_patterns: + for path in changed_files: + # TODO: consider running tests if a manifest changes + if path.endswith(".list") or path.endswith(".ini"): + continue + if path.endswith("^headers^"): + continue + + if mozpackmatch(path, pattern): + gpu = False + if type == "test-verify-e10s" or type == "test-coverage-e10s": + # file_patterns for test-verify will pick up some gpu tests, lets ignore + # in the case of reftest, we will not have any in the regular case + gpu_dirs = [ + "dom/canvas", + "gfx/tests", + "devtools/canvasdebugger", + "webgl", + ] + for gdir in gpu_dirs: + if len(path.split(gdir)) > 1: + gpu = True + + if not gpu: + test_count += 1 + + chunks = test_count / tests_per_chunk + chunks = int(math.ceil(chunks)) + + # Never return 0 chunks on try, so that per-file tests can be pushed to try with + # an explicit path, and also so "empty" runs can be checked on try. + if is_try and chunks == 0: + chunks = 1 + + return chunks diff --git a/taskcluster/gecko_taskgraph/util/platforms.py b/taskcluster/gecko_taskgraph/util/platforms.py new file mode 100644 index 0000000000..2c423223fe --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/platforms.py @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import re + +from taskgraph.util.attributes import keymatch + +# platform family is extracted from build platform by taking the alphabetic prefix +# and then translating win -> windows +_platform_re = re.compile(r"^[a-z]*") +_renames = {"win": "windows"} + + +_archive_formats = { + "linux": ".tar.bz2", + "macosx": ".tar.gz", + "windows": ".zip", +} + +_executable_extension = { + "linux": "", + "macosx": "", + "windows": ".exe", +} + +_architectures = { + r"linux\b.*": "x86", + r"linux64\b.*": "x86_64", + r"macosx64\b.*": "macos-x86_64-aarch64", + r"win32\b.*": "x86", + r"win64\b(?!-aarch64).*": "x86_64", + r"win64-aarch64\b.*": "aarch64", +} + + +def platform_family(build_platform): + """Given a build platform, return the platform family (linux, macosx, etc.)""" + family = _platform_re.match(build_platform).group(0) + return _renames.get(family, family) + + +def archive_format(build_platform): + """Given a build platform, return the archive format used on the platform.""" + return _archive_formats[platform_family(build_platform)] + + +def executable_extension(build_platform): + """Given a build platform, return the executable extension used on the platform.""" + return _executable_extension[platform_family(build_platform)] + + +def architecture(build_platform): + matches = keymatch(_architectures, build_platform) + if len(matches) == 1: + return matches[0] + raise Exception(f"Could not determine architecture of platform `{build_platform}`.") diff --git a/taskcluster/gecko_taskgraph/util/scriptworker.py b/taskcluster/gecko_taskgraph/util/scriptworker.py new file mode 100644 index 0000000000..3f1f64d9f8 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/scriptworker.py @@ -0,0 +1,859 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +"""Make scriptworker.cot.verify more user friendly by making scopes dynamic. + +Scriptworker uses certain scopes to determine which sets of credentials to use. +Certain scopes are restricted by branch in chain of trust verification, and are +checked again at the script level. This file provides functions to adjust +these scopes automatically by project; this makes pushing to try, forking a +project branch, and merge day uplifts more user friendly. + +In the future, we may adjust scopes by other settings as well, e.g. different +scopes for `push-to-candidates` rather than `push-to-releases`, even if both +happen on mozilla-beta and mozilla-release. + +Additional configuration is found in the :ref:`graph config <taskgraph-graph-config>`. +""" +import functools +import itertools +import json +import os +from datetime import datetime + +import jsone +from mozbuild.util import memoize +from taskgraph.util.schema import resolve_keyed_by +from taskgraph.util.taskcluster import get_artifact_prefix +from taskgraph.util.yaml import load_yaml + +from gecko_taskgraph.util.copy_task import copy_task + +# constants {{{1 +"""Map signing scope aliases to sets of projects. + +Currently m-c and DevEdition on m-b use nightly signing; Beta on m-b and m-r +use release signing. These data structures aren't set-up to handle different +scopes on the same repo, so we use a different set of them for DevEdition, and +callers are responsible for using the correct one (by calling the appropriate +helper below). More context on this in https://bugzilla.mozilla.org/show_bug.cgi?id=1358601. + +We will need to add esr support at some point. Eventually we want to add +nuance so certain m-b and m-r tasks use dep or nightly signing, and we only +release sign when we have a signed-off set of candidate builds. This current +approach works for now, though. + +This is a list of list-pairs, for ordering. +""" +SIGNING_SCOPE_ALIAS_TO_PROJECT = [ + [ + "all-nightly-branches", + { + "mozilla-central", + "comm-central", + }, + ], + [ + "all-release-branches", + { + "mozilla-beta", + "mozilla-release", + "mozilla-esr102", + "mozilla-esr115", + "comm-beta", + "comm-esr102", + "comm-esr115", + }, + ], +] + +"""Map the signing scope aliases to the actual scopes. +""" +SIGNING_CERT_SCOPES = { + "all-release-branches": "signing:cert:release-signing", + "all-nightly-branches": "signing:cert:nightly-signing", + "default": "signing:cert:dep-signing", +} + +DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT = [ + [ + "beta", + { + "mozilla-beta", + }, + ] +] + +DEVEDITION_SIGNING_CERT_SCOPES = { + "beta": "signing:cert:nightly-signing", + "default": "signing:cert:dep-signing", +} + +"""Map beetmover scope aliases to sets of projects. +""" +BEETMOVER_SCOPE_ALIAS_TO_PROJECT = [ + [ + "all-nightly-branches", + { + "mozilla-central", + "comm-central", + "oak", + }, + ], + [ + "all-release-branches", + { + "mozilla-beta", + "mozilla-release", + "mozilla-esr102", + "mozilla-esr115", + "comm-beta", + "comm-esr102", + "comm-esr115", + }, + ], +] + +"""Map the beetmover scope aliases to the actual scopes. +""" +BEETMOVER_BUCKET_SCOPES = { + "all-release-branches": "beetmover:bucket:release", + "all-nightly-branches": "beetmover:bucket:nightly", + "default": "beetmover:bucket:dep", +} + +"""Map the beetmover scope aliases to the actual scopes. +These are the scopes needed to import artifacts into the product delivery APT repos. +""" +BEETMOVER_APT_REPO_SCOPES = { + "all-release-branches": "beetmover:apt-repo:release", + "all-nightly-branches": "beetmover:apt-repo:nightly", + "default": "beetmover:apt-repo:dep", +} + +"""Map the beetmover tasks aliases to the actual action scopes. +""" +BEETMOVER_ACTION_SCOPES = { + "nightly": "beetmover:action:push-to-nightly", + "nightly-oak": "beetmover:action:push-to-nightly", + "default": "beetmover:action:push-to-candidates", +} + +"""Map the beetmover tasks aliases to the actual action scopes. +The action scopes are generic across different repo types. +""" +BEETMOVER_REPO_ACTION_SCOPES = { + "default": "beetmover:action:import-from-gcs-to-artifact-registry", +} + +"""Known balrog actions.""" +BALROG_ACTIONS = ( + "submit-locale", + "submit-toplevel", + "schedule", + "v2-submit-locale", + "v2-submit-toplevel", +) + +"""Map balrog scope aliases to sets of projects. + +This is a list of list-pairs, for ordering. +""" +BALROG_SCOPE_ALIAS_TO_PROJECT = [ + [ + "nightly", + { + "mozilla-central", + "comm-central", + "oak", + }, + ], + [ + "beta", + { + "mozilla-beta", + "comm-beta", + }, + ], + [ + "release", + { + "mozilla-release", + "comm-esr102", + "comm-esr115", + }, + ], + [ + "esr102", + { + "mozilla-esr102", + }, + ], + [ + "esr115", + { + "mozilla-esr115", + }, + ], +] + +"""Map the balrog scope aliases to the actual scopes. +""" +BALROG_SERVER_SCOPES = { + "nightly": "balrog:server:nightly", + "aurora": "balrog:server:aurora", + "beta": "balrog:server:beta", + "release": "balrog:server:release", + "esr102": "balrog:server:esr", + "esr115": "balrog:server:esr", + "default": "balrog:server:dep", +} + + +""" The list of the release promotion phases which we send notifications for +""" +RELEASE_NOTIFICATION_PHASES = ("promote", "push", "ship") + + +def add_scope_prefix(config, scope): + """ + Prepends the scriptworker scope prefix from the :ref:`graph config + <taskgraph-graph-config>`. + + Args: + config (TransformConfig): The configuration for the kind being transformed. + scope (string): The suffix of the scope + + Returns: + string: the scope to use. + """ + return "{prefix}:{scope}".format( + prefix=config.graph_config["scriptworker"]["scope-prefix"], + scope=scope, + ) + + +def with_scope_prefix(f): + """ + Wraps a function, calling :py:func:`add_scope_prefix` on the result of + calling the wrapped function. + + Args: + f (callable): A function that takes a ``config`` and some keyword + arguments, and returns a scope suffix. + + Returns: + callable: the wrapped function + """ + + @functools.wraps(f) + def wrapper(config, **kwargs): + scope_or_scopes = f(config, **kwargs) + if isinstance(scope_or_scopes, list): + return map(functools.partial(add_scope_prefix, config), scope_or_scopes) + return add_scope_prefix(config, scope_or_scopes) + + return wrapper + + +# scope functions {{{1 +@with_scope_prefix +def get_scope_from_project(config, alias_to_project_map, alias_to_scope_map): + """Determine the restricted scope from `config.params['project']`. + + Args: + config (TransformConfig): The configuration for the kind being transformed. + alias_to_project_map (list of lists): each list pair contains the + alias and the set of projects that match. This is ordered. + alias_to_scope_map (dict): the alias alias to scope + + Returns: + string: the scope to use. + """ + for alias, projects in alias_to_project_map: + if config.params["project"] in projects and alias in alias_to_scope_map: + return alias_to_scope_map[alias] + return alias_to_scope_map["default"] + + +@with_scope_prefix +def get_scope_from_release_type(config, release_type_to_scope_map): + """Determine the restricted scope from `config.params['target_tasks_method']`. + + Args: + config (TransformConfig): The configuration for the kind being transformed. + release_type_to_scope_map (dict): the maps release types to scopes + + Returns: + string: the scope to use. + """ + return release_type_to_scope_map.get( + config.params["release_type"], release_type_to_scope_map["default"] + ) + + +def get_phase_from_target_method(config, alias_to_tasks_map, alias_to_phase_map): + """Determine the phase from `config.params['target_tasks_method']`. + + Args: + config (TransformConfig): The configuration for the kind being transformed. + alias_to_tasks_map (list of lists): each list pair contains the + alias and the set of target methods that match. This is ordered. + alias_to_phase_map (dict): the alias to phase map + + Returns: + string: the phase to use. + """ + for alias, tasks in alias_to_tasks_map: + if ( + config.params["target_tasks_method"] in tasks + and alias in alias_to_phase_map + ): + return alias_to_phase_map[alias] + return alias_to_phase_map["default"] + + +get_signing_cert_scope = functools.partial( + get_scope_from_project, + alias_to_project_map=SIGNING_SCOPE_ALIAS_TO_PROJECT, + alias_to_scope_map=SIGNING_CERT_SCOPES, +) + +get_devedition_signing_cert_scope = functools.partial( + get_scope_from_project, + alias_to_project_map=DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT, + alias_to_scope_map=DEVEDITION_SIGNING_CERT_SCOPES, +) + +get_beetmover_bucket_scope = functools.partial( + get_scope_from_project, + alias_to_project_map=BEETMOVER_SCOPE_ALIAS_TO_PROJECT, + alias_to_scope_map=BEETMOVER_BUCKET_SCOPES, +) + +get_beetmover_apt_repo_scope = functools.partial( + get_scope_from_project, + alias_to_project_map=BEETMOVER_SCOPE_ALIAS_TO_PROJECT, + alias_to_scope_map=BEETMOVER_APT_REPO_SCOPES, +) + +get_beetmover_repo_action_scope = functools.partial( + get_scope_from_release_type, + release_type_to_scope_map=BEETMOVER_REPO_ACTION_SCOPES, +) + +get_beetmover_action_scope = functools.partial( + get_scope_from_release_type, + release_type_to_scope_map=BEETMOVER_ACTION_SCOPES, +) + +get_balrog_server_scope = functools.partial( + get_scope_from_project, + alias_to_project_map=BALROG_SCOPE_ALIAS_TO_PROJECT, + alias_to_scope_map=BALROG_SERVER_SCOPES, +) + +cached_load_yaml = memoize(load_yaml) + + +# release_config {{{1 +def get_release_config(config): + """Get the build number and version for a release task. + + Currently only applies to beetmover tasks. + + Args: + config (TransformConfig): The configuration for the kind being transformed. + + Returns: + dict: containing both `build_number` and `version`. This can be used to + update `task.payload`. + """ + release_config = {} + + partial_updates = os.environ.get("PARTIAL_UPDATES", "") + if partial_updates != "" and config.kind in ( + "release-bouncer-sub", + "release-bouncer-check", + "release-update-verify-config", + "release-secondary-update-verify-config", + "release-balrog-submit-toplevel", + "release-secondary-balrog-submit-toplevel", + ): + partial_updates = json.loads(partial_updates) + release_config["partial_versions"] = ", ".join( + [ + "{}build{}".format(v, info["buildNumber"]) + for v, info in partial_updates.items() + ] + ) + if release_config["partial_versions"] == "{}": + del release_config["partial_versions"] + + release_config["version"] = config.params["version"] + release_config["appVersion"] = config.params["app_version"] + + release_config["next_version"] = config.params["next_version"] + release_config["build_number"] = config.params["build_number"] + return release_config + + +def get_signing_cert_scope_per_platform(build_platform, is_shippable, config): + if "devedition" in build_platform: + return get_devedition_signing_cert_scope(config) + if is_shippable: + return get_signing_cert_scope(config) + return add_scope_prefix(config, "signing:cert:dep-signing") + + +# generate_beetmover_upstream_artifacts {{{1 +def generate_beetmover_upstream_artifacts( + config, job, platform, locale=None, dependencies=None, **kwargs +): + """Generate the upstream artifacts for beetmover, using the artifact map. + + Currently only applies to beetmover tasks. + + Args: + job (dict): The current job being generated + dependencies (list): A list of the job's dependency labels. + platform (str): The current build platform + locale (str): The current locale being beetmoved. + + Returns: + list: A list of dictionaries conforming to the upstream_artifacts spec. + """ + base_artifact_prefix = get_artifact_prefix(job) + resolve_keyed_by( + job, + "attributes.artifact_map", + "artifact map", + **{ + "release-type": config.params["release_type"], + "platform": platform, + }, + ) + map_config = copy_task(cached_load_yaml(job["attributes"]["artifact_map"])) + upstream_artifacts = list() + + if not locale: + locales = map_config["default_locales"] + elif isinstance(locale, list): + locales = locale + else: + locales = [locale] + + if not dependencies: + if job.get("dependencies"): + dependencies = job["dependencies"].keys() + elif job.get("primary-dependency"): + dependencies = [job["primary-dependency"].kind] + else: + raise Exception(f"Unsupported type of dependency. Got job: {job}") + + for locale, dep in itertools.product(locales, dependencies): + paths = list() + + for filename in map_config["mapping"]: + resolve_keyed_by( + map_config["mapping"][filename], + "from", + f"beetmover filename {filename}", + platform=platform, + ) + if dep not in map_config["mapping"][filename]["from"]: + continue + if locale != "en-US" and not map_config["mapping"][filename]["all_locales"]: + continue + if ( + "only_for_platforms" in map_config["mapping"][filename] + and platform + not in map_config["mapping"][filename]["only_for_platforms"] + ): + continue + if ( + "not_for_platforms" in map_config["mapping"][filename] + and platform in map_config["mapping"][filename]["not_for_platforms"] + ): + continue + if "partials_only" in map_config["mapping"][filename]: + continue + # The next time we look at this file it might be a different locale. + file_config = copy_task(map_config["mapping"][filename]) + resolve_keyed_by( + file_config, + "source_path_modifier", + "source path modifier", + locale=locale, + ) + + kwargs["locale"] = locale + + paths.append( + os.path.join( + base_artifact_prefix, + jsone.render(file_config["source_path_modifier"], kwargs), + jsone.render(filename, kwargs), + ) + ) + + if ( + job.get("dependencies") + and getattr(job["dependencies"][dep], "attributes", None) + and job["dependencies"][dep].attributes.get("release_artifacts") + ): + paths = [ + path + for path in paths + if path in job["dependencies"][dep].attributes["release_artifacts"] + ] + + if not paths: + continue + + upstream_artifacts.append( + { + "taskId": {"task-reference": f"<{dep}>"}, + "taskType": map_config["tasktype_map"].get(dep), + "paths": sorted(paths), + "locale": locale, + } + ) + + upstream_artifacts.sort(key=lambda u: u["paths"]) + return upstream_artifacts + + +def generate_artifact_registry_gcs_sources(dep): + gcs_sources = [] + locale = dep.attributes.get("locale") + if not locale: + repackage_deb_reference = "<repackage-deb>" + repackage_deb_artifact = "public/build/target.deb" + else: + repackage_deb_reference = "<repackage-deb-l10n>" + repackage_deb_artifact = f"public/build/{locale}/target.langpack.deb" + for config in dep.task["payload"]["artifactMap"]: + if ( + config["taskId"]["task-reference"] == repackage_deb_reference + and repackage_deb_artifact in config["paths"] + ): + gcs_sources.append( + config["paths"][repackage_deb_artifact]["destinations"][0] + ) + return gcs_sources + + +# generate_beetmover_artifact_map {{{1 +def generate_beetmover_artifact_map(config, job, **kwargs): + """Generate the beetmover artifact map. + + Currently only applies to beetmover tasks. + + Args: + config (): Current taskgraph configuration. + job (dict): The current job being generated + Common kwargs: + platform (str): The current build platform + locale (str): The current locale being beetmoved. + + Returns: + list: A list of dictionaries containing source->destination + maps for beetmover. + """ + platform = kwargs.get("platform", "") + resolve_keyed_by( + job, + "attributes.artifact_map", + job["label"], + **{ + "release-type": config.params["release_type"], + "platform": platform, + }, + ) + map_config = copy_task(cached_load_yaml(job["attributes"]["artifact_map"])) + base_artifact_prefix = map_config.get( + "base_artifact_prefix", get_artifact_prefix(job) + ) + + artifacts = list() + + dependencies = job["dependencies"].keys() + + if kwargs.get("locale"): + if isinstance(kwargs["locale"], list): + locales = kwargs["locale"] + else: + locales = [kwargs["locale"]] + else: + locales = map_config["default_locales"] + + resolve_keyed_by(map_config, "s3_bucket_paths", job["label"], platform=platform) + + for locale, dep in sorted(itertools.product(locales, dependencies)): + paths = dict() + for filename in map_config["mapping"]: + # Relevancy checks + resolve_keyed_by( + map_config["mapping"][filename], "from", "blah", platform=platform + ) + if dep not in map_config["mapping"][filename]["from"]: + # We don't get this file from this dependency. + continue + if locale != "en-US" and not map_config["mapping"][filename]["all_locales"]: + # This locale either doesn't produce or shouldn't upload this file. + continue + if ( + "only_for_platforms" in map_config["mapping"][filename] + and platform + not in map_config["mapping"][filename]["only_for_platforms"] + ): + # This platform either doesn't produce or shouldn't upload this file. + continue + if ( + "not_for_platforms" in map_config["mapping"][filename] + and platform in map_config["mapping"][filename]["not_for_platforms"] + ): + # This platform either doesn't produce or shouldn't upload this file. + continue + if "partials_only" in map_config["mapping"][filename]: + continue + + # copy_task because the next time we look at this file the locale will differ. + file_config = copy_task(map_config["mapping"][filename]) + + for field in [ + "destinations", + "locale_prefix", + "source_path_modifier", + "update_balrog_manifest", + "pretty_name", + "checksums_path", + ]: + resolve_keyed_by( + file_config, field, job["label"], locale=locale, platform=platform + ) + + # This format string should ideally be in the configuration file, + # but this would mean keeping variable names in sync between code + config. + destinations = [ + "{s3_bucket_path}/{dest_path}/{locale_prefix}{filename}".format( + s3_bucket_path=bucket_path, + dest_path=dest_path, + locale_prefix=file_config["locale_prefix"], + filename=file_config.get("pretty_name", filename), + ) + for dest_path, bucket_path in itertools.product( + file_config["destinations"], map_config["s3_bucket_paths"] + ) + ] + # Creating map entries + # Key must be artifact path, to avoid trampling duplicates, such + # as public/build/target.apk and public/build/en-US/target.apk + key = os.path.join( + base_artifact_prefix, + file_config["source_path_modifier"], + filename, + ) + + paths[key] = { + "destinations": destinations, + } + if file_config.get("checksums_path"): + paths[key]["checksums_path"] = file_config["checksums_path"] + + # optional flag: balrog manifest + if file_config.get("update_balrog_manifest"): + paths[key]["update_balrog_manifest"] = True + if file_config.get("balrog_format"): + paths[key]["balrog_format"] = file_config["balrog_format"] + + if not paths: + # No files for this dependency/locale combination. + continue + + # Render all variables for the artifact map + platforms = copy_task(map_config.get("platform_names", {})) + if platform: + for key in platforms.keys(): + resolve_keyed_by(platforms, key, job["label"], platform=platform) + + upload_date = datetime.fromtimestamp(config.params["build_date"]) + + kwargs.update( + { + "locale": locale, + "version": config.params["version"], + "branch": config.params["project"], + "build_number": config.params["build_number"], + "year": upload_date.year, + "month": upload_date.strftime("%m"), # zero-pad the month + "upload_date": upload_date.strftime("%Y-%m-%d-%H-%M-%S"), + } + ) + kwargs.update(**platforms) + paths = jsone.render(paths, kwargs) + artifacts.append( + { + "taskId": {"task-reference": f"<{dep}>"}, + "locale": locale, + "paths": paths, + } + ) + + return artifacts + + +# generate_beetmover_partials_artifact_map {{{1 +def generate_beetmover_partials_artifact_map(config, job, partials_info, **kwargs): + """Generate the beetmover partials artifact map. + + Currently only applies to beetmover tasks. + + Args: + config (): Current taskgraph configuration. + job (dict): The current job being generated + partials_info (dict): Current partials and information about them in a dict + Common kwargs: + platform (str): The current build platform + locale (str): The current locale being beetmoved. + + Returns: + list: A list of dictionaries containing source->destination + maps for beetmover. + """ + platform = kwargs.get("platform", "") + resolve_keyed_by( + job, + "attributes.artifact_map", + "artifact map", + **{ + "release-type": config.params["release_type"], + "platform": platform, + }, + ) + map_config = copy_task(cached_load_yaml(job["attributes"]["artifact_map"])) + base_artifact_prefix = map_config.get( + "base_artifact_prefix", get_artifact_prefix(job) + ) + + artifacts = list() + dependencies = job["dependencies"].keys() + + if kwargs.get("locale"): + locales = [kwargs["locale"]] + else: + locales = map_config["default_locales"] + + resolve_keyed_by( + map_config, "s3_bucket_paths", "s3_bucket_paths", platform=platform + ) + + platforms = copy_task(map_config.get("platform_names", {})) + if platform: + for key in platforms.keys(): + resolve_keyed_by(platforms, key, key, platform=platform) + upload_date = datetime.fromtimestamp(config.params["build_date"]) + + for locale, dep in itertools.product(locales, dependencies): + paths = dict() + for filename in map_config["mapping"]: + # Relevancy checks + if dep not in map_config["mapping"][filename]["from"]: + # We don't get this file from this dependency. + continue + if locale != "en-US" and not map_config["mapping"][filename]["all_locales"]: + # This locale either doesn't produce or shouldn't upload this file. + continue + if "partials_only" not in map_config["mapping"][filename]: + continue + # copy_task because the next time we look at this file the locale will differ. + file_config = copy_task(map_config["mapping"][filename]) + + for field in [ + "destinations", + "locale_prefix", + "source_path_modifier", + "update_balrog_manifest", + "from_buildid", + "pretty_name", + "checksums_path", + ]: + resolve_keyed_by( + file_config, field, field, locale=locale, platform=platform + ) + + # This format string should ideally be in the configuration file, + # but this would mean keeping variable names in sync between code + config. + destinations = [ + "{s3_bucket_path}/{dest_path}/{locale_prefix}{filename}".format( + s3_bucket_path=bucket_path, + dest_path=dest_path, + locale_prefix=file_config["locale_prefix"], + filename=file_config.get("pretty_name", filename), + ) + for dest_path, bucket_path in itertools.product( + file_config["destinations"], map_config["s3_bucket_paths"] + ) + ] + # Creating map entries + # Key must be artifact path, to avoid trampling duplicates, such + # as public/build/target.apk and public/build/en-US/target.apk + key = os.path.join( + base_artifact_prefix, + file_config["source_path_modifier"], + filename, + ) + partials_paths = {} + for pname, info in partials_info.items(): + partials_paths[key] = { + "destinations": destinations, + } + if file_config.get("checksums_path"): + partials_paths[key]["checksums_path"] = file_config[ + "checksums_path" + ] + + # optional flag: balrog manifest + if file_config.get("update_balrog_manifest"): + partials_paths[key]["update_balrog_manifest"] = True + if file_config.get("balrog_format"): + partials_paths[key]["balrog_format"] = file_config[ + "balrog_format" + ] + # optional flag: from_buildid + if file_config.get("from_buildid"): + partials_paths[key]["from_buildid"] = file_config["from_buildid"] + + # render buildid + kwargs.update( + { + "partial": pname, + "from_buildid": info["buildid"], + "previous_version": info.get("previousVersion"), + "buildid": str(config.params["moz_build_date"]), + "locale": locale, + "version": config.params["version"], + "branch": config.params["project"], + "build_number": config.params["build_number"], + "year": upload_date.year, + "month": upload_date.strftime("%m"), # zero-pad the month + "upload_date": upload_date.strftime("%Y-%m-%d-%H-%M-%S"), + } + ) + kwargs.update(**platforms) + paths.update(jsone.render(partials_paths, kwargs)) + + if not paths: + continue + + artifacts.append( + { + "taskId": {"task-reference": f"<{dep}>"}, + "locale": locale, + "paths": paths, + } + ) + + artifacts.sort(key=lambda a: sorted(a["paths"].items())) + return artifacts diff --git a/taskcluster/gecko_taskgraph/util/signed_artifacts.py b/taskcluster/gecko_taskgraph/util/signed_artifacts.py new file mode 100644 index 0000000000..2467ff8046 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/signed_artifacts.py @@ -0,0 +1,198 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Defines artifacts to sign before repackage. +""" + +from taskgraph.util.taskcluster import get_artifact_path + +from gecko_taskgraph.util.declarative_artifacts import get_geckoview_upstream_artifacts + +LANGPACK_SIGN_PLATFORMS = { # set + "linux64-shippable", + "linux64-devedition", + "macosx64-shippable", + "macosx64-devedition", +} + + +def is_partner_kind(kind): + if kind and kind.startswith(("release-partner", "release-eme-free")): + return True + + +def is_notarization_kind(kind): + if kind and "notarization" in kind: + return True + + +def is_mac_signing_king(kind): + return kind and "mac-signing" in kind + + +def generate_specifications_of_artifacts_to_sign( + config, job, keep_locale_template=True, kind=None, dep_kind=None +): + build_platform = job["attributes"].get("build_platform") + use_stub = job["attributes"].get("stub-installer") + # Get locales to know if we want to sign ja-JP-mac langpack + locales = job["attributes"].get("chunk_locales", []) + if kind == "release-source-signing": + artifacts_specifications = [ + { + "artifacts": [get_artifact_path(job, "source.tar.xz")], + "formats": ["autograph_gpg"], + } + ] + elif "android" in build_platform: + artifacts_specifications = [ + { + "artifacts": get_geckoview_artifacts_to_sign(config, job), + "formats": ["autograph_gpg"], + } + ] + # XXX: Mars aren't signed here (on any platform) because internals will be + # signed at after this stage of the release + elif "macosx" in build_platform: + langpack_formats = [] + if is_notarization_kind(config.kind): + formats = ["apple_notarization"] + artifacts_specifications = [ + { + "artifacts": [ + get_artifact_path(job, "{locale}/target.tar.gz"), + get_artifact_path(job, "{locale}/target.pkg"), + ], + "formats": formats, + } + ] + else: + # This task is mac-signing + if is_partner_kind(kind): + extension = "tar.gz" + else: + extension = "dmg" + artifacts_specifications = [ + { + "artifacts": [ + get_artifact_path(job, f"{{locale}}/target.{extension}") + ], + "formats": ["macapp", "autograph_widevine", "autograph_omnija"], + } + ] + langpack_formats = ["autograph_langpack"] + + if "ja-JP-mac" in locales and build_platform in LANGPACK_SIGN_PLATFORMS: + artifacts_specifications += [ + { + "artifacts": [ + get_artifact_path(job, "ja-JP-mac/target.langpack.xpi") + ], + "formats": langpack_formats, + } + ] + elif "win" in build_platform: + artifacts_specifications = [ + { + "artifacts": [ + get_artifact_path(job, "{locale}/setup.exe"), + ], + "formats": ["autograph_authenticode_sha2"], + }, + { + "artifacts": [ + get_artifact_path(job, "{locale}/target.zip"), + ], + "formats": [ + "autograph_authenticode_sha2", + "autograph_widevine", + "autograph_omnija", + ], + }, + ] + + if use_stub: + artifacts_specifications[0]["artifacts"] += [ + get_artifact_path(job, "{locale}/setup-stub.exe") + ] + elif "linux" in build_platform: + artifacts_specifications = [ + { + "artifacts": [get_artifact_path(job, "{locale}/target.tar.bz2")], + "formats": ["autograph_gpg", "autograph_widevine", "autograph_omnija"], + } + ] + if build_platform in LANGPACK_SIGN_PLATFORMS: + artifacts_specifications += [ + { + "artifacts": [ + get_artifact_path(job, "{locale}/target.langpack.xpi") + ], + "formats": ["autograph_langpack"], + } + ] + else: + raise Exception("Platform not implemented for signing") + + if not keep_locale_template: + artifacts_specifications = _strip_locale_template(artifacts_specifications) + + if is_partner_kind(kind): + artifacts_specifications = _strip_widevine_for_partners( + artifacts_specifications + ) + + return artifacts_specifications + + +def _strip_locale_template(artifacts_without_locales): + for spec in artifacts_without_locales: + for index, artifact in enumerate(spec["artifacts"]): + stripped_artifact = artifact.format(locale="") + stripped_artifact = stripped_artifact.replace("//", "/") + spec["artifacts"][index] = stripped_artifact + + return artifacts_without_locales + + +def _strip_widevine_for_partners(artifacts_specifications): + """Partner repacks should not resign that's previously signed for fear of breaking partial + updates + """ + for spec in artifacts_specifications: + if "autograph_widevine" in spec["formats"]: + spec["formats"].remove("autograph_widevine") + if "autograph_omnija" in spec["formats"]: + spec["formats"].remove("autograph_omnija") + + return artifacts_specifications + + +def get_signed_artifacts(input, formats, behavior=None): + """ + Get the list of signed artifacts for the given input and formats. + """ + artifacts = set() + if input.endswith(".dmg"): + artifacts.add(input.replace(".dmg", ".tar.gz")) + if behavior and behavior != "mac_sign": + artifacts.add(input.replace(".dmg", ".pkg")) + else: + artifacts.add(input) + if "autograph_gpg" in formats: + artifacts.add(f"{input}.asc") + + return artifacts + + +def get_geckoview_artifacts_to_sign(config, job): + upstream_artifacts = [] + for package in job["attributes"]["maven_packages"]: + upstream_artifacts += get_geckoview_upstream_artifacts(config, job, package) + return [ + path + for upstream_artifact in upstream_artifacts + for path in upstream_artifact["paths"] + if not path.endswith(".md5") and not path.endswith(".sha1") + ] diff --git a/taskcluster/gecko_taskgraph/util/taskcluster.py b/taskcluster/gecko_taskgraph/util/taskcluster.py new file mode 100644 index 0000000000..cddb01fd37 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/taskcluster.py @@ -0,0 +1,128 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging +import os + +import taskcluster_urls as liburls +from taskcluster import Hooks +from taskgraph.util import taskcluster as tc_util +from taskgraph.util.taskcluster import ( + _do_request, + get_index_url, + get_root_url, + get_task_definition, + get_task_url, +) + +logger = logging.getLogger(__name__) + + +def insert_index(index_path, task_id, data=None, use_proxy=False): + index_url = get_index_url(index_path, use_proxy=use_proxy) + + # Find task expiry. + expires = get_task_definition(task_id, use_proxy=use_proxy)["expires"] + + response = _do_request( + index_url, + method="put", + json={ + "taskId": task_id, + "rank": 0, + "data": data or {}, + "expires": expires, + }, + ) + return response + + +def status_task(task_id, use_proxy=False): + """Gets the status of a task given a task_id. + + In testing mode, just logs that it would have retrieved status. + + Args: + task_id (str): A task id. + use_proxy (bool): Whether to use taskcluster-proxy (default: False) + + Returns: + dict: A dictionary object as defined here: + https://docs.taskcluster.net/docs/reference/platform/queue/api#status + """ + if tc_util.testing: + logger.info(f"Would have gotten status for {task_id}.") + else: + resp = _do_request(get_task_url(task_id, use_proxy) + "/status") + status = resp.json().get("status", {}) + return status + + +def state_task(task_id, use_proxy=False): + """Gets the state of a task given a task_id. + + In testing mode, just logs that it would have retrieved state. This is a subset of the + data returned by :func:`status_task`. + + Args: + task_id (str): A task id. + use_proxy (bool): Whether to use taskcluster-proxy (default: False) + + Returns: + str: The state of the task, one of + ``pending, running, completed, failed, exception, unknown``. + """ + if tc_util.testing: + logger.info(f"Would have gotten state for {task_id}.") + else: + status = status_task(task_id, use_proxy=use_proxy).get("state") or "unknown" + return status + + +def trigger_hook(hook_group_id, hook_id, hook_payload): + hooks = Hooks({"rootUrl": get_root_url(True)}) + response = hooks.triggerHook(hook_group_id, hook_id, hook_payload) + + logger.info( + "Task seen here: {}/tasks/{}".format( + get_root_url(os.environ.get("TASKCLUSTER_PROXY_URL")), + response["status"]["taskId"], + ) + ) + + +def list_task_group_tasks(task_group_id): + """Generate the tasks in a task group""" + params = {} + while True: + url = liburls.api( + get_root_url(False), + "queue", + "v1", + f"task-group/{task_group_id}/list", + ) + resp = _do_request(url, method="get", params=params).json() + yield from resp["tasks"] + if resp.get("continuationToken"): + params = {"continuationToken": resp.get("continuationToken")} + else: + break + + +def list_task_group_incomplete_task_ids(task_group_id): + states = ("running", "pending", "unscheduled") + for task in [t["status"] for t in list_task_group_tasks(task_group_id)]: + if task["state"] in states: + yield task["taskId"] + + +def list_task_group_complete_tasks(task_group_id): + tasks = {} + for task in list_task_group_tasks(task_group_id): + if task.get("status", {}).get("state", "") == "completed": + tasks[task.get("task", {}).get("metadata", {}).get("name", "")] = task.get( + "status", {} + ).get("taskId", "") + return tasks diff --git a/taskcluster/gecko_taskgraph/util/taskgraph.py b/taskcluster/gecko_taskgraph/util/taskgraph.py new file mode 100644 index 0000000000..bac7b3fbb8 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/taskgraph.py @@ -0,0 +1,49 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Tools for interacting with existing taskgraphs. +""" + +from taskgraph.util.taskcluster import find_task_id, get_artifact + + +def find_decision_task(parameters, graph_config): + """Given the parameters for this action, find the taskId of the decision + task""" + head_rev_param = "{}head_rev".format(graph_config["project-repo-param-prefix"]) + return find_task_id( + "{}.v2.{}.revision.{}.taskgraph.decision".format( + graph_config["trust-domain"], + parameters["project"], + parameters[head_rev_param], + ) + ) + + +def find_existing_tasks(previous_graph_ids): + existing_tasks = {} + for previous_graph_id in previous_graph_ids: + label_to_taskid = get_artifact(previous_graph_id, "public/label-to-taskid.json") + existing_tasks.update(label_to_taskid) + return existing_tasks + + +def find_existing_tasks_from_previous_kinds( + full_task_graph, previous_graph_ids, rebuild_kinds +): + """Given a list of previous decision/action taskIds and kinds to ignore + from the previous graphs, return a dictionary of labels-to-taskids to use + as ``existing_tasks`` in the optimization step.""" + existing_tasks = find_existing_tasks(previous_graph_ids) + kind_labels = { + t.label + for t in full_task_graph.tasks.values() + if t.attributes["kind"] not in rebuild_kinds + } + return { + label: taskid + for (label, taskid) in existing_tasks.items() + if label in kind_labels + } diff --git a/taskcluster/gecko_taskgraph/util/templates.py b/taskcluster/gecko_taskgraph/util/templates.py new file mode 100644 index 0000000000..e6640a7edd --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/templates.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from gecko_taskgraph.util.copy_task import copy_task + + +def merge_to(source, dest): + """ + Merge dict and arrays (override scalar values) + + Keys from source override keys from dest, and elements from lists in source + are appended to lists in dest. + + :param dict source: to copy from + :param dict dest: to copy to (modified in place) + """ + + for key, value in source.items(): + if ( + isinstance(value, dict) + and len(value) == 1 + and list(value)[0].startswith("by-") + ): + # Do not merge by-* values as this is likely to confuse someone + dest[key] = value + continue + + # Override mismatching or empty types + if type(value) != type(dest.get(key)): # noqa + dest[key] = value + continue + + # Merge dict + if isinstance(value, dict): + merge_to(value, dest[key]) + continue + + if isinstance(value, list): + dest[key] = dest[key] + value + continue + + dest[key] = value + + return dest + + +def merge(*objects): + """ + Merge the given objects, using the semantics described for merge_to, with + objects later in the list taking precedence. From an inheritance + perspective, "parents" should be listed before "children". + + Returns the result without modifying any arguments. + """ + if len(objects) == 1: + return copy_task(objects[0]) + return merge_to(objects[-1], merge(*objects[:-1])) diff --git a/taskcluster/gecko_taskgraph/util/verify.py b/taskcluster/gecko_taskgraph/util/verify.py new file mode 100644 index 0000000000..75ddbda674 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/verify.py @@ -0,0 +1,454 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import logging +import os +import re +import sys + +import attr +from taskgraph.util.treeherder import join_symbol +from taskgraph.util.verify import VerificationSequence + +from gecko_taskgraph import GECKO +from gecko_taskgraph.util.attributes import ( + ALL_PROJECTS, + RELEASE_PROJECTS, + RUN_ON_PROJECT_ALIASES, +) + +logger = logging.getLogger(__name__) +doc_base_path = os.path.join(GECKO, "taskcluster", "docs") + + +verifications = VerificationSequence() + + +@attr.s(frozen=True) +class DocPaths: + _paths = attr.ib(factory=list) + + def get_files(self, filename): + rv = [] + for p in self._paths: + doc_path = os.path.join(p, filename) + if os.path.exists(doc_path): + rv.append(doc_path) + return rv + + def add(self, path): + """ + Projects that make use of Firefox's taskgraph can extend it with + their own task kinds by registering additional paths for documentation. + documentation_paths.add() needs to be called by the project's Taskgraph + registration function. See taskgraph.config. + """ + self._paths.append(path) + + +documentation_paths = DocPaths() +documentation_paths.add(doc_base_path) + + +def verify_docs(filename, identifiers, appearing_as): + """ + Look for identifiers of the type appearing_as in the files + returned by documentation_paths.get_files(). Firefox will have + a single file in a list, but projects such as Thunderbird can have + documentation in another location and may return multiple files. + """ + # We ignore identifiers starting with '_' for the sake of tests. + # Strings starting with "_" are ignored for doc verification + # hence they can be used for faking test values + doc_files = documentation_paths.get_files(filename) + doctext = "".join([open(d).read() for d in doc_files]) + + if appearing_as == "inline-literal": + expression_list = [ + "``" + identifier + "``" + for identifier in identifiers + if not identifier.startswith("_") + ] + elif appearing_as == "heading": + expression_list = [ + "\n" + identifier + "\n(?:(?:(?:-+\n)+)|(?:(?:.+\n)+))" + for identifier in identifiers + if not identifier.startswith("_") + ] + else: + raise Exception(f"appearing_as = `{appearing_as}` not defined") + + for expression, identifier in zip(expression_list, identifiers): + match_group = re.search(expression, doctext) + if not match_group: + raise Exception( + "{}: `{}` missing from doc file: `{}`".format( + appearing_as, identifier, filename + ) + ) + + +@verifications.add("initial") +def verify_run_using(): + from gecko_taskgraph.transforms.job import registry + + verify_docs( + filename="transforms/job.rst", + identifiers=registry.keys(), + appearing_as="inline-literal", + ) + + +@verifications.add("parameters") +def verify_parameters_docs(parameters): + if not parameters.strict: + return + + parameters_dict = dict(**parameters) + verify_docs( + filename="parameters.rst", + identifiers=list(parameters_dict), + appearing_as="inline-literal", + ) + + +@verifications.add("kinds") +def verify_kinds_docs(kinds): + verify_docs(filename="kinds.rst", identifiers=kinds.keys(), appearing_as="heading") + + +@verifications.add("full_task_set") +def verify_attributes(task, taskgraph, scratch_pad, graph_config, parameters): + if task is None: + verify_docs( + filename="attributes.rst", + identifiers=list(scratch_pad["attribute_set"]), + appearing_as="heading", + ) + return + scratch_pad.setdefault("attribute_set", set()).update(task.attributes.keys()) + + +@verifications.add("full_task_graph") +def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config, parameters): + """ + This function verifies that tuple + (collection.keys(), machine.platform, groupSymbol, symbol) is unique + for a target task graph. + """ + if task is None: + return + task_dict = task.task + if "extra" in task_dict: + extra = task_dict["extra"] + if "treeherder" in extra: + treeherder = extra["treeherder"] + + collection_keys = tuple(sorted(treeherder.get("collection", {}).keys())) + if len(collection_keys) != 1: + raise Exception( + "Task {} can't be in multiple treeherder collections " + "(the part of the platform after `/`): {}".format( + task.label, collection_keys + ) + ) + platform = treeherder.get("machine", {}).get("platform") + group_symbol = treeherder.get("groupSymbol") + symbol = treeherder.get("symbol") + + key = (platform, collection_keys[0], group_symbol, symbol) + if key in scratch_pad: + raise Exception( + "Duplicate treeherder platform and symbol in tasks " + "`{}`and `{}`: {} {}".format( + task.label, + scratch_pad[key], + f"{platform}/{collection_keys[0]}", + join_symbol(group_symbol, symbol), + ) + ) + else: + scratch_pad[key] = task.label + + +@verifications.add("full_task_graph") +def verify_trust_domain_v2_routes( + task, taskgraph, scratch_pad, graph_config, parameters +): + """ + This function ensures that any two tasks have distinct ``index.{trust-domain}.v2`` routes. + """ + if task is None: + return + route_prefix = "index.{}.v2".format(graph_config["trust-domain"]) + task_dict = task.task + routes = task_dict.get("routes", []) + + for route in routes: + if route.startswith(route_prefix): + if route in scratch_pad: + raise Exception( + "conflict between {}:{} for route: {}".format( + task.label, scratch_pad[route], route + ) + ) + else: + scratch_pad[route] = task.label + + +@verifications.add("full_task_graph") +def verify_routes_notification_filters( + task, taskgraph, scratch_pad, graph_config, parameters +): + """ + This function ensures that only understood filters for notifications are + specified. + + See: https://firefox-ci-tc.services.mozilla.com/docs/manual/using/task-notifications + """ + if task is None: + return + route_prefix = "notify." + valid_filters = ("on-any", "on-completed", "on-failed", "on-exception") + task_dict = task.task + routes = task_dict.get("routes", []) + + for route in routes: + if route.startswith(route_prefix): + # Get the filter of the route + route_filter = route.split(".")[-1] + if route_filter not in valid_filters: + raise Exception( + "{} has invalid notification filter ({})".format( + task.label, route_filter + ) + ) + + +@verifications.add("full_task_graph") +def verify_dependency_tiers(task, taskgraph, scratch_pad, graph_config, parameters): + tiers = scratch_pad + if task is not None: + tiers[task.label] = ( + task.task.get("extra", {}).get("treeherder", {}).get("tier", sys.maxsize) + ) + else: + + def printable_tier(tier): + if tier == sys.maxsize: + return "unknown" + return tier + + for task in taskgraph.tasks.values(): + tier = tiers[task.label] + for d in task.dependencies.values(): + if taskgraph[d].task.get("workerType") == "always-optimized": + continue + if "dummy" in taskgraph[d].kind: + continue + if tier < tiers[d]: + raise Exception( + "{} (tier {}) cannot depend on {} (tier {})".format( + task.label, + printable_tier(tier), + d, + printable_tier(tiers[d]), + ) + ) + + +@verifications.add("full_task_graph") +def verify_required_signoffs(task, taskgraph, scratch_pad, graph_config, parameters): + """ + Task with required signoffs can't be dependencies of tasks with less + required signoffs. + """ + all_required_signoffs = scratch_pad + if task is not None: + all_required_signoffs[task.label] = set( + task.attributes.get("required_signoffs", []) + ) + else: + + def printable_signoff(signoffs): + if len(signoffs) == 1: + return "required signoff {}".format(*signoffs) + if signoffs: + return "required signoffs {}".format(", ".join(signoffs)) + return "no required signoffs" + + for task in taskgraph.tasks.values(): + required_signoffs = all_required_signoffs[task.label] + for d in task.dependencies.values(): + if required_signoffs < all_required_signoffs[d]: + raise Exception( + "{} ({}) cannot depend on {} ({})".format( + task.label, + printable_signoff(required_signoffs), + d, + printable_signoff(all_required_signoffs[d]), + ) + ) + + +@verifications.add("full_task_graph") +def verify_aliases(task, taskgraph, scratch_pad, graph_config, parameters): + """ + This function verifies that aliases are not reused. + """ + if task is None: + return + if task.kind not in ("toolchain", "fetch"): + return + for_kind = scratch_pad.setdefault(task.kind, {}) + aliases = for_kind.setdefault("aliases", {}) + alias_attribute = f"{task.kind}-alias" + if task.label in aliases: + raise Exception( + "Task `{}` has a {} of `{}`, masking a task of that name.".format( + aliases[task.label], + alias_attribute, + task.label[len(task.kind) + 1 :], + ) + ) + labels = for_kind.setdefault("labels", set()) + labels.add(task.label) + attributes = task.attributes + if alias_attribute in attributes: + keys = attributes[alias_attribute] + if not keys: + keys = [] + elif isinstance(keys, str): + keys = [keys] + for key in keys: + full_key = f"{task.kind}-{key}" + if full_key in labels: + raise Exception( + "Task `{}` has a {} of `{}`," + " masking a task of that name.".format( + task.label, + alias_attribute, + key, + ) + ) + if full_key in aliases: + raise Exception( + "Duplicate {} in tasks `{}`and `{}`: {}".format( + alias_attribute, + task.label, + aliases[full_key], + key, + ) + ) + else: + aliases[full_key] = task.label + + +@verifications.add("optimized_task_graph") +def verify_always_optimized(task, taskgraph, scratch_pad, graph_config, parameters): + """ + This function ensures that always-optimized tasks have been optimized. + """ + if task is None: + return + if task.task.get("workerType") == "always-optimized": + raise Exception(f"Could not optimize the task {task.label!r}") + + +@verifications.add("full_task_graph", run_on_projects=RELEASE_PROJECTS) +def verify_shippable_no_sccache(task, taskgraph, scratch_pad, graph_config, parameters): + if task and task.attributes.get("shippable"): + if task.task.get("payload", {}).get("env", {}).get("USE_SCCACHE"): + raise Exception(f"Shippable job {task.label} cannot use sccache") + + +@verifications.add("full_task_graph") +def verify_test_packaging(task, taskgraph, scratch_pad, graph_config, parameters): + if task is None: + # In certain cases there are valid reasons for tests to be missing, + # don't error out when that happens. + missing_tests_allowed = any( + ( + # user specified `--target-kind` + parameters.get("target-kind") is not None, + # manifest scheduling is enabled + parameters["test_manifest_loader"] != "default", + ) + ) + + exceptions = [] + for task in taskgraph.tasks.values(): + if task.kind == "build" and not task.attributes.get( + "skip-verify-test-packaging" + ): + build_env = task.task.get("payload", {}).get("env", {}) + package_tests = build_env.get("MOZ_AUTOMATION_PACKAGE_TESTS") + shippable = task.attributes.get("shippable", False) + build_has_tests = scratch_pad.get(task.label) + + if package_tests != "1": + # Shippable builds should always package tests. + if shippable: + exceptions.append( + "Build job {} is shippable and does not specify " + "MOZ_AUTOMATION_PACKAGE_TESTS=1 in the " + "environment.".format(task.label) + ) + + # Build tasks in the scratch pad have tests dependent on + # them, so we need to package tests during build. + if build_has_tests: + exceptions.append( + "Build job {} has tests dependent on it and does not specify " + "MOZ_AUTOMATION_PACKAGE_TESTS=1 in the environment".format( + task.label + ) + ) + else: + # Build tasks that aren't in the scratch pad have no + # dependent tests, so we shouldn't package tests. + # With the caveat that we expect shippable jobs to always + # produce tests. + if not build_has_tests and not shippable: + # If we have not generated all task kinds, we can't verify that + # there are no dependent tests. + if not missing_tests_allowed: + exceptions.append( + "Build job {} has no tests, but specifies " + "MOZ_AUTOMATION_PACKAGE_TESTS={} in the environment. " + "Unset MOZ_AUTOMATION_PACKAGE_TESTS in the task definition " + "to fix.".format(task.label, package_tests) + ) + if exceptions: + raise Exception("\n".join(exceptions)) + return + if task.kind == "test": + build_task = taskgraph[task.dependencies["build"]] + scratch_pad[build_task.label] = 1 + + +@verifications.add("full_task_graph") +def verify_run_known_projects(task, taskgraph, scratch_pad, graph_config, parameters): + """Validates the inputs in run-on-projects. + + We should never let 'try' (or 'try-comm-central') be in run-on-projects even though it + is valid because it is not considered for try pushes. While here we also validate for + other unknown projects or typos. + """ + if task and task.attributes.get("run_on_projects"): + projects = set(task.attributes["run_on_projects"]) + if {"try", "try-comm-central"} & set(projects): + raise Exception( + "In task {}: using try in run-on-projects is invalid; use try " + "selectors to select this task on try".format(task.label) + ) + # try isn't valid, but by the time we get here its not an available project anyway. + valid_projects = ALL_PROJECTS | set(RUN_ON_PROJECT_ALIASES.keys()) + invalid_projects = projects - valid_projects + if invalid_projects: + raise Exception( + "Task '{}' has an invalid run-on-projects value: " + "{}".format(task.label, invalid_projects) + ) diff --git a/taskcluster/gecko_taskgraph/util/workertypes.py b/taskcluster/gecko_taskgraph/util/workertypes.py new file mode 100644 index 0000000000..b941ba85f6 --- /dev/null +++ b/taskcluster/gecko_taskgraph/util/workertypes.py @@ -0,0 +1,105 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozbuild.util import memoize +from taskgraph.util.attributes import keymatch +from taskgraph.util.keyed_by import evaluate_keyed_by + +from gecko_taskgraph.util.attributes import release_level as _release_level + +WORKER_TYPES = { + "gce/gecko-1-b-linux": ("docker-worker", "linux"), + "gce/gecko-2-b-linux": ("docker-worker", "linux"), + "gce/gecko-3-b-linux": ("docker-worker", "linux"), + "invalid/invalid": ("invalid", None), + "invalid/always-optimized": ("always-optimized", None), + "scriptworker-prov-v1/signing-linux-v1": ("scriptworker-signing", None), + "scriptworker-k8s/gecko-3-shipit": ("shipit", None), + "scriptworker-k8s/gecko-1-shipit": ("shipit", None), +} + + +@memoize +def _get(graph_config, alias, level, release_level, project): + """Get the configuration for this worker_type alias: {provisioner, + worker-type, implementation, os}""" + level = str(level) + + # handle the legacy (non-alias) format + if "/" in alias: + alias = alias.format(level=level) + provisioner, worker_type = alias.split("/", 1) + try: + implementation, os = WORKER_TYPES[alias] + return { + "provisioner": provisioner, + "worker-type": worker_type, + "implementation": implementation, + "os": os, + } + except KeyError: + return { + "provisioner": provisioner, + "worker-type": worker_type, + } + + matches = keymatch(graph_config["workers"]["aliases"], alias) + if len(matches) > 1: + raise KeyError("Multiple matches for worker-type alias " + alias) + elif not matches: + raise KeyError("No matches for worker-type alias " + alias) + worker_config = matches[0].copy() + + worker_config["provisioner"] = evaluate_keyed_by( + worker_config["provisioner"], + f"worker-type alias {alias} field provisioner", + {"level": level}, + ).format( + **{ + "trust-domain": graph_config["trust-domain"], + "level": level, + "alias": alias, + } + ) + attrs = {"level": level, "release-level": release_level} + if project: + attrs["project"] = project + worker_config["worker-type"] = evaluate_keyed_by( + worker_config["worker-type"], + f"worker-type alias {alias} field worker-type", + attrs, + ).format( + **{ + "trust-domain": graph_config["trust-domain"], + "level": level, + "alias": alias, + } + ) + + return worker_config + + +def worker_type_implementation(graph_config, parameters, worker_type): + """Get the worker implementation and OS for the given workerType, where the + OS represents the host system, not the target OS, in the case of + cross-compiles.""" + worker_config = _get( + graph_config, worker_type, "1", "staging", parameters["project"] + ) + return worker_config["implementation"], worker_config.get("os") + + +def get_worker_type(graph_config, parameters, worker_type): + """ + Get the worker type provisioner and worker-type, optionally evaluating + aliases from the graph config. + """ + worker_config = _get( + graph_config, + worker_type, + parameters["level"], + _release_level(parameters.get("project")), + parameters.get("project"), + ) + return worker_config["provisioner"], worker_config["worker-type"] |