From 6bf0a5cb5034a7e684dcc3500e841785237ce2dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 19:32:43 +0200 Subject: Adding upstream version 1:115.7.0. Signed-off-by: Daniel Baumann --- comm/taskcluster/comm_taskgraph/__init__.py | 45 +++ comm/taskcluster/comm_taskgraph/actions.py | 38 ++ comm/taskcluster/comm_taskgraph/decision.py | 162 ++++++++ comm/taskcluster/comm_taskgraph/documentation.py | 12 + comm/taskcluster/comm_taskgraph/files_changed.py | 114 ++++++ comm/taskcluster/comm_taskgraph/loader/__init__.py | 3 + comm/taskcluster/comm_taskgraph/loader/merge.py | 29 ++ .../taskcluster/comm_taskgraph/loader/reference.py | 93 +++++ .../comm_taskgraph/manifests/release_checksums.yml | 68 ++++ .../comm_taskgraph/manifests/source_checksums.yml | 46 +++ .../comm_taskgraph/manifests/source_files.yml | 46 +++ .../comm_taskgraph/manifests/strings_source.yml | 64 +++ .../manifests/thunderbird_candidates.yml | 334 ++++++++++++++++ .../manifests/thunderbird_candidates_checksums.yml | 83 ++++ .../manifests/thunderbird_nightly.yml | 434 +++++++++++++++++++++ .../manifests/thunderbird_nightly_checksums.yml | 48 +++ comm/taskcluster/comm_taskgraph/optimize.py | 151 +++++++ comm/taskcluster/comm_taskgraph/parameters.py | 42 ++ comm/taskcluster/comm_taskgraph/target_tasks.py | 75 ++++ .../test/cc_automationrelevance.json | 154 ++++++++ comm/taskcluster/comm_taskgraph/test/conftest.py | 7 + .../test/mc_automationrelevance.json | 309 +++++++++++++++ comm/taskcluster/comm_taskgraph/test/python.ini | 6 + .../comm_taskgraph/test/test_files_changed.py | 144 +++++++ .../test/test_optimization_strategies.py | 143 +++++++ .../comm_taskgraph/test/test_parameters.py | 102 +++++ .../comm_taskgraph/transforms/__init__.py | 3 + .../comm_taskgraph/transforms/job/__init__.py | 0 .../comm_taskgraph/transforms/job/toolchain.py | 165 ++++++++ comm/taskcluster/comm_taskgraph/transforms/l10n.py | 83 ++++ .../comm_taskgraph/transforms/l10n_pre.py | 45 +++ .../transforms/l10n_source_signing.py | 52 +++ .../comm_taskgraph/transforms/merge_automation.py | 58 +++ .../comm_taskgraph/transforms/partials.py | 33 ++ .../comm_taskgraph/transforms/push_langpacks.py | 231 +++++++++++ .../transforms/release_flatpak_push.py | 79 ++++ .../comm_taskgraph/transforms/release_started.py | 53 +++ .../comm_taskgraph/transforms/repackage_msix.py | 52 +++ .../comm_taskgraph/transforms/signing.py | 88 +++++ .../comm_taskgraph/transforms/source_test.py | 63 +++ .../comm_taskgraph/transforms/tb_build.py | 24 ++ .../comm_taskgraph/transforms/tb_cross_channel.py | 46 +++ .../taskcluster/comm_taskgraph/transforms/tests.py | 27 ++ .../transforms/update_verify_config.py | 134 +++++++ .../comm_taskgraph/try_option_syntax.py | 97 +++++ comm/taskcluster/comm_taskgraph/util/__init__.py | 15 + comm/taskcluster/comm_taskgraph/util/docker.py | 25 ++ comm/taskcluster/comm_taskgraph/util/hash.py | 76 ++++ 48 files changed, 4201 insertions(+) create mode 100644 comm/taskcluster/comm_taskgraph/__init__.py create mode 100644 comm/taskcluster/comm_taskgraph/actions.py create mode 100644 comm/taskcluster/comm_taskgraph/decision.py create mode 100644 comm/taskcluster/comm_taskgraph/documentation.py create mode 100644 comm/taskcluster/comm_taskgraph/files_changed.py create mode 100644 comm/taskcluster/comm_taskgraph/loader/__init__.py create mode 100644 comm/taskcluster/comm_taskgraph/loader/merge.py create mode 100644 comm/taskcluster/comm_taskgraph/loader/reference.py create mode 100644 comm/taskcluster/comm_taskgraph/manifests/release_checksums.yml create mode 100644 comm/taskcluster/comm_taskgraph/manifests/source_checksums.yml create mode 100644 comm/taskcluster/comm_taskgraph/manifests/source_files.yml create mode 100644 comm/taskcluster/comm_taskgraph/manifests/strings_source.yml create mode 100644 comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates.yml create mode 100644 comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates_checksums.yml create mode 100644 comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly.yml create mode 100644 comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly_checksums.yml create mode 100644 comm/taskcluster/comm_taskgraph/optimize.py create mode 100644 comm/taskcluster/comm_taskgraph/parameters.py create mode 100644 comm/taskcluster/comm_taskgraph/target_tasks.py create mode 100644 comm/taskcluster/comm_taskgraph/test/cc_automationrelevance.json create mode 100644 comm/taskcluster/comm_taskgraph/test/conftest.py create mode 100644 comm/taskcluster/comm_taskgraph/test/mc_automationrelevance.json create mode 100644 comm/taskcluster/comm_taskgraph/test/python.ini create mode 100644 comm/taskcluster/comm_taskgraph/test/test_files_changed.py create mode 100644 comm/taskcluster/comm_taskgraph/test/test_optimization_strategies.py create mode 100644 comm/taskcluster/comm_taskgraph/test/test_parameters.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/__init__.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/job/__init__.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/l10n.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/merge_automation.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/partials.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/release_started.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/signing.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/source_test.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/tb_build.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/tests.py create mode 100644 comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py create mode 100644 comm/taskcluster/comm_taskgraph/try_option_syntax.py create mode 100644 comm/taskcluster/comm_taskgraph/util/__init__.py create mode 100644 comm/taskcluster/comm_taskgraph/util/docker.py create mode 100644 comm/taskcluster/comm_taskgraph/util/hash.py (limited to 'comm/taskcluster/comm_taskgraph') diff --git a/comm/taskcluster/comm_taskgraph/__init__.py b/comm/taskcluster/comm_taskgraph/__init__.py new file mode 100644 index 0000000000..cd54612dfb --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/__init__.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os +from importlib import import_module + +from gecko_taskgraph import GECKO +from gecko_taskgraph.optimize.schema import set_optimization_schema # noqa: F401 + +from comm_taskgraph.optimize import thunderbird_optimizations + +logger = logging.getLogger(__name__) + +COMM = os.path.join(GECKO, "comm") +COMM_SCRIPTS = os.path.join(COMM, "taskcluster", "scripts") + + +def register(graph_config): + """ + Import all modules that are siblings of this one, triggering decorators in + the process. + """ + from comm_taskgraph.parameters import register_parameters + + logger.info("{} path registered".format(__name__)) + register_parameters() + + set_optimization_schema(thunderbird_optimizations) + + _import_modules( + [ + "documentation", + "util.docker", + "actions", + "target_tasks", + "transforms.job.toolchain", + ] + ) + + +def _import_modules(modules): + for module in modules: + import_module(".{}".format(module), package=__name__) diff --git a/comm/taskcluster/comm_taskgraph/actions.py b/comm/taskcluster/comm_taskgraph/actions.py new file mode 100644 index 0000000000..6095f3e6c8 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/actions.py @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from gecko_taskgraph.actions.registry import register_callback_action +from gecko_taskgraph.actions.util import create_tasks, fetch_graph_and_labels +from gecko_taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS + + +def is_release_promotion_available(parameters): + return parameters["project"] in RELEASE_PROMOTION_PROJECTS + + +@register_callback_action( + name="l10n-bump", + title="L10n Bumper Automation", + symbol="l10n_bump", + description="L10n bumper action.", + order=500, + context=[], + available=is_release_promotion_available, +) +def l10n_bump_action(parameters, graph_config, _input, task_group_id, task_id): + """ + Runs the 'l10n_bump' task. + """ + decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( + parameters, graph_config + ) + to_run = [label for label, entry in full_task_graph.tasks.items() if "l10n-bump" in entry.kind] + create_tasks( + graph_config, + to_run, + full_task_graph, + label_to_taskid, + parameters, + decision_task_id, + ) diff --git a/comm/taskcluster/comm_taskgraph/decision.py b/comm/taskcluster/comm_taskgraph/decision.py new file mode 100644 index 0000000000..a075b47cf5 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/decision.py @@ -0,0 +1,162 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os +import sys + +from redo import retry +from taskgraph.decision import ( + _determine_more_accurate_base_ref, + _determine_more_accurate_base_rev, + _get_env_prefix, +) +from taskgraph.taskgraph import TaskGraph +from taskgraph.util.taskcluster import get_artifact +from taskgraph.util.vcs import get_repository + +from gecko_taskgraph.util.backstop import is_backstop +from gecko_taskgraph.util.partials import populate_release_history +from gecko_taskgraph.util.taskgraph import ( + find_decision_task, + find_existing_tasks_from_previous_kinds, +) + +from . import COMM +from comm_taskgraph.parameters import get_defaults + +logger = logging.getLogger(__name__) + +BALROG_PRODUCT = "Thunderbird" + +# Backstop defaults +BACKSTOP_TIME_INTERVAL = 60 * 22 # minutes +INTEGRATION_PROJECTS = {"comm-central"} + +PER_PROJECT_PARAMETERS = { + "ash": { + "target_tasks_method": "ash_tasks", + }, + "jamun": { + "target_tasks_method": "nightly_desktop", + "release_type": "nightly", + }, + "try-comm-central": { + "enable_always_target": True, + "target_tasks_method": "try_cc_tasks", + }, + "comm-central": { + "target_tasks_method": "comm_central_tasks", + "release_type": "nightly", + }, + "comm-beta": { + "target_tasks_method": "mozilla_beta_tasks", + "release_type": "beta", + }, + "comm-esr115": { + "target_tasks_method": "mozilla_esr115_tasks", + "release_type": "release", + }, +} + +CRON_OPTIONS = { + "nightly_desktop": { + "existing_tasks": lambda parameters, graph_config: get_existing_tasks( + parameters, graph_config + ), + "release_history": lambda parameters, graph_config: populate_release_history( + BALROG_PRODUCT, parameters["project"] + ), + } +} + + +def get_decision_parameters(graph_config, parameters): + logger.info("{}.get_decision_parameters called".format(__name__)) + + # Apply default values for all Thunderbird CI projects + parameters.update(get_defaults(graph_config.vcs_root)) + + # If the target method is nightly, we should build partials. This means + # knowing what has been released previously. + # An empty release_history is fine, it just means no partials will be built + project = parameters["project"] + + if project in PER_PROJECT_PARAMETERS: + # Upstream will set target_tasks_method to "default" when nothing is set + if parameters["target_tasks_method"] == "default": + del parameters["target_tasks_method"] + + # If running from .cron.yml, do not overwrite existing parameters + update_parameters = [ + (_k, _v) + for _k, _v in PER_PROJECT_PARAMETERS[project].items() + if _k not in parameters or not parameters[_k] + ] + parameters.update(update_parameters) + logger.info("project parameters set for project {} from {}.".format(project, __file__)) + else: + # Projects without a target_tasks_method should not exist for Thunderbird CI + raise Exception("No target_tasks_method is defined for project {}.".format(project)) + + del parameters["backstop"] + parameters["backstop"] = is_backstop( + parameters, + trust_domain="comm", + time_interval=BACKSTOP_TIME_INTERVAL, + integration_projects=INTEGRATION_PROJECTS, + ) + for n in ( + "COMM_BASE_REPOSITORY", + "COMM_BASE_REV", + "COMM_HEAD_REPOSITORY", + "COMM_HEAD_REV", + "COMM_HEAD_REF", + ): + val = os.environ.get(n, "") + parameters[n.lower()] = val + + repo_path = COMM + repo = get_repository(repo_path) + logger.info("Determining comm_base_ref...") + parameters["comm_base_ref"] = _determine_more_accurate_base_ref( + repo, + candidate_base_ref="", + head_ref=parameters.get("comm_head_ref"), + base_rev=parameters.get("comm_base_rev"), + ) + logger.info("Determining comm_base_rev...") + parameters["comm_base_rev"] = _determine_more_accurate_base_rev( + repo, + base_ref=parameters["comm_base_ref"], + candidate_base_rev=parameters.get("comm_base_rev"), + head_rev=parameters.get("comm_head_rev"), + env_prefix=_get_env_prefix(graph_config), + ) + + parameters.setdefault("release_history", dict()) + if parameters.get("tasks_for", "") == "cron": + for key, _callable in CRON_OPTIONS.get(parameters["target_tasks_method"], {}).items(): + result = _callable(parameters, graph_config) + parameters[key] = result + + +def get_existing_tasks(parameters, graph_config): + """ + Find the decision task corresponding to the on-push graph, and return + a mapping of labels to task-ids from it. + """ + try: + decision_task = retry( + find_decision_task, + args=(parameters, graph_config), + attempts=4, + sleeptime=5 * 60, + ) + except Exception: + logger.exception("Didn't find existing push task.") + sys.exit(1) + + _, task_graph = TaskGraph.from_json(get_artifact(decision_task, "public/full-task-graph.json")) + return find_existing_tasks_from_previous_kinds(task_graph, [decision_task], []) diff --git a/comm/taskcluster/comm_taskgraph/documentation.py b/comm/taskcluster/comm_taskgraph/documentation.py new file mode 100644 index 0000000000..76e8f979cd --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/documentation.py @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os + +from gecko_taskgraph.util.verify import documentation_paths + +from . import COMM + +documentation_paths.add(os.path.join(COMM, "taskcluster", "docs")) diff --git a/comm/taskcluster/comm_taskgraph/files_changed.py b/comm/taskcluster/comm_taskgraph/files_changed.py new file mode 100644 index 0000000000..10deb65004 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/files_changed.py @@ -0,0 +1,114 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Support for optimizing tasks based on the set of files that have changed. +""" + +import logging +import os +from subprocess import CalledProcessError + +from taskgraph.util.memoize import memoize +from taskgraph.util.path import join as join_path +from taskgraph.util.path import match as match_path + +from gecko_taskgraph import GECKO +from gecko_taskgraph.util.hg import get_json_automationrelevance +from mozversioncontrol import InvalidRepoPath, get_repository_object + +logger = logging.getLogger(__name__) + + +@memoize +def get_changed_files(repository, revision): + """ + Get the set of files changed in the push headed by the given revision. + Responses are cached, so multiple calls with the same arguments are OK. + """ + contents = get_json_automationrelevance(repository, revision) + try: + changesets = contents["changesets"] + except KeyError: + # We shouldn't hit this error in CI. + if os.environ.get("MOZ_AUTOMATION"): + raise + + # We're likely on an unpublished commit, grab changed files from + # version control. + return get_locally_changed_files(GECKO) + + logger.debug("{} commits influencing task scheduling:".format(len(changesets))) + changed_files = set() + for c in changesets: + desc = "" # Support empty desc + if c["desc"]: + desc = c["desc"].splitlines()[0].encode("ascii", "ignore") + logger.debug(" {cset} {desc}".format(cset=c["node"][0:12], desc=desc)) + changed_files |= set(c["files"]) + + return changed_files + + +def get_files_changed_extended(params): + """ + Get the set of files changed in the push head from possibly multiple + head_repositories. + """ + changed_files = set() + + repo_keys = [key for key in params.keys() if key.endswith("head_repository")] + + def prefix_changed(_changed, prefix): + if not prefix: + return _changed + else: + return {join_path(prefix, file) for file in _changed} + + for repo_key in repo_keys: + repo_prefix = repo_key.replace("head_repository", "") + rev_key = f"{repo_prefix}head_rev" + repo_subdir_key = f"{repo_prefix}src_path" + + repository = params.get(repo_key) + revision = params.get(rev_key) + repo_subdir = params.get(repo_subdir_key, "") + + if not repository or not revision: + logger.warning( + f"Missing `{repo_key}` or `{rev_key}` parameters; " + "assuming all files have changed" + ) + return True + + changed_files |= prefix_changed(get_changed_files(repository, revision), repo_subdir) + + return changed_files + + +def check(params, file_patterns): + """Determine whether any of the files changed in the indicated push to + https://hg.mozilla.org match any of the given file patterns.""" + changed_files = get_files_changed_extended(params) + if not changed_files: + logger.warning( + "changed_files from automationrelevance is empty; assuming all files have changed" + ) + return True + + for pattern in file_patterns: + for path in changed_files: + if match_path(path, pattern): + return True + + return False + + +@memoize +def get_locally_changed_files(repo): + try: + vcs = get_repository_object(repo) + return set(vcs.get_outgoing_files("AM")) + except (InvalidRepoPath, CalledProcessError): + return set() diff --git a/comm/taskcluster/comm_taskgraph/loader/__init__.py b/comm/taskcluster/comm_taskgraph/loader/__init__.py new file mode 100644 index 0000000000..6fbe8159b2 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/loader/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/comm/taskcluster/comm_taskgraph/loader/merge.py b/comm/taskcluster/comm_taskgraph/loader/merge.py new file mode 100644 index 0000000000..00b600cd0d --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/loader/merge.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from gecko_taskgraph.loader.transform import loader as transform_loader + +from comm_taskgraph.loader.reference import loader as reference_loader + + +def loader(kind, path, config, params, loaded_tasks): + """ + Look up jobs via reference loader at reference-base-path using the list + reference-jobs-from, followed by jobs-from. + + This loader has been tested with "fetch" jobs successfully. Anything else + is likely to have bugs. + """ + # Make a copy of config for reference_loader. Use pop here to remove the + # fields that aren't used by the transform loader + reference_config = { + "kind-dependencies": config.get("kind-dependencies", None), + "reference-base-path": config.pop("reference-base-path"), + "reference-jobs": config.pop("reference-jobs", None), + } + for job in reference_loader(kind, path, reference_config, params, loaded_tasks): + yield job + + for job in transform_loader(kind, path, config, params, loaded_tasks): + yield job diff --git a/comm/taskcluster/comm_taskgraph/loader/reference.py b/comm/taskcluster/comm_taskgraph/loader/reference.py new file mode 100644 index 0000000000..cb4d8f0565 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/loader/reference.py @@ -0,0 +1,93 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os + +from taskgraph.util.python_path import find_object +from taskgraph.util.schema import resolve_keyed_by +from taskgraph.util.yaml import load_yaml + +logger = logging.getLogger(__name__) + + +def _get_aliases(kind, job, project): + aliases = {job["name"]} + + if kind == "toolchain": + if job["run"].get("toolchain-alias"): + resolve_keyed_by( + job["run"], + "toolchain-alias", + item_name=f"{kind}-{job['name']}", + project=project, + ) + aliaslist = job["run"].get("toolchain-alias") + if aliaslist is not None: + if isinstance(aliaslist, str): + aliaslist = [aliaslist] + for alias in aliaslist: + aliases.add(alias) + + return aliases + + +def _expand_aliases(kind, inputs, project): + """Given the list of all "reference-jobs" pulled in from upstream, return a + set with all job names and aliases. + For example "linux64-clang" is an alias of "linux64-clang-13", and both + of those names will be included in the returned set.""" + rv = set() + for input_job in inputs: + for alias in _get_aliases(kind, input_job, project): + rv.add(alias) + return rv + + +def _get_loader(path, config): + try: + _loader = config["loader"] + except KeyError: + raise KeyError("{!r} does not define `loader`".format(path)) + return find_object(_loader) + + +def loader(kind, path, config, params, loaded_tasks): + """ + Loads selected jobs from a different taskgraph hierarchy. + + This loads jobs of the given kind from the taskgraph rooted at `base-path`, + and includes all the jobs with names or aliases matching the names in the + `jobs` key. + """ + base_path = config.pop("reference-base-path") + sub_path = os.path.join(base_path, kind) + + logger.debug("Reference loader: load tasks from {}".format(sub_path)) + sub_config = load_yaml(sub_path, "kind.yml") + _loader = _get_loader(sub_path, sub_config) + inputs = _loader(kind, sub_path, sub_config, params, loaded_tasks) + + jobs = config.pop("reference-jobs", None) + + config.update(sub_config) + project = params["project"] + + if jobs is not None: + jobs = set(jobs) + + found_reference_jobs = [job for job in inputs if (_get_aliases(kind, job, project) & jobs)] + + # Check for jobs listed as a reference job in Thunderbird's config + # that do not exist in upstream. + reference_alias_names = _expand_aliases(kind, found_reference_jobs, project) + if reference_alias_names >= jobs: + return found_reference_jobs + else: + missing_jobs = jobs - reference_alias_names + raise Exception( + "Reference jobs not found in kind {}: {}".format(kind, ", ".join(missing_jobs)) + ) + else: + return inputs diff --git a/comm/taskcluster/comm_taskgraph/manifests/release_checksums.yml b/comm/taskcluster/comm_taskgraph/manifests/release_checksums.yml new file mode 100644 index 0000000000..81a11b2793 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/release_checksums.yml @@ -0,0 +1,68 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + by-platform: + thunderbird-release: + - pub/thunderbird/candidates +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + release-generate-checksums: build + release-generate-checksums-signing: signing + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - release-generate-checksums-signing + all_locales: true + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + - ${version}-candidates/build${build_number} + +# Configuration for individual files. Extends 'default', above. +mapping: + SHA256SUMMARY: + <<: *default + description: "Merkle-tree for the release artifacts with sha 256 hashes" + from: + - release-generate-checksums + pretty_name: SHA256SUMMARY + checksums_path: SHA256SUMMARY + SHA512SUMMARY: + <<: *default + description: "Merkle-tree for the release artifacts with sha 512 hashes" + from: + - release-generate-checksums + pretty_name: SHA512SUMMARY + checksums_path: SHA512SUMMARY + KEY: + <<: *default + description: "Public side of the key that was used to sign the release artifacts" + pretty_name: KEY + checksums_path: KEY + SHA256SUMS: + <<: *default + description: "Aggregated checksums with main installers details per platform in sha512 hashes" + pretty_name: SHA256SUMS + checksums_path: SHA256SUMS + SHA256SUMS.asc: + <<: *default + description: "Detached signature for the checksums file" + pretty_name: SHA256SUMS.asc + checksums_path: SHA256SUMS.asc + SHA512SUMS: + <<: *default + description: "Aggregated checksums with main installers details per platform in sha256 hashes" + pretty_name: SHA512SUMS + checksums_path: SHA512SUMS + SHA512SUMS.asc: + <<: *default + description: "Detached signature for the checksums file" + pretty_name: SHA512SUMS.asc + checksums_path: SHA512SUMS.asc diff --git a/comm/taskcluster/comm_taskgraph/manifests/source_checksums.yml b/comm/taskcluster/comm_taskgraph/manifests/source_checksums.yml new file mode 100644 index 0000000000..13fab07342 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/source_checksums.yml @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + by-platform: + thunderbird-source: + - pub/thunderbird/candidates +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + release-source-checksums-signing: signing + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - release-source-checksums-signing + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + - ${version}-candidates/build${build_number}/beetmover-checksums/source + +# Configuration for individual files. Extends 'default', above. +mapping: + target-source.checksums: + <<: *default + description: "Checksums file for the source zip files" + pretty_name: + by-platform: + thunderbird-source: thunderbird-${version}.checksums.beet + checksums_path: + by-platform: + thunderbird-source: thunderbird-${version}.checksums.beet + target-source.checksums.asc: + <<: *default + description: "Detached signature for the checksums file" + pretty_name: + by-platform: + thunderbird-source: thunderbird-${version}.checksums.asc + checksums_path: + by-platform: + thunderbird-source: thunderbird-${version}.checksums.asc diff --git a/comm/taskcluster/comm_taskgraph/manifests/source_files.yml b/comm/taskcluster/comm_taskgraph/manifests/source_files.yml new file mode 100644 index 0000000000..53270eb347 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/source_files.yml @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + by-platform: + thunderbird-source: + - pub/thunderbird/candidates +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + release-source-signing: signing + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - release-source-signing + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + - ${version}-candidates/build${build_number}/source + +# Configuration for individual files. Extends 'default', above. +mapping: + source.tar.xz: + <<: *default + description: "Source file with the in-tree code archived" + pretty_name: + by-platform: + thunderbird-source: thunderbird-${version}.source.tar.xz + checksums_path: + by-platform: + thunderbird-source: source/thunderbird-${version}.source.tar.xz + source.tar.xz.asc: + <<: *default + description: "Detached signature for the source file" + pretty_name: + by-platform: + thunderbird-source: thunderbird-${version}.source.tar.xz.asc + checksums_path: + by-platform: + thunderbird-source: source/thunderbird-${version}.source.tar.xz.asc diff --git a/comm/taskcluster/comm_taskgraph/manifests/strings_source.yml b/comm/taskcluster/comm_taskgraph/manifests/strings_source.yml new file mode 100644 index 0000000000..b6fa63ec84 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/strings_source.yml @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + by-platform: + shippable-l10n-pre: + - pub/thunderbird/candidates +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + shippable-l10n-pre-signing: signing + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - shippable-l10n-pre-signing + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + - ${version}-candidates/build${build_number}/source + +# Configuration for individual files. Extends 'default', above. +mapping: + strings_all.tar.zst: + <<: *default + description: "Strings sources for localuized builds and langpacks" + pretty_name: + by-platform: + shippable-l10n-pre: thunderbird-${version}.strings_all.tar.zst + checksums_path: + by-platform: + shippable-l10n-pre: source/thunderbird-${version}.strings_all.tar.zst + strings_all.tar.zst.asc: + <<: *default + description: "Detached signature for the strings source file" + pretty_name: + by-platform: + shippable-l10n-pre: thunderbird-${version}.strings_all.tar.zst.asc + checksums_path: + by-platform: + shippable-l10n-pre: source/thunderbird-${version}.strings_all.tar.zst.asc + l10n-changesets.json: + <<: *default + description: "Source info used to build strings_all.tar.zst" + pretty_name: + by-platform: + shippable-l10n-pre: thunderbird-${version}.l10n-changesets.json + checksums_path: + by-platform: + shippable-l10n-pre: source/thunderbird-${version}.l10n-changesets.json + l10n-changesets.json.asc: + <<: *default + description: "Detached signature for changesets file" + pretty_name: + by-platform: + shippable-l10n-pre: thunderbird-${version}.l10n-changesets.json.asc + checksums_path: + by-platform: + shippable-l10n-pre: source/thunderbird-${version}.l10n-changesets.json.asc diff --git a/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates.yml b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates.yml new file mode 100644 index 0000000000..26540e677c --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates.yml @@ -0,0 +1,334 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file contains exhaustive information about all the release artifacs that +# are needed within a type of release. +# +# Structure +# -------- +# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets +# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of +# destinations. If given an empty locale, it uses these locales instead. +# `tasktype_map` -- mapping between task reference and task type, particularly useful when +# composing the upstreamArtifacts for scriptworker. +# `platform_names` -- various platform mappings used in reckoning artifacts or other paths +# `default` -- a default entry, which the mappings extend and override in such a way that +# final path full-destinations will be a concatenation of the following: +# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name` +# `from` -- specifies the dependency(ies) from which to expect the particular artifact +# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected +# for all locales or just the default one +# `description` -- brief summary of what that artifact is +# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not +# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location +# for example `public/build` vs `public/build/ach/`. +# `destinations` -- final list of directories where to push the artifacts in S3 +# `pretty_name` -- the final name the artifact will have at destination +# `checksums_path` -- the name to identify one artifact within the checksums file +# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform +# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform +# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present +# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest +# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest + +s3_bucket_paths: + - pub/thunderbird/candidates +default_locales: + - en-US +tasktype_map: + build: build + signing: signing + mar-signing: signing + partials-signing: signing + repackage: repackage + repackage-signing: repackage + repackage-signing-msi: repackage + repackage-signing-shippable-l10n-msix: signing + release-sign-and-push-langpacks: scriptworker +platform_names: + path_platform: + by-platform: + linux-shippable: 'linux-i686' + linux64-shippable: 'linux-x86_64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'mac' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + tools_platform: + by-platform: + linux-shippable: 'linux' + linux64-shippable: 'linux64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'macosx64' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + filename_platform: + by-platform: + linux-shippable: 'linux' + linux64-shippable: 'linux64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'macosx64' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + +default: &default + from: + - build + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '${locale}/' + source_path_modifier: + by-locale: + default: '${locale}' + en-US: '' + destinations: + - ${version}-candidates/build${build_number}/${path_platform} + +mapping: + buildhub.json: + <<: *default + all_locales: false + description: "Build related information to be consumed by Buildhub service" + pretty_name: buildhub.json + checksums_path: ${path_platform}/${locale}/buildhub.json + target.common.tests.tar.gz: + <<: *default + description: "Mixture of reftests, mochitests, UI and others, commonly bundled together in a test suite" + pretty_name: thunderbird-${version}.common.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.common.tests.tar.gz + target.cppunittest.tests.tar.gz: + <<: *default + description: "C++ unittests related in-tree test infrastructure" + pretty_name: thunderbird-${version}.cppunittest.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.cppunittest.tests.tar.gz + target.crashreporter-symbols.zip: + <<: *default + description: "Crashreporter symbols to be consumed by Socorro" + pretty_name: thunderbird-${version}.crashreporter-symbols.zip + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.crashreporter-symbols.zip + target.json: + <<: *default + description: "Various compile and moz_app flags baked together in a json file" + pretty_name: thunderbird-${version}.json + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.json + target.mochitest.tests.tar.gz: + <<: *default + description: "Results for running the mochitest testing framework via Javascript function calls" + pretty_name: thunderbird-${version}.mochitest.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.mochitest.tests.tar.gz + target.mozinfo.json: + <<: *default + description: "Various compile and moz_app flags baked together in a json file" + pretty_name: thunderbird-${version}.mozinfo.json + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.mozinfo.json + target.reftest.tests.tar.gz: + <<: *default + description: "Results for running the reftest testing framework via display of two Web pages comparison" + pretty_name: thunderbird-${version}.reftest.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.reftest.tests.tar.gz + target.talos.tests.tar.gz: + <<: *default + description: "Results for running the talos testing framework to measure performance" + pretty_name: thunderbird-${version}.talos.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.talos.tests.tar.gz + target.awsy.tests.tar.gz: + <<: *default + description: "Results for running the awsy testing framework to track memory usage" + pretty_name: thunderbird-${version}.awsy.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.awsy.tests.tar.gz + target.test_packages.json: + <<: *default + description: "File containing metadata about all other files and testing harnesses specifics" + pretty_name: thunderbird-${version}.test_packages.json + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.test_packages.json + target.txt: + <<: *default + description: "File containing buildid and revision" + pretty_name: thunderbird-${version}.txt + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.txt + target.web-platform.tests.tar.gz: + <<: *default + description: "Results for running the webplatform testing framework to cover standard Web platform features" + pretty_name: thunderbird-${version}.web-platform.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.web-platform.tests.tar.gz + target.xpcshell.tests.tar.gz: + <<: *default + description: "Results for running the xpcshell testing framework to enable XPConnect console application" + pretty_name: thunderbird-${version}.xpcshell.tests.tar.gz + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.xpcshell.tests.tar.gz + target_info.txt: + <<: *default + description: "File containing the buildID" + locale_prefix: '' + pretty_name: ${filename_platform}_info.txt + checksums_path: ${filename_platform}_info.txt + destinations: + - ${version}-candidates/build${build_number} + mozharness.zip: + <<: *default + description: "File containing the mozharness set of scripts and configuration used by various automation tools" + pretty_name: mozharness.zip + checksums_path: ${path_platform}/${locale}/mozharness.zip + target.jsshell.zip: + <<: *default + description: "Set of shells to allow test snippets of Javascript code without needing to reload the page" + locale_prefix: '' + pretty_name: jsshell-${path_platform}.zip + checksums_path: jsshell/jsshell-${path_platform}.zip + destinations: + - ${version}-candidates/build${build_number}/jsshell + target.langpack.xpi: + <<: *default + all_locales: true + description: "Localized repack that grabs a packaged en-US Thunderbird and repackages it as locale-specific Thunderbird" + locale_prefix: '' + from: + - build + only_for_platforms: + - linux-shippable + - linux64-shippable + - macosx64-shippable + - win32-shippable + - win64-shippable + pretty_name: ${locale}.xpi + checksums_path: ${path_platform}/xpi/${locale}.xpi + destinations: + - ${version}-candidates/build${build_number}/${path_platform}/xpi + mar: + <<: *default + description: "Alongside `mbsdiff`, a tool used to generate partials" + locale_prefix: '' + source_path_modifier: 'host/bin' + pretty_name: ${tools_platform}/mar + checksums_path: mar-tools/${tools_platform}/mar + not_for_platforms: + - win32-shippable + - win64-shippable + destinations: + - ${version}-candidates/build${build_number}/mar-tools + mbsdiff: + <<: *default + description: "Alongside `mar`, a tool used to generate partials" + locale_prefix: '' + source_path_modifier: 'host/bin' + pretty_name: ${tools_platform}/mbsdiff + checksums_path: mar-tools/${tools_platform}/mbsdiff + not_for_platforms: + - win32-shippable + - win64-shippable + destinations: + - ${version}-candidates/build${build_number}/mar-tools + target.tar.bz2: + <<: *default + description: "Main installer for Linux platforms" + all_locales: true + from: + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + pretty_name: thunderbird-${version}.tar.bz2 + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.tar.bz2 + target.tar.bz2.asc: + <<: *default + description: "Detached signature for the checksums file" + all_locales: true + from: + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + pretty_name: thunderbird-${version}.tar.bz2.asc + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.tar.bz2.asc + target.pkg: + <<: *default + description: "Main package installer for Mac OS X platforms" + all_locales: true + from: + - signing + only_for_platforms: + - macosx64-shippable + pretty_name: Thunderbird ${version}.pkg + checksums_path: ${path_platform}/${locale}/Thunderbird ${version}.pkg + target.dmg: + <<: *default + description: "Main package installer for Mac OS X platforms" + all_locales: true + from: + - repackage + only_for_platforms: + - macosx64-shippable + pretty_name: Thunderbird ${version}.dmg + checksums_path: ${path_platform}/${locale}/Thunderbird ${version}.dmg + target.zip: + <<: *default + description: "Main package installer for Windows platforms" + all_locales: true + from: + - signing + only_for_platforms: + - win64-shippable + - win32-shippable + pretty_name: thunderbird-${version}.zip + checksums_path: ${path_platform}/${locale}/thunderbird-${version}.zip + target.installer.exe: + <<: *default + description: "Main installer for Windows platforms" + all_locales: true + from: + - repackage-signing + only_for_platforms: + - win64-shippable + - win32-shippable + pretty_name: Thunderbird Setup ${version}.exe + checksums_path: ${path_platform}/${locale}/Thunderbird Setup ${version}.exe + target.installer.msi: + <<: *default + description: "Windows installer for MSI platform" + all_locales: true + from: + - repackage-signing-msi + only_for_platforms: + - win64-shippable + - win32-shippable + pretty_name: Thunderbird Setup ${version}.msi + checksums_path: ${path_platform}/${locale}/Thunderbird Setup ${version}.msi + target.installer.msix: + <<: *default + description: "Windows MSIX installer" + from: + - repackage-signing-shippable-l10n-msix + only_for_platforms: + - win64-shippable + - win32-shippable + locale_prefix: 'multi/' + pretty_name: Thunderbird Setup ${version}.msix + checksums_path: ${path_platform}/multi/Thunderbird Setup ${version}.msix + target.complete.mar: + <<: *default + description: "Complete MAR to serve as updates" + all_locales: true + from: + - mar-signing + pretty_name: thunderbird-${version}.complete.mar + checksums_path: update/${path_platform}/${locale}/thunderbird-${version}.complete.mar + update_balrog_manifest: true + destinations: + - ${version}-candidates/build${build_number}/update/${path_platform} + ${partial}: + <<: *default + description: "Partials MAR files to serve as updates" + all_locales: true + from: + - partials-signing + partials_only: true + pretty_name: thunderbird-${previous_version}-${version}.partial.mar + checksums_path: update/${path_platform}/${locale}/thunderbird-${previous_version}-${version}.partial.mar + update_balrog_manifest: true + from_buildid: ${from_buildid} + destinations: + - ${version}-candidates/build${build_number}/update/${path_platform} diff --git a/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates_checksums.yml b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates_checksums.yml new file mode 100644 index 0000000000..f3b2225f96 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates_checksums.yml @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file contains exhaustive information about all the release artifacs that +# are needed within a type of release. +# +# Structure +# -------- +# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets +# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of +# destinations. If given an empty locale, it uses these locales instead. +# `tasktype_map` -- mapping between task reference and task type, particularly useful when +# composing the upstreamArtifacts for scriptworker. +# `platform_names` -- various platform mappings used in reckoning artifacts or other paths +# `default` -- a default entry, which the mappings extend and override in such a way that +# final path full-destinations will be a concatenation of the following: +# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name` +# `from` -- specifies the dependency(ies) from which to expect the particular artifact +# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected +# for all locales or just the default one +# `description` -- brief summary of what that artifact is +# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not +# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location +# for example `public/build` vs `public/build/ach/`. +# `destinations` -- final list of directories where to push the artifacts in S3 +# `pretty_name` -- the final name the artifact will have at destination +# `checksums_path` -- the name to identify one artifact within the checksums file +# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform +# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform +# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present +# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest +# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest + +s3_bucket_paths: + - pub/thunderbird/candidates +default_locales: + - en-US +tasktype_map: + beetmover-repackage: beetmover + release-beetmover-signed-langpacks: signing +platform_names: + path_platform: + by-platform: + linux-shippable: 'linux-i686' + linux64-shippable: 'linux-x86_64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'mac' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + linux: 'linux-i686' + linux64: 'linux-x86_64' + macosx64: 'mac' + win32: 'win32' + win64: 'win64' + +default: &default + from: + - beetmover-repackage + all_locales: true + description: "TO_BE_OVERRIDDEN" + locale_prefix: '${locale}/' + source_path_modifier: '' + destinations: + - ${version}-candidates/build${build_number}/beetmover-checksums/${path_platform} + +mapping: + target.checksums: + <<: *default + description: "Checksums file containing size, hash, sha algorithm and filename" + pretty_name: thunderbird-${version}.checksums.beet + checksums_path: beetmover-checksums/${path_platform}/${locale}/thunderbird-${version}.checksums.beet + target-langpack.checksums: + <<: *default + description: "Checksums file containing size, hash, sha algorithm and filename for the langpack" + locale_prefix: '' + from: + - release-beetmover-signed-langpacks + pretty_name: ${locale}.checksums.beet + checksums_path: beetmover-checksums/${path_platform}/xpi/${locale}.checksums.beet + destinations: + - ${version}-candidates/build${build_number}/beetmover-checksums/${path_platform}/xpi diff --git a/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly.yml b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly.yml new file mode 100644 index 0000000000..ee2ca6c800 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly.yml @@ -0,0 +1,434 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +# This file contains exhaustive information about all the release artifacs that +# are needed within a type of release. +# +# Structure +# -------- +# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets +# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of +# destinations. If given an empty locale, it uses these locales instead. +# `tasktype_map` -- mapping between task reference and task type, particularly useful when +# composing the upstreamArtifacts for scriptworker. +# `platform_names` -- various platform mappings used in reckoning artifacts or other paths +# `default` -- a default entry, which the mappings extend and override in such a way that +# final path full-destinations will be a concatenation of the following: +# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name` +# `from` -- specifies the dependency(ies) from which to expect the particular artifact +# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected +# for all locales or just the default one +# `description` -- brief summary of what that artifact is +# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not +# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location +# for example `public/build` vs `public/build/ach/`. +# `destinations` -- final list of directories where to push the artifacts in S3 +# `pretty_name` -- the final name the artifact will have at destination +# `checksums_path` -- the name to identify one artifact within the checksums file +# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform +# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform +# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present +# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest +# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest + +s3_bucket_paths: + - pub/thunderbird/nightly +default_locales: + - en-US +tasktype_map: + build: build + signing: signing + mar-signing: signing + partials-signing: signing + repackage: repackage + repackage-signing: repackage + repackage-signing-msi: repackage + repackage-signing-shippable-l10n-msix: signing +platform_names: + filename_platform: + by-platform: + linux-shippable: 'linux-i686' + linux64-shippable: 'linux-x86_64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'mac' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + stage_platform: + by-platform: + linux-shippable: 'linux' + linux64-shippable: 'linux64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'macosx64' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + +default: &default + from: + - build + all_locales: false + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: + by-locale: + default: '${locale}' + en-US: '' + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + +mapping: + buildhub.json: + <<: *default + description: "Build related information to be consumed by Buildhub service" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.buildhub.json + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.buildhub.json + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + KEY: + <<: *default + from: + - signing + description: "Public side of the key that was used to sign the release artifacts" + pretty_name: KEY + checksums_path: KEY + only_for_platforms: + - linux64-shippable + destinations: + - ${year}/${month}/${upload_date}-${branch} + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch} + - latest-${branch}-l10n + target.common.tests.tar.gz: + <<: *default + description: "Mixture of reftests, mochitests, UI and others, commonly bundled together in a test suite" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.common.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.common.tests.tar.gz + target.cppunittest.tests.tar.gz: + <<: *default + description: "C++ unittests related in-tree test infrastructure" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz + target.crashreporter-symbols.zip: + <<: *default + description: "Crashreporter symbols to be consumed by Socorro" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.crashreporter-symbols.zip + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.crashreporter-symbols.zip + not_for_platforms: + - linux64-asan-reporter-shippable + - win64-asan-reporter-shippable + target.json: + <<: *default + description: "Various compile and moz_app flags baked together in a json file" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.json + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.json + target.mochitest.tests.tar.gz: + <<: *default + description: "Results for running the mochitest testing framework via Javascript function calls" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz + target.mozinfo.json: + <<: *default + description: "Various compile and moz_app flags baked together in a json file" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.mozinfo.json + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.mozinfo.json + target.reftest.tests.tar.gz: + <<: *default + description: "Results for running the reftest testing framework via display of two Web pages comparison" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz + target.talos.tests.tar.gz: + <<: *default + description: "Results for running the talos testing framework to measure performance" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.talos.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.talos.tests.tar.gz + target.awsy.tests.tar.gz: + <<: *default + description: "Results for running the awsy testing framework to track memory usage" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz + target.test_packages.json: + <<: *default + description: "File containing metadata about all other files and testing harnesses specifics" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.test_packages.json + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.test_packages.json + target.txt: + <<: *default + description: "File containing buildid and revision" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.txt + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.txt + target.web-platform.tests.tar.gz: + <<: *default + description: "Results for running the webplatform testing framework to cover standard Web platform features" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz + target.xpcshell.tests.tar.gz: + <<: *default + description: "Results for running the xpcshell testing framework to enable XPConnect console application" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz + target_info.txt: + <<: *default + description: "File containing the buildID" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}_info.txt + checksums_path: thunderbird-${version}.${locale}.${filename_platform}_info.txt + mozharness.zip: + <<: *default + description: "File containing the mozharness set of scripts and configuration used by various automation tools" + pretty_name: mozharness.zip + checksums_path: mozharness.zip + target.jsshell.zip: + <<: *default + description: "Set of shells to allow test snippets of Javascript code without needing to reload the page" + pretty_name: jsshell-${filename_platform}.zip + checksums_path: jsshell-${filename_platform}.zip + not_for_platforms: + - linux64-asan-reporter-shippable + - win64-asan-reporter-shippable + target.langpack.xpi: + <<: *default + all_locales: true + description: "Localized repack that grabs a packaged en-US Thunderbird and repackages it as locale-specific Thunderbird" + from: + - build + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux64-asan-reporter-shippable + - macosx64-shippable + - win64-shippable + - win32-shippable + - win64-asan-reporter-shippable + pretty_name: thunderbird-${version}.${locale}.langpack.xpi + checksums_path: thunderbird-${version}.${locale}.langpack.xpi + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + default: + - ${year}/${month}/${upload_date}-${branch}-l10n/${filename_platform}/xpi + - latest-${branch}-l10n/${filename_platform}/xpi + mar: + <<: *default + description: "Alongside `mbsdiff`, a tool used to generate partials" + source_path_modifier: 'host/bin' + pretty_name: mar + checksums_path: mar + not_for_platforms: + - win32-shippable + - win64-shippable + - win64-asan-reporter-shippable + destinations: + - ${year}/${month}/${upload_date}-${branch}/mar-tools/${stage_platform} + - latest-${branch}/mar-tools/${stage_platform} + mbsdiff: + <<: *default + description: "Alongside `mar`, a tool used to generate partials" + source_path_modifier: 'host/bin' + pretty_name: mbsdiff + checksums_path: mbsdiff + not_for_platforms: + - win32-shippable + - win64-shippable + - win64-asan-reporter-shippable + destinations: + - ${year}/${month}/${upload_date}-${branch}/mar-tools/${stage_platform} + - latest-${branch}/mar-tools/${stage_platform} + target.tar.bz2: + <<: *default + description: "Main installer for Linux platforms" + all_locales: true + from: + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux64-asan-reporter-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.tar.bz2 + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.tar.bz2 + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.tar.bz2.asc: + <<: *default + description: "Detached signature for the checksums file" + all_locales: true + from: + - signing + only_for_platforms: + - linux-shippable + - linux64-shippable + - linux64-asan-reporter-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.tar.bz2.asc + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.tar.bz2.asc + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.pkg: + <<: *default + description: "Main package installer for Mac OS X platforms" + all_locales: true + from: + - signing + only_for_platforms: + - macosx64-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.pkg + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.pkg + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.dmg: + <<: *default + description: "Main package installer for Mac OS X platforms" + all_locales: true + from: + - repackage + only_for_platforms: + - macosx64-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.dmg + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.dmg + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.zip: + <<: *default + description: "Main package installer for Windows platforms" + all_locales: true + from: + - signing + only_for_platforms: + - win64-shippable + - win32-shippable + - win64-asan-reporter-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.zip + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.zip + target.installer.exe: + <<: *default + description: "Main installer for Windows platforms" + all_locales: true + from: + - repackage-signing + only_for_platforms: + - win64-shippable + - win32-shippable + - win64-asan-reporter-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.installer.exe + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.installer.exe + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.installer.msi: + <<: *default + description: "Windows installer for MSI platform" + all_locales: true + from: + - repackage-signing-msi + only_for_platforms: + - win64-shippable + - win32-shippable + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.installer.msi + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.installer.msi + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + target.installer.msix: + <<: *default + description: "Windows MSIX installer" + all_locales: true + from: + - repackage-signing-shippable-l10n-msix + only_for_platforms: + - win64-shippable + - win32-shippable + pretty_name: thunderbird-${version}.multi.${filename_platform}.installer.msix + checksums_path: thunderbird-${version}.multi.${filename_platform}.installer.msix + destinations: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + target.complete.mar: + <<: *default + description: "The main installer we ship our mobile products baked within" + all_locales: true + from: + - mar-signing + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.complete.mar + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.complete.mar + update_balrog_manifest: true + destinations: + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + ${partial}: + <<: *default + description: "Partials MAR files to serve as updates" + all_locales: true + from: + - partials-signing + partials_only: true + pretty_name: thunderbird-${branch}-${version}-${filename_platform}-${locale}-${from_buildid}-${buildid}.partial.mar + checksums_path: thunderbird-${branch}-${version}-${filename_platform}-${locale}-${from_buildid}-${buildid}.partial.mar + update_balrog_manifest: true + from_buildid: ${from_buildid} + destinations: + by-locale: + en-US: + - partials/${year}/${month}/${upload_date}-${branch} + default: + - partials/${year}/${month}/${upload_date}-${branch}-l10n diff --git a/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly_checksums.yml b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly_checksums.yml new file mode 100644 index 0000000000..4de3ae780e --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly_checksums.yml @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +s3_bucket_paths: + - pub/thunderbird/nightly +default_locales: # if given an empty locale, use these locales + - en-US +tasktype_map: # Map task reference to task type. + beetmover-repackage: beetmover +platform_names: + filename_platform: + by-platform: + linux-shippable: 'linux-i686' + linux64-shippable: 'linux-x86_64' + linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + macosx64-shippable: 'mac' + win32-shippable: 'win32' + win64-shippable: 'win64' + win64-asan-reporter-shippable: 'win64-asan-reporter' + +# A default entry, which the mappings below extend and override. +# Final 'destinations' will be the product of: +# s3_bucket_paths + destinations + locale_prefix + pretty_name +default: &default + from: + - beetmover-repackage + all_locales: true + description: "TO_BE_OVERRIDDEN" + locale_prefix: '' + source_path_modifier: '' + destinations: # locale_prefix is appended + by-locale: + en-US: + - ${year}/${month}/${upload_date}-${branch} + - latest-${branch} + - latest-${branch}-l10n + default: + - ${year}/${month}/${upload_date}-${branch}-l10n + - latest-${branch}-l10n + +# Configuration for individual files. Extends 'default', above. +mapping: + target.checksums: + <<: *default + description: "Checksums file containing size, hash, sha algorithm and filename" + pretty_name: thunderbird-${version}.${locale}.${filename_platform}.checksums + checksums_path: thunderbird-${version}.${locale}.${filename_platform}.checksums diff --git a/comm/taskcluster/comm_taskgraph/optimize.py b/comm/taskcluster/comm_taskgraph/optimize.py new file mode 100644 index 0000000000..24f7e67dfa --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/optimize.py @@ -0,0 +1,151 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Thunderbird specific taskgraph optimizers. +""" + +import logging + +from taskgraph.optimize.base import Any, OptimizationStrategy, register_strategy +from taskgraph.util.memoize import memoize +from taskgraph.util.path import join as join_path +from taskgraph.util.path import match as match_path +from taskgraph.util.yaml import load_yaml +from voluptuous import Optional, Required + +from gecko_taskgraph import GECKO +from gecko_taskgraph.optimize.schema import default_optimizations +from mozlint.pathutils import filterpaths + +from comm_taskgraph import files_changed + +logger = logging.getLogger(__name__) + + +def is_excluded(check_path, file_patterns): + for pattern in file_patterns: + if match_path(check_path, pattern): + return True + return False + + +def is_suite(check_path): + suite_patterns = ("editor", "suite") + return is_excluded(check_path, suite_patterns) + + +@memoize +def get_non_suite_changed_files(repository, revision): + """ + Returns the list of changed files from comm- repository (no prefixing) + with suite/** and editor/** files removed. + """ + return { + file + for file in files_changed.get_changed_files(repository, revision) + if not is_suite(file) + } + + +@register_strategy("comm-skip-unless-changed") +class SkipUnlessChanged(OptimizationStrategy): + def should_remove_task(self, task, params, file_patterns): + # pushlog_id == -1 - this is the case when run from a cron.yml job + if params.get("pushlog_id") == -1: + return False + + changed = files_changed.check(params, file_patterns) + if not changed: + logger.debug( + "no files found matching a pattern in `skip-unless-changed` for " + task.label + ) + return True + return False + + +@register_strategy("skip-suite-only") +class SkipSuiteOnly(OptimizationStrategy): + def should_remove_task(self, task, params, arg): + # pushlog_id == -1 - this is the case when run from a cron.yml job + if params.get("pushlog_id") == -1: + return False + + if params.get("project") == "try-comm-central": + # Do not try to use this optimization on try-c-c builds + return False + + repository = params.get("comm_head_repository") + revision = params.get("comm_head_rev") + non_suite_changed_files = get_non_suite_changed_files(repository, revision) + # non_suite_changed_files will be an empty set (Falsy) for suite-only pushes + # so "skip" this task + if not non_suite_changed_files: + return True + return False + + +@register_strategy("skip-unless-mozlint") +class SkipUnlessMozlint(OptimizationStrategy): + schema = { + "root-path": Optional(str), + "mozlint-config": Required(str), + } + + def should_remove_task(self, task, params, args): + include = [] + exclude = [] + extensions = [] + support_files = [] + + root_path = join_path(GECKO, args.get("root-path", "")) + mozlint_root = join_path(root_path, "tools", "lint") + mozlint_yaml = join_path(mozlint_root, args["mozlint-config"]) + + logger.info(f"Loading file patterns for {task.label} from {mozlint_yaml}.") + linter_config = load_yaml(mozlint_yaml) + for check, config in linter_config.items(): + include += config.get("include", []) + exclude += config.get("exclude", []) + extensions += [e.strip(".") for e in config.get("extensions", [])] + support_files += config.get("support-files", []) + + changed_files = files_changed.get_files_changed_extended(params) + + # Support files may not be part of "include" patterns, so check first + # Do not remove (return False) if any changed + for pattern in support_files: + for path in changed_files: + if match_path(path, pattern): + return False + + to_lint, to_exclude = filterpaths( + GECKO, + list(changed_files), + include=include, + exclude=exclude, + extensions=extensions, + ) + + # to_lint should be an empty list if there is nothing to check + if not to_lint: + return True + return False + + +register_strategy( + "skip-unless-backstop-no-suite", args=("skip-unless-backstop", "skip-suite-only") +)(Any) + +register_strategy( + "skip-unless-changed-no-suite", args=("comm-skip-unless-changed", "skip-suite-only") +)(Any) + +optimizations = ( + {"skip-suite-only": None}, + {"skip-unless-backstop-no-suite": None}, + {"skip-unless-changed-no-suite": [str]}, + {"skip-unless-mozlint": SkipUnlessMozlint.schema}, +) + +thunderbird_optimizations = default_optimizations + optimizations diff --git a/comm/taskcluster/comm_taskgraph/parameters.py b/comm/taskcluster/comm_taskgraph/parameters.py new file mode 100644 index 0000000000..a3cd3aebd5 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/parameters.py @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from taskgraph.parameters import extend_parameters_schema +from voluptuous import Required + +from gecko_taskgraph.parameters import gecko_parameters_schema as comm_parameters_schema +from gecko_taskgraph.parameters import get_app_version, get_version + +logger = logging.getLogger(__name__) + + +# Called at import time when comm_taskgraph:register is called +comm_parameters_schema.update( + { + Required("comm_base_repository"): str, + Required("comm_base_ref"): str, + Required("comm_base_rev"): str, + Required("comm_head_ref"): str, + Required("comm_head_repository"): str, + Required("comm_head_rev"): str, + Required("comm_src_path"): str, + } +) + + +def get_defaults(repo_root=None): + return { + "app_version": get_app_version(product_dir="comm/mail"), + "version": get_version("comm/mail"), + "comm_src_path": "comm/", + } + + +def register_parameters(): + """Register the additional comm_* parameters with taskgraph. Note that + defaults_fn is registered, but it does not actually run by design in the + decision task due to 'strict' being True in that case.""" + extend_parameters_schema(comm_parameters_schema, defaults_fn=get_defaults) diff --git a/comm/taskcluster/comm_taskgraph/target_tasks.py b/comm/taskcluster/comm_taskgraph/target_tasks.py new file mode 100644 index 0000000000..cfb758200d --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/target_tasks.py @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from gecko_taskgraph.target_tasks import ( + _target_task, + _try_task_config, + filter_on_platforms, + filter_out_shippable, + filter_out_shipping_phase, + standard_filter, +) + +from comm_taskgraph.try_option_syntax import _try_cc_option_syntax + +logger = logging.getLogger(__name__) + + +@_target_task("comm_searchfox_index") +def target_tasks_searchfox(full_task_graph, parameters, graph_config): + """Select tasks required for indexing Thunderbird for Searchfox web site each day""" + return [ + "searchfox-linux64-searchfox/debug", + "searchfox-macosx64-searchfox/debug", + "searchfox-win64-searchfox/debug", + ] + + +@_target_task("comm_central_tasks") +def target_tasks_default(full_task_graph, parameters, graph_config): + """Target the tasks which have indicated they should be run on this project + via the `run_on_projects` attributes.""" + return [ + l + for l, t in full_task_graph.tasks.items() + if standard_filter(t, parameters) and filter_out_shipping_phase(t, parameters) + ] + + +@_target_task("try_cc_tasks") +def target_tasks_try(full_task_graph, parameters, graph_config): + try_mode = parameters["try_mode"] + if try_mode == "try_task_config": + return _try_task_config(full_task_graph, parameters, graph_config) + elif try_mode == "try_option_syntax": + return _try_cc_option_syntax(full_task_graph, parameters, graph_config) + else: + # With no try mode, we schedule nothing, allowing the user to add tasks + # later via treeherder. + return [] + + +@_target_task("ash_tasks") +def target_tasks_ash(full_task_graph, parameters, graph_config): + run_for_platforms = ( + "linux64", + "macosx64", + "win64", + ) + + def filter_source_test(task): + return task.kind == "source-test" and task.attributes.get("always_target", False) + + def _filter(task, _parameters): + return all( + [ + filter_on_platforms(task, run_for_platforms) or filter_source_test(task), + standard_filter(task, _parameters), + filter_out_shippable(task), + ] + ) + + return [l for l, t in full_task_graph.tasks.items() if _filter(t, parameters)] diff --git a/comm/taskcluster/comm_taskgraph/test/cc_automationrelevance.json b/comm/taskcluster/comm_taskgraph/test/cc_automationrelevance.json new file mode 100644 index 0000000000..19751bad92 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/cc_automationrelevance.json @@ -0,0 +1,154 @@ +{ + "changesets": [ + { + "author": "Magnus Melin ", + "backsoutnodes": [], + "bugs": [ + { + "no": "1829065", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1829065" + } + ], + "date": [1681974919, 0], + "desc": "Bug 1829065 - re-enable browser_messageFilters.js. r=leftmostcat\n\n\nReenable the test.\nRemove the customizaton sub test since the button it's about is now different + it's using the old customization which is on its way out.\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175988", + "extra": { + "branch": "default" + }, + "files": [ + "mail/test/browser/folder-widget/browser.ini", + "mail/test/browser/folder-widget/browser_messageFilters.js" + ], + "node": "e6e51c25cd7162180080914b6bc46cb0620c4a87", + "parents": ["87bed05abb8f99147cf83ecb3bc0cec6ada7f39d"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=comm-central&originalRevision=d0c328a2cc3340ab6f6458442ea89e366e33bd8e&newProject=comm-central&newRevision=e6e51c25cd7162180080914b6bc46cb0620c4a87", + "phase": "public", + "pushdate": [1681986116, 0], + "pushhead": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e", + "pushid": 20934, + "pushuser": "geoff@darktrojan.net", + "rev": 38727, + "reviewers": [ + { + "name": "leftmostcat", + "revset": "reviewer(leftmostcat)" + } + ], + "treeherderrepo": "comm-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=comm-central" + }, + { + "author": "John Bieling ", + "backsoutnodes": [], + "bugs": [ + { + "no": "1828924", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1828924" + } + ], + "date": [1681911897, 0], + "desc": "Bug 1828924 - Fix imports of FileReader in the WebExtension API implementation files. r=mkmelin\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175899", + "extra": { + "amend_source": "83d21c8094985f1c8024d5966de1effa93b09b40", + "branch": "default" + }, + "files": [ + "mail/components/extensions/parent/ext-compose.js", + "mail/components/extensions/parent/ext-mail.js", + "mail/components/extensions/parent/ext-messages.js" + ], + "node": "04906b08eddb74d55b2fb4868290e4759a459158", + "parents": ["e6e51c25cd7162180080914b6bc46cb0620c4a87"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=comm-central&originalRevision=d0c328a2cc3340ab6f6458442ea89e366e33bd8e&newProject=comm-central&newRevision=e6e51c25cd7162180080914b6bc46cb0620c4a87", + "phase": "public", + "pushdate": [1681986116, 0], + "pushhead": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e", + "pushid": 20934, + "pushuser": "geoff@darktrojan.net", + "rev": 38728, + "reviewers": [ + { + "name": "mkmelin", + "revset": "reviewer(mkmelin)" + } + ], + "treeherderrepo": "comm-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=comm-central" + }, + { + "author": "Magnus Melin ", + "backsoutnodes": [], + "bugs": [ + { + "no": "1826744", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1826744" + } + ], + "date": [1681879233, 0], + "desc": "Bug 1826744 - \"Copy to Folder\" should be enabled for .eml files opened from file - was broken again. r=john.bieling\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175841", + "extra": { + "amend_source": "9b3f446beaa7132a9bfcd5542df376adb2af1337", + "branch": "default" + }, + "files": ["mail/base/content/mailWindowOverlay.js"], + "node": "b72ae83322b6a8d878ad710cbe1f4d59201404e1", + "parents": ["04906b08eddb74d55b2fb4868290e4759a459158"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=comm-central&originalRevision=d0c328a2cc3340ab6f6458442ea89e366e33bd8e&newProject=comm-central&newRevision=e6e51c25cd7162180080914b6bc46cb0620c4a87", + "phase": "public", + "pushdate": [1681986116, 0], + "pushhead": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e", + "pushid": 20934, + "pushuser": "geoff@darktrojan.net", + "rev": 38729, + "reviewers": [ + { + "name": "john.bieling", + "revset": "reviewer(john.bieling)" + } + ], + "treeherderrepo": "comm-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=comm-central" + }, + { + "author": "Geoff Lankow ", + "backsoutnodes": [], + "bugs": [ + { + "no": "1817367", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1817367" + } + ], + "date": [1681948136, -43200], + "desc": "Bug 1817367 - Wrap loading of virtual folder views in batch notifications. r=leftmostcat\nThis functionality was added to the base `nsMsgDBView`, but not these subclasses.\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175974", + "extra": { + "amend_source": "8722f310135f7c5f58cb1dcd4573189b3e225e6e", + "branch": "default", + "rebase_source": "4ca3cf372be8029d76bbfeecfcb406d197f0dbb6" + }, + "files": [ + "mailnews/base/src/nsMsgGroupView.cpp", + "mailnews/base/src/nsMsgQuickSearchDBView.cpp", + "mailnews/base/src/nsMsgSearchDBView.cpp", + "mailnews/base/src/nsMsgThreadedDBView.cpp", + "mailnews/base/src/nsMsgXFVirtualFolderDBView.cpp" + ], + "node": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e", + "parents": ["b72ae83322b6a8d878ad710cbe1f4d59201404e1"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=comm-central&originalRevision=d0c328a2cc3340ab6f6458442ea89e366e33bd8e&newProject=comm-central&newRevision=e6e51c25cd7162180080914b6bc46cb0620c4a87", + "phase": "public", + "pushdate": [1681986116, 0], + "pushhead": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e", + "pushid": 20934, + "pushuser": "geoff@darktrojan.net", + "rev": 38730, + "reviewers": [ + { + "name": "leftmostcat", + "revset": "reviewer(leftmostcat)" + } + ], + "treeherderrepo": "comm-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=comm-central" + } + ], + "visible": true +} diff --git a/comm/taskcluster/comm_taskgraph/test/conftest.py b/comm/taskcluster/comm_taskgraph/test/conftest.py new file mode 100644 index 0000000000..83ed5343f4 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/conftest.py @@ -0,0 +1,7 @@ +import os +import sys + +HERE = os.path.dirname(__file__) +EXT_PATH = os.path.abspath(os.path.join(HERE, "..", "..")) + +sys.path.insert(0, EXT_PATH) diff --git a/comm/taskcluster/comm_taskgraph/test/mc_automationrelevance.json b/comm/taskcluster/comm_taskgraph/test/mc_automationrelevance.json new file mode 100644 index 0000000000..8e90731d92 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/mc_automationrelevance.json @@ -0,0 +1,309 @@ +{ + "changesets": [ + { + "author": "Noah ", + "backsoutnodes": [], + "bugs": [ + { + "no": "1825749", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1825749" + } + ], + "date": [1681922538, 0], + "desc": "Bug 1825749 - Only run IdentityCredentialStorageCleaner if FedCM is enabled. r=hpeuckmann\n\nDifferential Revision: https://phabricator.services.mozilla.com/D174757", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": [ + "toolkit/components/cleardata/ClearDataService.sys.mjs", + "toolkit/components/cleardata/tests/unit/test_identity_credential_storage.js" + ], + "landingsystem": "lando", + "node": "e35b8568fd3f9d49b77e791b44159589b6bcb309", + "parents": ["680421823d4d4a444a0045996d16299691565aa7"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661160, + "reviewers": [ + { + "name": "hpeuckmann", + "revset": "reviewer(hpeuckmann)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Denis Palmeiro ", + "backsoutnodes": [], + "bugs": [ + { + "no": "1824772", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1824772" + } + ], + "date": [1681925585, 0], + "desc": "Bug 1824772: part 1 - Add jit option and static pref to toggle eager baseline hints. r=iain\n\nAdd a pref, javascript.options.jitHints, to toggle eager baseline hints.\n\nDepends on D175520\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175521", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": ["modules/libpref/init/StaticPrefList.yaml"], + "landingsystem": "lando", + "node": "3740c123c6b87c6712dda72f53b0a2a8feac5110", + "parents": ["e35b8568fd3f9d49b77e791b44159589b6bcb309"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661161, + "reviewers": [ + { + "name": "iain", + "revset": "reviewer(iain)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Tim Huang ", + "backsoutnodes": [], + "bugs": [ + { + "no": "1817463", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1817463" + } + ], + "date": [1681925941, 0], + "desc": "Bug 1817463 - Deferring SafeBrowsing updates if the browser is in idle mode. r=dimi\n\nThis patch implements the behavior for deferring SafeBrowsing updates\nwhen the browser is in idle mode. The update will be deferred until the\nnext user interaction active.\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175763", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": [ + "toolkit/components/url-classifier/UrlClassifierListManager.jsm" + ], + "landingsystem": "lando", + "node": "80b29d7af43145cdd678d4ea51e770271f0c56ce", + "parents": ["6895dba1dca74eee9f56b4ce8656b4696f7c386d"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661165, + "reviewers": [ + { + "name": "dimi", + "revset": "reviewer(dimi)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Sergey Galich ", + "backsoutnodes": [], + "bugs": [ + { + "no": "1828524", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1828524" + } + ], + "date": [1681926293, 0], + "desc": "Bug 1828524 - [Relay] Manage masks button leads to SUMO instead of relay.firefox.com r=credential-management-reviewers,dimi\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175705", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": [ + "modules/libpref/init/all.js", + "toolkit/components/passwordmgr/FirefoxRelay.sys.mjs" + ], + "landingsystem": "lando", + "node": "091fb26cb5df9e5a61d9a43883b33bdf1736ea6e", + "parents": ["ea52cdb6d093a5eb233b10a207b64d02b7489c23"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661167, + "reviewers": [ + { + "name": "credential-management-reviewers", + "revset": "reviewer(credential-management-reviewers)" + }, + { + "name": "dimi", + "revset": "reviewer(dimi)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Randell Jesup ", + "backsoutnodes": [], + "bugs": [ + { + "no": "1828973", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1828973" + } + ], + "date": [1681928684, 0], + "desc": "Bug 1828973: Disable OCSP for web-platform tests r=jgraham\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175936", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": [ + "testing/profiles/web-platform/user.js", + "testing/web-platform/meta/webtransport/__dir__.ini" + ], + "landingsystem": "lando", + "node": "ef5c4d384d2819bb857e1812ddca469fb0b87b86", + "parents": ["ccc54cebee26a52410f7c0cbd8ff8693078a620f"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661170, + "reviewers": [ + { + "name": "jgraham", + "revset": "reviewer(jgraham)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Stanca Serban ", + "backsoutnodes": [ + { + "node": "80b29d7af43145cdd678d4ea51e770271f0c56ce" + } + ], + "bugs": [ + { + "no": "1817463", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1817463" + } + ], + "date": [1681930381, -10800], + "desc": "Backed out changeset 80b29d7af431 (bug 1817463) for failures in toolkit/components/url-classifier/tests.", + "extra": { + "branch": "default", + "rebase_source": "9fc0fd23fdbaa4c30568ca895af3e1c9e3c08aea" + }, + "files": [ + "toolkit/components/url-classifier/UrlClassifierListManager.jsm" + ], + "node": "ae17e25d0f24bd1b6dee872c77c9770ad0ec7d87", + "parents": ["6ec708eb48569fcfb9bf708d91a8876383865a65"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661172, + "reviewers": [], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Bilal ", + "backsoutnodes": [], + "bugs": [ + { + "no": "1817002", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1817002" + } + ], + "date": [1681931937, 0], + "desc": "Bug 1817002 - ESMified browser/components/pocket/content. r=kpatenio\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175700", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": [ + "toolkit/components/normandy/content/AboutPages.sys.mjs", + "toolkit/modules/NewTabUtils.sys.mjs" + ], + "landingsystem": "lando", + "node": "eff25b124b5e2c29c10442e902e9c4a9c488672c", + "parents": ["dcb12f0cfb12a4d3ae666f1660276b9cea2419ba"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661174, + "reviewers": [ + { + "name": "kpatenio", + "revset": "reviewer(kpatenio)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + }, + { + "author": "Sammy Khamis ", + "backsoutnodes": [], + "bugs": [ + { + "no": "1825905", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1825905" + } + ], + "date": [1681937295, 0], + "desc": "Bug 1825905: Autofill sync engine should roundtrip data it doesn't know about r=sgalich,lina\n\nDifferential Revision: https://phabricator.services.mozilla.com/D174834", + "extra": { + "branch": "default", + "moz-landing-system": "lando" + }, + "files": [ + "toolkit/components/formautofill/FormAutofillStorageBase.sys.mjs" + ], + "landingsystem": "lando", + "node": "2d1f87aeb76249a22f8db78ac031305816544514", + "parents": ["5f3904256e0d38a7cffed5ef443b7b259d43ad49"], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041", + "phase": "public", + "pushdate": [1681982536, 0], + "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c", + "pushid": 40795, + "pushuser": "nfay@mozilla.com", + "rev": 661182, + "reviewers": [ + { + "name": "sgalich", + "revset": "reviewer(sgalich)" + }, + { + "name": "lina", + "revset": "reviewer(lina)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central" + } + ], + "visible": true +} diff --git a/comm/taskcluster/comm_taskgraph/test/python.ini b/comm/taskcluster/comm_taskgraph/test/python.ini new file mode 100644 index 0000000000..84548afeb0 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/python.ini @@ -0,0 +1,6 @@ +[DEFAULT] +subsuite = comm_taskgraph + +[test_files_changed.py] +[test_optimization_strategies.py] +[test_parameters.py] diff --git a/comm/taskcluster/comm_taskgraph/test/test_files_changed.py b/comm/taskcluster/comm_taskgraph/test/test_files_changed.py new file mode 100644 index 0000000000..5759520de6 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/test_files_changed.py @@ -0,0 +1,144 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import json +import os +import unittest + +import conftest # noqa: F401 +from mozunit import main + +from gecko_taskgraph.util import hg + +from comm_taskgraph import files_changed + +PARAMS = { + "comm_head_repository": "https://hg.mozilla.org/comm-central", + "comm_head_rev": "d0c328a2cc33", + "head_repository": "https://hg.mozilla.org/mozilla-central", + "head_rev": "6ca54f5a4a1b", + "comm_src_path": "comm/", +} + +FILES_CHANGED_CC = [ + "mail/base/content/mailWindowOverlay.js", + "mail/components/extensions/parent/ext-compose.js", + "mail/components/extensions/parent/ext-mail.js", + "mail/components/extensions/parent/ext-messages.js", + "mail/test/browser/folder-widget/browser.ini", + "mail/test/browser/folder-widget/browser_messageFilters.js", + "mailnews/base/src/nsMsgGroupView.cpp", + "mailnews/base/src/nsMsgQuickSearchDBView.cpp", + "mailnews/base/src/nsMsgSearchDBView.cpp", + "mailnews/base/src/nsMsgThreadedDBView.cpp", + "mailnews/base/src/nsMsgXFVirtualFolderDBView.cpp", +] + +FILES_CHANGED_MC = [ + "modules/libpref/init/StaticPrefList.yaml", + "modules/libpref/init/all.js", + "testing/profiles/web-platform/user.js", + "testing/web-platform/meta/webtransport/__dir__.ini", + "toolkit/components/cleardata/ClearDataService.sys.mjs", + "toolkit/components/cleardata/tests/unit/test_identity_credential_storage.js", + "toolkit/components/formautofill/FormAutofillStorageBase.sys.mjs", + "toolkit/components/normandy/content/AboutPages.sys.mjs", + "toolkit/components/passwordmgr/FirefoxRelay.sys.mjs", + "toolkit/components/url-classifier/UrlClassifierListManager.jsm", + "toolkit/modules/NewTabUtils.sys.mjs", +] + +FILES_CHANGED = sorted( + FILES_CHANGED_MC + [os.path.join("comm", file) for file in FILES_CHANGED_CC] +) + + +class FakeResponse: + def __init__(self, url, **kwargs): + if "comm-central" in url: + self.filename = "cc_automationrelevance.json" + elif "mozilla-central" in url: + self.filename = "mc_automationrelevance.json" + else: + raise Exception(f"Invalid automation URL: {url}") + + def json(self): + with open(os.path.join(os.path.dirname(__file__), self.filename)) as f: + return json.load(f) + + +class TestGetChangedFiles(unittest.TestCase): + def setUp(self): + files_changed.get_changed_files.clear() + self.old_get = hg.requests.get + + def fake_get(url, **kwargs): + return FakeResponse(url) + + hg.requests.get = fake_get + + def tearDown(self): + hg.requests.get = self.old_get + files_changed.get_changed_files.clear() + + def test_get_changed_files_mc(self): + """Get_changed_files correctly gets the list of changed files in a push. + This tests against the production hg.mozilla.org so that it will detect + any changes in the format of the returned data.""" + self.assertEqual( + sorted(files_changed.get_changed_files(PARAMS["head_repository"], PARAMS["head_rev"])), + FILES_CHANGED_MC, + ) + + def test_get_changed_files_cc(self): + """Get_changed_files correctly gets the list of changed files in a push. + This tests against the production hg.mozilla.org so that it will detect + any changes in the format of the returned data.""" + self.assertEqual( + sorted( + files_changed.get_changed_files( + PARAMS["comm_head_repository"], PARAMS["comm_head_rev"] + ) + ), + FILES_CHANGED_CC, + ) + + def test_get_changed_files_extended(self): + """Get_changed_files_extended correctly gets the list of changed files in a push. + This tests against the production hg.mozilla.org so that it will detect + any changes in the format of the returned data.""" + self.assertEqual( + sorted(files_changed.get_files_changed_extended(PARAMS)), + FILES_CHANGED, + ) + + +class TestCheck(unittest.TestCase): + def setUp(self): + files_changed.get_changed_files[PARAMS["head_repository"], PARAMS["head_rev"]] = set( + FILES_CHANGED_MC + ) + files_changed.get_changed_files[ + PARAMS["comm_head_repository"], PARAMS["comm_head_rev"] + ] = set(FILES_CHANGED_CC) + + def tearDown(self): + files_changed.get_changed_files.clear() + + def test_check_no_params(self): + self.assertTrue(files_changed.check({}, ["ignored"])) + + def test_check_no_match(self): + self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"])) + self.assertFalse(files_changed.check(PARAMS, ["comm/nosuch/**"])) + + def test_check_match_mc(self): + self.assertTrue(files_changed.check(PARAMS, ["toolkit/**"])) + + def test_check_match_cc(self): + self.assertTrue(files_changed.check(PARAMS, ["comm/mail/**"])) + + +if __name__ == "__main__": + main() diff --git a/comm/taskcluster/comm_taskgraph/test/test_optimization_strategies.py b/comm/taskcluster/comm_taskgraph/test/test_optimization_strategies.py new file mode 100644 index 0000000000..8e43760714 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/test_optimization_strategies.py @@ -0,0 +1,143 @@ +# Any copyright is dedicated to the public domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import hashlib +from datetime import datetime +from time import mktime + +import conftest # noqa: F401 +import pytest +import responses +from mozunit import main +from taskgraph.task import Task + +from gecko_taskgraph.optimize import registry + +from comm_taskgraph.optimize import SkipSuiteOnly + + +def generate_task(): + task = {} + task.setdefault("label", "task-label") + task.setdefault("kind", "build") + task.setdefault("task", {}) + task.setdefault("attributes", {}) + + for attr in ( + "dependencies", + "optimization", + "soft_dependencies", + "release_artifacts", + ): + task.setdefault(attr, None) + + task["task"].setdefault("label", task["label"]) + return Task.from_json(task) + + +def idfn(param): + if isinstance(param, tuple): + return param[0].__name__ + return None + + +def generate_json_push_data(files_changed): + return {"changesets": [{"desc": "commit comment", "files": files_changed, "node": "cdefgh"}]} + + +@pytest.fixture +def params(): + return { + "branch": "comm-central", + "head_repository": "https://hg.mozilla.org/mozilla-central", + "head_rev": "zyxwvu", + "comm_head_repository": "https://hg.mozilla.org/comm-central", + "comm_head_rev": "abcdef", + "comm_src_path": "comm/", + "project": "comm-central", + "pushlog_id": 1, + "pushdate": mktime(datetime.now().timetuple()), + } + + +def mk_rev(strings): + data = "".join(strings).encode("utf-8") + h = hashlib.new("sha1") + h.update(data) + return h.hexdigest() + + +@responses.activate +@pytest.mark.parametrize( + "pushed_files,expected", + [ + # suite-only push + pytest.param(["suite/a/b/c.txt", "suite/b/c/d.txt"], True), + # non-suite push + pytest.param(["mail/a/b/c.txt", "mailnews/b/c/d.txt"], False), + # mixed push + pytest.param(["suite/a/b/c.txt", "calendar/b/c/d.txt"], False), + ], + ids=idfn, +) +def test_suite_only_strategy(params, pushed_files, expected): + rev = mk_rev(pushed_files) + params["comm_head_rev"] = rev + + responses.add( + responses.GET, + "https://hg.mozilla.org/comm-central/json-automationrelevance/{}".format(rev), + json=generate_json_push_data(pushed_files), + status=200, + ) + task = generate_task() + + opt = SkipSuiteOnly() + remove = opt.should_remove_task(task, params, None) + + assert remove == expected + + +@responses.activate +@pytest.mark.parametrize( + "file_patterns,pushed_files,expected", + [ + # suite-only push, matches files-changed + pytest.param(["comm/**/*.txt"], ["suite/a/b/c.txt", "suite/b/c/d.js"], True), + # suite-only push, does not match files-changed + pytest.param(["comm/**/*.cpp"], ["suite/a/b/c.txt", "suite/b/c/d.js"], True), + # non-suite push, matches files changed + pytest.param(["comm/**/*.txt"], ["mail/a/b/c.txt", "mailnews/b/c/d.js"], False), + # non-suite push, does not match files changed + pytest.param(["comm/**/*.cpp"], ["mail/a/b/c.txt", "mailnews/b/c/d.js"], True), + ], + ids=idfn, +) +def test_suite_files_changed_strategy(params, file_patterns, pushed_files, expected): + rev = mk_rev(pushed_files) + params["comm_head_rev"] = rev + + # Fake the m-c json data + responses.add( + responses.GET, + "https://hg.mozilla.org/mozilla-central/json-automationrelevance/zyxwvu", + json=generate_json_push_data([]), + status=200, + ) + + responses.add( + responses.GET, + "https://hg.mozilla.org/comm-central/json-automationrelevance/{}".format(rev), + json=generate_json_push_data(pushed_files), + status=200, + ) + task = generate_task() + + opt = registry["skip-unless-changed-no-suite"] + remove = opt.should_remove_task(task, params, file_patterns) + + assert remove == expected + + +if __name__ == "__main__": + main() diff --git a/comm/taskcluster/comm_taskgraph/test/test_parameters.py b/comm/taskcluster/comm_taskgraph/test/test_parameters.py new file mode 100644 index 0000000000..8a9d78701b --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/test/test_parameters.py @@ -0,0 +1,102 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +import conftest # noqa: F401 +from mozunit import main +from taskgraph.parameters import Parameters + +from comm_taskgraph.parameters import register_parameters + + +class TestCommParameters(unittest.TestCase): + + vals = { + "app_version": "app_version", + "backstop": False, + "base_repository": "base_repository", + "base_ref": "base_ref", + "base_rev": "base_rev", + "build_date": 0, + "build_number": 0, + "comm_base_repository": "comm_base_repository", + "comm_base_ref": "comm_base_ref", + "comm_base_rev": "comm_base_rev", + "comm_head_ref": "comm_head_ref", + "comm_head_repository": "comm_head_repository", + "comm_head_rev": "comm_head_rev", + "comm_src_path": "comm/", + "do_not_optimize": [], + "enable_always_target": False, + "existing_tasks": {}, + "filters": [], + "head_ref": "head_ref", + "head_repository": "head_repository", + "head_rev": "head_rev", + "head_tag": "", + "hg_branch": "hg_branch", + "level": "level", + "message": "message", + "moz_build_date": "moz_build_date", + "next_version": "next_version", + "optimize_strategies": None, + "optimize_target_tasks": False, + "owner": "owner", + "phabricator_diff": "phabricator_diff", + "project": "project", + "pushdate": 0, + "pushlog_id": "pushlog_id", + "release_enable_emefree": False, + "release_enable_partner_repack": False, + "release_enable_partner_attribution": False, + "release_eta": None, + "release_history": {}, + "release_partners": [], + "release_partner_config": None, + "release_partner_build_number": 1, + "release_type": "release_type", + "release_product": None, + "repository_type": "hg", + "required_signoffs": [], + "signoff_urls": {}, + "target_tasks_method": "target_tasks_method", + "test_manifest_loader": "default", + "tasks_for": "tasks_for", + "try_mode": "try_mode", + "try_options": None, + "try_task_config": {}, + "version": "version", + } + + def setUp(self): + register_parameters() + + def test_Parameters_check(self): + """ + Specifying all of the gecko and comm parameters doesn't result in an error. + """ + p = Parameters(**self.vals) + p.check() # should not raise + + def test_Parameters_check_missing(self): + """ + If any of the comm parameters are specified, all of them must be specified. + """ + vals = self.vals.copy() + del vals["comm_base_repository"] + p = Parameters(**vals) + self.assertRaises(Exception, p.check) + + def test_Parameters_check_extra(self): + """ + If parameters other than the global and comm parameters are specified, + an error is reported. + """ + p = Parameters(extra="data", **self.vals) + self.assertRaises(Exception, p.check) + + +if __name__ == "__main__": + main() diff --git a/comm/taskcluster/comm_taskgraph/transforms/__init__.py b/comm/taskcluster/comm_taskgraph/transforms/__init__.py new file mode 100644 index 0000000000..6fbe8159b2 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/comm/taskcluster/comm_taskgraph/transforms/job/__init__.py b/comm/taskcluster/comm_taskgraph/transforms/job/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py b/comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py new file mode 100644 index 0000000000..a80c058a2b --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py @@ -0,0 +1,165 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running toolchain-building jobs via dedicated scripts in comm-central +""" + +import os.path + +import taskgraph +import taskgraph.util.path as util_path +from taskgraph.util.schema import resolve_keyed_by +from voluptuous import Any, Optional, Required + +from gecko_taskgraph import GECKO +from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using +from gecko_taskgraph.transforms.job.common import docker_worker_add_artifacts +from gecko_taskgraph.transforms.job.toolchain import toolchain_defaults, toolchain_run_schema +from gecko_taskgraph.util.attributes import RELEASE_PROJECTS +from gecko_taskgraph.util.hash import hash_paths as hash_paths_gecko_root + +from comm_taskgraph.util.hash import hash_paths_extended + +CACHE_TYPE = "toolchains.v3" + +TOOLCHAIN_SCRIPT_PATH = "comm/taskcluster/scripts" + + +comm_toolchain_run_schema = toolchain_run_schema.extend( + { + Required("using"): Any("comm-toolchain-script"), + Optional("script"): str, + } +) + + +def hash_paths(*args): + """ + Helper function while the single repository project is in development. + The extended version of hash_paths found in comm_taskgraph.util.hash is + not necessary (and does not work) with single-repo. This is a wrapper + function to pick the right function based on the presence of a comm/.hg + directory. + """ + comm_hg_path = util_path.join(GECKO, "comm", ".hg") + if os.path.exists(comm_hg_path): + return hash_paths_extended(*args) + else: + return hash_paths_gecko_root(*args) + + +def get_digest_data(config, run, taskdesc): + """ + Copied from gecko_taskgraph.transforms.job.toolchain, with minor + modifications to support the required script path. + """ + files = list(run.pop("resources", [])) + # This file + files.append("comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py") + # The script + if "script" in run: + files.append("{}/{}".format(TOOLCHAIN_SCRIPT_PATH, run["script"])) + # Tooltool manifest if any is defined: + tooltool_manifest = taskdesc["worker"]["env"].get("TOOLTOOL_MANIFEST") + if tooltool_manifest: + files.append(tooltool_manifest) + + # Accumulate dependency hashes for index generation. + data = [hash_paths(GECKO, files)] + + data.append(taskdesc["attributes"]["toolchain-artifact"]) + + # If the task uses an in-tree docker image, we want it to influence + # the index path as well. Ideally, the content of the docker image itself + # should have an influence, but at the moment, we can't get that + # information here. So use the docker image name as a proxy. Not a lot of + # changes to docker images actually have an impact on the resulting + # toolchain artifact, so we'll just rely on such important changes to be + # accompanied with a docker image name change. + image = taskdesc["worker"].get("docker-image", {}).get("in-tree") + if image: + data.append(image) + + # Likewise script arguments should influence the index. + args = run.get("arguments") + if args: + data.extend(args) + + if taskdesc["attributes"].get("rebuild-on-release"): + # Add whether this is a release branch or not + data.append(str(config.params["project"] in RELEASE_PROJECTS)) + return data + + +@run_job_using( + "docker-worker", + "comm-toolchain-script", + schema=comm_toolchain_run_schema, + defaults=toolchain_defaults, +) +def docker_worker_toolchain(config, job, taskdesc): + run = job["run"] + run["comm-checkout"] = True + + worker = taskdesc["worker"] = job["worker"] + worker["chain-of-trust"] = True + + # If the task doesn't have a docker-image, set a default + worker.setdefault("docker-image", {"in-tree": "deb11-toolchain-build"}) + + # Toolchain checkouts don't live under {workdir}/checkouts + workspace = "{workdir}/workspace/build".format(**run) + gecko_path = "{}/src".format(workspace) + + env = worker.setdefault("env", {}) + env.update( + { + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + "GECKO_PATH": gecko_path, + "TOOLCHAIN_ARTIFACT": run["toolchain-artifact"], + } + ) + + attributes = taskdesc.setdefault("attributes", {}) + attributes["toolchain-artifact"] = run.pop("toolchain-artifact") + toolchain_artifact = attributes["toolchain-artifact"] + if not toolchain_artifact.startswith("public/build/"): + attributes["artifact_prefix"] = os.path.dirname(toolchain_artifact) + + resolve_keyed_by( + run, + "toolchain-alias", + item_name=taskdesc["label"], + project=config.params["project"], + ) + alias = run.pop("toolchain-alias", None) + if alias: + attributes["toolchain-alias"] = alias + if "toolchain-env" in run: + attributes["toolchain-env"] = run.pop("toolchain-env") + + # Allow the job to specify where artifacts come from, but add + # public/build if it's not there already. + artifacts = worker.setdefault("artifacts", []) + if not artifacts: + docker_worker_add_artifacts(config, job, taskdesc) + + digest_data = get_digest_data(config, run, taskdesc) + + if job.get("attributes", {}).get("cached_task") is not False and not taskgraph.fast: + name = taskdesc["label"].replace(f"{config.kind}-", "", 1) + taskdesc["cache"] = { + "type": CACHE_TYPE, + "name": name, + "digest-data": digest_data, + } + + run["using"] = "run-task" + run["cwd"] = run["workdir"] + run["command"] = [ + "workspace/build/src/{}/{}".format(TOOLCHAIN_SCRIPT_PATH, run.pop("script")) + ] + run.pop("arguments", []) + + configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) diff --git a/comm/taskcluster/comm_taskgraph/transforms/l10n.py b/comm/taskcluster/comm_taskgraph/transforms/l10n.py new file mode 100644 index 0000000000..51246ad888 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/l10n.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Do transforms specific to l10n kind +""" + +from taskgraph.transforms.base import TransformSequence, ValidateSchema +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.transforms.l10n import ( + all_locales_attribute, + chunk_locales, + copy_in_useful_magic, + handle_artifact_prefix, + handle_keyed_by, + l10n_description_schema, + make_job_description, + set_extra_config, + setup_name, +) + + +def setup_signing_dependency(config, jobs): + """Sets up a task dependency to the signing job this relates to""" + for job in jobs: + job["dependencies"].update( + { + "build": job["dependent-tasks"]["build"].label, + } + ) + + if job["attributes"]["build_platform"].startswith("win"): + job["dependencies"].update( + { + "build-signing": job["dependent-tasks"]["build-signing"].label, + } + ) + + if "shippable" in job["attributes"]["build_platform"]: + if job["attributes"]["build_platform"].startswith("macosx"): + job["dependencies"].update( + {"repackage": job["dependent-tasks"]["repackage"].label} + ) + if job["attributes"]["build_platform"].startswith("linux"): + job["dependencies"].update( + { + "build-signing": job["dependent-tasks"]["build-signing"].label, + } + ) + yield job + + +def handle_keyed_by_local(config, jobs): + """Resolve fields that can be keyed by platform, etc.""" + for job in jobs: + resolve_keyed_by( + job, + "locales-file", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + yield job + + +transforms = TransformSequence() + + +for transform_func in ( + setup_name, + copy_in_useful_magic, + handle_keyed_by_local, + ValidateSchema(l10n_description_schema), + setup_signing_dependency, + handle_keyed_by, + handle_artifact_prefix, + all_locales_attribute, + chunk_locales, + ValidateSchema(l10n_description_schema), + set_extra_config, + make_job_description, +): + transforms.add(transform_func) diff --git a/comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py b/comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py new file mode 100644 index 0000000000..a642b0e592 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Create a strings build artifact to be consumed by shippable-l10n. +""" + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def handle_keyed_by(config, jobs): + """Resolve fields that can be keyed by platform, etc.""" + for job in jobs: + resolve_keyed_by( + job, + "locale-list", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + yield job + + +@transforms.add +def make_job_description(config, jobs): + for job in jobs: + locale_list = job.pop("locale-list") + comm_locales_file = job.pop("comm-locales-file") + browser_locales_file = job.pop("browser-locales-file") + job["run"].update( + { + "job-script": "comm/taskcluster/scripts/build-l10n-pre.sh", + "options": [ + f"locale-list={locale_list}", + f"comm-locales-file={comm_locales_file}", + f"browser-locales-file={browser_locales_file}", + ], + } + ) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py b/comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py new file mode 100644 index 0000000000..92a95cb612 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the signing task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.taskcluster import get_artifact_path + +from gecko_taskgraph.transforms.build_signing import add_signed_routes +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job + +transforms = TransformSequence() + +transforms.add(add_signed_routes) + + +@transforms.add +def define_upstream_artifacts(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + upstream_artifact_task = job.pop("upstream-artifact-task", dep_job) + + job["attributes"] = copy_attributes_from_dependent_job(dep_job) + + artifacts_specifications = [ + { + "artifacts": [ + get_artifact_path(job, "strings_all.tar.zst"), + get_artifact_path(job, "l10n-changesets.json"), + ], + "formats": ["autograph_gpg"], + } + ] + + task_ref = f"<{upstream_artifact_task.kind}>" + task_type = "build" + if "notarization" in upstream_artifact_task.kind: + task_type = "scriptworker" + + job["upstream-artifacts"] = [ + { + "taskId": {"task-reference": task_ref}, + "taskType": task_type, + "paths": spec["artifacts"], + "formats": spec["formats"], + } + for spec in artifacts_specifications + ] + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/merge_automation.py b/comm/taskcluster/comm_taskgraph/transforms/merge_automation.py new file mode 100644 index 0000000000..75ced147bb --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/merge_automation.py @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Fix-ups for comm-central merge automation +""" + +import os +import re + +from taskgraph.transforms.base import TransformSequence + +from comm_taskgraph import COMM + +transforms = TransformSequence() + + +def do_suite_verbump(replacements): + """Bump the minor version of suite version files.""" + allowed_files = ("suite/config/version.txt", "suite/config/version_display.txt") + old_version, new_version = None, None + + new_replacements = [] + for file, old, new in replacements: + if file not in allowed_files: + break + if old_version is None or new_version is None: + path = os.path.join(COMM, file) + data = open(path).read() + match = re.match(r"^(2)\.(\d+)(a1)$", data) + if match: + old_version = match.group(0) + + old_minor = match.group(2) + new_minor = str(int(old_minor) + 1) + + new_version = f"{match.group(1)}.{new_minor}{match.group(3)}" + + new_replacements.append([file, old_version, new_version]) + + if len(new_replacements) == len(replacements): + return new_replacements + else: + raise Exception(f"do_suite_version failed: {replacements}, {new_replacements}") + + +@transforms.add +def update_suite_versions(config, tasks): + for task in tasks: + if "merge_config" not in config.params: + break + behavior = config.params["merge_config"]["behavior"] + if behavior == "comm-bump-central": + merge_config = task["worker"]["merge-info"] + replacements = merge_config["replacements"] + merge_config["replacements"] = do_suite_verbump(replacements) + + yield task diff --git a/comm/taskcluster/comm_taskgraph/transforms/partials.py b/comm/taskcluster/comm_taskgraph/transforms/partials.py new file mode 100644 index 0000000000..39d6cf95a7 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/partials.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Thunderbird modifications to partial update building +""" +import logging + +from taskgraph.transforms.base import TransformSequence + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def update_scopes(config, jobs): + """ + Firefox does some caching when building partial updates, but there's no bucket for Thunderbird + at the moment. In the meantime, remove the scope from the task to avoid an error. + """ + # If no balrog release history, then don't run + if not config.params.get("release_history"): + return + + MBSDIFF_SCOPE = "auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/" + + for job in jobs: + task = job["task"] + if MBSDIFF_SCOPE in task["scopes"]: + task["scopes"].remove(MBSDIFF_SCOPE) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py b/comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py new file mode 100644 index 0000000000..8f6c7a5d5f --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py @@ -0,0 +1,231 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the release-push-langpacks task into an actual task description. +""" + +import json +import os +from contextlib import contextmanager + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, taskref_or_string +from taskgraph.util.treeherder import inherit_treeherder_from_dep +from voluptuous import Any, Optional, Required + +from gecko_taskgraph.loader.single_dep import schema +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import ( + copy_attributes_from_dependent_job, + release_level, +) +from mozbuild.action.langpack_manifest import get_version_maybe_buildid + +transforms = TransformSequence() + +langpack_push_description_schema = schema.extend( + { + Required("label"): str, + Required("description"): str, + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("worker"): { + Required("docker-image"): {"in-tree": str}, + Required("implementation"): "docker-worker", + Required("os"): "linux", + Optional("max-run-time"): int, + Required("env"): {str: taskref_or_string}, + Required("channel"): optionally_keyed_by( + "project", "platform", Any("listed", "unlisted") + ), + Required("command"): [taskref_or_string], + }, + Required("run-on-projects"): [], + Required("scopes"): optionally_keyed_by("release-level", [str]), + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + } +) + + +@transforms.add +def set_label(config, jobs): + for job in jobs: + label = "push-langpacks-{}".format(job["primary-dependency"].label) + job["label"] = label + + yield job + + +transforms.add_validate(langpack_push_description_schema) + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + resolve_keyed_by( + job, + "worker-type", + item_name=job["label"], + **{"release-level": release_level(config.params["project"])}, + ) + resolve_keyed_by( + job, + "scopes", + item_name=job["label"], + **{"release-level": release_level(config.params["project"])}, + ) + resolve_keyed_by( + job, + "worker.channel", + item_name=job["label"], + project=config.params["project"], + platform=job["primary-dependency"].attributes["build_platform"], + ) + + yield job + + +@transforms.add +def copy_attributes(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + job["attributes"] = copy_attributes_from_dependent_job(dep_job) + job["attributes"]["chunk_locales"] = dep_job.attributes.get("chunk_locales", ["en-US"]) + + yield job + + +@transforms.add +def filter_out_macos_jobs_but_mac_only_locales(config, jobs): + for job in jobs: + build_platform = job["primary-dependency"].attributes.get("build_platform") + + if build_platform == "linux64-shippable": + yield job + elif ( + build_platform == "macosx64-shippable" + and "ja-JP-mac" in job["attributes"]["chunk_locales"] + ): + # Other locales of the same job shouldn't be processed + job["attributes"]["chunk_locales"] = ["ja-JP-mac"] + job["label"] = job["label"].replace( + # Guard against a chunk 10 or chunk 1 (latter on try) weird munging + "-{}/".format(job["attributes"]["l10n_chunk"]), + "-ja-JP-mac/", + ) + yield job + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job["primary-dependency"] + + treeherder = inherit_treeherder_from_dep(job, dep_job) + treeherder.setdefault( + "symbol", "langpack(P{})".format(job["attributes"].get("l10n_chunk", "")) + ) + + job["description"] = job["description"].format( + locales="/".join(job["attributes"]["chunk_locales"]), + ) + + job["dependencies"] = {dep_job.kind: dep_job.label} + job["treeherder"] = treeherder + + yield job + + +def generate_upstream_artifacts(upstream_task_ref, locales): + return [ + { + "task": upstream_task_ref, + "extract": False, + "dest": f"{locale}", + "artifact": "public/build{locale}/target.langpack.xpi".format( + locale="" if locale == "en-US" else "/" + locale + ), + } + for locale in locales + ] + + +@transforms.add +def make_fetches(config, jobs): + for job in jobs: + upstream_task_ref = get_upstream_task_ref(job, expected_kinds=("build", "shippable-l10n")) + + worker = job.setdefault("worker", {}) + worker["taskcluster-proxy"] = True + + env = worker.setdefault("env", {}) + + job_fetches = generate_upstream_artifacts( + upstream_task_ref, job["attributes"]["chunk_locales"] + ) + env["MOZ_FETCHES"] = { + "task-reference": json.dumps( + sorted(job_fetches, key=lambda x: sorted(x.items())), sort_keys=True + ) + } + env["MOZ_SCM_LEVEL"] = config.params["level"] + + yield job + + +def get_upstream_task_ref(job, expected_kinds): + upstream_tasks = [ + job_kind for job_kind in job["dependencies"].keys() if job_kind in expected_kinds + ] + + if len(upstream_tasks) > 1: + raise Exception("Only one dependency expected") + + return f"<{upstream_tasks[0]}>" + + +@contextmanager +def environment(key, value): + """Set an environment variable in a context""" + old_value = None + if key in os.environ: + old_value = os.environ[key] + os.environ[key] = value + try: + yield True + finally: + if old_value is None: + del os.environ[key] + else: + os.environ[key] = old_value + + +@transforms.add +def set_env(config, jobs): + buildid = config.params["moz_build_date"] + app_version = config.params.get("app_version") + + with environment("MOZ_BUILD_DATE", buildid): + langpack_version = get_version_maybe_buildid(app_version) + + for job in jobs: + job["worker"].get("env", {}).update( + { + "LANGPACK_VERSION": langpack_version, + "LOCALES": json.dumps(job["attributes"]["chunk_locales"]), + "MOZ_FETCHES_DIR": "fetches", + "ATN_CHANNEL": job["worker"].get("channel"), + } + ) + + yield job + + +@transforms.add +def strip_unused_data(config, jobs): + for job in jobs: + del job["primary-dependency"] + del job["worker"]["channel"] + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py b/comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py new file mode 100644 index 0000000000..c4f39fc4eb --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the release-flatpak-push kind into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from voluptuous import Optional, Required + +from gecko_taskgraph.transforms.task import task_description_schema +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import add_scope_prefix + +push_flatpak_description_schema = Schema( + { + Required("name"): str, + Required("job-from"): task_description_schema["job-from"], + Required("dependencies"): task_description_schema["dependencies"], + Required("description"): task_description_schema["description"], + Required("treeherder"): task_description_schema["treeherder"], + Required("run-on-projects"): task_description_schema["run-on-projects"], + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("worker"): object, + Optional("scopes"): [str], + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + Required("flathub-scope"): str, + Optional("extra"): task_description_schema["extra"], + Optional("attributes"): task_description_schema["attributes"], + } +) + +transforms = TransformSequence() +transforms.add_validate(push_flatpak_description_schema) + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + if len(job["dependencies"]) != 1: + raise Exception("Exactly 1 dependency is required") + + job["worker"]["upstream-artifacts"] = generate_upstream_artifacts(job["dependencies"]) + + resolve_keyed_by( + job, + "worker.channel", + item_name=job["name"], + **{"release-type": config.params["release_type"]}, + ) + resolve_keyed_by( + job, + "worker-type", + item_name=job["name"], + **{"release-level": release_level(config.params["project"])}, + ) + if release_level(config.params["project"]) == "production": + job.setdefault("scopes", []).append( + add_scope_prefix( + config, + "{}:{}".format(job["flathub-scope"], job["worker"]["channel"]), + ) + ) + del job["flathub-scope"] + + yield job + + +def generate_upstream_artifacts(dependencies): + return [ + { + "taskId": {"task-reference": f"<{task_kind}>"}, + "taskType": "build", + "paths": ["public/build/target.flatpak.tar.xz"], + } + for task_kind in dependencies.keys() + ] diff --git a/comm/taskcluster/comm_taskgraph/transforms/release_started.py b/comm/taskcluster/comm_taskgraph/transforms/release_started.py new file mode 100644 index 0000000000..ce0cfa9fab --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/release_started.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Add notifications via taskcluster-notify for release tasks +""" + +from pipes import quote as shell_quote + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def add_notifications(config, jobs): + for job in jobs: + label = "{}-{}".format(config.kind, job["name"]) + + resolve_keyed_by(job, "emails", label, project=config.params["project"]) + emails = [email.format(config=config.__dict__) for email in job.pop("emails")] + + command = [ + "release", + "send-buglist-email", + "--version", + config.params["version"], + "--product", + job["shipping-product"], + "--revision", + config.params["comm_head_rev"], + "--build-number", + str(config.params["build_number"]), + "--repo", + config.params["comm_head_repository"], + ] + for address in emails: + command += ["--address", address] + command += [ + # We wrap this in `{'task-reference': ...}` below + "--task-group-id", + "", + ] + + job["scopes"] = ["notify:email:{}".format(address) for address in emails] + job["run"] = { + "using": "mach", + "sparse-profile": "mach", + "mach": {"task-reference": " ".join(map(shell_quote, command))}, + } + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py b/comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py new file mode 100644 index 0000000000..793519f6ff --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from taskgraph.transforms.base import TransformSequence + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def add_langpack_fetches(config, jobs): + """Adds the fetch configuration for the langpacks. This is done here + because Thunderbird langpacks are not signed and therefore not found as + artifacts of "shippable-l10n-signing" like they are for Firefox. Need to + use "shippable-l10n". + """ + + def depends_filter(dep_task): + return ( + dep_task.kind == "shippable-l10n" + and dep_task.attributes["build_platform"] == "linux64-shippable" + and dep_task.attributes["build_type"] == "opt" + ) + + for job in jobs: + dependencies = job.get("dependencies", {}) + fetches = job.setdefault("fetches", {}) + + # The keys are unique, like `shippable-l10n-linux64-shippable-1/opt`, so we + # can't ask for the tasks directly, we must filter for them. + for t in filter(depends_filter, config.kind_dependencies_tasks.values()): + dependencies.update({t.label: t.label}) + + fetches.update( + { + t.label: [ + { + "artifact": f"{loc}/target.langpack.xpi", + "extract": False, + # Otherwise we can't disambiguate locales! + "dest": f"distribution/extensions/{loc}", + } + for loc in t.attributes["chunk_locales"] + ] + } + ) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/signing.py b/comm/taskcluster/comm_taskgraph/transforms/signing.py new file mode 100644 index 0000000000..297fec0d2e --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/signing.py @@ -0,0 +1,88 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.signed_artifacts import is_notarization_kind + +transforms = TransformSequence() + + +def check_notarization(dependencies): + """ + Determine whether a signing job is the last step of a notarization + by looking at its dependencies. + """ + for dep in dependencies: + if is_notarization_kind(dep): + return True + + +@transforms.add +def remove_widevine(config, jobs): + """ + Remove references to widevine signing. + + This is to avoid adding special cases for handling signed artifacts + in mozilla-central code. Artifact signature formats are determined in + gecko_taskgraph.util.signed_artifacts. There's no override mechanism so we + remove the autograph_widevine format here. + """ + for job in jobs: + task = job["task"] + payload = task["payload"] + + widevine_scope = "project:comm:thunderbird:releng:signing:format:autograph_widevine" + if widevine_scope in task["scopes"]: + task["scopes"].remove(widevine_scope) + if "upstreamArtifacts" in payload: + for artifact in payload["upstreamArtifacts"]: + if "autograph_widevine" in artifact.get("formats", []): + artifact["formats"].remove("autograph_widevine") + + yield job + + +@transforms.add +def no_sign_langpacks(config, jobs): + """ + Remove langpacks from signing jobs after they are automatically added. + """ + for job in jobs: + task = job["task"] + payload = task["payload"] + + if "upstreamArtifacts" in payload: + for artifact in payload["upstreamArtifacts"]: + if "autograph_langpack" in artifact.get("formats", []): + artifact["formats"].remove("autograph_langpack") + + # Make sure that there are no .xpi files in the artifact list + if all([p.endswith("target.langpack.xpi") for p in artifact["paths"]]): + payload["upstreamArtifacts"].remove(artifact) + + yield job + + +@transforms.add +def check_for_no_formats(config, jobs): + """ + Check for signed artifacts without signature formats and remove them to + avoid scriptworker errors. + Signing jobs that use macOS notarization do not have formats, so keep + those. + """ + for job in jobs: + if not check_notarization(job["dependencies"]): + task = job["task"] + payload = task["payload"] + + if "upstreamArtifacts" in payload: + for artifact in payload["upstreamArtifacts"]: + if "formats" in artifact and not artifact["formats"]: + for remove_path in artifact["paths"]: + job["release-artifacts"].remove(remove_path) + + payload["upstreamArtifacts"].remove(artifact) + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/source_test.py b/comm/taskcluster/comm_taskgraph/transforms/source_test.py new file mode 100644 index 0000000000..d4ae39c134 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/source_test.py @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import shlex + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.path import join as join_path +from taskgraph.util.path import match as match_path + +from gecko_taskgraph.files_changed import get_changed_files + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +def get_patterns(job): + """Get the "run on-changed" file patterns.""" + optimization = job.get("optimization", {}) + if optimization: + return optimization.copy().popitem()[1] + return [] + + +def shlex_join(split_command): + """shlex.join from Python 3.8+""" + return " ".join(shlex.quote(arg) for arg in split_command) + + +@transforms.add +def changed_clang_format(config, jobs): + """ + Transform for clang-format job to set the commandline to only check + C++ files that were changed in the current push rather than running on + the entire repository. + """ + for job in jobs: + if job.get("name", "") == "clang-format": + repository = config.params.get("comm_head_repository") + revision = config.params.get("comm_head_rev") + + match_patterns = get_patterns(job) + changed_files = { + join_path("comm", file) for file in get_changed_files(repository, revision) + } + + cpp_files = [] + for pattern in match_patterns: + for path in changed_files: + if match_path(path, pattern): + cpp_files.append(path) + + # In the event that no C/C++ files were changed in the current push, + # the commandline will end up being invalid. But, the clang-format + # job will get dropped by optimization, so it doesn't really matter. + if cpp_files: + job["run"]["command"] = job["run"]["command"].format( + changed_files=shlex_join(cpp_files) + ) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/tb_build.py b/comm/taskcluster/comm_taskgraph/transforms/tb_build.py new file mode 100644 index 0000000000..eff16d8e69 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/tb_build.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from taskgraph.transforms.base import TransformSequence + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def munge_environment(config, jobs): + for job in jobs: + env = job["worker"]["env"] + # Remove MOZ_SOURCE_CHANGESET/REPO from the job environment and discard + # if present. Having these variables set in the environment causes problems + # with generating debug sym files. Bug 1747879. + env.pop("MOZ_SOURCE_CHANGESET", None) + env.pop("MOZ_SOURCE_REPO", None) + + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py b/comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py new file mode 100644 index 0000000000..ecdfd577af --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Build a command to run `mach tb-l10n-x-channel`. +""" + +from shlex import quote as shell_quote + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from rocbuild.notify import TB_BUILD_ADDR + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, jobs): + for job in jobs: + for item in ["ssh-key-secret", "run.actions"]: + resolve_keyed_by(job, item, item, **{"level": str(config.params["level"])}) + yield job + + +@transforms.add +def build_command(config, jobs): + for job in jobs: + command = [ + "tb-l10n-x-channel", + "-o", + "/builds/worker/artifacts/outgoing.diff", + "--attempts", + "5", + ] + ssh_key_secret = job.pop("ssh-key-secret") + if ssh_key_secret: + command.extend(["--ssh-secret", ssh_key_secret]) + job.setdefault("scopes", []).append(f"secrets:get:{ssh_key_secret}") + job["scopes"].append(f"notify:email:{TB_BUILD_ADDR}") + + command.extend(job["run"].pop("actions", [])) + job.setdefault("run", {}).update( + {"using": "mach", "mach": " ".join(map(shell_quote, command))} + ) + yield job diff --git a/comm/taskcluster/comm_taskgraph/transforms/tests.py b/comm/taskcluster/comm_taskgraph/transforms/tests.py new file mode 100644 index 0000000000..9c1df4663e --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/tests.py @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Thunderbird modifications to test jobs +""" + +import logging + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def optimization_keyed_by(config, tasks): + """Used to set the optimization strategy""" + for task in tasks: + resolve_keyed_by(task, "optimization", item_name=task["test-name"]) + yield task diff --git a/comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py b/comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py new file mode 100644 index 0000000000..2f9ee143fa --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py @@ -0,0 +1,134 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +from urllib.parse import urlsplit + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +from gecko_taskgraph.transforms.task import get_branch_repo, get_branch_rev +from gecko_taskgraph.transforms.update_verify_config import ensure_wrapped_singlequote +from gecko_taskgraph.util.attributes import release_level +from gecko_taskgraph.util.scriptworker import get_release_config + +transforms = TransformSequence() + + +# The beta regexes do not match point releases. +# In the rare event that we do ship a point +# release to beta, we need to either: +# 1) update these regexes to match that specific version +# 2) pass a second include version that matches that specific version +INCLUDE_VERSION_REGEXES = { + "beta": r"'^(\d+\.\d+b\d+)$'", + "nonbeta": r"'^\d+\.\d+(\.\d+)?$'", + # Previous major versions, for update testing before we update users to a new esr + "release-next": r"'^(91|102)\.\d+(\.\d+)?$'", +} + +MAR_CHANNEL_ID_OVERRIDE_REGEXES = { + "beta": r"'^\d+\.\d+(\.\d+)?$$,thunderbird-comm-beta,thunderbird-comm-release'", +} + + +ensure_wrapped_singlequote(INCLUDE_VERSION_REGEXES) +ensure_wrapped_singlequote(MAR_CHANNEL_ID_OVERRIDE_REGEXES) + + +@transforms.add +def add_command(config, tasks): + keyed_by_args = [ + "channel", + "archive-prefix", + "previous-archive-prefix", + "aus-server", + "override-certs", + "include-version", + "mar-channel-id-override", + "last-watershed", + ] + optional_args = [ + "updater-platform", + ] + + release_config = get_release_config(config) + + for task in tasks: + task["description"] = "generate update verify config for {}".format( + task["attributes"]["build_platform"] + ) + + command = [ + "python", + "testing/mozharness/scripts/release/update-verify-config-creator.py", + "--product", + task["extra"]["product"], + "--stage-product", + task["shipping-product"], + "--app-name", + task["extra"]["app-name"], + "--branch-prefix", + task["extra"]["branch-prefix"], + "--platform", + task["extra"]["platform"], + "--to-version", + release_config["version"], + "--to-app-version", + release_config["appVersion"], + "--to-build-number", + str(release_config["build_number"]), + "--to-buildid", + config.params["moz_build_date"], + "--to-revision", + get_branch_rev(config), + "--output-file", + "update-verify.cfg", + ] + + repo_path = urlsplit(get_branch_repo(config)).path.lstrip("/") + command.extend(["--repo-path", repo_path]) + + if release_config.get("partial_versions"): + for partial in release_config["partial_versions"].split(","): + command.extend(["--partial-version", partial.split("build")[0]]) + + for arg in optional_args: + if task["extra"].get(arg): + command.append(f"--{arg}") + command.append(task["extra"][arg]) + + for arg in keyed_by_args: + thing = f"extra.{arg}" + resolve_keyed_by( + task, + thing, + item_name=task["name"], + platform=task["attributes"]["build_platform"], + **{ + "release-type": config.params["release_type"], + "release-level": release_level(config.params["project"]), + }, + ) + # ignore things that resolved to null + if not task["extra"].get(arg): + continue + if arg == "include-version": + task["extra"][arg] = INCLUDE_VERSION_REGEXES[task["extra"][arg]] + if arg == "mar-channel-id-override": + task["extra"][arg] = MAR_CHANNEL_ID_OVERRIDE_REGEXES[task["extra"][arg]] + + command.append(f"--{arg}") + command.append(task["extra"][arg]) + + task["run"].update( + { + "using": "mach", + "mach": " ".join(command), + } + ) + + yield task diff --git a/comm/taskcluster/comm_taskgraph/try_option_syntax.py b/comm/taskcluster/comm_taskgraph/try_option_syntax.py new file mode 100644 index 0000000000..b401bfa7a2 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/try_option_syntax.py @@ -0,0 +1,97 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging + +from gecko_taskgraph.target_tasks import ( + filter_by_uncommon_try_tasks, + filter_unsupported_artifact_builds, +) +from gecko_taskgraph.try_option_syntax import TryOptionSyntax + +logger = logging.getLogger(__name__) + + +class TryCCOptionSyntax(TryOptionSyntax): + """ + Override parse_platforms in the superclass. Removed the attempt to replace + macosx64 jobs with macosx64-shippable. + """ + + def parse_platforms(self, options, full_task_graph): + platform_arg = options["platforms"] + if platform_arg == "all": + return None + + results = [] + for build in platform_arg.split(","): + results.append(build) + + test_platforms = { + t.attributes["test_platform"] + for t in full_task_graph.tasks.values() + if "test_platform" in t.attributes + } + build_platforms = { + t.attributes["build_platform"] + for t in full_task_graph.tasks.values() + if "build_platform" in t.attributes + } + all_platforms = test_platforms | build_platforms + bad_platforms = set(results) - all_platforms + if bad_platforms: + raise Exception("Unknown platform(s) [%s] specified for try" % ",".join(bad_platforms)) + + return results + + +def _try_cc_option_syntax(full_task_graph, parameters, graph_config): + """Generate a list of target tasks based on try syntax in + parameters['message'] and, for context, the full task graph. + + Based on gecko_taskgraph.target_tasks._try_option_syntax. Removed talos + and raptor references and use TryCCOptionSyntax. + """ + options = TryCCOptionSyntax(parameters, full_task_graph, graph_config) + target_tasks_labels = [ + t.label + for t in full_task_graph.tasks.values() + if options.task_matches(t) + and filter_by_uncommon_try_tasks(t.label) + and filter_unsupported_artifact_builds(t, parameters) + ] + + attributes = { + k: getattr(options, k) + for k in [ + "no_retry", + "tag", + ] + } + + for l in target_tasks_labels: + task = full_task_graph[l] + if "unittest_suite" in task.attributes: + task.attributes["task_duplicates"] = options.trigger_tests + + for l in target_tasks_labels: + task = full_task_graph[l] + # If the developer wants test jobs to be rebuilt N times we add that value here + if options.trigger_tests > 1 and "unittest_suite" in task.attributes: + task.attributes["task_duplicates"] = options.trigger_tests + + task.attributes.update(attributes) + + # Add notifications here as well + if options.notifications: + for task in full_task_graph: + owner = parameters.get("owner") + routes = task.task.setdefault("routes", []) + if options.notifications == "all": + routes.append(f"notify.email.{owner}.on-any") + elif options.notifications == "failure": + routes.append(f"notify.email.{owner}.on-failed") + routes.append(f"notify.email.{owner}.on-exception") + + return target_tasks_labels diff --git a/comm/taskcluster/comm_taskgraph/util/__init__.py b/comm/taskcluster/comm_taskgraph/util/__init__.py new file mode 100644 index 0000000000..71f5dc9cd0 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/util/__init__.py @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +def strip_comm_prefix(relpath): + """ + Returns relpath with 'comm/' prefix removed. + :param string relpath: relative path + :return string: stripped path + """ + if relpath[:5] == "comm/": + return relpath[5:] + else: + return relpath diff --git a/comm/taskcluster/comm_taskgraph/util/docker.py b/comm/taskcluster/comm_taskgraph/util/docker.py new file mode 100644 index 0000000000..07971f3f28 --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/util/docker.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os + +from gecko_taskgraph.util import docker as utildocker + +from .. import COMM + +logger = logging.getLogger(__name__) + +COMM_IMAGE_DIR = os.path.join(COMM, "taskcluster", "docker") + + +def register(): + logger.info("Registering comm docker image definition path.") + utildocker.image_paths.register( + "comm/taskcluster/ci/docker-image/docker-image.yml", COMM_IMAGE_DIR + ) + + +register() diff --git a/comm/taskcluster/comm_taskgraph/util/hash.py b/comm/taskcluster/comm_taskgraph/util/hash.py new file mode 100644 index 0000000000..d372a0192a --- /dev/null +++ b/comm/taskcluster/comm_taskgraph/util/hash.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import hashlib + +import taskgraph.util.path as util_path + +from gecko_taskgraph.util.hash import get_file_finder, hash_path + + +def split_patterns_list(patterns): + """ + Give a list of path patterns and return two lists. rv[0] corresponds to files from the + GECKO repository, rv[1] corresponds to COMM. + The pattern list for the COMM repository will have *not* the 'comm/' prefix stripped. + """ + return [ + [p for p in patterns if not p.startswith("comm/")], + [p for p in patterns if p.startswith("comm/")], + ] + + +def prefix_paths(prefix, paths): + """ + Prepend a prefix to a list of paths. + """ + return [util_path.join(prefix, p) for p in paths] + + +def process_found(found_gen, prefix=None): + """ + Transform the results from finder.find(pattern) into a list of files. + If prefix is given, prepend it to results. + """ + for filename, fileobj in found_gen: + if prefix: + yield util_path.join(prefix, filename) + else: + yield filename + + +def hash_paths_extended(base_path, patterns): + """ + Works like gecko_taskgraph.util.hash.hash_paths, except it is able to account for Thunderbird + source code being part of a separate repository. + Two file finders are created if necessary. + """ + gecko_patterns, comm_patterns = split_patterns_list(patterns) + gecko_finder = get_file_finder(base_path) + comm_finder = get_file_finder(util_path.join(base_path, "comm")) + + h = hashlib.sha256() + files = [] + for (patterns, finder, prefix) in [ + (gecko_patterns, gecko_finder, None), + (comm_patterns, comm_finder, "comm/"), + ]: + for pattern in patterns: + if prefix: + pattern = pattern.lstrip(prefix) + found = list(process_found(finder.find(pattern), prefix)) + if found: + files.extend(found) + else: + raise Exception("%s did not match anything" % pattern) + for path in sorted(files): + if path.endswith((".pyc", ".pyd", ".pyo")): + continue + h.update( + "{} {}\n".format( + hash_path(util_path.abspath(util_path.join(base_path, path))), + util_path.normsep(path), + ).encode("utf-8") + ) + return h.hexdigest() -- cgit v1.2.3