summaryrefslogtreecommitdiffstats
path: root/comm/taskcluster/comm_taskgraph
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
commit6bf0a5cb5034a7e684dcc3500e841785237ce2dd (patch)
treea68f146d7fa01f0134297619fbe7e33db084e0aa /comm/taskcluster/comm_taskgraph
parentInitial commit. (diff)
downloadthunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.tar.xz
thunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.zip
Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'comm/taskcluster/comm_taskgraph')
-rw-r--r--comm/taskcluster/comm_taskgraph/__init__.py45
-rw-r--r--comm/taskcluster/comm_taskgraph/actions.py38
-rw-r--r--comm/taskcluster/comm_taskgraph/decision.py162
-rw-r--r--comm/taskcluster/comm_taskgraph/documentation.py12
-rw-r--r--comm/taskcluster/comm_taskgraph/files_changed.py114
-rw-r--r--comm/taskcluster/comm_taskgraph/loader/__init__.py3
-rw-r--r--comm/taskcluster/comm_taskgraph/loader/merge.py29
-rw-r--r--comm/taskcluster/comm_taskgraph/loader/reference.py93
-rw-r--r--comm/taskcluster/comm_taskgraph/manifests/release_checksums.yml68
-rw-r--r--comm/taskcluster/comm_taskgraph/manifests/source_checksums.yml46
-rw-r--r--comm/taskcluster/comm_taskgraph/manifests/source_files.yml46
-rw-r--r--comm/taskcluster/comm_taskgraph/manifests/strings_source.yml64
-rw-r--r--comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates.yml334
-rw-r--r--comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates_checksums.yml83
-rw-r--r--comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly.yml434
-rw-r--r--comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly_checksums.yml48
-rw-r--r--comm/taskcluster/comm_taskgraph/optimize.py151
-rw-r--r--comm/taskcluster/comm_taskgraph/parameters.py42
-rw-r--r--comm/taskcluster/comm_taskgraph/target_tasks.py75
-rw-r--r--comm/taskcluster/comm_taskgraph/test/cc_automationrelevance.json154
-rw-r--r--comm/taskcluster/comm_taskgraph/test/conftest.py7
-rw-r--r--comm/taskcluster/comm_taskgraph/test/mc_automationrelevance.json309
-rw-r--r--comm/taskcluster/comm_taskgraph/test/python.ini6
-rw-r--r--comm/taskcluster/comm_taskgraph/test/test_files_changed.py144
-rw-r--r--comm/taskcluster/comm_taskgraph/test/test_optimization_strategies.py143
-rw-r--r--comm/taskcluster/comm_taskgraph/test/test_parameters.py102
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/__init__.py3
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/job/__init__.py0
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py165
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/l10n.py83
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py45
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py52
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/merge_automation.py58
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/partials.py33
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py231
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py79
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/release_started.py53
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py52
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/signing.py88
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/source_test.py63
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/tb_build.py24
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py46
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/tests.py27
-rw-r--r--comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py134
-rw-r--r--comm/taskcluster/comm_taskgraph/try_option_syntax.py97
-rw-r--r--comm/taskcluster/comm_taskgraph/util/__init__.py15
-rw-r--r--comm/taskcluster/comm_taskgraph/util/docker.py25
-rw-r--r--comm/taskcluster/comm_taskgraph/util/hash.py76
48 files changed, 4201 insertions, 0 deletions
diff --git a/comm/taskcluster/comm_taskgraph/__init__.py b/comm/taskcluster/comm_taskgraph/__init__.py
new file mode 100644
index 0000000000..cd54612dfb
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/__init__.py
@@ -0,0 +1,45 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+import os
+from importlib import import_module
+
+from gecko_taskgraph import GECKO
+from gecko_taskgraph.optimize.schema import set_optimization_schema # noqa: F401
+
+from comm_taskgraph.optimize import thunderbird_optimizations
+
+logger = logging.getLogger(__name__)
+
+COMM = os.path.join(GECKO, "comm")
+COMM_SCRIPTS = os.path.join(COMM, "taskcluster", "scripts")
+
+
+def register(graph_config):
+ """
+ Import all modules that are siblings of this one, triggering decorators in
+ the process.
+ """
+ from comm_taskgraph.parameters import register_parameters
+
+ logger.info("{} path registered".format(__name__))
+ register_parameters()
+
+ set_optimization_schema(thunderbird_optimizations)
+
+ _import_modules(
+ [
+ "documentation",
+ "util.docker",
+ "actions",
+ "target_tasks",
+ "transforms.job.toolchain",
+ ]
+ )
+
+
+def _import_modules(modules):
+ for module in modules:
+ import_module(".{}".format(module), package=__name__)
diff --git a/comm/taskcluster/comm_taskgraph/actions.py b/comm/taskcluster/comm_taskgraph/actions.py
new file mode 100644
index 0000000000..6095f3e6c8
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/actions.py
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from gecko_taskgraph.actions.registry import register_callback_action
+from gecko_taskgraph.actions.util import create_tasks, fetch_graph_and_labels
+from gecko_taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS
+
+
+def is_release_promotion_available(parameters):
+ return parameters["project"] in RELEASE_PROMOTION_PROJECTS
+
+
+@register_callback_action(
+ name="l10n-bump",
+ title="L10n Bumper Automation",
+ symbol="l10n_bump",
+ description="L10n bumper action.",
+ order=500,
+ context=[],
+ available=is_release_promotion_available,
+)
+def l10n_bump_action(parameters, graph_config, _input, task_group_id, task_id):
+ """
+ Runs the 'l10n_bump' task.
+ """
+ decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
+ parameters, graph_config
+ )
+ to_run = [label for label, entry in full_task_graph.tasks.items() if "l10n-bump" in entry.kind]
+ create_tasks(
+ graph_config,
+ to_run,
+ full_task_graph,
+ label_to_taskid,
+ parameters,
+ decision_task_id,
+ )
diff --git a/comm/taskcluster/comm_taskgraph/decision.py b/comm/taskcluster/comm_taskgraph/decision.py
new file mode 100644
index 0000000000..a075b47cf5
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/decision.py
@@ -0,0 +1,162 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+import os
+import sys
+
+from redo import retry
+from taskgraph.decision import (
+ _determine_more_accurate_base_ref,
+ _determine_more_accurate_base_rev,
+ _get_env_prefix,
+)
+from taskgraph.taskgraph import TaskGraph
+from taskgraph.util.taskcluster import get_artifact
+from taskgraph.util.vcs import get_repository
+
+from gecko_taskgraph.util.backstop import is_backstop
+from gecko_taskgraph.util.partials import populate_release_history
+from gecko_taskgraph.util.taskgraph import (
+ find_decision_task,
+ find_existing_tasks_from_previous_kinds,
+)
+
+from . import COMM
+from comm_taskgraph.parameters import get_defaults
+
+logger = logging.getLogger(__name__)
+
+BALROG_PRODUCT = "Thunderbird"
+
+# Backstop defaults
+BACKSTOP_TIME_INTERVAL = 60 * 22 # minutes
+INTEGRATION_PROJECTS = {"comm-central"}
+
+PER_PROJECT_PARAMETERS = {
+ "ash": {
+ "target_tasks_method": "ash_tasks",
+ },
+ "jamun": {
+ "target_tasks_method": "nightly_desktop",
+ "release_type": "nightly",
+ },
+ "try-comm-central": {
+ "enable_always_target": True,
+ "target_tasks_method": "try_cc_tasks",
+ },
+ "comm-central": {
+ "target_tasks_method": "comm_central_tasks",
+ "release_type": "nightly",
+ },
+ "comm-beta": {
+ "target_tasks_method": "mozilla_beta_tasks",
+ "release_type": "beta",
+ },
+ "comm-esr115": {
+ "target_tasks_method": "mozilla_esr115_tasks",
+ "release_type": "release",
+ },
+}
+
+CRON_OPTIONS = {
+ "nightly_desktop": {
+ "existing_tasks": lambda parameters, graph_config: get_existing_tasks(
+ parameters, graph_config
+ ),
+ "release_history": lambda parameters, graph_config: populate_release_history(
+ BALROG_PRODUCT, parameters["project"]
+ ),
+ }
+}
+
+
+def get_decision_parameters(graph_config, parameters):
+ logger.info("{}.get_decision_parameters called".format(__name__))
+
+ # Apply default values for all Thunderbird CI projects
+ parameters.update(get_defaults(graph_config.vcs_root))
+
+ # If the target method is nightly, we should build partials. This means
+ # knowing what has been released previously.
+ # An empty release_history is fine, it just means no partials will be built
+ project = parameters["project"]
+
+ if project in PER_PROJECT_PARAMETERS:
+ # Upstream will set target_tasks_method to "default" when nothing is set
+ if parameters["target_tasks_method"] == "default":
+ del parameters["target_tasks_method"]
+
+ # If running from .cron.yml, do not overwrite existing parameters
+ update_parameters = [
+ (_k, _v)
+ for _k, _v in PER_PROJECT_PARAMETERS[project].items()
+ if _k not in parameters or not parameters[_k]
+ ]
+ parameters.update(update_parameters)
+ logger.info("project parameters set for project {} from {}.".format(project, __file__))
+ else:
+ # Projects without a target_tasks_method should not exist for Thunderbird CI
+ raise Exception("No target_tasks_method is defined for project {}.".format(project))
+
+ del parameters["backstop"]
+ parameters["backstop"] = is_backstop(
+ parameters,
+ trust_domain="comm",
+ time_interval=BACKSTOP_TIME_INTERVAL,
+ integration_projects=INTEGRATION_PROJECTS,
+ )
+ for n in (
+ "COMM_BASE_REPOSITORY",
+ "COMM_BASE_REV",
+ "COMM_HEAD_REPOSITORY",
+ "COMM_HEAD_REV",
+ "COMM_HEAD_REF",
+ ):
+ val = os.environ.get(n, "")
+ parameters[n.lower()] = val
+
+ repo_path = COMM
+ repo = get_repository(repo_path)
+ logger.info("Determining comm_base_ref...")
+ parameters["comm_base_ref"] = _determine_more_accurate_base_ref(
+ repo,
+ candidate_base_ref="",
+ head_ref=parameters.get("comm_head_ref"),
+ base_rev=parameters.get("comm_base_rev"),
+ )
+ logger.info("Determining comm_base_rev...")
+ parameters["comm_base_rev"] = _determine_more_accurate_base_rev(
+ repo,
+ base_ref=parameters["comm_base_ref"],
+ candidate_base_rev=parameters.get("comm_base_rev"),
+ head_rev=parameters.get("comm_head_rev"),
+ env_prefix=_get_env_prefix(graph_config),
+ )
+
+ parameters.setdefault("release_history", dict())
+ if parameters.get("tasks_for", "") == "cron":
+ for key, _callable in CRON_OPTIONS.get(parameters["target_tasks_method"], {}).items():
+ result = _callable(parameters, graph_config)
+ parameters[key] = result
+
+
+def get_existing_tasks(parameters, graph_config):
+ """
+ Find the decision task corresponding to the on-push graph, and return
+ a mapping of labels to task-ids from it.
+ """
+ try:
+ decision_task = retry(
+ find_decision_task,
+ args=(parameters, graph_config),
+ attempts=4,
+ sleeptime=5 * 60,
+ )
+ except Exception:
+ logger.exception("Didn't find existing push task.")
+ sys.exit(1)
+
+ _, task_graph = TaskGraph.from_json(get_artifact(decision_task, "public/full-task-graph.json"))
+ return find_existing_tasks_from_previous_kinds(task_graph, [decision_task], [])
diff --git a/comm/taskcluster/comm_taskgraph/documentation.py b/comm/taskcluster/comm_taskgraph/documentation.py
new file mode 100644
index 0000000000..76e8f979cd
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/documentation.py
@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import os
+
+from gecko_taskgraph.util.verify import documentation_paths
+
+from . import COMM
+
+documentation_paths.add(os.path.join(COMM, "taskcluster", "docs"))
diff --git a/comm/taskcluster/comm_taskgraph/files_changed.py b/comm/taskcluster/comm_taskgraph/files_changed.py
new file mode 100644
index 0000000000..10deb65004
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/files_changed.py
@@ -0,0 +1,114 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Support for optimizing tasks based on the set of files that have changed.
+"""
+
+import logging
+import os
+from subprocess import CalledProcessError
+
+from taskgraph.util.memoize import memoize
+from taskgraph.util.path import join as join_path
+from taskgraph.util.path import match as match_path
+
+from gecko_taskgraph import GECKO
+from gecko_taskgraph.util.hg import get_json_automationrelevance
+from mozversioncontrol import InvalidRepoPath, get_repository_object
+
+logger = logging.getLogger(__name__)
+
+
+@memoize
+def get_changed_files(repository, revision):
+ """
+ Get the set of files changed in the push headed by the given revision.
+ Responses are cached, so multiple calls with the same arguments are OK.
+ """
+ contents = get_json_automationrelevance(repository, revision)
+ try:
+ changesets = contents["changesets"]
+ except KeyError:
+ # We shouldn't hit this error in CI.
+ if os.environ.get("MOZ_AUTOMATION"):
+ raise
+
+ # We're likely on an unpublished commit, grab changed files from
+ # version control.
+ return get_locally_changed_files(GECKO)
+
+ logger.debug("{} commits influencing task scheduling:".format(len(changesets)))
+ changed_files = set()
+ for c in changesets:
+ desc = "" # Support empty desc
+ if c["desc"]:
+ desc = c["desc"].splitlines()[0].encode("ascii", "ignore")
+ logger.debug(" {cset} {desc}".format(cset=c["node"][0:12], desc=desc))
+ changed_files |= set(c["files"])
+
+ return changed_files
+
+
+def get_files_changed_extended(params):
+ """
+ Get the set of files changed in the push head from possibly multiple
+ head_repositories.
+ """
+ changed_files = set()
+
+ repo_keys = [key for key in params.keys() if key.endswith("head_repository")]
+
+ def prefix_changed(_changed, prefix):
+ if not prefix:
+ return _changed
+ else:
+ return {join_path(prefix, file) for file in _changed}
+
+ for repo_key in repo_keys:
+ repo_prefix = repo_key.replace("head_repository", "")
+ rev_key = f"{repo_prefix}head_rev"
+ repo_subdir_key = f"{repo_prefix}src_path"
+
+ repository = params.get(repo_key)
+ revision = params.get(rev_key)
+ repo_subdir = params.get(repo_subdir_key, "")
+
+ if not repository or not revision:
+ logger.warning(
+ f"Missing `{repo_key}` or `{rev_key}` parameters; "
+ "assuming all files have changed"
+ )
+ return True
+
+ changed_files |= prefix_changed(get_changed_files(repository, revision), repo_subdir)
+
+ return changed_files
+
+
+def check(params, file_patterns):
+ """Determine whether any of the files changed in the indicated push to
+ https://hg.mozilla.org match any of the given file patterns."""
+ changed_files = get_files_changed_extended(params)
+ if not changed_files:
+ logger.warning(
+ "changed_files from automationrelevance is empty; assuming all files have changed"
+ )
+ return True
+
+ for pattern in file_patterns:
+ for path in changed_files:
+ if match_path(path, pattern):
+ return True
+
+ return False
+
+
+@memoize
+def get_locally_changed_files(repo):
+ try:
+ vcs = get_repository_object(repo)
+ return set(vcs.get_outgoing_files("AM"))
+ except (InvalidRepoPath, CalledProcessError):
+ return set()
diff --git a/comm/taskcluster/comm_taskgraph/loader/__init__.py b/comm/taskcluster/comm_taskgraph/loader/__init__.py
new file mode 100644
index 0000000000..6fbe8159b2
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/loader/__init__.py
@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
diff --git a/comm/taskcluster/comm_taskgraph/loader/merge.py b/comm/taskcluster/comm_taskgraph/loader/merge.py
new file mode 100644
index 0000000000..00b600cd0d
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/loader/merge.py
@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from gecko_taskgraph.loader.transform import loader as transform_loader
+
+from comm_taskgraph.loader.reference import loader as reference_loader
+
+
+def loader(kind, path, config, params, loaded_tasks):
+ """
+ Look up jobs via reference loader at reference-base-path using the list
+ reference-jobs-from, followed by jobs-from.
+
+ This loader has been tested with "fetch" jobs successfully. Anything else
+ is likely to have bugs.
+ """
+ # Make a copy of config for reference_loader. Use pop here to remove the
+ # fields that aren't used by the transform loader
+ reference_config = {
+ "kind-dependencies": config.get("kind-dependencies", None),
+ "reference-base-path": config.pop("reference-base-path"),
+ "reference-jobs": config.pop("reference-jobs", None),
+ }
+ for job in reference_loader(kind, path, reference_config, params, loaded_tasks):
+ yield job
+
+ for job in transform_loader(kind, path, config, params, loaded_tasks):
+ yield job
diff --git a/comm/taskcluster/comm_taskgraph/loader/reference.py b/comm/taskcluster/comm_taskgraph/loader/reference.py
new file mode 100644
index 0000000000..cb4d8f0565
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/loader/reference.py
@@ -0,0 +1,93 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+import os
+
+from taskgraph.util.python_path import find_object
+from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.util.yaml import load_yaml
+
+logger = logging.getLogger(__name__)
+
+
+def _get_aliases(kind, job, project):
+ aliases = {job["name"]}
+
+ if kind == "toolchain":
+ if job["run"].get("toolchain-alias"):
+ resolve_keyed_by(
+ job["run"],
+ "toolchain-alias",
+ item_name=f"{kind}-{job['name']}",
+ project=project,
+ )
+ aliaslist = job["run"].get("toolchain-alias")
+ if aliaslist is not None:
+ if isinstance(aliaslist, str):
+ aliaslist = [aliaslist]
+ for alias in aliaslist:
+ aliases.add(alias)
+
+ return aliases
+
+
+def _expand_aliases(kind, inputs, project):
+ """Given the list of all "reference-jobs" pulled in from upstream, return a
+ set with all job names and aliases.
+ For example "linux64-clang" is an alias of "linux64-clang-13", and both
+ of those names will be included in the returned set."""
+ rv = set()
+ for input_job in inputs:
+ for alias in _get_aliases(kind, input_job, project):
+ rv.add(alias)
+ return rv
+
+
+def _get_loader(path, config):
+ try:
+ _loader = config["loader"]
+ except KeyError:
+ raise KeyError("{!r} does not define `loader`".format(path))
+ return find_object(_loader)
+
+
+def loader(kind, path, config, params, loaded_tasks):
+ """
+ Loads selected jobs from a different taskgraph hierarchy.
+
+ This loads jobs of the given kind from the taskgraph rooted at `base-path`,
+ and includes all the jobs with names or aliases matching the names in the
+ `jobs` key.
+ """
+ base_path = config.pop("reference-base-path")
+ sub_path = os.path.join(base_path, kind)
+
+ logger.debug("Reference loader: load tasks from {}".format(sub_path))
+ sub_config = load_yaml(sub_path, "kind.yml")
+ _loader = _get_loader(sub_path, sub_config)
+ inputs = _loader(kind, sub_path, sub_config, params, loaded_tasks)
+
+ jobs = config.pop("reference-jobs", None)
+
+ config.update(sub_config)
+ project = params["project"]
+
+ if jobs is not None:
+ jobs = set(jobs)
+
+ found_reference_jobs = [job for job in inputs if (_get_aliases(kind, job, project) & jobs)]
+
+ # Check for jobs listed as a reference job in Thunderbird's config
+ # that do not exist in upstream.
+ reference_alias_names = _expand_aliases(kind, found_reference_jobs, project)
+ if reference_alias_names >= jobs:
+ return found_reference_jobs
+ else:
+ missing_jobs = jobs - reference_alias_names
+ raise Exception(
+ "Reference jobs not found in kind {}: {}".format(kind, ", ".join(missing_jobs))
+ )
+ else:
+ return inputs
diff --git a/comm/taskcluster/comm_taskgraph/manifests/release_checksums.yml b/comm/taskcluster/comm_taskgraph/manifests/release_checksums.yml
new file mode 100644
index 0000000000..81a11b2793
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/manifests/release_checksums.yml
@@ -0,0 +1,68 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+s3_bucket_paths:
+ by-platform:
+ thunderbird-release:
+ - pub/thunderbird/candidates
+default_locales: # if given an empty locale, use these locales
+ - en-US
+tasktype_map: # Map task reference to task type.
+ release-generate-checksums: build
+ release-generate-checksums-signing: signing
+
+# A default entry, which the mappings below extend and override.
+# Final 'destinations' will be the product of:
+# s3_bucket_paths + destinations + locale_prefix + pretty_name
+default: &default
+ from:
+ - release-generate-checksums-signing
+ all_locales: true
+ description: "TO_BE_OVERRIDDEN"
+ locale_prefix: ''
+ source_path_modifier: ''
+ destinations: # locale_prefix is appended
+ - ${version}-candidates/build${build_number}
+
+# Configuration for individual files. Extends 'default', above.
+mapping:
+ SHA256SUMMARY:
+ <<: *default
+ description: "Merkle-tree for the release artifacts with sha 256 hashes"
+ from:
+ - release-generate-checksums
+ pretty_name: SHA256SUMMARY
+ checksums_path: SHA256SUMMARY
+ SHA512SUMMARY:
+ <<: *default
+ description: "Merkle-tree for the release artifacts with sha 512 hashes"
+ from:
+ - release-generate-checksums
+ pretty_name: SHA512SUMMARY
+ checksums_path: SHA512SUMMARY
+ KEY:
+ <<: *default
+ description: "Public side of the key that was used to sign the release artifacts"
+ pretty_name: KEY
+ checksums_path: KEY
+ SHA256SUMS:
+ <<: *default
+ description: "Aggregated checksums with main installers details per platform in sha512 hashes"
+ pretty_name: SHA256SUMS
+ checksums_path: SHA256SUMS
+ SHA256SUMS.asc:
+ <<: *default
+ description: "Detached signature for the checksums file"
+ pretty_name: SHA256SUMS.asc
+ checksums_path: SHA256SUMS.asc
+ SHA512SUMS:
+ <<: *default
+ description: "Aggregated checksums with main installers details per platform in sha256 hashes"
+ pretty_name: SHA512SUMS
+ checksums_path: SHA512SUMS
+ SHA512SUMS.asc:
+ <<: *default
+ description: "Detached signature for the checksums file"
+ pretty_name: SHA512SUMS.asc
+ checksums_path: SHA512SUMS.asc
diff --git a/comm/taskcluster/comm_taskgraph/manifests/source_checksums.yml b/comm/taskcluster/comm_taskgraph/manifests/source_checksums.yml
new file mode 100644
index 0000000000..13fab07342
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/manifests/source_checksums.yml
@@ -0,0 +1,46 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+s3_bucket_paths:
+ by-platform:
+ thunderbird-source:
+ - pub/thunderbird/candidates
+default_locales: # if given an empty locale, use these locales
+ - en-US
+tasktype_map: # Map task reference to task type.
+ release-source-checksums-signing: signing
+
+# A default entry, which the mappings below extend and override.
+# Final 'destinations' will be the product of:
+# s3_bucket_paths + destinations + locale_prefix + pretty_name
+default: &default
+ from:
+ - release-source-checksums-signing
+ all_locales: false
+ description: "TO_BE_OVERRIDDEN"
+ locale_prefix: ''
+ source_path_modifier: ''
+ destinations: # locale_prefix is appended
+ - ${version}-candidates/build${build_number}/beetmover-checksums/source
+
+# Configuration for individual files. Extends 'default', above.
+mapping:
+ target-source.checksums:
+ <<: *default
+ description: "Checksums file for the source zip files"
+ pretty_name:
+ by-platform:
+ thunderbird-source: thunderbird-${version}.checksums.beet
+ checksums_path:
+ by-platform:
+ thunderbird-source: thunderbird-${version}.checksums.beet
+ target-source.checksums.asc:
+ <<: *default
+ description: "Detached signature for the checksums file"
+ pretty_name:
+ by-platform:
+ thunderbird-source: thunderbird-${version}.checksums.asc
+ checksums_path:
+ by-platform:
+ thunderbird-source: thunderbird-${version}.checksums.asc
diff --git a/comm/taskcluster/comm_taskgraph/manifests/source_files.yml b/comm/taskcluster/comm_taskgraph/manifests/source_files.yml
new file mode 100644
index 0000000000..53270eb347
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/manifests/source_files.yml
@@ -0,0 +1,46 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+s3_bucket_paths:
+ by-platform:
+ thunderbird-source:
+ - pub/thunderbird/candidates
+default_locales: # if given an empty locale, use these locales
+ - en-US
+tasktype_map: # Map task reference to task type.
+ release-source-signing: signing
+
+# A default entry, which the mappings below extend and override.
+# Final 'destinations' will be the product of:
+# s3_bucket_paths + destinations + locale_prefix + pretty_name
+default: &default
+ from:
+ - release-source-signing
+ all_locales: false
+ description: "TO_BE_OVERRIDDEN"
+ locale_prefix: ''
+ source_path_modifier: ''
+ destinations: # locale_prefix is appended
+ - ${version}-candidates/build${build_number}/source
+
+# Configuration for individual files. Extends 'default', above.
+mapping:
+ source.tar.xz:
+ <<: *default
+ description: "Source file with the in-tree code archived"
+ pretty_name:
+ by-platform:
+ thunderbird-source: thunderbird-${version}.source.tar.xz
+ checksums_path:
+ by-platform:
+ thunderbird-source: source/thunderbird-${version}.source.tar.xz
+ source.tar.xz.asc:
+ <<: *default
+ description: "Detached signature for the source file"
+ pretty_name:
+ by-platform:
+ thunderbird-source: thunderbird-${version}.source.tar.xz.asc
+ checksums_path:
+ by-platform:
+ thunderbird-source: source/thunderbird-${version}.source.tar.xz.asc
diff --git a/comm/taskcluster/comm_taskgraph/manifests/strings_source.yml b/comm/taskcluster/comm_taskgraph/manifests/strings_source.yml
new file mode 100644
index 0000000000..b6fa63ec84
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/manifests/strings_source.yml
@@ -0,0 +1,64 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+s3_bucket_paths:
+ by-platform:
+ shippable-l10n-pre:
+ - pub/thunderbird/candidates
+default_locales: # if given an empty locale, use these locales
+ - en-US
+tasktype_map: # Map task reference to task type.
+ shippable-l10n-pre-signing: signing
+
+# A default entry, which the mappings below extend and override.
+# Final 'destinations' will be the product of:
+# s3_bucket_paths + destinations + locale_prefix + pretty_name
+default: &default
+ from:
+ - shippable-l10n-pre-signing
+ all_locales: false
+ description: "TO_BE_OVERRIDDEN"
+ locale_prefix: ''
+ source_path_modifier: ''
+ destinations: # locale_prefix is appended
+ - ${version}-candidates/build${build_number}/source
+
+# Configuration for individual files. Extends 'default', above.
+mapping:
+ strings_all.tar.zst:
+ <<: *default
+ description: "Strings sources for localuized builds and langpacks"
+ pretty_name:
+ by-platform:
+ shippable-l10n-pre: thunderbird-${version}.strings_all.tar.zst
+ checksums_path:
+ by-platform:
+ shippable-l10n-pre: source/thunderbird-${version}.strings_all.tar.zst
+ strings_all.tar.zst.asc:
+ <<: *default
+ description: "Detached signature for the strings source file"
+ pretty_name:
+ by-platform:
+ shippable-l10n-pre: thunderbird-${version}.strings_all.tar.zst.asc
+ checksums_path:
+ by-platform:
+ shippable-l10n-pre: source/thunderbird-${version}.strings_all.tar.zst.asc
+ l10n-changesets.json:
+ <<: *default
+ description: "Source info used to build strings_all.tar.zst"
+ pretty_name:
+ by-platform:
+ shippable-l10n-pre: thunderbird-${version}.l10n-changesets.json
+ checksums_path:
+ by-platform:
+ shippable-l10n-pre: source/thunderbird-${version}.l10n-changesets.json
+ l10n-changesets.json.asc:
+ <<: *default
+ description: "Detached signature for changesets file"
+ pretty_name:
+ by-platform:
+ shippable-l10n-pre: thunderbird-${version}.l10n-changesets.json.asc
+ checksums_path:
+ by-platform:
+ shippable-l10n-pre: source/thunderbird-${version}.l10n-changesets.json.asc
diff --git a/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates.yml b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates.yml
new file mode 100644
index 0000000000..26540e677c
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates.yml
@@ -0,0 +1,334 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+# This file contains exhaustive information about all the release artifacs that
+# are needed within a type of release.
+#
+# Structure
+# --------
+# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets
+# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of
+# destinations. If given an empty locale, it uses these locales instead.
+# `tasktype_map` -- mapping between task reference and task type, particularly useful when
+# composing the upstreamArtifacts for scriptworker.
+# `platform_names` -- various platform mappings used in reckoning artifacts or other paths
+# `default` -- a default entry, which the mappings extend and override in such a way that
+# final path full-destinations will be a concatenation of the following:
+# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name`
+# `from` -- specifies the dependency(ies) from which to expect the particular artifact
+# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected
+# for all locales or just the default one
+# `description` -- brief summary of what that artifact is
+# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not
+# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location
+# for example `public/build` vs `public/build/ach/`.
+# `destinations` -- final list of directories where to push the artifacts in S3
+# `pretty_name` -- the final name the artifact will have at destination
+# `checksums_path` -- the name to identify one artifact within the checksums file
+# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform
+# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform
+# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present
+# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest
+# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest
+
+s3_bucket_paths:
+ - pub/thunderbird/candidates
+default_locales:
+ - en-US
+tasktype_map:
+ build: build
+ signing: signing
+ mar-signing: signing
+ partials-signing: signing
+ repackage: repackage
+ repackage-signing: repackage
+ repackage-signing-msi: repackage
+ repackage-signing-shippable-l10n-msix: signing
+ release-sign-and-push-langpacks: scriptworker
+platform_names:
+ path_platform:
+ by-platform:
+ linux-shippable: 'linux-i686'
+ linux64-shippable: 'linux-x86_64'
+ linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ macosx64-shippable: 'mac'
+ win32-shippable: 'win32'
+ win64-shippable: 'win64'
+ win64-asan-reporter-shippable: 'win64-asan-reporter'
+ tools_platform:
+ by-platform:
+ linux-shippable: 'linux'
+ linux64-shippable: 'linux64'
+ linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ macosx64-shippable: 'macosx64'
+ win32-shippable: 'win32'
+ win64-shippable: 'win64'
+ win64-asan-reporter-shippable: 'win64-asan-reporter'
+ filename_platform:
+ by-platform:
+ linux-shippable: 'linux'
+ linux64-shippable: 'linux64'
+ linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ macosx64-shippable: 'macosx64'
+ win32-shippable: 'win32'
+ win64-shippable: 'win64'
+ win64-asan-reporter-shippable: 'win64-asan-reporter'
+
+default: &default
+ from:
+ - build
+ all_locales: false
+ description: "TO_BE_OVERRIDDEN"
+ locale_prefix: '${locale}/'
+ source_path_modifier:
+ by-locale:
+ default: '${locale}'
+ en-US: ''
+ destinations:
+ - ${version}-candidates/build${build_number}/${path_platform}
+
+mapping:
+ buildhub.json:
+ <<: *default
+ all_locales: false
+ description: "Build related information to be consumed by Buildhub service"
+ pretty_name: buildhub.json
+ checksums_path: ${path_platform}/${locale}/buildhub.json
+ target.common.tests.tar.gz:
+ <<: *default
+ description: "Mixture of reftests, mochitests, UI and others, commonly bundled together in a test suite"
+ pretty_name: thunderbird-${version}.common.tests.tar.gz
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.common.tests.tar.gz
+ target.cppunittest.tests.tar.gz:
+ <<: *default
+ description: "C++ unittests related in-tree test infrastructure"
+ pretty_name: thunderbird-${version}.cppunittest.tests.tar.gz
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.cppunittest.tests.tar.gz
+ target.crashreporter-symbols.zip:
+ <<: *default
+ description: "Crashreporter symbols to be consumed by Socorro"
+ pretty_name: thunderbird-${version}.crashreporter-symbols.zip
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.crashreporter-symbols.zip
+ target.json:
+ <<: *default
+ description: "Various compile and moz_app flags baked together in a json file"
+ pretty_name: thunderbird-${version}.json
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.json
+ target.mochitest.tests.tar.gz:
+ <<: *default
+ description: "Results for running the mochitest testing framework via Javascript function calls"
+ pretty_name: thunderbird-${version}.mochitest.tests.tar.gz
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.mochitest.tests.tar.gz
+ target.mozinfo.json:
+ <<: *default
+ description: "Various compile and moz_app flags baked together in a json file"
+ pretty_name: thunderbird-${version}.mozinfo.json
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.mozinfo.json
+ target.reftest.tests.tar.gz:
+ <<: *default
+ description: "Results for running the reftest testing framework via display of two Web pages comparison"
+ pretty_name: thunderbird-${version}.reftest.tests.tar.gz
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.reftest.tests.tar.gz
+ target.talos.tests.tar.gz:
+ <<: *default
+ description: "Results for running the talos testing framework to measure performance"
+ pretty_name: thunderbird-${version}.talos.tests.tar.gz
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.talos.tests.tar.gz
+ target.awsy.tests.tar.gz:
+ <<: *default
+ description: "Results for running the awsy testing framework to track memory usage"
+ pretty_name: thunderbird-${version}.awsy.tests.tar.gz
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.awsy.tests.tar.gz
+ target.test_packages.json:
+ <<: *default
+ description: "File containing metadata about all other files and testing harnesses specifics"
+ pretty_name: thunderbird-${version}.test_packages.json
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.test_packages.json
+ target.txt:
+ <<: *default
+ description: "File containing buildid and revision"
+ pretty_name: thunderbird-${version}.txt
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.txt
+ target.web-platform.tests.tar.gz:
+ <<: *default
+ description: "Results for running the webplatform testing framework to cover standard Web platform features"
+ pretty_name: thunderbird-${version}.web-platform.tests.tar.gz
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.web-platform.tests.tar.gz
+ target.xpcshell.tests.tar.gz:
+ <<: *default
+ description: "Results for running the xpcshell testing framework to enable XPConnect console application"
+ pretty_name: thunderbird-${version}.xpcshell.tests.tar.gz
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.xpcshell.tests.tar.gz
+ target_info.txt:
+ <<: *default
+ description: "File containing the buildID"
+ locale_prefix: ''
+ pretty_name: ${filename_platform}_info.txt
+ checksums_path: ${filename_platform}_info.txt
+ destinations:
+ - ${version}-candidates/build${build_number}
+ mozharness.zip:
+ <<: *default
+ description: "File containing the mozharness set of scripts and configuration used by various automation tools"
+ pretty_name: mozharness.zip
+ checksums_path: ${path_platform}/${locale}/mozharness.zip
+ target.jsshell.zip:
+ <<: *default
+ description: "Set of shells to allow test snippets of Javascript code without needing to reload the page"
+ locale_prefix: ''
+ pretty_name: jsshell-${path_platform}.zip
+ checksums_path: jsshell/jsshell-${path_platform}.zip
+ destinations:
+ - ${version}-candidates/build${build_number}/jsshell
+ target.langpack.xpi:
+ <<: *default
+ all_locales: true
+ description: "Localized repack that grabs a packaged en-US Thunderbird and repackages it as locale-specific Thunderbird"
+ locale_prefix: ''
+ from:
+ - build
+ only_for_platforms:
+ - linux-shippable
+ - linux64-shippable
+ - macosx64-shippable
+ - win32-shippable
+ - win64-shippable
+ pretty_name: ${locale}.xpi
+ checksums_path: ${path_platform}/xpi/${locale}.xpi
+ destinations:
+ - ${version}-candidates/build${build_number}/${path_platform}/xpi
+ mar:
+ <<: *default
+ description: "Alongside `mbsdiff`, a tool used to generate partials"
+ locale_prefix: ''
+ source_path_modifier: 'host/bin'
+ pretty_name: ${tools_platform}/mar
+ checksums_path: mar-tools/${tools_platform}/mar
+ not_for_platforms:
+ - win32-shippable
+ - win64-shippable
+ destinations:
+ - ${version}-candidates/build${build_number}/mar-tools
+ mbsdiff:
+ <<: *default
+ description: "Alongside `mar`, a tool used to generate partials"
+ locale_prefix: ''
+ source_path_modifier: 'host/bin'
+ pretty_name: ${tools_platform}/mbsdiff
+ checksums_path: mar-tools/${tools_platform}/mbsdiff
+ not_for_platforms:
+ - win32-shippable
+ - win64-shippable
+ destinations:
+ - ${version}-candidates/build${build_number}/mar-tools
+ target.tar.bz2:
+ <<: *default
+ description: "Main installer for Linux platforms"
+ all_locales: true
+ from:
+ - signing
+ only_for_platforms:
+ - linux-shippable
+ - linux64-shippable
+ pretty_name: thunderbird-${version}.tar.bz2
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.tar.bz2
+ target.tar.bz2.asc:
+ <<: *default
+ description: "Detached signature for the checksums file"
+ all_locales: true
+ from:
+ - signing
+ only_for_platforms:
+ - linux-shippable
+ - linux64-shippable
+ pretty_name: thunderbird-${version}.tar.bz2.asc
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.tar.bz2.asc
+ target.pkg:
+ <<: *default
+ description: "Main package installer for Mac OS X platforms"
+ all_locales: true
+ from:
+ - signing
+ only_for_platforms:
+ - macosx64-shippable
+ pretty_name: Thunderbird ${version}.pkg
+ checksums_path: ${path_platform}/${locale}/Thunderbird ${version}.pkg
+ target.dmg:
+ <<: *default
+ description: "Main package installer for Mac OS X platforms"
+ all_locales: true
+ from:
+ - repackage
+ only_for_platforms:
+ - macosx64-shippable
+ pretty_name: Thunderbird ${version}.dmg
+ checksums_path: ${path_platform}/${locale}/Thunderbird ${version}.dmg
+ target.zip:
+ <<: *default
+ description: "Main package installer for Windows platforms"
+ all_locales: true
+ from:
+ - signing
+ only_for_platforms:
+ - win64-shippable
+ - win32-shippable
+ pretty_name: thunderbird-${version}.zip
+ checksums_path: ${path_platform}/${locale}/thunderbird-${version}.zip
+ target.installer.exe:
+ <<: *default
+ description: "Main installer for Windows platforms"
+ all_locales: true
+ from:
+ - repackage-signing
+ only_for_platforms:
+ - win64-shippable
+ - win32-shippable
+ pretty_name: Thunderbird Setup ${version}.exe
+ checksums_path: ${path_platform}/${locale}/Thunderbird Setup ${version}.exe
+ target.installer.msi:
+ <<: *default
+ description: "Windows installer for MSI platform"
+ all_locales: true
+ from:
+ - repackage-signing-msi
+ only_for_platforms:
+ - win64-shippable
+ - win32-shippable
+ pretty_name: Thunderbird Setup ${version}.msi
+ checksums_path: ${path_platform}/${locale}/Thunderbird Setup ${version}.msi
+ target.installer.msix:
+ <<: *default
+ description: "Windows MSIX installer"
+ from:
+ - repackage-signing-shippable-l10n-msix
+ only_for_platforms:
+ - win64-shippable
+ - win32-shippable
+ locale_prefix: 'multi/'
+ pretty_name: Thunderbird Setup ${version}.msix
+ checksums_path: ${path_platform}/multi/Thunderbird Setup ${version}.msix
+ target.complete.mar:
+ <<: *default
+ description: "Complete MAR to serve as updates"
+ all_locales: true
+ from:
+ - mar-signing
+ pretty_name: thunderbird-${version}.complete.mar
+ checksums_path: update/${path_platform}/${locale}/thunderbird-${version}.complete.mar
+ update_balrog_manifest: true
+ destinations:
+ - ${version}-candidates/build${build_number}/update/${path_platform}
+ ${partial}:
+ <<: *default
+ description: "Partials MAR files to serve as updates"
+ all_locales: true
+ from:
+ - partials-signing
+ partials_only: true
+ pretty_name: thunderbird-${previous_version}-${version}.partial.mar
+ checksums_path: update/${path_platform}/${locale}/thunderbird-${previous_version}-${version}.partial.mar
+ update_balrog_manifest: true
+ from_buildid: ${from_buildid}
+ destinations:
+ - ${version}-candidates/build${build_number}/update/${path_platform}
diff --git a/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates_checksums.yml b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates_checksums.yml
new file mode 100644
index 0000000000..f3b2225f96
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_candidates_checksums.yml
@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+# This file contains exhaustive information about all the release artifacs that
+# are needed within a type of release.
+#
+# Structure
+# --------
+# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets
+# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of
+# destinations. If given an empty locale, it uses these locales instead.
+# `tasktype_map` -- mapping between task reference and task type, particularly useful when
+# composing the upstreamArtifacts for scriptworker.
+# `platform_names` -- various platform mappings used in reckoning artifacts or other paths
+# `default` -- a default entry, which the mappings extend and override in such a way that
+# final path full-destinations will be a concatenation of the following:
+# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name`
+# `from` -- specifies the dependency(ies) from which to expect the particular artifact
+# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected
+# for all locales or just the default one
+# `description` -- brief summary of what that artifact is
+# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not
+# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location
+# for example `public/build` vs `public/build/ach/`.
+# `destinations` -- final list of directories where to push the artifacts in S3
+# `pretty_name` -- the final name the artifact will have at destination
+# `checksums_path` -- the name to identify one artifact within the checksums file
+# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform
+# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform
+# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present
+# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest
+# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest
+
+s3_bucket_paths:
+ - pub/thunderbird/candidates
+default_locales:
+ - en-US
+tasktype_map:
+ beetmover-repackage: beetmover
+ release-beetmover-signed-langpacks: signing
+platform_names:
+ path_platform:
+ by-platform:
+ linux-shippable: 'linux-i686'
+ linux64-shippable: 'linux-x86_64'
+ linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ macosx64-shippable: 'mac'
+ win32-shippable: 'win32'
+ win64-shippable: 'win64'
+ win64-asan-reporter-shippable: 'win64-asan-reporter'
+ linux: 'linux-i686'
+ linux64: 'linux-x86_64'
+ macosx64: 'mac'
+ win32: 'win32'
+ win64: 'win64'
+
+default: &default
+ from:
+ - beetmover-repackage
+ all_locales: true
+ description: "TO_BE_OVERRIDDEN"
+ locale_prefix: '${locale}/'
+ source_path_modifier: ''
+ destinations:
+ - ${version}-candidates/build${build_number}/beetmover-checksums/${path_platform}
+
+mapping:
+ target.checksums:
+ <<: *default
+ description: "Checksums file containing size, hash, sha algorithm and filename"
+ pretty_name: thunderbird-${version}.checksums.beet
+ checksums_path: beetmover-checksums/${path_platform}/${locale}/thunderbird-${version}.checksums.beet
+ target-langpack.checksums:
+ <<: *default
+ description: "Checksums file containing size, hash, sha algorithm and filename for the langpack"
+ locale_prefix: ''
+ from:
+ - release-beetmover-signed-langpacks
+ pretty_name: ${locale}.checksums.beet
+ checksums_path: beetmover-checksums/${path_platform}/xpi/${locale}.checksums.beet
+ destinations:
+ - ${version}-candidates/build${build_number}/beetmover-checksums/${path_platform}/xpi
diff --git a/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly.yml b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly.yml
new file mode 100644
index 0000000000..ee2ca6c800
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly.yml
@@ -0,0 +1,434 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+# This file contains exhaustive information about all the release artifacs that
+# are needed within a type of release.
+#
+# Structure
+# --------
+# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets
+# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of
+# destinations. If given an empty locale, it uses these locales instead.
+# `tasktype_map` -- mapping between task reference and task type, particularly useful when
+# composing the upstreamArtifacts for scriptworker.
+# `platform_names` -- various platform mappings used in reckoning artifacts or other paths
+# `default` -- a default entry, which the mappings extend and override in such a way that
+# final path full-destinations will be a concatenation of the following:
+# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name`
+# `from` -- specifies the dependency(ies) from which to expect the particular artifact
+# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected
+# for all locales or just the default one
+# `description` -- brief summary of what that artifact is
+# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not
+# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location
+# for example `public/build` vs `public/build/ach/`.
+# `destinations` -- final list of directories where to push the artifacts in S3
+# `pretty_name` -- the final name the artifact will have at destination
+# `checksums_path` -- the name to identify one artifact within the checksums file
+# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform
+# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform
+# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present
+# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest
+# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest
+
+s3_bucket_paths:
+ - pub/thunderbird/nightly
+default_locales:
+ - en-US
+tasktype_map:
+ build: build
+ signing: signing
+ mar-signing: signing
+ partials-signing: signing
+ repackage: repackage
+ repackage-signing: repackage
+ repackage-signing-msi: repackage
+ repackage-signing-shippable-l10n-msix: signing
+platform_names:
+ filename_platform:
+ by-platform:
+ linux-shippable: 'linux-i686'
+ linux64-shippable: 'linux-x86_64'
+ linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ macosx64-shippable: 'mac'
+ win32-shippable: 'win32'
+ win64-shippable: 'win64'
+ win64-asan-reporter-shippable: 'win64-asan-reporter'
+ stage_platform:
+ by-platform:
+ linux-shippable: 'linux'
+ linux64-shippable: 'linux64'
+ linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ macosx64-shippable: 'macosx64'
+ win32-shippable: 'win32'
+ win64-shippable: 'win64'
+ win64-asan-reporter-shippable: 'win64-asan-reporter'
+
+default: &default
+ from:
+ - build
+ all_locales: false
+ description: "TO_BE_OVERRIDDEN"
+ locale_prefix: ''
+ source_path_modifier:
+ by-locale:
+ default: '${locale}'
+ en-US: ''
+ destinations:
+ by-locale:
+ en-US:
+ - ${year}/${month}/${upload_date}-${branch}
+ - latest-${branch}
+ default:
+ - ${year}/${month}/${upload_date}-${branch}-l10n
+ - latest-${branch}-l10n
+
+mapping:
+ buildhub.json:
+ <<: *default
+ description: "Build related information to be consumed by Buildhub service"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.buildhub.json
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.buildhub.json
+ destinations:
+ by-locale:
+ en-US:
+ - ${year}/${month}/${upload_date}-${branch}
+ - latest-${branch}
+ - latest-${branch}-l10n
+ default:
+ - ${year}/${month}/${upload_date}-${branch}-l10n
+ - latest-${branch}-l10n
+ KEY:
+ <<: *default
+ from:
+ - signing
+ description: "Public side of the key that was used to sign the release artifacts"
+ pretty_name: KEY
+ checksums_path: KEY
+ only_for_platforms:
+ - linux64-shippable
+ destinations:
+ - ${year}/${month}/${upload_date}-${branch}
+ - ${year}/${month}/${upload_date}-${branch}-l10n
+ - latest-${branch}
+ - latest-${branch}-l10n
+ target.common.tests.tar.gz:
+ <<: *default
+ description: "Mixture of reftests, mochitests, UI and others, commonly bundled together in a test suite"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.common.tests.tar.gz
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.common.tests.tar.gz
+ target.cppunittest.tests.tar.gz:
+ <<: *default
+ description: "C++ unittests related in-tree test infrastructure"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz
+ target.crashreporter-symbols.zip:
+ <<: *default
+ description: "Crashreporter symbols to be consumed by Socorro"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.crashreporter-symbols.zip
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.crashreporter-symbols.zip
+ not_for_platforms:
+ - linux64-asan-reporter-shippable
+ - win64-asan-reporter-shippable
+ target.json:
+ <<: *default
+ description: "Various compile and moz_app flags baked together in a json file"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.json
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.json
+ target.mochitest.tests.tar.gz:
+ <<: *default
+ description: "Results for running the mochitest testing framework via Javascript function calls"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz
+ target.mozinfo.json:
+ <<: *default
+ description: "Various compile and moz_app flags baked together in a json file"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.mozinfo.json
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.mozinfo.json
+ target.reftest.tests.tar.gz:
+ <<: *default
+ description: "Results for running the reftest testing framework via display of two Web pages comparison"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz
+ target.talos.tests.tar.gz:
+ <<: *default
+ description: "Results for running the talos testing framework to measure performance"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.talos.tests.tar.gz
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.talos.tests.tar.gz
+ target.awsy.tests.tar.gz:
+ <<: *default
+ description: "Results for running the awsy testing framework to track memory usage"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz
+ target.test_packages.json:
+ <<: *default
+ description: "File containing metadata about all other files and testing harnesses specifics"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.test_packages.json
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.test_packages.json
+ target.txt:
+ <<: *default
+ description: "File containing buildid and revision"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.txt
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.txt
+ target.web-platform.tests.tar.gz:
+ <<: *default
+ description: "Results for running the webplatform testing framework to cover standard Web platform features"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz
+ target.xpcshell.tests.tar.gz:
+ <<: *default
+ description: "Results for running the xpcshell testing framework to enable XPConnect console application"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz
+ target_info.txt:
+ <<: *default
+ description: "File containing the buildID"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}_info.txt
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}_info.txt
+ mozharness.zip:
+ <<: *default
+ description: "File containing the mozharness set of scripts and configuration used by various automation tools"
+ pretty_name: mozharness.zip
+ checksums_path: mozharness.zip
+ target.jsshell.zip:
+ <<: *default
+ description: "Set of shells to allow test snippets of Javascript code without needing to reload the page"
+ pretty_name: jsshell-${filename_platform}.zip
+ checksums_path: jsshell-${filename_platform}.zip
+ not_for_platforms:
+ - linux64-asan-reporter-shippable
+ - win64-asan-reporter-shippable
+ target.langpack.xpi:
+ <<: *default
+ all_locales: true
+ description: "Localized repack that grabs a packaged en-US Thunderbird and repackages it as locale-specific Thunderbird"
+ from:
+ - build
+ only_for_platforms:
+ - linux-shippable
+ - linux64-shippable
+ - linux64-asan-reporter-shippable
+ - macosx64-shippable
+ - win64-shippable
+ - win32-shippable
+ - win64-asan-reporter-shippable
+ pretty_name: thunderbird-${version}.${locale}.langpack.xpi
+ checksums_path: thunderbird-${version}.${locale}.langpack.xpi
+ destinations:
+ by-locale:
+ en-US:
+ - ${year}/${month}/${upload_date}-${branch}
+ - latest-${branch}
+ default:
+ - ${year}/${month}/${upload_date}-${branch}-l10n/${filename_platform}/xpi
+ - latest-${branch}-l10n/${filename_platform}/xpi
+ mar:
+ <<: *default
+ description: "Alongside `mbsdiff`, a tool used to generate partials"
+ source_path_modifier: 'host/bin'
+ pretty_name: mar
+ checksums_path: mar
+ not_for_platforms:
+ - win32-shippable
+ - win64-shippable
+ - win64-asan-reporter-shippable
+ destinations:
+ - ${year}/${month}/${upload_date}-${branch}/mar-tools/${stage_platform}
+ - latest-${branch}/mar-tools/${stage_platform}
+ mbsdiff:
+ <<: *default
+ description: "Alongside `mar`, a tool used to generate partials"
+ source_path_modifier: 'host/bin'
+ pretty_name: mbsdiff
+ checksums_path: mbsdiff
+ not_for_platforms:
+ - win32-shippable
+ - win64-shippable
+ - win64-asan-reporter-shippable
+ destinations:
+ - ${year}/${month}/${upload_date}-${branch}/mar-tools/${stage_platform}
+ - latest-${branch}/mar-tools/${stage_platform}
+ target.tar.bz2:
+ <<: *default
+ description: "Main installer for Linux platforms"
+ all_locales: true
+ from:
+ - signing
+ only_for_platforms:
+ - linux-shippable
+ - linux64-shippable
+ - linux64-asan-reporter-shippable
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.tar.bz2
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.tar.bz2
+ destinations:
+ by-locale:
+ en-US:
+ - ${year}/${month}/${upload_date}-${branch}
+ - latest-${branch}
+ - latest-${branch}-l10n
+ default:
+ - ${year}/${month}/${upload_date}-${branch}-l10n
+ - latest-${branch}-l10n
+ target.tar.bz2.asc:
+ <<: *default
+ description: "Detached signature for the checksums file"
+ all_locales: true
+ from:
+ - signing
+ only_for_platforms:
+ - linux-shippable
+ - linux64-shippable
+ - linux64-asan-reporter-shippable
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.tar.bz2.asc
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.tar.bz2.asc
+ destinations:
+ by-locale:
+ en-US:
+ - ${year}/${month}/${upload_date}-${branch}
+ - latest-${branch}
+ - latest-${branch}-l10n
+ default:
+ - ${year}/${month}/${upload_date}-${branch}-l10n
+ - latest-${branch}-l10n
+ target.pkg:
+ <<: *default
+ description: "Main package installer for Mac OS X platforms"
+ all_locales: true
+ from:
+ - signing
+ only_for_platforms:
+ - macosx64-shippable
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.pkg
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.pkg
+ destinations:
+ by-locale:
+ en-US:
+ - ${year}/${month}/${upload_date}-${branch}
+ - latest-${branch}
+ - latest-${branch}-l10n
+ default:
+ - ${year}/${month}/${upload_date}-${branch}-l10n
+ - latest-${branch}-l10n
+ target.dmg:
+ <<: *default
+ description: "Main package installer for Mac OS X platforms"
+ all_locales: true
+ from:
+ - repackage
+ only_for_platforms:
+ - macosx64-shippable
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.dmg
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.dmg
+ destinations:
+ by-locale:
+ en-US:
+ - ${year}/${month}/${upload_date}-${branch}
+ - latest-${branch}
+ - latest-${branch}-l10n
+ default:
+ - ${year}/${month}/${upload_date}-${branch}-l10n
+ - latest-${branch}-l10n
+ target.zip:
+ <<: *default
+ description: "Main package installer for Windows platforms"
+ all_locales: true
+ from:
+ - signing
+ only_for_platforms:
+ - win64-shippable
+ - win32-shippable
+ - win64-asan-reporter-shippable
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.zip
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.zip
+ target.installer.exe:
+ <<: *default
+ description: "Main installer for Windows platforms"
+ all_locales: true
+ from:
+ - repackage-signing
+ only_for_platforms:
+ - win64-shippable
+ - win32-shippable
+ - win64-asan-reporter-shippable
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.installer.exe
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.installer.exe
+ destinations:
+ by-locale:
+ en-US:
+ - ${year}/${month}/${upload_date}-${branch}
+ - latest-${branch}
+ - latest-${branch}-l10n
+ default:
+ - ${year}/${month}/${upload_date}-${branch}-l10n
+ - latest-${branch}-l10n
+ target.installer.msi:
+ <<: *default
+ description: "Windows installer for MSI platform"
+ all_locales: true
+ from:
+ - repackage-signing-msi
+ only_for_platforms:
+ - win64-shippable
+ - win32-shippable
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.installer.msi
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.installer.msi
+ destinations:
+ by-locale:
+ en-US:
+ - ${year}/${month}/${upload_date}-${branch}
+ - latest-${branch}
+ - latest-${branch}-l10n
+ default:
+ - ${year}/${month}/${upload_date}-${branch}-l10n
+ - latest-${branch}-l10n
+ target.installer.msix:
+ <<: *default
+ description: "Windows MSIX installer"
+ all_locales: true
+ from:
+ - repackage-signing-shippable-l10n-msix
+ only_for_platforms:
+ - win64-shippable
+ - win32-shippable
+ pretty_name: thunderbird-${version}.multi.${filename_platform}.installer.msix
+ checksums_path: thunderbird-${version}.multi.${filename_platform}.installer.msix
+ destinations:
+ - ${year}/${month}/${upload_date}-${branch}
+ - latest-${branch}
+ - latest-${branch}-l10n
+ target.complete.mar:
+ <<: *default
+ description: "The main installer we ship our mobile products baked within"
+ all_locales: true
+ from:
+ - mar-signing
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.complete.mar
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.complete.mar
+ update_balrog_manifest: true
+ destinations:
+ by-locale:
+ en-US:
+ - ${year}/${month}/${upload_date}-${branch}
+ - latest-${branch}
+ - latest-${branch}-l10n
+ default:
+ - ${year}/${month}/${upload_date}-${branch}-l10n
+ - latest-${branch}-l10n
+ ${partial}:
+ <<: *default
+ description: "Partials MAR files to serve as updates"
+ all_locales: true
+ from:
+ - partials-signing
+ partials_only: true
+ pretty_name: thunderbird-${branch}-${version}-${filename_platform}-${locale}-${from_buildid}-${buildid}.partial.mar
+ checksums_path: thunderbird-${branch}-${version}-${filename_platform}-${locale}-${from_buildid}-${buildid}.partial.mar
+ update_balrog_manifest: true
+ from_buildid: ${from_buildid}
+ destinations:
+ by-locale:
+ en-US:
+ - partials/${year}/${month}/${upload_date}-${branch}
+ default:
+ - partials/${year}/${month}/${upload_date}-${branch}-l10n
diff --git a/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly_checksums.yml b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly_checksums.yml
new file mode 100644
index 0000000000..4de3ae780e
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/manifests/thunderbird_nightly_checksums.yml
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+s3_bucket_paths:
+ - pub/thunderbird/nightly
+default_locales: # if given an empty locale, use these locales
+ - en-US
+tasktype_map: # Map task reference to task type.
+ beetmover-repackage: beetmover
+platform_names:
+ filename_platform:
+ by-platform:
+ linux-shippable: 'linux-i686'
+ linux64-shippable: 'linux-x86_64'
+ linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ macosx64-shippable: 'mac'
+ win32-shippable: 'win32'
+ win64-shippable: 'win64'
+ win64-asan-reporter-shippable: 'win64-asan-reporter'
+
+# A default entry, which the mappings below extend and override.
+# Final 'destinations' will be the product of:
+# s3_bucket_paths + destinations + locale_prefix + pretty_name
+default: &default
+ from:
+ - beetmover-repackage
+ all_locales: true
+ description: "TO_BE_OVERRIDDEN"
+ locale_prefix: ''
+ source_path_modifier: ''
+ destinations: # locale_prefix is appended
+ by-locale:
+ en-US:
+ - ${year}/${month}/${upload_date}-${branch}
+ - latest-${branch}
+ - latest-${branch}-l10n
+ default:
+ - ${year}/${month}/${upload_date}-${branch}-l10n
+ - latest-${branch}-l10n
+
+# Configuration for individual files. Extends 'default', above.
+mapping:
+ target.checksums:
+ <<: *default
+ description: "Checksums file containing size, hash, sha algorithm and filename"
+ pretty_name: thunderbird-${version}.${locale}.${filename_platform}.checksums
+ checksums_path: thunderbird-${version}.${locale}.${filename_platform}.checksums
diff --git a/comm/taskcluster/comm_taskgraph/optimize.py b/comm/taskcluster/comm_taskgraph/optimize.py
new file mode 100644
index 0000000000..24f7e67dfa
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/optimize.py
@@ -0,0 +1,151 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Thunderbird specific taskgraph optimizers.
+"""
+
+import logging
+
+from taskgraph.optimize.base import Any, OptimizationStrategy, register_strategy
+from taskgraph.util.memoize import memoize
+from taskgraph.util.path import join as join_path
+from taskgraph.util.path import match as match_path
+from taskgraph.util.yaml import load_yaml
+from voluptuous import Optional, Required
+
+from gecko_taskgraph import GECKO
+from gecko_taskgraph.optimize.schema import default_optimizations
+from mozlint.pathutils import filterpaths
+
+from comm_taskgraph import files_changed
+
+logger = logging.getLogger(__name__)
+
+
+def is_excluded(check_path, file_patterns):
+ for pattern in file_patterns:
+ if match_path(check_path, pattern):
+ return True
+ return False
+
+
+def is_suite(check_path):
+ suite_patterns = ("editor", "suite")
+ return is_excluded(check_path, suite_patterns)
+
+
+@memoize
+def get_non_suite_changed_files(repository, revision):
+ """
+ Returns the list of changed files from comm- repository (no prefixing)
+ with suite/** and editor/** files removed.
+ """
+ return {
+ file
+ for file in files_changed.get_changed_files(repository, revision)
+ if not is_suite(file)
+ }
+
+
+@register_strategy("comm-skip-unless-changed")
+class SkipUnlessChanged(OptimizationStrategy):
+ def should_remove_task(self, task, params, file_patterns):
+ # pushlog_id == -1 - this is the case when run from a cron.yml job
+ if params.get("pushlog_id") == -1:
+ return False
+
+ changed = files_changed.check(params, file_patterns)
+ if not changed:
+ logger.debug(
+ "no files found matching a pattern in `skip-unless-changed` for " + task.label
+ )
+ return True
+ return False
+
+
+@register_strategy("skip-suite-only")
+class SkipSuiteOnly(OptimizationStrategy):
+ def should_remove_task(self, task, params, arg):
+ # pushlog_id == -1 - this is the case when run from a cron.yml job
+ if params.get("pushlog_id") == -1:
+ return False
+
+ if params.get("project") == "try-comm-central":
+ # Do not try to use this optimization on try-c-c builds
+ return False
+
+ repository = params.get("comm_head_repository")
+ revision = params.get("comm_head_rev")
+ non_suite_changed_files = get_non_suite_changed_files(repository, revision)
+ # non_suite_changed_files will be an empty set (Falsy) for suite-only pushes
+ # so "skip" this task
+ if not non_suite_changed_files:
+ return True
+ return False
+
+
+@register_strategy("skip-unless-mozlint")
+class SkipUnlessMozlint(OptimizationStrategy):
+ schema = {
+ "root-path": Optional(str),
+ "mozlint-config": Required(str),
+ }
+
+ def should_remove_task(self, task, params, args):
+ include = []
+ exclude = []
+ extensions = []
+ support_files = []
+
+ root_path = join_path(GECKO, args.get("root-path", ""))
+ mozlint_root = join_path(root_path, "tools", "lint")
+ mozlint_yaml = join_path(mozlint_root, args["mozlint-config"])
+
+ logger.info(f"Loading file patterns for {task.label} from {mozlint_yaml}.")
+ linter_config = load_yaml(mozlint_yaml)
+ for check, config in linter_config.items():
+ include += config.get("include", [])
+ exclude += config.get("exclude", [])
+ extensions += [e.strip(".") for e in config.get("extensions", [])]
+ support_files += config.get("support-files", [])
+
+ changed_files = files_changed.get_files_changed_extended(params)
+
+ # Support files may not be part of "include" patterns, so check first
+ # Do not remove (return False) if any changed
+ for pattern in support_files:
+ for path in changed_files:
+ if match_path(path, pattern):
+ return False
+
+ to_lint, to_exclude = filterpaths(
+ GECKO,
+ list(changed_files),
+ include=include,
+ exclude=exclude,
+ extensions=extensions,
+ )
+
+ # to_lint should be an empty list if there is nothing to check
+ if not to_lint:
+ return True
+ return False
+
+
+register_strategy(
+ "skip-unless-backstop-no-suite", args=("skip-unless-backstop", "skip-suite-only")
+)(Any)
+
+register_strategy(
+ "skip-unless-changed-no-suite", args=("comm-skip-unless-changed", "skip-suite-only")
+)(Any)
+
+optimizations = (
+ {"skip-suite-only": None},
+ {"skip-unless-backstop-no-suite": None},
+ {"skip-unless-changed-no-suite": [str]},
+ {"skip-unless-mozlint": SkipUnlessMozlint.schema},
+)
+
+thunderbird_optimizations = default_optimizations + optimizations
diff --git a/comm/taskcluster/comm_taskgraph/parameters.py b/comm/taskcluster/comm_taskgraph/parameters.py
new file mode 100644
index 0000000000..a3cd3aebd5
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/parameters.py
@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+
+from taskgraph.parameters import extend_parameters_schema
+from voluptuous import Required
+
+from gecko_taskgraph.parameters import gecko_parameters_schema as comm_parameters_schema
+from gecko_taskgraph.parameters import get_app_version, get_version
+
+logger = logging.getLogger(__name__)
+
+
+# Called at import time when comm_taskgraph:register is called
+comm_parameters_schema.update(
+ {
+ Required("comm_base_repository"): str,
+ Required("comm_base_ref"): str,
+ Required("comm_base_rev"): str,
+ Required("comm_head_ref"): str,
+ Required("comm_head_repository"): str,
+ Required("comm_head_rev"): str,
+ Required("comm_src_path"): str,
+ }
+)
+
+
+def get_defaults(repo_root=None):
+ return {
+ "app_version": get_app_version(product_dir="comm/mail"),
+ "version": get_version("comm/mail"),
+ "comm_src_path": "comm/",
+ }
+
+
+def register_parameters():
+ """Register the additional comm_* parameters with taskgraph. Note that
+ defaults_fn is registered, but it does not actually run by design in the
+ decision task due to 'strict' being True in that case."""
+ extend_parameters_schema(comm_parameters_schema, defaults_fn=get_defaults)
diff --git a/comm/taskcluster/comm_taskgraph/target_tasks.py b/comm/taskcluster/comm_taskgraph/target_tasks.py
new file mode 100644
index 0000000000..cfb758200d
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/target_tasks.py
@@ -0,0 +1,75 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+
+from gecko_taskgraph.target_tasks import (
+ _target_task,
+ _try_task_config,
+ filter_on_platforms,
+ filter_out_shippable,
+ filter_out_shipping_phase,
+ standard_filter,
+)
+
+from comm_taskgraph.try_option_syntax import _try_cc_option_syntax
+
+logger = logging.getLogger(__name__)
+
+
+@_target_task("comm_searchfox_index")
+def target_tasks_searchfox(full_task_graph, parameters, graph_config):
+ """Select tasks required for indexing Thunderbird for Searchfox web site each day"""
+ return [
+ "searchfox-linux64-searchfox/debug",
+ "searchfox-macosx64-searchfox/debug",
+ "searchfox-win64-searchfox/debug",
+ ]
+
+
+@_target_task("comm_central_tasks")
+def target_tasks_default(full_task_graph, parameters, graph_config):
+ """Target the tasks which have indicated they should be run on this project
+ via the `run_on_projects` attributes."""
+ return [
+ l
+ for l, t in full_task_graph.tasks.items()
+ if standard_filter(t, parameters) and filter_out_shipping_phase(t, parameters)
+ ]
+
+
+@_target_task("try_cc_tasks")
+def target_tasks_try(full_task_graph, parameters, graph_config):
+ try_mode = parameters["try_mode"]
+ if try_mode == "try_task_config":
+ return _try_task_config(full_task_graph, parameters, graph_config)
+ elif try_mode == "try_option_syntax":
+ return _try_cc_option_syntax(full_task_graph, parameters, graph_config)
+ else:
+ # With no try mode, we schedule nothing, allowing the user to add tasks
+ # later via treeherder.
+ return []
+
+
+@_target_task("ash_tasks")
+def target_tasks_ash(full_task_graph, parameters, graph_config):
+ run_for_platforms = (
+ "linux64",
+ "macosx64",
+ "win64",
+ )
+
+ def filter_source_test(task):
+ return task.kind == "source-test" and task.attributes.get("always_target", False)
+
+ def _filter(task, _parameters):
+ return all(
+ [
+ filter_on_platforms(task, run_for_platforms) or filter_source_test(task),
+ standard_filter(task, _parameters),
+ filter_out_shippable(task),
+ ]
+ )
+
+ return [l for l, t in full_task_graph.tasks.items() if _filter(t, parameters)]
diff --git a/comm/taskcluster/comm_taskgraph/test/cc_automationrelevance.json b/comm/taskcluster/comm_taskgraph/test/cc_automationrelevance.json
new file mode 100644
index 0000000000..19751bad92
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/test/cc_automationrelevance.json
@@ -0,0 +1,154 @@
+{
+ "changesets": [
+ {
+ "author": "Magnus Melin <mkmelin+mozilla@iki.fi>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1829065",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1829065"
+ }
+ ],
+ "date": [1681974919, 0],
+ "desc": "Bug 1829065 - re-enable browser_messageFilters.js. r=leftmostcat\n\n\nReenable the test.\nRemove the customizaton sub test since the button it's about is now different + it's using the old customization which is on its way out.\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175988",
+ "extra": {
+ "branch": "default"
+ },
+ "files": [
+ "mail/test/browser/folder-widget/browser.ini",
+ "mail/test/browser/folder-widget/browser_messageFilters.js"
+ ],
+ "node": "e6e51c25cd7162180080914b6bc46cb0620c4a87",
+ "parents": ["87bed05abb8f99147cf83ecb3bc0cec6ada7f39d"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=comm-central&originalRevision=d0c328a2cc3340ab6f6458442ea89e366e33bd8e&newProject=comm-central&newRevision=e6e51c25cd7162180080914b6bc46cb0620c4a87",
+ "phase": "public",
+ "pushdate": [1681986116, 0],
+ "pushhead": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e",
+ "pushid": 20934,
+ "pushuser": "geoff@darktrojan.net",
+ "rev": 38727,
+ "reviewers": [
+ {
+ "name": "leftmostcat",
+ "revset": "reviewer(leftmostcat)"
+ }
+ ],
+ "treeherderrepo": "comm-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=comm-central"
+ },
+ {
+ "author": "John Bieling <john@thunderbird.net>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1828924",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1828924"
+ }
+ ],
+ "date": [1681911897, 0],
+ "desc": "Bug 1828924 - Fix imports of FileReader in the WebExtension API implementation files. r=mkmelin\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175899",
+ "extra": {
+ "amend_source": "83d21c8094985f1c8024d5966de1effa93b09b40",
+ "branch": "default"
+ },
+ "files": [
+ "mail/components/extensions/parent/ext-compose.js",
+ "mail/components/extensions/parent/ext-mail.js",
+ "mail/components/extensions/parent/ext-messages.js"
+ ],
+ "node": "04906b08eddb74d55b2fb4868290e4759a459158",
+ "parents": ["e6e51c25cd7162180080914b6bc46cb0620c4a87"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=comm-central&originalRevision=d0c328a2cc3340ab6f6458442ea89e366e33bd8e&newProject=comm-central&newRevision=e6e51c25cd7162180080914b6bc46cb0620c4a87",
+ "phase": "public",
+ "pushdate": [1681986116, 0],
+ "pushhead": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e",
+ "pushid": 20934,
+ "pushuser": "geoff@darktrojan.net",
+ "rev": 38728,
+ "reviewers": [
+ {
+ "name": "mkmelin",
+ "revset": "reviewer(mkmelin)"
+ }
+ ],
+ "treeherderrepo": "comm-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=comm-central"
+ },
+ {
+ "author": "Magnus Melin <mkmelin+mozilla@iki.fi>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1826744",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1826744"
+ }
+ ],
+ "date": [1681879233, 0],
+ "desc": "Bug 1826744 - \"Copy to Folder\" should be enabled for .eml files opened from file - was broken again. r=john.bieling\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175841",
+ "extra": {
+ "amend_source": "9b3f446beaa7132a9bfcd5542df376adb2af1337",
+ "branch": "default"
+ },
+ "files": ["mail/base/content/mailWindowOverlay.js"],
+ "node": "b72ae83322b6a8d878ad710cbe1f4d59201404e1",
+ "parents": ["04906b08eddb74d55b2fb4868290e4759a459158"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=comm-central&originalRevision=d0c328a2cc3340ab6f6458442ea89e366e33bd8e&newProject=comm-central&newRevision=e6e51c25cd7162180080914b6bc46cb0620c4a87",
+ "phase": "public",
+ "pushdate": [1681986116, 0],
+ "pushhead": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e",
+ "pushid": 20934,
+ "pushuser": "geoff@darktrojan.net",
+ "rev": 38729,
+ "reviewers": [
+ {
+ "name": "john.bieling",
+ "revset": "reviewer(john.bieling)"
+ }
+ ],
+ "treeherderrepo": "comm-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=comm-central"
+ },
+ {
+ "author": "Geoff Lankow <geoff@darktrojan.net>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1817367",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1817367"
+ }
+ ],
+ "date": [1681948136, -43200],
+ "desc": "Bug 1817367 - Wrap loading of virtual folder views in batch notifications. r=leftmostcat\nThis functionality was added to the base `nsMsgDBView`, but not these subclasses.\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175974",
+ "extra": {
+ "amend_source": "8722f310135f7c5f58cb1dcd4573189b3e225e6e",
+ "branch": "default",
+ "rebase_source": "4ca3cf372be8029d76bbfeecfcb406d197f0dbb6"
+ },
+ "files": [
+ "mailnews/base/src/nsMsgGroupView.cpp",
+ "mailnews/base/src/nsMsgQuickSearchDBView.cpp",
+ "mailnews/base/src/nsMsgSearchDBView.cpp",
+ "mailnews/base/src/nsMsgThreadedDBView.cpp",
+ "mailnews/base/src/nsMsgXFVirtualFolderDBView.cpp"
+ ],
+ "node": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e",
+ "parents": ["b72ae83322b6a8d878ad710cbe1f4d59201404e1"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=comm-central&originalRevision=d0c328a2cc3340ab6f6458442ea89e366e33bd8e&newProject=comm-central&newRevision=e6e51c25cd7162180080914b6bc46cb0620c4a87",
+ "phase": "public",
+ "pushdate": [1681986116, 0],
+ "pushhead": "d0c328a2cc3340ab6f6458442ea89e366e33bd8e",
+ "pushid": 20934,
+ "pushuser": "geoff@darktrojan.net",
+ "rev": 38730,
+ "reviewers": [
+ {
+ "name": "leftmostcat",
+ "revset": "reviewer(leftmostcat)"
+ }
+ ],
+ "treeherderrepo": "comm-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=comm-central"
+ }
+ ],
+ "visible": true
+}
diff --git a/comm/taskcluster/comm_taskgraph/test/conftest.py b/comm/taskcluster/comm_taskgraph/test/conftest.py
new file mode 100644
index 0000000000..83ed5343f4
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/test/conftest.py
@@ -0,0 +1,7 @@
+import os
+import sys
+
+HERE = os.path.dirname(__file__)
+EXT_PATH = os.path.abspath(os.path.join(HERE, "..", ".."))
+
+sys.path.insert(0, EXT_PATH)
diff --git a/comm/taskcluster/comm_taskgraph/test/mc_automationrelevance.json b/comm/taskcluster/comm_taskgraph/test/mc_automationrelevance.json
new file mode 100644
index 0000000000..8e90731d92
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/test/mc_automationrelevance.json
@@ -0,0 +1,309 @@
+{
+ "changesets": [
+ {
+ "author": "Noah <osuolale49@gmail.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1825749",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1825749"
+ }
+ ],
+ "date": [1681922538, 0],
+ "desc": "Bug 1825749 - Only run IdentityCredentialStorageCleaner if FedCM is enabled. r=hpeuckmann\n\nDifferential Revision: https://phabricator.services.mozilla.com/D174757",
+ "extra": {
+ "branch": "default",
+ "moz-landing-system": "lando"
+ },
+ "files": [
+ "toolkit/components/cleardata/ClearDataService.sys.mjs",
+ "toolkit/components/cleardata/tests/unit/test_identity_credential_storage.js"
+ ],
+ "landingsystem": "lando",
+ "node": "e35b8568fd3f9d49b77e791b44159589b6bcb309",
+ "parents": ["680421823d4d4a444a0045996d16299691565aa7"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041",
+ "phase": "public",
+ "pushdate": [1681982536, 0],
+ "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c",
+ "pushid": 40795,
+ "pushuser": "nfay@mozilla.com",
+ "rev": 661160,
+ "reviewers": [
+ {
+ "name": "hpeuckmann",
+ "revset": "reviewer(hpeuckmann)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Denis Palmeiro <dpalmeiro@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1824772",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1824772"
+ }
+ ],
+ "date": [1681925585, 0],
+ "desc": "Bug 1824772: part 1 - Add jit option and static pref to toggle eager baseline hints. r=iain\n\nAdd a pref, javascript.options.jitHints, to toggle eager baseline hints.\n\nDepends on D175520\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175521",
+ "extra": {
+ "branch": "default",
+ "moz-landing-system": "lando"
+ },
+ "files": ["modules/libpref/init/StaticPrefList.yaml"],
+ "landingsystem": "lando",
+ "node": "3740c123c6b87c6712dda72f53b0a2a8feac5110",
+ "parents": ["e35b8568fd3f9d49b77e791b44159589b6bcb309"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041",
+ "phase": "public",
+ "pushdate": [1681982536, 0],
+ "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c",
+ "pushid": 40795,
+ "pushuser": "nfay@mozilla.com",
+ "rev": 661161,
+ "reviewers": [
+ {
+ "name": "iain",
+ "revset": "reviewer(iain)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Tim Huang <tihuang@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1817463",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1817463"
+ }
+ ],
+ "date": [1681925941, 0],
+ "desc": "Bug 1817463 - Deferring SafeBrowsing updates if the browser is in idle mode. r=dimi\n\nThis patch implements the behavior for deferring SafeBrowsing updates\nwhen the browser is in idle mode. The update will be deferred until the\nnext user interaction active.\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175763",
+ "extra": {
+ "branch": "default",
+ "moz-landing-system": "lando"
+ },
+ "files": [
+ "toolkit/components/url-classifier/UrlClassifierListManager.jsm"
+ ],
+ "landingsystem": "lando",
+ "node": "80b29d7af43145cdd678d4ea51e770271f0c56ce",
+ "parents": ["6895dba1dca74eee9f56b4ce8656b4696f7c386d"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041",
+ "phase": "public",
+ "pushdate": [1681982536, 0],
+ "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c",
+ "pushid": 40795,
+ "pushuser": "nfay@mozilla.com",
+ "rev": 661165,
+ "reviewers": [
+ {
+ "name": "dimi",
+ "revset": "reviewer(dimi)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Sergey Galich <sgalich@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1828524",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1828524"
+ }
+ ],
+ "date": [1681926293, 0],
+ "desc": "Bug 1828524 - [Relay] Manage masks button leads to SUMO instead of relay.firefox.com r=credential-management-reviewers,dimi\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175705",
+ "extra": {
+ "branch": "default",
+ "moz-landing-system": "lando"
+ },
+ "files": [
+ "modules/libpref/init/all.js",
+ "toolkit/components/passwordmgr/FirefoxRelay.sys.mjs"
+ ],
+ "landingsystem": "lando",
+ "node": "091fb26cb5df9e5a61d9a43883b33bdf1736ea6e",
+ "parents": ["ea52cdb6d093a5eb233b10a207b64d02b7489c23"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041",
+ "phase": "public",
+ "pushdate": [1681982536, 0],
+ "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c",
+ "pushid": 40795,
+ "pushuser": "nfay@mozilla.com",
+ "rev": 661167,
+ "reviewers": [
+ {
+ "name": "credential-management-reviewers",
+ "revset": "reviewer(credential-management-reviewers)"
+ },
+ {
+ "name": "dimi",
+ "revset": "reviewer(dimi)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Randell Jesup <rjesup@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1828973",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1828973"
+ }
+ ],
+ "date": [1681928684, 0],
+ "desc": "Bug 1828973: Disable OCSP for web-platform tests r=jgraham\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175936",
+ "extra": {
+ "branch": "default",
+ "moz-landing-system": "lando"
+ },
+ "files": [
+ "testing/profiles/web-platform/user.js",
+ "testing/web-platform/meta/webtransport/__dir__.ini"
+ ],
+ "landingsystem": "lando",
+ "node": "ef5c4d384d2819bb857e1812ddca469fb0b87b86",
+ "parents": ["ccc54cebee26a52410f7c0cbd8ff8693078a620f"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041",
+ "phase": "public",
+ "pushdate": [1681982536, 0],
+ "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c",
+ "pushid": 40795,
+ "pushuser": "nfay@mozilla.com",
+ "rev": 661170,
+ "reviewers": [
+ {
+ "name": "jgraham",
+ "revset": "reviewer(jgraham)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Stanca Serban <sstanca@mozilla.com>",
+ "backsoutnodes": [
+ {
+ "node": "80b29d7af43145cdd678d4ea51e770271f0c56ce"
+ }
+ ],
+ "bugs": [
+ {
+ "no": "1817463",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1817463"
+ }
+ ],
+ "date": [1681930381, -10800],
+ "desc": "Backed out changeset 80b29d7af431 (bug 1817463) for failures in toolkit/components/url-classifier/tests.",
+ "extra": {
+ "branch": "default",
+ "rebase_source": "9fc0fd23fdbaa4c30568ca895af3e1c9e3c08aea"
+ },
+ "files": [
+ "toolkit/components/url-classifier/UrlClassifierListManager.jsm"
+ ],
+ "node": "ae17e25d0f24bd1b6dee872c77c9770ad0ec7d87",
+ "parents": ["6ec708eb48569fcfb9bf708d91a8876383865a65"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041",
+ "phase": "public",
+ "pushdate": [1681982536, 0],
+ "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c",
+ "pushid": 40795,
+ "pushuser": "nfay@mozilla.com",
+ "rev": 661172,
+ "reviewers": [],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Bilal <bnasar@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1817002",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1817002"
+ }
+ ],
+ "date": [1681931937, 0],
+ "desc": "Bug 1817002 - ESMified browser/components/pocket/content. r=kpatenio\n\nDifferential Revision: https://phabricator.services.mozilla.com/D175700",
+ "extra": {
+ "branch": "default",
+ "moz-landing-system": "lando"
+ },
+ "files": [
+ "toolkit/components/normandy/content/AboutPages.sys.mjs",
+ "toolkit/modules/NewTabUtils.sys.mjs"
+ ],
+ "landingsystem": "lando",
+ "node": "eff25b124b5e2c29c10442e902e9c4a9c488672c",
+ "parents": ["dcb12f0cfb12a4d3ae666f1660276b9cea2419ba"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041",
+ "phase": "public",
+ "pushdate": [1681982536, 0],
+ "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c",
+ "pushid": 40795,
+ "pushuser": "nfay@mozilla.com",
+ "rev": 661174,
+ "reviewers": [
+ {
+ "name": "kpatenio",
+ "revset": "reviewer(kpatenio)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Sammy Khamis <skhamis@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1825905",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1825905"
+ }
+ ],
+ "date": [1681937295, 0],
+ "desc": "Bug 1825905: Autofill sync engine should roundtrip data it doesn't know about r=sgalich,lina\n\nDifferential Revision: https://phabricator.services.mozilla.com/D174834",
+ "extra": {
+ "branch": "default",
+ "moz-landing-system": "lando"
+ },
+ "files": [
+ "toolkit/components/formautofill/FormAutofillStorageBase.sys.mjs"
+ ],
+ "landingsystem": "lando",
+ "node": "2d1f87aeb76249a22f8db78ac031305816544514",
+ "parents": ["5f3904256e0d38a7cffed5ef443b7b259d43ad49"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=6ca54f5a4a1b2a12530001ea3f092c29810e803c&newProject=mozilla-central&newRevision=11fec803ea08a3e440016a5cdc9686937fd94041",
+ "phase": "public",
+ "pushdate": [1681982536, 0],
+ "pushhead": "6ca54f5a4a1b2a12530001ea3f092c29810e803c",
+ "pushid": 40795,
+ "pushuser": "nfay@mozilla.com",
+ "rev": 661182,
+ "reviewers": [
+ {
+ "name": "sgalich",
+ "revset": "reviewer(sgalich)"
+ },
+ {
+ "name": "lina",
+ "revset": "reviewer(lina)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/jobs?repo=mozilla-central"
+ }
+ ],
+ "visible": true
+}
diff --git a/comm/taskcluster/comm_taskgraph/test/python.ini b/comm/taskcluster/comm_taskgraph/test/python.ini
new file mode 100644
index 0000000000..84548afeb0
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/test/python.ini
@@ -0,0 +1,6 @@
+[DEFAULT]
+subsuite = comm_taskgraph
+
+[test_files_changed.py]
+[test_optimization_strategies.py]
+[test_parameters.py]
diff --git a/comm/taskcluster/comm_taskgraph/test/test_files_changed.py b/comm/taskcluster/comm_taskgraph/test/test_files_changed.py
new file mode 100644
index 0000000000..5759520de6
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/test/test_files_changed.py
@@ -0,0 +1,144 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import unittest
+
+import conftest # noqa: F401
+from mozunit import main
+
+from gecko_taskgraph.util import hg
+
+from comm_taskgraph import files_changed
+
+PARAMS = {
+ "comm_head_repository": "https://hg.mozilla.org/comm-central",
+ "comm_head_rev": "d0c328a2cc33",
+ "head_repository": "https://hg.mozilla.org/mozilla-central",
+ "head_rev": "6ca54f5a4a1b",
+ "comm_src_path": "comm/",
+}
+
+FILES_CHANGED_CC = [
+ "mail/base/content/mailWindowOverlay.js",
+ "mail/components/extensions/parent/ext-compose.js",
+ "mail/components/extensions/parent/ext-mail.js",
+ "mail/components/extensions/parent/ext-messages.js",
+ "mail/test/browser/folder-widget/browser.ini",
+ "mail/test/browser/folder-widget/browser_messageFilters.js",
+ "mailnews/base/src/nsMsgGroupView.cpp",
+ "mailnews/base/src/nsMsgQuickSearchDBView.cpp",
+ "mailnews/base/src/nsMsgSearchDBView.cpp",
+ "mailnews/base/src/nsMsgThreadedDBView.cpp",
+ "mailnews/base/src/nsMsgXFVirtualFolderDBView.cpp",
+]
+
+FILES_CHANGED_MC = [
+ "modules/libpref/init/StaticPrefList.yaml",
+ "modules/libpref/init/all.js",
+ "testing/profiles/web-platform/user.js",
+ "testing/web-platform/meta/webtransport/__dir__.ini",
+ "toolkit/components/cleardata/ClearDataService.sys.mjs",
+ "toolkit/components/cleardata/tests/unit/test_identity_credential_storage.js",
+ "toolkit/components/formautofill/FormAutofillStorageBase.sys.mjs",
+ "toolkit/components/normandy/content/AboutPages.sys.mjs",
+ "toolkit/components/passwordmgr/FirefoxRelay.sys.mjs",
+ "toolkit/components/url-classifier/UrlClassifierListManager.jsm",
+ "toolkit/modules/NewTabUtils.sys.mjs",
+]
+
+FILES_CHANGED = sorted(
+ FILES_CHANGED_MC + [os.path.join("comm", file) for file in FILES_CHANGED_CC]
+)
+
+
+class FakeResponse:
+ def __init__(self, url, **kwargs):
+ if "comm-central" in url:
+ self.filename = "cc_automationrelevance.json"
+ elif "mozilla-central" in url:
+ self.filename = "mc_automationrelevance.json"
+ else:
+ raise Exception(f"Invalid automation URL: {url}")
+
+ def json(self):
+ with open(os.path.join(os.path.dirname(__file__), self.filename)) as f:
+ return json.load(f)
+
+
+class TestGetChangedFiles(unittest.TestCase):
+ def setUp(self):
+ files_changed.get_changed_files.clear()
+ self.old_get = hg.requests.get
+
+ def fake_get(url, **kwargs):
+ return FakeResponse(url)
+
+ hg.requests.get = fake_get
+
+ def tearDown(self):
+ hg.requests.get = self.old_get
+ files_changed.get_changed_files.clear()
+
+ def test_get_changed_files_mc(self):
+ """Get_changed_files correctly gets the list of changed files in a push.
+ This tests against the production hg.mozilla.org so that it will detect
+ any changes in the format of the returned data."""
+ self.assertEqual(
+ sorted(files_changed.get_changed_files(PARAMS["head_repository"], PARAMS["head_rev"])),
+ FILES_CHANGED_MC,
+ )
+
+ def test_get_changed_files_cc(self):
+ """Get_changed_files correctly gets the list of changed files in a push.
+ This tests against the production hg.mozilla.org so that it will detect
+ any changes in the format of the returned data."""
+ self.assertEqual(
+ sorted(
+ files_changed.get_changed_files(
+ PARAMS["comm_head_repository"], PARAMS["comm_head_rev"]
+ )
+ ),
+ FILES_CHANGED_CC,
+ )
+
+ def test_get_changed_files_extended(self):
+ """Get_changed_files_extended correctly gets the list of changed files in a push.
+ This tests against the production hg.mozilla.org so that it will detect
+ any changes in the format of the returned data."""
+ self.assertEqual(
+ sorted(files_changed.get_files_changed_extended(PARAMS)),
+ FILES_CHANGED,
+ )
+
+
+class TestCheck(unittest.TestCase):
+ def setUp(self):
+ files_changed.get_changed_files[PARAMS["head_repository"], PARAMS["head_rev"]] = set(
+ FILES_CHANGED_MC
+ )
+ files_changed.get_changed_files[
+ PARAMS["comm_head_repository"], PARAMS["comm_head_rev"]
+ ] = set(FILES_CHANGED_CC)
+
+ def tearDown(self):
+ files_changed.get_changed_files.clear()
+
+ def test_check_no_params(self):
+ self.assertTrue(files_changed.check({}, ["ignored"]))
+
+ def test_check_no_match(self):
+ self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"]))
+ self.assertFalse(files_changed.check(PARAMS, ["comm/nosuch/**"]))
+
+ def test_check_match_mc(self):
+ self.assertTrue(files_changed.check(PARAMS, ["toolkit/**"]))
+
+ def test_check_match_cc(self):
+ self.assertTrue(files_changed.check(PARAMS, ["comm/mail/**"]))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/comm/taskcluster/comm_taskgraph/test/test_optimization_strategies.py b/comm/taskcluster/comm_taskgraph/test/test_optimization_strategies.py
new file mode 100644
index 0000000000..8e43760714
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/test/test_optimization_strategies.py
@@ -0,0 +1,143 @@
+# Any copyright is dedicated to the public domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+import hashlib
+from datetime import datetime
+from time import mktime
+
+import conftest # noqa: F401
+import pytest
+import responses
+from mozunit import main
+from taskgraph.task import Task
+
+from gecko_taskgraph.optimize import registry
+
+from comm_taskgraph.optimize import SkipSuiteOnly
+
+
+def generate_task():
+ task = {}
+ task.setdefault("label", "task-label")
+ task.setdefault("kind", "build")
+ task.setdefault("task", {})
+ task.setdefault("attributes", {})
+
+ for attr in (
+ "dependencies",
+ "optimization",
+ "soft_dependencies",
+ "release_artifacts",
+ ):
+ task.setdefault(attr, None)
+
+ task["task"].setdefault("label", task["label"])
+ return Task.from_json(task)
+
+
+def idfn(param):
+ if isinstance(param, tuple):
+ return param[0].__name__
+ return None
+
+
+def generate_json_push_data(files_changed):
+ return {"changesets": [{"desc": "commit comment", "files": files_changed, "node": "cdefgh"}]}
+
+
+@pytest.fixture
+def params():
+ return {
+ "branch": "comm-central",
+ "head_repository": "https://hg.mozilla.org/mozilla-central",
+ "head_rev": "zyxwvu",
+ "comm_head_repository": "https://hg.mozilla.org/comm-central",
+ "comm_head_rev": "abcdef",
+ "comm_src_path": "comm/",
+ "project": "comm-central",
+ "pushlog_id": 1,
+ "pushdate": mktime(datetime.now().timetuple()),
+ }
+
+
+def mk_rev(strings):
+ data = "".join(strings).encode("utf-8")
+ h = hashlib.new("sha1")
+ h.update(data)
+ return h.hexdigest()
+
+
+@responses.activate
+@pytest.mark.parametrize(
+ "pushed_files,expected",
+ [
+ # suite-only push
+ pytest.param(["suite/a/b/c.txt", "suite/b/c/d.txt"], True),
+ # non-suite push
+ pytest.param(["mail/a/b/c.txt", "mailnews/b/c/d.txt"], False),
+ # mixed push
+ pytest.param(["suite/a/b/c.txt", "calendar/b/c/d.txt"], False),
+ ],
+ ids=idfn,
+)
+def test_suite_only_strategy(params, pushed_files, expected):
+ rev = mk_rev(pushed_files)
+ params["comm_head_rev"] = rev
+
+ responses.add(
+ responses.GET,
+ "https://hg.mozilla.org/comm-central/json-automationrelevance/{}".format(rev),
+ json=generate_json_push_data(pushed_files),
+ status=200,
+ )
+ task = generate_task()
+
+ opt = SkipSuiteOnly()
+ remove = opt.should_remove_task(task, params, None)
+
+ assert remove == expected
+
+
+@responses.activate
+@pytest.mark.parametrize(
+ "file_patterns,pushed_files,expected",
+ [
+ # suite-only push, matches files-changed
+ pytest.param(["comm/**/*.txt"], ["suite/a/b/c.txt", "suite/b/c/d.js"], True),
+ # suite-only push, does not match files-changed
+ pytest.param(["comm/**/*.cpp"], ["suite/a/b/c.txt", "suite/b/c/d.js"], True),
+ # non-suite push, matches files changed
+ pytest.param(["comm/**/*.txt"], ["mail/a/b/c.txt", "mailnews/b/c/d.js"], False),
+ # non-suite push, does not match files changed
+ pytest.param(["comm/**/*.cpp"], ["mail/a/b/c.txt", "mailnews/b/c/d.js"], True),
+ ],
+ ids=idfn,
+)
+def test_suite_files_changed_strategy(params, file_patterns, pushed_files, expected):
+ rev = mk_rev(pushed_files)
+ params["comm_head_rev"] = rev
+
+ # Fake the m-c json data
+ responses.add(
+ responses.GET,
+ "https://hg.mozilla.org/mozilla-central/json-automationrelevance/zyxwvu",
+ json=generate_json_push_data([]),
+ status=200,
+ )
+
+ responses.add(
+ responses.GET,
+ "https://hg.mozilla.org/comm-central/json-automationrelevance/{}".format(rev),
+ json=generate_json_push_data(pushed_files),
+ status=200,
+ )
+ task = generate_task()
+
+ opt = registry["skip-unless-changed-no-suite"]
+ remove = opt.should_remove_task(task, params, file_patterns)
+
+ assert remove == expected
+
+
+if __name__ == "__main__":
+ main()
diff --git a/comm/taskcluster/comm_taskgraph/test/test_parameters.py b/comm/taskcluster/comm_taskgraph/test/test_parameters.py
new file mode 100644
index 0000000000..8a9d78701b
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/test/test_parameters.py
@@ -0,0 +1,102 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+import conftest # noqa: F401
+from mozunit import main
+from taskgraph.parameters import Parameters
+
+from comm_taskgraph.parameters import register_parameters
+
+
+class TestCommParameters(unittest.TestCase):
+
+ vals = {
+ "app_version": "app_version",
+ "backstop": False,
+ "base_repository": "base_repository",
+ "base_ref": "base_ref",
+ "base_rev": "base_rev",
+ "build_date": 0,
+ "build_number": 0,
+ "comm_base_repository": "comm_base_repository",
+ "comm_base_ref": "comm_base_ref",
+ "comm_base_rev": "comm_base_rev",
+ "comm_head_ref": "comm_head_ref",
+ "comm_head_repository": "comm_head_repository",
+ "comm_head_rev": "comm_head_rev",
+ "comm_src_path": "comm/",
+ "do_not_optimize": [],
+ "enable_always_target": False,
+ "existing_tasks": {},
+ "filters": [],
+ "head_ref": "head_ref",
+ "head_repository": "head_repository",
+ "head_rev": "head_rev",
+ "head_tag": "",
+ "hg_branch": "hg_branch",
+ "level": "level",
+ "message": "message",
+ "moz_build_date": "moz_build_date",
+ "next_version": "next_version",
+ "optimize_strategies": None,
+ "optimize_target_tasks": False,
+ "owner": "owner",
+ "phabricator_diff": "phabricator_diff",
+ "project": "project",
+ "pushdate": 0,
+ "pushlog_id": "pushlog_id",
+ "release_enable_emefree": False,
+ "release_enable_partner_repack": False,
+ "release_enable_partner_attribution": False,
+ "release_eta": None,
+ "release_history": {},
+ "release_partners": [],
+ "release_partner_config": None,
+ "release_partner_build_number": 1,
+ "release_type": "release_type",
+ "release_product": None,
+ "repository_type": "hg",
+ "required_signoffs": [],
+ "signoff_urls": {},
+ "target_tasks_method": "target_tasks_method",
+ "test_manifest_loader": "default",
+ "tasks_for": "tasks_for",
+ "try_mode": "try_mode",
+ "try_options": None,
+ "try_task_config": {},
+ "version": "version",
+ }
+
+ def setUp(self):
+ register_parameters()
+
+ def test_Parameters_check(self):
+ """
+ Specifying all of the gecko and comm parameters doesn't result in an error.
+ """
+ p = Parameters(**self.vals)
+ p.check() # should not raise
+
+ def test_Parameters_check_missing(self):
+ """
+ If any of the comm parameters are specified, all of them must be specified.
+ """
+ vals = self.vals.copy()
+ del vals["comm_base_repository"]
+ p = Parameters(**vals)
+ self.assertRaises(Exception, p.check)
+
+ def test_Parameters_check_extra(self):
+ """
+ If parameters other than the global and comm parameters are specified,
+ an error is reported.
+ """
+ p = Parameters(extra="data", **self.vals)
+ self.assertRaises(Exception, p.check)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/comm/taskcluster/comm_taskgraph/transforms/__init__.py b/comm/taskcluster/comm_taskgraph/transforms/__init__.py
new file mode 100644
index 0000000000..6fbe8159b2
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/__init__.py
@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
diff --git a/comm/taskcluster/comm_taskgraph/transforms/job/__init__.py b/comm/taskcluster/comm_taskgraph/transforms/job/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/job/__init__.py
diff --git a/comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py b/comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py
new file mode 100644
index 0000000000..a80c058a2b
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py
@@ -0,0 +1,165 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running toolchain-building jobs via dedicated scripts in comm-central
+"""
+
+import os.path
+
+import taskgraph
+import taskgraph.util.path as util_path
+from taskgraph.util.schema import resolve_keyed_by
+from voluptuous import Any, Optional, Required
+
+from gecko_taskgraph import GECKO
+from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using
+from gecko_taskgraph.transforms.job.common import docker_worker_add_artifacts
+from gecko_taskgraph.transforms.job.toolchain import toolchain_defaults, toolchain_run_schema
+from gecko_taskgraph.util.attributes import RELEASE_PROJECTS
+from gecko_taskgraph.util.hash import hash_paths as hash_paths_gecko_root
+
+from comm_taskgraph.util.hash import hash_paths_extended
+
+CACHE_TYPE = "toolchains.v3"
+
+TOOLCHAIN_SCRIPT_PATH = "comm/taskcluster/scripts"
+
+
+comm_toolchain_run_schema = toolchain_run_schema.extend(
+ {
+ Required("using"): Any("comm-toolchain-script"),
+ Optional("script"): str,
+ }
+)
+
+
+def hash_paths(*args):
+ """
+ Helper function while the single repository project is in development.
+ The extended version of hash_paths found in comm_taskgraph.util.hash is
+ not necessary (and does not work) with single-repo. This is a wrapper
+ function to pick the right function based on the presence of a comm/.hg
+ directory.
+ """
+ comm_hg_path = util_path.join(GECKO, "comm", ".hg")
+ if os.path.exists(comm_hg_path):
+ return hash_paths_extended(*args)
+ else:
+ return hash_paths_gecko_root(*args)
+
+
+def get_digest_data(config, run, taskdesc):
+ """
+ Copied from gecko_taskgraph.transforms.job.toolchain, with minor
+ modifications to support the required script path.
+ """
+ files = list(run.pop("resources", []))
+ # This file
+ files.append("comm/taskcluster/comm_taskgraph/transforms/job/toolchain.py")
+ # The script
+ if "script" in run:
+ files.append("{}/{}".format(TOOLCHAIN_SCRIPT_PATH, run["script"]))
+ # Tooltool manifest if any is defined:
+ tooltool_manifest = taskdesc["worker"]["env"].get("TOOLTOOL_MANIFEST")
+ if tooltool_manifest:
+ files.append(tooltool_manifest)
+
+ # Accumulate dependency hashes for index generation.
+ data = [hash_paths(GECKO, files)]
+
+ data.append(taskdesc["attributes"]["toolchain-artifact"])
+
+ # If the task uses an in-tree docker image, we want it to influence
+ # the index path as well. Ideally, the content of the docker image itself
+ # should have an influence, but at the moment, we can't get that
+ # information here. So use the docker image name as a proxy. Not a lot of
+ # changes to docker images actually have an impact on the resulting
+ # toolchain artifact, so we'll just rely on such important changes to be
+ # accompanied with a docker image name change.
+ image = taskdesc["worker"].get("docker-image", {}).get("in-tree")
+ if image:
+ data.append(image)
+
+ # Likewise script arguments should influence the index.
+ args = run.get("arguments")
+ if args:
+ data.extend(args)
+
+ if taskdesc["attributes"].get("rebuild-on-release"):
+ # Add whether this is a release branch or not
+ data.append(str(config.params["project"] in RELEASE_PROJECTS))
+ return data
+
+
+@run_job_using(
+ "docker-worker",
+ "comm-toolchain-script",
+ schema=comm_toolchain_run_schema,
+ defaults=toolchain_defaults,
+)
+def docker_worker_toolchain(config, job, taskdesc):
+ run = job["run"]
+ run["comm-checkout"] = True
+
+ worker = taskdesc["worker"] = job["worker"]
+ worker["chain-of-trust"] = True
+
+ # If the task doesn't have a docker-image, set a default
+ worker.setdefault("docker-image", {"in-tree": "deb11-toolchain-build"})
+
+ # Toolchain checkouts don't live under {workdir}/checkouts
+ workspace = "{workdir}/workspace/build".format(**run)
+ gecko_path = "{}/src".format(workspace)
+
+ env = worker.setdefault("env", {})
+ env.update(
+ {
+ "MOZ_BUILD_DATE": config.params["moz_build_date"],
+ "MOZ_SCM_LEVEL": config.params["level"],
+ "GECKO_PATH": gecko_path,
+ "TOOLCHAIN_ARTIFACT": run["toolchain-artifact"],
+ }
+ )
+
+ attributes = taskdesc.setdefault("attributes", {})
+ attributes["toolchain-artifact"] = run.pop("toolchain-artifact")
+ toolchain_artifact = attributes["toolchain-artifact"]
+ if not toolchain_artifact.startswith("public/build/"):
+ attributes["artifact_prefix"] = os.path.dirname(toolchain_artifact)
+
+ resolve_keyed_by(
+ run,
+ "toolchain-alias",
+ item_name=taskdesc["label"],
+ project=config.params["project"],
+ )
+ alias = run.pop("toolchain-alias", None)
+ if alias:
+ attributes["toolchain-alias"] = alias
+ if "toolchain-env" in run:
+ attributes["toolchain-env"] = run.pop("toolchain-env")
+
+ # Allow the job to specify where artifacts come from, but add
+ # public/build if it's not there already.
+ artifacts = worker.setdefault("artifacts", [])
+ if not artifacts:
+ docker_worker_add_artifacts(config, job, taskdesc)
+
+ digest_data = get_digest_data(config, run, taskdesc)
+
+ if job.get("attributes", {}).get("cached_task") is not False and not taskgraph.fast:
+ name = taskdesc["label"].replace(f"{config.kind}-", "", 1)
+ taskdesc["cache"] = {
+ "type": CACHE_TYPE,
+ "name": name,
+ "digest-data": digest_data,
+ }
+
+ run["using"] = "run-task"
+ run["cwd"] = run["workdir"]
+ run["command"] = [
+ "workspace/build/src/{}/{}".format(TOOLCHAIN_SCRIPT_PATH, run.pop("script"))
+ ] + run.pop("arguments", [])
+
+ configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"])
diff --git a/comm/taskcluster/comm_taskgraph/transforms/l10n.py b/comm/taskcluster/comm_taskgraph/transforms/l10n.py
new file mode 100644
index 0000000000..51246ad888
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/l10n.py
@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Do transforms specific to l10n kind
+"""
+
+from taskgraph.transforms.base import TransformSequence, ValidateSchema
+from taskgraph.util.schema import resolve_keyed_by
+
+from gecko_taskgraph.transforms.l10n import (
+ all_locales_attribute,
+ chunk_locales,
+ copy_in_useful_magic,
+ handle_artifact_prefix,
+ handle_keyed_by,
+ l10n_description_schema,
+ make_job_description,
+ set_extra_config,
+ setup_name,
+)
+
+
+def setup_signing_dependency(config, jobs):
+ """Sets up a task dependency to the signing job this relates to"""
+ for job in jobs:
+ job["dependencies"].update(
+ {
+ "build": job["dependent-tasks"]["build"].label,
+ }
+ )
+
+ if job["attributes"]["build_platform"].startswith("win"):
+ job["dependencies"].update(
+ {
+ "build-signing": job["dependent-tasks"]["build-signing"].label,
+ }
+ )
+
+ if "shippable" in job["attributes"]["build_platform"]:
+ if job["attributes"]["build_platform"].startswith("macosx"):
+ job["dependencies"].update(
+ {"repackage": job["dependent-tasks"]["repackage"].label}
+ )
+ if job["attributes"]["build_platform"].startswith("linux"):
+ job["dependencies"].update(
+ {
+ "build-signing": job["dependent-tasks"]["build-signing"].label,
+ }
+ )
+ yield job
+
+
+def handle_keyed_by_local(config, jobs):
+ """Resolve fields that can be keyed by platform, etc."""
+ for job in jobs:
+ resolve_keyed_by(
+ job,
+ "locales-file",
+ item_name=job["name"],
+ **{"release-type": config.params["release_type"]},
+ )
+ yield job
+
+
+transforms = TransformSequence()
+
+
+for transform_func in (
+ setup_name,
+ copy_in_useful_magic,
+ handle_keyed_by_local,
+ ValidateSchema(l10n_description_schema),
+ setup_signing_dependency,
+ handle_keyed_by,
+ handle_artifact_prefix,
+ all_locales_attribute,
+ chunk_locales,
+ ValidateSchema(l10n_description_schema),
+ set_extra_config,
+ make_job_description,
+):
+ transforms.add(transform_func)
diff --git a/comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py b/comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py
new file mode 100644
index 0000000000..a642b0e592
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/l10n_pre.py
@@ -0,0 +1,45 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Create a strings build artifact to be consumed by shippable-l10n.
+"""
+
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import resolve_keyed_by
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def handle_keyed_by(config, jobs):
+ """Resolve fields that can be keyed by platform, etc."""
+ for job in jobs:
+ resolve_keyed_by(
+ job,
+ "locale-list",
+ item_name=job["name"],
+ **{"release-type": config.params["release_type"]},
+ )
+ yield job
+
+
+@transforms.add
+def make_job_description(config, jobs):
+ for job in jobs:
+ locale_list = job.pop("locale-list")
+ comm_locales_file = job.pop("comm-locales-file")
+ browser_locales_file = job.pop("browser-locales-file")
+ job["run"].update(
+ {
+ "job-script": "comm/taskcluster/scripts/build-l10n-pre.sh",
+ "options": [
+ f"locale-list={locale_list}",
+ f"comm-locales-file={comm_locales_file}",
+ f"browser-locales-file={browser_locales_file}",
+ ],
+ }
+ )
+
+ yield job
diff --git a/comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py b/comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py
new file mode 100644
index 0000000000..92a95cb612
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/l10n_source_signing.py
@@ -0,0 +1,52 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the signing task into an actual task description.
+"""
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.taskcluster import get_artifact_path
+
+from gecko_taskgraph.transforms.build_signing import add_signed_routes
+from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job
+
+transforms = TransformSequence()
+
+transforms.add(add_signed_routes)
+
+
+@transforms.add
+def define_upstream_artifacts(config, jobs):
+ for job in jobs:
+ dep_job = job["primary-dependency"]
+ upstream_artifact_task = job.pop("upstream-artifact-task", dep_job)
+
+ job["attributes"] = copy_attributes_from_dependent_job(dep_job)
+
+ artifacts_specifications = [
+ {
+ "artifacts": [
+ get_artifact_path(job, "strings_all.tar.zst"),
+ get_artifact_path(job, "l10n-changesets.json"),
+ ],
+ "formats": ["autograph_gpg"],
+ }
+ ]
+
+ task_ref = f"<{upstream_artifact_task.kind}>"
+ task_type = "build"
+ if "notarization" in upstream_artifact_task.kind:
+ task_type = "scriptworker"
+
+ job["upstream-artifacts"] = [
+ {
+ "taskId": {"task-reference": task_ref},
+ "taskType": task_type,
+ "paths": spec["artifacts"],
+ "formats": spec["formats"],
+ }
+ for spec in artifacts_specifications
+ ]
+
+ yield job
diff --git a/comm/taskcluster/comm_taskgraph/transforms/merge_automation.py b/comm/taskcluster/comm_taskgraph/transforms/merge_automation.py
new file mode 100644
index 0000000000..75ced147bb
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/merge_automation.py
@@ -0,0 +1,58 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Fix-ups for comm-central merge automation
+"""
+
+import os
+import re
+
+from taskgraph.transforms.base import TransformSequence
+
+from comm_taskgraph import COMM
+
+transforms = TransformSequence()
+
+
+def do_suite_verbump(replacements):
+ """Bump the minor version of suite version files."""
+ allowed_files = ("suite/config/version.txt", "suite/config/version_display.txt")
+ old_version, new_version = None, None
+
+ new_replacements = []
+ for file, old, new in replacements:
+ if file not in allowed_files:
+ break
+ if old_version is None or new_version is None:
+ path = os.path.join(COMM, file)
+ data = open(path).read()
+ match = re.match(r"^(2)\.(\d+)(a1)$", data)
+ if match:
+ old_version = match.group(0)
+
+ old_minor = match.group(2)
+ new_minor = str(int(old_minor) + 1)
+
+ new_version = f"{match.group(1)}.{new_minor}{match.group(3)}"
+
+ new_replacements.append([file, old_version, new_version])
+
+ if len(new_replacements) == len(replacements):
+ return new_replacements
+ else:
+ raise Exception(f"do_suite_version failed: {replacements}, {new_replacements}")
+
+
+@transforms.add
+def update_suite_versions(config, tasks):
+ for task in tasks:
+ if "merge_config" not in config.params:
+ break
+ behavior = config.params["merge_config"]["behavior"]
+ if behavior == "comm-bump-central":
+ merge_config = task["worker"]["merge-info"]
+ replacements = merge_config["replacements"]
+ merge_config["replacements"] = do_suite_verbump(replacements)
+
+ yield task
diff --git a/comm/taskcluster/comm_taskgraph/transforms/partials.py b/comm/taskcluster/comm_taskgraph/transforms/partials.py
new file mode 100644
index 0000000000..39d6cf95a7
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/partials.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Thunderbird modifications to partial update building
+"""
+import logging
+
+from taskgraph.transforms.base import TransformSequence
+
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def update_scopes(config, jobs):
+ """
+ Firefox does some caching when building partial updates, but there's no bucket for Thunderbird
+ at the moment. In the meantime, remove the scope from the task to avoid an error.
+ """
+ # If no balrog release history, then don't run
+ if not config.params.get("release_history"):
+ return
+
+ MBSDIFF_SCOPE = "auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/"
+
+ for job in jobs:
+ task = job["task"]
+ if MBSDIFF_SCOPE in task["scopes"]:
+ task["scopes"].remove(MBSDIFF_SCOPE)
+
+ yield job
diff --git a/comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py b/comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py
new file mode 100644
index 0000000000..8f6c7a5d5f
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/push_langpacks.py
@@ -0,0 +1,231 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the release-push-langpacks task into an actual task description.
+"""
+
+import json
+import os
+from contextlib import contextmanager
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, taskref_or_string
+from taskgraph.util.treeherder import inherit_treeherder_from_dep
+from voluptuous import Any, Optional, Required
+
+from gecko_taskgraph.loader.single_dep import schema
+from gecko_taskgraph.transforms.task import task_description_schema
+from gecko_taskgraph.util.attributes import (
+ copy_attributes_from_dependent_job,
+ release_level,
+)
+from mozbuild.action.langpack_manifest import get_version_maybe_buildid
+
+transforms = TransformSequence()
+
+langpack_push_description_schema = schema.extend(
+ {
+ Required("label"): str,
+ Required("description"): str,
+ Required("worker-type"): optionally_keyed_by("release-level", str),
+ Required("worker"): {
+ Required("docker-image"): {"in-tree": str},
+ Required("implementation"): "docker-worker",
+ Required("os"): "linux",
+ Optional("max-run-time"): int,
+ Required("env"): {str: taskref_or_string},
+ Required("channel"): optionally_keyed_by(
+ "project", "platform", Any("listed", "unlisted")
+ ),
+ Required("command"): [taskref_or_string],
+ },
+ Required("run-on-projects"): [],
+ Required("scopes"): optionally_keyed_by("release-level", [str]),
+ Required("shipping-phase"): task_description_schema["shipping-phase"],
+ Required("shipping-product"): task_description_schema["shipping-product"],
+ }
+)
+
+
+@transforms.add
+def set_label(config, jobs):
+ for job in jobs:
+ label = "push-langpacks-{}".format(job["primary-dependency"].label)
+ job["label"] = label
+
+ yield job
+
+
+transforms.add_validate(langpack_push_description_schema)
+
+
+@transforms.add
+def resolve_keys(config, jobs):
+ for job in jobs:
+ resolve_keyed_by(
+ job,
+ "worker-type",
+ item_name=job["label"],
+ **{"release-level": release_level(config.params["project"])},
+ )
+ resolve_keyed_by(
+ job,
+ "scopes",
+ item_name=job["label"],
+ **{"release-level": release_level(config.params["project"])},
+ )
+ resolve_keyed_by(
+ job,
+ "worker.channel",
+ item_name=job["label"],
+ project=config.params["project"],
+ platform=job["primary-dependency"].attributes["build_platform"],
+ )
+
+ yield job
+
+
+@transforms.add
+def copy_attributes(config, jobs):
+ for job in jobs:
+ dep_job = job["primary-dependency"]
+ job["attributes"] = copy_attributes_from_dependent_job(dep_job)
+ job["attributes"]["chunk_locales"] = dep_job.attributes.get("chunk_locales", ["en-US"])
+
+ yield job
+
+
+@transforms.add
+def filter_out_macos_jobs_but_mac_only_locales(config, jobs):
+ for job in jobs:
+ build_platform = job["primary-dependency"].attributes.get("build_platform")
+
+ if build_platform == "linux64-shippable":
+ yield job
+ elif (
+ build_platform == "macosx64-shippable"
+ and "ja-JP-mac" in job["attributes"]["chunk_locales"]
+ ):
+ # Other locales of the same job shouldn't be processed
+ job["attributes"]["chunk_locales"] = ["ja-JP-mac"]
+ job["label"] = job["label"].replace(
+ # Guard against a chunk 10 or chunk 1 (latter on try) weird munging
+ "-{}/".format(job["attributes"]["l10n_chunk"]),
+ "-ja-JP-mac/",
+ )
+ yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+ for job in jobs:
+ dep_job = job["primary-dependency"]
+
+ treeherder = inherit_treeherder_from_dep(job, dep_job)
+ treeherder.setdefault(
+ "symbol", "langpack(P{})".format(job["attributes"].get("l10n_chunk", ""))
+ )
+
+ job["description"] = job["description"].format(
+ locales="/".join(job["attributes"]["chunk_locales"]),
+ )
+
+ job["dependencies"] = {dep_job.kind: dep_job.label}
+ job["treeherder"] = treeherder
+
+ yield job
+
+
+def generate_upstream_artifacts(upstream_task_ref, locales):
+ return [
+ {
+ "task": upstream_task_ref,
+ "extract": False,
+ "dest": f"{locale}",
+ "artifact": "public/build{locale}/target.langpack.xpi".format(
+ locale="" if locale == "en-US" else "/" + locale
+ ),
+ }
+ for locale in locales
+ ]
+
+
+@transforms.add
+def make_fetches(config, jobs):
+ for job in jobs:
+ upstream_task_ref = get_upstream_task_ref(job, expected_kinds=("build", "shippable-l10n"))
+
+ worker = job.setdefault("worker", {})
+ worker["taskcluster-proxy"] = True
+
+ env = worker.setdefault("env", {})
+
+ job_fetches = generate_upstream_artifacts(
+ upstream_task_ref, job["attributes"]["chunk_locales"]
+ )
+ env["MOZ_FETCHES"] = {
+ "task-reference": json.dumps(
+ sorted(job_fetches, key=lambda x: sorted(x.items())), sort_keys=True
+ )
+ }
+ env["MOZ_SCM_LEVEL"] = config.params["level"]
+
+ yield job
+
+
+def get_upstream_task_ref(job, expected_kinds):
+ upstream_tasks = [
+ job_kind for job_kind in job["dependencies"].keys() if job_kind in expected_kinds
+ ]
+
+ if len(upstream_tasks) > 1:
+ raise Exception("Only one dependency expected")
+
+ return f"<{upstream_tasks[0]}>"
+
+
+@contextmanager
+def environment(key, value):
+ """Set an environment variable in a context"""
+ old_value = None
+ if key in os.environ:
+ old_value = os.environ[key]
+ os.environ[key] = value
+ try:
+ yield True
+ finally:
+ if old_value is None:
+ del os.environ[key]
+ else:
+ os.environ[key] = old_value
+
+
+@transforms.add
+def set_env(config, jobs):
+ buildid = config.params["moz_build_date"]
+ app_version = config.params.get("app_version")
+
+ with environment("MOZ_BUILD_DATE", buildid):
+ langpack_version = get_version_maybe_buildid(app_version)
+
+ for job in jobs:
+ job["worker"].get("env", {}).update(
+ {
+ "LANGPACK_VERSION": langpack_version,
+ "LOCALES": json.dumps(job["attributes"]["chunk_locales"]),
+ "MOZ_FETCHES_DIR": "fetches",
+ "ATN_CHANNEL": job["worker"].get("channel"),
+ }
+ )
+
+ yield job
+
+
+@transforms.add
+def strip_unused_data(config, jobs):
+ for job in jobs:
+ del job["primary-dependency"]
+ del job["worker"]["channel"]
+
+ yield job
diff --git a/comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py b/comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py
new file mode 100644
index 0000000000..c4f39fc4eb
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/release_flatpak_push.py
@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the release-flatpak-push kind into an actual task description.
+"""
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by
+from voluptuous import Optional, Required
+
+from gecko_taskgraph.transforms.task import task_description_schema
+from gecko_taskgraph.util.attributes import release_level
+from gecko_taskgraph.util.scriptworker import add_scope_prefix
+
+push_flatpak_description_schema = Schema(
+ {
+ Required("name"): str,
+ Required("job-from"): task_description_schema["job-from"],
+ Required("dependencies"): task_description_schema["dependencies"],
+ Required("description"): task_description_schema["description"],
+ Required("treeherder"): task_description_schema["treeherder"],
+ Required("run-on-projects"): task_description_schema["run-on-projects"],
+ Required("worker-type"): optionally_keyed_by("release-level", str),
+ Required("worker"): object,
+ Optional("scopes"): [str],
+ Required("shipping-phase"): task_description_schema["shipping-phase"],
+ Required("shipping-product"): task_description_schema["shipping-product"],
+ Required("flathub-scope"): str,
+ Optional("extra"): task_description_schema["extra"],
+ Optional("attributes"): task_description_schema["attributes"],
+ }
+)
+
+transforms = TransformSequence()
+transforms.add_validate(push_flatpak_description_schema)
+
+
+@transforms.add
+def make_task_description(config, jobs):
+ for job in jobs:
+ if len(job["dependencies"]) != 1:
+ raise Exception("Exactly 1 dependency is required")
+
+ job["worker"]["upstream-artifacts"] = generate_upstream_artifacts(job["dependencies"])
+
+ resolve_keyed_by(
+ job,
+ "worker.channel",
+ item_name=job["name"],
+ **{"release-type": config.params["release_type"]},
+ )
+ resolve_keyed_by(
+ job,
+ "worker-type",
+ item_name=job["name"],
+ **{"release-level": release_level(config.params["project"])},
+ )
+ if release_level(config.params["project"]) == "production":
+ job.setdefault("scopes", []).append(
+ add_scope_prefix(
+ config,
+ "{}:{}".format(job["flathub-scope"], job["worker"]["channel"]),
+ )
+ )
+ del job["flathub-scope"]
+
+ yield job
+
+
+def generate_upstream_artifacts(dependencies):
+ return [
+ {
+ "taskId": {"task-reference": f"<{task_kind}>"},
+ "taskType": "build",
+ "paths": ["public/build/target.flatpak.tar.xz"],
+ }
+ for task_kind in dependencies.keys()
+ ]
diff --git a/comm/taskcluster/comm_taskgraph/transforms/release_started.py b/comm/taskcluster/comm_taskgraph/transforms/release_started.py
new file mode 100644
index 0000000000..ce0cfa9fab
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/release_started.py
@@ -0,0 +1,53 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Add notifications via taskcluster-notify for release tasks
+"""
+
+from pipes import quote as shell_quote
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import resolve_keyed_by
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def add_notifications(config, jobs):
+ for job in jobs:
+ label = "{}-{}".format(config.kind, job["name"])
+
+ resolve_keyed_by(job, "emails", label, project=config.params["project"])
+ emails = [email.format(config=config.__dict__) for email in job.pop("emails")]
+
+ command = [
+ "release",
+ "send-buglist-email",
+ "--version",
+ config.params["version"],
+ "--product",
+ job["shipping-product"],
+ "--revision",
+ config.params["comm_head_rev"],
+ "--build-number",
+ str(config.params["build_number"]),
+ "--repo",
+ config.params["comm_head_repository"],
+ ]
+ for address in emails:
+ command += ["--address", address]
+ command += [
+ # We wrap this in `{'task-reference': ...}` below
+ "--task-group-id",
+ "<decision>",
+ ]
+
+ job["scopes"] = ["notify:email:{}".format(address) for address in emails]
+ job["run"] = {
+ "using": "mach",
+ "sparse-profile": "mach",
+ "mach": {"task-reference": " ".join(map(shell_quote, command))},
+ }
+
+ yield job
diff --git a/comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py b/comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py
new file mode 100644
index 0000000000..793519f6ff
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/repackage_msix.py
@@ -0,0 +1,52 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+
+from taskgraph.transforms.base import TransformSequence
+
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def add_langpack_fetches(config, jobs):
+ """Adds the fetch configuration for the langpacks. This is done here
+ because Thunderbird langpacks are not signed and therefore not found as
+ artifacts of "shippable-l10n-signing" like they are for Firefox. Need to
+ use "shippable-l10n".
+ """
+
+ def depends_filter(dep_task):
+ return (
+ dep_task.kind == "shippable-l10n"
+ and dep_task.attributes["build_platform"] == "linux64-shippable"
+ and dep_task.attributes["build_type"] == "opt"
+ )
+
+ for job in jobs:
+ dependencies = job.get("dependencies", {})
+ fetches = job.setdefault("fetches", {})
+
+ # The keys are unique, like `shippable-l10n-linux64-shippable-1/opt`, so we
+ # can't ask for the tasks directly, we must filter for them.
+ for t in filter(depends_filter, config.kind_dependencies_tasks.values()):
+ dependencies.update({t.label: t.label})
+
+ fetches.update(
+ {
+ t.label: [
+ {
+ "artifact": f"{loc}/target.langpack.xpi",
+ "extract": False,
+ # Otherwise we can't disambiguate locales!
+ "dest": f"distribution/extensions/{loc}",
+ }
+ for loc in t.attributes["chunk_locales"]
+ ]
+ }
+ )
+
+ yield job
diff --git a/comm/taskcluster/comm_taskgraph/transforms/signing.py b/comm/taskcluster/comm_taskgraph/transforms/signing.py
new file mode 100644
index 0000000000..297fec0d2e
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/signing.py
@@ -0,0 +1,88 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from taskgraph.transforms.base import TransformSequence
+
+from gecko_taskgraph.util.signed_artifacts import is_notarization_kind
+
+transforms = TransformSequence()
+
+
+def check_notarization(dependencies):
+ """
+ Determine whether a signing job is the last step of a notarization
+ by looking at its dependencies.
+ """
+ for dep in dependencies:
+ if is_notarization_kind(dep):
+ return True
+
+
+@transforms.add
+def remove_widevine(config, jobs):
+ """
+ Remove references to widevine signing.
+
+ This is to avoid adding special cases for handling signed artifacts
+ in mozilla-central code. Artifact signature formats are determined in
+ gecko_taskgraph.util.signed_artifacts. There's no override mechanism so we
+ remove the autograph_widevine format here.
+ """
+ for job in jobs:
+ task = job["task"]
+ payload = task["payload"]
+
+ widevine_scope = "project:comm:thunderbird:releng:signing:format:autograph_widevine"
+ if widevine_scope in task["scopes"]:
+ task["scopes"].remove(widevine_scope)
+ if "upstreamArtifacts" in payload:
+ for artifact in payload["upstreamArtifacts"]:
+ if "autograph_widevine" in artifact.get("formats", []):
+ artifact["formats"].remove("autograph_widevine")
+
+ yield job
+
+
+@transforms.add
+def no_sign_langpacks(config, jobs):
+ """
+ Remove langpacks from signing jobs after they are automatically added.
+ """
+ for job in jobs:
+ task = job["task"]
+ payload = task["payload"]
+
+ if "upstreamArtifacts" in payload:
+ for artifact in payload["upstreamArtifacts"]:
+ if "autograph_langpack" in artifact.get("formats", []):
+ artifact["formats"].remove("autograph_langpack")
+
+ # Make sure that there are no .xpi files in the artifact list
+ if all([p.endswith("target.langpack.xpi") for p in artifact["paths"]]):
+ payload["upstreamArtifacts"].remove(artifact)
+
+ yield job
+
+
+@transforms.add
+def check_for_no_formats(config, jobs):
+ """
+ Check for signed artifacts without signature formats and remove them to
+ avoid scriptworker errors.
+ Signing jobs that use macOS notarization do not have formats, so keep
+ those.
+ """
+ for job in jobs:
+ if not check_notarization(job["dependencies"]):
+ task = job["task"]
+ payload = task["payload"]
+
+ if "upstreamArtifacts" in payload:
+ for artifact in payload["upstreamArtifacts"]:
+ if "formats" in artifact and not artifact["formats"]:
+ for remove_path in artifact["paths"]:
+ job["release-artifacts"].remove(remove_path)
+
+ payload["upstreamArtifacts"].remove(artifact)
+ yield job
diff --git a/comm/taskcluster/comm_taskgraph/transforms/source_test.py b/comm/taskcluster/comm_taskgraph/transforms/source_test.py
new file mode 100644
index 0000000000..d4ae39c134
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/source_test.py
@@ -0,0 +1,63 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+import shlex
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.path import join as join_path
+from taskgraph.util.path import match as match_path
+
+from gecko_taskgraph.files_changed import get_changed_files
+
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+def get_patterns(job):
+ """Get the "run on-changed" file patterns."""
+ optimization = job.get("optimization", {})
+ if optimization:
+ return optimization.copy().popitem()[1]
+ return []
+
+
+def shlex_join(split_command):
+ """shlex.join from Python 3.8+"""
+ return " ".join(shlex.quote(arg) for arg in split_command)
+
+
+@transforms.add
+def changed_clang_format(config, jobs):
+ """
+ Transform for clang-format job to set the commandline to only check
+ C++ files that were changed in the current push rather than running on
+ the entire repository.
+ """
+ for job in jobs:
+ if job.get("name", "") == "clang-format":
+ repository = config.params.get("comm_head_repository")
+ revision = config.params.get("comm_head_rev")
+
+ match_patterns = get_patterns(job)
+ changed_files = {
+ join_path("comm", file) for file in get_changed_files(repository, revision)
+ }
+
+ cpp_files = []
+ for pattern in match_patterns:
+ for path in changed_files:
+ if match_path(path, pattern):
+ cpp_files.append(path)
+
+ # In the event that no C/C++ files were changed in the current push,
+ # the commandline will end up being invalid. But, the clang-format
+ # job will get dropped by optimization, so it doesn't really matter.
+ if cpp_files:
+ job["run"]["command"] = job["run"]["command"].format(
+ changed_files=shlex_join(cpp_files)
+ )
+
+ yield job
diff --git a/comm/taskcluster/comm_taskgraph/transforms/tb_build.py b/comm/taskcluster/comm_taskgraph/transforms/tb_build.py
new file mode 100644
index 0000000000..eff16d8e69
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/tb_build.py
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+
+from taskgraph.transforms.base import TransformSequence
+
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def munge_environment(config, jobs):
+ for job in jobs:
+ env = job["worker"]["env"]
+ # Remove MOZ_SOURCE_CHANGESET/REPO from the job environment and discard
+ # if present. Having these variables set in the environment causes problems
+ # with generating debug sym files. Bug 1747879.
+ env.pop("MOZ_SOURCE_CHANGESET", None)
+ env.pop("MOZ_SOURCE_REPO", None)
+
+ yield job
diff --git a/comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py b/comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py
new file mode 100644
index 0000000000..ecdfd577af
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/tb_cross_channel.py
@@ -0,0 +1,46 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Build a command to run `mach tb-l10n-x-channel`.
+"""
+
+from shlex import quote as shell_quote
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import resolve_keyed_by
+
+from rocbuild.notify import TB_BUILD_ADDR
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def resolve_keys(config, jobs):
+ for job in jobs:
+ for item in ["ssh-key-secret", "run.actions"]:
+ resolve_keyed_by(job, item, item, **{"level": str(config.params["level"])})
+ yield job
+
+
+@transforms.add
+def build_command(config, jobs):
+ for job in jobs:
+ command = [
+ "tb-l10n-x-channel",
+ "-o",
+ "/builds/worker/artifacts/outgoing.diff",
+ "--attempts",
+ "5",
+ ]
+ ssh_key_secret = job.pop("ssh-key-secret")
+ if ssh_key_secret:
+ command.extend(["--ssh-secret", ssh_key_secret])
+ job.setdefault("scopes", []).append(f"secrets:get:{ssh_key_secret}")
+ job["scopes"].append(f"notify:email:{TB_BUILD_ADDR}")
+
+ command.extend(job["run"].pop("actions", []))
+ job.setdefault("run", {}).update(
+ {"using": "mach", "mach": " ".join(map(shell_quote, command))}
+ )
+ yield job
diff --git a/comm/taskcluster/comm_taskgraph/transforms/tests.py b/comm/taskcluster/comm_taskgraph/transforms/tests.py
new file mode 100644
index 0000000000..9c1df4663e
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/tests.py
@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Thunderbird modifications to test jobs
+"""
+
+import logging
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import resolve_keyed_by
+
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def optimization_keyed_by(config, tasks):
+ """Used to set the optimization strategy"""
+ for task in tasks:
+ resolve_keyed_by(task, "optimization", item_name=task["test-name"])
+ yield task
diff --git a/comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py b/comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py
new file mode 100644
index 0000000000..2f9ee143fa
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/transforms/update_verify_config.py
@@ -0,0 +1,134 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from urllib.parse import urlsplit
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import resolve_keyed_by
+
+from gecko_taskgraph.transforms.task import get_branch_repo, get_branch_rev
+from gecko_taskgraph.transforms.update_verify_config import ensure_wrapped_singlequote
+from gecko_taskgraph.util.attributes import release_level
+from gecko_taskgraph.util.scriptworker import get_release_config
+
+transforms = TransformSequence()
+
+
+# The beta regexes do not match point releases.
+# In the rare event that we do ship a point
+# release to beta, we need to either:
+# 1) update these regexes to match that specific version
+# 2) pass a second include version that matches that specific version
+INCLUDE_VERSION_REGEXES = {
+ "beta": r"'^(\d+\.\d+b\d+)$'",
+ "nonbeta": r"'^\d+\.\d+(\.\d+)?$'",
+ # Previous major versions, for update testing before we update users to a new esr
+ "release-next": r"'^(91|102)\.\d+(\.\d+)?$'",
+}
+
+MAR_CHANNEL_ID_OVERRIDE_REGEXES = {
+ "beta": r"'^\d+\.\d+(\.\d+)?$$,thunderbird-comm-beta,thunderbird-comm-release'",
+}
+
+
+ensure_wrapped_singlequote(INCLUDE_VERSION_REGEXES)
+ensure_wrapped_singlequote(MAR_CHANNEL_ID_OVERRIDE_REGEXES)
+
+
+@transforms.add
+def add_command(config, tasks):
+ keyed_by_args = [
+ "channel",
+ "archive-prefix",
+ "previous-archive-prefix",
+ "aus-server",
+ "override-certs",
+ "include-version",
+ "mar-channel-id-override",
+ "last-watershed",
+ ]
+ optional_args = [
+ "updater-platform",
+ ]
+
+ release_config = get_release_config(config)
+
+ for task in tasks:
+ task["description"] = "generate update verify config for {}".format(
+ task["attributes"]["build_platform"]
+ )
+
+ command = [
+ "python",
+ "testing/mozharness/scripts/release/update-verify-config-creator.py",
+ "--product",
+ task["extra"]["product"],
+ "--stage-product",
+ task["shipping-product"],
+ "--app-name",
+ task["extra"]["app-name"],
+ "--branch-prefix",
+ task["extra"]["branch-prefix"],
+ "--platform",
+ task["extra"]["platform"],
+ "--to-version",
+ release_config["version"],
+ "--to-app-version",
+ release_config["appVersion"],
+ "--to-build-number",
+ str(release_config["build_number"]),
+ "--to-buildid",
+ config.params["moz_build_date"],
+ "--to-revision",
+ get_branch_rev(config),
+ "--output-file",
+ "update-verify.cfg",
+ ]
+
+ repo_path = urlsplit(get_branch_repo(config)).path.lstrip("/")
+ command.extend(["--repo-path", repo_path])
+
+ if release_config.get("partial_versions"):
+ for partial in release_config["partial_versions"].split(","):
+ command.extend(["--partial-version", partial.split("build")[0]])
+
+ for arg in optional_args:
+ if task["extra"].get(arg):
+ command.append(f"--{arg}")
+ command.append(task["extra"][arg])
+
+ for arg in keyed_by_args:
+ thing = f"extra.{arg}"
+ resolve_keyed_by(
+ task,
+ thing,
+ item_name=task["name"],
+ platform=task["attributes"]["build_platform"],
+ **{
+ "release-type": config.params["release_type"],
+ "release-level": release_level(config.params["project"]),
+ },
+ )
+ # ignore things that resolved to null
+ if not task["extra"].get(arg):
+ continue
+ if arg == "include-version":
+ task["extra"][arg] = INCLUDE_VERSION_REGEXES[task["extra"][arg]]
+ if arg == "mar-channel-id-override":
+ task["extra"][arg] = MAR_CHANNEL_ID_OVERRIDE_REGEXES[task["extra"][arg]]
+
+ command.append(f"--{arg}")
+ command.append(task["extra"][arg])
+
+ task["run"].update(
+ {
+ "using": "mach",
+ "mach": " ".join(command),
+ }
+ )
+
+ yield task
diff --git a/comm/taskcluster/comm_taskgraph/try_option_syntax.py b/comm/taskcluster/comm_taskgraph/try_option_syntax.py
new file mode 100644
index 0000000000..b401bfa7a2
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/try_option_syntax.py
@@ -0,0 +1,97 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+
+from gecko_taskgraph.target_tasks import (
+ filter_by_uncommon_try_tasks,
+ filter_unsupported_artifact_builds,
+)
+from gecko_taskgraph.try_option_syntax import TryOptionSyntax
+
+logger = logging.getLogger(__name__)
+
+
+class TryCCOptionSyntax(TryOptionSyntax):
+ """
+ Override parse_platforms in the superclass. Removed the attempt to replace
+ macosx64 jobs with macosx64-shippable.
+ """
+
+ def parse_platforms(self, options, full_task_graph):
+ platform_arg = options["platforms"]
+ if platform_arg == "all":
+ return None
+
+ results = []
+ for build in platform_arg.split(","):
+ results.append(build)
+
+ test_platforms = {
+ t.attributes["test_platform"]
+ for t in full_task_graph.tasks.values()
+ if "test_platform" in t.attributes
+ }
+ build_platforms = {
+ t.attributes["build_platform"]
+ for t in full_task_graph.tasks.values()
+ if "build_platform" in t.attributes
+ }
+ all_platforms = test_platforms | build_platforms
+ bad_platforms = set(results) - all_platforms
+ if bad_platforms:
+ raise Exception("Unknown platform(s) [%s] specified for try" % ",".join(bad_platforms))
+
+ return results
+
+
+def _try_cc_option_syntax(full_task_graph, parameters, graph_config):
+ """Generate a list of target tasks based on try syntax in
+ parameters['message'] and, for context, the full task graph.
+
+ Based on gecko_taskgraph.target_tasks._try_option_syntax. Removed talos
+ and raptor references and use TryCCOptionSyntax.
+ """
+ options = TryCCOptionSyntax(parameters, full_task_graph, graph_config)
+ target_tasks_labels = [
+ t.label
+ for t in full_task_graph.tasks.values()
+ if options.task_matches(t)
+ and filter_by_uncommon_try_tasks(t.label)
+ and filter_unsupported_artifact_builds(t, parameters)
+ ]
+
+ attributes = {
+ k: getattr(options, k)
+ for k in [
+ "no_retry",
+ "tag",
+ ]
+ }
+
+ for l in target_tasks_labels:
+ task = full_task_graph[l]
+ if "unittest_suite" in task.attributes:
+ task.attributes["task_duplicates"] = options.trigger_tests
+
+ for l in target_tasks_labels:
+ task = full_task_graph[l]
+ # If the developer wants test jobs to be rebuilt N times we add that value here
+ if options.trigger_tests > 1 and "unittest_suite" in task.attributes:
+ task.attributes["task_duplicates"] = options.trigger_tests
+
+ task.attributes.update(attributes)
+
+ # Add notifications here as well
+ if options.notifications:
+ for task in full_task_graph:
+ owner = parameters.get("owner")
+ routes = task.task.setdefault("routes", [])
+ if options.notifications == "all":
+ routes.append(f"notify.email.{owner}.on-any")
+ elif options.notifications == "failure":
+ routes.append(f"notify.email.{owner}.on-failed")
+ routes.append(f"notify.email.{owner}.on-exception")
+
+ return target_tasks_labels
diff --git a/comm/taskcluster/comm_taskgraph/util/__init__.py b/comm/taskcluster/comm_taskgraph/util/__init__.py
new file mode 100644
index 0000000000..71f5dc9cd0
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/util/__init__.py
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+def strip_comm_prefix(relpath):
+ """
+ Returns relpath with 'comm/' prefix removed.
+ :param string relpath: relative path
+ :return string: stripped path
+ """
+ if relpath[:5] == "comm/":
+ return relpath[5:]
+ else:
+ return relpath
diff --git a/comm/taskcluster/comm_taskgraph/util/docker.py b/comm/taskcluster/comm_taskgraph/util/docker.py
new file mode 100644
index 0000000000..07971f3f28
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/util/docker.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+import os
+
+from gecko_taskgraph.util import docker as utildocker
+
+from .. import COMM
+
+logger = logging.getLogger(__name__)
+
+COMM_IMAGE_DIR = os.path.join(COMM, "taskcluster", "docker")
+
+
+def register():
+ logger.info("Registering comm docker image definition path.")
+ utildocker.image_paths.register(
+ "comm/taskcluster/ci/docker-image/docker-image.yml", COMM_IMAGE_DIR
+ )
+
+
+register()
diff --git a/comm/taskcluster/comm_taskgraph/util/hash.py b/comm/taskcluster/comm_taskgraph/util/hash.py
new file mode 100644
index 0000000000..d372a0192a
--- /dev/null
+++ b/comm/taskcluster/comm_taskgraph/util/hash.py
@@ -0,0 +1,76 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import hashlib
+
+import taskgraph.util.path as util_path
+
+from gecko_taskgraph.util.hash import get_file_finder, hash_path
+
+
+def split_patterns_list(patterns):
+ """
+ Give a list of path patterns and return two lists. rv[0] corresponds to files from the
+ GECKO repository, rv[1] corresponds to COMM.
+ The pattern list for the COMM repository will have *not* the 'comm/' prefix stripped.
+ """
+ return [
+ [p for p in patterns if not p.startswith("comm/")],
+ [p for p in patterns if p.startswith("comm/")],
+ ]
+
+
+def prefix_paths(prefix, paths):
+ """
+ Prepend a prefix to a list of paths.
+ """
+ return [util_path.join(prefix, p) for p in paths]
+
+
+def process_found(found_gen, prefix=None):
+ """
+ Transform the results from finder.find(pattern) into a list of files.
+ If prefix is given, prepend it to results.
+ """
+ for filename, fileobj in found_gen:
+ if prefix:
+ yield util_path.join(prefix, filename)
+ else:
+ yield filename
+
+
+def hash_paths_extended(base_path, patterns):
+ """
+ Works like gecko_taskgraph.util.hash.hash_paths, except it is able to account for Thunderbird
+ source code being part of a separate repository.
+ Two file finders are created if necessary.
+ """
+ gecko_patterns, comm_patterns = split_patterns_list(patterns)
+ gecko_finder = get_file_finder(base_path)
+ comm_finder = get_file_finder(util_path.join(base_path, "comm"))
+
+ h = hashlib.sha256()
+ files = []
+ for (patterns, finder, prefix) in [
+ (gecko_patterns, gecko_finder, None),
+ (comm_patterns, comm_finder, "comm/"),
+ ]:
+ for pattern in patterns:
+ if prefix:
+ pattern = pattern.lstrip(prefix)
+ found = list(process_found(finder.find(pattern), prefix))
+ if found:
+ files.extend(found)
+ else:
+ raise Exception("%s did not match anything" % pattern)
+ for path in sorted(files):
+ if path.endswith((".pyc", ".pyd", ".pyo")):
+ continue
+ h.update(
+ "{} {}\n".format(
+ hash_path(util_path.abspath(util_path.join(base_path, path))),
+ util_path.normsep(path),
+ ).encode("utf-8")
+ )
+ return h.hexdigest()