diff options
Diffstat (limited to 'taskcluster/gecko_taskgraph/transforms')
15 files changed, 71 insertions, 62 deletions
diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py index fb5b17d3b3..1434b4013a 100644 --- a/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py +++ b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py @@ -101,6 +101,7 @@ CONFIG_PER_BOUNCER_PRODUCT = { "file_names": { "win": "{pretty_product}%20Setup%20{version}.msix", "win64": "{pretty_product}%20Setup%20{version}.msix", + "win64-aarch64": "{pretty_product}%20Setup%20{version}.msix", }, }, "pkg": { diff --git a/taskcluster/gecko_taskgraph/transforms/build.py b/taskcluster/gecko_taskgraph/transforms/build.py index 4e73c5aef2..3ab22f2923 100644 --- a/taskcluster/gecko_taskgraph/transforms/build.py +++ b/taskcluster/gecko_taskgraph/transforms/build.py @@ -139,6 +139,10 @@ def use_artifact(config, jobs): and job.get("index", {}).get("job-name") in ARTIFACT_JOBS # If tests aren't packaged, then we are not able to rebuild all the packages and job["worker"]["env"].get("MOZ_AUTOMATION_PACKAGE_TESTS") == "1" + # Android shippable artifact builds are not supported + and not ( + "android" in job["name"] and job["attributes"].get("shippable", False) + ) ): job["treeherder"]["symbol"] = add_suffix(job["treeherder"]["symbol"], "a") job["worker"]["env"]["USE_ARTIFACT"] = "1" diff --git a/taskcluster/gecko_taskgraph/transforms/build_schedules.py b/taskcluster/gecko_taskgraph/transforms/build_schedules.py index ed6262b8b2..f9621adf77 100644 --- a/taskcluster/gecko_taskgraph/transforms/build_schedules.py +++ b/taskcluster/gecko_taskgraph/transforms/build_schedules.py @@ -19,15 +19,7 @@ def set_build_schedules_optimization(config, tasks): continue schedules = [] - if config.kind == "build": - family = platform_family(task["attributes"]["build_platform"]) - schedules = [family] - - if "android" not in family: - # These are not GeckoView builds, so are associated with Firefox. - schedules.append("firefox") - - elif config.kind in ( + if config.kind in ( "build-components", "build-samples-browser", "test-components", @@ -44,5 +36,13 @@ def set_build_schedules_optimization(config, tasks): elif "focus" in task["name"] or "klar" in task["name"]: schedules.append("focus-android") + else: + family = platform_family(task["attributes"]["build_platform"]) + schedules = [family] + + if "android" not in family: + # These are not GeckoView builds, so are associated with Firefox. + schedules.append("firefox") + task["optimization"] = {"build": schedules} yield task diff --git a/taskcluster/gecko_taskgraph/transforms/diffoscope.py b/taskcluster/gecko_taskgraph/transforms/diffoscope.py index b74dc5bb8f..05c6617950 100644 --- a/taskcluster/gecko_taskgraph/transforms/diffoscope.py +++ b/taskcluster/gecko_taskgraph/transforms/diffoscope.py @@ -32,7 +32,7 @@ diff_description_schema = Schema( Required("original"): index_or_string, Required("new"): index_or_string, # Arguments to pass to diffoscope, used for job-defaults in - # taskcluster/ci/diffoscope/kind.yml + # taskcluster/kinds/diffoscope/kind.yml Optional("args"): str, # Extra arguments to pass to diffoscope, that can be set per job. Optional("extra-args"): str, diff --git a/taskcluster/gecko_taskgraph/transforms/github_sync.py b/taskcluster/gecko_taskgraph/transforms/github_sync.py deleted file mode 100644 index 6f48f794ce..0000000000 --- a/taskcluster/gecko_taskgraph/transforms/github_sync.py +++ /dev/null @@ -1,23 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from taskgraph.transforms.base import TransformSequence - -transforms = TransformSequence() - - -@transforms.add -def sync_github(config, tasks): - """Do transforms specific to github-sync tasks.""" - for task in tasks: - # Add the secret to the scopes, only in m-c. - # Doing this on any other tree will result in decision task failure - # because m-c is the only one allowed to have that scope. - secret = task["secret"] - if config.params["project"] == "mozilla-central": - task.setdefault("scopes", []) - task["scopes"].append("secrets:get:" + secret) - task["worker"].setdefault("env", {})["GITHUB_SECRET"] = secret - del task["secret"] - yield task diff --git a/taskcluster/gecko_taskgraph/transforms/perftest.py b/taskcluster/gecko_taskgraph/transforms/perftest.py index 47baafdad7..066db956f7 100644 --- a/taskcluster/gecko_taskgraph/transforms/perftest.py +++ b/taskcluster/gecko_taskgraph/transforms/perftest.py @@ -1,3 +1,4 @@ +# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ diff --git a/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py b/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py index ea2cac3a68..48465b3ab7 100644 --- a/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py +++ b/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Transform the reprocess-symbols task description template, -taskcluster/ci/reprocess-symbols/job-template.yml into an actual task description. +taskcluster/kinds/reprocess-symbols/job-template.yml into an actual task description. """ diff --git a/taskcluster/gecko_taskgraph/transforms/signing.py b/taskcluster/gecko_taskgraph/transforms/signing.py index 1bf91effd1..760f9f122f 100644 --- a/taskcluster/gecko_taskgraph/transforms/signing.py +++ b/taskcluster/gecko_taskgraph/transforms/signing.py @@ -61,6 +61,14 @@ signing_description_schema = Schema( ) +def get_locales_description(attributes, default): + """Returns the [list] of locales for task description usage""" + chunk_locales = attributes.get("chunk_locales") + if chunk_locales: + return ", ".join(chunk_locales) + return attributes.get("locale", default) + + @transforms.add def delete_name(config, jobs): """Delete the 'name' key if it exists, we don't use it.""" @@ -77,6 +85,7 @@ transforms.add_validate(signing_description_schema) def add_requirements_link(config, jobs): for job in jobs: dep_job = get_primary_dependency(config, job) + assert dep_job requirements_path = evaluate_keyed_by( config.graph_config["mac-signing"]["mac-requirements"], "mac requirements", @@ -95,6 +104,7 @@ def add_requirements_link(config, jobs): def make_task_description(config, jobs): for job in jobs: dep_job = get_primary_dependency(config, job) + assert dep_job attributes = dep_job.attributes signing_format_scopes = [] @@ -105,6 +115,7 @@ def make_task_description(config, jobs): is_shippable = dep_job.attributes.get("shippable", False) build_platform = dep_job.attributes.get("build_platform") + assert build_platform treeherder = None if "partner" not in config.kind and "eme-free" not in config.kind: treeherder = job.get("treeherder", {}) @@ -139,9 +150,9 @@ def make_task_description(config, jobs): label = job["label"] description = ( - "Initial Signing for locale '{locale}' for build '" + "Signing of locale(s) '{locale}' for build '" "{build_platform}/{build_type}'".format( - locale=attributes.get("locale", "en-US"), + locale=get_locales_description(attributes, "en-US"), build_platform=build_platform, build_type=attributes.get("build_type"), ) @@ -187,10 +198,16 @@ def make_task_description(config, jobs): # build-mac-{signing,notarization} uses signingscript instead of iscript if "macosx" in build_platform and config.kind.endswith("-mac-notarization"): - task["worker"]["mac-behavior"] = "apple_notarization" task["scopes"] = [ add_scope_prefix(config, "signing:cert:release-apple-notarization") ] + task[ + "description" + ] = "Notarization of '{}' locales for build '{}/{}'".format( + get_locales_description(attributes, "en-US"), + build_platform, + attributes.get("build_type"), + ) elif "macosx" in build_platform: # iscript overrides task["worker"]["mac-behavior"] = "mac_sign_and_pkg" diff --git a/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py b/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py index 0c31f48cc0..2b846604f1 100644 --- a/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py +++ b/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py @@ -1,3 +1,4 @@ +# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ diff --git a/taskcluster/gecko_taskgraph/transforms/source_test.py b/taskcluster/gecko_taskgraph/transforms/source_test.py index e3eeb7c819..266637a7a0 100644 --- a/taskcluster/gecko_taskgraph/transforms/source_test.py +++ b/taskcluster/gecko_taskgraph/transforms/source_test.py @@ -1,3 +1,4 @@ +# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ @@ -264,8 +265,8 @@ def remove_optimization_on_central(config, jobs): if not job.get("attributes", {}).get("code-review", False): yield job continue - if "when" not in job: - yield job - continue - del job["when"] + if "when" in job: + del job["when"] + if "optimization" in job and "skip-unless-mozlint" in job["optimization"]: + del job["optimization"] yield job diff --git a/taskcluster/gecko_taskgraph/transforms/task.py b/taskcluster/gecko_taskgraph/transforms/task.py index 4bfe0e9f6d..aa77c46d65 100644 --- a/taskcluster/gecko_taskgraph/transforms/task.py +++ b/taskcluster/gecko_taskgraph/transforms/task.py @@ -208,7 +208,7 @@ TC_TREEHERDER_SCHEMA_URL = ( UNKNOWN_GROUP_NAME = ( - "Treeherder group {} (from {}) has no name; " "add it to taskcluster/ci/config.yml" + "Treeherder group {} (from {}) has no name; " "add it to taskcluster/config.yml" ) V2_ROUTE_TEMPLATES = [ @@ -313,7 +313,7 @@ def index_builder(name): UNSUPPORTED_INDEX_PRODUCT_ERROR = """\ The gecko-v2 product {product} is not in the list of configured products in -`taskcluster/ci/config.yml'. +`taskcluster/config.yml'. """ @@ -842,6 +842,7 @@ def build_generic_worker_payload(config, task, task_def): # behavior for mac iscript Optional("mac-behavior"): Any( "apple_notarization", + "apple_notarization_stacked", "mac_sign_and_pkg", "mac_sign_and_pkg_hardened", "mac_geckodriver", @@ -1612,7 +1613,7 @@ def task_name_from_label(config, tasks): UNSUPPORTED_SHIPPING_PRODUCT_ERROR = """\ The shipping product {product} is not in the list of configured products in -`taskcluster/ci/config.yml'. +`taskcluster/config.yml'. """ diff --git a/taskcluster/gecko_taskgraph/transforms/test/other.py b/taskcluster/gecko_taskgraph/transforms/test/other.py index 5d54467001..e01691c05a 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/other.py +++ b/taskcluster/gecko_taskgraph/transforms/test/other.py @@ -419,26 +419,20 @@ def setup_browsertime(config, tasks): "win32-chromedriver-122", "win32-chromedriver-123", ], - "windows.*-32.*": [ - "win32-chromedriver-122", - "win32-chromedriver-123", - "win32-chromedriver-124", - ], "windows.*-64.*": [ - "win32-chromedriver-122", - "win32-chromedriver-123", + "win64-chromedriver-123", "win64-chromedriver-124", ], } chromium_fetches = { - "linux.*": ["linux64-chromium"], - "macosx1015.*": ["mac-chromium"], - "macosx1400.*": ["mac-chromium-arm"], - "windows.*aarch64.*": ["win32-chromium"], - "windows.*-32.*": ["win32-chromium"], - "windows.*-64.*": ["win64-chromium"], - "android.*": ["linux64-chromium"], + "linux.*": ["linux64-chromiumdriver"], + "macosx1015.*": ["mac-chromiumdriver"], + "macosx1400.*": ["mac-chromiumdriver-arm"], + "windows.*aarch64.*": ["win32-chromiumdriver"], + "windows.*-32.*": ["win32-chromiumdriver"], + "windows.*-64.*": ["win64-chromiumdriver"], + "android.*": ["linux64-chromiumdriver"], } cd_extracted_name = { @@ -581,7 +575,9 @@ def enable_code_coverage(config, tasks): yield task continue task["mozharness"].setdefault("extra-options", []).append("--code-coverage") - task["instance-size"] = "xlarge" + task["instance-size"] = "xlarge-noscratch" + if "jittest" in task["test-name"]: + task["instance-size"] = "xlarge" # Temporarily disable Mac tests on mozilla-central if "mac" in task["build-platform"]: diff --git a/taskcluster/gecko_taskgraph/transforms/test/variant.py b/taskcluster/gecko_taskgraph/transforms/test/variant.py index bda91b2f25..6ee6c429f0 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/variant.py +++ b/taskcluster/gecko_taskgraph/transforms/test/variant.py @@ -78,6 +78,14 @@ def split_variants(config, tasks): remaining_variants.append(name) return remaining_variants + def replace_task_items(task_key, variant_key): + for item in variant_key: + if isinstance(variant_key[item], dict): + task_key[item] = replace_task_items(task_key[item], variant_key[item]) + else: + task_key[item] = variant_key[item] + return task_key + def apply_variant(variant, task): task["description"] = variant["description"].format(**task) @@ -94,7 +102,9 @@ def split_variants(config, tasks): task["variant-suffix"] += suffix # Replace and/or merge the configuration. - task.update(variant.get("replace", {})) + + # we only want to update the leaf node, the the entire top level dict + task = replace_task_items(task, variant.get("replace", {})) return merge(task, variant.get("merge", {})) expired_variants = find_expired_variants(TEST_VARIANTS) diff --git a/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py b/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py index aea948f90e..3828e912ec 100644 --- a/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py +++ b/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Transform the upload-generated-files task description template, -taskcluster/ci/upload-generated-sources/kind.yml, into an actual task description. +taskcluster/kinds/upload-generated-sources/kind.yml, into an actual task description. """ from taskgraph.transforms.base import TransformSequence diff --git a/taskcluster/gecko_taskgraph/transforms/upload_symbols.py b/taskcluster/gecko_taskgraph/transforms/upload_symbols.py index 5c0bf18cb0..b216d2c8e1 100644 --- a/taskcluster/gecko_taskgraph/transforms/upload_symbols.py +++ b/taskcluster/gecko_taskgraph/transforms/upload_symbols.py @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Transform the upload-symbols task description template, -taskcluster/ci/upload-symbols/job-template.yml into an actual task description. +taskcluster/kinds/upload-symbols/job-template.yml into an actual task description. """ |