diff options
Diffstat (limited to 'taskcluster/gecko_taskgraph')
67 files changed, 1073 insertions, 964 deletions
diff --git a/taskcluster/gecko_taskgraph/__init__.py b/taskcluster/gecko_taskgraph/__init__.py index f1de1e9120..c169eae023 100644 --- a/taskcluster/gecko_taskgraph/__init__.py +++ b/taskcluster/gecko_taskgraph/__init__.py @@ -51,12 +51,23 @@ def register(graph_config): Args: graph_config: The graph configuration object. """ + import android_taskgraph from taskgraph import generator + # TODO: Remove along with + # `gecko_taskgraph.optimize.strategies.SkipUnlessChanged` + # (see comment over there) + from taskgraph.optimize.base import registry + + del registry["skip-unless-changed"] + from gecko_taskgraph import ( # noqa: trigger target task method registration morph, # noqa: trigger morph registration target_tasks, ) + + android_taskgraph.register(graph_config) + from gecko_taskgraph.parameters import register_parameters from gecko_taskgraph.util import dependencies # noqa: trigger group_by registration from gecko_taskgraph.util.verify import verifications diff --git a/taskcluster/gecko_taskgraph/config.py b/taskcluster/gecko_taskgraph/config.py index 5045963b46..df4a5665e3 100644 --- a/taskcluster/gecko_taskgraph/config.py +++ b/taskcluster/gecko_taskgraph/config.py @@ -4,6 +4,7 @@ from taskgraph.util.schema import Schema, optionally_keyed_by from voluptuous import Any, Optional, Required +from voluptuous.validators import Length graph_config_schema = Schema( { @@ -20,7 +21,7 @@ graph_config_schema = Schema( Required("product-dir"): str, Required("treeherder"): { # Mapping of treeherder group symbols to descriptive names - Required("group-names"): {str: str} + Required("group-names"): {str: Length(max=100)} }, Required("index"): {Required("products"): [str]}, Required("try"): { @@ -105,13 +106,8 @@ graph_config_schema = Schema( } }, }, - Required("mac-notarization"): { - Required("mac-entitlements"): optionally_keyed_by( - "platform", "release-level", str - ), - Required("mac-requirements"): optionally_keyed_by("platform", str), - }, Required("mac-signing"): { + Required("mac-requirements"): optionally_keyed_by("platform", str), Required("hardened-sign-config"): optionally_keyed_by( "hardened-signing-type", [ @@ -128,7 +124,7 @@ graph_config_schema = Schema( Required("globs"): [str], } ], - ) + ), }, Required("taskgraph"): { Optional( diff --git a/taskcluster/gecko_taskgraph/decision.py b/taskcluster/gecko_taskgraph/decision.py index e0bc9e3ca8..9fd8a5b5c1 100644 --- a/taskcluster/gecko_taskgraph/decision.py +++ b/taskcluster/gecko_taskgraph/decision.py @@ -32,6 +32,7 @@ from taskgraph.util.yaml import load_yaml from . import GECKO from .actions import render_actions_json +from .files_changed import get_changed_files from .parameters import get_app_version, get_version from .try_option_syntax import parse_message from .util.backstop import BACKSTOP_INDEX, is_backstop @@ -308,6 +309,9 @@ def get_decision_parameters(graph_config, options): parameters["hg_branch"] = get_hg_revision_branch( GECKO, revision=parameters["head_rev"] ) + parameters["files_changed"] = sorted( + get_changed_files(parameters["head_repository"], parameters["head_rev"]) + ) parameters["next_version"] = None parameters["optimize_strategies"] = None parameters["optimize_target_tasks"] = True @@ -448,14 +452,17 @@ def set_try_config(parameters, task_config_file): def set_decision_indexes(decision_task_id, params, graph_config): index_paths = [] if params["backstop"]: - index_paths.append(BACKSTOP_INDEX) + # When two Decision tasks run at nearly the same time, it's possible + # they both end up being backstops if the second checks the backstop + # index before the first inserts it. Insert this index first to reduce + # the chances of that happening. + index_paths.insert(0, BACKSTOP_INDEX) subs = params.copy() subs["trust-domain"] = graph_config["trust-domain"] - index_paths = [i.format(**subs) for i in index_paths] for index_path in index_paths: - insert_index(index_path, decision_task_id, use_proxy=True) + insert_index(index_path.format(**subs), decision_task_id, use_proxy=True) def write_artifact(filename, data): diff --git a/taskcluster/gecko_taskgraph/files_changed.py b/taskcluster/gecko_taskgraph/files_changed.py index c814df0806..0352fb9d7e 100644 --- a/taskcluster/gecko_taskgraph/files_changed.py +++ b/taskcluster/gecko_taskgraph/files_changed.py @@ -16,7 +16,7 @@ from mozpack.path import match as mozpackmatch from mozversioncontrol import InvalidRepoPath, get_repository_object from gecko_taskgraph import GECKO -from gecko_taskgraph.util.hg import get_json_automationrelevance +from gecko_taskgraph.util.hg import get_json_pushchangedfiles logger = logging.getLogger(__name__) @@ -27,9 +27,8 @@ def get_changed_files(repository, revision): Get the set of files changed in the push headed by the given revision. Responses are cached, so multiple calls with the same arguments are OK. """ - contents = get_json_automationrelevance(repository, revision) try: - changesets = contents["changesets"] + return get_json_pushchangedfiles(repository, revision)["files"] except KeyError: # We shouldn't hit this error in CI. if os.environ.get("MOZ_AUTOMATION"): @@ -39,17 +38,6 @@ def get_changed_files(repository, revision): # version control. return get_locally_changed_files(GECKO) - logger.debug("{} commits influencing task scheduling:".format(len(changesets))) - changed_files = set() - for c in changesets: - desc = "" # Support empty desc - if c["desc"]: - desc = c["desc"].splitlines()[0].encode("ascii", "ignore") - logger.debug(" {cset} {desc}".format(cset=c["node"][0:12], desc=desc)) - changed_files |= set(c["files"]) - - return changed_files - def check(params, file_patterns): """Determine whether any of the files changed in the indicated push to diff --git a/taskcluster/gecko_taskgraph/main.py b/taskcluster/gecko_taskgraph/main.py index e9a353f246..c6d7a4a3c8 100644 --- a/taskcluster/gecko_taskgraph/main.py +++ b/taskcluster/gecko_taskgraph/main.py @@ -21,6 +21,9 @@ from typing import Any, List import appdirs import yaml +from gecko_taskgraph import GECKO +from gecko_taskgraph.files_changed import get_locally_changed_files + Command = namedtuple("Command", ["func", "args", "kwargs", "defaults"]) commands = {} @@ -105,7 +108,7 @@ def get_filtered_taskgraph(taskgraph, tasksregex, exclude_keys): for key in exclude_keys: obj = task_dict attrs = key.split(".") - while attrs[0] in obj: + while obj and attrs[0] in obj: if len(attrs) == 1: del obj[attrs[0]] break @@ -130,7 +133,7 @@ def get_taskgraph_generator(root, parameters): return TaskGraphGenerator(root_dir=root, parameters=parameters) -def format_taskgraph(options, parameters, logfile=None): +def format_taskgraph(options, parameters, overrides, logfile=None): import taskgraph from taskgraph.parameters import parameters_loader @@ -148,7 +151,7 @@ def format_taskgraph(options, parameters, logfile=None): if isinstance(parameters, str): parameters = parameters_loader( parameters, - overrides={"target-kinds": options.get("target_kinds")}, + overrides=overrides, strict=False, ) @@ -182,7 +185,7 @@ def dump_output(out, path=None, params_spec=None): print(out + "\n", file=fh) -def generate_taskgraph(options, parameters, logdir): +def generate_taskgraph(options, parameters, overrides, logdir): from taskgraph.parameters import Parameters def logfile(spec): @@ -198,14 +201,16 @@ def generate_taskgraph(options, parameters, logdir): # tracebacks a little more readable and avoids additional process overhead. if len(parameters) == 1: spec = parameters[0] - out = format_taskgraph(options, spec, logfile(spec)) + out = format_taskgraph(options, spec, overrides, logfile(spec)) dump_output(out, options["output_file"]) return futures = {} with ProcessPoolExecutor(max_workers=options["max_workers"]) as executor: for spec in parameters: - f = executor.submit(format_taskgraph, options, spec, logfile(spec)) + f = executor.submit( + format_taskgraph, options, spec, overrides, logfile(spec) + ) futures[f] = spec for future in as_completed(futures): @@ -299,6 +304,15 @@ def generate_taskgraph(options, parameters, logdir): "specified).", ) @argument( + "--force-local-files-changed", + default=False, + action="store_true", + help="Compute the 'files-changed' parameter from local version control, " + "even when explicitly using a parameter set that already has it defined. " + "Note that this is already the default behaviour when no parameters are " + "specified.", +) +@argument( "--no-optimize", dest="optimize", action="store_false", @@ -372,10 +386,10 @@ def show_taskgraph(options): output_file = options["output_file"] if options["diff"]: - # --root argument is taskgraph's config at <repo>/taskcluster/ci + # --root argument is taskgraph's config at <repo>/taskcluster repo_root = os.getcwd() if options["root"]: - repo_root = f"{options['root']}/../.." + repo_root = f"{options['root']}/.." repo = get_repository(repo_root) if not repo.working_directory_clean(): @@ -400,15 +414,21 @@ def show_taskgraph(options): ) print(f"Generating {options['graph_attr']} @ {cur_ref}", file=sys.stderr) + overrides = { + "target-kinds": options.get("target_kinds"), + } parameters: List[Any[str, Parameters]] = options.pop("parameters") if not parameters: - overrides = { - "target-kinds": options.get("target_kinds"), - } parameters = [ parameters_loader(None, strict=False, overrides=overrides) ] # will use default values + # This is the default behaviour anyway, so no need to re-compute. + options["force_local_files_changed"] = False + + elif options["force_local_files_changed"]: + overrides["files-changed"] = sorted(get_locally_changed_files(GECKO)) + for param in parameters[:]: if isinstance(param, str) and os.path.isdir(param): parameters.remove(param) @@ -434,7 +454,7 @@ def show_taskgraph(options): # to setup its `mach` based logging. setup_logging() - generate_taskgraph(options, parameters, logdir) + generate_taskgraph(options, parameters, overrides, logdir) if options["diff"]: assert diffdir is not None @@ -464,7 +484,7 @@ def show_taskgraph(options): diffdir, f"{options['graph_attr']}_{base_ref}" ) print(f"Generating {options['graph_attr']} @ {base_ref}", file=sys.stderr) - generate_taskgraph(options, parameters, logdir) + generate_taskgraph(options, parameters, overrides, logdir) finally: repo.update(cur_ref) @@ -687,7 +707,7 @@ def decision(options): @argument( "--root", "-r", - default="taskcluster/ci", + default="taskcluster", help="root of the taskgraph definition relative to topsrcdir", ) def action_callback(options): @@ -723,7 +743,7 @@ def action_callback(options): @argument( "--root", "-r", - default="taskcluster/ci", + default="taskcluster", help="root of the taskgraph definition relative to topsrcdir", ) @argument( diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml b/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml index fb58439509..d847b4fcdc 100644 --- a/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml +++ b/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml @@ -62,6 +62,8 @@ platform_names: linux64-shippable: 'linux-x86_64' linux64-devedition: 'linux-x86_64' linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + linux64-aarch64-shippable: 'linux-aarch64' + linux64-aarch64-devedition: 'linux-aarch64' macosx64-shippable: 'mac' macosx64-devedition: 'mac' win32-shippable: 'win32' @@ -78,6 +80,8 @@ platform_names: linux64-shippable: 'linux64' linux64-devedition: 'linux64-devedition' linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + linux64-aarch64-shippable: 'linux64-aarch64' + linux64-aarch64-devedition: 'linux64-aarch64-devedition' macosx64-shippable: 'macosx64' macosx64-devedition: 'macosx64-devedition' win32-shippable: 'win32' @@ -94,6 +98,8 @@ platform_names: linux64-shippable: 'linux64' linux64-devedition: 'linux64' linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + linux64-aarch64-shippable: 'linux64-aarch64' + linux64-aarch64-devedition: 'linux64-aarch64' macosx64-shippable: 'macosx64' macosx64-devedition: 'macosx64' win32-shippable: 'win32' @@ -391,6 +397,8 @@ mapping: - win32-shippable - win64-devedition - win32-devedition + - win64-aarch64-shippable + - win64-aarch64-devedition locale_prefix: 'multi/' pretty_name: Firefox Setup ${version}.msix checksums_path: ${path_platform}/multi/Firefox Setup ${version}.msix diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml b/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml index 43ba4cbf15..7ca17662e2 100644 --- a/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml +++ b/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml @@ -51,6 +51,8 @@ platform_names: linux64-shippable: 'linux-x86_64' linux64-devedition: 'linux-x86_64' linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' + linux64-aarch64-shippable: 'linux-aarch64' + linux64-aarch64-devedition: 'linux-aarch64' macosx64-shippable: 'mac' macosx64-devedition: 'mac' win32-shippable: 'win32' diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml b/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml index d413ede3bd..60c39e7c53 100644 --- a/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml +++ b/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml @@ -57,6 +57,8 @@ platform_names: linux-devedition: 'linux-i686' linux64-shippable: 'linux-x86_64' linux64-devedition: 'linux-x86_64' + linux64-aarch64-shippable: 'linux-aarch64' + linux64-aarch64-devedition: 'linux-aarch64' linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' macosx64-shippable: 'mac' macosx64-devedition: 'mac' @@ -74,6 +76,8 @@ platform_names: linux64-asan-reporter-shippable: 'linux64-asan-reporter' linux64-shippable: 'linux64' linux64-devedition: 'linux64' + linux64-aarch64-shippable: 'linux64-aarch64' + linux64-aarch64-devedition: 'linux64-aarch64' macosx64-shippable: 'macosx64' macosx64-devedition: 'macosx64' win32-shippable: 'win32' @@ -127,19 +131,10 @@ mapping: checksums_path: KEY only_for_platforms: - linux64-shippable + - linux64-aarch64-shippable destinations: - ${year}/${month}/${upload_date}-${branch} - latest-${branch} - target.common.tests.tar.gz: - <<: *default - description: "Mixture of reftests, mochitests, UI and others, commonly bundled together in a test suite" - pretty_name: firefox-${version}.${locale}.${filename_platform}.common.tests.tar.gz - checksums_path: firefox-${version}.${locale}.${filename_platform}.common.tests.tar.gz - target.cppunittest.tests.tar.gz: - <<: *default - description: "C++ unittests related in-tree test infrastructure" - pretty_name: firefox-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz - checksums_path: firefox-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz target.crashreporter-symbols.zip: <<: *default description: "Crashreporter symbols to be consumed by Socorro" @@ -153,51 +148,16 @@ mapping: description: "Various compile and moz_app flags baked together in a json file" pretty_name: firefox-${version}.${locale}.${filename_platform}.json checksums_path: firefox-${version}.${locale}.${filename_platform}.json - target.mochitest.tests.tar.gz: - <<: *default - description: "Results for running the mochitest testing framework via Javascript function calls" - pretty_name: firefox-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz - checksums_path: firefox-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz target.mozinfo.json: <<: *default description: "Various compile and moz_app flags baked together in a json file" pretty_name: firefox-${version}.${locale}.${filename_platform}.mozinfo.json checksums_path: firefox-${version}.${locale}.${filename_platform}.mozinfo.json - target.reftest.tests.tar.gz: - <<: *default - description: "Results for running the reftest testing framework via display of two Web pages comparison" - pretty_name: firefox-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz - checksums_path: firefox-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz - target.talos.tests.tar.gz: - <<: *default - description: "Results for running the talos testing framework to measure performance" - pretty_name: firefox-${version}.${locale}.${filename_platform}.talos.tests.tar.gz - checksums_path: firefox-${version}.${locale}.${filename_platform}.talos.tests.tar.gz - target.awsy.tests.tar.gz: - <<: *default - description: "Results for running the awsy testing framework to track memory usage" - pretty_name: firefox-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz - checksums_path: firefox-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz - target.test_packages.json: - <<: *default - description: "File containing metadata about all other files and testing harnesses specifics" - pretty_name: firefox-${version}.${locale}.${filename_platform}.test_packages.json - checksums_path: firefox-${version}.${locale}.${filename_platform}.test_packages.json target.txt: <<: *default description: "File containing buildid and revision" pretty_name: firefox-${version}.${locale}.${filename_platform}.txt checksums_path: firefox-${version}.${locale}.${filename_platform}.txt - target.web-platform.tests.tar.gz: - <<: *default - description: "Results for running the webplatform testing framework to cover standard Web platform features" - pretty_name: firefox-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz - checksums_path: firefox-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz - target.xpcshell.tests.tar.gz: - <<: *default - description: "Results for running the xpcshell testing framework to enable XPConnect console application" - pretty_name: firefox-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz - checksums_path: firefox-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz target_info.txt: <<: *default description: "File containing the buildID" @@ -226,6 +186,7 @@ mapping: only_for_platforms: - linux-shippable - linux64-shippable + - linux64-aarch64-shippable - macosx64-shippable - win64-shippable - win32-shippable @@ -252,6 +213,8 @@ mapping: only_for_platforms: - linux-shippable - linux64-shippable + - linux64-aarch64-shippable + - linux64-aarch64-devedition - linux-devedition - linux64-devedition pretty_name: firefox-${version}.${locale}.langpack.deb @@ -301,6 +264,7 @@ mapping: only_for_platforms: - linux-shippable - linux64-shippable + - linux64-aarch64-shippable - linux64-asan-reporter-shippable pretty_name: firefox-${version}.${locale}.${filename_platform}.tar.bz2 checksums_path: firefox-${version}.${locale}.${filename_platform}.tar.bz2 @@ -322,6 +286,7 @@ mapping: only_for_platforms: - linux-shippable - linux64-shippable + - linux64-aarch64-shippable - linux64-asan-reporter-shippable pretty_name: firefox-${version}.${locale}.${filename_platform}.tar.bz2.asc checksums_path: firefox-${version}.${locale}.${filename_platform}.tar.bz2.asc @@ -465,6 +430,7 @@ mapping: - repackage-signing-shippable-l10n-msix only_for_platforms: - win64-shippable + - win64-aarch64-shippable - win32-shippable pretty_name: firefox-${version}.multi.${filename_platform}.installer.msix checksums_path: firefox-${version}.multi.${filename_platform}.installer.msix @@ -498,6 +464,7 @@ mapping: only_for_platforms: - linux-shippable - linux64-shippable + - linux64-aarch64-shippable pretty_name: firefox-${version}.${locale}.${filename_platform}.deb checksums_path: firefox-${version}.${locale}.${filename_platform}.deb update_balrog_manifest: false diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml b/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml index f1b81572ab..caa31f556e 100644 --- a/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml +++ b/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml @@ -18,6 +18,8 @@ platform_names: linux-shippable: 'linux-i686' linux-devedition: 'linux-i686' linux64-shippable: 'linux-x86_64' + linux64-aarch64-shippable: 'linux-aarch64' + linux64-aarch64-devedition: 'linux-aarch64' linux64-devedition: 'linux-x86_64' linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter' macosx64-shippable: 'mac' diff --git a/taskcluster/gecko_taskgraph/optimize/mozlint.py b/taskcluster/gecko_taskgraph/optimize/mozlint.py new file mode 100644 index 0000000000..0fe42ea70d --- /dev/null +++ b/taskcluster/gecko_taskgraph/optimize/mozlint.py @@ -0,0 +1,96 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +from pathlib import Path +from typing import List, Union + +from mozlint.parser import Parser +from mozlint.pathutils import filterpaths +from taskgraph.optimize.base import OptimizationStrategy +from taskgraph.util.path import match as match_path + +from gecko_taskgraph import GECKO + +logger = logging.getLogger(__name__) + + +class TGMozlintParser(Parser): + """ + Mozlint Parser that skips validation. This is needed because decision + tasks use sparse clones and the files themselves are not present. + """ + + def _validate(self, linter): + pass + + +class SkipUnlessMozlint(OptimizationStrategy): + """ + Optimization strategy for mozlint tests. + + Uses the mozlint YAML file for each test to determine whether or not a test + should run. + + Using: + - The optimization relies on having `files_changed` set in the decision task + parameters. + - Register with register_strategy. The argument is the path to MozLint YAML + configuration files ("/tools/lint" for mozilla-central): + - In source-test/mozlint.yml, set the optimization strategy for each job by filename: + - For Mozlint jobs that run multiple linters at once use a list of filenames: + """ + + def __init__(self, linters_path: str): + self.mozlint_parser = TGMozlintParser(GECKO) + self.linters_path = Path(GECKO) / linters_path + + def should_remove_task( + self, task, params, mozlint_confs: Union[str, List[str]] + ) -> bool: + include = [] + exclude = [] + extensions = [] + exclude_extensions = [] + support_files = ["python/mozlint/**", "tools/lint/**"] + + files_changed = params["files_changed"] + + if isinstance(mozlint_confs, str): + mozlint_confs = [mozlint_confs] + + for mozlint_conf in mozlint_confs: + mozlint_yaml = str(self.linters_path / mozlint_conf) + logger.info(f"Loading file patterns for {task.label} from {mozlint_yaml}.") + linter_config = self.mozlint_parser(mozlint_yaml) + + for config in linter_config: + include += config.get("include", []) + exclude += config.get("exclude", []) + extensions += [e.strip(".") for e in config.get("extensions", [])] + exclude_extensions += [ + e.strip(".") for e in config.get("exclude_extensions", []) + ] + support_files += config.get("support-files", []) + + # Support files may not be part of "include" patterns, so check first + # Do not remove (return False) if any changed + for pattern in set(support_files): + for path in files_changed: + if match_path(path, pattern): + return False + + to_lint, to_exclude = filterpaths( + GECKO, + list(files_changed), + include=include, + exclude=exclude, + extensions=extensions, + exclude_extensions=exclude_extensions, + ) + + # to_lint should be an empty list if there is nothing to check + if not to_lint: + return True + return False diff --git a/taskcluster/gecko_taskgraph/optimize/schema.py b/taskcluster/gecko_taskgraph/optimize/schema.py index a7f878cf60..a348fb177c 100644 --- a/taskcluster/gecko_taskgraph/optimize/schema.py +++ b/taskcluster/gecko_taskgraph/optimize/schema.py @@ -39,6 +39,8 @@ default_optimizations = ( {"upload-symbols": None}, # optimize strategy alias for reprocess-symbols tasks {"reprocess-symbols": None}, + # optimization strategy for mozlint tests + {"skip-unless-mozlint": voluptuous.Any(str, [str])}, ) OptimizationSchema = voluptuous.Any(*default_optimizations) diff --git a/taskcluster/gecko_taskgraph/optimize/strategies.py b/taskcluster/gecko_taskgraph/optimize/strategies.py index 4d0d23a5ac..f1ade38672 100644 --- a/taskcluster/gecko_taskgraph/optimize/strategies.py +++ b/taskcluster/gecko_taskgraph/optimize/strategies.py @@ -9,8 +9,9 @@ import mozpack.path as mozpath from mozbuild.base import MozbuildObject from mozbuild.util import memoize from taskgraph.optimize.base import OptimizationStrategy, register_strategy +from taskgraph.util.path import match as match_path -from gecko_taskgraph import files_changed +from gecko_taskgraph.optimize.mozlint import SkipUnlessMozlint logger = logging.getLogger(__name__) @@ -18,9 +19,7 @@ logger = logging.getLogger(__name__) @register_strategy("skip-unless-schedules") class SkipUnlessSchedules(OptimizationStrategy): @memoize - def scheduled_by_push(self, repository, revision): - changed_files = files_changed.get_changed_files(repository, revision) - + def scheduled_by_push(self, files_changed): mbo = MozbuildObject.from_environment() # the decision task has a sparse checkout, so, mozbuild_reader will use # a MercurialRevisionFinder with revision '.', which should be the same @@ -28,7 +27,7 @@ class SkipUnlessSchedules(OptimizationStrategy): rdr = mbo.mozbuild_reader(config_mode="empty") components = set() - for p, m in rdr.files_info(changed_files).items(): + for p, m in rdr.files_info(files_changed).items(): components |= set(m["SCHEDULES"].components) return components @@ -37,9 +36,7 @@ class SkipUnlessSchedules(OptimizationStrategy): if params.get("pushlog_id") == -1: return False - scheduled = self.scheduled_by_push( - params["head_repository"], params["head_rev"] - ) + scheduled = self.scheduled_by_push(frozenset(params["files_changed"])) conditions = set(conditions) # if *any* of the condition components are scheduled, do not optimize if conditions & scheduled: @@ -55,8 +52,8 @@ class SkipUnlessHasRelevantTests(OptimizationStrategy): """ @memoize - def get_changed_dirs(self, repo, rev): - changed = map(mozpath.dirname, files_changed.get_changed_files(repo, rev)) + def get_changed_dirs(self, files_changed): + changed = map(mozpath.dirname, files_changed) # Filter out empty directories (from files modified in the root). # Otherwise all tasks would be scheduled. return {d for d in changed if d} @@ -65,7 +62,7 @@ class SkipUnlessHasRelevantTests(OptimizationStrategy): if not task.attributes.get("test_manifests"): return True - for d in self.get_changed_dirs(params["head_repository"], params["head_rev"]): + for d in self.get_changed_dirs(frozenset(params["files_changed"])): for t in task.attributes["test_manifests"]: if t.startswith(d): logger.debug( @@ -75,3 +72,36 @@ class SkipUnlessHasRelevantTests(OptimizationStrategy): ) return False return True + + +# TODO: This overwrites upstream Taskgraph's `skip-unless-changed` +# optimization. Once the firefox-android migration is landed and we upgrade +# upstream Taskgraph to a version that doesn't call files_changed.check`, this +# class can be deleted. Also remove the `taskgraph.optimize.base.registry` tweak +# in `gecko_taskgraph.register` at the same time. +@register_strategy("skip-unless-changed") +class SkipUnlessChanged(OptimizationStrategy): + def check(self, files_changed, patterns): + for pattern in patterns: + for path in files_changed: + if match_path(path, pattern): + return True + return False + + def should_remove_task(self, task, params, file_patterns): + # pushlog_id == -1 - this is the case when run from a cron.yml job or on a git repository + if params.get("repository_type") == "hg" and params.get("pushlog_id") == -1: + return False + + changed = self.check(params["files_changed"], file_patterns) + if not changed: + logger.debug( + 'no files found matching a pattern in `skip-unless-changed` for "{}"'.format( + task.label + ) + ) + return True + return False + + +register_strategy("skip-unless-mozlint", args=("tools/lint",))(SkipUnlessMozlint) diff --git a/taskcluster/gecko_taskgraph/parameters.py b/taskcluster/gecko_taskgraph/parameters.py index 7e3de1372f..c9cca8acf1 100644 --- a/taskcluster/gecko_taskgraph/parameters.py +++ b/taskcluster/gecko_taskgraph/parameters.py @@ -9,6 +9,7 @@ from taskgraph.parameters import extend_parameters_schema from voluptuous import Any, Optional, Required from gecko_taskgraph import GECKO +from gecko_taskgraph.files_changed import get_locally_changed_files logger = logging.getLogger(__name__) @@ -18,6 +19,7 @@ gecko_parameters_schema = { Required("backstop"): bool, Required("build_number"): int, Required("enable_always_target"): Any(bool, [str]), + Required("files_changed"): [str], Required("hg_branch"): str, Required("message"): str, Required("next_version"): Any(None, str), @@ -107,6 +109,7 @@ def get_defaults(repo_root=None): "base_repository": "https://hg.mozilla.org/mozilla-unified", "build_number": 1, "enable_always_target": ["docker-image"], + "files_changed": sorted(get_locally_changed_files(repo_root)), "head_repository": "https://hg.mozilla.org/mozilla-central", "hg_branch": "default", "message": "", diff --git a/taskcluster/gecko_taskgraph/target_tasks.py b/taskcluster/gecko_taskgraph/target_tasks.py index fcbfab4e17..5469f9eb0d 100644 --- a/taskcluster/gecko_taskgraph/target_tasks.py +++ b/taskcluster/gecko_taskgraph/target_tasks.py @@ -11,7 +11,7 @@ from datetime import datetime, timedelta from redo import retry from taskgraph.parameters import Parameters -from taskgraph.target_tasks import _target_task, get_method +from taskgraph.target_tasks import get_method, register_target_task from taskgraph.util.taskcluster import find_task_id from gecko_taskgraph import GECKO, try_option_syntax @@ -37,13 +37,13 @@ UNCOMMON_TRY_TASK_LABELS = [ r"android-geckoview-docs", r"android-hw", # Windows tasks - r"windows10-64-ref-hw", + r"windows11-64-2009-hw-ref", r"windows10-aarch64-qr", # Linux tasks r"linux-", # hide all linux32 tasks by default - bug 1599197 r"linux1804-32", # hide linux32 tests - bug 1599197 # Test tasks - r"web-platform-tests.*backlog", # hide wpt jobs that are not implemented yet - bug 1572820 + r"web-platform-tests(?!-webgpu).*backlog", # hide wpt jobs that are not implemented yet - bug 1572820 r"-ccov", r"-profiling-", # talos/raptor profiling jobs are run too often r"-32-.*-webgpu", # webgpu gets little benefit from these tests. @@ -52,6 +52,9 @@ UNCOMMON_TRY_TASK_LABELS = [ # Hide shippable versions of tests we have opt versions of because the non-shippable # versions are faster to run. This is mostly perf tests. r"-shippable(?!.*(awsy|browsertime|marionette-headless|mochitest-devtools-chrome-fis|raptor|talos|web-platform-tests-wdspec-headless|mochitest-plain-headless))", # noqa - too long + r"nightly-simulation", + # Can't actually run on try + r"notarization", ] @@ -393,7 +396,7 @@ def _try_option_syntax(full_task_graph, parameters, graph_config): return target_tasks_labels -@_target_task("try_tasks") +@register_target_task("try_tasks") def target_tasks_try(full_task_graph, parameters, graph_config): try_mode = parameters["try_mode"] if try_mode == "try_task_config": @@ -405,13 +408,13 @@ def target_tasks_try(full_task_graph, parameters, graph_config): return [] -@_target_task("try_select_tasks") +@register_target_task("try_select_tasks") def target_tasks_try_select(full_task_graph, parameters, graph_config): tasks = target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config) return [l for l in tasks if filter_by_uncommon_try_tasks(l)] -@_target_task("try_select_tasks_uncommon") +@register_target_task("try_select_tasks_uncommon") def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config): from gecko_taskgraph.decision import PER_PROJECT_PARAMETERS @@ -437,7 +440,7 @@ def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config): return sorted(tasks) -@_target_task("try_auto") +@register_target_task("try_auto") def target_tasks_try_auto(full_task_graph, parameters, graph_config): """Target the tasks which have indicated they should be run on autoland (rather than try) via the `run_on_projects` attributes. @@ -468,7 +471,7 @@ def target_tasks_try_auto(full_task_graph, parameters, graph_config): ] -@_target_task("default") +@register_target_task("default") def target_tasks_default(full_task_graph, parameters, graph_config): """Target the tasks which have indicated they should be run on this project via the `run_on_projects` attributes.""" @@ -481,7 +484,7 @@ def target_tasks_default(full_task_graph, parameters, graph_config): ] -@_target_task("autoland_tasks") +@register_target_task("autoland_tasks") def target_tasks_autoland(full_task_graph, parameters, graph_config): """In addition to doing the filtering by project that the 'default' filter does, also remove any tests running against shippable builds @@ -507,7 +510,7 @@ def target_tasks_autoland(full_task_graph, parameters, graph_config): return [l for l in filtered_for_project if filter(full_task_graph[l])] -@_target_task("mozilla_central_tasks") +@register_target_task("mozilla_central_tasks") def target_tasks_mozilla_central(full_task_graph, parameters, graph_config): """In addition to doing the filtering by project that the 'default' filter does, also remove any tests running against regular (aka not shippable, @@ -547,7 +550,7 @@ def target_tasks_mozilla_central(full_task_graph, parameters, graph_config): return [l for l in filtered_for_project if filter(full_task_graph[l])] -@_target_task("graphics_tasks") +@register_target_task("graphics_tasks") def target_tasks_graphics(full_task_graph, parameters, graph_config): """In addition to doing the filtering by project that the 'default' filter does, also remove artifact builds because we have csets on @@ -565,7 +568,7 @@ def target_tasks_graphics(full_task_graph, parameters, graph_config): return [l for l in filtered_for_project if filter(full_task_graph[l])] -@_target_task("mozilla_beta_tasks") +@register_target_task("mozilla_beta_tasks") def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config): """Select the set of tasks required for a promotable beta or release build of desktop, plus android CI. The candidates build process involves a pipeline @@ -578,7 +581,7 @@ def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config): ] -@_target_task("mozilla_release_tasks") +@register_target_task("mozilla_release_tasks") def target_tasks_mozilla_release(full_task_graph, parameters, graph_config): """Select the set of tasks required for a promotable beta or release build of desktop, plus android CI. The candidates build process involves a pipeline @@ -591,7 +594,7 @@ def target_tasks_mozilla_release(full_task_graph, parameters, graph_config): ] -@_target_task("mozilla_esr115_tasks") +@register_target_task("mozilla_esr115_tasks") def target_tasks_mozilla_esr115(full_task_graph, parameters, graph_config): """Select the set of tasks required for a promotable beta or release build of desktop, without android CI. The candidates build process involves a pipeline @@ -615,19 +618,13 @@ def target_tasks_mozilla_esr115(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("promote_desktop") +@register_target_task("promote_desktop") def target_tasks_promote_desktop(full_task_graph, parameters, graph_config): """Select the superset of tasks required to promote a beta or release build of a desktop product. This should include all non-android mozilla_{beta,release} tasks, plus l10n, beetmover, balrog, etc.""" def filter(task): - # Bug 1758507 - geckoview ships in the promote phase - if not parameters["release_type"].startswith("esr") and is_geckoview( - task, parameters - ): - return True - if task.attributes.get("shipping_product") != parameters["release_product"]: return False @@ -645,15 +642,7 @@ def target_tasks_promote_desktop(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -def is_geckoview(task, parameters): - return ( - task.attributes.get("shipping_product") == "fennec" - and task.kind in ("beetmover-geckoview", "upload-symbols") - and parameters["release_product"] == "firefox" - ) - - -@_target_task("push_desktop") +@register_target_task("push_desktop") def target_tasks_push_desktop(full_task_graph, parameters, graph_config): """Select the set of tasks required to push a build of desktop to cdns. Previous build deps will be optimized out via action task.""" @@ -679,7 +668,7 @@ def target_tasks_push_desktop(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("ship_desktop") +@register_target_task("ship_desktop") def target_tasks_ship_desktop(full_task_graph, parameters, graph_config): """Select the set of tasks required to ship desktop. Previous build deps will be optimized out via action task.""" @@ -720,7 +709,7 @@ def target_tasks_ship_desktop(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("pine_tasks") +@register_target_task("pine_tasks") def target_tasks_pine(full_task_graph, parameters, graph_config): """Bug 1879960 - no reftests or wpt needed""" filtered_for_project = target_tasks_default( @@ -728,6 +717,8 @@ def target_tasks_pine(full_task_graph, parameters, graph_config): ) def filter(task): + if "android" in task.attributes.get("build_platform", ""): + return False suite = task.attributes.get("unittest_suite", "") if "reftest" in suite or "web-platform" in suite: return False @@ -736,7 +727,7 @@ def target_tasks_pine(full_task_graph, parameters, graph_config): return [l for l in filtered_for_project if filter(full_task_graph[l])] -@_target_task("larch_tasks") +@register_target_task("larch_tasks") def target_tasks_larch(full_task_graph, parameters, graph_config): """Bug 1879213 - only run necessary tasks on larch""" filtered_for_project = target_tasks_default( @@ -760,7 +751,7 @@ def target_tasks_larch(full_task_graph, parameters, graph_config): return [l for l in filtered_for_project if filter(full_task_graph[l])] -@_target_task("kaios_tasks") +@register_target_task("kaios_tasks") def target_tasks_kaios(full_task_graph, parameters, graph_config): """The set of tasks to run for kaios integration""" @@ -771,36 +762,7 @@ def target_tasks_kaios(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("ship_geckoview") -def target_tasks_ship_geckoview(full_task_graph, parameters, graph_config): - """Select the set of tasks required to ship geckoview nightly. The - nightly build process involves a pipeline of builds and an upload to - maven.mozilla.org.""" - index_path = ( - f"{graph_config['trust-domain']}.v2.{parameters['project']}.revision." - f"{parameters['head_rev']}.taskgraph.decision-ship-geckoview" - ) - if os.environ.get("MOZ_AUTOMATION") and retry( - index_exists, - args=(index_path,), - kwargs={ - "reason": "to avoid triggering multiple nightlies off the same revision", - }, - ): - return [] - - def filter(task): - # XXX Starting 69, we don't ship Fennec Nightly anymore. We just want geckoview to be - # uploaded - return task.attributes.get("shipping_product") == "fennec" and task.kind in ( - "beetmover-geckoview", - "upload-symbols", - ) - - return [l for l, t in full_task_graph.tasks.items() if filter(t)] - - -@_target_task("custom-car_perf_testing") +@register_target_task("custom-car_perf_testing") def target_tasks_custom_car_perf_testing(full_task_graph, parameters, graph_config): """Select tasks required for running daily performance tests for custom chromium-as-release.""" @@ -827,7 +789,7 @@ def target_tasks_custom_car_perf_testing(full_task_graph, parameters, graph_conf return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("general_perf_testing") +@register_target_task("general_perf_testing") def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config): """ Select tasks required for running performance tests 3 times a week. @@ -844,6 +806,9 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config) if "tp6-bench" in try_name: return False + if "tp7" in try_name: + return False + # Bug 1867669 - Temporarily disable all live site tests if "live" in try_name and "sheriffed" not in try_name: return False @@ -856,11 +821,7 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config) if "tp6" in try_name and "essential" not in try_name: return False return True - if "chromium" in try_name: - if "tp6" in try_name and "essential" not in try_name: - return False - return True - # chromium-as-release has it's own cron + # chromium-as-release has its own cron if "custom-car" in try_name: return False if "-live" in try_name: @@ -920,8 +881,13 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config) # Don't run android CaR sp tests as we already have a cron for this. if "m-car" in try_name: return False + if "fenix" in try_name: + return False if "speedometer" in try_name: return True + if "motionmark" in try_name and "1-3" in try_name: + if "chrome-m" in try_name: + return True return False return [l for l, t in full_task_graph.tasks.items() if filter(t)] @@ -946,7 +912,7 @@ def make_desktop_nightly_filter(platforms): return filter -@_target_task("sp-perftests") +@register_target_task("sp-perftests") def target_tasks_speedometer_tests(full_task_graph, parameters, graph_config): def filter(task): platform = task.attributes.get("test_platform") @@ -970,18 +936,18 @@ def target_tasks_speedometer_tests(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("nightly_linux") +@register_target_task("nightly_linux") def target_tasks_nightly_linux(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of linux. The nightly build process involves a pipeline of builds, signing, and, eventually, uploading the tasks to balrog.""" filter = make_desktop_nightly_filter( - {"linux64-shippable", "linux-shippable", "linux-aarch64-shippable"} + {"linux64-shippable", "linux-shippable", "linux64-aarch64-shippable"} ) return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@_target_task("nightly_macosx") +@register_target_task("nightly_macosx") def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of macosx. The nightly build process involves a pipeline of builds, signing, @@ -990,7 +956,7 @@ def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@_target_task("nightly_win32") +@register_target_task("nightly_win32") def target_tasks_nightly_win32(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of win32 and win64. The nightly build process involves a pipeline of builds, signing, @@ -999,7 +965,7 @@ def target_tasks_nightly_win32(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@_target_task("nightly_win64") +@register_target_task("nightly_win64") def target_tasks_nightly_win64(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of win32 and win64. The nightly build process involves a pipeline of builds, signing, @@ -1008,7 +974,7 @@ def target_tasks_nightly_win64(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@_target_task("nightly_win64_aarch64") +@register_target_task("nightly_win64_aarch64") def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of win32 and win64. The nightly build process involves a pipeline of builds, signing, @@ -1017,7 +983,7 @@ def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@_target_task("nightly_asan") +@register_target_task("nightly_asan") def target_tasks_nightly_asan(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of asan. The nightly build process involves a pipeline of builds, signing, @@ -1028,7 +994,7 @@ def target_tasks_nightly_asan(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@_target_task("daily_releases") +@register_target_task("daily_releases") def target_tasks_daily_releases(full_task_graph, parameters, graph_config): """Select the set of tasks required to identify if we should release. If we determine that we should the task will communicate to ship-it to @@ -1040,7 +1006,7 @@ def target_tasks_daily_releases(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("nightly_desktop") +@register_target_task("nightly_desktop") def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of linux, mac, windows.""" @@ -1078,8 +1044,30 @@ def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config): ) +@register_target_task("nightly_all") +def target_tasks_nightly_all(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a nightly build of firefox desktop and android""" + index_path = ( + f"{graph_config['trust-domain']}.v2.{parameters['project']}.revision." + f"{parameters['head_rev']}.taskgraph.decision-nightly-all" + ) + if os.environ.get("MOZ_AUTOMATION") and retry( + index_exists, + args=(index_path,), + kwargs={ + "reason": "to avoid triggering multiple nightlies off the same revision", + }, + ): + return [] + + return list( + set(target_tasks_nightly_desktop(full_task_graph, parameters, graph_config)) + | set(target_tasks_nightly_android(full_task_graph, parameters, graph_config)) + ) + + # Run Searchfox analysis once daily. -@_target_task("searchfox_index") +@register_target_task("searchfox_index") def target_tasks_searchfox(full_task_graph, parameters, graph_config): """Select tasks required for indexing Firefox for Searchfox web site each day""" return [ @@ -1095,7 +1083,7 @@ def target_tasks_searchfox(full_task_graph, parameters, graph_config): # Run build linux64-plain-clang-trunk/opt on mozilla-central/beta with perf tests -@_target_task("linux64_clang_trunk_perf") +@register_target_task("linux64_clang_trunk_perf") def target_tasks_build_linux64_clang_trunk_perf( full_task_graph, parameters, graph_config ): @@ -1111,19 +1099,19 @@ def target_tasks_build_linux64_clang_trunk_perf( # Run Updatebot's cron job 4 times daily. -@_target_task("updatebot_cron") +@register_target_task("updatebot_cron") def target_tasks_updatebot_cron(full_task_graph, parameters, graph_config): """Select tasks required to run Updatebot's cron job""" return ["updatebot-cron"] -@_target_task("customv8_update") +@register_target_task("customv8_update") def target_tasks_customv8_update(full_task_graph, parameters, graph_config): """Select tasks required for building latest d8/v8 version.""" return ["toolchain-linux64-custom-v8"] -@_target_task("file_update") +@register_target_task("file_update") def target_tasks_file_update(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform nightly in-tree file updates""" @@ -1134,7 +1122,7 @@ def target_tasks_file_update(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("l10n_bump") +@register_target_task("l10n_bump") def target_tasks_l10n_bump(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform l10n bumping.""" @@ -1145,7 +1133,7 @@ def target_tasks_l10n_bump(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("merge_automation") +@register_target_task("merge_automation") def target_tasks_merge_automation(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform repository merges.""" @@ -1156,7 +1144,7 @@ def target_tasks_merge_automation(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("scriptworker_canary") +@register_target_task("scriptworker_canary") def target_tasks_scriptworker_canary(full_task_graph, parameters, graph_config): """Select the set of tasks required to run scriptworker canaries.""" @@ -1167,7 +1155,7 @@ def target_tasks_scriptworker_canary(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("cron_bouncer_check") +@register_target_task("cron_bouncer_check") def target_tasks_bouncer_check(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform bouncer version verification.""" @@ -1180,7 +1168,7 @@ def target_tasks_bouncer_check(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("staging_release_builds") +@register_target_task("staging_release_builds") def target_tasks_staging_release(full_task_graph, parameters, graph_config): """ Select all builds that are part of releases. @@ -1204,7 +1192,7 @@ def target_tasks_staging_release(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("release_simulation") +@register_target_task("release_simulation") def target_tasks_release_simulation(full_task_graph, parameters, graph_config): """ Select builds that would run on push on a release branch. @@ -1241,7 +1229,7 @@ def target_tasks_release_simulation(full_task_graph, parameters, graph_config): ] -@_target_task("codereview") +@register_target_task("codereview") def target_tasks_codereview(full_task_graph, parameters, graph_config): """Select all code review tasks needed to produce a report""" @@ -1259,13 +1247,13 @@ def target_tasks_codereview(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("nothing") +@register_target_task("nothing") def target_tasks_nothing(full_task_graph, parameters, graph_config): """Select nothing, for DONTBUILD pushes""" return [] -@_target_task("daily_beta_perf") +@register_target_task("daily_beta_perf") def target_tasks_daily_beta_perf(full_task_graph, parameters, graph_config): """ Select performance tests on the beta branch to be run daily @@ -1367,7 +1355,7 @@ def target_tasks_daily_beta_perf(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("weekly_release_perf") +@register_target_task("weekly_release_perf") def target_tasks_weekly_release_perf(full_task_graph, parameters, graph_config): """ Select performance tests on the release branch to be run weekly @@ -1435,7 +1423,7 @@ def target_tasks_weekly_release_perf(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("raptor_tp6m") +@register_target_task("raptor_tp6m") def target_tasks_raptor_tp6m(full_task_graph, parameters, graph_config): """ Select tasks required for running raptor cold page-load tests on fenix and refbrow @@ -1456,14 +1444,13 @@ def target_tasks_raptor_tp6m(full_task_graph, parameters, graph_config): "browsertime" in try_name and "amazon" in try_name and "search" not in try_name - and "fenix" in try_name ): return True return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("backfill_all_browsertime") +@register_target_task("backfill_all_browsertime") def target_tasks_backfill_all_browsertime(full_task_graph, parameters, graph_config): """ Search for revisions that contains patches that were reviewed by perftest reviewers @@ -1531,7 +1518,7 @@ def target_tasks_backfill_all_browsertime(full_task_graph, parameters, graph_con return [] -@_target_task("condprof") +@register_target_task("condprof") def target_tasks_condprof(full_task_graph, parameters, graph_config): """ Select tasks required for building conditioned profiles. @@ -1542,7 +1529,7 @@ def target_tasks_condprof(full_task_graph, parameters, graph_config): yield name -@_target_task("system_symbols") +@register_target_task("system_symbols") def target_tasks_system_symbols(full_task_graph, parameters, graph_config): """ Select tasks for scraping and uploading system symbols. @@ -1556,7 +1543,7 @@ def target_tasks_system_symbols(full_task_graph, parameters, graph_config): yield name -@_target_task("perftest") +@register_target_task("perftest") def target_tasks_perftest(full_task_graph, parameters, graph_config): """ Select perftest tasks we want to run daily @@ -1568,7 +1555,7 @@ def target_tasks_perftest(full_task_graph, parameters, graph_config): yield name -@_target_task("perftest-on-autoland") +@register_target_task("perftest-on-autoland") def target_tasks_perftest_autoland(full_task_graph, parameters, graph_config): """ Select perftest tasks we want to run daily @@ -1582,7 +1569,7 @@ def target_tasks_perftest_autoland(full_task_graph, parameters, graph_config): yield name -@_target_task("l10n-cross-channel") +@register_target_task("l10n-cross-channel") def target_tasks_l10n_cross_channel(full_task_graph, parameters, graph_config): """Select the set of tasks required to run l10n cross-channel.""" @@ -1592,17 +1579,7 @@ def target_tasks_l10n_cross_channel(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("are-we-esmified-yet") -def target_tasks_are_we_esmified_yet(full_task_graph, parameters, graph_config): - """ - select the task to track the progress of the esmification project - """ - return [ - l for l, t in full_task_graph.tasks.items() if t.kind == "are-we-esmified-yet" - ] - - -@_target_task("eslint-build") +@register_target_task("eslint-build") def target_tasks_eslint_build(full_task_graph, parameters, graph_config): """Select the task to run additional ESLint rules which require a build.""" @@ -1613,7 +1590,7 @@ def target_tasks_eslint_build(full_task_graph, parameters, graph_config): yield name -@_target_task("holly_tasks") +@register_target_task("holly_tasks") def target_tasks_holly(full_task_graph, parameters, graph_config): """Bug 1814661: only run updatebot tasks on holly""" @@ -1623,7 +1600,7 @@ def target_tasks_holly(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("snap_upstream_tests") +@register_target_task("snap_upstream_tests") def target_tasks_snap_upstream_tests(full_task_graph, parameters, graph_config): """ Select tasks for testing Snap package built as upstream. Omit -try because @@ -1632,3 +1609,49 @@ def target_tasks_snap_upstream_tests(full_task_graph, parameters, graph_config): for name, task in full_task_graph.tasks.items(): if "snap-upstream-test" in name and not "-try" in name: yield name + + +@register_target_task("nightly-android") +def target_tasks_nightly_android(full_task_graph, parameters, graph_config): + def filter(task, parameters): + # geckoview + if task.attributes.get("shipping_product") == "fennec" and task.kind in ( + "beetmover-geckoview", + "upload-symbols", + ): + return True + + # fenix/focus/a-c + build_type = task.attributes.get("build-type", "") + return build_type in ( + "nightly", + "focus-nightly", + "fenix-nightly", + "fenix-nightly-firebase", + "focus-nightly-firebase", + ) + + index_path = ( + f"{graph_config['trust-domain']}.v2.{parameters['project']}.branch." + f"{parameters['head_ref']}.revision.{parameters['head_rev']}.taskgraph.decision-nightly-android" + ) + if os.environ.get("MOZ_AUTOMATION") and retry( + index_exists, + args=(index_path,), + kwargs={ + "reason": "to avoid triggering multiple nightlies off the same revision", + }, + ): + return [] + + return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] + + +@register_target_task("android-l10n-import") +def target_tasks_android_l10n_import(full_task_graph, parameters, graph_config): + return [l for l, t in full_task_graph.tasks.items() if l == "android-l10n-import"] + + +@register_target_task("android-l10n-sync") +def target_tasks_android_l10n_sync(full_task_graph, parameters, graph_config): + return [l for l, t in full_task_graph.tasks.items() if l == "android-l10n-sync"] diff --git a/taskcluster/gecko_taskgraph/test/automationrelevance.json b/taskcluster/gecko_taskgraph/test/automationrelevance.json deleted file mode 100644 index 3bdfa9ed9e..0000000000 --- a/taskcluster/gecko_taskgraph/test/automationrelevance.json +++ /dev/null @@ -1,358 +0,0 @@ -{ - "changesets": [ - { - "author": "James Long <longster@gmail.com>", - "backsoutnodes": [], - "bugs": [ - { - "no": "1300866", - "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866" - } - ], - "date": [1473196655.0, 14400], - "desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins", - "extra": { - "branch": "default" - }, - "files": ["devtools/client/debugger/index.html"], - "node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "parents": ["37c9349b4e8167a61b08b7e119c21ea177b98942"], - "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "pushdate": [1473261248, 0], - "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", - "pushid": 30664, - "pushnodes": [ - "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", - "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", - "99c542fa43a72ee863c813b5624048d1b443549b", - "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", - "541c9086c0f27fba60beecc9bc94543103895c86", - "041a925171e431bf51fb50193ab19d156088c89a", - "a14f88a9af7a59e677478694bafd9375ac53683e" - ], - "pushuser": "cbook@mozilla.com", - "rev": 312890, - "reviewers": [ - { - "name": "jlast", - "revset": "reviewer(jlast)" - }, - { - "name": "bgrins", - "revset": "reviewer(bgrins)" - } - ], - "treeherderrepo": "mozilla-central", - "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" - }, - { - "author": "Wes Kocher <wkocher@mozilla.com>", - "backsoutnodes": [], - "bugs": [], - "date": [1473208638.0, 25200], - "desc": "Merge m-c to fx-team, a=merge", - "extra": { - "branch": "default" - }, - "files": ["taskcluster/scripts/builder/build-l10n.sh"], - "node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", - "parents": [ - "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "91c2b9d5c1354ca79e5b174591dbb03b32b15bbf" - ], - "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "pushdate": [1473261248, 0], - "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", - "pushid": 30664, - "pushnodes": [ - "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", - "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", - "99c542fa43a72ee863c813b5624048d1b443549b", - "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", - "541c9086c0f27fba60beecc9bc94543103895c86", - "041a925171e431bf51fb50193ab19d156088c89a", - "a14f88a9af7a59e677478694bafd9375ac53683e" - ], - "pushuser": "cbook@mozilla.com", - "rev": 312891, - "reviewers": [ - { - "name": "merge", - "revset": "reviewer(merge)" - } - ], - "treeherderrepo": "mozilla-central", - "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" - }, - { - "author": "Towkir Ahmed <towkir17@gmail.com>", - "backsoutnodes": [], - "bugs": [ - { - "no": "1296648", - "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648" - } - ], - "date": [1472957580.0, 14400], - "desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim", - "extra": { - "branch": "default" - }, - "files": ["devtools/client/themes/rules.css"], - "node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", - "parents": ["73a6a267a50a0e1c41e689b265ad3eebe43d7ac6"], - "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "pushdate": [1473261248, 0], - "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", - "pushid": 30664, - "pushnodes": [ - "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", - "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", - "99c542fa43a72ee863c813b5624048d1b443549b", - "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", - "541c9086c0f27fba60beecc9bc94543103895c86", - "041a925171e431bf51fb50193ab19d156088c89a", - "a14f88a9af7a59e677478694bafd9375ac53683e" - ], - "pushuser": "cbook@mozilla.com", - "rev": 312892, - "reviewers": [ - { - "name": "ntim", - "revset": "reviewer(ntim)" - } - ], - "treeherderrepo": "mozilla-central", - "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" - }, - { - "author": "Oriol <oriol-bugzilla@hotmail.com>", - "backsoutnodes": [], - "bugs": [ - { - "no": "1300336", - "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336" - } - ], - "date": [1472921160.0, 14400], - "desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen", - "extra": { - "branch": "default" - }, - "files": [ - "devtools/client/webconsole/test/browser_webconsole_output_06.js", - "devtools/server/actors/object.js" - ], - "node": "99c542fa43a72ee863c813b5624048d1b443549b", - "parents": ["16a1a91f9269ab95dd83eb29dc5d0227665f7d94"], - "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "pushdate": [1473261248, 0], - "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", - "pushid": 30664, - "pushnodes": [ - "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", - "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", - "99c542fa43a72ee863c813b5624048d1b443549b", - "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", - "541c9086c0f27fba60beecc9bc94543103895c86", - "041a925171e431bf51fb50193ab19d156088c89a", - "a14f88a9af7a59e677478694bafd9375ac53683e" - ], - "pushuser": "cbook@mozilla.com", - "rev": 312893, - "reviewers": [ - { - "name": "fitzgen", - "revset": "reviewer(fitzgen)" - } - ], - "treeherderrepo": "mozilla-central", - "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" - }, - { - "author": "Ruturaj Vartak <ruturaj@gmail.com>", - "backsoutnodes": [], - "bugs": [ - { - "no": "1295010", - "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010" - } - ], - "date": [1472854020.0, -7200], - "desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK", - "extra": { - "amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd", - "branch": "default", - "histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba", - "rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651" - }, - "files": [ - "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js", - "devtools/client/inspector/test/head.js", - "devtools/server/actors/highlighters/eye-dropper.js" - ], - "node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", - "parents": ["99c542fa43a72ee863c813b5624048d1b443549b"], - "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "pushdate": [1473261248, 0], - "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", - "pushid": 30664, - "pushnodes": [ - "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", - "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", - "99c542fa43a72ee863c813b5624048d1b443549b", - "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", - "541c9086c0f27fba60beecc9bc94543103895c86", - "041a925171e431bf51fb50193ab19d156088c89a", - "a14f88a9af7a59e677478694bafd9375ac53683e" - ], - "pushuser": "cbook@mozilla.com", - "rev": 312894, - "reviewers": [ - { - "name": "pbro", - "revset": "reviewer(pbro)" - } - ], - "treeherderrepo": "mozilla-central", - "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" - }, - { - "author": "Matteo Ferretti <mferretti@mozilla.com>", - "backsoutnodes": [], - "bugs": [ - { - "no": "1299154", - "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154" - } - ], - "date": [1472629906.0, -7200], - "desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n", - "extra": { - "branch": "default", - "committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200" - }, - "files": [ - "docshell/base/nsDocShell.cpp", - "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html" - ], - "node": "541c9086c0f27fba60beecc9bc94543103895c86", - "parents": ["a6b6a93eb41a05e310a11f0172f01ba9b21d3eac"], - "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "pushdate": [1473261248, 0], - "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", - "pushid": 30664, - "pushnodes": [ - "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", - "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", - "99c542fa43a72ee863c813b5624048d1b443549b", - "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", - "541c9086c0f27fba60beecc9bc94543103895c86", - "041a925171e431bf51fb50193ab19d156088c89a", - "a14f88a9af7a59e677478694bafd9375ac53683e" - ], - "pushuser": "cbook@mozilla.com", - "rev": 312895, - "reviewers": [ - { - "name": "mstange", - "revset": "reviewer(mstange)" - } - ], - "treeherderrepo": "mozilla-central", - "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" - }, - { - "author": "Patrick Brosset <pbrosset@mozilla.com>", - "backsoutnodes": [], - "bugs": [ - { - "no": "1295010", - "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010" - } - ], - "date": [1473239449.0, -7200], - "desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp", - "extra": { - "branch": "default" - }, - "files": [ - "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js", - "devtools/client/inspector/test/head.js" - ], - "node": "041a925171e431bf51fb50193ab19d156088c89a", - "parents": ["541c9086c0f27fba60beecc9bc94543103895c86"], - "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "pushdate": [1473261248, 0], - "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", - "pushid": 30664, - "pushnodes": [ - "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", - "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", - "99c542fa43a72ee863c813b5624048d1b443549b", - "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", - "541c9086c0f27fba60beecc9bc94543103895c86", - "041a925171e431bf51fb50193ab19d156088c89a", - "a14f88a9af7a59e677478694bafd9375ac53683e" - ], - "pushuser": "cbook@mozilla.com", - "rev": 312896, - "reviewers": [ - { - "name": "me", - "revset": "reviewer(me)" - } - ], - "treeherderrepo": "mozilla-central", - "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" - }, - { - "author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>", - "backsoutnodes": [], - "bugs": [], - "date": [1473261233.0, -7200], - "desc": "merge fx-team to mozilla-central a=merge", - "extra": { - "branch": "default" - }, - "files": [], - "node": "a14f88a9af7a59e677478694bafd9375ac53683e", - "parents": [ - "3d0b41fdd93bd8233745eadb4e0358e385bf2cb9", - "041a925171e431bf51fb50193ab19d156088c89a" - ], - "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "pushdate": [1473261248, 0], - "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", - "pushid": 30664, - "pushnodes": [ - "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", - "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", - "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", - "99c542fa43a72ee863c813b5624048d1b443549b", - "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", - "541c9086c0f27fba60beecc9bc94543103895c86", - "041a925171e431bf51fb50193ab19d156088c89a", - "a14f88a9af7a59e677478694bafd9375ac53683e" - ], - "pushuser": "cbook@mozilla.com", - "rev": 312897, - "reviewers": [ - { - "name": "merge", - "revset": "reviewer(merge)" - } - ], - "treeherderrepo": "mozilla-central", - "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" - } - ], - "visible": true -} diff --git a/taskcluster/gecko_taskgraph/test/conftest.py b/taskcluster/gecko_taskgraph/test/conftest.py index 360c2da65e..758d3402b5 100644 --- a/taskcluster/gecko_taskgraph/test/conftest.py +++ b/taskcluster/gecko_taskgraph/test/conftest.py @@ -47,7 +47,7 @@ def enable_logging(): @pytest.fixture(scope="session") def graph_config(): - return load_graph_config(os.path.join(GECKO, "taskcluster", "ci")) + return load_graph_config(os.path.join(GECKO, "taskcluster")) @pytest.fixture(scope="session") @@ -151,6 +151,20 @@ class FakeOptimization(OptimizationStrategy): return False +class FakeTransformConfig: + kind = "fake-kind" + path = "/root/ci/fake-kind" + config = {} + params = FakeParameters() + kind_dependencies_tasks = {} + graph_config = {} + write_artifacts = False + + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + @pytest.fixture def maketgg(monkeypatch): def inner(target_tasks=None, kinds=[("_fake", [])], params=None): @@ -195,12 +209,16 @@ def maketgg(monkeypatch): @pytest.fixture def run_transform(): graph_config = fake_load_graph_config("/root") - kind = FakeKind.create("fake", {}, graph_config) + config = FakeTransformConfig(graph_config=graph_config) + + def inner(xform, tasks, **extra_config): + if extra_config: + for k, v in extra_config.items(): + setattr(config, k, v) - def inner(xform, tasks): if isinstance(tasks, dict): tasks = [tasks] - return xform(kind.config, tasks) + return xform(config, tasks) return inner diff --git a/taskcluster/gecko_taskgraph/test/python.toml b/taskcluster/gecko_taskgraph/test/python.toml index 597a02d8aa..fd71a9c2bd 100644 --- a/taskcluster/gecko_taskgraph/test/python.toml +++ b/taskcluster/gecko_taskgraph/test/python.toml @@ -17,6 +17,8 @@ subsuite = "taskgraph" ["test_taskcluster_yml.py"] +["test_transforms_build_schedules.py"] + ["test_transforms_job.py"] ["test_transforms_test.py"] diff --git a/taskcluster/gecko_taskgraph/test/test_actions_util.py b/taskcluster/gecko_taskgraph/test/test_actions_util.py index 7c38caea57..0838dafc11 100644 --- a/taskcluster/gecko_taskgraph/test/test_actions_util.py +++ b/taskcluster/gecko_taskgraph/test/test_actions_util.py @@ -155,6 +155,7 @@ def is_subset(subset, superset): def test_extract_applicable_action( responses, monkeypatch, actions_json, task_def, expected ): + actions.util.get_task_definition.cache_clear() base_url = "https://taskcluster" decision_task_id = "dddd" task_id = "tttt" diff --git a/taskcluster/gecko_taskgraph/test/test_decision.py b/taskcluster/gecko_taskgraph/test/test_decision.py index 8440b8e13f..53186b70fb 100644 --- a/taskcluster/gecko_taskgraph/test/test_decision.py +++ b/taskcluster/gecko_taskgraph/test/test_decision.py @@ -7,7 +7,6 @@ import json import os import shutil import tempfile -import unittest from unittest.mock import patch import pytest @@ -18,6 +17,7 @@ from gecko_taskgraph import decision from gecko_taskgraph.parameters import register_parameters FAKE_GRAPH_CONFIG = {"product-dir": "browser", "taskgraph": {}} +TTC_FILE = os.path.join(os.getcwd(), "try_task_config.json") @pytest.fixture(scope="module", autouse=True) @@ -25,150 +25,172 @@ def register(): register_parameters() -class TestDecision(unittest.TestCase): - def test_write_artifact_json(self): - data = [{"some": "data"}] - tmpdir = tempfile.mkdtemp() - try: - decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts") - decision.write_artifact("artifact.json", data) - with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.json")) as f: - self.assertEqual(json.load(f), data) - finally: - if os.path.exists(tmpdir): - shutil.rmtree(tmpdir) - decision.ARTIFACTS_DIR = "artifacts" - - def test_write_artifact_yml(self): - data = [{"some": "data"}] - tmpdir = tempfile.mkdtemp() - try: - decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts") - decision.write_artifact("artifact.yml", data) - self.assertEqual(load_yaml(decision.ARTIFACTS_DIR, "artifact.yml"), data) - finally: - if os.path.exists(tmpdir): - shutil.rmtree(tmpdir) - decision.ARTIFACTS_DIR = "artifacts" - - -class TestGetDecisionParameters(unittest.TestCase): - ttc_file = os.path.join(os.getcwd(), "try_task_config.json") - - def setUp(self): - self.options = { - "base_repository": "https://hg.mozilla.org/mozilla-unified", - "head_repository": "https://hg.mozilla.org/mozilla-central", - "head_rev": "abcd", - "head_ref": "ef01", - "head_tag": "", - "message": "", - "project": "mozilla-central", - "pushlog_id": "143", - "pushdate": 1503691511, - "owner": "nobody@mozilla.com", - "repository_type": "hg", - "tasks_for": "hg-push", - "level": "3", - } - - @patch("gecko_taskgraph.decision.get_hg_revision_branch") - @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev") - def test_simple_options( - self, mock_determine_more_accurate_base_rev, mock_get_hg_revision_branch - ): - mock_get_hg_revision_branch.return_value = "default" - mock_determine_more_accurate_base_rev.return_value = "baserev" - with MockedOpen({self.ttc_file: None}): - params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) - self.assertEqual(params["pushlog_id"], "143") - self.assertEqual(params["build_date"], 1503691511) - self.assertEqual(params["hg_branch"], "default") - self.assertEqual(params["moz_build_date"], "20170825200511") - self.assertEqual(params["try_mode"], None) - self.assertEqual(params["try_options"], None) - self.assertEqual(params["try_task_config"], {}) - - @patch("gecko_taskgraph.decision.get_hg_revision_branch") - @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev") - def test_no_email_owner( - self, mock_determine_more_accurate_base_rev, mock_get_hg_revision_branch - ): - mock_get_hg_revision_branch.return_value = "default" - mock_determine_more_accurate_base_rev.return_value = "baserev" - self.options["owner"] = "ffxbld" - with MockedOpen({self.ttc_file: None}): - params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) - self.assertEqual(params["owner"], "ffxbld@noreply.mozilla.org") - - @patch("gecko_taskgraph.decision.get_hg_revision_branch") - @patch("gecko_taskgraph.decision.get_hg_commit_message") - @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev") - def test_try_options( - self, - mock_determine_more_accurate_base_rev, - mock_get_hg_commit_message, - mock_get_hg_revision_branch, - ): - mock_get_hg_commit_message.return_value = "try: -b do -t all --artifact" - mock_get_hg_revision_branch.return_value = "default" - mock_determine_more_accurate_base_rev.return_value = "baserev" - self.options["project"] = "try" - with MockedOpen({self.ttc_file: None}): - params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) - self.assertEqual(params["try_mode"], "try_option_syntax") - self.assertEqual(params["try_options"]["build_types"], "do") - self.assertEqual(params["try_options"]["unittests"], "all") - self.assertEqual( - params["try_task_config"], +@pytest.fixture(scope="module") +def options(): + return { + "base_repository": "https://hg.mozilla.org/mozilla-unified", + "head_repository": "https://hg.mozilla.org/mozilla-central", + "head_rev": "abcd", + "head_ref": "ef01", + "head_tag": "", + "message": "", + "project": "mozilla-central", + "pushlog_id": "143", + "pushdate": 1503691511, + "owner": "nobody@mozilla.com", + "repository_type": "hg", + "tasks_for": "hg-push", + "level": "3", + } + + +def test_write_artifact_json(): + data = [{"some": "data"}] + tmpdir = tempfile.mkdtemp() + try: + decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts") + decision.write_artifact("artifact.json", data) + with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.json")) as f: + assert json.load(f) == data + finally: + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + decision.ARTIFACTS_DIR = "artifacts" + + +def test_write_artifact_yml(): + data = [{"some": "data"}] + tmpdir = tempfile.mkdtemp() + try: + decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts") + decision.write_artifact("artifact.yml", data) + assert load_yaml(decision.ARTIFACTS_DIR, "artifact.yml") == data + finally: + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + decision.ARTIFACTS_DIR = "artifacts" + + +@patch("gecko_taskgraph.decision.get_hg_revision_branch") +@patch("gecko_taskgraph.decision.get_hg_commit_message") +@patch("gecko_taskgraph.decision._determine_more_accurate_base_rev") +@patch("gecko_taskgraph.decision.get_changed_files") +@pytest.mark.parametrize( + "extra_options,commit_msg,ttc,expected", + ( + pytest.param( + {}, + None, + None, { - "gecko-profile": False, - "use-artifact-builds": True, - "env": {}, + "pushlog_id": "143", + "build_date": 1503691511, + "files_changed": ["bar/baz.md", "foo.txt"], + "hg_branch": "default", + "moz_build_date": "20170825200511", + "try_mode": None, + "try_options": None, + "try_task_config": {}, }, - ) - - @patch("gecko_taskgraph.decision.get_hg_revision_branch") - @patch("gecko_taskgraph.decision.get_hg_commit_message") - @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev") - def test_try_task_config( - self, - mock_get_hg_commit_message, - mock_get_hg_revision_branch, - mock_determine_more_accurate_base_rev, - ): - mock_get_hg_commit_message.return_value = "Fuzzy query=foo" - mock_get_hg_revision_branch.return_value = "default" - mock_determine_more_accurate_base_rev.return_value = "baserev" - ttc = {"tasks": ["a", "b"]} - self.options["project"] = "try" - with MockedOpen({self.ttc_file: json.dumps(ttc)}): - params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options) - self.assertEqual(params["try_mode"], "try_task_config") - self.assertEqual(params["try_options"], None) - self.assertEqual(params["try_task_config"], ttc) - - def test_try_syntax_from_message_empty(self): - self.assertEqual(decision.try_syntax_from_message(""), "") - - def test_try_syntax_from_message_no_try_syntax(self): - self.assertEqual(decision.try_syntax_from_message("abc | def"), "") - - def test_try_syntax_from_message_initial_try_syntax(self): - self.assertEqual( - decision.try_syntax_from_message("try: -f -o -o"), "try: -f -o -o" - ) - - def test_try_syntax_from_message_initial_try_syntax_multiline(self): - self.assertEqual( - decision.try_syntax_from_message("try: -f -o -o\nabc\ndef"), "try: -f -o -o" - ) - - def test_try_syntax_from_message_embedded_try_syntax_multiline(self): - self.assertEqual( - decision.try_syntax_from_message("some stuff\ntry: -f -o -o\nabc\ndef"), + id="simple_options", + ), + pytest.param( + {"owner": "ffxbld"}, + None, + None, + { + "owner": "ffxbld@noreply.mozilla.org", + }, + id="no_email_owner", + ), + pytest.param( + {"project": "try"}, + "try: -b do -t all --artifact", + None, + { + "try_mode": "try_option_syntax", + "try_options": { + "build_types": "do", + "include_nightly": False, + "interactive": False, + "jobs": None, + "no_retry": False, + "notifications": None, + "platforms": "all", + "raptor": "none", + "raptor_trigger_tests": 1, + "tag": None, + "talos": "all", + "talos_trigger_tests": 1, + "taskcluster_worker": False, + "trigger_tests": 1, + "unittests": "all", + }, + "try_task_config": { + "gecko-profile": False, + "use-artifact-builds": True, + "env": {}, + }, + }, + id="try_options", + ), + pytest.param( + { + "project": "try", + }, + "Fuzzy query=foo", + {"tasks": ["a", "b"]}, + { + "try_mode": "try_task_config", + "try_options": None, + "try_task_config": {"tasks": ["a", "b"]}, + }, + id="try_task_config", + ), + ), +) +def test_get_decision_parameters( + mock_get_changed_files, + mock_determine_more_accurate_base_rev, + mock_get_hg_commit_message, + mock_get_hg_revision_branch, + options, + extra_options, + commit_msg, + ttc, + expected, +): + mock_get_hg_revision_branch.return_value = "default" + mock_get_hg_commit_message.return_value = commit_msg or "commit message" + mock_determine_more_accurate_base_rev.return_value = "baserev" + mock_get_changed_files.return_value = ["foo.txt", "bar/baz.md"] + + options.update(extra_options) + contents = None + if ttc: + contents = json.dumps(ttc) + with MockedOpen({TTC_FILE: contents}): + params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, options) + + for key in expected: + assert params[key] == expected[key], f"key {key} does not match!" + + +@pytest.mark.parametrize( + "msg, expected", + ( + pytest.param("", "", id="empty"), + pytest.param("abc | def", "", id="no_try_syntax"), + pytest.param("try: -f -o -o", "try: -f -o -o", id="initial_try_syntax"), + pytest.param( + "some stuff\ntry: -f -o -o\nabc\ndef", "try: -f -o -o", - ) + id="embedded_try_syntax_multiline", + ), + ), +) +def test_try_syntax_from_message(msg, expected): + assert decision.try_syntax_from_message(msg) == expected if __name__ == "__main__": diff --git a/taskcluster/gecko_taskgraph/test/test_files_changed.py b/taskcluster/gecko_taskgraph/test/test_files_changed.py index 5b9a016649..389dbb4093 100644 --- a/taskcluster/gecko_taskgraph/test/test_files_changed.py +++ b/taskcluster/gecko_taskgraph/test/test_files_changed.py @@ -3,14 +3,11 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. -import json -import os import unittest from mozunit import main from gecko_taskgraph import files_changed -from gecko_taskgraph.util import hg PARAMS = { "head_repository": "https://hg.mozilla.org/mozilla-central", @@ -31,40 +28,17 @@ FILES_CHANGED = [ ] -class FakeResponse: - def json(self): - with open( - os.path.join(os.path.dirname(__file__), "automationrelevance.json") - ) as f: - return json.load(f) - - -class TestGetChangedFiles(unittest.TestCase): - def setUp(self): - files_changed.get_changed_files.clear() - self.old_get = hg.requests.get - - def fake_get(url, **kwargs): - return FakeResponse() - - hg.requests.get = fake_get - - def tearDown(self): - hg.requests.get = self.old_get - files_changed.get_changed_files.clear() - - def test_get_changed_files(self): - """Get_changed_files correctly gets the list of changed files in a push. - This tests against the production hg.mozilla.org so that it will detect - any changes in the format of the returned data.""" - self.assertEqual( - sorted( - files_changed.get_changed_files( - PARAMS["head_repository"], PARAMS["head_rev"] - ) - ), - FILES_CHANGED, +def test_get_changed_files(responses): + url = f"{PARAMS['head_repository']}/json-pushchangedfiles/{PARAMS['head_rev']}" + responses.add(responses.GET, url, status=200, json={"files": FILES_CHANGED}) + assert ( + sorted( + files_changed.get_changed_files( + PARAMS["head_repository"], PARAMS["head_rev"] + ) ) + == FILES_CHANGED + ) class TestCheck(unittest.TestCase): diff --git a/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py b/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py index 1240d71cf8..2c10ba449a 100644 --- a/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py +++ b/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py @@ -19,6 +19,7 @@ from gecko_taskgraph.optimize.bugbug import ( DisperseGroups, SkipUnlessDebug, ) +from gecko_taskgraph.optimize.mozlint import SkipUnlessMozlint from gecko_taskgraph.optimize.strategies import SkipUnlessSchedules from gecko_taskgraph.util.backstop import BACKSTOP_PUSH_INTERVAL from gecko_taskgraph.util.bugbug import ( @@ -511,5 +512,79 @@ def test_project_autoland_test(monkeypatch, responses, params): assert scheduled == {"task-0-label", "task-1-label"} +@pytest.mark.parametrize( + "pushed_files,to_lint,expected", + [ + pytest.param( + ["a/b/c.txt"], + [], + True, + ), + pytest.param( + ["python/mozlint/a/support_file.txt", "b/c/d.txt"], + ["python/mozlint/a/support_file.txt"], + False, + ), + ], + ids=idfn, +) +def test_mozlint_should_remove_task( + monkeypatch, params, pushed_files, to_lint, expected +): + import mozlint.pathutils + + class MockParser: + def __call__(self, *args, **kwargs): + return [] + + def mock_filterpaths(*args, **kwargs): + return to_lint, None + + monkeypatch.setattr(mozlint.pathutils, "filterpaths", mock_filterpaths) + + opt = SkipUnlessMozlint("") + monkeypatch.setattr(opt, "mozlint_parser", MockParser()) + params["files_changed"] = pushed_files + + result = opt.should_remove_task(default_tasks[0], params, "") + assert result == expected + + +@pytest.mark.parametrize( + "pushed_files,linter_config,expected", + [ + pytest.param( + ["a/b/c.txt"], + [{"include": ["b/c"]}], + True, + ), + pytest.param( + ["a/b/c.txt"], + [{"include": ["a/b"], "exclude": ["a/b/c.txt"]}], + True, + ), + pytest.param( + ["python/mozlint/a/support_file.txt", "b/c/d.txt"], + [{}], + False, + ), + ], + ids=idfn, +) +def test_mozlint_should_remove_task2( + monkeypatch, params, pushed_files, linter_config, expected +): + class MockParser: + def __call__(self, *args, **kwargs): + return linter_config + + opt = SkipUnlessMozlint("") + monkeypatch.setattr(opt, "mozlint_parser", MockParser()) + params["files_changed"] = pushed_files + + result = opt.should_remove_task(default_tasks[0], params, "") + assert result == expected + + if __name__ == "__main__": main() diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_build_schedules.py b/taskcluster/gecko_taskgraph/test/test_transforms_build_schedules.py new file mode 100644 index 0000000000..a693461c68 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_transforms_build_schedules.py @@ -0,0 +1,56 @@ +import pytest +from mozunit import main + +from gecko_taskgraph.transforms.build_schedules import set_build_schedules_optimization + + +@pytest.mark.parametrize( + "kind,task,expected", + ( + pytest.param("build", {"when": "foo"}, None, id="no-op"), + pytest.param( + "build", + {"attributes": {"build_platform": "linux64/opt"}}, + {"build": ["linux", "firefox"]}, + id="build", + ), + pytest.param( + "build-components", + {}, + {"build": ["android", "fenix", "focus-android"]}, + id="build-components", + ), + pytest.param( + "build-bundle", + {"name": "build-bundle-fenix"}, + {"build": ["android", "fenix"]}, + id="build-bundle-fenix", + ), + pytest.param( + "build-apk", + {"name": "fenix"}, + {"build": ["android", "fenix"]}, + id="build-apk-fenix", + ), + pytest.param( + "build-apk", + {"name": "build-apk-focus"}, + {"build": ["android", "focus-android"]}, + id="build-apk-focus", + ), + pytest.param( + "build-apk", + {"name": "build-apk-klar"}, + {"build": ["android", "focus-android"]}, + id="build-apk-klar", + ), + ), +) +def test_build_schedules(run_transform, kind, task, expected): + tasks = list(run_transform(set_build_schedules_optimization, [task], kind=kind)) + assert len(tasks) == 1 + assert tasks[0].get("optimization") == expected + + +if __name__ == "__main__": + main() diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_job.py b/taskcluster/gecko_taskgraph/test/test_transforms_job.py index b032307ea6..6161a36054 100644 --- a/taskcluster/gecko_taskgraph/test/test_transforms_job.py +++ b/taskcluster/gecko_taskgraph/test/test_transforms_job.py @@ -37,7 +37,7 @@ TASK_DEFAULTS = { @pytest.fixture(scope="module") def config(): - graph_config = load_graph_config(os.path.join(GECKO, "taskcluster", "ci")) + graph_config = load_graph_config(os.path.join(GECKO, "taskcluster")) params = FakeParameters( { "base_repository": "http://hg.example.com", diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_test.py b/taskcluster/gecko_taskgraph/test/test_transforms_test.py index 1e5067a2b5..d61eff5769 100644 --- a/taskcluster/gecko_taskgraph/test/test_transforms_test.py +++ b/taskcluster/gecko_taskgraph/test/test_transforms_test.py @@ -235,16 +235,16 @@ def test_split_variants(monkeypatch, run_full_config_transform, make_test_task): pytest.param( { "attributes": {}, - "test-platform": "windows10-64-2004-ref-hw-2017-ccov/debug", + "test-platform": "windows11-64-2009-hw-ref-ccov/debug", }, { "platform": { "arch": "64", - "machine": "ref-hw-2017", + "machine": "hw-ref", "os": { - "build": "2004", + "build": "2009", "name": "windows", - "version": "10", + "version": "11", }, }, "build": { diff --git a/taskcluster/gecko_taskgraph/test/test_util_backstop.py b/taskcluster/gecko_taskgraph/test/test_util_backstop.py index af9aabd5af..0a2bdc6ae4 100644 --- a/taskcluster/gecko_taskgraph/test/test_util_backstop.py +++ b/taskcluster/gecko_taskgraph/test/test_util_backstop.py @@ -18,20 +18,22 @@ from gecko_taskgraph.util.backstop import ( is_backstop, ) -LAST_BACKSTOP_ID = 0 +LAST_BACKSTOP_PUSHID = 1 LAST_BACKSTOP_PUSHDATE = mktime(datetime.now().timetuple()) DEFAULT_RESPONSES = { "index": { "status": 200, - "json": {"taskId": LAST_BACKSTOP_ID}, + "json": {"taskId": LAST_BACKSTOP_PUSHID}, }, "artifact": { "status": 200, "body": dedent( """ pushdate: {} + pushlog_id: "{}" """.format( - LAST_BACKSTOP_PUSHDATE + LAST_BACKSTOP_PUSHDATE, + LAST_BACKSTOP_PUSHID, ) ), }, @@ -50,7 +52,8 @@ def params(): "head_rev": "abcdef", "project": "autoland", "pushdate": LAST_BACKSTOP_PUSHDATE + 1, - "pushlog_id": LAST_BACKSTOP_ID + 1, + "pushlog_id": f"{LAST_BACKSTOP_PUSHID + 1}", + "target_tasks_method": "default", } @@ -61,7 +64,7 @@ def params(): { "index": {"status": 404}, }, - {"pushlog_id": 1}, + {"pushlog_id": "1"}, True, id="no previous backstop", ), @@ -78,8 +81,8 @@ def params(): pytest.param( DEFAULT_RESPONSES, { - "pushlog_id": LAST_BACKSTOP_ID + 1, - "pushdate": LAST_BACKSTOP_PUSHDATE + 1, + "pushlog_id": f"{LAST_BACKSTOP_PUSHID + BACKSTOP_PUSH_INTERVAL - 1}", + "pushdate": LAST_BACKSTOP_PUSHDATE + (BACKSTOP_TIME_INTERVAL * 60) - 1, }, False, id="not a backstop", @@ -87,10 +90,26 @@ def params(): pytest.param( {}, { - "pushlog_id": BACKSTOP_PUSH_INTERVAL, + "target_tasks_method": "nothing", + }, + False, + id="dontbuild", + ), + pytest.param( + DEFAULT_RESPONSES, + { + "pushlog_id": f"{LAST_BACKSTOP_PUSHID + BACKSTOP_PUSH_INTERVAL}", + }, + True, + id="interval", + ), + pytest.param( + DEFAULT_RESPONSES, + { + "pushlog_id": f"{LAST_BACKSTOP_PUSHID + BACKSTOP_PUSH_INTERVAL + 1}", }, True, - id="backstop interval", + id="greater than interval", ), pytest.param( DEFAULT_RESPONSES, @@ -104,7 +123,7 @@ def params(): {}, { "project": "try", - "pushlog_id": BACKSTOP_PUSH_INTERVAL, + "pushlog_id": f"{BACKSTOP_PUSH_INTERVAL}", }, False, id="try not a backstop", @@ -138,13 +157,12 @@ def test_is_backstop(responses, params, response_args, extra_params, expected): **{"trust-domain": "gecko", "project": params["project"]} ) ), - "artifact": get_artifact_url(LAST_BACKSTOP_ID, "public/parameters.yml"), - "status": get_task_url(LAST_BACKSTOP_ID) + "/status", + "artifact": get_artifact_url(LAST_BACKSTOP_PUSHID, "public/parameters.yml"), + "status": get_task_url(LAST_BACKSTOP_PUSHID) + "/status", } for key in ("index", "status", "artifact"): if key in response_args: - print(urls[key]) responses.add(responses.GET, urls[key], **response_args[key]) params.update(extra_params) diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py index fb5b17d3b3..1434b4013a 100644 --- a/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py +++ b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py @@ -101,6 +101,7 @@ CONFIG_PER_BOUNCER_PRODUCT = { "file_names": { "win": "{pretty_product}%20Setup%20{version}.msix", "win64": "{pretty_product}%20Setup%20{version}.msix", + "win64-aarch64": "{pretty_product}%20Setup%20{version}.msix", }, }, "pkg": { diff --git a/taskcluster/gecko_taskgraph/transforms/build.py b/taskcluster/gecko_taskgraph/transforms/build.py index 4e73c5aef2..3ab22f2923 100644 --- a/taskcluster/gecko_taskgraph/transforms/build.py +++ b/taskcluster/gecko_taskgraph/transforms/build.py @@ -139,6 +139,10 @@ def use_artifact(config, jobs): and job.get("index", {}).get("job-name") in ARTIFACT_JOBS # If tests aren't packaged, then we are not able to rebuild all the packages and job["worker"]["env"].get("MOZ_AUTOMATION_PACKAGE_TESTS") == "1" + # Android shippable artifact builds are not supported + and not ( + "android" in job["name"] and job["attributes"].get("shippable", False) + ) ): job["treeherder"]["symbol"] = add_suffix(job["treeherder"]["symbol"], "a") job["worker"]["env"]["USE_ARTIFACT"] = "1" diff --git a/taskcluster/gecko_taskgraph/transforms/build_attrs.py b/taskcluster/gecko_taskgraph/transforms/build_attrs.py index 9cda71718a..fda9888fe1 100644 --- a/taskcluster/gecko_taskgraph/transforms/build_attrs.py +++ b/taskcluster/gecko_taskgraph/transforms/build_attrs.py @@ -4,8 +4,6 @@ from taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.platforms import platform_family - transforms = TransformSequence() @@ -34,17 +32,3 @@ def set_build_attributes(config, jobs): ) yield job - - -@transforms.add -def set_schedules_optimization(config, jobs): - """Set the `skip-unless-affected` optimization based on the build platform.""" - for job in jobs: - # don't add skip-unless-schedules if there's already a when defined - if "when" in job: - yield job - continue - - build_platform = job["attributes"]["build_platform"] - job.setdefault("optimization", {"build": [platform_family(build_platform)]}) - yield job diff --git a/taskcluster/gecko_taskgraph/transforms/build_schedules.py b/taskcluster/gecko_taskgraph/transforms/build_schedules.py new file mode 100644 index 0000000000..f9621adf77 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/build_schedules.py @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.util.platforms import platform_family + +transforms = TransformSequence() + + +@transforms.add +def set_build_schedules_optimization(config, tasks): + """Set the `build` optimization based on the build platform.""" + for task in tasks: + # don't add an optimization if there's already one defined + if "when" in task or "optimization" in task: + yield task + continue + + schedules = [] + if config.kind in ( + "build-components", + "build-samples-browser", + "test-components", + ): + # These are Android components builds and can only impact Fenix or Focus. + schedules = ["android", "fenix", "focus-android"] + + elif config.kind in ("build-apk", "build-bundle", "test-apk", "ui-test-apk"): + # These are APK builds for Fenix or Focus + schedules = ["android"] + + if "fenix" in task["name"]: + schedules.append("fenix") + elif "focus" in task["name"] or "klar" in task["name"]: + schedules.append("focus-android") + + else: + family = platform_family(task["attributes"]["build_platform"]) + schedules = [family] + + if "android" not in family: + # These are not GeckoView builds, so are associated with Firefox. + schedules.append("firefox") + + task["optimization"] = {"build": schedules} + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/condprof.py b/taskcluster/gecko_taskgraph/transforms/condprof.py index 516c1d8f20..f39a9b09ab 100644 --- a/taskcluster/gecko_taskgraph/transforms/condprof.py +++ b/taskcluster/gecko_taskgraph/transforms/condprof.py @@ -28,7 +28,7 @@ diff_description_schema = Schema( Optional("run-on-projects"): task_description_schema["run-on-projects"], Optional("scopes"): task_description_schema["scopes"], Optional("treeherder"): task_description_schema["treeherder"], - Optional("use-system-python"): bool, + Optional("use-python"): job_description_schema["use-python"], Optional("worker"): job_description_schema["worker"], Optional("worker-type"): task_description_schema["worker-type"], } @@ -84,8 +84,8 @@ def generate_scenarios(config, tasks): "fetches": copy_task(task["fetches"]), } - use_system_python = task.get("use-system-python", None) - if use_system_python is not None: - taskdesc["use-system-python"] = use_system_python + use_taskcluster_python = task.get("use-python", "system") + if use_taskcluster_python != "system": + taskdesc["use-python"] = use_taskcluster_python yield taskdesc diff --git a/taskcluster/gecko_taskgraph/transforms/diffoscope.py b/taskcluster/gecko_taskgraph/transforms/diffoscope.py index b74dc5bb8f..05c6617950 100644 --- a/taskcluster/gecko_taskgraph/transforms/diffoscope.py +++ b/taskcluster/gecko_taskgraph/transforms/diffoscope.py @@ -32,7 +32,7 @@ diff_description_schema = Schema( Required("original"): index_or_string, Required("new"): index_or_string, # Arguments to pass to diffoscope, used for job-defaults in - # taskcluster/ci/diffoscope/kind.yml + # taskcluster/kinds/diffoscope/kind.yml Optional("args"): str, # Extra arguments to pass to diffoscope, that can be set per job. Optional("extra-args"): str, diff --git a/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py b/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py index 2f0d8dd2aa..016d642b9b 100644 --- a/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py +++ b/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ -Transform the repackage signing task into an actual task description. +Transform the geckodriver notarization task into an actual task description. """ from taskgraph.transforms.base import TransformSequence @@ -14,7 +14,7 @@ from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job from gecko_taskgraph.util.scriptworker import add_scope_prefix -repackage_signing_description_schema = Schema( +geckodriver_notarization_description_schema = Schema( { Optional("label"): str, Optional("treeherder"): task_description_schema["treeherder"], @@ -38,7 +38,7 @@ def remove_name(config, jobs): yield job -transforms.add_validate(repackage_signing_description_schema) +transforms.add_validate(geckodriver_notarization_description_schema) @transforms.add diff --git a/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py b/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py index 95b8d3dd54..25f861e662 100644 --- a/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py +++ b/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py @@ -121,7 +121,7 @@ def make_signing_description(config, jobs): def _craft_upstream_artifacts(dep_job, dependency_kind, build_platform): if build_platform.startswith("win"): - signing_format = "autograph_authenticode_sha2" + signing_format = "autograph_authenticode_202404" elif build_platform.startswith("linux"): signing_format = "autograph_gpg" elif build_platform.startswith("macosx"): diff --git a/taskcluster/gecko_taskgraph/transforms/github_sync.py b/taskcluster/gecko_taskgraph/transforms/github_sync.py deleted file mode 100644 index 6f48f794ce..0000000000 --- a/taskcluster/gecko_taskgraph/transforms/github_sync.py +++ /dev/null @@ -1,23 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from taskgraph.transforms.base import TransformSequence - -transforms = TransformSequence() - - -@transforms.add -def sync_github(config, tasks): - """Do transforms specific to github-sync tasks.""" - for task in tasks: - # Add the secret to the scopes, only in m-c. - # Doing this on any other tree will result in decision task failure - # because m-c is the only one allowed to have that scope. - secret = task["secret"] - if config.params["project"] == "mozilla-central": - task.setdefault("scopes", []) - task["scopes"].append("secrets:get:" + secret) - task["worker"].setdefault("env", {})["GITHUB_SECRET"] = secret - del task["secret"] - yield task diff --git a/taskcluster/gecko_taskgraph/transforms/job/__init__.py b/taskcluster/gecko_taskgraph/transforms/job/__init__.py index b87f7e0955..54cedf513a 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/__init__.py +++ b/taskcluster/gecko_taskgraph/transforms/job/__init__.py @@ -14,11 +14,12 @@ import json import logging import mozpack.path as mozpath +from packaging.version import Version from taskgraph.transforms.base import TransformSequence from taskgraph.util.python_path import import_sibling_modules from taskgraph.util.schema import Schema, validate_schema from taskgraph.util.taskcluster import get_artifact_prefix -from voluptuous import Any, Exclusive, Extra, Optional, Required +from voluptuous import Any, Coerce, Exclusive, Extra, Optional, Required from gecko_taskgraph.transforms.cached_tasks import order_tasks from gecko_taskgraph.transforms.task import task_description_schema @@ -62,7 +63,7 @@ job_description_schema = Schema( "optimization" ], Optional("use-sccache"): task_description_schema["use-sccache"], - Optional("use-system-python"): bool, + Optional("use-python"): Any("system", "default", Coerce(Version)), Optional("priority"): task_description_schema["priority"], # The "when" section contains descriptions of the circumstances under which # this task should be included in the task graph. This will be converted @@ -245,9 +246,15 @@ def get_attribute(dict, key, attributes, attribute_name): @transforms.add def use_system_python(config, jobs): for job in jobs: - if job.pop("use-system-python", True): + taskcluster_python = job.pop("use-python", "system") + if taskcluster_python == "system": yield job else: + if taskcluster_python == "default": + python_version = "python" # the taskcluster default alias + else: + python_version = f"python-{taskcluster_python}" + fetches = job.setdefault("fetches", {}) toolchain = fetches.setdefault("toolchain", []) if "win" in job["worker"]["os"]: @@ -259,7 +266,7 @@ def use_system_python(config, jobs): else: raise ValueError("unexpected worker.os value {}".format(platform)) - toolchain.append("{}-python".format(platform)) + toolchain.append(f"{platform}-{python_version}") worker = job.setdefault("worker", {}) env = worker.setdefault("env", {}) diff --git a/taskcluster/gecko_taskgraph/transforms/job/mach.py b/taskcluster/gecko_taskgraph/transforms/job/mach.py index 775213f8fe..5f830ec04b 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/mach.py +++ b/taskcluster/gecko_taskgraph/transforms/job/mach.py @@ -50,7 +50,7 @@ def configure_mach(config, job, taskdesc): if python: del run["python-version"] - if taskdesc.get("use-system-python"): + if taskdesc.get("use-python", "system") == "system": if worker["os"] == "macosx" and python == 3: python = "/usr/local/bin/python3" diff --git a/taskcluster/gecko_taskgraph/transforms/job/mozharness.py b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py index 4d7293ec51..ada5b85ea3 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/mozharness.py +++ b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py @@ -289,7 +289,7 @@ def mozharness_on_generic_worker(config, job, taskdesc): system_python_dir = "" gecko_path = "$GECKO_PATH" - if run.get("use-system-python", True): + if run.get("use-python", "system") == "system": python_bindir = system_python_dir else: # $MOZ_PYTHON_HOME is going to be substituted in run-task, when we diff --git a/taskcluster/gecko_taskgraph/transforms/openh264_signing.py b/taskcluster/gecko_taskgraph/transforms/openh264_signing.py index 00a55dad41..74aebbd90b 100644 --- a/taskcluster/gecko_taskgraph/transforms/openh264_signing.py +++ b/taskcluster/gecko_taskgraph/transforms/openh264_signing.py @@ -81,7 +81,7 @@ def make_signing_description(config, jobs): } if "win" in build_platform: - upstream_artifact["formats"] = ["autograph_authenticode_sha2"] + upstream_artifact["formats"] = ["autograph_authenticode_202404"] elif "mac" in build_platform: upstream_artifact["formats"] = ["mac_single_file"] upstream_artifact["singleFileGlobs"] = ["libgmpopenh264.dylib"] diff --git a/taskcluster/gecko_taskgraph/transforms/perftest.py b/taskcluster/gecko_taskgraph/transforms/perftest.py index 47baafdad7..066db956f7 100644 --- a/taskcluster/gecko_taskgraph/transforms/perftest.py +++ b/taskcluster/gecko_taskgraph/transforms/perftest.py @@ -1,3 +1,4 @@ +# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ diff --git a/taskcluster/gecko_taskgraph/transforms/release_deps.py b/taskcluster/gecko_taskgraph/transforms/release_deps.py index e44af576eb..aab8a2f60a 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_deps.py +++ b/taskcluster/gecko_taskgraph/transforms/release_deps.py @@ -44,6 +44,17 @@ def add_dependencies(config, jobs): != job["attributes"]["build_platform"] ): continue + + # TODO get rid of the release-type match + if product == "firefox-android": + # exclude beta tasks from release graph and vice versa + from android_taskgraph.release_type import does_task_match_release_type + + if not does_task_match_release_type( + dep_task, config.params["release_type"] + ): + continue + # Add matching product tasks to deps if ( dep_task.task.get("shipping-product") == product diff --git a/taskcluster/gecko_taskgraph/transforms/release_notifications.py b/taskcluster/gecko_taskgraph/transforms/release_notifications.py index 86109ec5ed..071e5de8a3 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_notifications.py +++ b/taskcluster/gecko_taskgraph/transforms/release_notifications.py @@ -39,6 +39,9 @@ def add_notifications(config, jobs): resolve_keyed_by( notifications, "emails", label, project=config.params["project"] ) + resolve_keyed_by( + notifications, "message", label, project=config.params["project"] + ) emails = notifications["emails"] format_kwargs = dict( task=job, diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_signing.py b/taskcluster/gecko_taskgraph/transforms/repackage_signing.py index 66c1f87d70..d4106efb29 100644 --- a/taskcluster/gecko_taskgraph/transforms/repackage_signing.py +++ b/taskcluster/gecko_taskgraph/transforms/repackage_signing.py @@ -29,10 +29,10 @@ repackage_signing_description_schema = Schema( ) SIGNING_FORMATS = { - "target.installer.exe": ["autograph_authenticode_sha2_stub"], - "target.stub-installer.exe": ["autograph_authenticode_sha2_stub"], - "target.installer.msi": ["autograph_authenticode_sha2"], - "target.installer.msix": ["autograph_authenticode_sha2"], + "target.installer.exe": ["autograph_authenticode_202404_stub"], + "target.stub-installer.exe": ["autograph_authenticode_202404_stub"], + "target.installer.msi": ["autograph_authenticode_202404"], + "target.installer.msix": ["autograph_authenticode_202404"], } transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py b/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py index e3940fd846..5f7c893e92 100644 --- a/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py +++ b/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py @@ -93,7 +93,7 @@ def make_repackage_signing_description(config, jobs): "paths": [ get_artifact_path(dep_job, f"{repack_id}/target.installer.exe"), ], - "formats": ["autograph_authenticode_sha2", "autograph_gpg"], + "formats": ["autograph_authenticode_202404", "autograph_gpg"], } ] @@ -113,7 +113,7 @@ def make_repackage_signing_description(config, jobs): f"{repack_id}/target.stub-installer.exe", ), ], - "formats": ["autograph_authenticode_sha2", "autograph_gpg"], + "formats": ["autograph_authenticode_202404", "autograph_gpg"], } ) elif "mac" in build_platform: diff --git a/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py b/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py index ea2cac3a68..48465b3ab7 100644 --- a/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py +++ b/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Transform the reprocess-symbols task description template, -taskcluster/ci/reprocess-symbols/job-template.yml into an actual task description. +taskcluster/kinds/reprocess-symbols/job-template.yml into an actual task description. """ diff --git a/taskcluster/gecko_taskgraph/transforms/signing.py b/taskcluster/gecko_taskgraph/transforms/signing.py index e55ad47f42..760f9f122f 100644 --- a/taskcluster/gecko_taskgraph/transforms/signing.py +++ b/taskcluster/gecko_taskgraph/transforms/signing.py @@ -12,10 +12,7 @@ from taskgraph.util.schema import Schema, taskref_or_string from voluptuous import Optional, Required from gecko_taskgraph.transforms.task import task_description_schema -from gecko_taskgraph.util.attributes import ( - copy_attributes_from_dependent_job, - release_level, -) +from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job from gecko_taskgraph.util.scriptworker import ( add_scope_prefix, get_signing_cert_scope_per_platform, @@ -64,6 +61,14 @@ signing_description_schema = Schema( ) +def get_locales_description(attributes, default): + """Returns the [list] of locales for task description usage""" + chunk_locales = attributes.get("chunk_locales") + if chunk_locales: + return ", ".join(chunk_locales) + return attributes.get("locale", default) + + @transforms.add def delete_name(config, jobs): """Delete the 'name' key if it exists, we don't use it.""" @@ -77,30 +82,12 @@ transforms.add_validate(signing_description_schema) @transforms.add -def add_entitlements_link(config, jobs): - for job in jobs: - dep_job = get_primary_dependency(config, job) - entitlements_path = evaluate_keyed_by( - config.graph_config["mac-notarization"]["mac-entitlements"], - "mac entitlements", - { - "platform": dep_job.attributes.get("build_platform"), - "release-level": release_level(config.params["project"]), - }, - ) - if entitlements_path: - job["entitlements-url"] = config.params.file_url( - entitlements_path, - ) - yield job - - -@transforms.add def add_requirements_link(config, jobs): for job in jobs: dep_job = get_primary_dependency(config, job) + assert dep_job requirements_path = evaluate_keyed_by( - config.graph_config["mac-notarization"]["mac-requirements"], + config.graph_config["mac-signing"]["mac-requirements"], "mac requirements", { "platform": dep_job.attributes.get("build_platform"), @@ -117,6 +104,7 @@ def add_requirements_link(config, jobs): def make_task_description(config, jobs): for job in jobs: dep_job = get_primary_dependency(config, job) + assert dep_job attributes = dep_job.attributes signing_format_scopes = [] @@ -127,6 +115,7 @@ def make_task_description(config, jobs): is_shippable = dep_job.attributes.get("shippable", False) build_platform = dep_job.attributes.get("build_platform") + assert build_platform treeherder = None if "partner" not in config.kind and "eme-free" not in config.kind: treeherder = job.get("treeherder", {}) @@ -161,9 +150,9 @@ def make_task_description(config, jobs): label = job["label"] description = ( - "Initial Signing for locale '{locale}' for build '" + "Signing of locale(s) '{locale}' for build '" "{build_platform}/{build_type}'".format( - locale=attributes.get("locale", "en-US"), + locale=get_locales_description(attributes, "en-US"), build_platform=build_platform, build_type=attributes.get("build_type"), ) @@ -209,10 +198,16 @@ def make_task_description(config, jobs): # build-mac-{signing,notarization} uses signingscript instead of iscript if "macosx" in build_platform and config.kind.endswith("-mac-notarization"): - task["worker"]["mac-behavior"] = "apple_notarization" task["scopes"] = [ add_scope_prefix(config, "signing:cert:release-apple-notarization") ] + task[ + "description" + ] = "Notarization of '{}' locales for build '{}/{}'".format( + get_locales_description(attributes, "en-US"), + build_platform, + attributes.get("build_type"), + ) elif "macosx" in build_platform: # iscript overrides task["worker"]["mac-behavior"] = "mac_sign_and_pkg" diff --git a/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py b/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py index 0c31f48cc0..2b846604f1 100644 --- a/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py +++ b/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py @@ -1,3 +1,4 @@ +# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ diff --git a/taskcluster/gecko_taskgraph/transforms/source_test.py b/taskcluster/gecko_taskgraph/transforms/source_test.py index e3eeb7c819..266637a7a0 100644 --- a/taskcluster/gecko_taskgraph/transforms/source_test.py +++ b/taskcluster/gecko_taskgraph/transforms/source_test.py @@ -1,3 +1,4 @@ +# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ @@ -264,8 +265,8 @@ def remove_optimization_on_central(config, jobs): if not job.get("attributes", {}).get("code-review", False): yield job continue - if "when" not in job: - yield job - continue - del job["when"] + if "when" in job: + del job["when"] + if "optimization" in job and "skip-unless-mozlint" in job["optimization"]: + del job["optimization"] yield job diff --git a/taskcluster/gecko_taskgraph/transforms/task.py b/taskcluster/gecko_taskgraph/transforms/task.py index 3129742ea9..aa77c46d65 100644 --- a/taskcluster/gecko_taskgraph/transforms/task.py +++ b/taskcluster/gecko_taskgraph/transforms/task.py @@ -208,7 +208,7 @@ TC_TREEHERDER_SCHEMA_URL = ( UNKNOWN_GROUP_NAME = ( - "Treeherder group {} (from {}) has no name; " "add it to taskcluster/ci/config.yml" + "Treeherder group {} (from {}) has no name; " "add it to taskcluster/config.yml" ) V2_ROUTE_TEMPLATES = [ @@ -313,7 +313,7 @@ def index_builder(name): UNSUPPORTED_INDEX_PRODUCT_ERROR = """\ The gecko-v2 product {product} is not in the list of configured products in -`taskcluster/ci/config.yml'. +`taskcluster/config.yml'. """ @@ -842,6 +842,7 @@ def build_generic_worker_payload(config, task, task_def): # behavior for mac iscript Optional("mac-behavior"): Any( "apple_notarization", + "apple_notarization_stacked", "mac_sign_and_pkg", "mac_sign_and_pkg_hardened", "mac_geckodriver", @@ -1349,6 +1350,23 @@ def build_push_addons_payload(config, task, task_def): ], }, Optional("merge-info"): object, + Optional("android-l10n-import-info"): { + Required("from-repo-url"): str, + Required("toml-info"): [ + { + Required("toml-path"): str, + Required("dest-path"): str, + } + ], + }, + Optional("android-l10n-sync-info"): { + Required("from-repo-url"): str, + Required("toml-info"): [ + { + Required("toml-path"): str, + } + ], + }, }, ) def build_treescript_payload(config, task, task_def): @@ -1412,6 +1430,38 @@ def build_treescript_payload(config, task, task_def): task_def["payload"]["merge_info"] = merge_info actions.append("merge_day") + if worker.get("android-l10n-import-info"): + android_l10n_import_info = {} + for k, v in worker["android-l10n-import-info"].items(): + android_l10n_import_info[k.replace("-", "_")] = worker[ + "android-l10n-import-info" + ][k] + android_l10n_import_info["toml_info"] = [ + { + param_name.replace("-", "_"): param_value + for param_name, param_value in entry.items() + } + for entry in worker["android-l10n-import-info"]["toml-info"] + ] + task_def["payload"]["android_l10n_import_info"] = android_l10n_import_info + actions.append("android_l10n_import") + + if worker.get("android-l10n-sync-info"): + android_l10n_sync_info = {} + for k, v in worker["android-l10n-sync-info"].items(): + android_l10n_sync_info[k.replace("-", "_")] = worker[ + "android-l10n-sync-info" + ][k] + android_l10n_sync_info["toml_info"] = [ + { + param_name.replace("-", "_"): param_value + for param_name, param_value in entry.items() + } + for entry in worker["android-l10n-sync-info"]["toml-info"] + ] + task_def["payload"]["android_l10n_sync_info"] = android_l10n_sync_info + actions.append("android_l10n_sync") + if worker["push"]: actions.append("push") @@ -1563,7 +1613,7 @@ def task_name_from_label(config, tasks): UNSUPPORTED_SHIPPING_PRODUCT_ERROR = """\ The shipping product {product} is not in the list of configured products in -`taskcluster/ci/config.yml'. +`taskcluster/config.yml'. """ diff --git a/taskcluster/gecko_taskgraph/transforms/test/__init__.py b/taskcluster/gecko_taskgraph/transforms/test/__init__.py index 92704bf18c..19ab8d289f 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/__init__.py +++ b/taskcluster/gecko_taskgraph/transforms/test/__init__.py @@ -27,6 +27,7 @@ from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by from voluptuous import Any, Exclusive, Optional, Required from gecko_taskgraph.optimize.schema import OptimizationSchema +from gecko_taskgraph.transforms.job import job_description_schema from gecko_taskgraph.transforms.test.other import get_mobile_project from gecko_taskgraph.util.chunking import manifest_loaders @@ -118,7 +119,9 @@ test_description_schema = Schema( Required("run-without-variant"): optionally_keyed_by("test-platform", bool), # The EC2 instance size to run these tests on. Required("instance-size"): optionally_keyed_by( - "test-platform", Any("default", "large", "xlarge") + "test-platform", + "variant", + Any("default", "large", "large-noscratch", "xlarge", "xlarge-noscratch"), ), # type of virtualization or hardware required by test. Required("virtualization"): optionally_keyed_by( @@ -265,11 +268,14 @@ test_description_schema = Schema( str, None, {Required("index"): str, Required("name"): str}, + {Required("upstream-task"): str, Required("name"): str}, ), ), # A list of artifacts to install from 'fetch' tasks. Validation deferred # to 'job' transforms. Optional("fetches"): object, + # A list of extra dependencies + Optional("dependencies"): object, # Raptor / browsertime specific keys, defer validation to 'raptor.py' # transform. Optional("raptor"): object, @@ -279,6 +285,8 @@ test_description_schema = Schema( Optional("subtest"): str, # Define if a given task supports artifact builds or not, see bug 1695325. Optional("supports-artifact-builds"): bool, + # Version of python used to run the task + Optional("use-python"): job_description_schema["use-python"], } ) @@ -346,6 +354,7 @@ def set_defaults(config, tasks): task.setdefault("run-without-variant", True) task.setdefault("variants", []) task.setdefault("supports-artifact-builds", True) + task.setdefault("use-python", "system") task["mozharness"].setdefault("extra-options", []) task["mozharness"].setdefault("requires-signed-builds", False) @@ -484,6 +493,9 @@ def make_job_description(config, tasks): if task["mozharness"]["requires-signed-builds"] is True: jobdesc["dependencies"]["build-signing"] = task["build-signing-label"] + if "dependencies" in task: + jobdesc["dependencies"].update(task["dependencies"]) + if "expires-after" in task: jobdesc["expires-after"] = task["expires-after"] diff --git a/taskcluster/gecko_taskgraph/transforms/test/chunk.py b/taskcluster/gecko_taskgraph/transforms/test/chunk.py index 7f832c57df..8219c41664 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/chunk.py +++ b/taskcluster/gecko_taskgraph/transforms/test/chunk.py @@ -44,8 +44,7 @@ def set_test_verify_chunks(config, tasks): task["chunks"] = perfile_number_of_chunks( is_try(config.params), env.get("MOZHARNESS_TEST_PATHS", ""), - config.params.get("head_repository", ""), - config.params.get("head_rev", ""), + frozenset(config.params["files_changed"]), task["test-name"], ) diff --git a/taskcluster/gecko_taskgraph/transforms/test/other.py b/taskcluster/gecko_taskgraph/transforms/test/other.py index b8cb95cff7..e01691c05a 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/other.py +++ b/taskcluster/gecko_taskgraph/transforms/test/other.py @@ -2,6 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import copy import hashlib import json import re @@ -12,7 +13,11 @@ from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import keymatch from taskgraph.util.keyed_by import evaluate_keyed_by from taskgraph.util.schema import Schema, resolve_keyed_by -from taskgraph.util.taskcluster import get_artifact_path, get_index_url +from taskgraph.util.taskcluster import ( + get_artifact_path, + get_artifact_url, + get_index_url, +) from voluptuous import Any, Optional, Required from gecko_taskgraph.transforms.test.variant import TEST_VARIANTS @@ -99,6 +104,25 @@ def setup_talos(config, tasks): if config.params.get("project", None): extra_options.append("--project=%s" % config.params["project"]) + if "pdfpaint" in task["try-name"]: + max_chunks = 10 + for chunk in range(1, max_chunks + 1): + new_task = copy.deepcopy(task) + new_task["mozharness"]["extra-options"].append( + f"--pdfPaintChunk={chunk}" + ) + new_task["test-name"] = task["test-name"].replace( + "pdfpaint", f"pdfpaint-{chunk}" + ) + new_task["try-name"] = task["try-name"].replace( + "pdfpaint", f"pdfpaint-{chunk}" + ) + new_task["treeherder-symbol"] = task["treeherder-symbol"].replace( + "pdfpaint", f"pdfpaint-{chunk}" + ) + yield new_task + continue + yield task @@ -246,6 +270,7 @@ def handle_keyed_by(config, tasks): "webrender-run-on-projects", "mozharness.requires-signed-builds", "build-signing-label", + "dependencies", ] for task in tasks: for field in fields: @@ -292,10 +317,17 @@ def set_target(config, tasks): target = "target.tar.bz2" if isinstance(target, dict): - # TODO Remove hardcoded mobile artifact prefix - index_url = get_index_url(target["index"]) - installer_url = "{}/artifacts/public/{}".format(index_url, target["name"]) - task["mozharness"]["installer-url"] = installer_url + if "index" in target: + # TODO Remove hardcoded mobile artifact prefix + index_url = get_index_url(target["index"]) + installer_url = "{}/artifacts/public/{}".format( + index_url, target["name"] + ) + task["mozharness"]["installer-url"] = installer_url + else: + task["mozharness"]["installer-url"] = get_artifact_url( + f'<{target["upstream-task"]}>', target["name"] + ) else: task["mozharness"]["build-artifact-name"] = get_artifact_path(task, target) @@ -363,50 +395,44 @@ def setup_browsertime(config, tasks): cd_fetches = { "android.*": [ - "linux64-chromedriver-120", - "linux64-chromedriver-121", "linux64-chromedriver-122", + "linux64-chromedriver-123", + "linux64-chromedriver-124", ], "linux.*": [ - "linux64-chromedriver-120", - "linux64-chromedriver-121", "linux64-chromedriver-122", + "linux64-chromedriver-123", + "linux64-chromedriver-124", ], "macosx1015.*": [ - "mac64-chromedriver-120", - "mac64-chromedriver-121", "mac64-chromedriver-122", + "mac64-chromedriver-123", + "mac64-chromedriver-124", ], "macosx1400.*": [ - "mac-arm-chromedriver-120", - "mac-arm-chromedriver-121", "mac-arm-chromedriver-122", + "mac-arm-chromedriver-123", + "mac-arm-chromedriver-124", ], "windows.*aarch64.*": [ - "win32-chromedriver-120", - "win32-chromedriver-121", - "win32-chromedriver-122", - ], - "windows.*-32.*": [ - "win32-chromedriver-120", "win32-chromedriver-121", "win32-chromedriver-122", + "win32-chromedriver-123", ], "windows.*-64.*": [ - "win32-chromedriver-120", - "win32-chromedriver-121", - "win32-chromedriver-122", + "win64-chromedriver-123", + "win64-chromedriver-124", ], } chromium_fetches = { - "linux.*": ["linux64-chromium"], - "macosx1015.*": ["mac-chromium"], - "macosx1400.*": ["mac-chromium-arm"], - "windows.*aarch64.*": ["win32-chromium"], - "windows.*-32.*": ["win32-chromium"], - "windows.*-64.*": ["win64-chromium"], - "android.*": ["linux64-chromium"], + "linux.*": ["linux64-chromiumdriver"], + "macosx1015.*": ["mac-chromiumdriver"], + "macosx1400.*": ["mac-chromiumdriver-arm"], + "windows.*aarch64.*": ["win32-chromiumdriver"], + "windows.*-32.*": ["win32-chromiumdriver"], + "windows.*-64.*": ["win64-chromiumdriver"], + "android.*": ["linux64-chromiumdriver"], } cd_extracted_name = { @@ -419,11 +445,7 @@ def setup_browsertime(config, tasks): # Only add the chromedriver fetches when chrome is running for platform in cd_fetches: fs["by-test-platform"][platform].extend(cd_fetches[platform]) - if ( - "--app=chromium" in extra_options - or "--app=custom-car" in extra_options - or "--app=cstm-car-m" in extra_options - ): + if "--app=custom-car" in extra_options or "--app=cstm-car-m" in extra_options: for platform in chromium_fetches: fs["by-test-platform"][platform].extend(chromium_fetches[platform]) @@ -553,7 +575,9 @@ def enable_code_coverage(config, tasks): yield task continue task["mozharness"].setdefault("extra-options", []).append("--code-coverage") - task["instance-size"] = "xlarge" + task["instance-size"] = "xlarge-noscratch" + if "jittest" in task["test-name"]: + task["instance-size"] = "xlarge" # Temporarily disable Mac tests on mozilla-central if "mac" in task["build-platform"]: @@ -791,7 +815,7 @@ test_setting_description_schema = Schema( }, Optional("device"): str, Optional("display"): "wayland", - Optional("machine"): Any("ref-hw-2017", "hw-ref"), + Optional("machine"): "hw-ref", }, "build": { Required("type"): Any("opt", "debug", "debug-isolated-process"), @@ -852,7 +876,6 @@ def set_test_setting(config, tasks): # TODO Rename these so they don't have a dash. dash_attrs = [ "clang-trunk", - "ref-hw-2017", "hw-ref", ] dash_token = "%D%" @@ -908,9 +931,6 @@ def set_test_setting(config, tasks): if parts[0].isdigit(): os_build = parts.pop(0) - if parts and parts[0] == "ref-hw-2017": - machine = parts.pop(0) - if parts and parts[0] == "hw-ref": machine = parts.pop(0) @@ -1102,6 +1122,7 @@ def set_schedules_components(config, tasks): schedules.add(category) schedules.add(platform_family(task["build-platform"])) + schedules.add("firefox") task["schedules-component"] = sorted(schedules) yield task diff --git a/taskcluster/gecko_taskgraph/transforms/test/raptor.py b/taskcluster/gecko_taskgraph/transforms/test/raptor.py index 18e21e6a1e..ca35749037 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/raptor.py +++ b/taskcluster/gecko_taskgraph/transforms/test/raptor.py @@ -76,7 +76,6 @@ def split_apps(config, tests): app_symbols = { "chrome": "ChR", "chrome-m": "ChR", - "chromium": "Cr", "fenix": "fenix", "refbrow": "refbrow", "safari": "Saf", diff --git a/taskcluster/gecko_taskgraph/transforms/test/variant.py b/taskcluster/gecko_taskgraph/transforms/test/variant.py index bda91b2f25..6ee6c429f0 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/variant.py +++ b/taskcluster/gecko_taskgraph/transforms/test/variant.py @@ -78,6 +78,14 @@ def split_variants(config, tasks): remaining_variants.append(name) return remaining_variants + def replace_task_items(task_key, variant_key): + for item in variant_key: + if isinstance(variant_key[item], dict): + task_key[item] = replace_task_items(task_key[item], variant_key[item]) + else: + task_key[item] = variant_key[item] + return task_key + def apply_variant(variant, task): task["description"] = variant["description"].format(**task) @@ -94,7 +102,9 @@ def split_variants(config, tasks): task["variant-suffix"] += suffix # Replace and/or merge the configuration. - task.update(variant.get("replace", {})) + + # we only want to update the leaf node, the the entire top level dict + task = replace_task_items(task, variant.get("replace", {})) return merge(task, variant.get("merge", {})) expired_variants = find_expired_variants(TEST_VARIANTS) diff --git a/taskcluster/gecko_taskgraph/transforms/test/worker.py b/taskcluster/gecko_taskgraph/transforms/test/worker.py index 873347459c..51b12de51d 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/worker.py +++ b/taskcluster/gecko_taskgraph/transforms/test/worker.py @@ -7,8 +7,10 @@ from taskgraph.transforms.base import TransformSequence # default worker types keyed by instance-size LINUX_WORKER_TYPES = { "large": "t-linux-large", + "large-noscratch": "t-linux-large-noscratch", "xlarge": "t-linux-xlarge", - "default": "t-linux-large", + "xlarge-noscratch": "t-linux-xlarge-noscratch", + "default": "t-linux-large-noscratch", } # windows worker types keyed by test-platform and virtualization @@ -23,11 +25,6 @@ WINDOWS_WORKER_TYPES = { "virtual-with-gpu": "t-win10-64-gpu-s", "hardware": "t-win10-64-1803-hw", }, - "windows10-64-ref-hw-2017": { - "virtual": "t-win10-64", - "virtual-with-gpu": "t-win10-64-gpu-s", - "hardware": "t-win10-64-ref-hw", - }, "windows11-64-2009-hw-ref-shippable": { "virtual": "win11-64-2009-hw-ref", "virtual-with-gpu": "win11-64-2009-hw-ref", @@ -130,12 +127,8 @@ def set_worker_type(config, tasks): elif test_platform.startswith("win"): # figure out what platform the job needs to run on if task["virtualization"] == "hardware": - # some jobs like talos and reftest run on real h/w - those are all win10 - if test_platform.startswith("windows10-64-ref-hw-2017"): - win_worker_type_platform = WINDOWS_WORKER_TYPES[ - "windows10-64-ref-hw-2017" - ] - elif test_platform.startswith("windows11-64-2009-hw-ref"): + # some jobs like talos and reftest run on real h/w + if test_platform.startswith("windows11-64-2009-hw-ref"): win_worker_type_platform = WINDOWS_WORKER_TYPES[ "windows11-64-2009-hw-ref" ] diff --git a/taskcluster/gecko_taskgraph/transforms/test_apk.py b/taskcluster/gecko_taskgraph/transforms/test_apk.py new file mode 100644 index 0000000000..b00657b91e --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/test_apk.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the test +kinds. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ( + "routes", + "scopes", + "extra.notify", + ): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + "level": config.params["level"], + } + ) + yield task diff --git a/taskcluster/gecko_taskgraph/transforms/update_verify_config.py b/taskcluster/gecko_taskgraph/transforms/update_verify_config.py index 2d1cd40877..4e516f173c 100644 --- a/taskcluster/gecko_taskgraph/transforms/update_verify_config.py +++ b/taskcluster/gecko_taskgraph/transforms/update_verify_config.py @@ -101,6 +101,8 @@ def add_command(config, tasks): get_branch_rev(config), "--output-file", "update-verify.cfg", + "--local-repo", + ".", ] repo_path = urlsplit(get_branch_repo(config)).path.lstrip("/") diff --git a/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py b/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py index aea948f90e..3828e912ec 100644 --- a/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py +++ b/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Transform the upload-generated-files task description template, -taskcluster/ci/upload-generated-sources/kind.yml, into an actual task description. +taskcluster/kinds/upload-generated-sources/kind.yml, into an actual task description. """ from taskgraph.transforms.base import TransformSequence diff --git a/taskcluster/gecko_taskgraph/transforms/upload_symbols.py b/taskcluster/gecko_taskgraph/transforms/upload_symbols.py index 5c0bf18cb0..b216d2c8e1 100644 --- a/taskcluster/gecko_taskgraph/transforms/upload_symbols.py +++ b/taskcluster/gecko_taskgraph/transforms/upload_symbols.py @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Transform the upload-symbols task description template, -taskcluster/ci/upload-symbols/job-template.yml into an actual task description. +taskcluster/kinds/upload-symbols/job-template.yml into an actual task description. """ diff --git a/taskcluster/gecko_taskgraph/util/backstop.py b/taskcluster/gecko_taskgraph/util/backstop.py index 26c9a4fb91..18c9166083 100644 --- a/taskcluster/gecko_taskgraph/util/backstop.py +++ b/taskcluster/gecko_taskgraph/util/backstop.py @@ -37,22 +37,17 @@ def is_backstop( return True project = params["project"] - pushid = int(params["pushlog_id"]) - pushdate = int(params["pushdate"]) - if project in TRY_PROJECTS: return False if project not in integration_projects: return True - # On every Nth push, want to run all tasks. - if pushid % push_interval == 0: - return True - - if time_interval <= 0: + # This push was explicitly set to run nothing (e.g via DONTBUILD), so + # shouldn't be a backstop candidate. + if params["target_tasks_method"] == "nothing": return False - # We also want to ensure we run all tasks at least once per N minutes. + # Find the last backstop to compute push and time intervals. subs = {"trust-domain": trust_domain, "project": project} index = BACKSTOP_INDEX.format(**subs) @@ -67,9 +62,7 @@ def is_backstop( return True try: - last_pushdate = get_artifact(last_backstop_id, "public/parameters.yml")[ - "pushdate" - ] + last_params = get_artifact(last_backstop_id, "public/parameters.yml") except HTTPError as e: # If the last backstop decision task exists in the index, but # parameters.yml isn't available yet, it means the decision task is @@ -79,6 +72,15 @@ def is_backstop( return False raise - if (pushdate - last_pushdate) / 60 >= time_interval: + # On every Nth push, want to run all tasks. + if int(params["pushlog_id"]) - int(last_params["pushlog_id"]) >= push_interval: + return True + + if time_interval <= 0: + return False + + # We also want to ensure we run all tasks at least once per N minutes. + if (params["pushdate"] - last_params["pushdate"]) / 60 >= time_interval: return True + return False diff --git a/taskcluster/gecko_taskgraph/util/chunking.py b/taskcluster/gecko_taskgraph/util/chunking.py index a0ed56de78..9d8b98e07b 100644 --- a/taskcluster/gecko_taskgraph/util/chunking.py +++ b/taskcluster/gecko_taskgraph/util/chunking.py @@ -24,8 +24,8 @@ here = os.path.abspath(os.path.dirname(__file__)) resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader) TEST_VARIANTS = {} -if os.path.exists(os.path.join(GECKO, "taskcluster", "ci", "test", "variants.yml")): - TEST_VARIANTS = load_yaml(GECKO, "taskcluster", "ci", "test", "variants.yml") +if os.path.exists(os.path.join(GECKO, "taskcluster", "kinds", "test", "variants.yml")): + TEST_VARIANTS = load_yaml(GECKO, "taskcluster", "kinds", "test", "variants.yml") WPT_SUBSUITES = { "canvas": "html/canvas", @@ -101,8 +101,8 @@ def guess_mozinfo_from_task(task, repo=""): ("linux", "1804"): "18.04", ("macosx", "1015"): "10.15", ("macosx", "1100"): "11.00", - ("windows", "7"): "6.1", ("windows", "10"): "10.0", + ("windows", "11"): "11.0", } for (name, old_ver), new_ver in os_versions.items(): if p_os["name"] == name and p_os["version"] == old_ver: diff --git a/taskcluster/gecko_taskgraph/util/docker.py b/taskcluster/gecko_taskgraph/util/docker.py index e8de7d1fdb..51e3f7e484 100644 --- a/taskcluster/gecko_taskgraph/util/docker.py +++ b/taskcluster/gecko_taskgraph/util/docker.py @@ -298,7 +298,7 @@ class ImagePathsMap(Mapping): self.__update_image_paths(jobs, image_dir) -image_paths = ImagePathsMap("taskcluster/ci/docker-image/kind.yml") +image_paths = ImagePathsMap("taskcluster/kinds/docker-image/kind.yml") def image_path(name): diff --git a/taskcluster/gecko_taskgraph/util/hg.py b/taskcluster/gecko_taskgraph/util/hg.py index 18a92fbd0d..d03af87f13 100644 --- a/taskcluster/gecko_taskgraph/util/hg.py +++ b/taskcluster/gecko_taskgraph/util/hg.py @@ -96,15 +96,15 @@ def get_push_data(repository, project, push_id_start, push_id_end): @memoize -def get_json_automationrelevance(repository, revision): - url = "{}/json-automationrelevance/{}".format(repository.rstrip("/"), revision) +def get_json_pushchangedfiles(repository, revision): + url = "{}/json-pushchangedfiles/{}".format(repository.rstrip("/"), revision) logger.debug("Querying version control for metadata: %s", url) - def get_automationrelevance(): - response = requests.get(url, timeout=30) + def get_pushchangedfiles(): + response = requests.get(url, timeout=60) return response.json() - return retry(get_automationrelevance, attempts=10, sleeptime=10) + return retry(get_pushchangedfiles, attempts=10, sleeptime=10) def get_hg_revision_branch(root, revision): diff --git a/taskcluster/gecko_taskgraph/util/partners.py b/taskcluster/gecko_taskgraph/util/partners.py index 2546e1ae88..f4b8b187cf 100644 --- a/taskcluster/gecko_taskgraph/util/partners.py +++ b/taskcluster/gecko_taskgraph/util/partners.py @@ -523,7 +523,7 @@ def apply_partner_priority(config, jobs): # Reduce the priority of the partner repack jobs because they don't block QE. Meanwhile # leave EME-free jobs alone because they do, and they'll get the branch priority like the rest # of the release. Only bother with this in production, not on staging releases on try. - # medium is the same as mozilla-central, see taskcluster/ci/config.yml. ie higher than + # medium is the same as mozilla-central, see taskcluster/config.yml. ie higher than # integration branches because we don't want to wait a lot for the graph to be done, but # for multiple releases the partner tasks always wait for non-partner. if ( diff --git a/taskcluster/gecko_taskgraph/util/perfile.py b/taskcluster/gecko_taskgraph/util/perfile.py index 4e82d87dad..4c18ca98be 100644 --- a/taskcluster/gecko_taskgraph/util/perfile.py +++ b/taskcluster/gecko_taskgraph/util/perfile.py @@ -12,15 +12,12 @@ import taskgraph from mozbuild.util import memoize from mozpack.path import match as mozpackmatch -from gecko_taskgraph import files_changed - -from .. import GECKO - logger = logging.getLogger(__name__) @memoize -def perfile_number_of_chunks(is_try, try_task_config, head_repository, head_rev, type): +def perfile_number_of_chunks(is_try, try_task_config, files_changed, type): + changed_files = set(files_changed) if taskgraph.fast and not is_try: # When iterating on taskgraph changes, the exact number of chunks that # test-verify runs usually isn't important, so skip it when going fast. @@ -55,17 +52,11 @@ def perfile_number_of_chunks(is_try, try_task_config, head_repository, head_rev, # Returning 0 means no tests to run, this captures non test-verify tasks return 1 - changed_files = set() if try_task_config: suite_to_paths = json.loads(try_task_config) specified_files = itertools.chain.from_iterable(suite_to_paths.values()) changed_files.update(specified_files) - if is_try: - changed_files.update(files_changed.get_locally_changed_files(GECKO)) - else: - changed_files.update(files_changed.get_changed_files(head_repository, head_rev)) - test_count = 0 for pattern in file_patterns: for path in changed_files: diff --git a/taskcluster/gecko_taskgraph/util/platforms.py b/taskcluster/gecko_taskgraph/util/platforms.py index 2c423223fe..3010b32792 100644 --- a/taskcluster/gecko_taskgraph/util/platforms.py +++ b/taskcluster/gecko_taskgraph/util/platforms.py @@ -27,7 +27,8 @@ _executable_extension = { _architectures = { r"linux\b.*": "x86", - r"linux64\b.*": "x86_64", + r"linux64\b(?!-aarch64).*": "x86_64", + r"linux64-aarch64\b.*": "aarch64", r"macosx64\b.*": "macos-x86_64-aarch64", r"win32\b.*": "x86", r"win64\b(?!-aarch64).*": "x86_64", diff --git a/taskcluster/gecko_taskgraph/util/signed_artifacts.py b/taskcluster/gecko_taskgraph/util/signed_artifacts.py index 2467ff8046..302a2e09f7 100644 --- a/taskcluster/gecko_taskgraph/util/signed_artifacts.py +++ b/taskcluster/gecko_taskgraph/util/signed_artifacts.py @@ -57,7 +57,7 @@ def generate_specifications_of_artifacts_to_sign( elif "macosx" in build_platform: langpack_formats = [] if is_notarization_kind(config.kind): - formats = ["apple_notarization"] + formats = ["apple_notarization_stacked"] artifacts_specifications = [ { "artifacts": [ @@ -98,14 +98,14 @@ def generate_specifications_of_artifacts_to_sign( "artifacts": [ get_artifact_path(job, "{locale}/setup.exe"), ], - "formats": ["autograph_authenticode_sha2"], + "formats": ["autograph_authenticode_202404"], }, { "artifacts": [ get_artifact_path(job, "{locale}/target.zip"), ], "formats": [ - "autograph_authenticode_sha2", + "autograph_authenticode_202404", "autograph_widevine", "autograph_omnija", ], diff --git a/taskcluster/gecko_taskgraph/util/taskcluster.py b/taskcluster/gecko_taskgraph/util/taskcluster.py index cddb01fd37..826eee4cdb 100644 --- a/taskcluster/gecko_taskgraph/util/taskcluster.py +++ b/taskcluster/gecko_taskgraph/util/taskcluster.py @@ -24,7 +24,7 @@ def insert_index(index_path, task_id, data=None, use_proxy=False): index_url = get_index_url(index_path, use_proxy=use_proxy) # Find task expiry. - expires = get_task_definition(task_id, use_proxy=use_proxy)["expires"] + expires = get_task_definition(task_id, use_proxy)["expires"] response = _do_request( index_url, |