summaryrefslogtreecommitdiffstats
path: root/taskcluster/gecko_taskgraph
diff options
context:
space:
mode:
Diffstat (limited to 'taskcluster/gecko_taskgraph')
-rw-r--r--taskcluster/gecko_taskgraph/__init__.py11
-rw-r--r--taskcluster/gecko_taskgraph/config.py12
-rw-r--r--taskcluster/gecko_taskgraph/decision.py13
-rw-r--r--taskcluster/gecko_taskgraph/files_changed.py16
-rw-r--r--taskcluster/gecko_taskgraph/main.py50
-rw-r--r--taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml8
-rw-r--r--taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml2
-rw-r--r--taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml57
-rw-r--r--taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml2
-rw-r--r--taskcluster/gecko_taskgraph/optimize/mozlint.py96
-rw-r--r--taskcluster/gecko_taskgraph/optimize/schema.py2
-rw-r--r--taskcluster/gecko_taskgraph/optimize/strategies.py52
-rw-r--r--taskcluster/gecko_taskgraph/parameters.py3
-rw-r--r--taskcluster/gecko_taskgraph/target_tasks.py255
-rw-r--r--taskcluster/gecko_taskgraph/test/automationrelevance.json358
-rw-r--r--taskcluster/gecko_taskgraph/test/conftest.py26
-rw-r--r--taskcluster/gecko_taskgraph/test/python.toml2
-rw-r--r--taskcluster/gecko_taskgraph/test/test_actions_util.py1
-rw-r--r--taskcluster/gecko_taskgraph/test/test_decision.py306
-rw-r--r--taskcluster/gecko_taskgraph/test/test_files_changed.py46
-rw-r--r--taskcluster/gecko_taskgraph/test/test_optimize_strategies.py75
-rw-r--r--taskcluster/gecko_taskgraph/test/test_transforms_build_schedules.py56
-rw-r--r--taskcluster/gecko_taskgraph/test/test_transforms_job.py2
-rw-r--r--taskcluster/gecko_taskgraph/test/test_transforms_test.py8
-rw-r--r--taskcluster/gecko_taskgraph/test/test_util_backstop.py44
-rw-r--r--taskcluster/gecko_taskgraph/transforms/bouncer_submission.py1
-rw-r--r--taskcluster/gecko_taskgraph/transforms/build.py4
-rw-r--r--taskcluster/gecko_taskgraph/transforms/build_attrs.py16
-rw-r--r--taskcluster/gecko_taskgraph/transforms/build_schedules.py48
-rw-r--r--taskcluster/gecko_taskgraph/transforms/condprof.py8
-rw-r--r--taskcluster/gecko_taskgraph/transforms/diffoscope.py2
-rw-r--r--taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py6
-rw-r--r--taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py2
-rw-r--r--taskcluster/gecko_taskgraph/transforms/github_sync.py23
-rw-r--r--taskcluster/gecko_taskgraph/transforms/job/__init__.py15
-rw-r--r--taskcluster/gecko_taskgraph/transforms/job/mach.py2
-rw-r--r--taskcluster/gecko_taskgraph/transforms/job/mozharness.py2
-rw-r--r--taskcluster/gecko_taskgraph/transforms/openh264_signing.py2
-rw-r--r--taskcluster/gecko_taskgraph/transforms/perftest.py1
-rw-r--r--taskcluster/gecko_taskgraph/transforms/release_deps.py11
-rw-r--r--taskcluster/gecko_taskgraph/transforms/release_notifications.py3
-rw-r--r--taskcluster/gecko_taskgraph/transforms/repackage_signing.py8
-rw-r--r--taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py4
-rw-r--r--taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py2
-rw-r--r--taskcluster/gecko_taskgraph/transforms/signing.py49
-rw-r--r--taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py1
-rw-r--r--taskcluster/gecko_taskgraph/transforms/source_test.py9
-rw-r--r--taskcluster/gecko_taskgraph/transforms/task.py56
-rw-r--r--taskcluster/gecko_taskgraph/transforms/test/__init__.py14
-rw-r--r--taskcluster/gecko_taskgraph/transforms/test/chunk.py3
-rw-r--r--taskcluster/gecko_taskgraph/transforms/test/other.py101
-rw-r--r--taskcluster/gecko_taskgraph/transforms/test/raptor.py1
-rw-r--r--taskcluster/gecko_taskgraph/transforms/test/variant.py12
-rw-r--r--taskcluster/gecko_taskgraph/transforms/test/worker.py17
-rw-r--r--taskcluster/gecko_taskgraph/transforms/test_apk.py33
-rw-r--r--taskcluster/gecko_taskgraph/transforms/update_verify_config.py2
-rw-r--r--taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py2
-rw-r--r--taskcluster/gecko_taskgraph/transforms/upload_symbols.py2
-rw-r--r--taskcluster/gecko_taskgraph/util/backstop.py28
-rw-r--r--taskcluster/gecko_taskgraph/util/chunking.py6
-rw-r--r--taskcluster/gecko_taskgraph/util/docker.py2
-rw-r--r--taskcluster/gecko_taskgraph/util/hg.py10
-rw-r--r--taskcluster/gecko_taskgraph/util/partners.py2
-rw-r--r--taskcluster/gecko_taskgraph/util/perfile.py13
-rw-r--r--taskcluster/gecko_taskgraph/util/platforms.py3
-rw-r--r--taskcluster/gecko_taskgraph/util/signed_artifacts.py6
-rw-r--r--taskcluster/gecko_taskgraph/util/taskcluster.py2
67 files changed, 1073 insertions, 964 deletions
diff --git a/taskcluster/gecko_taskgraph/__init__.py b/taskcluster/gecko_taskgraph/__init__.py
index f1de1e9120..c169eae023 100644
--- a/taskcluster/gecko_taskgraph/__init__.py
+++ b/taskcluster/gecko_taskgraph/__init__.py
@@ -51,12 +51,23 @@ def register(graph_config):
Args:
graph_config: The graph configuration object.
"""
+ import android_taskgraph
from taskgraph import generator
+ # TODO: Remove along with
+ # `gecko_taskgraph.optimize.strategies.SkipUnlessChanged`
+ # (see comment over there)
+ from taskgraph.optimize.base import registry
+
+ del registry["skip-unless-changed"]
+
from gecko_taskgraph import ( # noqa: trigger target task method registration
morph, # noqa: trigger morph registration
target_tasks,
)
+
+ android_taskgraph.register(graph_config)
+
from gecko_taskgraph.parameters import register_parameters
from gecko_taskgraph.util import dependencies # noqa: trigger group_by registration
from gecko_taskgraph.util.verify import verifications
diff --git a/taskcluster/gecko_taskgraph/config.py b/taskcluster/gecko_taskgraph/config.py
index 5045963b46..df4a5665e3 100644
--- a/taskcluster/gecko_taskgraph/config.py
+++ b/taskcluster/gecko_taskgraph/config.py
@@ -4,6 +4,7 @@
from taskgraph.util.schema import Schema, optionally_keyed_by
from voluptuous import Any, Optional, Required
+from voluptuous.validators import Length
graph_config_schema = Schema(
{
@@ -20,7 +21,7 @@ graph_config_schema = Schema(
Required("product-dir"): str,
Required("treeherder"): {
# Mapping of treeherder group symbols to descriptive names
- Required("group-names"): {str: str}
+ Required("group-names"): {str: Length(max=100)}
},
Required("index"): {Required("products"): [str]},
Required("try"): {
@@ -105,13 +106,8 @@ graph_config_schema = Schema(
}
},
},
- Required("mac-notarization"): {
- Required("mac-entitlements"): optionally_keyed_by(
- "platform", "release-level", str
- ),
- Required("mac-requirements"): optionally_keyed_by("platform", str),
- },
Required("mac-signing"): {
+ Required("mac-requirements"): optionally_keyed_by("platform", str),
Required("hardened-sign-config"): optionally_keyed_by(
"hardened-signing-type",
[
@@ -128,7 +124,7 @@ graph_config_schema = Schema(
Required("globs"): [str],
}
],
- )
+ ),
},
Required("taskgraph"): {
Optional(
diff --git a/taskcluster/gecko_taskgraph/decision.py b/taskcluster/gecko_taskgraph/decision.py
index e0bc9e3ca8..9fd8a5b5c1 100644
--- a/taskcluster/gecko_taskgraph/decision.py
+++ b/taskcluster/gecko_taskgraph/decision.py
@@ -32,6 +32,7 @@ from taskgraph.util.yaml import load_yaml
from . import GECKO
from .actions import render_actions_json
+from .files_changed import get_changed_files
from .parameters import get_app_version, get_version
from .try_option_syntax import parse_message
from .util.backstop import BACKSTOP_INDEX, is_backstop
@@ -308,6 +309,9 @@ def get_decision_parameters(graph_config, options):
parameters["hg_branch"] = get_hg_revision_branch(
GECKO, revision=parameters["head_rev"]
)
+ parameters["files_changed"] = sorted(
+ get_changed_files(parameters["head_repository"], parameters["head_rev"])
+ )
parameters["next_version"] = None
parameters["optimize_strategies"] = None
parameters["optimize_target_tasks"] = True
@@ -448,14 +452,17 @@ def set_try_config(parameters, task_config_file):
def set_decision_indexes(decision_task_id, params, graph_config):
index_paths = []
if params["backstop"]:
- index_paths.append(BACKSTOP_INDEX)
+ # When two Decision tasks run at nearly the same time, it's possible
+ # they both end up being backstops if the second checks the backstop
+ # index before the first inserts it. Insert this index first to reduce
+ # the chances of that happening.
+ index_paths.insert(0, BACKSTOP_INDEX)
subs = params.copy()
subs["trust-domain"] = graph_config["trust-domain"]
- index_paths = [i.format(**subs) for i in index_paths]
for index_path in index_paths:
- insert_index(index_path, decision_task_id, use_proxy=True)
+ insert_index(index_path.format(**subs), decision_task_id, use_proxy=True)
def write_artifact(filename, data):
diff --git a/taskcluster/gecko_taskgraph/files_changed.py b/taskcluster/gecko_taskgraph/files_changed.py
index c814df0806..0352fb9d7e 100644
--- a/taskcluster/gecko_taskgraph/files_changed.py
+++ b/taskcluster/gecko_taskgraph/files_changed.py
@@ -16,7 +16,7 @@ from mozpack.path import match as mozpackmatch
from mozversioncontrol import InvalidRepoPath, get_repository_object
from gecko_taskgraph import GECKO
-from gecko_taskgraph.util.hg import get_json_automationrelevance
+from gecko_taskgraph.util.hg import get_json_pushchangedfiles
logger = logging.getLogger(__name__)
@@ -27,9 +27,8 @@ def get_changed_files(repository, revision):
Get the set of files changed in the push headed by the given revision.
Responses are cached, so multiple calls with the same arguments are OK.
"""
- contents = get_json_automationrelevance(repository, revision)
try:
- changesets = contents["changesets"]
+ return get_json_pushchangedfiles(repository, revision)["files"]
except KeyError:
# We shouldn't hit this error in CI.
if os.environ.get("MOZ_AUTOMATION"):
@@ -39,17 +38,6 @@ def get_changed_files(repository, revision):
# version control.
return get_locally_changed_files(GECKO)
- logger.debug("{} commits influencing task scheduling:".format(len(changesets)))
- changed_files = set()
- for c in changesets:
- desc = "" # Support empty desc
- if c["desc"]:
- desc = c["desc"].splitlines()[0].encode("ascii", "ignore")
- logger.debug(" {cset} {desc}".format(cset=c["node"][0:12], desc=desc))
- changed_files |= set(c["files"])
-
- return changed_files
-
def check(params, file_patterns):
"""Determine whether any of the files changed in the indicated push to
diff --git a/taskcluster/gecko_taskgraph/main.py b/taskcluster/gecko_taskgraph/main.py
index e9a353f246..c6d7a4a3c8 100644
--- a/taskcluster/gecko_taskgraph/main.py
+++ b/taskcluster/gecko_taskgraph/main.py
@@ -21,6 +21,9 @@ from typing import Any, List
import appdirs
import yaml
+from gecko_taskgraph import GECKO
+from gecko_taskgraph.files_changed import get_locally_changed_files
+
Command = namedtuple("Command", ["func", "args", "kwargs", "defaults"])
commands = {}
@@ -105,7 +108,7 @@ def get_filtered_taskgraph(taskgraph, tasksregex, exclude_keys):
for key in exclude_keys:
obj = task_dict
attrs = key.split(".")
- while attrs[0] in obj:
+ while obj and attrs[0] in obj:
if len(attrs) == 1:
del obj[attrs[0]]
break
@@ -130,7 +133,7 @@ def get_taskgraph_generator(root, parameters):
return TaskGraphGenerator(root_dir=root, parameters=parameters)
-def format_taskgraph(options, parameters, logfile=None):
+def format_taskgraph(options, parameters, overrides, logfile=None):
import taskgraph
from taskgraph.parameters import parameters_loader
@@ -148,7 +151,7 @@ def format_taskgraph(options, parameters, logfile=None):
if isinstance(parameters, str):
parameters = parameters_loader(
parameters,
- overrides={"target-kinds": options.get("target_kinds")},
+ overrides=overrides,
strict=False,
)
@@ -182,7 +185,7 @@ def dump_output(out, path=None, params_spec=None):
print(out + "\n", file=fh)
-def generate_taskgraph(options, parameters, logdir):
+def generate_taskgraph(options, parameters, overrides, logdir):
from taskgraph.parameters import Parameters
def logfile(spec):
@@ -198,14 +201,16 @@ def generate_taskgraph(options, parameters, logdir):
# tracebacks a little more readable and avoids additional process overhead.
if len(parameters) == 1:
spec = parameters[0]
- out = format_taskgraph(options, spec, logfile(spec))
+ out = format_taskgraph(options, spec, overrides, logfile(spec))
dump_output(out, options["output_file"])
return
futures = {}
with ProcessPoolExecutor(max_workers=options["max_workers"]) as executor:
for spec in parameters:
- f = executor.submit(format_taskgraph, options, spec, logfile(spec))
+ f = executor.submit(
+ format_taskgraph, options, spec, overrides, logfile(spec)
+ )
futures[f] = spec
for future in as_completed(futures):
@@ -299,6 +304,15 @@ def generate_taskgraph(options, parameters, logdir):
"specified).",
)
@argument(
+ "--force-local-files-changed",
+ default=False,
+ action="store_true",
+ help="Compute the 'files-changed' parameter from local version control, "
+ "even when explicitly using a parameter set that already has it defined. "
+ "Note that this is already the default behaviour when no parameters are "
+ "specified.",
+)
+@argument(
"--no-optimize",
dest="optimize",
action="store_false",
@@ -372,10 +386,10 @@ def show_taskgraph(options):
output_file = options["output_file"]
if options["diff"]:
- # --root argument is taskgraph's config at <repo>/taskcluster/ci
+ # --root argument is taskgraph's config at <repo>/taskcluster
repo_root = os.getcwd()
if options["root"]:
- repo_root = f"{options['root']}/../.."
+ repo_root = f"{options['root']}/.."
repo = get_repository(repo_root)
if not repo.working_directory_clean():
@@ -400,15 +414,21 @@ def show_taskgraph(options):
)
print(f"Generating {options['graph_attr']} @ {cur_ref}", file=sys.stderr)
+ overrides = {
+ "target-kinds": options.get("target_kinds"),
+ }
parameters: List[Any[str, Parameters]] = options.pop("parameters")
if not parameters:
- overrides = {
- "target-kinds": options.get("target_kinds"),
- }
parameters = [
parameters_loader(None, strict=False, overrides=overrides)
] # will use default values
+ # This is the default behaviour anyway, so no need to re-compute.
+ options["force_local_files_changed"] = False
+
+ elif options["force_local_files_changed"]:
+ overrides["files-changed"] = sorted(get_locally_changed_files(GECKO))
+
for param in parameters[:]:
if isinstance(param, str) and os.path.isdir(param):
parameters.remove(param)
@@ -434,7 +454,7 @@ def show_taskgraph(options):
# to setup its `mach` based logging.
setup_logging()
- generate_taskgraph(options, parameters, logdir)
+ generate_taskgraph(options, parameters, overrides, logdir)
if options["diff"]:
assert diffdir is not None
@@ -464,7 +484,7 @@ def show_taskgraph(options):
diffdir, f"{options['graph_attr']}_{base_ref}"
)
print(f"Generating {options['graph_attr']} @ {base_ref}", file=sys.stderr)
- generate_taskgraph(options, parameters, logdir)
+ generate_taskgraph(options, parameters, overrides, logdir)
finally:
repo.update(cur_ref)
@@ -687,7 +707,7 @@ def decision(options):
@argument(
"--root",
"-r",
- default="taskcluster/ci",
+ default="taskcluster",
help="root of the taskgraph definition relative to topsrcdir",
)
def action_callback(options):
@@ -723,7 +743,7 @@ def action_callback(options):
@argument(
"--root",
"-r",
- default="taskcluster/ci",
+ default="taskcluster",
help="root of the taskgraph definition relative to topsrcdir",
)
@argument(
diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml b/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml
index fb58439509..d847b4fcdc 100644
--- a/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml
+++ b/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml
@@ -62,6 +62,8 @@ platform_names:
linux64-shippable: 'linux-x86_64'
linux64-devedition: 'linux-x86_64'
linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ linux64-aarch64-shippable: 'linux-aarch64'
+ linux64-aarch64-devedition: 'linux-aarch64'
macosx64-shippable: 'mac'
macosx64-devedition: 'mac'
win32-shippable: 'win32'
@@ -78,6 +80,8 @@ platform_names:
linux64-shippable: 'linux64'
linux64-devedition: 'linux64-devedition'
linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ linux64-aarch64-shippable: 'linux64-aarch64'
+ linux64-aarch64-devedition: 'linux64-aarch64-devedition'
macosx64-shippable: 'macosx64'
macosx64-devedition: 'macosx64-devedition'
win32-shippable: 'win32'
@@ -94,6 +98,8 @@ platform_names:
linux64-shippable: 'linux64'
linux64-devedition: 'linux64'
linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ linux64-aarch64-shippable: 'linux64-aarch64'
+ linux64-aarch64-devedition: 'linux64-aarch64'
macosx64-shippable: 'macosx64'
macosx64-devedition: 'macosx64'
win32-shippable: 'win32'
@@ -391,6 +397,8 @@ mapping:
- win32-shippable
- win64-devedition
- win32-devedition
+ - win64-aarch64-shippable
+ - win64-aarch64-devedition
locale_prefix: 'multi/'
pretty_name: Firefox Setup ${version}.msix
checksums_path: ${path_platform}/multi/Firefox Setup ${version}.msix
diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml b/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml
index 43ba4cbf15..7ca17662e2 100644
--- a/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml
+++ b/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml
@@ -51,6 +51,8 @@ platform_names:
linux64-shippable: 'linux-x86_64'
linux64-devedition: 'linux-x86_64'
linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ linux64-aarch64-shippable: 'linux-aarch64'
+ linux64-aarch64-devedition: 'linux-aarch64'
macosx64-shippable: 'mac'
macosx64-devedition: 'mac'
win32-shippable: 'win32'
diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml b/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml
index d413ede3bd..60c39e7c53 100644
--- a/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml
+++ b/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml
@@ -57,6 +57,8 @@ platform_names:
linux-devedition: 'linux-i686'
linux64-shippable: 'linux-x86_64'
linux64-devedition: 'linux-x86_64'
+ linux64-aarch64-shippable: 'linux-aarch64'
+ linux64-aarch64-devedition: 'linux-aarch64'
linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
macosx64-shippable: 'mac'
macosx64-devedition: 'mac'
@@ -74,6 +76,8 @@ platform_names:
linux64-asan-reporter-shippable: 'linux64-asan-reporter'
linux64-shippable: 'linux64'
linux64-devedition: 'linux64'
+ linux64-aarch64-shippable: 'linux64-aarch64'
+ linux64-aarch64-devedition: 'linux64-aarch64'
macosx64-shippable: 'macosx64'
macosx64-devedition: 'macosx64'
win32-shippable: 'win32'
@@ -127,19 +131,10 @@ mapping:
checksums_path: KEY
only_for_platforms:
- linux64-shippable
+ - linux64-aarch64-shippable
destinations:
- ${year}/${month}/${upload_date}-${branch}
- latest-${branch}
- target.common.tests.tar.gz:
- <<: *default
- description: "Mixture of reftests, mochitests, UI and others, commonly bundled together in a test suite"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.common.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.common.tests.tar.gz
- target.cppunittest.tests.tar.gz:
- <<: *default
- description: "C++ unittests related in-tree test infrastructure"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz
target.crashreporter-symbols.zip:
<<: *default
description: "Crashreporter symbols to be consumed by Socorro"
@@ -153,51 +148,16 @@ mapping:
description: "Various compile and moz_app flags baked together in a json file"
pretty_name: firefox-${version}.${locale}.${filename_platform}.json
checksums_path: firefox-${version}.${locale}.${filename_platform}.json
- target.mochitest.tests.tar.gz:
- <<: *default
- description: "Results for running the mochitest testing framework via Javascript function calls"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz
target.mozinfo.json:
<<: *default
description: "Various compile and moz_app flags baked together in a json file"
pretty_name: firefox-${version}.${locale}.${filename_platform}.mozinfo.json
checksums_path: firefox-${version}.${locale}.${filename_platform}.mozinfo.json
- target.reftest.tests.tar.gz:
- <<: *default
- description: "Results for running the reftest testing framework via display of two Web pages comparison"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz
- target.talos.tests.tar.gz:
- <<: *default
- description: "Results for running the talos testing framework to measure performance"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.talos.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.talos.tests.tar.gz
- target.awsy.tests.tar.gz:
- <<: *default
- description: "Results for running the awsy testing framework to track memory usage"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz
- target.test_packages.json:
- <<: *default
- description: "File containing metadata about all other files and testing harnesses specifics"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.test_packages.json
- checksums_path: firefox-${version}.${locale}.${filename_platform}.test_packages.json
target.txt:
<<: *default
description: "File containing buildid and revision"
pretty_name: firefox-${version}.${locale}.${filename_platform}.txt
checksums_path: firefox-${version}.${locale}.${filename_platform}.txt
- target.web-platform.tests.tar.gz:
- <<: *default
- description: "Results for running the webplatform testing framework to cover standard Web platform features"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz
- target.xpcshell.tests.tar.gz:
- <<: *default
- description: "Results for running the xpcshell testing framework to enable XPConnect console application"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz
target_info.txt:
<<: *default
description: "File containing the buildID"
@@ -226,6 +186,7 @@ mapping:
only_for_platforms:
- linux-shippable
- linux64-shippable
+ - linux64-aarch64-shippable
- macosx64-shippable
- win64-shippable
- win32-shippable
@@ -252,6 +213,8 @@ mapping:
only_for_platforms:
- linux-shippable
- linux64-shippable
+ - linux64-aarch64-shippable
+ - linux64-aarch64-devedition
- linux-devedition
- linux64-devedition
pretty_name: firefox-${version}.${locale}.langpack.deb
@@ -301,6 +264,7 @@ mapping:
only_for_platforms:
- linux-shippable
- linux64-shippable
+ - linux64-aarch64-shippable
- linux64-asan-reporter-shippable
pretty_name: firefox-${version}.${locale}.${filename_platform}.tar.bz2
checksums_path: firefox-${version}.${locale}.${filename_platform}.tar.bz2
@@ -322,6 +286,7 @@ mapping:
only_for_platforms:
- linux-shippable
- linux64-shippable
+ - linux64-aarch64-shippable
- linux64-asan-reporter-shippable
pretty_name: firefox-${version}.${locale}.${filename_platform}.tar.bz2.asc
checksums_path: firefox-${version}.${locale}.${filename_platform}.tar.bz2.asc
@@ -465,6 +430,7 @@ mapping:
- repackage-signing-shippable-l10n-msix
only_for_platforms:
- win64-shippable
+ - win64-aarch64-shippable
- win32-shippable
pretty_name: firefox-${version}.multi.${filename_platform}.installer.msix
checksums_path: firefox-${version}.multi.${filename_platform}.installer.msix
@@ -498,6 +464,7 @@ mapping:
only_for_platforms:
- linux-shippable
- linux64-shippable
+ - linux64-aarch64-shippable
pretty_name: firefox-${version}.${locale}.${filename_platform}.deb
checksums_path: firefox-${version}.${locale}.${filename_platform}.deb
update_balrog_manifest: false
diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml b/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml
index f1b81572ab..caa31f556e 100644
--- a/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml
+++ b/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml
@@ -18,6 +18,8 @@ platform_names:
linux-shippable: 'linux-i686'
linux-devedition: 'linux-i686'
linux64-shippable: 'linux-x86_64'
+ linux64-aarch64-shippable: 'linux-aarch64'
+ linux64-aarch64-devedition: 'linux-aarch64'
linux64-devedition: 'linux-x86_64'
linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
macosx64-shippable: 'mac'
diff --git a/taskcluster/gecko_taskgraph/optimize/mozlint.py b/taskcluster/gecko_taskgraph/optimize/mozlint.py
new file mode 100644
index 0000000000..0fe42ea70d
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/optimize/mozlint.py
@@ -0,0 +1,96 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+from pathlib import Path
+from typing import List, Union
+
+from mozlint.parser import Parser
+from mozlint.pathutils import filterpaths
+from taskgraph.optimize.base import OptimizationStrategy
+from taskgraph.util.path import match as match_path
+
+from gecko_taskgraph import GECKO
+
+logger = logging.getLogger(__name__)
+
+
+class TGMozlintParser(Parser):
+ """
+ Mozlint Parser that skips validation. This is needed because decision
+ tasks use sparse clones and the files themselves are not present.
+ """
+
+ def _validate(self, linter):
+ pass
+
+
+class SkipUnlessMozlint(OptimizationStrategy):
+ """
+ Optimization strategy for mozlint tests.
+
+ Uses the mozlint YAML file for each test to determine whether or not a test
+ should run.
+
+ Using:
+ - The optimization relies on having `files_changed` set in the decision task
+ parameters.
+ - Register with register_strategy. The argument is the path to MozLint YAML
+ configuration files ("/tools/lint" for mozilla-central):
+ - In source-test/mozlint.yml, set the optimization strategy for each job by filename:
+ - For Mozlint jobs that run multiple linters at once use a list of filenames:
+ """
+
+ def __init__(self, linters_path: str):
+ self.mozlint_parser = TGMozlintParser(GECKO)
+ self.linters_path = Path(GECKO) / linters_path
+
+ def should_remove_task(
+ self, task, params, mozlint_confs: Union[str, List[str]]
+ ) -> bool:
+ include = []
+ exclude = []
+ extensions = []
+ exclude_extensions = []
+ support_files = ["python/mozlint/**", "tools/lint/**"]
+
+ files_changed = params["files_changed"]
+
+ if isinstance(mozlint_confs, str):
+ mozlint_confs = [mozlint_confs]
+
+ for mozlint_conf in mozlint_confs:
+ mozlint_yaml = str(self.linters_path / mozlint_conf)
+ logger.info(f"Loading file patterns for {task.label} from {mozlint_yaml}.")
+ linter_config = self.mozlint_parser(mozlint_yaml)
+
+ for config in linter_config:
+ include += config.get("include", [])
+ exclude += config.get("exclude", [])
+ extensions += [e.strip(".") for e in config.get("extensions", [])]
+ exclude_extensions += [
+ e.strip(".") for e in config.get("exclude_extensions", [])
+ ]
+ support_files += config.get("support-files", [])
+
+ # Support files may not be part of "include" patterns, so check first
+ # Do not remove (return False) if any changed
+ for pattern in set(support_files):
+ for path in files_changed:
+ if match_path(path, pattern):
+ return False
+
+ to_lint, to_exclude = filterpaths(
+ GECKO,
+ list(files_changed),
+ include=include,
+ exclude=exclude,
+ extensions=extensions,
+ exclude_extensions=exclude_extensions,
+ )
+
+ # to_lint should be an empty list if there is nothing to check
+ if not to_lint:
+ return True
+ return False
diff --git a/taskcluster/gecko_taskgraph/optimize/schema.py b/taskcluster/gecko_taskgraph/optimize/schema.py
index a7f878cf60..a348fb177c 100644
--- a/taskcluster/gecko_taskgraph/optimize/schema.py
+++ b/taskcluster/gecko_taskgraph/optimize/schema.py
@@ -39,6 +39,8 @@ default_optimizations = (
{"upload-symbols": None},
# optimize strategy alias for reprocess-symbols tasks
{"reprocess-symbols": None},
+ # optimization strategy for mozlint tests
+ {"skip-unless-mozlint": voluptuous.Any(str, [str])},
)
OptimizationSchema = voluptuous.Any(*default_optimizations)
diff --git a/taskcluster/gecko_taskgraph/optimize/strategies.py b/taskcluster/gecko_taskgraph/optimize/strategies.py
index 4d0d23a5ac..f1ade38672 100644
--- a/taskcluster/gecko_taskgraph/optimize/strategies.py
+++ b/taskcluster/gecko_taskgraph/optimize/strategies.py
@@ -9,8 +9,9 @@ import mozpack.path as mozpath
from mozbuild.base import MozbuildObject
from mozbuild.util import memoize
from taskgraph.optimize.base import OptimizationStrategy, register_strategy
+from taskgraph.util.path import match as match_path
-from gecko_taskgraph import files_changed
+from gecko_taskgraph.optimize.mozlint import SkipUnlessMozlint
logger = logging.getLogger(__name__)
@@ -18,9 +19,7 @@ logger = logging.getLogger(__name__)
@register_strategy("skip-unless-schedules")
class SkipUnlessSchedules(OptimizationStrategy):
@memoize
- def scheduled_by_push(self, repository, revision):
- changed_files = files_changed.get_changed_files(repository, revision)
-
+ def scheduled_by_push(self, files_changed):
mbo = MozbuildObject.from_environment()
# the decision task has a sparse checkout, so, mozbuild_reader will use
# a MercurialRevisionFinder with revision '.', which should be the same
@@ -28,7 +27,7 @@ class SkipUnlessSchedules(OptimizationStrategy):
rdr = mbo.mozbuild_reader(config_mode="empty")
components = set()
- for p, m in rdr.files_info(changed_files).items():
+ for p, m in rdr.files_info(files_changed).items():
components |= set(m["SCHEDULES"].components)
return components
@@ -37,9 +36,7 @@ class SkipUnlessSchedules(OptimizationStrategy):
if params.get("pushlog_id") == -1:
return False
- scheduled = self.scheduled_by_push(
- params["head_repository"], params["head_rev"]
- )
+ scheduled = self.scheduled_by_push(frozenset(params["files_changed"]))
conditions = set(conditions)
# if *any* of the condition components are scheduled, do not optimize
if conditions & scheduled:
@@ -55,8 +52,8 @@ class SkipUnlessHasRelevantTests(OptimizationStrategy):
"""
@memoize
- def get_changed_dirs(self, repo, rev):
- changed = map(mozpath.dirname, files_changed.get_changed_files(repo, rev))
+ def get_changed_dirs(self, files_changed):
+ changed = map(mozpath.dirname, files_changed)
# Filter out empty directories (from files modified in the root).
# Otherwise all tasks would be scheduled.
return {d for d in changed if d}
@@ -65,7 +62,7 @@ class SkipUnlessHasRelevantTests(OptimizationStrategy):
if not task.attributes.get("test_manifests"):
return True
- for d in self.get_changed_dirs(params["head_repository"], params["head_rev"]):
+ for d in self.get_changed_dirs(frozenset(params["files_changed"])):
for t in task.attributes["test_manifests"]:
if t.startswith(d):
logger.debug(
@@ -75,3 +72,36 @@ class SkipUnlessHasRelevantTests(OptimizationStrategy):
)
return False
return True
+
+
+# TODO: This overwrites upstream Taskgraph's `skip-unless-changed`
+# optimization. Once the firefox-android migration is landed and we upgrade
+# upstream Taskgraph to a version that doesn't call files_changed.check`, this
+# class can be deleted. Also remove the `taskgraph.optimize.base.registry` tweak
+# in `gecko_taskgraph.register` at the same time.
+@register_strategy("skip-unless-changed")
+class SkipUnlessChanged(OptimizationStrategy):
+ def check(self, files_changed, patterns):
+ for pattern in patterns:
+ for path in files_changed:
+ if match_path(path, pattern):
+ return True
+ return False
+
+ def should_remove_task(self, task, params, file_patterns):
+ # pushlog_id == -1 - this is the case when run from a cron.yml job or on a git repository
+ if params.get("repository_type") == "hg" and params.get("pushlog_id") == -1:
+ return False
+
+ changed = self.check(params["files_changed"], file_patterns)
+ if not changed:
+ logger.debug(
+ 'no files found matching a pattern in `skip-unless-changed` for "{}"'.format(
+ task.label
+ )
+ )
+ return True
+ return False
+
+
+register_strategy("skip-unless-mozlint", args=("tools/lint",))(SkipUnlessMozlint)
diff --git a/taskcluster/gecko_taskgraph/parameters.py b/taskcluster/gecko_taskgraph/parameters.py
index 7e3de1372f..c9cca8acf1 100644
--- a/taskcluster/gecko_taskgraph/parameters.py
+++ b/taskcluster/gecko_taskgraph/parameters.py
@@ -9,6 +9,7 @@ from taskgraph.parameters import extend_parameters_schema
from voluptuous import Any, Optional, Required
from gecko_taskgraph import GECKO
+from gecko_taskgraph.files_changed import get_locally_changed_files
logger = logging.getLogger(__name__)
@@ -18,6 +19,7 @@ gecko_parameters_schema = {
Required("backstop"): bool,
Required("build_number"): int,
Required("enable_always_target"): Any(bool, [str]),
+ Required("files_changed"): [str],
Required("hg_branch"): str,
Required("message"): str,
Required("next_version"): Any(None, str),
@@ -107,6 +109,7 @@ def get_defaults(repo_root=None):
"base_repository": "https://hg.mozilla.org/mozilla-unified",
"build_number": 1,
"enable_always_target": ["docker-image"],
+ "files_changed": sorted(get_locally_changed_files(repo_root)),
"head_repository": "https://hg.mozilla.org/mozilla-central",
"hg_branch": "default",
"message": "",
diff --git a/taskcluster/gecko_taskgraph/target_tasks.py b/taskcluster/gecko_taskgraph/target_tasks.py
index fcbfab4e17..5469f9eb0d 100644
--- a/taskcluster/gecko_taskgraph/target_tasks.py
+++ b/taskcluster/gecko_taskgraph/target_tasks.py
@@ -11,7 +11,7 @@ from datetime import datetime, timedelta
from redo import retry
from taskgraph.parameters import Parameters
-from taskgraph.target_tasks import _target_task, get_method
+from taskgraph.target_tasks import get_method, register_target_task
from taskgraph.util.taskcluster import find_task_id
from gecko_taskgraph import GECKO, try_option_syntax
@@ -37,13 +37,13 @@ UNCOMMON_TRY_TASK_LABELS = [
r"android-geckoview-docs",
r"android-hw",
# Windows tasks
- r"windows10-64-ref-hw",
+ r"windows11-64-2009-hw-ref",
r"windows10-aarch64-qr",
# Linux tasks
r"linux-", # hide all linux32 tasks by default - bug 1599197
r"linux1804-32", # hide linux32 tests - bug 1599197
# Test tasks
- r"web-platform-tests.*backlog", # hide wpt jobs that are not implemented yet - bug 1572820
+ r"web-platform-tests(?!-webgpu).*backlog", # hide wpt jobs that are not implemented yet - bug 1572820
r"-ccov",
r"-profiling-", # talos/raptor profiling jobs are run too often
r"-32-.*-webgpu", # webgpu gets little benefit from these tests.
@@ -52,6 +52,9 @@ UNCOMMON_TRY_TASK_LABELS = [
# Hide shippable versions of tests we have opt versions of because the non-shippable
# versions are faster to run. This is mostly perf tests.
r"-shippable(?!.*(awsy|browsertime|marionette-headless|mochitest-devtools-chrome-fis|raptor|talos|web-platform-tests-wdspec-headless|mochitest-plain-headless))", # noqa - too long
+ r"nightly-simulation",
+ # Can't actually run on try
+ r"notarization",
]
@@ -393,7 +396,7 @@ def _try_option_syntax(full_task_graph, parameters, graph_config):
return target_tasks_labels
-@_target_task("try_tasks")
+@register_target_task("try_tasks")
def target_tasks_try(full_task_graph, parameters, graph_config):
try_mode = parameters["try_mode"]
if try_mode == "try_task_config":
@@ -405,13 +408,13 @@ def target_tasks_try(full_task_graph, parameters, graph_config):
return []
-@_target_task("try_select_tasks")
+@register_target_task("try_select_tasks")
def target_tasks_try_select(full_task_graph, parameters, graph_config):
tasks = target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config)
return [l for l in tasks if filter_by_uncommon_try_tasks(l)]
-@_target_task("try_select_tasks_uncommon")
+@register_target_task("try_select_tasks_uncommon")
def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config):
from gecko_taskgraph.decision import PER_PROJECT_PARAMETERS
@@ -437,7 +440,7 @@ def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config):
return sorted(tasks)
-@_target_task("try_auto")
+@register_target_task("try_auto")
def target_tasks_try_auto(full_task_graph, parameters, graph_config):
"""Target the tasks which have indicated they should be run on autoland
(rather than try) via the `run_on_projects` attributes.
@@ -468,7 +471,7 @@ def target_tasks_try_auto(full_task_graph, parameters, graph_config):
]
-@_target_task("default")
+@register_target_task("default")
def target_tasks_default(full_task_graph, parameters, graph_config):
"""Target the tasks which have indicated they should be run on this project
via the `run_on_projects` attributes."""
@@ -481,7 +484,7 @@ def target_tasks_default(full_task_graph, parameters, graph_config):
]
-@_target_task("autoland_tasks")
+@register_target_task("autoland_tasks")
def target_tasks_autoland(full_task_graph, parameters, graph_config):
"""In addition to doing the filtering by project that the 'default'
filter does, also remove any tests running against shippable builds
@@ -507,7 +510,7 @@ def target_tasks_autoland(full_task_graph, parameters, graph_config):
return [l for l in filtered_for_project if filter(full_task_graph[l])]
-@_target_task("mozilla_central_tasks")
+@register_target_task("mozilla_central_tasks")
def target_tasks_mozilla_central(full_task_graph, parameters, graph_config):
"""In addition to doing the filtering by project that the 'default'
filter does, also remove any tests running against regular (aka not shippable,
@@ -547,7 +550,7 @@ def target_tasks_mozilla_central(full_task_graph, parameters, graph_config):
return [l for l in filtered_for_project if filter(full_task_graph[l])]
-@_target_task("graphics_tasks")
+@register_target_task("graphics_tasks")
def target_tasks_graphics(full_task_graph, parameters, graph_config):
"""In addition to doing the filtering by project that the 'default'
filter does, also remove artifact builds because we have csets on
@@ -565,7 +568,7 @@ def target_tasks_graphics(full_task_graph, parameters, graph_config):
return [l for l in filtered_for_project if filter(full_task_graph[l])]
-@_target_task("mozilla_beta_tasks")
+@register_target_task("mozilla_beta_tasks")
def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a promotable beta or release build
of desktop, plus android CI. The candidates build process involves a pipeline
@@ -578,7 +581,7 @@ def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config):
]
-@_target_task("mozilla_release_tasks")
+@register_target_task("mozilla_release_tasks")
def target_tasks_mozilla_release(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a promotable beta or release build
of desktop, plus android CI. The candidates build process involves a pipeline
@@ -591,7 +594,7 @@ def target_tasks_mozilla_release(full_task_graph, parameters, graph_config):
]
-@_target_task("mozilla_esr115_tasks")
+@register_target_task("mozilla_esr115_tasks")
def target_tasks_mozilla_esr115(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a promotable beta or release build
of desktop, without android CI. The candidates build process involves a pipeline
@@ -615,19 +618,13 @@ def target_tasks_mozilla_esr115(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("promote_desktop")
+@register_target_task("promote_desktop")
def target_tasks_promote_desktop(full_task_graph, parameters, graph_config):
"""Select the superset of tasks required to promote a beta or release build
of a desktop product. This should include all non-android
mozilla_{beta,release} tasks, plus l10n, beetmover, balrog, etc."""
def filter(task):
- # Bug 1758507 - geckoview ships in the promote phase
- if not parameters["release_type"].startswith("esr") and is_geckoview(
- task, parameters
- ):
- return True
-
if task.attributes.get("shipping_product") != parameters["release_product"]:
return False
@@ -645,15 +642,7 @@ def target_tasks_promote_desktop(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-def is_geckoview(task, parameters):
- return (
- task.attributes.get("shipping_product") == "fennec"
- and task.kind in ("beetmover-geckoview", "upload-symbols")
- and parameters["release_product"] == "firefox"
- )
-
-
-@_target_task("push_desktop")
+@register_target_task("push_desktop")
def target_tasks_push_desktop(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to push a build of desktop to cdns.
Previous build deps will be optimized out via action task."""
@@ -679,7 +668,7 @@ def target_tasks_push_desktop(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("ship_desktop")
+@register_target_task("ship_desktop")
def target_tasks_ship_desktop(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to ship desktop.
Previous build deps will be optimized out via action task."""
@@ -720,7 +709,7 @@ def target_tasks_ship_desktop(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("pine_tasks")
+@register_target_task("pine_tasks")
def target_tasks_pine(full_task_graph, parameters, graph_config):
"""Bug 1879960 - no reftests or wpt needed"""
filtered_for_project = target_tasks_default(
@@ -728,6 +717,8 @@ def target_tasks_pine(full_task_graph, parameters, graph_config):
)
def filter(task):
+ if "android" in task.attributes.get("build_platform", ""):
+ return False
suite = task.attributes.get("unittest_suite", "")
if "reftest" in suite or "web-platform" in suite:
return False
@@ -736,7 +727,7 @@ def target_tasks_pine(full_task_graph, parameters, graph_config):
return [l for l in filtered_for_project if filter(full_task_graph[l])]
-@_target_task("larch_tasks")
+@register_target_task("larch_tasks")
def target_tasks_larch(full_task_graph, parameters, graph_config):
"""Bug 1879213 - only run necessary tasks on larch"""
filtered_for_project = target_tasks_default(
@@ -760,7 +751,7 @@ def target_tasks_larch(full_task_graph, parameters, graph_config):
return [l for l in filtered_for_project if filter(full_task_graph[l])]
-@_target_task("kaios_tasks")
+@register_target_task("kaios_tasks")
def target_tasks_kaios(full_task_graph, parameters, graph_config):
"""The set of tasks to run for kaios integration"""
@@ -771,36 +762,7 @@ def target_tasks_kaios(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("ship_geckoview")
-def target_tasks_ship_geckoview(full_task_graph, parameters, graph_config):
- """Select the set of tasks required to ship geckoview nightly. The
- nightly build process involves a pipeline of builds and an upload to
- maven.mozilla.org."""
- index_path = (
- f"{graph_config['trust-domain']}.v2.{parameters['project']}.revision."
- f"{parameters['head_rev']}.taskgraph.decision-ship-geckoview"
- )
- if os.environ.get("MOZ_AUTOMATION") and retry(
- index_exists,
- args=(index_path,),
- kwargs={
- "reason": "to avoid triggering multiple nightlies off the same revision",
- },
- ):
- return []
-
- def filter(task):
- # XXX Starting 69, we don't ship Fennec Nightly anymore. We just want geckoview to be
- # uploaded
- return task.attributes.get("shipping_product") == "fennec" and task.kind in (
- "beetmover-geckoview",
- "upload-symbols",
- )
-
- return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-
-
-@_target_task("custom-car_perf_testing")
+@register_target_task("custom-car_perf_testing")
def target_tasks_custom_car_perf_testing(full_task_graph, parameters, graph_config):
"""Select tasks required for running daily performance tests for custom chromium-as-release."""
@@ -827,7 +789,7 @@ def target_tasks_custom_car_perf_testing(full_task_graph, parameters, graph_conf
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("general_perf_testing")
+@register_target_task("general_perf_testing")
def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config):
"""
Select tasks required for running performance tests 3 times a week.
@@ -844,6 +806,9 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config)
if "tp6-bench" in try_name:
return False
+ if "tp7" in try_name:
+ return False
+
# Bug 1867669 - Temporarily disable all live site tests
if "live" in try_name and "sheriffed" not in try_name:
return False
@@ -856,11 +821,7 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config)
if "tp6" in try_name and "essential" not in try_name:
return False
return True
- if "chromium" in try_name:
- if "tp6" in try_name and "essential" not in try_name:
- return False
- return True
- # chromium-as-release has it's own cron
+ # chromium-as-release has its own cron
if "custom-car" in try_name:
return False
if "-live" in try_name:
@@ -920,8 +881,13 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config)
# Don't run android CaR sp tests as we already have a cron for this.
if "m-car" in try_name:
return False
+ if "fenix" in try_name:
+ return False
if "speedometer" in try_name:
return True
+ if "motionmark" in try_name and "1-3" in try_name:
+ if "chrome-m" in try_name:
+ return True
return False
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
@@ -946,7 +912,7 @@ def make_desktop_nightly_filter(platforms):
return filter
-@_target_task("sp-perftests")
+@register_target_task("sp-perftests")
def target_tasks_speedometer_tests(full_task_graph, parameters, graph_config):
def filter(task):
platform = task.attributes.get("test_platform")
@@ -970,18 +936,18 @@ def target_tasks_speedometer_tests(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("nightly_linux")
+@register_target_task("nightly_linux")
def target_tasks_nightly_linux(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of linux. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
filter = make_desktop_nightly_filter(
- {"linux64-shippable", "linux-shippable", "linux-aarch64-shippable"}
+ {"linux64-shippable", "linux-shippable", "linux64-aarch64-shippable"}
)
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("nightly_macosx")
+@register_target_task("nightly_macosx")
def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of macosx. The
nightly build process involves a pipeline of builds, signing,
@@ -990,7 +956,7 @@ def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("nightly_win32")
+@register_target_task("nightly_win32")
def target_tasks_nightly_win32(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of win32 and win64.
The nightly build process involves a pipeline of builds, signing,
@@ -999,7 +965,7 @@ def target_tasks_nightly_win32(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("nightly_win64")
+@register_target_task("nightly_win64")
def target_tasks_nightly_win64(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of win32 and win64.
The nightly build process involves a pipeline of builds, signing,
@@ -1008,7 +974,7 @@ def target_tasks_nightly_win64(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("nightly_win64_aarch64")
+@register_target_task("nightly_win64_aarch64")
def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of win32 and win64.
The nightly build process involves a pipeline of builds, signing,
@@ -1017,7 +983,7 @@ def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("nightly_asan")
+@register_target_task("nightly_asan")
def target_tasks_nightly_asan(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of asan. The
nightly build process involves a pipeline of builds, signing,
@@ -1028,7 +994,7 @@ def target_tasks_nightly_asan(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("daily_releases")
+@register_target_task("daily_releases")
def target_tasks_daily_releases(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to identify if we should release.
If we determine that we should the task will communicate to ship-it to
@@ -1040,7 +1006,7 @@ def target_tasks_daily_releases(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("nightly_desktop")
+@register_target_task("nightly_desktop")
def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of linux, mac,
windows."""
@@ -1078,8 +1044,30 @@ def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config):
)
+@register_target_task("nightly_all")
+def target_tasks_nightly_all(full_task_graph, parameters, graph_config):
+ """Select the set of tasks required for a nightly build of firefox desktop and android"""
+ index_path = (
+ f"{graph_config['trust-domain']}.v2.{parameters['project']}.revision."
+ f"{parameters['head_rev']}.taskgraph.decision-nightly-all"
+ )
+ if os.environ.get("MOZ_AUTOMATION") and retry(
+ index_exists,
+ args=(index_path,),
+ kwargs={
+ "reason": "to avoid triggering multiple nightlies off the same revision",
+ },
+ ):
+ return []
+
+ return list(
+ set(target_tasks_nightly_desktop(full_task_graph, parameters, graph_config))
+ | set(target_tasks_nightly_android(full_task_graph, parameters, graph_config))
+ )
+
+
# Run Searchfox analysis once daily.
-@_target_task("searchfox_index")
+@register_target_task("searchfox_index")
def target_tasks_searchfox(full_task_graph, parameters, graph_config):
"""Select tasks required for indexing Firefox for Searchfox web site each day"""
return [
@@ -1095,7 +1083,7 @@ def target_tasks_searchfox(full_task_graph, parameters, graph_config):
# Run build linux64-plain-clang-trunk/opt on mozilla-central/beta with perf tests
-@_target_task("linux64_clang_trunk_perf")
+@register_target_task("linux64_clang_trunk_perf")
def target_tasks_build_linux64_clang_trunk_perf(
full_task_graph, parameters, graph_config
):
@@ -1111,19 +1099,19 @@ def target_tasks_build_linux64_clang_trunk_perf(
# Run Updatebot's cron job 4 times daily.
-@_target_task("updatebot_cron")
+@register_target_task("updatebot_cron")
def target_tasks_updatebot_cron(full_task_graph, parameters, graph_config):
"""Select tasks required to run Updatebot's cron job"""
return ["updatebot-cron"]
-@_target_task("customv8_update")
+@register_target_task("customv8_update")
def target_tasks_customv8_update(full_task_graph, parameters, graph_config):
"""Select tasks required for building latest d8/v8 version."""
return ["toolchain-linux64-custom-v8"]
-@_target_task("file_update")
+@register_target_task("file_update")
def target_tasks_file_update(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to perform nightly in-tree file updates"""
@@ -1134,7 +1122,7 @@ def target_tasks_file_update(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("l10n_bump")
+@register_target_task("l10n_bump")
def target_tasks_l10n_bump(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to perform l10n bumping."""
@@ -1145,7 +1133,7 @@ def target_tasks_l10n_bump(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("merge_automation")
+@register_target_task("merge_automation")
def target_tasks_merge_automation(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to perform repository merges."""
@@ -1156,7 +1144,7 @@ def target_tasks_merge_automation(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("scriptworker_canary")
+@register_target_task("scriptworker_canary")
def target_tasks_scriptworker_canary(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to run scriptworker canaries."""
@@ -1167,7 +1155,7 @@ def target_tasks_scriptworker_canary(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("cron_bouncer_check")
+@register_target_task("cron_bouncer_check")
def target_tasks_bouncer_check(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to perform bouncer version verification."""
@@ -1180,7 +1168,7 @@ def target_tasks_bouncer_check(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("staging_release_builds")
+@register_target_task("staging_release_builds")
def target_tasks_staging_release(full_task_graph, parameters, graph_config):
"""
Select all builds that are part of releases.
@@ -1204,7 +1192,7 @@ def target_tasks_staging_release(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("release_simulation")
+@register_target_task("release_simulation")
def target_tasks_release_simulation(full_task_graph, parameters, graph_config):
"""
Select builds that would run on push on a release branch.
@@ -1241,7 +1229,7 @@ def target_tasks_release_simulation(full_task_graph, parameters, graph_config):
]
-@_target_task("codereview")
+@register_target_task("codereview")
def target_tasks_codereview(full_task_graph, parameters, graph_config):
"""Select all code review tasks needed to produce a report"""
@@ -1259,13 +1247,13 @@ def target_tasks_codereview(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("nothing")
+@register_target_task("nothing")
def target_tasks_nothing(full_task_graph, parameters, graph_config):
"""Select nothing, for DONTBUILD pushes"""
return []
-@_target_task("daily_beta_perf")
+@register_target_task("daily_beta_perf")
def target_tasks_daily_beta_perf(full_task_graph, parameters, graph_config):
"""
Select performance tests on the beta branch to be run daily
@@ -1367,7 +1355,7 @@ def target_tasks_daily_beta_perf(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("weekly_release_perf")
+@register_target_task("weekly_release_perf")
def target_tasks_weekly_release_perf(full_task_graph, parameters, graph_config):
"""
Select performance tests on the release branch to be run weekly
@@ -1435,7 +1423,7 @@ def target_tasks_weekly_release_perf(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("raptor_tp6m")
+@register_target_task("raptor_tp6m")
def target_tasks_raptor_tp6m(full_task_graph, parameters, graph_config):
"""
Select tasks required for running raptor cold page-load tests on fenix and refbrow
@@ -1456,14 +1444,13 @@ def target_tasks_raptor_tp6m(full_task_graph, parameters, graph_config):
"browsertime" in try_name
and "amazon" in try_name
and "search" not in try_name
- and "fenix" in try_name
):
return True
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("backfill_all_browsertime")
+@register_target_task("backfill_all_browsertime")
def target_tasks_backfill_all_browsertime(full_task_graph, parameters, graph_config):
"""
Search for revisions that contains patches that were reviewed by perftest reviewers
@@ -1531,7 +1518,7 @@ def target_tasks_backfill_all_browsertime(full_task_graph, parameters, graph_con
return []
-@_target_task("condprof")
+@register_target_task("condprof")
def target_tasks_condprof(full_task_graph, parameters, graph_config):
"""
Select tasks required for building conditioned profiles.
@@ -1542,7 +1529,7 @@ def target_tasks_condprof(full_task_graph, parameters, graph_config):
yield name
-@_target_task("system_symbols")
+@register_target_task("system_symbols")
def target_tasks_system_symbols(full_task_graph, parameters, graph_config):
"""
Select tasks for scraping and uploading system symbols.
@@ -1556,7 +1543,7 @@ def target_tasks_system_symbols(full_task_graph, parameters, graph_config):
yield name
-@_target_task("perftest")
+@register_target_task("perftest")
def target_tasks_perftest(full_task_graph, parameters, graph_config):
"""
Select perftest tasks we want to run daily
@@ -1568,7 +1555,7 @@ def target_tasks_perftest(full_task_graph, parameters, graph_config):
yield name
-@_target_task("perftest-on-autoland")
+@register_target_task("perftest-on-autoland")
def target_tasks_perftest_autoland(full_task_graph, parameters, graph_config):
"""
Select perftest tasks we want to run daily
@@ -1582,7 +1569,7 @@ def target_tasks_perftest_autoland(full_task_graph, parameters, graph_config):
yield name
-@_target_task("l10n-cross-channel")
+@register_target_task("l10n-cross-channel")
def target_tasks_l10n_cross_channel(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to run l10n cross-channel."""
@@ -1592,17 +1579,7 @@ def target_tasks_l10n_cross_channel(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("are-we-esmified-yet")
-def target_tasks_are_we_esmified_yet(full_task_graph, parameters, graph_config):
- """
- select the task to track the progress of the esmification project
- """
- return [
- l for l, t in full_task_graph.tasks.items() if t.kind == "are-we-esmified-yet"
- ]
-
-
-@_target_task("eslint-build")
+@register_target_task("eslint-build")
def target_tasks_eslint_build(full_task_graph, parameters, graph_config):
"""Select the task to run additional ESLint rules which require a build."""
@@ -1613,7 +1590,7 @@ def target_tasks_eslint_build(full_task_graph, parameters, graph_config):
yield name
-@_target_task("holly_tasks")
+@register_target_task("holly_tasks")
def target_tasks_holly(full_task_graph, parameters, graph_config):
"""Bug 1814661: only run updatebot tasks on holly"""
@@ -1623,7 +1600,7 @@ def target_tasks_holly(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("snap_upstream_tests")
+@register_target_task("snap_upstream_tests")
def target_tasks_snap_upstream_tests(full_task_graph, parameters, graph_config):
"""
Select tasks for testing Snap package built as upstream. Omit -try because
@@ -1632,3 +1609,49 @@ def target_tasks_snap_upstream_tests(full_task_graph, parameters, graph_config):
for name, task in full_task_graph.tasks.items():
if "snap-upstream-test" in name and not "-try" in name:
yield name
+
+
+@register_target_task("nightly-android")
+def target_tasks_nightly_android(full_task_graph, parameters, graph_config):
+ def filter(task, parameters):
+ # geckoview
+ if task.attributes.get("shipping_product") == "fennec" and task.kind in (
+ "beetmover-geckoview",
+ "upload-symbols",
+ ):
+ return True
+
+ # fenix/focus/a-c
+ build_type = task.attributes.get("build-type", "")
+ return build_type in (
+ "nightly",
+ "focus-nightly",
+ "fenix-nightly",
+ "fenix-nightly-firebase",
+ "focus-nightly-firebase",
+ )
+
+ index_path = (
+ f"{graph_config['trust-domain']}.v2.{parameters['project']}.branch."
+ f"{parameters['head_ref']}.revision.{parameters['head_rev']}.taskgraph.decision-nightly-android"
+ )
+ if os.environ.get("MOZ_AUTOMATION") and retry(
+ index_exists,
+ args=(index_path,),
+ kwargs={
+ "reason": "to avoid triggering multiple nightlies off the same revision",
+ },
+ ):
+ return []
+
+ return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
+
+
+@register_target_task("android-l10n-import")
+def target_tasks_android_l10n_import(full_task_graph, parameters, graph_config):
+ return [l for l, t in full_task_graph.tasks.items() if l == "android-l10n-import"]
+
+
+@register_target_task("android-l10n-sync")
+def target_tasks_android_l10n_sync(full_task_graph, parameters, graph_config):
+ return [l for l, t in full_task_graph.tasks.items() if l == "android-l10n-sync"]
diff --git a/taskcluster/gecko_taskgraph/test/automationrelevance.json b/taskcluster/gecko_taskgraph/test/automationrelevance.json
deleted file mode 100644
index 3bdfa9ed9e..0000000000
--- a/taskcluster/gecko_taskgraph/test/automationrelevance.json
+++ /dev/null
@@ -1,358 +0,0 @@
-{
- "changesets": [
- {
- "author": "James Long <longster@gmail.com>",
- "backsoutnodes": [],
- "bugs": [
- {
- "no": "1300866",
- "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866"
- }
- ],
- "date": [1473196655.0, 14400],
- "desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins",
- "extra": {
- "branch": "default"
- },
- "files": ["devtools/client/debugger/index.html"],
- "node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "parents": ["37c9349b4e8167a61b08b7e119c21ea177b98942"],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312890,
- "reviewers": [
- {
- "name": "jlast",
- "revset": "reviewer(jlast)"
- },
- {
- "name": "bgrins",
- "revset": "reviewer(bgrins)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Wes Kocher <wkocher@mozilla.com>",
- "backsoutnodes": [],
- "bugs": [],
- "date": [1473208638.0, 25200],
- "desc": "Merge m-c to fx-team, a=merge",
- "extra": {
- "branch": "default"
- },
- "files": ["taskcluster/scripts/builder/build-l10n.sh"],
- "node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "parents": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "91c2b9d5c1354ca79e5b174591dbb03b32b15bbf"
- ],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312891,
- "reviewers": [
- {
- "name": "merge",
- "revset": "reviewer(merge)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Towkir Ahmed <towkir17@gmail.com>",
- "backsoutnodes": [],
- "bugs": [
- {
- "no": "1296648",
- "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648"
- }
- ],
- "date": [1472957580.0, 14400],
- "desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim",
- "extra": {
- "branch": "default"
- },
- "files": ["devtools/client/themes/rules.css"],
- "node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "parents": ["73a6a267a50a0e1c41e689b265ad3eebe43d7ac6"],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312892,
- "reviewers": [
- {
- "name": "ntim",
- "revset": "reviewer(ntim)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Oriol <oriol-bugzilla@hotmail.com>",
- "backsoutnodes": [],
- "bugs": [
- {
- "no": "1300336",
- "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336"
- }
- ],
- "date": [1472921160.0, 14400],
- "desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen",
- "extra": {
- "branch": "default"
- },
- "files": [
- "devtools/client/webconsole/test/browser_webconsole_output_06.js",
- "devtools/server/actors/object.js"
- ],
- "node": "99c542fa43a72ee863c813b5624048d1b443549b",
- "parents": ["16a1a91f9269ab95dd83eb29dc5d0227665f7d94"],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312893,
- "reviewers": [
- {
- "name": "fitzgen",
- "revset": "reviewer(fitzgen)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Ruturaj Vartak <ruturaj@gmail.com>",
- "backsoutnodes": [],
- "bugs": [
- {
- "no": "1295010",
- "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
- }
- ],
- "date": [1472854020.0, -7200],
- "desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK",
- "extra": {
- "amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd",
- "branch": "default",
- "histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba",
- "rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651"
- },
- "files": [
- "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
- "devtools/client/inspector/test/head.js",
- "devtools/server/actors/highlighters/eye-dropper.js"
- ],
- "node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "parents": ["99c542fa43a72ee863c813b5624048d1b443549b"],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312894,
- "reviewers": [
- {
- "name": "pbro",
- "revset": "reviewer(pbro)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Matteo Ferretti <mferretti@mozilla.com>",
- "backsoutnodes": [],
- "bugs": [
- {
- "no": "1299154",
- "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154"
- }
- ],
- "date": [1472629906.0, -7200],
- "desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n",
- "extra": {
- "branch": "default",
- "committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200"
- },
- "files": [
- "docshell/base/nsDocShell.cpp",
- "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html"
- ],
- "node": "541c9086c0f27fba60beecc9bc94543103895c86",
- "parents": ["a6b6a93eb41a05e310a11f0172f01ba9b21d3eac"],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312895,
- "reviewers": [
- {
- "name": "mstange",
- "revset": "reviewer(mstange)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Patrick Brosset <pbrosset@mozilla.com>",
- "backsoutnodes": [],
- "bugs": [
- {
- "no": "1295010",
- "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
- }
- ],
- "date": [1473239449.0, -7200],
- "desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp",
- "extra": {
- "branch": "default"
- },
- "files": [
- "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
- "devtools/client/inspector/test/head.js"
- ],
- "node": "041a925171e431bf51fb50193ab19d156088c89a",
- "parents": ["541c9086c0f27fba60beecc9bc94543103895c86"],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312896,
- "reviewers": [
- {
- "name": "me",
- "revset": "reviewer(me)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>",
- "backsoutnodes": [],
- "bugs": [],
- "date": [1473261233.0, -7200],
- "desc": "merge fx-team to mozilla-central a=merge",
- "extra": {
- "branch": "default"
- },
- "files": [],
- "node": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "parents": [
- "3d0b41fdd93bd8233745eadb4e0358e385bf2cb9",
- "041a925171e431bf51fb50193ab19d156088c89a"
- ],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312897,
- "reviewers": [
- {
- "name": "merge",
- "revset": "reviewer(merge)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- }
- ],
- "visible": true
-}
diff --git a/taskcluster/gecko_taskgraph/test/conftest.py b/taskcluster/gecko_taskgraph/test/conftest.py
index 360c2da65e..758d3402b5 100644
--- a/taskcluster/gecko_taskgraph/test/conftest.py
+++ b/taskcluster/gecko_taskgraph/test/conftest.py
@@ -47,7 +47,7 @@ def enable_logging():
@pytest.fixture(scope="session")
def graph_config():
- return load_graph_config(os.path.join(GECKO, "taskcluster", "ci"))
+ return load_graph_config(os.path.join(GECKO, "taskcluster"))
@pytest.fixture(scope="session")
@@ -151,6 +151,20 @@ class FakeOptimization(OptimizationStrategy):
return False
+class FakeTransformConfig:
+ kind = "fake-kind"
+ path = "/root/ci/fake-kind"
+ config = {}
+ params = FakeParameters()
+ kind_dependencies_tasks = {}
+ graph_config = {}
+ write_artifacts = False
+
+ def __init__(self, **kwargs):
+ for k, v in kwargs.items():
+ setattr(self, k, v)
+
+
@pytest.fixture
def maketgg(monkeypatch):
def inner(target_tasks=None, kinds=[("_fake", [])], params=None):
@@ -195,12 +209,16 @@ def maketgg(monkeypatch):
@pytest.fixture
def run_transform():
graph_config = fake_load_graph_config("/root")
- kind = FakeKind.create("fake", {}, graph_config)
+ config = FakeTransformConfig(graph_config=graph_config)
+
+ def inner(xform, tasks, **extra_config):
+ if extra_config:
+ for k, v in extra_config.items():
+ setattr(config, k, v)
- def inner(xform, tasks):
if isinstance(tasks, dict):
tasks = [tasks]
- return xform(kind.config, tasks)
+ return xform(config, tasks)
return inner
diff --git a/taskcluster/gecko_taskgraph/test/python.toml b/taskcluster/gecko_taskgraph/test/python.toml
index 597a02d8aa..fd71a9c2bd 100644
--- a/taskcluster/gecko_taskgraph/test/python.toml
+++ b/taskcluster/gecko_taskgraph/test/python.toml
@@ -17,6 +17,8 @@ subsuite = "taskgraph"
["test_taskcluster_yml.py"]
+["test_transforms_build_schedules.py"]
+
["test_transforms_job.py"]
["test_transforms_test.py"]
diff --git a/taskcluster/gecko_taskgraph/test/test_actions_util.py b/taskcluster/gecko_taskgraph/test/test_actions_util.py
index 7c38caea57..0838dafc11 100644
--- a/taskcluster/gecko_taskgraph/test/test_actions_util.py
+++ b/taskcluster/gecko_taskgraph/test/test_actions_util.py
@@ -155,6 +155,7 @@ def is_subset(subset, superset):
def test_extract_applicable_action(
responses, monkeypatch, actions_json, task_def, expected
):
+ actions.util.get_task_definition.cache_clear()
base_url = "https://taskcluster"
decision_task_id = "dddd"
task_id = "tttt"
diff --git a/taskcluster/gecko_taskgraph/test/test_decision.py b/taskcluster/gecko_taskgraph/test/test_decision.py
index 8440b8e13f..53186b70fb 100644
--- a/taskcluster/gecko_taskgraph/test/test_decision.py
+++ b/taskcluster/gecko_taskgraph/test/test_decision.py
@@ -7,7 +7,6 @@ import json
import os
import shutil
import tempfile
-import unittest
from unittest.mock import patch
import pytest
@@ -18,6 +17,7 @@ from gecko_taskgraph import decision
from gecko_taskgraph.parameters import register_parameters
FAKE_GRAPH_CONFIG = {"product-dir": "browser", "taskgraph": {}}
+TTC_FILE = os.path.join(os.getcwd(), "try_task_config.json")
@pytest.fixture(scope="module", autouse=True)
@@ -25,150 +25,172 @@ def register():
register_parameters()
-class TestDecision(unittest.TestCase):
- def test_write_artifact_json(self):
- data = [{"some": "data"}]
- tmpdir = tempfile.mkdtemp()
- try:
- decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
- decision.write_artifact("artifact.json", data)
- with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.json")) as f:
- self.assertEqual(json.load(f), data)
- finally:
- if os.path.exists(tmpdir):
- shutil.rmtree(tmpdir)
- decision.ARTIFACTS_DIR = "artifacts"
-
- def test_write_artifact_yml(self):
- data = [{"some": "data"}]
- tmpdir = tempfile.mkdtemp()
- try:
- decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
- decision.write_artifact("artifact.yml", data)
- self.assertEqual(load_yaml(decision.ARTIFACTS_DIR, "artifact.yml"), data)
- finally:
- if os.path.exists(tmpdir):
- shutil.rmtree(tmpdir)
- decision.ARTIFACTS_DIR = "artifacts"
-
-
-class TestGetDecisionParameters(unittest.TestCase):
- ttc_file = os.path.join(os.getcwd(), "try_task_config.json")
-
- def setUp(self):
- self.options = {
- "base_repository": "https://hg.mozilla.org/mozilla-unified",
- "head_repository": "https://hg.mozilla.org/mozilla-central",
- "head_rev": "abcd",
- "head_ref": "ef01",
- "head_tag": "",
- "message": "",
- "project": "mozilla-central",
- "pushlog_id": "143",
- "pushdate": 1503691511,
- "owner": "nobody@mozilla.com",
- "repository_type": "hg",
- "tasks_for": "hg-push",
- "level": "3",
- }
-
- @patch("gecko_taskgraph.decision.get_hg_revision_branch")
- @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
- def test_simple_options(
- self, mock_determine_more_accurate_base_rev, mock_get_hg_revision_branch
- ):
- mock_get_hg_revision_branch.return_value = "default"
- mock_determine_more_accurate_base_rev.return_value = "baserev"
- with MockedOpen({self.ttc_file: None}):
- params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
- self.assertEqual(params["pushlog_id"], "143")
- self.assertEqual(params["build_date"], 1503691511)
- self.assertEqual(params["hg_branch"], "default")
- self.assertEqual(params["moz_build_date"], "20170825200511")
- self.assertEqual(params["try_mode"], None)
- self.assertEqual(params["try_options"], None)
- self.assertEqual(params["try_task_config"], {})
-
- @patch("gecko_taskgraph.decision.get_hg_revision_branch")
- @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
- def test_no_email_owner(
- self, mock_determine_more_accurate_base_rev, mock_get_hg_revision_branch
- ):
- mock_get_hg_revision_branch.return_value = "default"
- mock_determine_more_accurate_base_rev.return_value = "baserev"
- self.options["owner"] = "ffxbld"
- with MockedOpen({self.ttc_file: None}):
- params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
- self.assertEqual(params["owner"], "ffxbld@noreply.mozilla.org")
-
- @patch("gecko_taskgraph.decision.get_hg_revision_branch")
- @patch("gecko_taskgraph.decision.get_hg_commit_message")
- @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
- def test_try_options(
- self,
- mock_determine_more_accurate_base_rev,
- mock_get_hg_commit_message,
- mock_get_hg_revision_branch,
- ):
- mock_get_hg_commit_message.return_value = "try: -b do -t all --artifact"
- mock_get_hg_revision_branch.return_value = "default"
- mock_determine_more_accurate_base_rev.return_value = "baserev"
- self.options["project"] = "try"
- with MockedOpen({self.ttc_file: None}):
- params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
- self.assertEqual(params["try_mode"], "try_option_syntax")
- self.assertEqual(params["try_options"]["build_types"], "do")
- self.assertEqual(params["try_options"]["unittests"], "all")
- self.assertEqual(
- params["try_task_config"],
+@pytest.fixture(scope="module")
+def options():
+ return {
+ "base_repository": "https://hg.mozilla.org/mozilla-unified",
+ "head_repository": "https://hg.mozilla.org/mozilla-central",
+ "head_rev": "abcd",
+ "head_ref": "ef01",
+ "head_tag": "",
+ "message": "",
+ "project": "mozilla-central",
+ "pushlog_id": "143",
+ "pushdate": 1503691511,
+ "owner": "nobody@mozilla.com",
+ "repository_type": "hg",
+ "tasks_for": "hg-push",
+ "level": "3",
+ }
+
+
+def test_write_artifact_json():
+ data = [{"some": "data"}]
+ tmpdir = tempfile.mkdtemp()
+ try:
+ decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
+ decision.write_artifact("artifact.json", data)
+ with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.json")) as f:
+ assert json.load(f) == data
+ finally:
+ if os.path.exists(tmpdir):
+ shutil.rmtree(tmpdir)
+ decision.ARTIFACTS_DIR = "artifacts"
+
+
+def test_write_artifact_yml():
+ data = [{"some": "data"}]
+ tmpdir = tempfile.mkdtemp()
+ try:
+ decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
+ decision.write_artifact("artifact.yml", data)
+ assert load_yaml(decision.ARTIFACTS_DIR, "artifact.yml") == data
+ finally:
+ if os.path.exists(tmpdir):
+ shutil.rmtree(tmpdir)
+ decision.ARTIFACTS_DIR = "artifacts"
+
+
+@patch("gecko_taskgraph.decision.get_hg_revision_branch")
+@patch("gecko_taskgraph.decision.get_hg_commit_message")
+@patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
+@patch("gecko_taskgraph.decision.get_changed_files")
+@pytest.mark.parametrize(
+ "extra_options,commit_msg,ttc,expected",
+ (
+ pytest.param(
+ {},
+ None,
+ None,
{
- "gecko-profile": False,
- "use-artifact-builds": True,
- "env": {},
+ "pushlog_id": "143",
+ "build_date": 1503691511,
+ "files_changed": ["bar/baz.md", "foo.txt"],
+ "hg_branch": "default",
+ "moz_build_date": "20170825200511",
+ "try_mode": None,
+ "try_options": None,
+ "try_task_config": {},
},
- )
-
- @patch("gecko_taskgraph.decision.get_hg_revision_branch")
- @patch("gecko_taskgraph.decision.get_hg_commit_message")
- @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
- def test_try_task_config(
- self,
- mock_get_hg_commit_message,
- mock_get_hg_revision_branch,
- mock_determine_more_accurate_base_rev,
- ):
- mock_get_hg_commit_message.return_value = "Fuzzy query=foo"
- mock_get_hg_revision_branch.return_value = "default"
- mock_determine_more_accurate_base_rev.return_value = "baserev"
- ttc = {"tasks": ["a", "b"]}
- self.options["project"] = "try"
- with MockedOpen({self.ttc_file: json.dumps(ttc)}):
- params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
- self.assertEqual(params["try_mode"], "try_task_config")
- self.assertEqual(params["try_options"], None)
- self.assertEqual(params["try_task_config"], ttc)
-
- def test_try_syntax_from_message_empty(self):
- self.assertEqual(decision.try_syntax_from_message(""), "")
-
- def test_try_syntax_from_message_no_try_syntax(self):
- self.assertEqual(decision.try_syntax_from_message("abc | def"), "")
-
- def test_try_syntax_from_message_initial_try_syntax(self):
- self.assertEqual(
- decision.try_syntax_from_message("try: -f -o -o"), "try: -f -o -o"
- )
-
- def test_try_syntax_from_message_initial_try_syntax_multiline(self):
- self.assertEqual(
- decision.try_syntax_from_message("try: -f -o -o\nabc\ndef"), "try: -f -o -o"
- )
-
- def test_try_syntax_from_message_embedded_try_syntax_multiline(self):
- self.assertEqual(
- decision.try_syntax_from_message("some stuff\ntry: -f -o -o\nabc\ndef"),
+ id="simple_options",
+ ),
+ pytest.param(
+ {"owner": "ffxbld"},
+ None,
+ None,
+ {
+ "owner": "ffxbld@noreply.mozilla.org",
+ },
+ id="no_email_owner",
+ ),
+ pytest.param(
+ {"project": "try"},
+ "try: -b do -t all --artifact",
+ None,
+ {
+ "try_mode": "try_option_syntax",
+ "try_options": {
+ "build_types": "do",
+ "include_nightly": False,
+ "interactive": False,
+ "jobs": None,
+ "no_retry": False,
+ "notifications": None,
+ "platforms": "all",
+ "raptor": "none",
+ "raptor_trigger_tests": 1,
+ "tag": None,
+ "talos": "all",
+ "talos_trigger_tests": 1,
+ "taskcluster_worker": False,
+ "trigger_tests": 1,
+ "unittests": "all",
+ },
+ "try_task_config": {
+ "gecko-profile": False,
+ "use-artifact-builds": True,
+ "env": {},
+ },
+ },
+ id="try_options",
+ ),
+ pytest.param(
+ {
+ "project": "try",
+ },
+ "Fuzzy query=foo",
+ {"tasks": ["a", "b"]},
+ {
+ "try_mode": "try_task_config",
+ "try_options": None,
+ "try_task_config": {"tasks": ["a", "b"]},
+ },
+ id="try_task_config",
+ ),
+ ),
+)
+def test_get_decision_parameters(
+ mock_get_changed_files,
+ mock_determine_more_accurate_base_rev,
+ mock_get_hg_commit_message,
+ mock_get_hg_revision_branch,
+ options,
+ extra_options,
+ commit_msg,
+ ttc,
+ expected,
+):
+ mock_get_hg_revision_branch.return_value = "default"
+ mock_get_hg_commit_message.return_value = commit_msg or "commit message"
+ mock_determine_more_accurate_base_rev.return_value = "baserev"
+ mock_get_changed_files.return_value = ["foo.txt", "bar/baz.md"]
+
+ options.update(extra_options)
+ contents = None
+ if ttc:
+ contents = json.dumps(ttc)
+ with MockedOpen({TTC_FILE: contents}):
+ params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, options)
+
+ for key in expected:
+ assert params[key] == expected[key], f"key {key} does not match!"
+
+
+@pytest.mark.parametrize(
+ "msg, expected",
+ (
+ pytest.param("", "", id="empty"),
+ pytest.param("abc | def", "", id="no_try_syntax"),
+ pytest.param("try: -f -o -o", "try: -f -o -o", id="initial_try_syntax"),
+ pytest.param(
+ "some stuff\ntry: -f -o -o\nabc\ndef",
"try: -f -o -o",
- )
+ id="embedded_try_syntax_multiline",
+ ),
+ ),
+)
+def test_try_syntax_from_message(msg, expected):
+ assert decision.try_syntax_from_message(msg) == expected
if __name__ == "__main__":
diff --git a/taskcluster/gecko_taskgraph/test/test_files_changed.py b/taskcluster/gecko_taskgraph/test/test_files_changed.py
index 5b9a016649..389dbb4093 100644
--- a/taskcluster/gecko_taskgraph/test/test_files_changed.py
+++ b/taskcluster/gecko_taskgraph/test/test_files_changed.py
@@ -3,14 +3,11 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-import json
-import os
import unittest
from mozunit import main
from gecko_taskgraph import files_changed
-from gecko_taskgraph.util import hg
PARAMS = {
"head_repository": "https://hg.mozilla.org/mozilla-central",
@@ -31,40 +28,17 @@ FILES_CHANGED = [
]
-class FakeResponse:
- def json(self):
- with open(
- os.path.join(os.path.dirname(__file__), "automationrelevance.json")
- ) as f:
- return json.load(f)
-
-
-class TestGetChangedFiles(unittest.TestCase):
- def setUp(self):
- files_changed.get_changed_files.clear()
- self.old_get = hg.requests.get
-
- def fake_get(url, **kwargs):
- return FakeResponse()
-
- hg.requests.get = fake_get
-
- def tearDown(self):
- hg.requests.get = self.old_get
- files_changed.get_changed_files.clear()
-
- def test_get_changed_files(self):
- """Get_changed_files correctly gets the list of changed files in a push.
- This tests against the production hg.mozilla.org so that it will detect
- any changes in the format of the returned data."""
- self.assertEqual(
- sorted(
- files_changed.get_changed_files(
- PARAMS["head_repository"], PARAMS["head_rev"]
- )
- ),
- FILES_CHANGED,
+def test_get_changed_files(responses):
+ url = f"{PARAMS['head_repository']}/json-pushchangedfiles/{PARAMS['head_rev']}"
+ responses.add(responses.GET, url, status=200, json={"files": FILES_CHANGED})
+ assert (
+ sorted(
+ files_changed.get_changed_files(
+ PARAMS["head_repository"], PARAMS["head_rev"]
+ )
)
+ == FILES_CHANGED
+ )
class TestCheck(unittest.TestCase):
diff --git a/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py b/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py
index 1240d71cf8..2c10ba449a 100644
--- a/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py
+++ b/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py
@@ -19,6 +19,7 @@ from gecko_taskgraph.optimize.bugbug import (
DisperseGroups,
SkipUnlessDebug,
)
+from gecko_taskgraph.optimize.mozlint import SkipUnlessMozlint
from gecko_taskgraph.optimize.strategies import SkipUnlessSchedules
from gecko_taskgraph.util.backstop import BACKSTOP_PUSH_INTERVAL
from gecko_taskgraph.util.bugbug import (
@@ -511,5 +512,79 @@ def test_project_autoland_test(monkeypatch, responses, params):
assert scheduled == {"task-0-label", "task-1-label"}
+@pytest.mark.parametrize(
+ "pushed_files,to_lint,expected",
+ [
+ pytest.param(
+ ["a/b/c.txt"],
+ [],
+ True,
+ ),
+ pytest.param(
+ ["python/mozlint/a/support_file.txt", "b/c/d.txt"],
+ ["python/mozlint/a/support_file.txt"],
+ False,
+ ),
+ ],
+ ids=idfn,
+)
+def test_mozlint_should_remove_task(
+ monkeypatch, params, pushed_files, to_lint, expected
+):
+ import mozlint.pathutils
+
+ class MockParser:
+ def __call__(self, *args, **kwargs):
+ return []
+
+ def mock_filterpaths(*args, **kwargs):
+ return to_lint, None
+
+ monkeypatch.setattr(mozlint.pathutils, "filterpaths", mock_filterpaths)
+
+ opt = SkipUnlessMozlint("")
+ monkeypatch.setattr(opt, "mozlint_parser", MockParser())
+ params["files_changed"] = pushed_files
+
+ result = opt.should_remove_task(default_tasks[0], params, "")
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "pushed_files,linter_config,expected",
+ [
+ pytest.param(
+ ["a/b/c.txt"],
+ [{"include": ["b/c"]}],
+ True,
+ ),
+ pytest.param(
+ ["a/b/c.txt"],
+ [{"include": ["a/b"], "exclude": ["a/b/c.txt"]}],
+ True,
+ ),
+ pytest.param(
+ ["python/mozlint/a/support_file.txt", "b/c/d.txt"],
+ [{}],
+ False,
+ ),
+ ],
+ ids=idfn,
+)
+def test_mozlint_should_remove_task2(
+ monkeypatch, params, pushed_files, linter_config, expected
+):
+ class MockParser:
+ def __call__(self, *args, **kwargs):
+ return linter_config
+
+ opt = SkipUnlessMozlint("")
+ monkeypatch.setattr(opt, "mozlint_parser", MockParser())
+ params["files_changed"] = pushed_files
+
+ result = opt.should_remove_task(default_tasks[0], params, "")
+ assert result == expected
+
+
if __name__ == "__main__":
main()
diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_build_schedules.py b/taskcluster/gecko_taskgraph/test/test_transforms_build_schedules.py
new file mode 100644
index 0000000000..a693461c68
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_transforms_build_schedules.py
@@ -0,0 +1,56 @@
+import pytest
+from mozunit import main
+
+from gecko_taskgraph.transforms.build_schedules import set_build_schedules_optimization
+
+
+@pytest.mark.parametrize(
+ "kind,task,expected",
+ (
+ pytest.param("build", {"when": "foo"}, None, id="no-op"),
+ pytest.param(
+ "build",
+ {"attributes": {"build_platform": "linux64/opt"}},
+ {"build": ["linux", "firefox"]},
+ id="build",
+ ),
+ pytest.param(
+ "build-components",
+ {},
+ {"build": ["android", "fenix", "focus-android"]},
+ id="build-components",
+ ),
+ pytest.param(
+ "build-bundle",
+ {"name": "build-bundle-fenix"},
+ {"build": ["android", "fenix"]},
+ id="build-bundle-fenix",
+ ),
+ pytest.param(
+ "build-apk",
+ {"name": "fenix"},
+ {"build": ["android", "fenix"]},
+ id="build-apk-fenix",
+ ),
+ pytest.param(
+ "build-apk",
+ {"name": "build-apk-focus"},
+ {"build": ["android", "focus-android"]},
+ id="build-apk-focus",
+ ),
+ pytest.param(
+ "build-apk",
+ {"name": "build-apk-klar"},
+ {"build": ["android", "focus-android"]},
+ id="build-apk-klar",
+ ),
+ ),
+)
+def test_build_schedules(run_transform, kind, task, expected):
+ tasks = list(run_transform(set_build_schedules_optimization, [task], kind=kind))
+ assert len(tasks) == 1
+ assert tasks[0].get("optimization") == expected
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_job.py b/taskcluster/gecko_taskgraph/test/test_transforms_job.py
index b032307ea6..6161a36054 100644
--- a/taskcluster/gecko_taskgraph/test/test_transforms_job.py
+++ b/taskcluster/gecko_taskgraph/test/test_transforms_job.py
@@ -37,7 +37,7 @@ TASK_DEFAULTS = {
@pytest.fixture(scope="module")
def config():
- graph_config = load_graph_config(os.path.join(GECKO, "taskcluster", "ci"))
+ graph_config = load_graph_config(os.path.join(GECKO, "taskcluster"))
params = FakeParameters(
{
"base_repository": "http://hg.example.com",
diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_test.py b/taskcluster/gecko_taskgraph/test/test_transforms_test.py
index 1e5067a2b5..d61eff5769 100644
--- a/taskcluster/gecko_taskgraph/test/test_transforms_test.py
+++ b/taskcluster/gecko_taskgraph/test/test_transforms_test.py
@@ -235,16 +235,16 @@ def test_split_variants(monkeypatch, run_full_config_transform, make_test_task):
pytest.param(
{
"attributes": {},
- "test-platform": "windows10-64-2004-ref-hw-2017-ccov/debug",
+ "test-platform": "windows11-64-2009-hw-ref-ccov/debug",
},
{
"platform": {
"arch": "64",
- "machine": "ref-hw-2017",
+ "machine": "hw-ref",
"os": {
- "build": "2004",
+ "build": "2009",
"name": "windows",
- "version": "10",
+ "version": "11",
},
},
"build": {
diff --git a/taskcluster/gecko_taskgraph/test/test_util_backstop.py b/taskcluster/gecko_taskgraph/test/test_util_backstop.py
index af9aabd5af..0a2bdc6ae4 100644
--- a/taskcluster/gecko_taskgraph/test/test_util_backstop.py
+++ b/taskcluster/gecko_taskgraph/test/test_util_backstop.py
@@ -18,20 +18,22 @@ from gecko_taskgraph.util.backstop import (
is_backstop,
)
-LAST_BACKSTOP_ID = 0
+LAST_BACKSTOP_PUSHID = 1
LAST_BACKSTOP_PUSHDATE = mktime(datetime.now().timetuple())
DEFAULT_RESPONSES = {
"index": {
"status": 200,
- "json": {"taskId": LAST_BACKSTOP_ID},
+ "json": {"taskId": LAST_BACKSTOP_PUSHID},
},
"artifact": {
"status": 200,
"body": dedent(
"""
pushdate: {}
+ pushlog_id: "{}"
""".format(
- LAST_BACKSTOP_PUSHDATE
+ LAST_BACKSTOP_PUSHDATE,
+ LAST_BACKSTOP_PUSHID,
)
),
},
@@ -50,7 +52,8 @@ def params():
"head_rev": "abcdef",
"project": "autoland",
"pushdate": LAST_BACKSTOP_PUSHDATE + 1,
- "pushlog_id": LAST_BACKSTOP_ID + 1,
+ "pushlog_id": f"{LAST_BACKSTOP_PUSHID + 1}",
+ "target_tasks_method": "default",
}
@@ -61,7 +64,7 @@ def params():
{
"index": {"status": 404},
},
- {"pushlog_id": 1},
+ {"pushlog_id": "1"},
True,
id="no previous backstop",
),
@@ -78,8 +81,8 @@ def params():
pytest.param(
DEFAULT_RESPONSES,
{
- "pushlog_id": LAST_BACKSTOP_ID + 1,
- "pushdate": LAST_BACKSTOP_PUSHDATE + 1,
+ "pushlog_id": f"{LAST_BACKSTOP_PUSHID + BACKSTOP_PUSH_INTERVAL - 1}",
+ "pushdate": LAST_BACKSTOP_PUSHDATE + (BACKSTOP_TIME_INTERVAL * 60) - 1,
},
False,
id="not a backstop",
@@ -87,10 +90,26 @@ def params():
pytest.param(
{},
{
- "pushlog_id": BACKSTOP_PUSH_INTERVAL,
+ "target_tasks_method": "nothing",
+ },
+ False,
+ id="dontbuild",
+ ),
+ pytest.param(
+ DEFAULT_RESPONSES,
+ {
+ "pushlog_id": f"{LAST_BACKSTOP_PUSHID + BACKSTOP_PUSH_INTERVAL}",
+ },
+ True,
+ id="interval",
+ ),
+ pytest.param(
+ DEFAULT_RESPONSES,
+ {
+ "pushlog_id": f"{LAST_BACKSTOP_PUSHID + BACKSTOP_PUSH_INTERVAL + 1}",
},
True,
- id="backstop interval",
+ id="greater than interval",
),
pytest.param(
DEFAULT_RESPONSES,
@@ -104,7 +123,7 @@ def params():
{},
{
"project": "try",
- "pushlog_id": BACKSTOP_PUSH_INTERVAL,
+ "pushlog_id": f"{BACKSTOP_PUSH_INTERVAL}",
},
False,
id="try not a backstop",
@@ -138,13 +157,12 @@ def test_is_backstop(responses, params, response_args, extra_params, expected):
**{"trust-domain": "gecko", "project": params["project"]}
)
),
- "artifact": get_artifact_url(LAST_BACKSTOP_ID, "public/parameters.yml"),
- "status": get_task_url(LAST_BACKSTOP_ID) + "/status",
+ "artifact": get_artifact_url(LAST_BACKSTOP_PUSHID, "public/parameters.yml"),
+ "status": get_task_url(LAST_BACKSTOP_PUSHID) + "/status",
}
for key in ("index", "status", "artifact"):
if key in response_args:
- print(urls[key])
responses.add(responses.GET, urls[key], **response_args[key])
params.update(extra_params)
diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py
index fb5b17d3b3..1434b4013a 100644
--- a/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py
+++ b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py
@@ -101,6 +101,7 @@ CONFIG_PER_BOUNCER_PRODUCT = {
"file_names": {
"win": "{pretty_product}%20Setup%20{version}.msix",
"win64": "{pretty_product}%20Setup%20{version}.msix",
+ "win64-aarch64": "{pretty_product}%20Setup%20{version}.msix",
},
},
"pkg": {
diff --git a/taskcluster/gecko_taskgraph/transforms/build.py b/taskcluster/gecko_taskgraph/transforms/build.py
index 4e73c5aef2..3ab22f2923 100644
--- a/taskcluster/gecko_taskgraph/transforms/build.py
+++ b/taskcluster/gecko_taskgraph/transforms/build.py
@@ -139,6 +139,10 @@ def use_artifact(config, jobs):
and job.get("index", {}).get("job-name") in ARTIFACT_JOBS
# If tests aren't packaged, then we are not able to rebuild all the packages
and job["worker"]["env"].get("MOZ_AUTOMATION_PACKAGE_TESTS") == "1"
+ # Android shippable artifact builds are not supported
+ and not (
+ "android" in job["name"] and job["attributes"].get("shippable", False)
+ )
):
job["treeherder"]["symbol"] = add_suffix(job["treeherder"]["symbol"], "a")
job["worker"]["env"]["USE_ARTIFACT"] = "1"
diff --git a/taskcluster/gecko_taskgraph/transforms/build_attrs.py b/taskcluster/gecko_taskgraph/transforms/build_attrs.py
index 9cda71718a..fda9888fe1 100644
--- a/taskcluster/gecko_taskgraph/transforms/build_attrs.py
+++ b/taskcluster/gecko_taskgraph/transforms/build_attrs.py
@@ -4,8 +4,6 @@
from taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.platforms import platform_family
-
transforms = TransformSequence()
@@ -34,17 +32,3 @@ def set_build_attributes(config, jobs):
)
yield job
-
-
-@transforms.add
-def set_schedules_optimization(config, jobs):
- """Set the `skip-unless-affected` optimization based on the build platform."""
- for job in jobs:
- # don't add skip-unless-schedules if there's already a when defined
- if "when" in job:
- yield job
- continue
-
- build_platform = job["attributes"]["build_platform"]
- job.setdefault("optimization", {"build": [platform_family(build_platform)]})
- yield job
diff --git a/taskcluster/gecko_taskgraph/transforms/build_schedules.py b/taskcluster/gecko_taskgraph/transforms/build_schedules.py
new file mode 100644
index 0000000000..f9621adf77
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/transforms/build_schedules.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from taskgraph.transforms.base import TransformSequence
+
+from gecko_taskgraph.util.platforms import platform_family
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_build_schedules_optimization(config, tasks):
+ """Set the `build` optimization based on the build platform."""
+ for task in tasks:
+ # don't add an optimization if there's already one defined
+ if "when" in task or "optimization" in task:
+ yield task
+ continue
+
+ schedules = []
+ if config.kind in (
+ "build-components",
+ "build-samples-browser",
+ "test-components",
+ ):
+ # These are Android components builds and can only impact Fenix or Focus.
+ schedules = ["android", "fenix", "focus-android"]
+
+ elif config.kind in ("build-apk", "build-bundle", "test-apk", "ui-test-apk"):
+ # These are APK builds for Fenix or Focus
+ schedules = ["android"]
+
+ if "fenix" in task["name"]:
+ schedules.append("fenix")
+ elif "focus" in task["name"] or "klar" in task["name"]:
+ schedules.append("focus-android")
+
+ else:
+ family = platform_family(task["attributes"]["build_platform"])
+ schedules = [family]
+
+ if "android" not in family:
+ # These are not GeckoView builds, so are associated with Firefox.
+ schedules.append("firefox")
+
+ task["optimization"] = {"build": schedules}
+ yield task
diff --git a/taskcluster/gecko_taskgraph/transforms/condprof.py b/taskcluster/gecko_taskgraph/transforms/condprof.py
index 516c1d8f20..f39a9b09ab 100644
--- a/taskcluster/gecko_taskgraph/transforms/condprof.py
+++ b/taskcluster/gecko_taskgraph/transforms/condprof.py
@@ -28,7 +28,7 @@ diff_description_schema = Schema(
Optional("run-on-projects"): task_description_schema["run-on-projects"],
Optional("scopes"): task_description_schema["scopes"],
Optional("treeherder"): task_description_schema["treeherder"],
- Optional("use-system-python"): bool,
+ Optional("use-python"): job_description_schema["use-python"],
Optional("worker"): job_description_schema["worker"],
Optional("worker-type"): task_description_schema["worker-type"],
}
@@ -84,8 +84,8 @@ def generate_scenarios(config, tasks):
"fetches": copy_task(task["fetches"]),
}
- use_system_python = task.get("use-system-python", None)
- if use_system_python is not None:
- taskdesc["use-system-python"] = use_system_python
+ use_taskcluster_python = task.get("use-python", "system")
+ if use_taskcluster_python != "system":
+ taskdesc["use-python"] = use_taskcluster_python
yield taskdesc
diff --git a/taskcluster/gecko_taskgraph/transforms/diffoscope.py b/taskcluster/gecko_taskgraph/transforms/diffoscope.py
index b74dc5bb8f..05c6617950 100644
--- a/taskcluster/gecko_taskgraph/transforms/diffoscope.py
+++ b/taskcluster/gecko_taskgraph/transforms/diffoscope.py
@@ -32,7 +32,7 @@ diff_description_schema = Schema(
Required("original"): index_or_string,
Required("new"): index_or_string,
# Arguments to pass to diffoscope, used for job-defaults in
- # taskcluster/ci/diffoscope/kind.yml
+ # taskcluster/kinds/diffoscope/kind.yml
Optional("args"): str,
# Extra arguments to pass to diffoscope, that can be set per job.
Optional("extra-args"): str,
diff --git a/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py b/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py
index 2f0d8dd2aa..016d642b9b 100644
--- a/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py
+++ b/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py
@@ -2,7 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
-Transform the repackage signing task into an actual task description.
+Transform the geckodriver notarization task into an actual task description.
"""
from taskgraph.transforms.base import TransformSequence
@@ -14,7 +14,7 @@ from gecko_taskgraph.transforms.task import task_description_schema
from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job
from gecko_taskgraph.util.scriptworker import add_scope_prefix
-repackage_signing_description_schema = Schema(
+geckodriver_notarization_description_schema = Schema(
{
Optional("label"): str,
Optional("treeherder"): task_description_schema["treeherder"],
@@ -38,7 +38,7 @@ def remove_name(config, jobs):
yield job
-transforms.add_validate(repackage_signing_description_schema)
+transforms.add_validate(geckodriver_notarization_description_schema)
@transforms.add
diff --git a/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py b/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py
index 95b8d3dd54..25f861e662 100644
--- a/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py
+++ b/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py
@@ -121,7 +121,7 @@ def make_signing_description(config, jobs):
def _craft_upstream_artifacts(dep_job, dependency_kind, build_platform):
if build_platform.startswith("win"):
- signing_format = "autograph_authenticode_sha2"
+ signing_format = "autograph_authenticode_202404"
elif build_platform.startswith("linux"):
signing_format = "autograph_gpg"
elif build_platform.startswith("macosx"):
diff --git a/taskcluster/gecko_taskgraph/transforms/github_sync.py b/taskcluster/gecko_taskgraph/transforms/github_sync.py
deleted file mode 100644
index 6f48f794ce..0000000000
--- a/taskcluster/gecko_taskgraph/transforms/github_sync.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from taskgraph.transforms.base import TransformSequence
-
-transforms = TransformSequence()
-
-
-@transforms.add
-def sync_github(config, tasks):
- """Do transforms specific to github-sync tasks."""
- for task in tasks:
- # Add the secret to the scopes, only in m-c.
- # Doing this on any other tree will result in decision task failure
- # because m-c is the only one allowed to have that scope.
- secret = task["secret"]
- if config.params["project"] == "mozilla-central":
- task.setdefault("scopes", [])
- task["scopes"].append("secrets:get:" + secret)
- task["worker"].setdefault("env", {})["GITHUB_SECRET"] = secret
- del task["secret"]
- yield task
diff --git a/taskcluster/gecko_taskgraph/transforms/job/__init__.py b/taskcluster/gecko_taskgraph/transforms/job/__init__.py
index b87f7e0955..54cedf513a 100644
--- a/taskcluster/gecko_taskgraph/transforms/job/__init__.py
+++ b/taskcluster/gecko_taskgraph/transforms/job/__init__.py
@@ -14,11 +14,12 @@ import json
import logging
import mozpack.path as mozpath
+from packaging.version import Version
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.python_path import import_sibling_modules
from taskgraph.util.schema import Schema, validate_schema
from taskgraph.util.taskcluster import get_artifact_prefix
-from voluptuous import Any, Exclusive, Extra, Optional, Required
+from voluptuous import Any, Coerce, Exclusive, Extra, Optional, Required
from gecko_taskgraph.transforms.cached_tasks import order_tasks
from gecko_taskgraph.transforms.task import task_description_schema
@@ -62,7 +63,7 @@ job_description_schema = Schema(
"optimization"
],
Optional("use-sccache"): task_description_schema["use-sccache"],
- Optional("use-system-python"): bool,
+ Optional("use-python"): Any("system", "default", Coerce(Version)),
Optional("priority"): task_description_schema["priority"],
# The "when" section contains descriptions of the circumstances under which
# this task should be included in the task graph. This will be converted
@@ -245,9 +246,15 @@ def get_attribute(dict, key, attributes, attribute_name):
@transforms.add
def use_system_python(config, jobs):
for job in jobs:
- if job.pop("use-system-python", True):
+ taskcluster_python = job.pop("use-python", "system")
+ if taskcluster_python == "system":
yield job
else:
+ if taskcluster_python == "default":
+ python_version = "python" # the taskcluster default alias
+ else:
+ python_version = f"python-{taskcluster_python}"
+
fetches = job.setdefault("fetches", {})
toolchain = fetches.setdefault("toolchain", [])
if "win" in job["worker"]["os"]:
@@ -259,7 +266,7 @@ def use_system_python(config, jobs):
else:
raise ValueError("unexpected worker.os value {}".format(platform))
- toolchain.append("{}-python".format(platform))
+ toolchain.append(f"{platform}-{python_version}")
worker = job.setdefault("worker", {})
env = worker.setdefault("env", {})
diff --git a/taskcluster/gecko_taskgraph/transforms/job/mach.py b/taskcluster/gecko_taskgraph/transforms/job/mach.py
index 775213f8fe..5f830ec04b 100644
--- a/taskcluster/gecko_taskgraph/transforms/job/mach.py
+++ b/taskcluster/gecko_taskgraph/transforms/job/mach.py
@@ -50,7 +50,7 @@ def configure_mach(config, job, taskdesc):
if python:
del run["python-version"]
- if taskdesc.get("use-system-python"):
+ if taskdesc.get("use-python", "system") == "system":
if worker["os"] == "macosx" and python == 3:
python = "/usr/local/bin/python3"
diff --git a/taskcluster/gecko_taskgraph/transforms/job/mozharness.py b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py
index 4d7293ec51..ada5b85ea3 100644
--- a/taskcluster/gecko_taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py
@@ -289,7 +289,7 @@ def mozharness_on_generic_worker(config, job, taskdesc):
system_python_dir = ""
gecko_path = "$GECKO_PATH"
- if run.get("use-system-python", True):
+ if run.get("use-python", "system") == "system":
python_bindir = system_python_dir
else:
# $MOZ_PYTHON_HOME is going to be substituted in run-task, when we
diff --git a/taskcluster/gecko_taskgraph/transforms/openh264_signing.py b/taskcluster/gecko_taskgraph/transforms/openh264_signing.py
index 00a55dad41..74aebbd90b 100644
--- a/taskcluster/gecko_taskgraph/transforms/openh264_signing.py
+++ b/taskcluster/gecko_taskgraph/transforms/openh264_signing.py
@@ -81,7 +81,7 @@ def make_signing_description(config, jobs):
}
if "win" in build_platform:
- upstream_artifact["formats"] = ["autograph_authenticode_sha2"]
+ upstream_artifact["formats"] = ["autograph_authenticode_202404"]
elif "mac" in build_platform:
upstream_artifact["formats"] = ["mac_single_file"]
upstream_artifact["singleFileGlobs"] = ["libgmpopenh264.dylib"]
diff --git a/taskcluster/gecko_taskgraph/transforms/perftest.py b/taskcluster/gecko_taskgraph/transforms/perftest.py
index 47baafdad7..066db956f7 100644
--- a/taskcluster/gecko_taskgraph/transforms/perftest.py
+++ b/taskcluster/gecko_taskgraph/transforms/perftest.py
@@ -1,3 +1,4 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
diff --git a/taskcluster/gecko_taskgraph/transforms/release_deps.py b/taskcluster/gecko_taskgraph/transforms/release_deps.py
index e44af576eb..aab8a2f60a 100644
--- a/taskcluster/gecko_taskgraph/transforms/release_deps.py
+++ b/taskcluster/gecko_taskgraph/transforms/release_deps.py
@@ -44,6 +44,17 @@ def add_dependencies(config, jobs):
!= job["attributes"]["build_platform"]
):
continue
+
+ # TODO get rid of the release-type match
+ if product == "firefox-android":
+ # exclude beta tasks from release graph and vice versa
+ from android_taskgraph.release_type import does_task_match_release_type
+
+ if not does_task_match_release_type(
+ dep_task, config.params["release_type"]
+ ):
+ continue
+
# Add matching product tasks to deps
if (
dep_task.task.get("shipping-product") == product
diff --git a/taskcluster/gecko_taskgraph/transforms/release_notifications.py b/taskcluster/gecko_taskgraph/transforms/release_notifications.py
index 86109ec5ed..071e5de8a3 100644
--- a/taskcluster/gecko_taskgraph/transforms/release_notifications.py
+++ b/taskcluster/gecko_taskgraph/transforms/release_notifications.py
@@ -39,6 +39,9 @@ def add_notifications(config, jobs):
resolve_keyed_by(
notifications, "emails", label, project=config.params["project"]
)
+ resolve_keyed_by(
+ notifications, "message", label, project=config.params["project"]
+ )
emails = notifications["emails"]
format_kwargs = dict(
task=job,
diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_signing.py b/taskcluster/gecko_taskgraph/transforms/repackage_signing.py
index 66c1f87d70..d4106efb29 100644
--- a/taskcluster/gecko_taskgraph/transforms/repackage_signing.py
+++ b/taskcluster/gecko_taskgraph/transforms/repackage_signing.py
@@ -29,10 +29,10 @@ repackage_signing_description_schema = Schema(
)
SIGNING_FORMATS = {
- "target.installer.exe": ["autograph_authenticode_sha2_stub"],
- "target.stub-installer.exe": ["autograph_authenticode_sha2_stub"],
- "target.installer.msi": ["autograph_authenticode_sha2"],
- "target.installer.msix": ["autograph_authenticode_sha2"],
+ "target.installer.exe": ["autograph_authenticode_202404_stub"],
+ "target.stub-installer.exe": ["autograph_authenticode_202404_stub"],
+ "target.installer.msi": ["autograph_authenticode_202404"],
+ "target.installer.msix": ["autograph_authenticode_202404"],
}
transforms = TransformSequence()
diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py b/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py
index e3940fd846..5f7c893e92 100644
--- a/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py
+++ b/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py
@@ -93,7 +93,7 @@ def make_repackage_signing_description(config, jobs):
"paths": [
get_artifact_path(dep_job, f"{repack_id}/target.installer.exe"),
],
- "formats": ["autograph_authenticode_sha2", "autograph_gpg"],
+ "formats": ["autograph_authenticode_202404", "autograph_gpg"],
}
]
@@ -113,7 +113,7 @@ def make_repackage_signing_description(config, jobs):
f"{repack_id}/target.stub-installer.exe",
),
],
- "formats": ["autograph_authenticode_sha2", "autograph_gpg"],
+ "formats": ["autograph_authenticode_202404", "autograph_gpg"],
}
)
elif "mac" in build_platform:
diff --git a/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py b/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py
index ea2cac3a68..48465b3ab7 100644
--- a/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py
+++ b/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py
@@ -3,7 +3,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the reprocess-symbols task description template,
-taskcluster/ci/reprocess-symbols/job-template.yml into an actual task description.
+taskcluster/kinds/reprocess-symbols/job-template.yml into an actual task description.
"""
diff --git a/taskcluster/gecko_taskgraph/transforms/signing.py b/taskcluster/gecko_taskgraph/transforms/signing.py
index e55ad47f42..760f9f122f 100644
--- a/taskcluster/gecko_taskgraph/transforms/signing.py
+++ b/taskcluster/gecko_taskgraph/transforms/signing.py
@@ -12,10 +12,7 @@ from taskgraph.util.schema import Schema, taskref_or_string
from voluptuous import Optional, Required
from gecko_taskgraph.transforms.task import task_description_schema
-from gecko_taskgraph.util.attributes import (
- copy_attributes_from_dependent_job,
- release_level,
-)
+from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job
from gecko_taskgraph.util.scriptworker import (
add_scope_prefix,
get_signing_cert_scope_per_platform,
@@ -64,6 +61,14 @@ signing_description_schema = Schema(
)
+def get_locales_description(attributes, default):
+ """Returns the [list] of locales for task description usage"""
+ chunk_locales = attributes.get("chunk_locales")
+ if chunk_locales:
+ return ", ".join(chunk_locales)
+ return attributes.get("locale", default)
+
+
@transforms.add
def delete_name(config, jobs):
"""Delete the 'name' key if it exists, we don't use it."""
@@ -77,30 +82,12 @@ transforms.add_validate(signing_description_schema)
@transforms.add
-def add_entitlements_link(config, jobs):
- for job in jobs:
- dep_job = get_primary_dependency(config, job)
- entitlements_path = evaluate_keyed_by(
- config.graph_config["mac-notarization"]["mac-entitlements"],
- "mac entitlements",
- {
- "platform": dep_job.attributes.get("build_platform"),
- "release-level": release_level(config.params["project"]),
- },
- )
- if entitlements_path:
- job["entitlements-url"] = config.params.file_url(
- entitlements_path,
- )
- yield job
-
-
-@transforms.add
def add_requirements_link(config, jobs):
for job in jobs:
dep_job = get_primary_dependency(config, job)
+ assert dep_job
requirements_path = evaluate_keyed_by(
- config.graph_config["mac-notarization"]["mac-requirements"],
+ config.graph_config["mac-signing"]["mac-requirements"],
"mac requirements",
{
"platform": dep_job.attributes.get("build_platform"),
@@ -117,6 +104,7 @@ def add_requirements_link(config, jobs):
def make_task_description(config, jobs):
for job in jobs:
dep_job = get_primary_dependency(config, job)
+ assert dep_job
attributes = dep_job.attributes
signing_format_scopes = []
@@ -127,6 +115,7 @@ def make_task_description(config, jobs):
is_shippable = dep_job.attributes.get("shippable", False)
build_platform = dep_job.attributes.get("build_platform")
+ assert build_platform
treeherder = None
if "partner" not in config.kind and "eme-free" not in config.kind:
treeherder = job.get("treeherder", {})
@@ -161,9 +150,9 @@ def make_task_description(config, jobs):
label = job["label"]
description = (
- "Initial Signing for locale '{locale}' for build '"
+ "Signing of locale(s) '{locale}' for build '"
"{build_platform}/{build_type}'".format(
- locale=attributes.get("locale", "en-US"),
+ locale=get_locales_description(attributes, "en-US"),
build_platform=build_platform,
build_type=attributes.get("build_type"),
)
@@ -209,10 +198,16 @@ def make_task_description(config, jobs):
# build-mac-{signing,notarization} uses signingscript instead of iscript
if "macosx" in build_platform and config.kind.endswith("-mac-notarization"):
- task["worker"]["mac-behavior"] = "apple_notarization"
task["scopes"] = [
add_scope_prefix(config, "signing:cert:release-apple-notarization")
]
+ task[
+ "description"
+ ] = "Notarization of '{}' locales for build '{}/{}'".format(
+ get_locales_description(attributes, "en-US"),
+ build_platform,
+ attributes.get("build_type"),
+ )
elif "macosx" in build_platform:
# iscript overrides
task["worker"]["mac-behavior"] = "mac_sign_and_pkg"
diff --git a/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py b/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py
index 0c31f48cc0..2b846604f1 100644
--- a/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py
+++ b/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py
@@ -1,3 +1,4 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
diff --git a/taskcluster/gecko_taskgraph/transforms/source_test.py b/taskcluster/gecko_taskgraph/transforms/source_test.py
index e3eeb7c819..266637a7a0 100644
--- a/taskcluster/gecko_taskgraph/transforms/source_test.py
+++ b/taskcluster/gecko_taskgraph/transforms/source_test.py
@@ -1,3 +1,4 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
@@ -264,8 +265,8 @@ def remove_optimization_on_central(config, jobs):
if not job.get("attributes", {}).get("code-review", False):
yield job
continue
- if "when" not in job:
- yield job
- continue
- del job["when"]
+ if "when" in job:
+ del job["when"]
+ if "optimization" in job and "skip-unless-mozlint" in job["optimization"]:
+ del job["optimization"]
yield job
diff --git a/taskcluster/gecko_taskgraph/transforms/task.py b/taskcluster/gecko_taskgraph/transforms/task.py
index 3129742ea9..aa77c46d65 100644
--- a/taskcluster/gecko_taskgraph/transforms/task.py
+++ b/taskcluster/gecko_taskgraph/transforms/task.py
@@ -208,7 +208,7 @@ TC_TREEHERDER_SCHEMA_URL = (
UNKNOWN_GROUP_NAME = (
- "Treeherder group {} (from {}) has no name; " "add it to taskcluster/ci/config.yml"
+ "Treeherder group {} (from {}) has no name; " "add it to taskcluster/config.yml"
)
V2_ROUTE_TEMPLATES = [
@@ -313,7 +313,7 @@ def index_builder(name):
UNSUPPORTED_INDEX_PRODUCT_ERROR = """\
The gecko-v2 product {product} is not in the list of configured products in
-`taskcluster/ci/config.yml'.
+`taskcluster/config.yml'.
"""
@@ -842,6 +842,7 @@ def build_generic_worker_payload(config, task, task_def):
# behavior for mac iscript
Optional("mac-behavior"): Any(
"apple_notarization",
+ "apple_notarization_stacked",
"mac_sign_and_pkg",
"mac_sign_and_pkg_hardened",
"mac_geckodriver",
@@ -1349,6 +1350,23 @@ def build_push_addons_payload(config, task, task_def):
],
},
Optional("merge-info"): object,
+ Optional("android-l10n-import-info"): {
+ Required("from-repo-url"): str,
+ Required("toml-info"): [
+ {
+ Required("toml-path"): str,
+ Required("dest-path"): str,
+ }
+ ],
+ },
+ Optional("android-l10n-sync-info"): {
+ Required("from-repo-url"): str,
+ Required("toml-info"): [
+ {
+ Required("toml-path"): str,
+ }
+ ],
+ },
},
)
def build_treescript_payload(config, task, task_def):
@@ -1412,6 +1430,38 @@ def build_treescript_payload(config, task, task_def):
task_def["payload"]["merge_info"] = merge_info
actions.append("merge_day")
+ if worker.get("android-l10n-import-info"):
+ android_l10n_import_info = {}
+ for k, v in worker["android-l10n-import-info"].items():
+ android_l10n_import_info[k.replace("-", "_")] = worker[
+ "android-l10n-import-info"
+ ][k]
+ android_l10n_import_info["toml_info"] = [
+ {
+ param_name.replace("-", "_"): param_value
+ for param_name, param_value in entry.items()
+ }
+ for entry in worker["android-l10n-import-info"]["toml-info"]
+ ]
+ task_def["payload"]["android_l10n_import_info"] = android_l10n_import_info
+ actions.append("android_l10n_import")
+
+ if worker.get("android-l10n-sync-info"):
+ android_l10n_sync_info = {}
+ for k, v in worker["android-l10n-sync-info"].items():
+ android_l10n_sync_info[k.replace("-", "_")] = worker[
+ "android-l10n-sync-info"
+ ][k]
+ android_l10n_sync_info["toml_info"] = [
+ {
+ param_name.replace("-", "_"): param_value
+ for param_name, param_value in entry.items()
+ }
+ for entry in worker["android-l10n-sync-info"]["toml-info"]
+ ]
+ task_def["payload"]["android_l10n_sync_info"] = android_l10n_sync_info
+ actions.append("android_l10n_sync")
+
if worker["push"]:
actions.append("push")
@@ -1563,7 +1613,7 @@ def task_name_from_label(config, tasks):
UNSUPPORTED_SHIPPING_PRODUCT_ERROR = """\
The shipping product {product} is not in the list of configured products in
-`taskcluster/ci/config.yml'.
+`taskcluster/config.yml'.
"""
diff --git a/taskcluster/gecko_taskgraph/transforms/test/__init__.py b/taskcluster/gecko_taskgraph/transforms/test/__init__.py
index 92704bf18c..19ab8d289f 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/__init__.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/__init__.py
@@ -27,6 +27,7 @@ from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by
from voluptuous import Any, Exclusive, Optional, Required
from gecko_taskgraph.optimize.schema import OptimizationSchema
+from gecko_taskgraph.transforms.job import job_description_schema
from gecko_taskgraph.transforms.test.other import get_mobile_project
from gecko_taskgraph.util.chunking import manifest_loaders
@@ -118,7 +119,9 @@ test_description_schema = Schema(
Required("run-without-variant"): optionally_keyed_by("test-platform", bool),
# The EC2 instance size to run these tests on.
Required("instance-size"): optionally_keyed_by(
- "test-platform", Any("default", "large", "xlarge")
+ "test-platform",
+ "variant",
+ Any("default", "large", "large-noscratch", "xlarge", "xlarge-noscratch"),
),
# type of virtualization or hardware required by test.
Required("virtualization"): optionally_keyed_by(
@@ -265,11 +268,14 @@ test_description_schema = Schema(
str,
None,
{Required("index"): str, Required("name"): str},
+ {Required("upstream-task"): str, Required("name"): str},
),
),
# A list of artifacts to install from 'fetch' tasks. Validation deferred
# to 'job' transforms.
Optional("fetches"): object,
+ # A list of extra dependencies
+ Optional("dependencies"): object,
# Raptor / browsertime specific keys, defer validation to 'raptor.py'
# transform.
Optional("raptor"): object,
@@ -279,6 +285,8 @@ test_description_schema = Schema(
Optional("subtest"): str,
# Define if a given task supports artifact builds or not, see bug 1695325.
Optional("supports-artifact-builds"): bool,
+ # Version of python used to run the task
+ Optional("use-python"): job_description_schema["use-python"],
}
)
@@ -346,6 +354,7 @@ def set_defaults(config, tasks):
task.setdefault("run-without-variant", True)
task.setdefault("variants", [])
task.setdefault("supports-artifact-builds", True)
+ task.setdefault("use-python", "system")
task["mozharness"].setdefault("extra-options", [])
task["mozharness"].setdefault("requires-signed-builds", False)
@@ -484,6 +493,9 @@ def make_job_description(config, tasks):
if task["mozharness"]["requires-signed-builds"] is True:
jobdesc["dependencies"]["build-signing"] = task["build-signing-label"]
+ if "dependencies" in task:
+ jobdesc["dependencies"].update(task["dependencies"])
+
if "expires-after" in task:
jobdesc["expires-after"] = task["expires-after"]
diff --git a/taskcluster/gecko_taskgraph/transforms/test/chunk.py b/taskcluster/gecko_taskgraph/transforms/test/chunk.py
index 7f832c57df..8219c41664 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/chunk.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/chunk.py
@@ -44,8 +44,7 @@ def set_test_verify_chunks(config, tasks):
task["chunks"] = perfile_number_of_chunks(
is_try(config.params),
env.get("MOZHARNESS_TEST_PATHS", ""),
- config.params.get("head_repository", ""),
- config.params.get("head_rev", ""),
+ frozenset(config.params["files_changed"]),
task["test-name"],
)
diff --git a/taskcluster/gecko_taskgraph/transforms/test/other.py b/taskcluster/gecko_taskgraph/transforms/test/other.py
index b8cb95cff7..e01691c05a 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/other.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/other.py
@@ -2,6 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import copy
import hashlib
import json
import re
@@ -12,7 +13,11 @@ from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import keymatch
from taskgraph.util.keyed_by import evaluate_keyed_by
from taskgraph.util.schema import Schema, resolve_keyed_by
-from taskgraph.util.taskcluster import get_artifact_path, get_index_url
+from taskgraph.util.taskcluster import (
+ get_artifact_path,
+ get_artifact_url,
+ get_index_url,
+)
from voluptuous import Any, Optional, Required
from gecko_taskgraph.transforms.test.variant import TEST_VARIANTS
@@ -99,6 +104,25 @@ def setup_talos(config, tasks):
if config.params.get("project", None):
extra_options.append("--project=%s" % config.params["project"])
+ if "pdfpaint" in task["try-name"]:
+ max_chunks = 10
+ for chunk in range(1, max_chunks + 1):
+ new_task = copy.deepcopy(task)
+ new_task["mozharness"]["extra-options"].append(
+ f"--pdfPaintChunk={chunk}"
+ )
+ new_task["test-name"] = task["test-name"].replace(
+ "pdfpaint", f"pdfpaint-{chunk}"
+ )
+ new_task["try-name"] = task["try-name"].replace(
+ "pdfpaint", f"pdfpaint-{chunk}"
+ )
+ new_task["treeherder-symbol"] = task["treeherder-symbol"].replace(
+ "pdfpaint", f"pdfpaint-{chunk}"
+ )
+ yield new_task
+ continue
+
yield task
@@ -246,6 +270,7 @@ def handle_keyed_by(config, tasks):
"webrender-run-on-projects",
"mozharness.requires-signed-builds",
"build-signing-label",
+ "dependencies",
]
for task in tasks:
for field in fields:
@@ -292,10 +317,17 @@ def set_target(config, tasks):
target = "target.tar.bz2"
if isinstance(target, dict):
- # TODO Remove hardcoded mobile artifact prefix
- index_url = get_index_url(target["index"])
- installer_url = "{}/artifacts/public/{}".format(index_url, target["name"])
- task["mozharness"]["installer-url"] = installer_url
+ if "index" in target:
+ # TODO Remove hardcoded mobile artifact prefix
+ index_url = get_index_url(target["index"])
+ installer_url = "{}/artifacts/public/{}".format(
+ index_url, target["name"]
+ )
+ task["mozharness"]["installer-url"] = installer_url
+ else:
+ task["mozharness"]["installer-url"] = get_artifact_url(
+ f'<{target["upstream-task"]}>', target["name"]
+ )
else:
task["mozharness"]["build-artifact-name"] = get_artifact_path(task, target)
@@ -363,50 +395,44 @@ def setup_browsertime(config, tasks):
cd_fetches = {
"android.*": [
- "linux64-chromedriver-120",
- "linux64-chromedriver-121",
"linux64-chromedriver-122",
+ "linux64-chromedriver-123",
+ "linux64-chromedriver-124",
],
"linux.*": [
- "linux64-chromedriver-120",
- "linux64-chromedriver-121",
"linux64-chromedriver-122",
+ "linux64-chromedriver-123",
+ "linux64-chromedriver-124",
],
"macosx1015.*": [
- "mac64-chromedriver-120",
- "mac64-chromedriver-121",
"mac64-chromedriver-122",
+ "mac64-chromedriver-123",
+ "mac64-chromedriver-124",
],
"macosx1400.*": [
- "mac-arm-chromedriver-120",
- "mac-arm-chromedriver-121",
"mac-arm-chromedriver-122",
+ "mac-arm-chromedriver-123",
+ "mac-arm-chromedriver-124",
],
"windows.*aarch64.*": [
- "win32-chromedriver-120",
- "win32-chromedriver-121",
- "win32-chromedriver-122",
- ],
- "windows.*-32.*": [
- "win32-chromedriver-120",
"win32-chromedriver-121",
"win32-chromedriver-122",
+ "win32-chromedriver-123",
],
"windows.*-64.*": [
- "win32-chromedriver-120",
- "win32-chromedriver-121",
- "win32-chromedriver-122",
+ "win64-chromedriver-123",
+ "win64-chromedriver-124",
],
}
chromium_fetches = {
- "linux.*": ["linux64-chromium"],
- "macosx1015.*": ["mac-chromium"],
- "macosx1400.*": ["mac-chromium-arm"],
- "windows.*aarch64.*": ["win32-chromium"],
- "windows.*-32.*": ["win32-chromium"],
- "windows.*-64.*": ["win64-chromium"],
- "android.*": ["linux64-chromium"],
+ "linux.*": ["linux64-chromiumdriver"],
+ "macosx1015.*": ["mac-chromiumdriver"],
+ "macosx1400.*": ["mac-chromiumdriver-arm"],
+ "windows.*aarch64.*": ["win32-chromiumdriver"],
+ "windows.*-32.*": ["win32-chromiumdriver"],
+ "windows.*-64.*": ["win64-chromiumdriver"],
+ "android.*": ["linux64-chromiumdriver"],
}
cd_extracted_name = {
@@ -419,11 +445,7 @@ def setup_browsertime(config, tasks):
# Only add the chromedriver fetches when chrome is running
for platform in cd_fetches:
fs["by-test-platform"][platform].extend(cd_fetches[platform])
- if (
- "--app=chromium" in extra_options
- or "--app=custom-car" in extra_options
- or "--app=cstm-car-m" in extra_options
- ):
+ if "--app=custom-car" in extra_options or "--app=cstm-car-m" in extra_options:
for platform in chromium_fetches:
fs["by-test-platform"][platform].extend(chromium_fetches[platform])
@@ -553,7 +575,9 @@ def enable_code_coverage(config, tasks):
yield task
continue
task["mozharness"].setdefault("extra-options", []).append("--code-coverage")
- task["instance-size"] = "xlarge"
+ task["instance-size"] = "xlarge-noscratch"
+ if "jittest" in task["test-name"]:
+ task["instance-size"] = "xlarge"
# Temporarily disable Mac tests on mozilla-central
if "mac" in task["build-platform"]:
@@ -791,7 +815,7 @@ test_setting_description_schema = Schema(
},
Optional("device"): str,
Optional("display"): "wayland",
- Optional("machine"): Any("ref-hw-2017", "hw-ref"),
+ Optional("machine"): "hw-ref",
},
"build": {
Required("type"): Any("opt", "debug", "debug-isolated-process"),
@@ -852,7 +876,6 @@ def set_test_setting(config, tasks):
# TODO Rename these so they don't have a dash.
dash_attrs = [
"clang-trunk",
- "ref-hw-2017",
"hw-ref",
]
dash_token = "%D%"
@@ -908,9 +931,6 @@ def set_test_setting(config, tasks):
if parts[0].isdigit():
os_build = parts.pop(0)
- if parts and parts[0] == "ref-hw-2017":
- machine = parts.pop(0)
-
if parts and parts[0] == "hw-ref":
machine = parts.pop(0)
@@ -1102,6 +1122,7 @@ def set_schedules_components(config, tasks):
schedules.add(category)
schedules.add(platform_family(task["build-platform"]))
+ schedules.add("firefox")
task["schedules-component"] = sorted(schedules)
yield task
diff --git a/taskcluster/gecko_taskgraph/transforms/test/raptor.py b/taskcluster/gecko_taskgraph/transforms/test/raptor.py
index 18e21e6a1e..ca35749037 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/raptor.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/raptor.py
@@ -76,7 +76,6 @@ def split_apps(config, tests):
app_symbols = {
"chrome": "ChR",
"chrome-m": "ChR",
- "chromium": "Cr",
"fenix": "fenix",
"refbrow": "refbrow",
"safari": "Saf",
diff --git a/taskcluster/gecko_taskgraph/transforms/test/variant.py b/taskcluster/gecko_taskgraph/transforms/test/variant.py
index bda91b2f25..6ee6c429f0 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/variant.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/variant.py
@@ -78,6 +78,14 @@ def split_variants(config, tasks):
remaining_variants.append(name)
return remaining_variants
+ def replace_task_items(task_key, variant_key):
+ for item in variant_key:
+ if isinstance(variant_key[item], dict):
+ task_key[item] = replace_task_items(task_key[item], variant_key[item])
+ else:
+ task_key[item] = variant_key[item]
+ return task_key
+
def apply_variant(variant, task):
task["description"] = variant["description"].format(**task)
@@ -94,7 +102,9 @@ def split_variants(config, tasks):
task["variant-suffix"] += suffix
# Replace and/or merge the configuration.
- task.update(variant.get("replace", {}))
+
+ # we only want to update the leaf node, the the entire top level dict
+ task = replace_task_items(task, variant.get("replace", {}))
return merge(task, variant.get("merge", {}))
expired_variants = find_expired_variants(TEST_VARIANTS)
diff --git a/taskcluster/gecko_taskgraph/transforms/test/worker.py b/taskcluster/gecko_taskgraph/transforms/test/worker.py
index 873347459c..51b12de51d 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/worker.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/worker.py
@@ -7,8 +7,10 @@ from taskgraph.transforms.base import TransformSequence
# default worker types keyed by instance-size
LINUX_WORKER_TYPES = {
"large": "t-linux-large",
+ "large-noscratch": "t-linux-large-noscratch",
"xlarge": "t-linux-xlarge",
- "default": "t-linux-large",
+ "xlarge-noscratch": "t-linux-xlarge-noscratch",
+ "default": "t-linux-large-noscratch",
}
# windows worker types keyed by test-platform and virtualization
@@ -23,11 +25,6 @@ WINDOWS_WORKER_TYPES = {
"virtual-with-gpu": "t-win10-64-gpu-s",
"hardware": "t-win10-64-1803-hw",
},
- "windows10-64-ref-hw-2017": {
- "virtual": "t-win10-64",
- "virtual-with-gpu": "t-win10-64-gpu-s",
- "hardware": "t-win10-64-ref-hw",
- },
"windows11-64-2009-hw-ref-shippable": {
"virtual": "win11-64-2009-hw-ref",
"virtual-with-gpu": "win11-64-2009-hw-ref",
@@ -130,12 +127,8 @@ def set_worker_type(config, tasks):
elif test_platform.startswith("win"):
# figure out what platform the job needs to run on
if task["virtualization"] == "hardware":
- # some jobs like talos and reftest run on real h/w - those are all win10
- if test_platform.startswith("windows10-64-ref-hw-2017"):
- win_worker_type_platform = WINDOWS_WORKER_TYPES[
- "windows10-64-ref-hw-2017"
- ]
- elif test_platform.startswith("windows11-64-2009-hw-ref"):
+ # some jobs like talos and reftest run on real h/w
+ if test_platform.startswith("windows11-64-2009-hw-ref"):
win_worker_type_platform = WINDOWS_WORKER_TYPES[
"windows11-64-2009-hw-ref"
]
diff --git a/taskcluster/gecko_taskgraph/transforms/test_apk.py b/taskcluster/gecko_taskgraph/transforms/test_apk.py
new file mode 100644
index 0000000000..b00657b91e
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/transforms/test_apk.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Apply some defaults and minor modifications to the jobs defined in the test
+kinds.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import resolve_keyed_by
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def resolve_keys(config, tasks):
+ for task in tasks:
+ for key in (
+ "routes",
+ "scopes",
+ "extra.notify",
+ ):
+ resolve_keyed_by(
+ task,
+ key,
+ item_name=task["name"],
+ **{
+ "level": config.params["level"],
+ }
+ )
+ yield task
diff --git a/taskcluster/gecko_taskgraph/transforms/update_verify_config.py b/taskcluster/gecko_taskgraph/transforms/update_verify_config.py
index 2d1cd40877..4e516f173c 100644
--- a/taskcluster/gecko_taskgraph/transforms/update_verify_config.py
+++ b/taskcluster/gecko_taskgraph/transforms/update_verify_config.py
@@ -101,6 +101,8 @@ def add_command(config, tasks):
get_branch_rev(config),
"--output-file",
"update-verify.cfg",
+ "--local-repo",
+ ".",
]
repo_path = urlsplit(get_branch_repo(config)).path.lstrip("/")
diff --git a/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py b/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py
index aea948f90e..3828e912ec 100644
--- a/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py
+++ b/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py
@@ -3,7 +3,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the upload-generated-files task description template,
-taskcluster/ci/upload-generated-sources/kind.yml, into an actual task description.
+taskcluster/kinds/upload-generated-sources/kind.yml, into an actual task description.
"""
from taskgraph.transforms.base import TransformSequence
diff --git a/taskcluster/gecko_taskgraph/transforms/upload_symbols.py b/taskcluster/gecko_taskgraph/transforms/upload_symbols.py
index 5c0bf18cb0..b216d2c8e1 100644
--- a/taskcluster/gecko_taskgraph/transforms/upload_symbols.py
+++ b/taskcluster/gecko_taskgraph/transforms/upload_symbols.py
@@ -3,7 +3,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the upload-symbols task description template,
-taskcluster/ci/upload-symbols/job-template.yml into an actual task description.
+taskcluster/kinds/upload-symbols/job-template.yml into an actual task description.
"""
diff --git a/taskcluster/gecko_taskgraph/util/backstop.py b/taskcluster/gecko_taskgraph/util/backstop.py
index 26c9a4fb91..18c9166083 100644
--- a/taskcluster/gecko_taskgraph/util/backstop.py
+++ b/taskcluster/gecko_taskgraph/util/backstop.py
@@ -37,22 +37,17 @@ def is_backstop(
return True
project = params["project"]
- pushid = int(params["pushlog_id"])
- pushdate = int(params["pushdate"])
-
if project in TRY_PROJECTS:
return False
if project not in integration_projects:
return True
- # On every Nth push, want to run all tasks.
- if pushid % push_interval == 0:
- return True
-
- if time_interval <= 0:
+ # This push was explicitly set to run nothing (e.g via DONTBUILD), so
+ # shouldn't be a backstop candidate.
+ if params["target_tasks_method"] == "nothing":
return False
- # We also want to ensure we run all tasks at least once per N minutes.
+ # Find the last backstop to compute push and time intervals.
subs = {"trust-domain": trust_domain, "project": project}
index = BACKSTOP_INDEX.format(**subs)
@@ -67,9 +62,7 @@ def is_backstop(
return True
try:
- last_pushdate = get_artifact(last_backstop_id, "public/parameters.yml")[
- "pushdate"
- ]
+ last_params = get_artifact(last_backstop_id, "public/parameters.yml")
except HTTPError as e:
# If the last backstop decision task exists in the index, but
# parameters.yml isn't available yet, it means the decision task is
@@ -79,6 +72,15 @@ def is_backstop(
return False
raise
- if (pushdate - last_pushdate) / 60 >= time_interval:
+ # On every Nth push, want to run all tasks.
+ if int(params["pushlog_id"]) - int(last_params["pushlog_id"]) >= push_interval:
+ return True
+
+ if time_interval <= 0:
+ return False
+
+ # We also want to ensure we run all tasks at least once per N minutes.
+ if (params["pushdate"] - last_params["pushdate"]) / 60 >= time_interval:
return True
+
return False
diff --git a/taskcluster/gecko_taskgraph/util/chunking.py b/taskcluster/gecko_taskgraph/util/chunking.py
index a0ed56de78..9d8b98e07b 100644
--- a/taskcluster/gecko_taskgraph/util/chunking.py
+++ b/taskcluster/gecko_taskgraph/util/chunking.py
@@ -24,8 +24,8 @@ here = os.path.abspath(os.path.dirname(__file__))
resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader)
TEST_VARIANTS = {}
-if os.path.exists(os.path.join(GECKO, "taskcluster", "ci", "test", "variants.yml")):
- TEST_VARIANTS = load_yaml(GECKO, "taskcluster", "ci", "test", "variants.yml")
+if os.path.exists(os.path.join(GECKO, "taskcluster", "kinds", "test", "variants.yml")):
+ TEST_VARIANTS = load_yaml(GECKO, "taskcluster", "kinds", "test", "variants.yml")
WPT_SUBSUITES = {
"canvas": "html/canvas",
@@ -101,8 +101,8 @@ def guess_mozinfo_from_task(task, repo=""):
("linux", "1804"): "18.04",
("macosx", "1015"): "10.15",
("macosx", "1100"): "11.00",
- ("windows", "7"): "6.1",
("windows", "10"): "10.0",
+ ("windows", "11"): "11.0",
}
for (name, old_ver), new_ver in os_versions.items():
if p_os["name"] == name and p_os["version"] == old_ver:
diff --git a/taskcluster/gecko_taskgraph/util/docker.py b/taskcluster/gecko_taskgraph/util/docker.py
index e8de7d1fdb..51e3f7e484 100644
--- a/taskcluster/gecko_taskgraph/util/docker.py
+++ b/taskcluster/gecko_taskgraph/util/docker.py
@@ -298,7 +298,7 @@ class ImagePathsMap(Mapping):
self.__update_image_paths(jobs, image_dir)
-image_paths = ImagePathsMap("taskcluster/ci/docker-image/kind.yml")
+image_paths = ImagePathsMap("taskcluster/kinds/docker-image/kind.yml")
def image_path(name):
diff --git a/taskcluster/gecko_taskgraph/util/hg.py b/taskcluster/gecko_taskgraph/util/hg.py
index 18a92fbd0d..d03af87f13 100644
--- a/taskcluster/gecko_taskgraph/util/hg.py
+++ b/taskcluster/gecko_taskgraph/util/hg.py
@@ -96,15 +96,15 @@ def get_push_data(repository, project, push_id_start, push_id_end):
@memoize
-def get_json_automationrelevance(repository, revision):
- url = "{}/json-automationrelevance/{}".format(repository.rstrip("/"), revision)
+def get_json_pushchangedfiles(repository, revision):
+ url = "{}/json-pushchangedfiles/{}".format(repository.rstrip("/"), revision)
logger.debug("Querying version control for metadata: %s", url)
- def get_automationrelevance():
- response = requests.get(url, timeout=30)
+ def get_pushchangedfiles():
+ response = requests.get(url, timeout=60)
return response.json()
- return retry(get_automationrelevance, attempts=10, sleeptime=10)
+ return retry(get_pushchangedfiles, attempts=10, sleeptime=10)
def get_hg_revision_branch(root, revision):
diff --git a/taskcluster/gecko_taskgraph/util/partners.py b/taskcluster/gecko_taskgraph/util/partners.py
index 2546e1ae88..f4b8b187cf 100644
--- a/taskcluster/gecko_taskgraph/util/partners.py
+++ b/taskcluster/gecko_taskgraph/util/partners.py
@@ -523,7 +523,7 @@ def apply_partner_priority(config, jobs):
# Reduce the priority of the partner repack jobs because they don't block QE. Meanwhile
# leave EME-free jobs alone because they do, and they'll get the branch priority like the rest
# of the release. Only bother with this in production, not on staging releases on try.
- # medium is the same as mozilla-central, see taskcluster/ci/config.yml. ie higher than
+ # medium is the same as mozilla-central, see taskcluster/config.yml. ie higher than
# integration branches because we don't want to wait a lot for the graph to be done, but
# for multiple releases the partner tasks always wait for non-partner.
if (
diff --git a/taskcluster/gecko_taskgraph/util/perfile.py b/taskcluster/gecko_taskgraph/util/perfile.py
index 4e82d87dad..4c18ca98be 100644
--- a/taskcluster/gecko_taskgraph/util/perfile.py
+++ b/taskcluster/gecko_taskgraph/util/perfile.py
@@ -12,15 +12,12 @@ import taskgraph
from mozbuild.util import memoize
from mozpack.path import match as mozpackmatch
-from gecko_taskgraph import files_changed
-
-from .. import GECKO
-
logger = logging.getLogger(__name__)
@memoize
-def perfile_number_of_chunks(is_try, try_task_config, head_repository, head_rev, type):
+def perfile_number_of_chunks(is_try, try_task_config, files_changed, type):
+ changed_files = set(files_changed)
if taskgraph.fast and not is_try:
# When iterating on taskgraph changes, the exact number of chunks that
# test-verify runs usually isn't important, so skip it when going fast.
@@ -55,17 +52,11 @@ def perfile_number_of_chunks(is_try, try_task_config, head_repository, head_rev,
# Returning 0 means no tests to run, this captures non test-verify tasks
return 1
- changed_files = set()
if try_task_config:
suite_to_paths = json.loads(try_task_config)
specified_files = itertools.chain.from_iterable(suite_to_paths.values())
changed_files.update(specified_files)
- if is_try:
- changed_files.update(files_changed.get_locally_changed_files(GECKO))
- else:
- changed_files.update(files_changed.get_changed_files(head_repository, head_rev))
-
test_count = 0
for pattern in file_patterns:
for path in changed_files:
diff --git a/taskcluster/gecko_taskgraph/util/platforms.py b/taskcluster/gecko_taskgraph/util/platforms.py
index 2c423223fe..3010b32792 100644
--- a/taskcluster/gecko_taskgraph/util/platforms.py
+++ b/taskcluster/gecko_taskgraph/util/platforms.py
@@ -27,7 +27,8 @@ _executable_extension = {
_architectures = {
r"linux\b.*": "x86",
- r"linux64\b.*": "x86_64",
+ r"linux64\b(?!-aarch64).*": "x86_64",
+ r"linux64-aarch64\b.*": "aarch64",
r"macosx64\b.*": "macos-x86_64-aarch64",
r"win32\b.*": "x86",
r"win64\b(?!-aarch64).*": "x86_64",
diff --git a/taskcluster/gecko_taskgraph/util/signed_artifacts.py b/taskcluster/gecko_taskgraph/util/signed_artifacts.py
index 2467ff8046..302a2e09f7 100644
--- a/taskcluster/gecko_taskgraph/util/signed_artifacts.py
+++ b/taskcluster/gecko_taskgraph/util/signed_artifacts.py
@@ -57,7 +57,7 @@ def generate_specifications_of_artifacts_to_sign(
elif "macosx" in build_platform:
langpack_formats = []
if is_notarization_kind(config.kind):
- formats = ["apple_notarization"]
+ formats = ["apple_notarization_stacked"]
artifacts_specifications = [
{
"artifacts": [
@@ -98,14 +98,14 @@ def generate_specifications_of_artifacts_to_sign(
"artifacts": [
get_artifact_path(job, "{locale}/setup.exe"),
],
- "formats": ["autograph_authenticode_sha2"],
+ "formats": ["autograph_authenticode_202404"],
},
{
"artifacts": [
get_artifact_path(job, "{locale}/target.zip"),
],
"formats": [
- "autograph_authenticode_sha2",
+ "autograph_authenticode_202404",
"autograph_widevine",
"autograph_omnija",
],
diff --git a/taskcluster/gecko_taskgraph/util/taskcluster.py b/taskcluster/gecko_taskgraph/util/taskcluster.py
index cddb01fd37..826eee4cdb 100644
--- a/taskcluster/gecko_taskgraph/util/taskcluster.py
+++ b/taskcluster/gecko_taskgraph/util/taskcluster.py
@@ -24,7 +24,7 @@ def insert_index(index_path, task_id, data=None, use_proxy=False):
index_url = get_index_url(index_path, use_proxy=use_proxy)
# Find task expiry.
- expires = get_task_definition(task_id, use_proxy=use_proxy)["expires"]
+ expires = get_task_definition(task_id, use_proxy)["expires"]
response = _do_request(
index_url,