summaryrefslogtreecommitdiffstats
path: root/taskcluster/gecko_taskgraph
diff options
context:
space:
mode:
Diffstat (limited to 'taskcluster/gecko_taskgraph')
-rw-r--r--taskcluster/gecko_taskgraph/config.py3
-rw-r--r--taskcluster/gecko_taskgraph/files_changed.py16
-rw-r--r--taskcluster/gecko_taskgraph/main.py10
-rw-r--r--taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml6
-rw-r--r--taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml2
-rw-r--r--taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml56
-rw-r--r--taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml2
-rw-r--r--taskcluster/gecko_taskgraph/optimize/mozlint.py96
-rw-r--r--taskcluster/gecko_taskgraph/optimize/schema.py2
-rw-r--r--taskcluster/gecko_taskgraph/optimize/strategies.py5
-rw-r--r--taskcluster/gecko_taskgraph/target_tasks.py120
-rw-r--r--taskcluster/gecko_taskgraph/test/automationrelevance.json358
-rw-r--r--taskcluster/gecko_taskgraph/test/conftest.py2
-rw-r--r--taskcluster/gecko_taskgraph/test/test_actions_util.py1
-rw-r--r--taskcluster/gecko_taskgraph/test/test_files_changed.py46
-rw-r--r--taskcluster/gecko_taskgraph/test/test_optimize_strategies.py75
-rw-r--r--taskcluster/gecko_taskgraph/test/test_transforms_job.py2
-rw-r--r--taskcluster/gecko_taskgraph/transforms/bouncer_submission.py1
-rw-r--r--taskcluster/gecko_taskgraph/transforms/build.py4
-rw-r--r--taskcluster/gecko_taskgraph/transforms/build_schedules.py18
-rw-r--r--taskcluster/gecko_taskgraph/transforms/diffoscope.py2
-rw-r--r--taskcluster/gecko_taskgraph/transforms/github_sync.py23
-rw-r--r--taskcluster/gecko_taskgraph/transforms/perftest.py1
-rw-r--r--taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py2
-rw-r--r--taskcluster/gecko_taskgraph/transforms/signing.py23
-rw-r--r--taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py1
-rw-r--r--taskcluster/gecko_taskgraph/transforms/source_test.py9
-rw-r--r--taskcluster/gecko_taskgraph/transforms/task.py7
-rw-r--r--taskcluster/gecko_taskgraph/transforms/test/other.py26
-rw-r--r--taskcluster/gecko_taskgraph/transforms/test/variant.py12
-rw-r--r--taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py2
-rw-r--r--taskcluster/gecko_taskgraph/transforms/upload_symbols.py2
-rw-r--r--taskcluster/gecko_taskgraph/util/chunking.py4
-rw-r--r--taskcluster/gecko_taskgraph/util/docker.py2
-rw-r--r--taskcluster/gecko_taskgraph/util/hg.py8
-rw-r--r--taskcluster/gecko_taskgraph/util/partners.py2
-rw-r--r--taskcluster/gecko_taskgraph/util/platforms.py3
-rw-r--r--taskcluster/gecko_taskgraph/util/signed_artifacts.py2
-rw-r--r--taskcluster/gecko_taskgraph/util/taskcluster.py2
39 files changed, 365 insertions, 593 deletions
diff --git a/taskcluster/gecko_taskgraph/config.py b/taskcluster/gecko_taskgraph/config.py
index 7b6237f9fb..df4a5665e3 100644
--- a/taskcluster/gecko_taskgraph/config.py
+++ b/taskcluster/gecko_taskgraph/config.py
@@ -4,6 +4,7 @@
from taskgraph.util.schema import Schema, optionally_keyed_by
from voluptuous import Any, Optional, Required
+from voluptuous.validators import Length
graph_config_schema = Schema(
{
@@ -20,7 +21,7 @@ graph_config_schema = Schema(
Required("product-dir"): str,
Required("treeherder"): {
# Mapping of treeherder group symbols to descriptive names
- Required("group-names"): {str: str}
+ Required("group-names"): {str: Length(max=100)}
},
Required("index"): {Required("products"): [str]},
Required("try"): {
diff --git a/taskcluster/gecko_taskgraph/files_changed.py b/taskcluster/gecko_taskgraph/files_changed.py
index c814df0806..0352fb9d7e 100644
--- a/taskcluster/gecko_taskgraph/files_changed.py
+++ b/taskcluster/gecko_taskgraph/files_changed.py
@@ -16,7 +16,7 @@ from mozpack.path import match as mozpackmatch
from mozversioncontrol import InvalidRepoPath, get_repository_object
from gecko_taskgraph import GECKO
-from gecko_taskgraph.util.hg import get_json_automationrelevance
+from gecko_taskgraph.util.hg import get_json_pushchangedfiles
logger = logging.getLogger(__name__)
@@ -27,9 +27,8 @@ def get_changed_files(repository, revision):
Get the set of files changed in the push headed by the given revision.
Responses are cached, so multiple calls with the same arguments are OK.
"""
- contents = get_json_automationrelevance(repository, revision)
try:
- changesets = contents["changesets"]
+ return get_json_pushchangedfiles(repository, revision)["files"]
except KeyError:
# We shouldn't hit this error in CI.
if os.environ.get("MOZ_AUTOMATION"):
@@ -39,17 +38,6 @@ def get_changed_files(repository, revision):
# version control.
return get_locally_changed_files(GECKO)
- logger.debug("{} commits influencing task scheduling:".format(len(changesets)))
- changed_files = set()
- for c in changesets:
- desc = "" # Support empty desc
- if c["desc"]:
- desc = c["desc"].splitlines()[0].encode("ascii", "ignore")
- logger.debug(" {cset} {desc}".format(cset=c["node"][0:12], desc=desc))
- changed_files |= set(c["files"])
-
- return changed_files
-
def check(params, file_patterns):
"""Determine whether any of the files changed in the indicated push to
diff --git a/taskcluster/gecko_taskgraph/main.py b/taskcluster/gecko_taskgraph/main.py
index e261f26c80..c6d7a4a3c8 100644
--- a/taskcluster/gecko_taskgraph/main.py
+++ b/taskcluster/gecko_taskgraph/main.py
@@ -108,7 +108,7 @@ def get_filtered_taskgraph(taskgraph, tasksregex, exclude_keys):
for key in exclude_keys:
obj = task_dict
attrs = key.split(".")
- while attrs[0] in obj:
+ while obj and attrs[0] in obj:
if len(attrs) == 1:
del obj[attrs[0]]
break
@@ -386,10 +386,10 @@ def show_taskgraph(options):
output_file = options["output_file"]
if options["diff"]:
- # --root argument is taskgraph's config at <repo>/taskcluster/ci
+ # --root argument is taskgraph's config at <repo>/taskcluster
repo_root = os.getcwd()
if options["root"]:
- repo_root = f"{options['root']}/../.."
+ repo_root = f"{options['root']}/.."
repo = get_repository(repo_root)
if not repo.working_directory_clean():
@@ -707,7 +707,7 @@ def decision(options):
@argument(
"--root",
"-r",
- default="taskcluster/ci",
+ default="taskcluster",
help="root of the taskgraph definition relative to topsrcdir",
)
def action_callback(options):
@@ -743,7 +743,7 @@ def action_callback(options):
@argument(
"--root",
"-r",
- default="taskcluster/ci",
+ default="taskcluster",
help="root of the taskgraph definition relative to topsrcdir",
)
@argument(
diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml b/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml
index af0ea79aa4..d847b4fcdc 100644
--- a/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml
+++ b/taskcluster/gecko_taskgraph/manifests/firefox_candidates.yml
@@ -62,6 +62,8 @@ platform_names:
linux64-shippable: 'linux-x86_64'
linux64-devedition: 'linux-x86_64'
linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ linux64-aarch64-shippable: 'linux-aarch64'
+ linux64-aarch64-devedition: 'linux-aarch64'
macosx64-shippable: 'mac'
macosx64-devedition: 'mac'
win32-shippable: 'win32'
@@ -78,6 +80,8 @@ platform_names:
linux64-shippable: 'linux64'
linux64-devedition: 'linux64-devedition'
linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ linux64-aarch64-shippable: 'linux64-aarch64'
+ linux64-aarch64-devedition: 'linux64-aarch64-devedition'
macosx64-shippable: 'macosx64'
macosx64-devedition: 'macosx64-devedition'
win32-shippable: 'win32'
@@ -94,6 +98,8 @@ platform_names:
linux64-shippable: 'linux64'
linux64-devedition: 'linux64'
linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ linux64-aarch64-shippable: 'linux64-aarch64'
+ linux64-aarch64-devedition: 'linux64-aarch64'
macosx64-shippable: 'macosx64'
macosx64-devedition: 'macosx64'
win32-shippable: 'win32'
diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml b/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml
index 43ba4cbf15..7ca17662e2 100644
--- a/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml
+++ b/taskcluster/gecko_taskgraph/manifests/firefox_candidates_checksums.yml
@@ -51,6 +51,8 @@ platform_names:
linux64-shippable: 'linux-x86_64'
linux64-devedition: 'linux-x86_64'
linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ linux64-aarch64-shippable: 'linux-aarch64'
+ linux64-aarch64-devedition: 'linux-aarch64'
macosx64-shippable: 'mac'
macosx64-devedition: 'mac'
win32-shippable: 'win32'
diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml b/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml
index 421caba08f..60c39e7c53 100644
--- a/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml
+++ b/taskcluster/gecko_taskgraph/manifests/firefox_nightly.yml
@@ -57,6 +57,8 @@ platform_names:
linux-devedition: 'linux-i686'
linux64-shippable: 'linux-x86_64'
linux64-devedition: 'linux-x86_64'
+ linux64-aarch64-shippable: 'linux-aarch64'
+ linux64-aarch64-devedition: 'linux-aarch64'
linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
macosx64-shippable: 'mac'
macosx64-devedition: 'mac'
@@ -74,6 +76,8 @@ platform_names:
linux64-asan-reporter-shippable: 'linux64-asan-reporter'
linux64-shippable: 'linux64'
linux64-devedition: 'linux64'
+ linux64-aarch64-shippable: 'linux64-aarch64'
+ linux64-aarch64-devedition: 'linux64-aarch64'
macosx64-shippable: 'macosx64'
macosx64-devedition: 'macosx64'
win32-shippable: 'win32'
@@ -127,19 +131,10 @@ mapping:
checksums_path: KEY
only_for_platforms:
- linux64-shippable
+ - linux64-aarch64-shippable
destinations:
- ${year}/${month}/${upload_date}-${branch}
- latest-${branch}
- target.common.tests.tar.gz:
- <<: *default
- description: "Mixture of reftests, mochitests, UI and others, commonly bundled together in a test suite"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.common.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.common.tests.tar.gz
- target.cppunittest.tests.tar.gz:
- <<: *default
- description: "C++ unittests related in-tree test infrastructure"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.cppunittest.tests.tar.gz
target.crashreporter-symbols.zip:
<<: *default
description: "Crashreporter symbols to be consumed by Socorro"
@@ -153,51 +148,16 @@ mapping:
description: "Various compile and moz_app flags baked together in a json file"
pretty_name: firefox-${version}.${locale}.${filename_platform}.json
checksums_path: firefox-${version}.${locale}.${filename_platform}.json
- target.mochitest.tests.tar.gz:
- <<: *default
- description: "Results for running the mochitest testing framework via Javascript function calls"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.mochitest.tests.tar.gz
target.mozinfo.json:
<<: *default
description: "Various compile and moz_app flags baked together in a json file"
pretty_name: firefox-${version}.${locale}.${filename_platform}.mozinfo.json
checksums_path: firefox-${version}.${locale}.${filename_platform}.mozinfo.json
- target.reftest.tests.tar.gz:
- <<: *default
- description: "Results for running the reftest testing framework via display of two Web pages comparison"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.reftest.tests.tar.gz
- target.talos.tests.tar.gz:
- <<: *default
- description: "Results for running the talos testing framework to measure performance"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.talos.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.talos.tests.tar.gz
- target.awsy.tests.tar.gz:
- <<: *default
- description: "Results for running the awsy testing framework to track memory usage"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.awsy.tests.tar.gz
- target.test_packages.json:
- <<: *default
- description: "File containing metadata about all other files and testing harnesses specifics"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.test_packages.json
- checksums_path: firefox-${version}.${locale}.${filename_platform}.test_packages.json
target.txt:
<<: *default
description: "File containing buildid and revision"
pretty_name: firefox-${version}.${locale}.${filename_platform}.txt
checksums_path: firefox-${version}.${locale}.${filename_platform}.txt
- target.web-platform.tests.tar.gz:
- <<: *default
- description: "Results for running the webplatform testing framework to cover standard Web platform features"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.web-platform.tests.tar.gz
- target.xpcshell.tests.tar.gz:
- <<: *default
- description: "Results for running the xpcshell testing framework to enable XPConnect console application"
- pretty_name: firefox-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz
- checksums_path: firefox-${version}.${locale}.${filename_platform}.xpcshell.tests.tar.gz
target_info.txt:
<<: *default
description: "File containing the buildID"
@@ -226,6 +186,7 @@ mapping:
only_for_platforms:
- linux-shippable
- linux64-shippable
+ - linux64-aarch64-shippable
- macosx64-shippable
- win64-shippable
- win32-shippable
@@ -252,6 +213,8 @@ mapping:
only_for_platforms:
- linux-shippable
- linux64-shippable
+ - linux64-aarch64-shippable
+ - linux64-aarch64-devedition
- linux-devedition
- linux64-devedition
pretty_name: firefox-${version}.${locale}.langpack.deb
@@ -301,6 +264,7 @@ mapping:
only_for_platforms:
- linux-shippable
- linux64-shippable
+ - linux64-aarch64-shippable
- linux64-asan-reporter-shippable
pretty_name: firefox-${version}.${locale}.${filename_platform}.tar.bz2
checksums_path: firefox-${version}.${locale}.${filename_platform}.tar.bz2
@@ -322,6 +286,7 @@ mapping:
only_for_platforms:
- linux-shippable
- linux64-shippable
+ - linux64-aarch64-shippable
- linux64-asan-reporter-shippable
pretty_name: firefox-${version}.${locale}.${filename_platform}.tar.bz2.asc
checksums_path: firefox-${version}.${locale}.${filename_platform}.tar.bz2.asc
@@ -499,6 +464,7 @@ mapping:
only_for_platforms:
- linux-shippable
- linux64-shippable
+ - linux64-aarch64-shippable
pretty_name: firefox-${version}.${locale}.${filename_platform}.deb
checksums_path: firefox-${version}.${locale}.${filename_platform}.deb
update_balrog_manifest: false
diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml b/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml
index f1b81572ab..caa31f556e 100644
--- a/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml
+++ b/taskcluster/gecko_taskgraph/manifests/firefox_nightly_checksums.yml
@@ -18,6 +18,8 @@ platform_names:
linux-shippable: 'linux-i686'
linux-devedition: 'linux-i686'
linux64-shippable: 'linux-x86_64'
+ linux64-aarch64-shippable: 'linux-aarch64'
+ linux64-aarch64-devedition: 'linux-aarch64'
linux64-devedition: 'linux-x86_64'
linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
macosx64-shippable: 'mac'
diff --git a/taskcluster/gecko_taskgraph/optimize/mozlint.py b/taskcluster/gecko_taskgraph/optimize/mozlint.py
new file mode 100644
index 0000000000..0fe42ea70d
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/optimize/mozlint.py
@@ -0,0 +1,96 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+from pathlib import Path
+from typing import List, Union
+
+from mozlint.parser import Parser
+from mozlint.pathutils import filterpaths
+from taskgraph.optimize.base import OptimizationStrategy
+from taskgraph.util.path import match as match_path
+
+from gecko_taskgraph import GECKO
+
+logger = logging.getLogger(__name__)
+
+
+class TGMozlintParser(Parser):
+ """
+ Mozlint Parser that skips validation. This is needed because decision
+ tasks use sparse clones and the files themselves are not present.
+ """
+
+ def _validate(self, linter):
+ pass
+
+
+class SkipUnlessMozlint(OptimizationStrategy):
+ """
+ Optimization strategy for mozlint tests.
+
+ Uses the mozlint YAML file for each test to determine whether or not a test
+ should run.
+
+ Using:
+ - The optimization relies on having `files_changed` set in the decision task
+ parameters.
+ - Register with register_strategy. The argument is the path to MozLint YAML
+ configuration files ("/tools/lint" for mozilla-central):
+ - In source-test/mozlint.yml, set the optimization strategy for each job by filename:
+ - For Mozlint jobs that run multiple linters at once use a list of filenames:
+ """
+
+ def __init__(self, linters_path: str):
+ self.mozlint_parser = TGMozlintParser(GECKO)
+ self.linters_path = Path(GECKO) / linters_path
+
+ def should_remove_task(
+ self, task, params, mozlint_confs: Union[str, List[str]]
+ ) -> bool:
+ include = []
+ exclude = []
+ extensions = []
+ exclude_extensions = []
+ support_files = ["python/mozlint/**", "tools/lint/**"]
+
+ files_changed = params["files_changed"]
+
+ if isinstance(mozlint_confs, str):
+ mozlint_confs = [mozlint_confs]
+
+ for mozlint_conf in mozlint_confs:
+ mozlint_yaml = str(self.linters_path / mozlint_conf)
+ logger.info(f"Loading file patterns for {task.label} from {mozlint_yaml}.")
+ linter_config = self.mozlint_parser(mozlint_yaml)
+
+ for config in linter_config:
+ include += config.get("include", [])
+ exclude += config.get("exclude", [])
+ extensions += [e.strip(".") for e in config.get("extensions", [])]
+ exclude_extensions += [
+ e.strip(".") for e in config.get("exclude_extensions", [])
+ ]
+ support_files += config.get("support-files", [])
+
+ # Support files may not be part of "include" patterns, so check first
+ # Do not remove (return False) if any changed
+ for pattern in set(support_files):
+ for path in files_changed:
+ if match_path(path, pattern):
+ return False
+
+ to_lint, to_exclude = filterpaths(
+ GECKO,
+ list(files_changed),
+ include=include,
+ exclude=exclude,
+ extensions=extensions,
+ exclude_extensions=exclude_extensions,
+ )
+
+ # to_lint should be an empty list if there is nothing to check
+ if not to_lint:
+ return True
+ return False
diff --git a/taskcluster/gecko_taskgraph/optimize/schema.py b/taskcluster/gecko_taskgraph/optimize/schema.py
index a7f878cf60..a348fb177c 100644
--- a/taskcluster/gecko_taskgraph/optimize/schema.py
+++ b/taskcluster/gecko_taskgraph/optimize/schema.py
@@ -39,6 +39,8 @@ default_optimizations = (
{"upload-symbols": None},
# optimize strategy alias for reprocess-symbols tasks
{"reprocess-symbols": None},
+ # optimization strategy for mozlint tests
+ {"skip-unless-mozlint": voluptuous.Any(str, [str])},
)
OptimizationSchema = voluptuous.Any(*default_optimizations)
diff --git a/taskcluster/gecko_taskgraph/optimize/strategies.py b/taskcluster/gecko_taskgraph/optimize/strategies.py
index ffc395385b..f1ade38672 100644
--- a/taskcluster/gecko_taskgraph/optimize/strategies.py
+++ b/taskcluster/gecko_taskgraph/optimize/strategies.py
@@ -11,6 +11,8 @@ from mozbuild.util import memoize
from taskgraph.optimize.base import OptimizationStrategy, register_strategy
from taskgraph.util.path import match as match_path
+from gecko_taskgraph.optimize.mozlint import SkipUnlessMozlint
+
logger = logging.getLogger(__name__)
@@ -100,3 +102,6 @@ class SkipUnlessChanged(OptimizationStrategy):
)
return True
return False
+
+
+register_strategy("skip-unless-mozlint", args=("tools/lint",))(SkipUnlessMozlint)
diff --git a/taskcluster/gecko_taskgraph/target_tasks.py b/taskcluster/gecko_taskgraph/target_tasks.py
index e004cfb3e2..5469f9eb0d 100644
--- a/taskcluster/gecko_taskgraph/target_tasks.py
+++ b/taskcluster/gecko_taskgraph/target_tasks.py
@@ -11,7 +11,7 @@ from datetime import datetime, timedelta
from redo import retry
from taskgraph.parameters import Parameters
-from taskgraph.target_tasks import _target_task, get_method
+from taskgraph.target_tasks import get_method, register_target_task
from taskgraph.util.taskcluster import find_task_id
from gecko_taskgraph import GECKO, try_option_syntax
@@ -43,7 +43,7 @@ UNCOMMON_TRY_TASK_LABELS = [
r"linux-", # hide all linux32 tasks by default - bug 1599197
r"linux1804-32", # hide linux32 tests - bug 1599197
# Test tasks
- r"web-platform-tests.*backlog", # hide wpt jobs that are not implemented yet - bug 1572820
+ r"web-platform-tests(?!-webgpu).*backlog", # hide wpt jobs that are not implemented yet - bug 1572820
r"-ccov",
r"-profiling-", # talos/raptor profiling jobs are run too often
r"-32-.*-webgpu", # webgpu gets little benefit from these tests.
@@ -396,7 +396,7 @@ def _try_option_syntax(full_task_graph, parameters, graph_config):
return target_tasks_labels
-@_target_task("try_tasks")
+@register_target_task("try_tasks")
def target_tasks_try(full_task_graph, parameters, graph_config):
try_mode = parameters["try_mode"]
if try_mode == "try_task_config":
@@ -408,13 +408,13 @@ def target_tasks_try(full_task_graph, parameters, graph_config):
return []
-@_target_task("try_select_tasks")
+@register_target_task("try_select_tasks")
def target_tasks_try_select(full_task_graph, parameters, graph_config):
tasks = target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config)
return [l for l in tasks if filter_by_uncommon_try_tasks(l)]
-@_target_task("try_select_tasks_uncommon")
+@register_target_task("try_select_tasks_uncommon")
def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config):
from gecko_taskgraph.decision import PER_PROJECT_PARAMETERS
@@ -440,7 +440,7 @@ def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config):
return sorted(tasks)
-@_target_task("try_auto")
+@register_target_task("try_auto")
def target_tasks_try_auto(full_task_graph, parameters, graph_config):
"""Target the tasks which have indicated they should be run on autoland
(rather than try) via the `run_on_projects` attributes.
@@ -471,7 +471,7 @@ def target_tasks_try_auto(full_task_graph, parameters, graph_config):
]
-@_target_task("default")
+@register_target_task("default")
def target_tasks_default(full_task_graph, parameters, graph_config):
"""Target the tasks which have indicated they should be run on this project
via the `run_on_projects` attributes."""
@@ -484,7 +484,7 @@ def target_tasks_default(full_task_graph, parameters, graph_config):
]
-@_target_task("autoland_tasks")
+@register_target_task("autoland_tasks")
def target_tasks_autoland(full_task_graph, parameters, graph_config):
"""In addition to doing the filtering by project that the 'default'
filter does, also remove any tests running against shippable builds
@@ -510,7 +510,7 @@ def target_tasks_autoland(full_task_graph, parameters, graph_config):
return [l for l in filtered_for_project if filter(full_task_graph[l])]
-@_target_task("mozilla_central_tasks")
+@register_target_task("mozilla_central_tasks")
def target_tasks_mozilla_central(full_task_graph, parameters, graph_config):
"""In addition to doing the filtering by project that the 'default'
filter does, also remove any tests running against regular (aka not shippable,
@@ -550,7 +550,7 @@ def target_tasks_mozilla_central(full_task_graph, parameters, graph_config):
return [l for l in filtered_for_project if filter(full_task_graph[l])]
-@_target_task("graphics_tasks")
+@register_target_task("graphics_tasks")
def target_tasks_graphics(full_task_graph, parameters, graph_config):
"""In addition to doing the filtering by project that the 'default'
filter does, also remove artifact builds because we have csets on
@@ -568,7 +568,7 @@ def target_tasks_graphics(full_task_graph, parameters, graph_config):
return [l for l in filtered_for_project if filter(full_task_graph[l])]
-@_target_task("mozilla_beta_tasks")
+@register_target_task("mozilla_beta_tasks")
def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a promotable beta or release build
of desktop, plus android CI. The candidates build process involves a pipeline
@@ -581,7 +581,7 @@ def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config):
]
-@_target_task("mozilla_release_tasks")
+@register_target_task("mozilla_release_tasks")
def target_tasks_mozilla_release(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a promotable beta or release build
of desktop, plus android CI. The candidates build process involves a pipeline
@@ -594,7 +594,7 @@ def target_tasks_mozilla_release(full_task_graph, parameters, graph_config):
]
-@_target_task("mozilla_esr115_tasks")
+@register_target_task("mozilla_esr115_tasks")
def target_tasks_mozilla_esr115(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a promotable beta or release build
of desktop, without android CI. The candidates build process involves a pipeline
@@ -618,7 +618,7 @@ def target_tasks_mozilla_esr115(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("promote_desktop")
+@register_target_task("promote_desktop")
def target_tasks_promote_desktop(full_task_graph, parameters, graph_config):
"""Select the superset of tasks required to promote a beta or release build
of a desktop product. This should include all non-android
@@ -642,7 +642,7 @@ def target_tasks_promote_desktop(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("push_desktop")
+@register_target_task("push_desktop")
def target_tasks_push_desktop(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to push a build of desktop to cdns.
Previous build deps will be optimized out via action task."""
@@ -668,7 +668,7 @@ def target_tasks_push_desktop(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("ship_desktop")
+@register_target_task("ship_desktop")
def target_tasks_ship_desktop(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to ship desktop.
Previous build deps will be optimized out via action task."""
@@ -709,7 +709,7 @@ def target_tasks_ship_desktop(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("pine_tasks")
+@register_target_task("pine_tasks")
def target_tasks_pine(full_task_graph, parameters, graph_config):
"""Bug 1879960 - no reftests or wpt needed"""
filtered_for_project = target_tasks_default(
@@ -717,6 +717,8 @@ def target_tasks_pine(full_task_graph, parameters, graph_config):
)
def filter(task):
+ if "android" in task.attributes.get("build_platform", ""):
+ return False
suite = task.attributes.get("unittest_suite", "")
if "reftest" in suite or "web-platform" in suite:
return False
@@ -725,7 +727,7 @@ def target_tasks_pine(full_task_graph, parameters, graph_config):
return [l for l in filtered_for_project if filter(full_task_graph[l])]
-@_target_task("larch_tasks")
+@register_target_task("larch_tasks")
def target_tasks_larch(full_task_graph, parameters, graph_config):
"""Bug 1879213 - only run necessary tasks on larch"""
filtered_for_project = target_tasks_default(
@@ -749,7 +751,7 @@ def target_tasks_larch(full_task_graph, parameters, graph_config):
return [l for l in filtered_for_project if filter(full_task_graph[l])]
-@_target_task("kaios_tasks")
+@register_target_task("kaios_tasks")
def target_tasks_kaios(full_task_graph, parameters, graph_config):
"""The set of tasks to run for kaios integration"""
@@ -760,7 +762,7 @@ def target_tasks_kaios(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("custom-car_perf_testing")
+@register_target_task("custom-car_perf_testing")
def target_tasks_custom_car_perf_testing(full_task_graph, parameters, graph_config):
"""Select tasks required for running daily performance tests for custom chromium-as-release."""
@@ -787,7 +789,7 @@ def target_tasks_custom_car_perf_testing(full_task_graph, parameters, graph_conf
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("general_perf_testing")
+@register_target_task("general_perf_testing")
def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config):
"""
Select tasks required for running performance tests 3 times a week.
@@ -910,7 +912,7 @@ def make_desktop_nightly_filter(platforms):
return filter
-@_target_task("sp-perftests")
+@register_target_task("sp-perftests")
def target_tasks_speedometer_tests(full_task_graph, parameters, graph_config):
def filter(task):
platform = task.attributes.get("test_platform")
@@ -934,7 +936,7 @@ def target_tasks_speedometer_tests(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("nightly_linux")
+@register_target_task("nightly_linux")
def target_tasks_nightly_linux(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of linux. The
nightly build process involves a pipeline of builds, signing,
@@ -945,7 +947,7 @@ def target_tasks_nightly_linux(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("nightly_macosx")
+@register_target_task("nightly_macosx")
def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of macosx. The
nightly build process involves a pipeline of builds, signing,
@@ -954,7 +956,7 @@ def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("nightly_win32")
+@register_target_task("nightly_win32")
def target_tasks_nightly_win32(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of win32 and win64.
The nightly build process involves a pipeline of builds, signing,
@@ -963,7 +965,7 @@ def target_tasks_nightly_win32(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("nightly_win64")
+@register_target_task("nightly_win64")
def target_tasks_nightly_win64(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of win32 and win64.
The nightly build process involves a pipeline of builds, signing,
@@ -972,7 +974,7 @@ def target_tasks_nightly_win64(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("nightly_win64_aarch64")
+@register_target_task("nightly_win64_aarch64")
def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of win32 and win64.
The nightly build process involves a pipeline of builds, signing,
@@ -981,7 +983,7 @@ def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("nightly_asan")
+@register_target_task("nightly_asan")
def target_tasks_nightly_asan(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of asan. The
nightly build process involves a pipeline of builds, signing,
@@ -992,7 +994,7 @@ def target_tasks_nightly_asan(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("daily_releases")
+@register_target_task("daily_releases")
def target_tasks_daily_releases(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to identify if we should release.
If we determine that we should the task will communicate to ship-it to
@@ -1004,7 +1006,7 @@ def target_tasks_daily_releases(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("nightly_desktop")
+@register_target_task("nightly_desktop")
def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of linux, mac,
windows."""
@@ -1042,7 +1044,7 @@ def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config):
)
-@_target_task("nightly_all")
+@register_target_task("nightly_all")
def target_tasks_nightly_all(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of firefox desktop and android"""
index_path = (
@@ -1065,7 +1067,7 @@ def target_tasks_nightly_all(full_task_graph, parameters, graph_config):
# Run Searchfox analysis once daily.
-@_target_task("searchfox_index")
+@register_target_task("searchfox_index")
def target_tasks_searchfox(full_task_graph, parameters, graph_config):
"""Select tasks required for indexing Firefox for Searchfox web site each day"""
return [
@@ -1081,7 +1083,7 @@ def target_tasks_searchfox(full_task_graph, parameters, graph_config):
# Run build linux64-plain-clang-trunk/opt on mozilla-central/beta with perf tests
-@_target_task("linux64_clang_trunk_perf")
+@register_target_task("linux64_clang_trunk_perf")
def target_tasks_build_linux64_clang_trunk_perf(
full_task_graph, parameters, graph_config
):
@@ -1097,19 +1099,19 @@ def target_tasks_build_linux64_clang_trunk_perf(
# Run Updatebot's cron job 4 times daily.
-@_target_task("updatebot_cron")
+@register_target_task("updatebot_cron")
def target_tasks_updatebot_cron(full_task_graph, parameters, graph_config):
"""Select tasks required to run Updatebot's cron job"""
return ["updatebot-cron"]
-@_target_task("customv8_update")
+@register_target_task("customv8_update")
def target_tasks_customv8_update(full_task_graph, parameters, graph_config):
"""Select tasks required for building latest d8/v8 version."""
return ["toolchain-linux64-custom-v8"]
-@_target_task("file_update")
+@register_target_task("file_update")
def target_tasks_file_update(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to perform nightly in-tree file updates"""
@@ -1120,7 +1122,7 @@ def target_tasks_file_update(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("l10n_bump")
+@register_target_task("l10n_bump")
def target_tasks_l10n_bump(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to perform l10n bumping."""
@@ -1131,7 +1133,7 @@ def target_tasks_l10n_bump(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("merge_automation")
+@register_target_task("merge_automation")
def target_tasks_merge_automation(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to perform repository merges."""
@@ -1142,7 +1144,7 @@ def target_tasks_merge_automation(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("scriptworker_canary")
+@register_target_task("scriptworker_canary")
def target_tasks_scriptworker_canary(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to run scriptworker canaries."""
@@ -1153,7 +1155,7 @@ def target_tasks_scriptworker_canary(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("cron_bouncer_check")
+@register_target_task("cron_bouncer_check")
def target_tasks_bouncer_check(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to perform bouncer version verification."""
@@ -1166,7 +1168,7 @@ def target_tasks_bouncer_check(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("staging_release_builds")
+@register_target_task("staging_release_builds")
def target_tasks_staging_release(full_task_graph, parameters, graph_config):
"""
Select all builds that are part of releases.
@@ -1190,7 +1192,7 @@ def target_tasks_staging_release(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("release_simulation")
+@register_target_task("release_simulation")
def target_tasks_release_simulation(full_task_graph, parameters, graph_config):
"""
Select builds that would run on push on a release branch.
@@ -1227,7 +1229,7 @@ def target_tasks_release_simulation(full_task_graph, parameters, graph_config):
]
-@_target_task("codereview")
+@register_target_task("codereview")
def target_tasks_codereview(full_task_graph, parameters, graph_config):
"""Select all code review tasks needed to produce a report"""
@@ -1245,13 +1247,13 @@ def target_tasks_codereview(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("nothing")
+@register_target_task("nothing")
def target_tasks_nothing(full_task_graph, parameters, graph_config):
"""Select nothing, for DONTBUILD pushes"""
return []
-@_target_task("daily_beta_perf")
+@register_target_task("daily_beta_perf")
def target_tasks_daily_beta_perf(full_task_graph, parameters, graph_config):
"""
Select performance tests on the beta branch to be run daily
@@ -1353,7 +1355,7 @@ def target_tasks_daily_beta_perf(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("weekly_release_perf")
+@register_target_task("weekly_release_perf")
def target_tasks_weekly_release_perf(full_task_graph, parameters, graph_config):
"""
Select performance tests on the release branch to be run weekly
@@ -1421,7 +1423,7 @@ def target_tasks_weekly_release_perf(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("raptor_tp6m")
+@register_target_task("raptor_tp6m")
def target_tasks_raptor_tp6m(full_task_graph, parameters, graph_config):
"""
Select tasks required for running raptor cold page-load tests on fenix and refbrow
@@ -1448,7 +1450,7 @@ def target_tasks_raptor_tp6m(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("backfill_all_browsertime")
+@register_target_task("backfill_all_browsertime")
def target_tasks_backfill_all_browsertime(full_task_graph, parameters, graph_config):
"""
Search for revisions that contains patches that were reviewed by perftest reviewers
@@ -1516,7 +1518,7 @@ def target_tasks_backfill_all_browsertime(full_task_graph, parameters, graph_con
return []
-@_target_task("condprof")
+@register_target_task("condprof")
def target_tasks_condprof(full_task_graph, parameters, graph_config):
"""
Select tasks required for building conditioned profiles.
@@ -1527,7 +1529,7 @@ def target_tasks_condprof(full_task_graph, parameters, graph_config):
yield name
-@_target_task("system_symbols")
+@register_target_task("system_symbols")
def target_tasks_system_symbols(full_task_graph, parameters, graph_config):
"""
Select tasks for scraping and uploading system symbols.
@@ -1541,7 +1543,7 @@ def target_tasks_system_symbols(full_task_graph, parameters, graph_config):
yield name
-@_target_task("perftest")
+@register_target_task("perftest")
def target_tasks_perftest(full_task_graph, parameters, graph_config):
"""
Select perftest tasks we want to run daily
@@ -1553,7 +1555,7 @@ def target_tasks_perftest(full_task_graph, parameters, graph_config):
yield name
-@_target_task("perftest-on-autoland")
+@register_target_task("perftest-on-autoland")
def target_tasks_perftest_autoland(full_task_graph, parameters, graph_config):
"""
Select perftest tasks we want to run daily
@@ -1567,7 +1569,7 @@ def target_tasks_perftest_autoland(full_task_graph, parameters, graph_config):
yield name
-@_target_task("l10n-cross-channel")
+@register_target_task("l10n-cross-channel")
def target_tasks_l10n_cross_channel(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to run l10n cross-channel."""
@@ -1577,7 +1579,7 @@ def target_tasks_l10n_cross_channel(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("eslint-build")
+@register_target_task("eslint-build")
def target_tasks_eslint_build(full_task_graph, parameters, graph_config):
"""Select the task to run additional ESLint rules which require a build."""
@@ -1588,7 +1590,7 @@ def target_tasks_eslint_build(full_task_graph, parameters, graph_config):
yield name
-@_target_task("holly_tasks")
+@register_target_task("holly_tasks")
def target_tasks_holly(full_task_graph, parameters, graph_config):
"""Bug 1814661: only run updatebot tasks on holly"""
@@ -1598,7 +1600,7 @@ def target_tasks_holly(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t)]
-@_target_task("snap_upstream_tests")
+@register_target_task("snap_upstream_tests")
def target_tasks_snap_upstream_tests(full_task_graph, parameters, graph_config):
"""
Select tasks for testing Snap package built as upstream. Omit -try because
@@ -1609,7 +1611,7 @@ def target_tasks_snap_upstream_tests(full_task_graph, parameters, graph_config):
yield name
-@_target_task("nightly-android")
+@register_target_task("nightly-android")
def target_tasks_nightly_android(full_task_graph, parameters, graph_config):
def filter(task, parameters):
# geckoview
@@ -1645,11 +1647,11 @@ def target_tasks_nightly_android(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)]
-@_target_task("android-l10n-import")
+@register_target_task("android-l10n-import")
def target_tasks_android_l10n_import(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if l == "android-l10n-import"]
-@_target_task("android-l10n-sync")
+@register_target_task("android-l10n-sync")
def target_tasks_android_l10n_sync(full_task_graph, parameters, graph_config):
return [l for l, t in full_task_graph.tasks.items() if l == "android-l10n-sync"]
diff --git a/taskcluster/gecko_taskgraph/test/automationrelevance.json b/taskcluster/gecko_taskgraph/test/automationrelevance.json
deleted file mode 100644
index 3bdfa9ed9e..0000000000
--- a/taskcluster/gecko_taskgraph/test/automationrelevance.json
+++ /dev/null
@@ -1,358 +0,0 @@
-{
- "changesets": [
- {
- "author": "James Long <longster@gmail.com>",
- "backsoutnodes": [],
- "bugs": [
- {
- "no": "1300866",
- "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866"
- }
- ],
- "date": [1473196655.0, 14400],
- "desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins",
- "extra": {
- "branch": "default"
- },
- "files": ["devtools/client/debugger/index.html"],
- "node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "parents": ["37c9349b4e8167a61b08b7e119c21ea177b98942"],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312890,
- "reviewers": [
- {
- "name": "jlast",
- "revset": "reviewer(jlast)"
- },
- {
- "name": "bgrins",
- "revset": "reviewer(bgrins)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Wes Kocher <wkocher@mozilla.com>",
- "backsoutnodes": [],
- "bugs": [],
- "date": [1473208638.0, 25200],
- "desc": "Merge m-c to fx-team, a=merge",
- "extra": {
- "branch": "default"
- },
- "files": ["taskcluster/scripts/builder/build-l10n.sh"],
- "node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "parents": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "91c2b9d5c1354ca79e5b174591dbb03b32b15bbf"
- ],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312891,
- "reviewers": [
- {
- "name": "merge",
- "revset": "reviewer(merge)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Towkir Ahmed <towkir17@gmail.com>",
- "backsoutnodes": [],
- "bugs": [
- {
- "no": "1296648",
- "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648"
- }
- ],
- "date": [1472957580.0, 14400],
- "desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim",
- "extra": {
- "branch": "default"
- },
- "files": ["devtools/client/themes/rules.css"],
- "node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "parents": ["73a6a267a50a0e1c41e689b265ad3eebe43d7ac6"],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312892,
- "reviewers": [
- {
- "name": "ntim",
- "revset": "reviewer(ntim)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Oriol <oriol-bugzilla@hotmail.com>",
- "backsoutnodes": [],
- "bugs": [
- {
- "no": "1300336",
- "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336"
- }
- ],
- "date": [1472921160.0, 14400],
- "desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen",
- "extra": {
- "branch": "default"
- },
- "files": [
- "devtools/client/webconsole/test/browser_webconsole_output_06.js",
- "devtools/server/actors/object.js"
- ],
- "node": "99c542fa43a72ee863c813b5624048d1b443549b",
- "parents": ["16a1a91f9269ab95dd83eb29dc5d0227665f7d94"],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312893,
- "reviewers": [
- {
- "name": "fitzgen",
- "revset": "reviewer(fitzgen)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Ruturaj Vartak <ruturaj@gmail.com>",
- "backsoutnodes": [],
- "bugs": [
- {
- "no": "1295010",
- "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
- }
- ],
- "date": [1472854020.0, -7200],
- "desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK",
- "extra": {
- "amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd",
- "branch": "default",
- "histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba",
- "rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651"
- },
- "files": [
- "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
- "devtools/client/inspector/test/head.js",
- "devtools/server/actors/highlighters/eye-dropper.js"
- ],
- "node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "parents": ["99c542fa43a72ee863c813b5624048d1b443549b"],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312894,
- "reviewers": [
- {
- "name": "pbro",
- "revset": "reviewer(pbro)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Matteo Ferretti <mferretti@mozilla.com>",
- "backsoutnodes": [],
- "bugs": [
- {
- "no": "1299154",
- "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154"
- }
- ],
- "date": [1472629906.0, -7200],
- "desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n",
- "extra": {
- "branch": "default",
- "committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200"
- },
- "files": [
- "docshell/base/nsDocShell.cpp",
- "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html"
- ],
- "node": "541c9086c0f27fba60beecc9bc94543103895c86",
- "parents": ["a6b6a93eb41a05e310a11f0172f01ba9b21d3eac"],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312895,
- "reviewers": [
- {
- "name": "mstange",
- "revset": "reviewer(mstange)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Patrick Brosset <pbrosset@mozilla.com>",
- "backsoutnodes": [],
- "bugs": [
- {
- "no": "1295010",
- "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
- }
- ],
- "date": [1473239449.0, -7200],
- "desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp",
- "extra": {
- "branch": "default"
- },
- "files": [
- "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
- "devtools/client/inspector/test/head.js"
- ],
- "node": "041a925171e431bf51fb50193ab19d156088c89a",
- "parents": ["541c9086c0f27fba60beecc9bc94543103895c86"],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312896,
- "reviewers": [
- {
- "name": "me",
- "revset": "reviewer(me)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- },
- {
- "author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>",
- "backsoutnodes": [],
- "bugs": [],
- "date": [1473261233.0, -7200],
- "desc": "merge fx-team to mozilla-central a=merge",
- "extra": {
- "branch": "default"
- },
- "files": [],
- "node": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "parents": [
- "3d0b41fdd93bd8233745eadb4e0358e385bf2cb9",
- "041a925171e431bf51fb50193ab19d156088c89a"
- ],
- "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "pushdate": [1473261248, 0],
- "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
- "pushid": 30664,
- "pushnodes": [
- "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
- "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
- "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
- "99c542fa43a72ee863c813b5624048d1b443549b",
- "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
- "541c9086c0f27fba60beecc9bc94543103895c86",
- "041a925171e431bf51fb50193ab19d156088c89a",
- "a14f88a9af7a59e677478694bafd9375ac53683e"
- ],
- "pushuser": "cbook@mozilla.com",
- "rev": 312897,
- "reviewers": [
- {
- "name": "merge",
- "revset": "reviewer(merge)"
- }
- ],
- "treeherderrepo": "mozilla-central",
- "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
- }
- ],
- "visible": true
-}
diff --git a/taskcluster/gecko_taskgraph/test/conftest.py b/taskcluster/gecko_taskgraph/test/conftest.py
index ff3d6ce2bd..758d3402b5 100644
--- a/taskcluster/gecko_taskgraph/test/conftest.py
+++ b/taskcluster/gecko_taskgraph/test/conftest.py
@@ -47,7 +47,7 @@ def enable_logging():
@pytest.fixture(scope="session")
def graph_config():
- return load_graph_config(os.path.join(GECKO, "taskcluster", "ci"))
+ return load_graph_config(os.path.join(GECKO, "taskcluster"))
@pytest.fixture(scope="session")
diff --git a/taskcluster/gecko_taskgraph/test/test_actions_util.py b/taskcluster/gecko_taskgraph/test/test_actions_util.py
index 7c38caea57..0838dafc11 100644
--- a/taskcluster/gecko_taskgraph/test/test_actions_util.py
+++ b/taskcluster/gecko_taskgraph/test/test_actions_util.py
@@ -155,6 +155,7 @@ def is_subset(subset, superset):
def test_extract_applicable_action(
responses, monkeypatch, actions_json, task_def, expected
):
+ actions.util.get_task_definition.cache_clear()
base_url = "https://taskcluster"
decision_task_id = "dddd"
task_id = "tttt"
diff --git a/taskcluster/gecko_taskgraph/test/test_files_changed.py b/taskcluster/gecko_taskgraph/test/test_files_changed.py
index 5b9a016649..389dbb4093 100644
--- a/taskcluster/gecko_taskgraph/test/test_files_changed.py
+++ b/taskcluster/gecko_taskgraph/test/test_files_changed.py
@@ -3,14 +3,11 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-import json
-import os
import unittest
from mozunit import main
from gecko_taskgraph import files_changed
-from gecko_taskgraph.util import hg
PARAMS = {
"head_repository": "https://hg.mozilla.org/mozilla-central",
@@ -31,40 +28,17 @@ FILES_CHANGED = [
]
-class FakeResponse:
- def json(self):
- with open(
- os.path.join(os.path.dirname(__file__), "automationrelevance.json")
- ) as f:
- return json.load(f)
-
-
-class TestGetChangedFiles(unittest.TestCase):
- def setUp(self):
- files_changed.get_changed_files.clear()
- self.old_get = hg.requests.get
-
- def fake_get(url, **kwargs):
- return FakeResponse()
-
- hg.requests.get = fake_get
-
- def tearDown(self):
- hg.requests.get = self.old_get
- files_changed.get_changed_files.clear()
-
- def test_get_changed_files(self):
- """Get_changed_files correctly gets the list of changed files in a push.
- This tests against the production hg.mozilla.org so that it will detect
- any changes in the format of the returned data."""
- self.assertEqual(
- sorted(
- files_changed.get_changed_files(
- PARAMS["head_repository"], PARAMS["head_rev"]
- )
- ),
- FILES_CHANGED,
+def test_get_changed_files(responses):
+ url = f"{PARAMS['head_repository']}/json-pushchangedfiles/{PARAMS['head_rev']}"
+ responses.add(responses.GET, url, status=200, json={"files": FILES_CHANGED})
+ assert (
+ sorted(
+ files_changed.get_changed_files(
+ PARAMS["head_repository"], PARAMS["head_rev"]
+ )
)
+ == FILES_CHANGED
+ )
class TestCheck(unittest.TestCase):
diff --git a/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py b/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py
index 1240d71cf8..2c10ba449a 100644
--- a/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py
+++ b/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py
@@ -19,6 +19,7 @@ from gecko_taskgraph.optimize.bugbug import (
DisperseGroups,
SkipUnlessDebug,
)
+from gecko_taskgraph.optimize.mozlint import SkipUnlessMozlint
from gecko_taskgraph.optimize.strategies import SkipUnlessSchedules
from gecko_taskgraph.util.backstop import BACKSTOP_PUSH_INTERVAL
from gecko_taskgraph.util.bugbug import (
@@ -511,5 +512,79 @@ def test_project_autoland_test(monkeypatch, responses, params):
assert scheduled == {"task-0-label", "task-1-label"}
+@pytest.mark.parametrize(
+ "pushed_files,to_lint,expected",
+ [
+ pytest.param(
+ ["a/b/c.txt"],
+ [],
+ True,
+ ),
+ pytest.param(
+ ["python/mozlint/a/support_file.txt", "b/c/d.txt"],
+ ["python/mozlint/a/support_file.txt"],
+ False,
+ ),
+ ],
+ ids=idfn,
+)
+def test_mozlint_should_remove_task(
+ monkeypatch, params, pushed_files, to_lint, expected
+):
+ import mozlint.pathutils
+
+ class MockParser:
+ def __call__(self, *args, **kwargs):
+ return []
+
+ def mock_filterpaths(*args, **kwargs):
+ return to_lint, None
+
+ monkeypatch.setattr(mozlint.pathutils, "filterpaths", mock_filterpaths)
+
+ opt = SkipUnlessMozlint("")
+ monkeypatch.setattr(opt, "mozlint_parser", MockParser())
+ params["files_changed"] = pushed_files
+
+ result = opt.should_remove_task(default_tasks[0], params, "")
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "pushed_files,linter_config,expected",
+ [
+ pytest.param(
+ ["a/b/c.txt"],
+ [{"include": ["b/c"]}],
+ True,
+ ),
+ pytest.param(
+ ["a/b/c.txt"],
+ [{"include": ["a/b"], "exclude": ["a/b/c.txt"]}],
+ True,
+ ),
+ pytest.param(
+ ["python/mozlint/a/support_file.txt", "b/c/d.txt"],
+ [{}],
+ False,
+ ),
+ ],
+ ids=idfn,
+)
+def test_mozlint_should_remove_task2(
+ monkeypatch, params, pushed_files, linter_config, expected
+):
+ class MockParser:
+ def __call__(self, *args, **kwargs):
+ return linter_config
+
+ opt = SkipUnlessMozlint("")
+ monkeypatch.setattr(opt, "mozlint_parser", MockParser())
+ params["files_changed"] = pushed_files
+
+ result = opt.should_remove_task(default_tasks[0], params, "")
+ assert result == expected
+
+
if __name__ == "__main__":
main()
diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_job.py b/taskcluster/gecko_taskgraph/test/test_transforms_job.py
index b032307ea6..6161a36054 100644
--- a/taskcluster/gecko_taskgraph/test/test_transforms_job.py
+++ b/taskcluster/gecko_taskgraph/test/test_transforms_job.py
@@ -37,7 +37,7 @@ TASK_DEFAULTS = {
@pytest.fixture(scope="module")
def config():
- graph_config = load_graph_config(os.path.join(GECKO, "taskcluster", "ci"))
+ graph_config = load_graph_config(os.path.join(GECKO, "taskcluster"))
params = FakeParameters(
{
"base_repository": "http://hg.example.com",
diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py
index fb5b17d3b3..1434b4013a 100644
--- a/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py
+++ b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py
@@ -101,6 +101,7 @@ CONFIG_PER_BOUNCER_PRODUCT = {
"file_names": {
"win": "{pretty_product}%20Setup%20{version}.msix",
"win64": "{pretty_product}%20Setup%20{version}.msix",
+ "win64-aarch64": "{pretty_product}%20Setup%20{version}.msix",
},
},
"pkg": {
diff --git a/taskcluster/gecko_taskgraph/transforms/build.py b/taskcluster/gecko_taskgraph/transforms/build.py
index 4e73c5aef2..3ab22f2923 100644
--- a/taskcluster/gecko_taskgraph/transforms/build.py
+++ b/taskcluster/gecko_taskgraph/transforms/build.py
@@ -139,6 +139,10 @@ def use_artifact(config, jobs):
and job.get("index", {}).get("job-name") in ARTIFACT_JOBS
# If tests aren't packaged, then we are not able to rebuild all the packages
and job["worker"]["env"].get("MOZ_AUTOMATION_PACKAGE_TESTS") == "1"
+ # Android shippable artifact builds are not supported
+ and not (
+ "android" in job["name"] and job["attributes"].get("shippable", False)
+ )
):
job["treeherder"]["symbol"] = add_suffix(job["treeherder"]["symbol"], "a")
job["worker"]["env"]["USE_ARTIFACT"] = "1"
diff --git a/taskcluster/gecko_taskgraph/transforms/build_schedules.py b/taskcluster/gecko_taskgraph/transforms/build_schedules.py
index ed6262b8b2..f9621adf77 100644
--- a/taskcluster/gecko_taskgraph/transforms/build_schedules.py
+++ b/taskcluster/gecko_taskgraph/transforms/build_schedules.py
@@ -19,15 +19,7 @@ def set_build_schedules_optimization(config, tasks):
continue
schedules = []
- if config.kind == "build":
- family = platform_family(task["attributes"]["build_platform"])
- schedules = [family]
-
- if "android" not in family:
- # These are not GeckoView builds, so are associated with Firefox.
- schedules.append("firefox")
-
- elif config.kind in (
+ if config.kind in (
"build-components",
"build-samples-browser",
"test-components",
@@ -44,5 +36,13 @@ def set_build_schedules_optimization(config, tasks):
elif "focus" in task["name"] or "klar" in task["name"]:
schedules.append("focus-android")
+ else:
+ family = platform_family(task["attributes"]["build_platform"])
+ schedules = [family]
+
+ if "android" not in family:
+ # These are not GeckoView builds, so are associated with Firefox.
+ schedules.append("firefox")
+
task["optimization"] = {"build": schedules}
yield task
diff --git a/taskcluster/gecko_taskgraph/transforms/diffoscope.py b/taskcluster/gecko_taskgraph/transforms/diffoscope.py
index b74dc5bb8f..05c6617950 100644
--- a/taskcluster/gecko_taskgraph/transforms/diffoscope.py
+++ b/taskcluster/gecko_taskgraph/transforms/diffoscope.py
@@ -32,7 +32,7 @@ diff_description_schema = Schema(
Required("original"): index_or_string,
Required("new"): index_or_string,
# Arguments to pass to diffoscope, used for job-defaults in
- # taskcluster/ci/diffoscope/kind.yml
+ # taskcluster/kinds/diffoscope/kind.yml
Optional("args"): str,
# Extra arguments to pass to diffoscope, that can be set per job.
Optional("extra-args"): str,
diff --git a/taskcluster/gecko_taskgraph/transforms/github_sync.py b/taskcluster/gecko_taskgraph/transforms/github_sync.py
deleted file mode 100644
index 6f48f794ce..0000000000
--- a/taskcluster/gecko_taskgraph/transforms/github_sync.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from taskgraph.transforms.base import TransformSequence
-
-transforms = TransformSequence()
-
-
-@transforms.add
-def sync_github(config, tasks):
- """Do transforms specific to github-sync tasks."""
- for task in tasks:
- # Add the secret to the scopes, only in m-c.
- # Doing this on any other tree will result in decision task failure
- # because m-c is the only one allowed to have that scope.
- secret = task["secret"]
- if config.params["project"] == "mozilla-central":
- task.setdefault("scopes", [])
- task["scopes"].append("secrets:get:" + secret)
- task["worker"].setdefault("env", {})["GITHUB_SECRET"] = secret
- del task["secret"]
- yield task
diff --git a/taskcluster/gecko_taskgraph/transforms/perftest.py b/taskcluster/gecko_taskgraph/transforms/perftest.py
index 47baafdad7..066db956f7 100644
--- a/taskcluster/gecko_taskgraph/transforms/perftest.py
+++ b/taskcluster/gecko_taskgraph/transforms/perftest.py
@@ -1,3 +1,4 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
diff --git a/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py b/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py
index ea2cac3a68..48465b3ab7 100644
--- a/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py
+++ b/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py
@@ -3,7 +3,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the reprocess-symbols task description template,
-taskcluster/ci/reprocess-symbols/job-template.yml into an actual task description.
+taskcluster/kinds/reprocess-symbols/job-template.yml into an actual task description.
"""
diff --git a/taskcluster/gecko_taskgraph/transforms/signing.py b/taskcluster/gecko_taskgraph/transforms/signing.py
index 1bf91effd1..760f9f122f 100644
--- a/taskcluster/gecko_taskgraph/transforms/signing.py
+++ b/taskcluster/gecko_taskgraph/transforms/signing.py
@@ -61,6 +61,14 @@ signing_description_schema = Schema(
)
+def get_locales_description(attributes, default):
+ """Returns the [list] of locales for task description usage"""
+ chunk_locales = attributes.get("chunk_locales")
+ if chunk_locales:
+ return ", ".join(chunk_locales)
+ return attributes.get("locale", default)
+
+
@transforms.add
def delete_name(config, jobs):
"""Delete the 'name' key if it exists, we don't use it."""
@@ -77,6 +85,7 @@ transforms.add_validate(signing_description_schema)
def add_requirements_link(config, jobs):
for job in jobs:
dep_job = get_primary_dependency(config, job)
+ assert dep_job
requirements_path = evaluate_keyed_by(
config.graph_config["mac-signing"]["mac-requirements"],
"mac requirements",
@@ -95,6 +104,7 @@ def add_requirements_link(config, jobs):
def make_task_description(config, jobs):
for job in jobs:
dep_job = get_primary_dependency(config, job)
+ assert dep_job
attributes = dep_job.attributes
signing_format_scopes = []
@@ -105,6 +115,7 @@ def make_task_description(config, jobs):
is_shippable = dep_job.attributes.get("shippable", False)
build_platform = dep_job.attributes.get("build_platform")
+ assert build_platform
treeherder = None
if "partner" not in config.kind and "eme-free" not in config.kind:
treeherder = job.get("treeherder", {})
@@ -139,9 +150,9 @@ def make_task_description(config, jobs):
label = job["label"]
description = (
- "Initial Signing for locale '{locale}' for build '"
+ "Signing of locale(s) '{locale}' for build '"
"{build_platform}/{build_type}'".format(
- locale=attributes.get("locale", "en-US"),
+ locale=get_locales_description(attributes, "en-US"),
build_platform=build_platform,
build_type=attributes.get("build_type"),
)
@@ -187,10 +198,16 @@ def make_task_description(config, jobs):
# build-mac-{signing,notarization} uses signingscript instead of iscript
if "macosx" in build_platform and config.kind.endswith("-mac-notarization"):
- task["worker"]["mac-behavior"] = "apple_notarization"
task["scopes"] = [
add_scope_prefix(config, "signing:cert:release-apple-notarization")
]
+ task[
+ "description"
+ ] = "Notarization of '{}' locales for build '{}/{}'".format(
+ get_locales_description(attributes, "en-US"),
+ build_platform,
+ attributes.get("build_type"),
+ )
elif "macosx" in build_platform:
# iscript overrides
task["worker"]["mac-behavior"] = "mac_sign_and_pkg"
diff --git a/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py b/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py
index 0c31f48cc0..2b846604f1 100644
--- a/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py
+++ b/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py
@@ -1,3 +1,4 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
diff --git a/taskcluster/gecko_taskgraph/transforms/source_test.py b/taskcluster/gecko_taskgraph/transforms/source_test.py
index e3eeb7c819..266637a7a0 100644
--- a/taskcluster/gecko_taskgraph/transforms/source_test.py
+++ b/taskcluster/gecko_taskgraph/transforms/source_test.py
@@ -1,3 +1,4 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
@@ -264,8 +265,8 @@ def remove_optimization_on_central(config, jobs):
if not job.get("attributes", {}).get("code-review", False):
yield job
continue
- if "when" not in job:
- yield job
- continue
- del job["when"]
+ if "when" in job:
+ del job["when"]
+ if "optimization" in job and "skip-unless-mozlint" in job["optimization"]:
+ del job["optimization"]
yield job
diff --git a/taskcluster/gecko_taskgraph/transforms/task.py b/taskcluster/gecko_taskgraph/transforms/task.py
index 4bfe0e9f6d..aa77c46d65 100644
--- a/taskcluster/gecko_taskgraph/transforms/task.py
+++ b/taskcluster/gecko_taskgraph/transforms/task.py
@@ -208,7 +208,7 @@ TC_TREEHERDER_SCHEMA_URL = (
UNKNOWN_GROUP_NAME = (
- "Treeherder group {} (from {}) has no name; " "add it to taskcluster/ci/config.yml"
+ "Treeherder group {} (from {}) has no name; " "add it to taskcluster/config.yml"
)
V2_ROUTE_TEMPLATES = [
@@ -313,7 +313,7 @@ def index_builder(name):
UNSUPPORTED_INDEX_PRODUCT_ERROR = """\
The gecko-v2 product {product} is not in the list of configured products in
-`taskcluster/ci/config.yml'.
+`taskcluster/config.yml'.
"""
@@ -842,6 +842,7 @@ def build_generic_worker_payload(config, task, task_def):
# behavior for mac iscript
Optional("mac-behavior"): Any(
"apple_notarization",
+ "apple_notarization_stacked",
"mac_sign_and_pkg",
"mac_sign_and_pkg_hardened",
"mac_geckodriver",
@@ -1612,7 +1613,7 @@ def task_name_from_label(config, tasks):
UNSUPPORTED_SHIPPING_PRODUCT_ERROR = """\
The shipping product {product} is not in the list of configured products in
-`taskcluster/ci/config.yml'.
+`taskcluster/config.yml'.
"""
diff --git a/taskcluster/gecko_taskgraph/transforms/test/other.py b/taskcluster/gecko_taskgraph/transforms/test/other.py
index 5d54467001..e01691c05a 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/other.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/other.py
@@ -419,26 +419,20 @@ def setup_browsertime(config, tasks):
"win32-chromedriver-122",
"win32-chromedriver-123",
],
- "windows.*-32.*": [
- "win32-chromedriver-122",
- "win32-chromedriver-123",
- "win32-chromedriver-124",
- ],
"windows.*-64.*": [
- "win32-chromedriver-122",
- "win32-chromedriver-123",
+ "win64-chromedriver-123",
"win64-chromedriver-124",
],
}
chromium_fetches = {
- "linux.*": ["linux64-chromium"],
- "macosx1015.*": ["mac-chromium"],
- "macosx1400.*": ["mac-chromium-arm"],
- "windows.*aarch64.*": ["win32-chromium"],
- "windows.*-32.*": ["win32-chromium"],
- "windows.*-64.*": ["win64-chromium"],
- "android.*": ["linux64-chromium"],
+ "linux.*": ["linux64-chromiumdriver"],
+ "macosx1015.*": ["mac-chromiumdriver"],
+ "macosx1400.*": ["mac-chromiumdriver-arm"],
+ "windows.*aarch64.*": ["win32-chromiumdriver"],
+ "windows.*-32.*": ["win32-chromiumdriver"],
+ "windows.*-64.*": ["win64-chromiumdriver"],
+ "android.*": ["linux64-chromiumdriver"],
}
cd_extracted_name = {
@@ -581,7 +575,9 @@ def enable_code_coverage(config, tasks):
yield task
continue
task["mozharness"].setdefault("extra-options", []).append("--code-coverage")
- task["instance-size"] = "xlarge"
+ task["instance-size"] = "xlarge-noscratch"
+ if "jittest" in task["test-name"]:
+ task["instance-size"] = "xlarge"
# Temporarily disable Mac tests on mozilla-central
if "mac" in task["build-platform"]:
diff --git a/taskcluster/gecko_taskgraph/transforms/test/variant.py b/taskcluster/gecko_taskgraph/transforms/test/variant.py
index bda91b2f25..6ee6c429f0 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/variant.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/variant.py
@@ -78,6 +78,14 @@ def split_variants(config, tasks):
remaining_variants.append(name)
return remaining_variants
+ def replace_task_items(task_key, variant_key):
+ for item in variant_key:
+ if isinstance(variant_key[item], dict):
+ task_key[item] = replace_task_items(task_key[item], variant_key[item])
+ else:
+ task_key[item] = variant_key[item]
+ return task_key
+
def apply_variant(variant, task):
task["description"] = variant["description"].format(**task)
@@ -94,7 +102,9 @@ def split_variants(config, tasks):
task["variant-suffix"] += suffix
# Replace and/or merge the configuration.
- task.update(variant.get("replace", {}))
+
+ # we only want to update the leaf node, the the entire top level dict
+ task = replace_task_items(task, variant.get("replace", {}))
return merge(task, variant.get("merge", {}))
expired_variants = find_expired_variants(TEST_VARIANTS)
diff --git a/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py b/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py
index aea948f90e..3828e912ec 100644
--- a/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py
+++ b/taskcluster/gecko_taskgraph/transforms/upload_generated_sources.py
@@ -3,7 +3,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the upload-generated-files task description template,
-taskcluster/ci/upload-generated-sources/kind.yml, into an actual task description.
+taskcluster/kinds/upload-generated-sources/kind.yml, into an actual task description.
"""
from taskgraph.transforms.base import TransformSequence
diff --git a/taskcluster/gecko_taskgraph/transforms/upload_symbols.py b/taskcluster/gecko_taskgraph/transforms/upload_symbols.py
index 5c0bf18cb0..b216d2c8e1 100644
--- a/taskcluster/gecko_taskgraph/transforms/upload_symbols.py
+++ b/taskcluster/gecko_taskgraph/transforms/upload_symbols.py
@@ -3,7 +3,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the upload-symbols task description template,
-taskcluster/ci/upload-symbols/job-template.yml into an actual task description.
+taskcluster/kinds/upload-symbols/job-template.yml into an actual task description.
"""
diff --git a/taskcluster/gecko_taskgraph/util/chunking.py b/taskcluster/gecko_taskgraph/util/chunking.py
index a8ae4d8b6b..9d8b98e07b 100644
--- a/taskcluster/gecko_taskgraph/util/chunking.py
+++ b/taskcluster/gecko_taskgraph/util/chunking.py
@@ -24,8 +24,8 @@ here = os.path.abspath(os.path.dirname(__file__))
resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader)
TEST_VARIANTS = {}
-if os.path.exists(os.path.join(GECKO, "taskcluster", "ci", "test", "variants.yml")):
- TEST_VARIANTS = load_yaml(GECKO, "taskcluster", "ci", "test", "variants.yml")
+if os.path.exists(os.path.join(GECKO, "taskcluster", "kinds", "test", "variants.yml")):
+ TEST_VARIANTS = load_yaml(GECKO, "taskcluster", "kinds", "test", "variants.yml")
WPT_SUBSUITES = {
"canvas": "html/canvas",
diff --git a/taskcluster/gecko_taskgraph/util/docker.py b/taskcluster/gecko_taskgraph/util/docker.py
index e8de7d1fdb..51e3f7e484 100644
--- a/taskcluster/gecko_taskgraph/util/docker.py
+++ b/taskcluster/gecko_taskgraph/util/docker.py
@@ -298,7 +298,7 @@ class ImagePathsMap(Mapping):
self.__update_image_paths(jobs, image_dir)
-image_paths = ImagePathsMap("taskcluster/ci/docker-image/kind.yml")
+image_paths = ImagePathsMap("taskcluster/kinds/docker-image/kind.yml")
def image_path(name):
diff --git a/taskcluster/gecko_taskgraph/util/hg.py b/taskcluster/gecko_taskgraph/util/hg.py
index 17d341cdc0..d03af87f13 100644
--- a/taskcluster/gecko_taskgraph/util/hg.py
+++ b/taskcluster/gecko_taskgraph/util/hg.py
@@ -96,15 +96,15 @@ def get_push_data(repository, project, push_id_start, push_id_end):
@memoize
-def get_json_automationrelevance(repository, revision):
- url = "{}/json-automationrelevance/{}".format(repository.rstrip("/"), revision)
+def get_json_pushchangedfiles(repository, revision):
+ url = "{}/json-pushchangedfiles/{}".format(repository.rstrip("/"), revision)
logger.debug("Querying version control for metadata: %s", url)
- def get_automationrelevance():
+ def get_pushchangedfiles():
response = requests.get(url, timeout=60)
return response.json()
- return retry(get_automationrelevance, attempts=10, sleeptime=10)
+ return retry(get_pushchangedfiles, attempts=10, sleeptime=10)
def get_hg_revision_branch(root, revision):
diff --git a/taskcluster/gecko_taskgraph/util/partners.py b/taskcluster/gecko_taskgraph/util/partners.py
index 2546e1ae88..f4b8b187cf 100644
--- a/taskcluster/gecko_taskgraph/util/partners.py
+++ b/taskcluster/gecko_taskgraph/util/partners.py
@@ -523,7 +523,7 @@ def apply_partner_priority(config, jobs):
# Reduce the priority of the partner repack jobs because they don't block QE. Meanwhile
# leave EME-free jobs alone because they do, and they'll get the branch priority like the rest
# of the release. Only bother with this in production, not on staging releases on try.
- # medium is the same as mozilla-central, see taskcluster/ci/config.yml. ie higher than
+ # medium is the same as mozilla-central, see taskcluster/config.yml. ie higher than
# integration branches because we don't want to wait a lot for the graph to be done, but
# for multiple releases the partner tasks always wait for non-partner.
if (
diff --git a/taskcluster/gecko_taskgraph/util/platforms.py b/taskcluster/gecko_taskgraph/util/platforms.py
index 2c423223fe..3010b32792 100644
--- a/taskcluster/gecko_taskgraph/util/platforms.py
+++ b/taskcluster/gecko_taskgraph/util/platforms.py
@@ -27,7 +27,8 @@ _executable_extension = {
_architectures = {
r"linux\b.*": "x86",
- r"linux64\b.*": "x86_64",
+ r"linux64\b(?!-aarch64).*": "x86_64",
+ r"linux64-aarch64\b.*": "aarch64",
r"macosx64\b.*": "macos-x86_64-aarch64",
r"win32\b.*": "x86",
r"win64\b(?!-aarch64).*": "x86_64",
diff --git a/taskcluster/gecko_taskgraph/util/signed_artifacts.py b/taskcluster/gecko_taskgraph/util/signed_artifacts.py
index 2467ff8046..61dad14abf 100644
--- a/taskcluster/gecko_taskgraph/util/signed_artifacts.py
+++ b/taskcluster/gecko_taskgraph/util/signed_artifacts.py
@@ -57,7 +57,7 @@ def generate_specifications_of_artifacts_to_sign(
elif "macosx" in build_platform:
langpack_formats = []
if is_notarization_kind(config.kind):
- formats = ["apple_notarization"]
+ formats = ["apple_notarization_stacked"]
artifacts_specifications = [
{
"artifacts": [
diff --git a/taskcluster/gecko_taskgraph/util/taskcluster.py b/taskcluster/gecko_taskgraph/util/taskcluster.py
index cddb01fd37..826eee4cdb 100644
--- a/taskcluster/gecko_taskgraph/util/taskcluster.py
+++ b/taskcluster/gecko_taskgraph/util/taskcluster.py
@@ -24,7 +24,7 @@ def insert_index(index_path, task_id, data=None, use_proxy=False):
index_url = get_index_url(index_path, use_proxy=use_proxy)
# Find task expiry.
- expires = get_task_definition(task_id, use_proxy=use_proxy)["expires"]
+ expires = get_task_definition(task_id, use_proxy)["expires"]
response = _do_request(
index_url,