Merging upstream version 126.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 72 insertions, 44 deletions

diff --git a/taskcluster/gecko_taskgraph/transforms/test/__init__.py b/taskcluster/gecko_taskgraph/transforms/test/__init__.py
index 92704bf18c..19ab8d289f 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/__init__.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/__init__.py
@@ -27,6 +27,7 @@ from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by
 from voluptuous import Any, Exclusive, Optional, Required
 
 from gecko_taskgraph.optimize.schema import OptimizationSchema
+from gecko_taskgraph.transforms.job import job_description_schema
 from gecko_taskgraph.transforms.test.other import get_mobile_project
 from gecko_taskgraph.util.chunking import manifest_loaders
 
@@ -118,7 +119,9 @@ test_description_schema = Schema(
         Required("run-without-variant"): optionally_keyed_by("test-platform", bool),
         # The EC2 instance size to run these tests on.
         Required("instance-size"): optionally_keyed_by(
-            "test-platform", Any("default", "large", "xlarge")
+            "test-platform",
+            "variant",
+            Any("default", "large", "large-noscratch", "xlarge", "xlarge-noscratch"),
         ),
         # type of virtualization or hardware required by test.
         Required("virtualization"): optionally_keyed_by(
@@ -265,11 +268,14 @@ test_description_schema = Schema(
                 str,
                 None,
                 {Required("index"): str, Required("name"): str},
+                {Required("upstream-task"): str, Required("name"): str},
             ),
         ),
         # A list of artifacts to install from 'fetch' tasks. Validation deferred
         # to 'job' transforms.
         Optional("fetches"): object,
+        # A list of extra dependencies
+        Optional("dependencies"): object,
         # Raptor / browsertime specific keys, defer validation to 'raptor.py'
         # transform.
         Optional("raptor"): object,
@@ -279,6 +285,8 @@ test_description_schema = Schema(
         Optional("subtest"): str,
         # Define if a given task supports artifact builds or not, see bug 1695325.
         Optional("supports-artifact-builds"): bool,
+        # Version of python used to run the task
+        Optional("use-python"): job_description_schema["use-python"],
     }
 )
 
@@ -346,6 +354,7 @@ def set_defaults(config, tasks):
         task.setdefault("run-without-variant", True)
         task.setdefault("variants", [])
         task.setdefault("supports-artifact-builds", True)
+        task.setdefault("use-python", "system")
 
         task["mozharness"].setdefault("extra-options", [])
         task["mozharness"].setdefault("requires-signed-builds", False)
@@ -484,6 +493,9 @@ def make_job_description(config, tasks):
         if task["mozharness"]["requires-signed-builds"] is True:
             jobdesc["dependencies"]["build-signing"] = task["build-signing-label"]
 
+        if "dependencies" in task:
+            jobdesc["dependencies"].update(task["dependencies"])
+
         if "expires-after" in task:
             jobdesc["expires-after"] = task["expires-after"]
 
diff --git a/taskcluster/gecko_taskgraph/transforms/test/chunk.py b/taskcluster/gecko_taskgraph/transforms/test/chunk.py
index 7f832c57df..8219c41664 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/chunk.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/chunk.py
@@ -44,8 +44,7 @@ def set_test_verify_chunks(config, tasks):
             task["chunks"] = perfile_number_of_chunks(
                 is_try(config.params),
                 env.get("MOZHARNESS_TEST_PATHS", ""),
-                config.params.get("head_repository", ""),
-                config.params.get("head_rev", ""),
+                frozenset(config.params["files_changed"]),
                 task["test-name"],
             )
 
diff --git a/taskcluster/gecko_taskgraph/transforms/test/other.py b/taskcluster/gecko_taskgraph/transforms/test/other.py
index b8cb95cff7..5d54467001 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/other.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/other.py
@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import copy
 import hashlib
 import json
 import re
@@ -12,7 +13,11 @@ from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import keymatch
 from taskgraph.util.keyed_by import evaluate_keyed_by
 from taskgraph.util.schema import Schema, resolve_keyed_by
-from taskgraph.util.taskcluster import get_artifact_path, get_index_url
+from taskgraph.util.taskcluster import (
+    get_artifact_path,
+    get_artifact_url,
+    get_index_url,
+)
 from voluptuous import Any, Optional, Required
 
 from gecko_taskgraph.transforms.test.variant import TEST_VARIANTS
@@ -99,6 +104,25 @@ def setup_talos(config, tasks):
         if config.params.get("project", None):
             extra_options.append("--project=%s" % config.params["project"])
 
+        if "pdfpaint" in task["try-name"]:
+            max_chunks = 10
+            for chunk in range(1, max_chunks + 1):
+                new_task = copy.deepcopy(task)
+                new_task["mozharness"]["extra-options"].append(
+                    f"--pdfPaintChunk={chunk}"
+                )
+                new_task["test-name"] = task["test-name"].replace(
+                    "pdfpaint", f"pdfpaint-{chunk}"
+                )
+                new_task["try-name"] = task["try-name"].replace(
+                    "pdfpaint", f"pdfpaint-{chunk}"
+                )
+                new_task["treeherder-symbol"] = task["treeherder-symbol"].replace(
+                    "pdfpaint", f"pdfpaint-{chunk}"
+                )
+                yield new_task
+            continue
+
         yield task
 
 
@@ -246,6 +270,7 @@ def handle_keyed_by(config, tasks):
         "webrender-run-on-projects",
         "mozharness.requires-signed-builds",
         "build-signing-label",
+        "dependencies",
     ]
     for task in tasks:
         for field in fields:
@@ -292,10 +317,17 @@ def set_target(config, tasks):
                 target = "target.tar.bz2"
 
         if isinstance(target, dict):
-            # TODO Remove hardcoded mobile artifact prefix
-            index_url = get_index_url(target["index"])
-            installer_url = "{}/artifacts/public/{}".format(index_url, target["name"])
-            task["mozharness"]["installer-url"] = installer_url
+            if "index" in target:
+                # TODO Remove hardcoded mobile artifact prefix
+                index_url = get_index_url(target["index"])
+                installer_url = "{}/artifacts/public/{}".format(
+                    index_url, target["name"]
+                )
+                task["mozharness"]["installer-url"] = installer_url
+            else:
+                task["mozharness"]["installer-url"] = get_artifact_url(
+                    f'<{target["upstream-task"]}>', target["name"]
+                )
         else:
             task["mozharness"]["build-artifact-name"] = get_artifact_path(task, target)
 
@@ -363,39 +395,39 @@ def setup_browsertime(config, tasks):
 
         cd_fetches = {
             "android.*": [
-                "linux64-chromedriver-120",
-                "linux64-chromedriver-121",
                 "linux64-chromedriver-122",
+                "linux64-chromedriver-123",
+                "linux64-chromedriver-124",
             ],
             "linux.*": [
-                "linux64-chromedriver-120",
-                "linux64-chromedriver-121",
                 "linux64-chromedriver-122",
+                "linux64-chromedriver-123",
+                "linux64-chromedriver-124",
             ],
             "macosx1015.*": [
-                "mac64-chromedriver-120",
-                "mac64-chromedriver-121",
                 "mac64-chromedriver-122",
+                "mac64-chromedriver-123",
+                "mac64-chromedriver-124",
             ],
             "macosx1400.*": [
-                "mac-arm-chromedriver-120",
-                "mac-arm-chromedriver-121",
                 "mac-arm-chromedriver-122",
+                "mac-arm-chromedriver-123",
+                "mac-arm-chromedriver-124",
             ],
             "windows.*aarch64.*": [
-                "win32-chromedriver-120",
                 "win32-chromedriver-121",
                 "win32-chromedriver-122",
+                "win32-chromedriver-123",
             ],
             "windows.*-32.*": [
-                "win32-chromedriver-120",
-                "win32-chromedriver-121",
                 "win32-chromedriver-122",
+                "win32-chromedriver-123",
+                "win32-chromedriver-124",
             ],
             "windows.*-64.*": [
-                "win32-chromedriver-120",
-                "win32-chromedriver-121",
                 "win32-chromedriver-122",
+                "win32-chromedriver-123",
+                "win64-chromedriver-124",
             ],
         }
 
@@ -419,11 +451,7 @@ def setup_browsertime(config, tasks):
             # Only add the chromedriver fetches when chrome is running
             for platform in cd_fetches:
                 fs["by-test-platform"][platform].extend(cd_fetches[platform])
-        if (
-            "--app=chromium" in extra_options
-            or "--app=custom-car" in extra_options
-            or "--app=cstm-car-m" in extra_options
-        ):
+        if "--app=custom-car" in extra_options or "--app=cstm-car-m" in extra_options:
             for platform in chromium_fetches:
                 fs["by-test-platform"][platform].extend(chromium_fetches[platform])
 
@@ -791,7 +819,7 @@ test_setting_description_schema = Schema(
             },
             Optional("device"): str,
             Optional("display"): "wayland",
-            Optional("machine"): Any("ref-hw-2017", "hw-ref"),
+            Optional("machine"): "hw-ref",
         },
         "build": {
             Required("type"): Any("opt", "debug", "debug-isolated-process"),
@@ -852,7 +880,6 @@ def set_test_setting(config, tasks):
     # TODO Rename these so they don't have a dash.
     dash_attrs = [
         "clang-trunk",
-        "ref-hw-2017",
         "hw-ref",
     ]
     dash_token = "%D%"
@@ -908,9 +935,6 @@ def set_test_setting(config, tasks):
             if parts[0].isdigit():
                 os_build = parts.pop(0)
 
-            if parts and parts[0] == "ref-hw-2017":
-                machine = parts.pop(0)
-
             if parts and parts[0] == "hw-ref":
                 machine = parts.pop(0)
 
@@ -1102,6 +1126,7 @@ def set_schedules_components(config, tasks):
 
         schedules.add(category)
         schedules.add(platform_family(task["build-platform"]))
+        schedules.add("firefox")
 
         task["schedules-component"] = sorted(schedules)
         yield task
diff --git a/taskcluster/gecko_taskgraph/transforms/test/raptor.py b/taskcluster/gecko_taskgraph/transforms/test/raptor.py
index 18e21e6a1e..ca35749037 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/raptor.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/raptor.py
@@ -76,7 +76,6 @@ def split_apps(config, tests):
     app_symbols = {
         "chrome": "ChR",
         "chrome-m": "ChR",
-        "chromium": "Cr",
         "fenix": "fenix",
         "refbrow": "refbrow",
         "safari": "Saf",
diff --git a/taskcluster/gecko_taskgraph/transforms/test/worker.py b/taskcluster/gecko_taskgraph/transforms/test/worker.py
index 873347459c..51b12de51d 100644
--- a/taskcluster/gecko_taskgraph/transforms/test/worker.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/worker.py
@@ -7,8 +7,10 @@ from taskgraph.transforms.base import TransformSequence
 # default worker types keyed by instance-size
 LINUX_WORKER_TYPES = {
     "large": "t-linux-large",
+    "large-noscratch": "t-linux-large-noscratch",
     "xlarge": "t-linux-xlarge",
-    "default": "t-linux-large",
+    "xlarge-noscratch": "t-linux-xlarge-noscratch",
+    "default": "t-linux-large-noscratch",
 }
 
 # windows worker types keyed by test-platform and virtualization
@@ -23,11 +25,6 @@ WINDOWS_WORKER_TYPES = {
         "virtual-with-gpu": "t-win10-64-gpu-s",
         "hardware": "t-win10-64-1803-hw",
     },
-    "windows10-64-ref-hw-2017": {
-        "virtual": "t-win10-64",
-        "virtual-with-gpu": "t-win10-64-gpu-s",
-        "hardware": "t-win10-64-ref-hw",
-    },
     "windows11-64-2009-hw-ref-shippable": {
         "virtual": "win11-64-2009-hw-ref",
         "virtual-with-gpu": "win11-64-2009-hw-ref",
@@ -130,12 +127,8 @@ def set_worker_type(config, tasks):
         elif test_platform.startswith("win"):
             # figure out what platform the job needs to run on
             if task["virtualization"] == "hardware":
-                # some jobs like talos and reftest run on real h/w - those are all win10
-                if test_platform.startswith("windows10-64-ref-hw-2017"):
-                    win_worker_type_platform = WINDOWS_WORKER_TYPES[
-                        "windows10-64-ref-hw-2017"
-                    ]
-                elif test_platform.startswith("windows11-64-2009-hw-ref"):
+                # some jobs like talos and reftest run on real h/w
+                if test_platform.startswith("windows11-64-2009-hw-ref"):
                     win_worker_type_platform = WINDOWS_WORKER_TYPES[
                         "windows11-64-2009-hw-ref"
                     ]