Merging upstream version 126.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

6 files changed, 278 insertions, 162 deletions

diff --git a/taskcluster/gecko_taskgraph/test/conftest.py b/taskcluster/gecko_taskgraph/test/conftest.py
index 360c2da65e..ff3d6ce2bd 100644
--- a/taskcluster/gecko_taskgraph/test/conftest.py
+++ b/taskcluster/gecko_taskgraph/test/conftest.py
@@ -151,6 +151,20 @@ class FakeOptimization(OptimizationStrategy):
         return False
 
 
+class FakeTransformConfig:
+    kind = "fake-kind"
+    path = "/root/ci/fake-kind"
+    config = {}
+    params = FakeParameters()
+    kind_dependencies_tasks = {}
+    graph_config = {}
+    write_artifacts = False
+
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+
 @pytest.fixture
 def maketgg(monkeypatch):
     def inner(target_tasks=None, kinds=[("_fake", [])], params=None):
@@ -195,12 +209,16 @@ def maketgg(monkeypatch):
 @pytest.fixture
 def run_transform():
     graph_config = fake_load_graph_config("/root")
-    kind = FakeKind.create("fake", {}, graph_config)
+    config = FakeTransformConfig(graph_config=graph_config)
+
+    def inner(xform, tasks, **extra_config):
+        if extra_config:
+            for k, v in extra_config.items():
+                setattr(config, k, v)
 
-    def inner(xform, tasks):
         if isinstance(tasks, dict):
             tasks = [tasks]
-        return xform(kind.config, tasks)
+        return xform(config, tasks)
 
     return inner
 
diff --git a/taskcluster/gecko_taskgraph/test/python.toml b/taskcluster/gecko_taskgraph/test/python.toml
index 597a02d8aa..fd71a9c2bd 100644
--- a/taskcluster/gecko_taskgraph/test/python.toml
+++ b/taskcluster/gecko_taskgraph/test/python.toml
@@ -17,6 +17,8 @@ subsuite = "taskgraph"
 
 ["test_taskcluster_yml.py"]
 
+["test_transforms_build_schedules.py"]
+
 ["test_transforms_job.py"]
 
 ["test_transforms_test.py"]
diff --git a/taskcluster/gecko_taskgraph/test/test_decision.py b/taskcluster/gecko_taskgraph/test/test_decision.py
index 8440b8e13f..53186b70fb 100644
--- a/taskcluster/gecko_taskgraph/test/test_decision.py
+++ b/taskcluster/gecko_taskgraph/test/test_decision.py
@@ -7,7 +7,6 @@ import json
 import os
 import shutil
 import tempfile
-import unittest
 from unittest.mock import patch
 
 import pytest
@@ -18,6 +17,7 @@ from gecko_taskgraph import decision
 from gecko_taskgraph.parameters import register_parameters
 
 FAKE_GRAPH_CONFIG = {"product-dir": "browser", "taskgraph": {}}
+TTC_FILE = os.path.join(os.getcwd(), "try_task_config.json")
 
 
 @pytest.fixture(scope="module", autouse=True)
@@ -25,150 +25,172 @@ def register():
     register_parameters()
 
 
-class TestDecision(unittest.TestCase):
-    def test_write_artifact_json(self):
-        data = [{"some": "data"}]
-        tmpdir = tempfile.mkdtemp()
-        try:
-            decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
-            decision.write_artifact("artifact.json", data)
-            with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.json")) as f:
-                self.assertEqual(json.load(f), data)
-        finally:
-            if os.path.exists(tmpdir):
-                shutil.rmtree(tmpdir)
-            decision.ARTIFACTS_DIR = "artifacts"
-
-    def test_write_artifact_yml(self):
-        data = [{"some": "data"}]
-        tmpdir = tempfile.mkdtemp()
-        try:
-            decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
-            decision.write_artifact("artifact.yml", data)
-            self.assertEqual(load_yaml(decision.ARTIFACTS_DIR, "artifact.yml"), data)
-        finally:
-            if os.path.exists(tmpdir):
-                shutil.rmtree(tmpdir)
-            decision.ARTIFACTS_DIR = "artifacts"
-
-
-class TestGetDecisionParameters(unittest.TestCase):
-    ttc_file = os.path.join(os.getcwd(), "try_task_config.json")
-
-    def setUp(self):
-        self.options = {
-            "base_repository": "https://hg.mozilla.org/mozilla-unified",
-            "head_repository": "https://hg.mozilla.org/mozilla-central",
-            "head_rev": "abcd",
-            "head_ref": "ef01",
-            "head_tag": "",
-            "message": "",
-            "project": "mozilla-central",
-            "pushlog_id": "143",
-            "pushdate": 1503691511,
-            "owner": "nobody@mozilla.com",
-            "repository_type": "hg",
-            "tasks_for": "hg-push",
-            "level": "3",
-        }
-
-    @patch("gecko_taskgraph.decision.get_hg_revision_branch")
-    @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
-    def test_simple_options(
-        self, mock_determine_more_accurate_base_rev, mock_get_hg_revision_branch
-    ):
-        mock_get_hg_revision_branch.return_value = "default"
-        mock_determine_more_accurate_base_rev.return_value = "baserev"
-        with MockedOpen({self.ttc_file: None}):
-            params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
-        self.assertEqual(params["pushlog_id"], "143")
-        self.assertEqual(params["build_date"], 1503691511)
-        self.assertEqual(params["hg_branch"], "default")
-        self.assertEqual(params["moz_build_date"], "20170825200511")
-        self.assertEqual(params["try_mode"], None)
-        self.assertEqual(params["try_options"], None)
-        self.assertEqual(params["try_task_config"], {})
-
-    @patch("gecko_taskgraph.decision.get_hg_revision_branch")
-    @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
-    def test_no_email_owner(
-        self, mock_determine_more_accurate_base_rev, mock_get_hg_revision_branch
-    ):
-        mock_get_hg_revision_branch.return_value = "default"
-        mock_determine_more_accurate_base_rev.return_value = "baserev"
-        self.options["owner"] = "ffxbld"
-        with MockedOpen({self.ttc_file: None}):
-            params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
-        self.assertEqual(params["owner"], "ffxbld@noreply.mozilla.org")
-
-    @patch("gecko_taskgraph.decision.get_hg_revision_branch")
-    @patch("gecko_taskgraph.decision.get_hg_commit_message")
-    @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
-    def test_try_options(
-        self,
-        mock_determine_more_accurate_base_rev,
-        mock_get_hg_commit_message,
-        mock_get_hg_revision_branch,
-    ):
-        mock_get_hg_commit_message.return_value = "try: -b do -t all --artifact"
-        mock_get_hg_revision_branch.return_value = "default"
-        mock_determine_more_accurate_base_rev.return_value = "baserev"
-        self.options["project"] = "try"
-        with MockedOpen({self.ttc_file: None}):
-            params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
-        self.assertEqual(params["try_mode"], "try_option_syntax")
-        self.assertEqual(params["try_options"]["build_types"], "do")
-        self.assertEqual(params["try_options"]["unittests"], "all")
-        self.assertEqual(
-            params["try_task_config"],
+@pytest.fixture(scope="module")
+def options():
+    return {
+        "base_repository": "https://hg.mozilla.org/mozilla-unified",
+        "head_repository": "https://hg.mozilla.org/mozilla-central",
+        "head_rev": "abcd",
+        "head_ref": "ef01",
+        "head_tag": "",
+        "message": "",
+        "project": "mozilla-central",
+        "pushlog_id": "143",
+        "pushdate": 1503691511,
+        "owner": "nobody@mozilla.com",
+        "repository_type": "hg",
+        "tasks_for": "hg-push",
+        "level": "3",
+    }
+
+
+def test_write_artifact_json():
+    data = [{"some": "data"}]
+    tmpdir = tempfile.mkdtemp()
+    try:
+        decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
+        decision.write_artifact("artifact.json", data)
+        with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.json")) as f:
+            assert json.load(f) == data
+    finally:
+        if os.path.exists(tmpdir):
+            shutil.rmtree(tmpdir)
+        decision.ARTIFACTS_DIR = "artifacts"
+
+
+def test_write_artifact_yml():
+    data = [{"some": "data"}]
+    tmpdir = tempfile.mkdtemp()
+    try:
+        decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
+        decision.write_artifact("artifact.yml", data)
+        assert load_yaml(decision.ARTIFACTS_DIR, "artifact.yml") == data
+    finally:
+        if os.path.exists(tmpdir):
+            shutil.rmtree(tmpdir)
+        decision.ARTIFACTS_DIR = "artifacts"
+
+
+@patch("gecko_taskgraph.decision.get_hg_revision_branch")
+@patch("gecko_taskgraph.decision.get_hg_commit_message")
+@patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
+@patch("gecko_taskgraph.decision.get_changed_files")
+@pytest.mark.parametrize(
+    "extra_options,commit_msg,ttc,expected",
+    (
+        pytest.param(
+            {},
+            None,
+            None,
             {
-                "gecko-profile": False,
-                "use-artifact-builds": True,
-                "env": {},
+                "pushlog_id": "143",
+                "build_date": 1503691511,
+                "files_changed": ["bar/baz.md", "foo.txt"],
+                "hg_branch": "default",
+                "moz_build_date": "20170825200511",
+                "try_mode": None,
+                "try_options": None,
+                "try_task_config": {},
             },
-        )
-
-    @patch("gecko_taskgraph.decision.get_hg_revision_branch")
-    @patch("gecko_taskgraph.decision.get_hg_commit_message")
-    @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
-    def test_try_task_config(
-        self,
-        mock_get_hg_commit_message,
-        mock_get_hg_revision_branch,
-        mock_determine_more_accurate_base_rev,
-    ):
-        mock_get_hg_commit_message.return_value = "Fuzzy query=foo"
-        mock_get_hg_revision_branch.return_value = "default"
-        mock_determine_more_accurate_base_rev.return_value = "baserev"
-        ttc = {"tasks": ["a", "b"]}
-        self.options["project"] = "try"
-        with MockedOpen({self.ttc_file: json.dumps(ttc)}):
-            params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
-            self.assertEqual(params["try_mode"], "try_task_config")
-            self.assertEqual(params["try_options"], None)
-            self.assertEqual(params["try_task_config"], ttc)
-
-    def test_try_syntax_from_message_empty(self):
-        self.assertEqual(decision.try_syntax_from_message(""), "")
-
-    def test_try_syntax_from_message_no_try_syntax(self):
-        self.assertEqual(decision.try_syntax_from_message("abc | def"), "")
-
-    def test_try_syntax_from_message_initial_try_syntax(self):
-        self.assertEqual(
-            decision.try_syntax_from_message("try: -f -o -o"), "try: -f -o -o"
-        )
-
-    def test_try_syntax_from_message_initial_try_syntax_multiline(self):
-        self.assertEqual(
-            decision.try_syntax_from_message("try: -f -o -o\nabc\ndef"), "try: -f -o -o"
-        )
-
-    def test_try_syntax_from_message_embedded_try_syntax_multiline(self):
-        self.assertEqual(
-            decision.try_syntax_from_message("some stuff\ntry: -f -o -o\nabc\ndef"),
+            id="simple_options",
+        ),
+        pytest.param(
+            {"owner": "ffxbld"},
+            None,
+            None,
+            {
+                "owner": "ffxbld@noreply.mozilla.org",
+            },
+            id="no_email_owner",
+        ),
+        pytest.param(
+            {"project": "try"},
+            "try: -b do -t all --artifact",
+            None,
+            {
+                "try_mode": "try_option_syntax",
+                "try_options": {
+                    "build_types": "do",
+                    "include_nightly": False,
+                    "interactive": False,
+                    "jobs": None,
+                    "no_retry": False,
+                    "notifications": None,
+                    "platforms": "all",
+                    "raptor": "none",
+                    "raptor_trigger_tests": 1,
+                    "tag": None,
+                    "talos": "all",
+                    "talos_trigger_tests": 1,
+                    "taskcluster_worker": False,
+                    "trigger_tests": 1,
+                    "unittests": "all",
+                },
+                "try_task_config": {
+                    "gecko-profile": False,
+                    "use-artifact-builds": True,
+                    "env": {},
+                },
+            },
+            id="try_options",
+        ),
+        pytest.param(
+            {
+                "project": "try",
+            },
+            "Fuzzy query=foo",
+            {"tasks": ["a", "b"]},
+            {
+                "try_mode": "try_task_config",
+                "try_options": None,
+                "try_task_config": {"tasks": ["a", "b"]},
+            },
+            id="try_task_config",
+        ),
+    ),
+)
+def test_get_decision_parameters(
+    mock_get_changed_files,
+    mock_determine_more_accurate_base_rev,
+    mock_get_hg_commit_message,
+    mock_get_hg_revision_branch,
+    options,
+    extra_options,
+    commit_msg,
+    ttc,
+    expected,
+):
+    mock_get_hg_revision_branch.return_value = "default"
+    mock_get_hg_commit_message.return_value = commit_msg or "commit message"
+    mock_determine_more_accurate_base_rev.return_value = "baserev"
+    mock_get_changed_files.return_value = ["foo.txt", "bar/baz.md"]
+
+    options.update(extra_options)
+    contents = None
+    if ttc:
+        contents = json.dumps(ttc)
+    with MockedOpen({TTC_FILE: contents}):
+        params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, options)
+
+    for key in expected:
+        assert params[key] == expected[key], f"key {key} does not match!"
+
+
+@pytest.mark.parametrize(
+    "msg, expected",
+    (
+        pytest.param("", "", id="empty"),
+        pytest.param("abc | def", "", id="no_try_syntax"),
+        pytest.param("try: -f -o -o", "try: -f -o -o", id="initial_try_syntax"),
+        pytest.param(
+            "some stuff\ntry: -f -o -o\nabc\ndef",
             "try: -f -o -o",
-        )
+            id="embedded_try_syntax_multiline",
+        ),
+    ),
+)
+def test_try_syntax_from_message(msg, expected):
+    assert decision.try_syntax_from_message(msg) == expected
 
 
 if __name__ == "__main__":
diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_build_schedules.py b/taskcluster/gecko_taskgraph/test/test_transforms_build_schedules.py
new file mode 100644
index 0000000000..a693461c68
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_transforms_build_schedules.py
@@ -0,0 +1,56 @@
+import pytest
+from mozunit import main
+
+from gecko_taskgraph.transforms.build_schedules import set_build_schedules_optimization
+
+
+@pytest.mark.parametrize(
+    "kind,task,expected",
+    (
+        pytest.param("build", {"when": "foo"}, None, id="no-op"),
+        pytest.param(
+            "build",
+            {"attributes": {"build_platform": "linux64/opt"}},
+            {"build": ["linux", "firefox"]},
+            id="build",
+        ),
+        pytest.param(
+            "build-components",
+            {},
+            {"build": ["android", "fenix", "focus-android"]},
+            id="build-components",
+        ),
+        pytest.param(
+            "build-bundle",
+            {"name": "build-bundle-fenix"},
+            {"build": ["android", "fenix"]},
+            id="build-bundle-fenix",
+        ),
+        pytest.param(
+            "build-apk",
+            {"name": "fenix"},
+            {"build": ["android", "fenix"]},
+            id="build-apk-fenix",
+        ),
+        pytest.param(
+            "build-apk",
+            {"name": "build-apk-focus"},
+            {"build": ["android", "focus-android"]},
+            id="build-apk-focus",
+        ),
+        pytest.param(
+            "build-apk",
+            {"name": "build-apk-klar"},
+            {"build": ["android", "focus-android"]},
+            id="build-apk-klar",
+        ),
+    ),
+)
+def test_build_schedules(run_transform, kind, task, expected):
+    tasks = list(run_transform(set_build_schedules_optimization, [task], kind=kind))
+    assert len(tasks) == 1
+    assert tasks[0].get("optimization") == expected
+
+
+if __name__ == "__main__":
+    main()
diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_test.py b/taskcluster/gecko_taskgraph/test/test_transforms_test.py
index 1e5067a2b5..d61eff5769 100644
--- a/taskcluster/gecko_taskgraph/test/test_transforms_test.py
+++ b/taskcluster/gecko_taskgraph/test/test_transforms_test.py
@@ -235,16 +235,16 @@ def test_split_variants(monkeypatch, run_full_config_transform, make_test_task):
         pytest.param(
             {
                 "attributes": {},
-                "test-platform": "windows10-64-2004-ref-hw-2017-ccov/debug",
+                "test-platform": "windows11-64-2009-hw-ref-ccov/debug",
             },
             {
                 "platform": {
                     "arch": "64",
-                    "machine": "ref-hw-2017",
+                    "machine": "hw-ref",
                     "os": {
-                        "build": "2004",
+                        "build": "2009",
                         "name": "windows",
-                        "version": "10",
+                        "version": "11",
                     },
                 },
                 "build": {
diff --git a/taskcluster/gecko_taskgraph/test/test_util_backstop.py b/taskcluster/gecko_taskgraph/test/test_util_backstop.py
index af9aabd5af..0a2bdc6ae4 100644
--- a/taskcluster/gecko_taskgraph/test/test_util_backstop.py
+++ b/taskcluster/gecko_taskgraph/test/test_util_backstop.py
@@ -18,20 +18,22 @@ from gecko_taskgraph.util.backstop import (
     is_backstop,
 )
 
-LAST_BACKSTOP_ID = 0
+LAST_BACKSTOP_PUSHID = 1
 LAST_BACKSTOP_PUSHDATE = mktime(datetime.now().timetuple())
 DEFAULT_RESPONSES = {
     "index": {
         "status": 200,
-        "json": {"taskId": LAST_BACKSTOP_ID},
+        "json": {"taskId": LAST_BACKSTOP_PUSHID},
     },
     "artifact": {
         "status": 200,
         "body": dedent(
             """
             pushdate: {}
+            pushlog_id: "{}"
         """.format(
-                LAST_BACKSTOP_PUSHDATE
+                LAST_BACKSTOP_PUSHDATE,
+                LAST_BACKSTOP_PUSHID,
             )
         ),
     },
@@ -50,7 +52,8 @@ def params():
         "head_rev": "abcdef",
         "project": "autoland",
         "pushdate": LAST_BACKSTOP_PUSHDATE + 1,
-        "pushlog_id": LAST_BACKSTOP_ID + 1,
+        "pushlog_id": f"{LAST_BACKSTOP_PUSHID + 1}",
+        "target_tasks_method": "default",
     }
 
 
@@ -61,7 +64,7 @@ def params():
             {
                 "index": {"status": 404},
             },
-            {"pushlog_id": 1},
+            {"pushlog_id": "1"},
             True,
             id="no previous backstop",
         ),
@@ -78,8 +81,8 @@ def params():
         pytest.param(
             DEFAULT_RESPONSES,
             {
-                "pushlog_id": LAST_BACKSTOP_ID + 1,
-                "pushdate": LAST_BACKSTOP_PUSHDATE + 1,
+                "pushlog_id": f"{LAST_BACKSTOP_PUSHID + BACKSTOP_PUSH_INTERVAL - 1}",
+                "pushdate": LAST_BACKSTOP_PUSHDATE + (BACKSTOP_TIME_INTERVAL * 60) - 1,
             },
             False,
             id="not a backstop",
@@ -87,10 +90,26 @@ def params():
         pytest.param(
             {},
             {
-                "pushlog_id": BACKSTOP_PUSH_INTERVAL,
+                "target_tasks_method": "nothing",
+            },
+            False,
+            id="dontbuild",
+        ),
+        pytest.param(
+            DEFAULT_RESPONSES,
+            {
+                "pushlog_id": f"{LAST_BACKSTOP_PUSHID + BACKSTOP_PUSH_INTERVAL}",
+            },
+            True,
+            id="interval",
+        ),
+        pytest.param(
+            DEFAULT_RESPONSES,
+            {
+                "pushlog_id": f"{LAST_BACKSTOP_PUSHID + BACKSTOP_PUSH_INTERVAL + 1}",
             },
             True,
-            id="backstop interval",
+            id="greater than interval",
         ),
         pytest.param(
             DEFAULT_RESPONSES,
@@ -104,7 +123,7 @@ def params():
             {},
             {
                 "project": "try",
-                "pushlog_id": BACKSTOP_PUSH_INTERVAL,
+                "pushlog_id": f"{BACKSTOP_PUSH_INTERVAL}",
             },
             False,
             id="try not a backstop",
@@ -138,13 +157,12 @@ def test_is_backstop(responses, params, response_args, extra_params, expected):
                 **{"trust-domain": "gecko", "project": params["project"]}
             )
         ),
-        "artifact": get_artifact_url(LAST_BACKSTOP_ID, "public/parameters.yml"),
-        "status": get_task_url(LAST_BACKSTOP_ID) + "/status",
+        "artifact": get_artifact_url(LAST_BACKSTOP_PUSHID, "public/parameters.yml"),
+        "status": get_task_url(LAST_BACKSTOP_PUSHID) + "/status",
     }
 
     for key in ("index", "status", "artifact"):
         if key in response_args:
-            print(urls[key])
             responses.add(responses.GET, urls[key], **response_args[key])
 
     params.update(extra_params)